feat(dbt): extract dbt model lineage from Jinja-templated SQL
A dbt model is an ordinary .sql file whose dependencies are written
{{ ref('other_model') }} or {{ source('group','table') }}, never as
literal table names. The SQL grammar cannot read those -- FROM {{ ref(
'x') }} is a parse error to it -- so the dependency structure of an
entire dbt project was invisible to the graph.
extract_dbt.c runs as a sub-extractor inside the normal indexing
pipeline. Per qualifying file it emits one Model definition (named by
the file stem, dbt's own model identity) plus one usage per ref()/
source() call; pass_usages resolves those into model -> relation
lineage edges like any other reference.
Model is a relation label alongside Table and View, which is what makes
the rest work without new plumbing: registry seeding, the central
relation veto, the incremental surface hash, search ranking and the
architecture queries all pick it up from cbm_label_is_relation. Sharing
one label class also means a model's source('raw','customers') resolves
onto a Table declared in a plain DDL migration in the same repository,
so dbt lineage and SQL lineage form one graph rather than two, while
the veto keeps model names out of every non-lineage consumer.
The pass is self-gating: SQL files only, and only those carrying a real
ref()/source() call. The dbt builtins are themselves the evidence, which
is cheaper than a dbt_project.yml lookup and more precise -- templated
SQL that is not dbt (an Airflow {{ ds }} parameter) produces no Model
node and no usages even inside a dbt repository.
{% macro %} definitions are deliberately excluded: the vendored
tree-sitter-jinja2 grammar has no node types for {% %} statements, so
they can only be recovered by a hand-written scanner that cannot see
comments or find {% endmacro %} for a correct span. Filed as follow-up
rather than approximated.
Tests cover the lineage, the last-string-argument semantics of both
builtins, the gate (against a plain-SQL control extraction), plain DDL
staying untouched, and an end-to-end pipeline case asserting model ->
model across files, model -> Table onto plain DDL, and cross-language
isolation. Disabling the pass reddens three of them; removing Model from
the relation set breaks lineage outright.
Implements the lineage half of #575.
Co-authored-by: alexisperinger-ux <alexis.peringer@iss-stoxx.com>
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
@@ -245,6 +245,7 @@ EXTRACTION_SRCS = \
|
||||
$(CBM_DIR)/extract_env_accesses.c \
|
||||
$(CBM_DIR)/extract_channels.c \
|
||||
$(CBM_DIR)/extract_k8s.c \
|
||||
$(CBM_DIR)/extract_dbt.c \
|
||||
$(CBM_DIR)/helpers.c \
|
||||
$(CBM_DIR)/lang_specs.c \
|
||||
$(CBM_DIR)/macro_table.c \
|
||||
|
||||
@@ -1303,6 +1303,13 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
|
||||
cbm_extract_k8s(&ctx);
|
||||
}
|
||||
|
||||
// dbt lineage pass: a dbt model's dependencies live in Jinja ({{ ref(...) }}),
|
||||
// which the SQL grammar cannot read. Self-gated — SQL files only, and only
|
||||
// those carrying a real dbt builtin call.
|
||||
if (ctx.language == CBM_LANG_SQL) {
|
||||
cbm_extract_dbt(&ctx);
|
||||
}
|
||||
|
||||
// LSP type-aware call/usage resolution (per-file). Runs in every mode;
|
||||
// refines the tree-sitter + textual-resolution graph with type info.
|
||||
uint64_t lsp_start = now_ns();
|
||||
|
||||
@@ -726,6 +726,10 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch);
|
||||
// --- Sub-extractor entry points ---
|
||||
|
||||
void cbm_extract_definitions(CBMExtractCtx *ctx);
|
||||
// dbt lineage for Jinja-templated SQL models: emits a Model def plus one usage
|
||||
// per ref()/source() call. No-op unless the file parses as SQL and actually
|
||||
// contains a dbt builtin call. Defined in extract_dbt.c.
|
||||
void cbm_extract_dbt(CBMExtractCtx *ctx);
|
||||
void cbm_extract_imports(CBMExtractCtx *ctx);
|
||||
void cbm_extract_usages(CBMExtractCtx *ctx);
|
||||
void cbm_extract_semantic(CBMExtractCtx *ctx);
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
// extract_dbt.c — dbt lineage extractor for Jinja-templated SQL models.
|
||||
//
|
||||
// A dbt model is an ordinary .sql file in the repository whose SELECT is
|
||||
// templated with Jinja: dependencies are written `{{ ref('other_model') }}`
|
||||
// (another model in the project) or `{{ source('group', 'table') }}` (a raw
|
||||
// warehouse table), never as literal table names. The SQL grammar cannot read
|
||||
// those — `FROM {{ ref('x') }}` is a parse error to it — so the dependency
|
||||
// structure of an entire dbt project is invisible to the code graph without
|
||||
// this pass.
|
||||
//
|
||||
// The vendored tree-sitter-jinja2 grammar models `{{ ... }}` expressions
|
||||
// (jinja_expression / fn_call / lit_string) but has no node types at all for
|
||||
// `{% ... %}` statements, so this extractor covers lineage only. `{% macro %}`
|
||||
// definitions would need a hand-written scanner and are deliberately left out
|
||||
// rather than recovered approximately.
|
||||
//
|
||||
// What it emits, per qualifying file:
|
||||
// - one Model definition, named by the file stem (dbt's own model identity)
|
||||
// - one usage per ref()/source() call, scoped to that Model
|
||||
// pass_usages then resolves each usage against the shared definition registry
|
||||
// and emits a `model -USAGE-> relation` lineage edge. Model is a relation label
|
||||
// (cbm_label_is_relation), so lineage joins dbt models to Table/View nodes
|
||||
// declared in plain DDL elsewhere in the same repository, and the registry's
|
||||
// relation veto keeps these names out of every non-lineage consumer.
|
||||
//
|
||||
// Gate: the file must parse as SQL, contain a Jinja expression delimiter, and
|
||||
// contain at least one real ref()/source() call. Generic templated SQL (an
|
||||
// Airflow `{{ ds }}` parameter, say) therefore produces nothing at all — the
|
||||
// dbt builtins are the evidence that this is a dbt model, so no dbt_project.yml
|
||||
// lookup is needed and no non-dbt repository pays for a Model node it did not
|
||||
// ask for.
|
||||
|
||||
#include "cbm.h"
|
||||
#include "arena.h"
|
||||
#include "helpers.h"
|
||||
#include "lang_specs.h"
|
||||
#include "tree_sitter/api.h"
|
||||
#include "foundation/constants.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Local constants. */
|
||||
enum { DBT_FIRST_LINE = 1 };
|
||||
|
||||
/* Cheap pre-filter: does the source contain a Jinja expression opener? Files
|
||||
* without one cannot hold a ref()/source() call, and skipping them keeps the
|
||||
* second parse off every ordinary .sql file in the repository. */
|
||||
static bool source_has_jinja_expr(const char *s, int len) {
|
||||
for (int i = 0; i + 1 < len; i++) {
|
||||
if (s[i] == '{' && s[i + 1] == '{') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Strip one pair of surrounding quotes from a jinja lit_string. The grammar
|
||||
* hands back the token with its quotes attached. */
|
||||
static char *dbt_unquote(CBMArena *a, char *s) {
|
||||
if (!s) {
|
||||
return NULL;
|
||||
}
|
||||
size_t n = strlen(s);
|
||||
if (n >= 2 && (s[0] == '\'' || s[0] == '"') && s[n - 1] == s[0]) {
|
||||
char *inner = cbm_arena_strdup(a, s + 1);
|
||||
if (!inner) {
|
||||
return s;
|
||||
}
|
||||
size_t m = strlen(inner);
|
||||
if (m > 0) {
|
||||
inner[m - 1] = '\0';
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/* Rightmost lit_string under `node` in DFS order. Both dbt builtins put the
|
||||
* referenced relation last: ref('model'), ref('package', 'model') and
|
||||
* source('group', 'table') all name the relation in their final string
|
||||
* argument. The callee itself is an identifier, never a lit_string, so it
|
||||
* cannot be mistaken for one. */
|
||||
static TSNode dbt_last_lit_string(TSNode node, TSNode best, bool *found) {
|
||||
if (strcmp(ts_node_type(node), "lit_string") == 0) {
|
||||
best = node;
|
||||
*found = true;
|
||||
}
|
||||
uint32_t cc = ts_node_child_count(node);
|
||||
for (uint32_t i = 0; i < cc; i++) {
|
||||
best = dbt_last_lit_string(ts_node_child(node, i), best, found);
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/* Collect ref()/source() targets from a jinja2 parse tree into `out`, which is
|
||||
* the file's usage array. Usages are scoped to enclosing_qn (the Model). */
|
||||
static void collect_dbt_refs(CBMExtractCtx *ctx, TSNode node, const char *enclosing_qn,
|
||||
CBMUsageArray *out) {
|
||||
if (strcmp(ts_node_type(node), "fn_call") == 0) {
|
||||
TSNode fn = ts_node_child_by_field_name(node, "fn_name", (uint32_t)strlen("fn_name"));
|
||||
if (ts_node_is_null(fn)) {
|
||||
fn = cbm_find_child_by_kind(node, "identifier");
|
||||
}
|
||||
if (!ts_node_is_null(fn)) {
|
||||
char *fname = cbm_node_text(ctx->arena, fn, ctx->source);
|
||||
if (fname && (strcmp(fname, "ref") == 0 || strcmp(fname, "source") == 0)) {
|
||||
bool found = false;
|
||||
TSNode empty = {0};
|
||||
TSNode strn = dbt_last_lit_string(node, empty, &found);
|
||||
if (found) {
|
||||
char *name =
|
||||
dbt_unquote(ctx->arena, cbm_node_text(ctx->arena, strn, ctx->source));
|
||||
if (name && name[0]) {
|
||||
CBMUsage usage = {0};
|
||||
usage.ref_name = name;
|
||||
usage.enclosing_func_qn = enclosing_qn;
|
||||
usage.site_start_byte = ts_node_start_byte(strn);
|
||||
usage.site_end_byte = ts_node_end_byte(strn);
|
||||
cbm_usages_push(out, ctx->arena, usage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
uint32_t cc = ts_node_child_count(node);
|
||||
for (uint32_t i = 0; i < cc; i++) {
|
||||
collect_dbt_refs(ctx, ts_node_child(node, i), enclosing_qn, out);
|
||||
}
|
||||
}
|
||||
|
||||
/* dbt model identity is the file stem: models/staging/stg_users.sql is the
|
||||
* model `stg_users`, and `{{ ref('stg_users') }}` anywhere in the project
|
||||
* addresses it by that bare name (dbt requires model names to be unique across
|
||||
* a project, so the directory is deliberately not part of the identity). */
|
||||
static char *dbt_name_from_path(CBMArena *a, const char *rel_path) {
|
||||
if (!rel_path) {
|
||||
return NULL;
|
||||
}
|
||||
const char *base = rel_path;
|
||||
for (const char *p = rel_path; *p; p++) {
|
||||
if (*p == '/' || *p == '\\') {
|
||||
base = p + 1;
|
||||
}
|
||||
}
|
||||
const char *dot = NULL;
|
||||
for (const char *p = base; *p; p++) {
|
||||
if (*p == '.') {
|
||||
dot = p;
|
||||
}
|
||||
}
|
||||
size_t n = dot ? (size_t)(dot - base) : strlen(base);
|
||||
if (n == 0) {
|
||||
return NULL;
|
||||
}
|
||||
char *out = cbm_arena_strdup(a, base);
|
||||
if (!out) {
|
||||
return NULL;
|
||||
}
|
||||
out[n] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
void cbm_extract_dbt(CBMExtractCtx *ctx) {
|
||||
if (!ctx || ctx->language != CBM_LANG_SQL || !ctx->source || ctx->source_len <= 0) {
|
||||
return;
|
||||
}
|
||||
if (!source_has_jinja_expr(ctx->source, ctx->source_len)) {
|
||||
return;
|
||||
}
|
||||
const TSLanguage *jl = cbm_ts_language(CBM_LANG_JINJA2);
|
||||
if (!jl) {
|
||||
return;
|
||||
}
|
||||
char *model_name = dbt_name_from_path(ctx->arena, ctx->rel_path);
|
||||
if (!model_name || !model_name[0]) {
|
||||
return;
|
||||
}
|
||||
const char *model_qn = cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, model_name);
|
||||
if (!model_qn) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* A fresh parser: the primary SQL pass owns the thread-local one, and this
|
||||
* runs inside its walk. */
|
||||
TSParser *parser = ts_parser_new();
|
||||
if (!parser) {
|
||||
return;
|
||||
}
|
||||
/* Refs are staged locally so a file with Jinja but no dbt builtins commits
|
||||
* nothing at all — neither usages nor a Model node. */
|
||||
CBMUsageArray staged = {0};
|
||||
if (ts_parser_set_language(parser, jl)) {
|
||||
TSTree *tree = ts_parser_parse_string(parser, NULL, ctx->source, (uint32_t)ctx->source_len);
|
||||
if (tree) {
|
||||
collect_dbt_refs(ctx, ts_tree_root_node(tree), model_qn, &staged);
|
||||
ts_tree_delete(tree);
|
||||
}
|
||||
}
|
||||
ts_parser_delete(parser);
|
||||
if (staged.count == 0) {
|
||||
return; /* templated SQL, but not dbt — emit nothing */
|
||||
}
|
||||
|
||||
CBMDefinition def;
|
||||
memset(&def, 0, sizeof(def));
|
||||
def.name = model_name;
|
||||
def.qualified_name = model_qn;
|
||||
def.label = "Model";
|
||||
def.file_path = ctx->rel_path;
|
||||
def.start_line = DBT_FIRST_LINE;
|
||||
def.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
|
||||
def.is_exported = true;
|
||||
cbm_defs_push(&ctx->result->defs, ctx->arena, def);
|
||||
|
||||
for (int i = 0; i < staged.count; i++) {
|
||||
cbm_usages_push(&ctx->result->usages, ctx->arena, staged.items[i]);
|
||||
}
|
||||
}
|
||||
+10
-6
@@ -166,16 +166,20 @@ bool cbm_label_is_type_like(const char *label) {
|
||||
strcmp(label, "Type") == 0 || strcmp(label, "Trait") == 0;
|
||||
}
|
||||
|
||||
// True when `label` names a data relation (SQL CREATE TABLE / CREATE VIEW).
|
||||
// Relations live in the registry so FROM/JOIN lineage can resolve, but they are
|
||||
// deliberately NOT type-like: they must never satisfy inheritance, impl-receiver,
|
||||
// semantic-type, or LSP-registrar lookups, and resolver fallbacks treat them as
|
||||
// lineage-only targets (see cbm_label_is_registry_symbol call sites).
|
||||
// True when `label` names a data relation: SQL CREATE TABLE / CREATE VIEW, and
|
||||
// a dbt Model (a Jinja-templated .sql file, which materializes as a warehouse
|
||||
// table or view). Relations live in the registry so FROM/JOIN and dbt ref()
|
||||
// lineage can resolve, and sharing one label class is what lets a dbt model's
|
||||
// ref() reach a Table declared in plain DDL elsewhere in the same repository.
|
||||
// They are deliberately NOT type-like: they must never satisfy inheritance,
|
||||
// impl-receiver, semantic-type, or LSP-registrar lookups, and resolver
|
||||
// fallbacks treat them as lineage-only targets (see cbm_label_is_registry_symbol
|
||||
// call sites).
|
||||
bool cbm_label_is_relation(const char *label) {
|
||||
if (!label) {
|
||||
return false;
|
||||
}
|
||||
return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0;
|
||||
return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0 || strcmp(label, "Model") == 0;
|
||||
}
|
||||
|
||||
// True when `label` belongs in the cross-file name registry (see cbm.h). Single
|
||||
|
||||
@@ -108,8 +108,8 @@ enum { SKIP_ONE = 1, PAIR_LEN = 2 };
|
||||
#define CBM_SQL_TYPE_LIKE_LABELS "'Class','Struct','Interface','Enum','Type','Trait'"
|
||||
#define CBM_SQL_CALLABLE_LABELS "'Function','Method'"
|
||||
#define CBM_SQL_CALLABLE_OR_TYPE_LABELS CBM_SQL_CALLABLE_LABELS "," CBM_SQL_TYPE_LIKE_LABELS
|
||||
/* SQL mirror of cbm_label_is_relation() (Table/View — data-lineage nodes),
|
||||
* pinned by tests/test_store_nodes.c the same way as the sets above. */
|
||||
#define CBM_SQL_RELATION_LABELS "'Table','View'"
|
||||
/* SQL mirror of cbm_label_is_relation() (Table/View/Model — data-lineage
|
||||
* nodes), pinned by tests/test_store_nodes.c the same way as the sets above. */
|
||||
#define CBM_SQL_RELATION_LABELS "'Table','View','Model'"
|
||||
|
||||
#endif /* CBM_CONSTANTS_H */
|
||||
|
||||
@@ -1868,6 +1868,100 @@ TEST(sql_schema_qualified_name) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* --- dbt Jinja lineage --- */
|
||||
|
||||
/* Helper: does the file's usage list carry `name`? */
|
||||
static int has_usage(CBMFileResult *r, const char *name) {
|
||||
for (int i = 0; i < r->usages.count; i++) {
|
||||
if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, name) == 0) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
TEST(dbt_model_and_ref_lineage) {
|
||||
/* A dbt model: the file stem is the model identity, and each ref() is a
|
||||
* dependency on another model. The SQL grammar cannot read `{{ ref(..) }}`
|
||||
* at all, so without the dbt pass this file yields no lineage whatsoever. */
|
||||
CBMFileResult *r = extract("SELECT o.id, c.name\n"
|
||||
"FROM {{ ref('stg_orders') }} o\n"
|
||||
"JOIN {{ ref('stg_customers') }} c ON c.id = o.customer_id\n",
|
||||
CBM_LANG_SQL, "t", "models/marts/orders_enriched.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT(has_def(r, "Model", "orders_enriched"));
|
||||
ASSERT(has_usage(r, "stg_orders"));
|
||||
ASSERT(has_usage(r, "stg_customers"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(dbt_source_and_two_arg_ref) {
|
||||
/* Both dbt builtins name the relation in their LAST string argument:
|
||||
* source('group','table') -> table, and the two-argument
|
||||
* ref('package','model') form -> model. */
|
||||
CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n"
|
||||
"UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n",
|
||||
CBM_LANG_SQL, "t", "models/stg_customers.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT(has_def(r, "Model", "stg_customers"));
|
||||
ASSERT(has_usage(r, "customers"));
|
||||
ASSERT(has_usage(r, "legacy_customers"));
|
||||
/* the group/package argument is not the relation */
|
||||
ASSERT_FALSE(has_usage(r, "raw"));
|
||||
ASSERT_FALSE(has_usage(r, "analytics"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(dbt_ignores_non_dbt_jinja) {
|
||||
/* Templated SQL is not dbt SQL. An Airflow-style parameter substitution has
|
||||
* Jinja but no dbt builtin, so the dbt pass must contribute NOTHING — no
|
||||
* Model node named after the file, and no usage minted from the template
|
||||
* variables. This is the gate that keeps every non-dbt repository free of
|
||||
* fabricated data-lineage vocabulary.
|
||||
*
|
||||
* The ordinary SQL identifier path is unaffected and still sees the literal
|
||||
* `FROM events`; the second extraction below is the control proving that
|
||||
* usage is pre-existing SQL behaviour rather than anything dbt added. */
|
||||
CBMFileResult *r = extract("SELECT * FROM events WHERE day = '{{ ds }}'\n"
|
||||
" AND region = '{{ params.region_code }}'\n",
|
||||
CBM_LANG_SQL, "t", "queries/daily_events.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(has_def(r, "Model", "daily_events"));
|
||||
|
||||
/* Control: the same statement with the templates replaced by plain string
|
||||
* literals. Both parse as SQL identically, so an equal usage count is the
|
||||
* precise statement of "the dbt pass contributed nothing here" — stronger
|
||||
* than naming individual identifiers, and immune to how SQL happens to
|
||||
* tokenize the template text. */
|
||||
CBMFileResult *plain = extract("SELECT * FROM events WHERE day = '2026-01-01'\n"
|
||||
" AND region = 'eu-west'\n",
|
||||
CBM_LANG_SQL, "t", "queries/daily_events.sql");
|
||||
ASSERT_NOT_NULL(plain);
|
||||
ASSERT_FALSE(has_def(plain, "Model", "daily_events"));
|
||||
ASSERT_EQ(r->usages.count, plain->usages.count);
|
||||
ASSERT_EQ(r->defs.count, plain->defs.count);
|
||||
cbm_free_result(plain);
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(dbt_plain_sql_untouched) {
|
||||
/* Plain DDL keeps producing exactly the Table/View relations it did before
|
||||
* the dbt pass existed — no Model node, and the FROM lineage is unchanged. */
|
||||
CBMFileResult *r = extract("CREATE TABLE users (id INTEGER);\n"
|
||||
"CREATE VIEW active_users AS SELECT * FROM users;\n",
|
||||
CBM_LANG_SQL, "t", "schema.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(r->has_error);
|
||||
ASSERT(has_def(r, "Table", "users"));
|
||||
ASSERT(has_def(r, "View", "active_users"));
|
||||
ASSERT_FALSE(has_def(r, "Model", "schema"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* --- Meson project --- */
|
||||
TEST(meson_project) {
|
||||
CBMFileResult *r = extract(
|
||||
@@ -5555,6 +5649,10 @@ SUITE(extraction) {
|
||||
RUN_TEST(sql_ddl_node_labels);
|
||||
RUN_TEST(sql_view_lineage_usages);
|
||||
RUN_TEST(sql_schema_qualified_name);
|
||||
RUN_TEST(dbt_model_and_ref_lineage);
|
||||
RUN_TEST(dbt_source_and_two_arg_ref);
|
||||
RUN_TEST(dbt_ignores_non_dbt_jinja);
|
||||
RUN_TEST(dbt_plain_sql_untouched);
|
||||
RUN_TEST(meson_project);
|
||||
RUN_TEST(css_rules);
|
||||
RUN_TEST(scss_rules);
|
||||
|
||||
@@ -2250,6 +2250,48 @@ TEST(pipeline_sql_lineage_and_relation_isolation) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* dbt lineage end-to-end. A dbt project's dependency structure lives entirely
|
||||
* in Jinja ({{ ref('x') }}), which the SQL grammar cannot read, so this is the
|
||||
* whole value: model -> model edges across files, plus the join onto a Table
|
||||
* declared in ordinary DDL — Model and Table are both relation labels, so one
|
||||
* lineage layer spans both. The Python file is the isolation control: `stg_orders`
|
||||
* exists project-wide only as a dbt model, and the registry's relation veto must
|
||||
* keep a same-named call out of the lineage layer. */
|
||||
TEST(pipeline_dbt_jinja_lineage) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dbt_lineage_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmp)) {
|
||||
FAIL("tmpdir");
|
||||
}
|
||||
write_temp_file(tmp, "raw_schema.sql", "CREATE TABLE customers (id INTEGER, name TEXT);\n");
|
||||
write_temp_file(tmp, "stg_orders.sql",
|
||||
"SELECT id, customer_id FROM {{ source('raw', 'customers') }}\n");
|
||||
write_temp_file(tmp, "orders_enriched.sql",
|
||||
"SELECT o.id, c.name\n"
|
||||
"FROM {{ ref('stg_orders') }} o\n"
|
||||
"JOIN {{ ref('stg_orders') }} c ON c.id = o.customer_id\n");
|
||||
write_temp_file(tmp, "app.py", "def load():\n return stg_orders()\n");
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/dbt.db", tmp);
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
const char *project = cbm_pipeline_project_name(p);
|
||||
cbm_store_t *s = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(s);
|
||||
/* model -> model: the ref() lineage the SQL grammar cannot see */
|
||||
ASSERT_TRUE(named_edge_count(s, project, "USAGE", "orders_enriched", "stg_orders") >= 1);
|
||||
/* model -> table: source() joining dbt onto plain DDL in the same repo */
|
||||
ASSERT_EQ(named_edge_count(s, project, "USAGE", "stg_orders", "customers"), 1);
|
||||
/* isolation: the Python call must not reach the model */
|
||||
ASSERT_EQ(named_edge_count(s, project, "CALLS", "load", "stg_orders"), 0);
|
||||
ASSERT_EQ(named_edge_count(s, project, "USAGE", "load", "stg_orders"), 0);
|
||||
cbm_store_close(s);
|
||||
cbm_pipeline_free(p);
|
||||
th_rmtree(tmp);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Renaming a table must drop lineage from DEPENDENT (unchanged) SQL files on
|
||||
* the incremental path. Table/View participate in the per-file LSP surface
|
||||
* hash as registry-only labels (lsp_surface.c), so tables.sql's def change
|
||||
@@ -12348,6 +12390,7 @@ SUITE(pipeline_semantic_manifest_repro) {
|
||||
RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge);
|
||||
RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
|
||||
RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
|
||||
RUN_TEST(pipeline_dbt_jinja_lineage);
|
||||
RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold);
|
||||
RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full);
|
||||
RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge);
|
||||
|
||||
@@ -57,7 +57,7 @@ TEST(sql_label_allowlists_match_cbm_label_is_type_like) {
|
||||
* cbm_registry_resolve vetoes them, so a code identifier sharing a table's
|
||||
* name never binds into the lineage layer. */
|
||||
TEST(sql_relation_labels_match_cbm_label_is_relation) {
|
||||
static const char *const relations[] = {"Table", "View"};
|
||||
static const char *const relations[] = {"Table", "View", "Model"};
|
||||
for (size_t i = 0; i < sizeof(relations) / sizeof(relations[0]); i++) {
|
||||
ASSERT_TRUE(cbm_label_is_relation(relations[i]));
|
||||
ASSERT_TRUE(cbm_label_is_registry_symbol(relations[i]));
|
||||
|
||||
Reference in New Issue
Block a user