Merge branch 'main' into fix/pi-extension-tool-schemas
This commit is contained in:
@@ -20,13 +20,13 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run OpenSSF Scorecard
|
||||
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
|
||||
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
publish_results: true
|
||||
|
||||
- name: Upload SARIF results
|
||||
uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
|
||||
- uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
|
||||
with:
|
||||
only-labels: 'awaiting-reporter'
|
||||
stale-issue-label: 'stale'
|
||||
|
||||
@@ -245,6 +245,7 @@ EXTRACTION_SRCS = \
|
||||
$(CBM_DIR)/extract_env_accesses.c \
|
||||
$(CBM_DIR)/extract_channels.c \
|
||||
$(CBM_DIR)/extract_k8s.c \
|
||||
$(CBM_DIR)/extract_dbt.c \
|
||||
$(CBM_DIR)/helpers.c \
|
||||
$(CBM_DIR)/lang_specs.c \
|
||||
$(CBM_DIR)/macro_table.c \
|
||||
|
||||
@@ -7,6 +7,57 @@
|
||||
let
|
||||
systems = [ "aarch64-darwin" "x86_64-darwin" "aarch64-linux" "x86_64-linux" ];
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
|
||||
|
||||
# Cross dev shell (macOS only): build the *other* darwin arch locally.
|
||||
# The native clang already cross-emits object code via -arch (scripts/env.sh
|
||||
# exports ARCHFLAGS), so the only thing missing for a cross link is zlib:
|
||||
# it is the product's single link-time dependency (`-lz`), and the host-arch
|
||||
# copy cannot satisfy an x86_64 link on arm64 or vice versa. This shell puts
|
||||
# the TARGET-arch zlib on the include and link paths, and nothing else.
|
||||
#
|
||||
# Building needs no Rosetta; *running* an x86_64 binary on Apple Silicon
|
||||
# does. The target-arch libs come from prebuilt substitutes, so entering the
|
||||
# shell is a download rather than a cross-build of the dependency tree.
|
||||
#
|
||||
# Returns {} off macOS, where there is no second darwin arch to target.
|
||||
crossDevShells = pkgs:
|
||||
let
|
||||
inherit (nixpkgs) lib;
|
||||
crossTarget =
|
||||
{
|
||||
"aarch64-darwin" = "x86_64-darwin";
|
||||
"x86_64-darwin" = "aarch64-darwin";
|
||||
}
|
||||
.${pkgs.system} or null;
|
||||
mkCrossShell =
|
||||
targetSystem:
|
||||
let
|
||||
tpkgs = nixpkgs.legacyPackages.${targetSystem};
|
||||
targetArch = if targetSystem == "x86_64-darwin" then "x86_64" else "arm64";
|
||||
in
|
||||
pkgs.mkShell {
|
||||
# Host toolchain only — clang comes from the shell stdenv. Deliberately
|
||||
# NOT inputsFrom the default package: that would put the HOST-arch zlib
|
||||
# on the link path, which is exactly the wrong-arch copy this shell
|
||||
# exists to displace.
|
||||
nativeBuildInputs = [ pkgs.gnumake ];
|
||||
shellHook = ''
|
||||
# zlib is consumed as a bare `-lz` / `#include <zlib.h>`, so its
|
||||
# include dir (the header is arch-independent) and lib dir are
|
||||
# supplied by hand. NIX_LDFLAGS is searched ahead of the link's own
|
||||
# -L, so the target-arch slice wins; a host-arch copy that still
|
||||
# reaches ld is skipped as "wrong architecture" rather than
|
||||
# producing a silently mislinked binary.
|
||||
export NIX_CFLAGS_COMPILE="-I${lib.getDev tpkgs.zlib}/include ''${NIX_CFLAGS_COMPILE:-}"
|
||||
export NIX_LDFLAGS="-L${lib.getLib tpkgs.zlib}/lib ''${NIX_LDFLAGS:-}"
|
||||
echo "[cross] target ${targetSystem} — build with: scripts/build.sh --arch ${targetArch}"
|
||||
'';
|
||||
};
|
||||
in
|
||||
lib.optionalAttrs (crossTarget != null) {
|
||||
# `nix develop .#cross` then `scripts/build.sh --arch <target>`.
|
||||
cross = mkCrossShell crossTarget;
|
||||
};
|
||||
in
|
||||
{
|
||||
packages = forAllSystems (pkgs: {
|
||||
@@ -45,6 +96,6 @@
|
||||
default = pkgs.mkShell {
|
||||
inputsFrom = [ self.packages.${pkgs.system}.default ];
|
||||
};
|
||||
});
|
||||
} // crossDevShells pkgs);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1303,6 +1303,13 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
|
||||
cbm_extract_k8s(&ctx);
|
||||
}
|
||||
|
||||
// dbt lineage pass: a dbt model's dependencies live in Jinja ({{ ref(...) }}),
|
||||
// which the SQL grammar cannot read. Self-gated — SQL files only, and only
|
||||
// those carrying a real dbt builtin call.
|
||||
if (ctx.language == CBM_LANG_SQL) {
|
||||
cbm_extract_dbt(&ctx);
|
||||
}
|
||||
|
||||
// LSP type-aware call/usage resolution (per-file). Runs in every mode;
|
||||
// refines the tree-sitter + textual-resolution graph with type info.
|
||||
uint64_t lsp_start = now_ns();
|
||||
|
||||
@@ -726,6 +726,10 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch);
|
||||
// --- Sub-extractor entry points ---
|
||||
|
||||
void cbm_extract_definitions(CBMExtractCtx *ctx);
|
||||
// dbt lineage for Jinja-templated SQL models: emits a Model def plus one usage
|
||||
// per ref()/source() call. No-op unless the file parses as SQL and actually
|
||||
// contains a dbt builtin call. Defined in extract_dbt.c.
|
||||
void cbm_extract_dbt(CBMExtractCtx *ctx);
|
||||
void cbm_extract_imports(CBMExtractCtx *ctx);
|
||||
void cbm_extract_usages(CBMExtractCtx *ctx);
|
||||
void cbm_extract_semantic(CBMExtractCtx *ctx);
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
// extract_dbt.c — dbt lineage extractor for Jinja-templated SQL models.
|
||||
//
|
||||
// A dbt model is an ordinary .sql file in the repository whose SELECT is
|
||||
// templated with Jinja: dependencies are written `{{ ref('other_model') }}`
|
||||
// (another model in the project) or `{{ source('group', 'table') }}` (a raw
|
||||
// warehouse table), never as literal table names. The SQL grammar cannot read
|
||||
// those — `FROM {{ ref('x') }}` is a parse error to it — so the dependency
|
||||
// structure of an entire dbt project is invisible to the code graph without
|
||||
// this pass.
|
||||
//
|
||||
// The vendored tree-sitter-jinja2 grammar models `{{ ... }}` expressions
|
||||
// (jinja_expression / fn_call / lit_string) but has no node types at all for
|
||||
// `{% ... %}` statements, so this extractor covers lineage only. `{% macro %}`
|
||||
// definitions would need a hand-written scanner and are deliberately left out
|
||||
// rather than recovered approximately.
|
||||
//
|
||||
// What it emits, per qualifying file:
|
||||
// - one Model definition, named by the file stem (dbt's own model identity)
|
||||
// - one usage per ref()/source() call, scoped to that Model
|
||||
// pass_usages then resolves each usage against the shared definition registry
|
||||
// and emits a `model -USAGE-> relation` lineage edge. Model is a relation label
|
||||
// (cbm_label_is_relation), so lineage joins dbt models to Table/View nodes
|
||||
// declared in plain DDL elsewhere in the same repository, and the registry's
|
||||
// relation veto keeps these names out of every non-lineage consumer.
|
||||
//
|
||||
// Gate: the file must parse as SQL, contain a Jinja expression delimiter, and
|
||||
// contain at least one real ref()/source() call. Generic templated SQL (an
|
||||
// Airflow `{{ ds }}` parameter, say) therefore produces nothing at all — the
|
||||
// dbt builtins are the evidence that this is a dbt model, so no dbt_project.yml
|
||||
// lookup is needed and no non-dbt repository pays for a Model node it did not
|
||||
// ask for.
|
||||
|
||||
#include "cbm.h"
|
||||
#include "arena.h"
|
||||
#include "helpers.h"
|
||||
#include "lang_specs.h"
|
||||
#include "tree_sitter/api.h"
|
||||
#include "foundation/constants.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Local constants. */
|
||||
enum { DBT_FIRST_LINE = 1 };
|
||||
|
||||
/* Cheap pre-filter: does the source contain a Jinja expression opener? Files
|
||||
* without one cannot hold a ref()/source() call, and skipping them keeps the
|
||||
* second parse off every ordinary .sql file in the repository. */
|
||||
static bool source_has_jinja_expr(const char *s, int len) {
|
||||
for (int i = 0; i + 1 < len; i++) {
|
||||
if (s[i] == '{' && s[i + 1] == '{') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Strip one pair of surrounding quotes from a jinja lit_string. The grammar
|
||||
* hands back the token with its quotes attached. */
|
||||
static char *dbt_unquote(CBMArena *a, char *s) {
|
||||
if (!s) {
|
||||
return NULL;
|
||||
}
|
||||
size_t n = strlen(s);
|
||||
if (n >= 2 && (s[0] == '\'' || s[0] == '"') && s[n - 1] == s[0]) {
|
||||
char *inner = cbm_arena_strdup(a, s + 1);
|
||||
if (!inner) {
|
||||
return s;
|
||||
}
|
||||
size_t m = strlen(inner);
|
||||
if (m > 0) {
|
||||
inner[m - 1] = '\0';
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/* Rightmost lit_string under `node` in DFS order. Both dbt builtins put the
|
||||
* referenced relation last: ref('model'), ref('package', 'model') and
|
||||
* source('group', 'table') all name the relation in their final string
|
||||
* argument. The callee itself is an identifier, never a lit_string, so it
|
||||
* cannot be mistaken for one. */
|
||||
static TSNode dbt_last_lit_string(TSNode node, TSNode best, bool *found) {
|
||||
if (strcmp(ts_node_type(node), "lit_string") == 0) {
|
||||
best = node;
|
||||
*found = true;
|
||||
}
|
||||
uint32_t cc = ts_node_child_count(node);
|
||||
for (uint32_t i = 0; i < cc; i++) {
|
||||
best = dbt_last_lit_string(ts_node_child(node, i), best, found);
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/* Collect ref()/source() targets from a jinja2 parse tree into `out`, which is
|
||||
* the file's usage array. Usages are scoped to enclosing_qn (the Model). */
|
||||
static void collect_dbt_refs(CBMExtractCtx *ctx, TSNode node, const char *enclosing_qn,
|
||||
CBMUsageArray *out) {
|
||||
if (strcmp(ts_node_type(node), "fn_call") == 0) {
|
||||
TSNode fn = ts_node_child_by_field_name(node, "fn_name", (uint32_t)strlen("fn_name"));
|
||||
if (ts_node_is_null(fn)) {
|
||||
fn = cbm_find_child_by_kind(node, "identifier");
|
||||
}
|
||||
if (!ts_node_is_null(fn)) {
|
||||
char *fname = cbm_node_text(ctx->arena, fn, ctx->source);
|
||||
if (fname && (strcmp(fname, "ref") == 0 || strcmp(fname, "source") == 0)) {
|
||||
bool found = false;
|
||||
TSNode empty = {0};
|
||||
TSNode strn = dbt_last_lit_string(node, empty, &found);
|
||||
if (found) {
|
||||
char *name =
|
||||
dbt_unquote(ctx->arena, cbm_node_text(ctx->arena, strn, ctx->source));
|
||||
if (name && name[0]) {
|
||||
CBMUsage usage = {0};
|
||||
usage.ref_name = name;
|
||||
usage.enclosing_func_qn = enclosing_qn;
|
||||
usage.site_start_byte = ts_node_start_byte(strn);
|
||||
usage.site_end_byte = ts_node_end_byte(strn);
|
||||
cbm_usages_push(out, ctx->arena, usage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
uint32_t cc = ts_node_child_count(node);
|
||||
for (uint32_t i = 0; i < cc; i++) {
|
||||
collect_dbt_refs(ctx, ts_node_child(node, i), enclosing_qn, out);
|
||||
}
|
||||
}
|
||||
|
||||
/* dbt model identity is the file stem: models/staging/stg_users.sql is the
|
||||
* model `stg_users`, and `{{ ref('stg_users') }}` anywhere in the project
|
||||
* addresses it by that bare name (dbt requires model names to be unique across
|
||||
* a project, so the directory is deliberately not part of the identity). */
|
||||
static char *dbt_name_from_path(CBMArena *a, const char *rel_path) {
|
||||
if (!rel_path) {
|
||||
return NULL;
|
||||
}
|
||||
const char *base = rel_path;
|
||||
for (const char *p = rel_path; *p; p++) {
|
||||
if (*p == '/' || *p == '\\') {
|
||||
base = p + 1;
|
||||
}
|
||||
}
|
||||
const char *dot = NULL;
|
||||
for (const char *p = base; *p; p++) {
|
||||
if (*p == '.') {
|
||||
dot = p;
|
||||
}
|
||||
}
|
||||
size_t n = dot ? (size_t)(dot - base) : strlen(base);
|
||||
if (n == 0) {
|
||||
return NULL;
|
||||
}
|
||||
char *out = cbm_arena_strdup(a, base);
|
||||
if (!out) {
|
||||
return NULL;
|
||||
}
|
||||
out[n] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
void cbm_extract_dbt(CBMExtractCtx *ctx) {
|
||||
if (!ctx || ctx->language != CBM_LANG_SQL || !ctx->source || ctx->source_len <= 0) {
|
||||
return;
|
||||
}
|
||||
if (!source_has_jinja_expr(ctx->source, ctx->source_len)) {
|
||||
return;
|
||||
}
|
||||
const TSLanguage *jl = cbm_ts_language(CBM_LANG_JINJA2);
|
||||
if (!jl) {
|
||||
return;
|
||||
}
|
||||
char *model_name = dbt_name_from_path(ctx->arena, ctx->rel_path);
|
||||
if (!model_name || !model_name[0]) {
|
||||
return;
|
||||
}
|
||||
const char *model_qn = cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, model_name);
|
||||
if (!model_qn) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* A fresh parser: the primary SQL pass owns the thread-local one, and this
|
||||
* runs inside its walk. */
|
||||
TSParser *parser = ts_parser_new();
|
||||
if (!parser) {
|
||||
return;
|
||||
}
|
||||
/* Refs are staged locally so a file with Jinja but no dbt builtins commits
|
||||
* nothing at all — neither usages nor a Model node. */
|
||||
CBMUsageArray staged = {0};
|
||||
if (ts_parser_set_language(parser, jl)) {
|
||||
TSTree *tree = ts_parser_parse_string(parser, NULL, ctx->source, (uint32_t)ctx->source_len);
|
||||
if (tree) {
|
||||
collect_dbt_refs(ctx, ts_tree_root_node(tree), model_qn, &staged);
|
||||
ts_tree_delete(tree);
|
||||
}
|
||||
}
|
||||
ts_parser_delete(parser);
|
||||
if (staged.count == 0) {
|
||||
return; /* templated SQL, but not dbt — emit nothing */
|
||||
}
|
||||
|
||||
CBMDefinition def;
|
||||
memset(&def, 0, sizeof(def));
|
||||
def.name = model_name;
|
||||
def.qualified_name = model_qn;
|
||||
def.label = "Model";
|
||||
def.file_path = ctx->rel_path;
|
||||
def.start_line = DBT_FIRST_LINE;
|
||||
def.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
|
||||
def.is_exported = true;
|
||||
cbm_defs_push(&ctx->result->defs, ctx->arena, def);
|
||||
|
||||
for (int i = 0; i < staged.count; i++) {
|
||||
cbm_usages_push(&ctx->result->usages, ctx->arena, staged.items[i]);
|
||||
}
|
||||
}
|
||||
@@ -745,9 +745,14 @@ TSNode cbm_resolve_func_name(TSNode node, CBMLanguage lang) {
|
||||
}
|
||||
|
||||
/* Swift and newer tree-sitter-kotlin: function_declaration has no `name`
|
||||
* field; the function name is a `simple_identifier` child. */
|
||||
* field; the function name is a `simple_identifier` child. A Swift
|
||||
* protocol requirement (a bodyless `func` inside a protocol) is a
|
||||
* separate node type with the same shape; it is named here as well as in
|
||||
* resolve_method_name because swift_func_types now admits it, so it can
|
||||
* reach the free-function path too and would otherwise land unnamed. */
|
||||
if ((lang == CBM_LANG_SWIFT || lang == CBM_LANG_KOTLIN) &&
|
||||
strcmp(kind, "function_declaration") == 0) {
|
||||
(strcmp(kind, "function_declaration") == 0 ||
|
||||
strcmp(kind, "protocol_function_declaration") == 0)) {
|
||||
TSNode si = cbm_find_child_by_kind(node, "simple_identifier");
|
||||
if (!ts_node_is_null(si)) {
|
||||
return si;
|
||||
@@ -4579,6 +4584,7 @@ static TSNode find_class_body(TSNode class_node, CBMLanguage lang) {
|
||||
static const char *body_types[] = {"class_body",
|
||||
"interface_body",
|
||||
"enum_body",
|
||||
"protocol_body",
|
||||
"template_body",
|
||||
"interface_type",
|
||||
"struct_type",
|
||||
@@ -4699,7 +4705,8 @@ static TSNode resolve_method_name(TSNode child, CBMLanguage lang) {
|
||||
}
|
||||
|
||||
if ((lang == CBM_LANG_SWIFT || lang == CBM_LANG_KOTLIN) &&
|
||||
strcmp(ck, "function_declaration") == 0) {
|
||||
(strcmp(ck, "function_declaration") == 0 ||
|
||||
strcmp(ck, "protocol_function_declaration") == 0)) {
|
||||
return cbm_find_child_by_kind(child, "simple_identifier");
|
||||
}
|
||||
|
||||
|
||||
+10
-6
@@ -166,16 +166,20 @@ bool cbm_label_is_type_like(const char *label) {
|
||||
strcmp(label, "Type") == 0 || strcmp(label, "Trait") == 0;
|
||||
}
|
||||
|
||||
// True when `label` names a data relation (SQL CREATE TABLE / CREATE VIEW).
|
||||
// Relations live in the registry so FROM/JOIN lineage can resolve, but they are
|
||||
// deliberately NOT type-like: they must never satisfy inheritance, impl-receiver,
|
||||
// semantic-type, or LSP-registrar lookups, and resolver fallbacks treat them as
|
||||
// lineage-only targets (see cbm_label_is_registry_symbol call sites).
|
||||
// True when `label` names a data relation: SQL CREATE TABLE / CREATE VIEW, and
|
||||
// a dbt Model (a Jinja-templated .sql file, which materializes as a warehouse
|
||||
// table or view). Relations live in the registry so FROM/JOIN and dbt ref()
|
||||
// lineage can resolve, and sharing one label class is what lets a dbt model's
|
||||
// ref() reach a Table declared in plain DDL elsewhere in the same repository.
|
||||
// They are deliberately NOT type-like: they must never satisfy inheritance,
|
||||
// impl-receiver, semantic-type, or LSP-registrar lookups, and resolver
|
||||
// fallbacks treat them as lineage-only targets (see cbm_label_is_registry_symbol
|
||||
// call sites).
|
||||
bool cbm_label_is_relation(const char *label) {
|
||||
if (!label) {
|
||||
return false;
|
||||
}
|
||||
return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0;
|
||||
return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0 || strcmp(label, "Model") == 0;
|
||||
}
|
||||
|
||||
// True when `label` belongs in the cross-file name registry (see cbm.h). Single
|
||||
|
||||
@@ -547,7 +547,16 @@ static const char *objc_var_types[] = {"declaration", NULL};
|
||||
static const char *objc_assign_types[] = {"assignment_expression", NULL};
|
||||
|
||||
// ==================== SWIFT ====================
|
||||
static const char *swift_func_types[] = {"function_declaration", "macro_declaration", NULL};
|
||||
// protocol_function_declaration: a protocol's method requirements. Needed here as
|
||||
// well as in the name resolvers — extract_class_methods gates on this set, so
|
||||
// without the entry the requirement is walked and then discarded.
|
||||
static const char *swift_func_types[] = {"function_declaration", "protocol_function_declaration",
|
||||
"macro_declaration", NULL};
|
||||
// KNOWN GAP: struct_declaration and enum_declaration are not node types in the
|
||||
// vendored tree-sitter-swift grammar — it models both as class_declaration — so
|
||||
// these two entries never match anything, and a bare Swift `enum` is labeled
|
||||
// Class rather than Enum. Left in place deliberately: they are the only marker
|
||||
// of that modelling gap, and deleting them would hide it. Tracked separately.
|
||||
static const char *swift_class_types[] = {"class_declaration", "protocol_declaration",
|
||||
"struct_declaration", "enum_declaration", NULL};
|
||||
static const char *swift_field_types[] = {"property_declaration", NULL};
|
||||
|
||||
+3
-3
@@ -7,7 +7,7 @@
|
||||
|
||||
**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack.
|
||||
|
||||
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 159 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 43 automatic/conditional client surfaces.
|
||||
High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 158 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 43 automatic/conditional client surfaces.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -29,7 +29,7 @@ Restart your agent. Say **"Index this project"** — done.
|
||||
|
||||
- **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline with LZ4 compression and in-memory SQLite.
|
||||
- **Plug and play** — verified native runtime sets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). No Docker or API keys; Node.js owns package download, cache repair, and launch.
|
||||
- **159 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks.
|
||||
- **158 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks.
|
||||
- **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search.
|
||||
- **43 supported automatic/conditional client surfaces** — `install` configures the appropriate MCP, durable-context, and documented hook surfaces without widening client permissions.
|
||||
- **Detected automatically (37)** — Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob Shell, Pochi, and Pi.
|
||||
@@ -38,7 +38,7 @@ Restart your agent. Say **"Index this project"** — done.
|
||||
- **Lifecycle hooks stay conservative** — Kimi uses `UserPromptSubmit`; on macOS/Linux, GitLab Duo gets a fail-open user `SessionStart`, while Devin gets `UserPromptSubmit`, `PostCompaction`, and a deduplicated `SessionStart` when Claude does not already provide it. Qoder, GitLab Duo, Devin, and Factory hooks are withheld on Windows without a documented shell/executor contract. Cline's auto-activating file hooks are withheld because their context output is not reliably consumed, CodeBuddy beta hooks are not auto-installed, and Cursor context hooks remain withheld.
|
||||
- **Subagent access is explicit** — Claude, Gemini, Kiro, Qwen, CodeBuddy, KiloCode, Mistral Vibe, Qoder, Junie, and Factory get documented graph profiles with the narrowest tool/server filters their schemas support. KiloCode and Vibe enumerate read-only query tools rather than using server wildcards. Cursor, Rovo, Pochi, and Cline use explicit parent handoff where child MCP is unavailable or unsafe; IBM Bob receives no invented hook or agent.
|
||||
- **Manual, UI, cloud, or repository-managed (not counted)** — Qodo, Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection remains manual.
|
||||
- **14 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more.
|
||||
- **15 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more.
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
|
||||
@@ -332,9 +332,18 @@ static inline const int8_t *pretrained_vec_at(int i) {{
|
||||
|
||||
|
||||
def write_blob_s(path: str, incbin_path: str):
|
||||
"""Write assembler .incbin directive."""
|
||||
with open(path, "w") as f:
|
||||
f.write(f"""/* nomic-embed-code vector blob embedded via assembler. */
|
||||
"""Write the assembler .incbin wrapper for every target object format.
|
||||
|
||||
Must stay byte-identical to the tracked vendored/nomic/code_vectors_blob.S:
|
||||
tests/test_nomic_blob_generator_contract.sh fails if the two drift. The ELF
|
||||
branch's .note.GNU-stack is load-bearing -- regenerating without it puts an
|
||||
executable stack back into every Linux release binary.
|
||||
"""
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(f"""/* nomic-embed-code vector blob embedded via assembler.
|
||||
* Cross-platform: macOS (Mach-O) vs Linux (ELF) vs Windows (COFF). */
|
||||
|
||||
#if defined(__APPLE__)
|
||||
.section __DATA,__const
|
||||
.globl _PRETRAINED_VECTOR_BLOB
|
||||
.globl _PRETRAINED_VECTOR_BLOB_LEN
|
||||
@@ -347,6 +356,46 @@ _PRETRAINED_VECTOR_BLOB_END:
|
||||
.p2align 2
|
||||
_PRETRAINED_VECTOR_BLOB_LEN:
|
||||
.long _PRETRAINED_VECTOR_BLOB_END - _PRETRAINED_VECTOR_BLOB
|
||||
|
||||
#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__)
|
||||
.section .rdata,"dr"
|
||||
.globl PRETRAINED_VECTOR_BLOB
|
||||
.globl PRETRAINED_VECTOR_BLOB_LEN
|
||||
.p2align 4
|
||||
PRETRAINED_VECTOR_BLOB:
|
||||
.incbin "{incbin_path}"
|
||||
PRETRAINED_VECTOR_BLOB_END:
|
||||
|
||||
.section .rdata,"dr"
|
||||
.p2align 2
|
||||
PRETRAINED_VECTOR_BLOB_LEN:
|
||||
.long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB
|
||||
|
||||
#else
|
||||
/* WHY: an ELF object that carries no .note.GNU-stack tells the linker
|
||||
* nothing about its stack requirement, and GNU ld then assumes the WORST
|
||||
* for the whole link — every Linux release binary shipped GNU_STACK RWE
|
||||
* because of this one omission. This is the only assembly source in the
|
||||
* build, so it alone decided that property. The note must stay even though
|
||||
* a blob of constant data obviously never executes: absence is the signal,
|
||||
* not the contents. -Wl,-z,noexecstack in the link flags enforces the
|
||||
* outcome, and scripts/ci/check-binary-composition.sh fails the release if
|
||||
* an executable stack ever comes back. */
|
||||
.section .note.GNU-stack,"",@progbits
|
||||
|
||||
.section .rodata,"a",@progbits
|
||||
.globl PRETRAINED_VECTOR_BLOB
|
||||
.globl PRETRAINED_VECTOR_BLOB_LEN
|
||||
.p2align 4
|
||||
PRETRAINED_VECTOR_BLOB:
|
||||
.incbin "{incbin_path}"
|
||||
PRETRAINED_VECTOR_BLOB_END:
|
||||
|
||||
.section .rodata,"a",@progbits
|
||||
.p2align 2
|
||||
PRETRAINED_VECTOR_BLOB_LEN:
|
||||
.long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB
|
||||
#endif
|
||||
""")
|
||||
print(f" {path}: written")
|
||||
|
||||
|
||||
@@ -263,6 +263,9 @@ bash "$ROOT/tests/test_runtime_isolation_contract.sh"
|
||||
echo "=== Step 0u: shell line-ending contract ==="
|
||||
bash "$ROOT/tests/test_shell_line_endings.sh"
|
||||
|
||||
echo "=== Step 0v: nomic blob generator contract ==="
|
||||
bash "$ROOT/tests/test_nomic_blob_generator_contract.sh"
|
||||
|
||||
# Verify compiler supports target arch
|
||||
verify_compiler "$CC"
|
||||
|
||||
|
||||
@@ -108,8 +108,8 @@ enum { SKIP_ONE = 1, PAIR_LEN = 2 };
|
||||
#define CBM_SQL_TYPE_LIKE_LABELS "'Class','Struct','Interface','Enum','Type','Trait'"
|
||||
#define CBM_SQL_CALLABLE_LABELS "'Function','Method'"
|
||||
#define CBM_SQL_CALLABLE_OR_TYPE_LABELS CBM_SQL_CALLABLE_LABELS "," CBM_SQL_TYPE_LIKE_LABELS
|
||||
/* SQL mirror of cbm_label_is_relation() (Table/View — data-lineage nodes),
|
||||
* pinned by tests/test_store_nodes.c the same way as the sets above. */
|
||||
#define CBM_SQL_RELATION_LABELS "'Table','View'"
|
||||
/* SQL mirror of cbm_label_is_relation() (Table/View/Model — data-lineage
|
||||
* nodes), pinned by tests/test_store_nodes.c the same way as the sets above. */
|
||||
#define CBM_SQL_RELATION_LABELS "'Table','View','Model'"
|
||||
|
||||
#endif /* CBM_CONSTANTS_H */
|
||||
|
||||
+60
-8
@@ -44,6 +44,12 @@ enum {
|
||||
MAIN_PATH_CAP = 4096,
|
||||
MAIN_CONNECT_TIMEOUT_MS = 1000,
|
||||
MAIN_STARTUP_TIMEOUT_MS = 10000,
|
||||
/* Backstop for waiting out a held startup transition — see
|
||||
* main_local_transition_acquire. Not a budget for healthy contention: a busy
|
||||
* lock always resolves, either because the live holder finishes or because
|
||||
* the OS finishes reclaiming a dead holder's lock. This only bounds a peer
|
||||
* that never finishes, so a command cannot hang indefinitely. */
|
||||
MAIN_STARTUP_CONTENTION_CEILING_MS = 120000,
|
||||
MAIN_MCP_STARTUP_TIMEOUT_MS = 30000,
|
||||
MAIN_REQUEST_TIMEOUT_MS = 24 * 60 * 60 * 1000,
|
||||
MAIN_HOOK_CONNECT_TIMEOUT_MS = 250,
|
||||
@@ -1288,14 +1294,47 @@ static bool main_local_cli_feedback_enabled(int argc, char **argv) {
|
||||
return cbm_cli_progress_enabled(requested, cli_isatty(2) != 0);
|
||||
}
|
||||
|
||||
/* Acquire the exclusive startup transition, waiting out whatever currently holds it.
|
||||
*
|
||||
* try_acquire answers one of three things, and they must not be conflated:
|
||||
* 1 acquired
|
||||
* -1 coordination is unsafe or unverifiable — fail now, waiting cannot help
|
||||
* 0 BUSY: the lock is held
|
||||
*
|
||||
* A busy lock always resolves, by one of two routes:
|
||||
* - a live peer holds it, and releases when its command finishes;
|
||||
* - the holder is already dead and the operating system has not finished
|
||||
* reclaiming the lock yet.
|
||||
*
|
||||
* The second route is Windows-specific and is why this wait needs room. On POSIX
|
||||
* the kernel drops flock the instant the owner dies. Windows byte-range locks do
|
||||
* not work that way: Microsoft documents that after a process terminates holding
|
||||
* one, "the time it takes for the operating system to unlock these locks depends
|
||||
* upon available system resources", and that until then "access to these files
|
||||
* may be denied". A loaded CI runner is precisely where those resources are
|
||||
* scarce, and `tests/windows/test_daemon_stability.py` manufactures the situation
|
||||
* deliberately — it hard-kills daemons with `taskkill /F`, including a crash
|
||||
* recovery section, so the next client meets a lock whose owner no longer exists.
|
||||
*
|
||||
* Waiting was previously capped at MAIN_STARTUP_TIMEOUT_MS (10s), which let a
|
||||
* clock decide a user-visible outcome: a command was refused with "coordination
|
||||
* remained busy" while the only thing wrong was that the OS had not yet swept up
|
||||
* after a killed process. Since both routes above terminate, the correct response
|
||||
* to busy is to keep waiting; the ceiling below is a backstop against a peer that
|
||||
* never finishes, not a budget for healthy contention. Clean exits already
|
||||
* release through main_local_transition_close, so this path is only reached after
|
||||
* an abrupt termination or under genuine concurrency. */
|
||||
static int main_local_transition_acquire(const cbm_daemon_ipc_endpoint_t *endpoint, FILE *feedback,
|
||||
cbm_daemon_ipc_local_transition_t **transition_out) {
|
||||
uint64_t deadline = main_deadline_after(MAIN_STARTUP_TIMEOUT_MS);
|
||||
uint64_t ceiling = main_deadline_after(MAIN_STARTUP_CONTENTION_CEILING_MS);
|
||||
bool waiting_reported = false;
|
||||
for (;;) {
|
||||
int status = cbm_daemon_ipc_local_transition_try_acquire(endpoint, transition_out);
|
||||
if (status != 0 || cbm_now_ms() >= deadline) {
|
||||
return status;
|
||||
if (status != 0) {
|
||||
return status; /* acquired, or a genuine coordination failure */
|
||||
}
|
||||
if (cbm_now_ms() >= ceiling) {
|
||||
return 0; /* still busy after the backstop */
|
||||
}
|
||||
if (feedback && !waiting_reported) {
|
||||
(void)fputs("Waiting for CBM startup coordination...\n", feedback);
|
||||
@@ -2542,11 +2581,24 @@ int main(int argc, char **argv) {
|
||||
int transition_status =
|
||||
main_local_transition_acquire(local_endpoint, feedback, &local_transition);
|
||||
if (transition_status != 1 || !local_transition) {
|
||||
(void)fprintf(stderr,
|
||||
"codebase-memory-mcp: CLI startup coordination %s; retry after the "
|
||||
"active CBM transition exits\n",
|
||||
transition_status == 0 ? "remained busy"
|
||||
: "could not be verified safely");
|
||||
if (transition_status == 0) {
|
||||
/* The backstop fired. Name both explanations: after this much
|
||||
* waiting the likely causes are a peer that is genuinely stuck,
|
||||
* or (on Windows) a lock left behind by a force-killed process
|
||||
* that the OS has not reclaimed. "Busy" alone sent reporters
|
||||
* hunting for a CBM session that had already exited. */
|
||||
(void)fprintf(stderr,
|
||||
"codebase-memory-mcp: CLI startup coordination stayed busy for "
|
||||
"%d seconds. Either another CBM command is still running, or a "
|
||||
"previous one was force-killed and the operating system has not "
|
||||
"released its lock yet. Check for running CBM processes; if there "
|
||||
"are none, retry shortly.\n",
|
||||
MAIN_STARTUP_CONTENTION_CEILING_MS / 1000);
|
||||
} else {
|
||||
(void)fprintf(stderr, "codebase-memory-mcp: CLI startup coordination could not "
|
||||
"be verified safely; retry after active CBM sessions "
|
||||
"exit\n");
|
||||
}
|
||||
goto local_cli_cleanup;
|
||||
}
|
||||
int seal_status = cbm_daemon_ipc_local_transition_seal_legacy(local_transition);
|
||||
|
||||
@@ -24,6 +24,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/limits.h"
|
||||
#include "foundation/str_util.h"
|
||||
#include "cbm.h"
|
||||
#include "arena.h"
|
||||
#include "iris_export_xml.h"
|
||||
@@ -464,9 +465,10 @@ static int create_import_edges_for_file(cbm_pipeline_ctx_t *ctx, const CBMFileRe
|
||||
const cbm_gbuf_node_t *target =
|
||||
cbm_pipeline_resolve_import_node(ctx, rel, file_qn, imp, namespace_map);
|
||||
if (target && target->id != source_node->id) {
|
||||
char esc_ln[CBM_SZ_128];
|
||||
cbm_json_escape(esc_ln, sizeof(esc_ln), imp->local_name ? imp->local_name : "");
|
||||
char imp_props[CBM_SZ_256];
|
||||
snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}",
|
||||
imp->local_name ? imp->local_name : "");
|
||||
snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}", esc_ln);
|
||||
cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, target->id, "IMPORTS", imp_props);
|
||||
count++;
|
||||
}
|
||||
|
||||
@@ -1941,10 +1941,12 @@ static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, c
|
||||
cbm_gbuf_upsert_node(gbuf, "Route", p, route_qn, "", 0, 0, "{\"source\":\"graphql\"}");
|
||||
|
||||
char esc_c[CBM_SZ_256];
|
||||
char esc_op[CBM_SZ_512];
|
||||
cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name);
|
||||
cbm_json_escape(esc_op, sizeof(esc_op), p);
|
||||
char props[CBM_SZ_1K];
|
||||
snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"operation\":\"%s\",\"confidence\":%.2f}",
|
||||
esc_c, p, res->confidence);
|
||||
esc_c, esc_op, res->confidence);
|
||||
cbm_gbuf_insert_edge(gbuf, source->id, route_id, "GRAPHQL_CALLS", props);
|
||||
}
|
||||
|
||||
|
||||
+76
-2
@@ -54,6 +54,7 @@
|
||||
#include <windows.h>
|
||||
#include <process.h>
|
||||
#include <psapi.h> /* GetProcessMemoryInfo */
|
||||
#include <tlhelp32.h> /* CreateToolhelp32Snapshot, Process32First/Next */
|
||||
#else
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
@@ -560,7 +561,7 @@ static void handle_processes(cbm_http_conn_t *c) {
|
||||
int pos = 0;
|
||||
|
||||
#ifdef _WIN32
|
||||
/* Windows: GetProcessMemoryInfo + GetProcessTimes */
|
||||
/* Windows: GetProcessMemoryInfo + GetProcessTimes for the current process */
|
||||
PROCESS_MEMORY_COUNTERS pmc;
|
||||
FILETIME ft_create, ft_exit, ft_kernel, ft_user;
|
||||
double user_s = 0, sys_s = 0;
|
||||
@@ -578,8 +579,81 @@ static void handle_processes(cbm_http_conn_t *c) {
|
||||
}
|
||||
http_appendf(buf, sizeof(buf), &pos,
|
||||
"{\"self_pid\":%d,\"self_rss_mb\":%.1f,"
|
||||
"\"self_user_cpu_s\":%.1f,\"self_sys_cpu_s\":%.1f,\"processes\":[]}",
|
||||
"\"self_user_cpu_s\":%.1f,\"self_sys_cpu_s\":%.1f,\"processes\":[",
|
||||
(int)_getpid(), (double)rss_bytes / (1024.0 * 1024.0), user_s, sys_s);
|
||||
|
||||
/* Enumerate all codebase-memory-mcp.exe processes via toolhelp snapshot */
|
||||
int proc_count = 0;
|
||||
HANDLE hSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
|
||||
if (hSnap != INVALID_HANDLE_VALUE) {
|
||||
PROCESSENTRY32 pe;
|
||||
pe.dwSize = sizeof(pe);
|
||||
for (BOOL ok = Process32First(hSnap, &pe); ok; ok = Process32Next(hSnap, &pe)) {
|
||||
if (_stricmp(pe.szExeFile, "codebase-memory-mcp.exe") == 0) {
|
||||
HANDLE hProc = OpenProcess(
|
||||
PROCESS_QUERY_INFORMATION | PROCESS_VM_READ,
|
||||
FALSE, pe.th32ProcessID);
|
||||
if (hProc) {
|
||||
PROCESS_MEMORY_COUNTERS ppmc;
|
||||
FILETIME ftc, fte, ftk, ftu;
|
||||
double cpu_user = 0, cpu_sys = 0;
|
||||
size_t proc_rss = 0;
|
||||
DWORD elapsed_sec = 0;
|
||||
|
||||
if (GetProcessMemoryInfo(hProc, &ppmc, sizeof(ppmc)))
|
||||
proc_rss = ppmc.WorkingSetSize;
|
||||
|
||||
if (GetProcessTimes(hProc, &ftc, &fte, &ftk, &ftu)) {
|
||||
ULARGE_INTEGER pu, pk;
|
||||
pu.LowPart = ftu.dwLowDateTime;
|
||||
pu.HighPart = ftu.dwHighDateTime;
|
||||
pk.LowPart = ftk.dwLowDateTime;
|
||||
pk.HighPart = ftk.dwHighDateTime;
|
||||
cpu_user = (double)pu.QuadPart / 1e7;
|
||||
cpu_sys = (double)pk.QuadPart / 1e7;
|
||||
|
||||
/* 进程创建以来的运行时间 */
|
||||
FILETIME now_ft;
|
||||
GetSystemTimeAsFileTime(&now_ft);
|
||||
ULARGE_INTEGER now_uli, start_uli;
|
||||
now_uli.LowPart = now_ft.dwLowDateTime;
|
||||
now_uli.HighPart = now_ft.dwHighDateTime;
|
||||
start_uli.LowPart = ftc.dwLowDateTime;
|
||||
start_uli.HighPart = ftc.dwHighDateTime;
|
||||
/* 时钟偏移保护:防止创建时间微偏未来导致 unsigned 下溢 */
|
||||
if (now_uli.QuadPart > start_uli.QuadPart) {
|
||||
ULONGLONG elapsed_100ns = now_uli.QuadPart - start_uli.QuadPart;
|
||||
elapsed_sec = (DWORD)(elapsed_100ns / 10000000ULL);
|
||||
}
|
||||
}
|
||||
|
||||
if (proc_count > 0)
|
||||
buf[pos++] = ',';
|
||||
http_appendf(buf, sizeof(buf), &pos,
|
||||
"{\"pid\":%lu,\"cpu\":%.1f,\"rss_mb\":%.1f,"
|
||||
"\"elapsed\":\"%lu-%02lu:%02lu:%02lu\","
|
||||
"\"command\":\"codebase-memory-mcp\","
|
||||
"\"is_self\":%s}",
|
||||
pe.th32ProcessID, cpu_user + cpu_sys,
|
||||
(double)proc_rss / (1024.0 * 1024.0),
|
||||
elapsed_sec / 86400,
|
||||
(elapsed_sec % 86400) / 3600,
|
||||
(elapsed_sec % 3600) / 60,
|
||||
elapsed_sec % 60,
|
||||
pe.th32ProcessID == (DWORD)_getpid()
|
||||
? "true" : "false");
|
||||
if (pos >= (int)sizeof(buf)) {
|
||||
pos = (int)sizeof(buf) - 1;
|
||||
}
|
||||
proc_count++;
|
||||
CloseHandle(hProc);
|
||||
}
|
||||
}
|
||||
}
|
||||
CloseHandle(hSnap);
|
||||
}
|
||||
|
||||
http_appendf(buf, sizeof(buf), &pos, "]}");
|
||||
#else
|
||||
struct rusage ru;
|
||||
getrusage(RUSAGE_SELF, &ru);
|
||||
|
||||
+133
-6
@@ -75,6 +75,10 @@ typedef struct {
|
||||
uint64_t last_dirty_sig; /* committed dirty-state signature */
|
||||
uint64_t pending_dirty_sig; /* observed at check time */
|
||||
char pending_head[CBM_SZ_128]; /* HEAD observed at check time */
|
||||
/* Hop from root_path up to the repository root ("" when they are the same),
|
||||
* from `rev-parse --show-cdup`. Porcelain paths are repository-relative, so
|
||||
* the signature needs this to stat them. Resolved once at baseline. */
|
||||
char repo_cdup[CBM_SZ_4K];
|
||||
} project_state_t;
|
||||
|
||||
/* ── Watcher struct ─────────────────────────────────────────────── */
|
||||
@@ -452,6 +456,86 @@ static watcher_git_status_t git_repo_status(cbm_watcher_t *w, project_state_t *s
|
||||
return watcher_git_run(w, state, argv, 0, NULL);
|
||||
}
|
||||
|
||||
/* True when root_path carries its OWN repository marker: a `.git` directory, or
|
||||
* a `.git` file (the gitlink form used by linked worktrees and initialized
|
||||
* submodules). Deliberately a filesystem check, not a git invocation — the whole
|
||||
* point is to learn something `rev-parse` cannot tell us, because it walks up. */
|
||||
static bool git_has_own_dot_git(const char *root_path) {
|
||||
char path[CBM_SZ_4K];
|
||||
int written = snprintf(path, sizeof(path), "%s/.git", root_path);
|
||||
if (written <= 0 || (size_t)written >= sizeof(path)) {
|
||||
return false;
|
||||
}
|
||||
struct stat st;
|
||||
return stat(path, &st) == 0 && (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode));
|
||||
}
|
||||
|
||||
/* Does the ancestor repository actually track anything inside this directory?
|
||||
* Emptiness distinguishes "a scratch folder that merely sits under a repo" from
|
||||
* "a genuine subdirectory of one". Output is capped: we only care whether the
|
||||
* first byte exists, never what it is. */
|
||||
static watcher_git_status_t git_tracks_anything_here(cbm_watcher_t *w, project_state_t *state,
|
||||
bool *tracked_out) {
|
||||
*tracked_out = false;
|
||||
const char *argv[] = {"git", "-C", state->root_path, "ls-files", "-z", "--", ".", NULL};
|
||||
watcher_git_output_t output;
|
||||
watcher_git_status_t status = watcher_git_run(w, state, argv, WATCHER_GIT_HEAD_MAX, &output);
|
||||
if (status != WATCHER_GIT_OK) {
|
||||
return status;
|
||||
}
|
||||
FILE *fp = cbm_fopen(output.path, "rb");
|
||||
if (fp) {
|
||||
*tracked_out = fgetc(fp) != EOF;
|
||||
(void)fclose(fp);
|
||||
}
|
||||
watcher_git_output_cleanup(&output);
|
||||
return fp ? WATCHER_GIT_OK : WATCHER_GIT_SUPERVISION_FAILED;
|
||||
}
|
||||
|
||||
/* Relative hop from root_path up to the repository root, as `git rev-parse
|
||||
* --show-cdup` reports it ("" at the root, "../" one level down, and so on).
|
||||
*
|
||||
* This matters because `git status --porcelain` prints paths relative to the
|
||||
* REPOSITORY root, while the signature stats them relative to root_path. For a
|
||||
* project watched at the repository root the two coincide and the bug is
|
||||
* invisible; for a subdirectory project every stat silently misses, and the
|
||||
* signature quietly degrades to text-only — losing the size/mtime component
|
||||
* that makes an edit to an already-dirty file detectable.
|
||||
*
|
||||
* --show-cdup rather than --show-toplevel: under MSYS/Cygwin git, --show-toplevel
|
||||
* returns a translated absolute path (/c/... or a drive-letter form) that does
|
||||
* not join cleanly onto the native root_path we hold. A relative hop composes
|
||||
* correctly on every platform because it never leaves our own path space. */
|
||||
static watcher_git_status_t git_repo_cdup(cbm_watcher_t *w, project_state_t *state, char *out,
|
||||
size_t out_size) {
|
||||
if (!out || out_size < 2) {
|
||||
return WATCHER_GIT_SUPERVISION_FAILED;
|
||||
}
|
||||
out[0] = '\0';
|
||||
const char *argv[] = {"git", "-C", state->root_path, "rev-parse", "--show-cdup", NULL};
|
||||
watcher_git_output_t output;
|
||||
watcher_git_status_t status = watcher_git_run(w, state, argv, WATCHER_GIT_HEAD_MAX, &output);
|
||||
if (status != WATCHER_GIT_OK) {
|
||||
return status;
|
||||
}
|
||||
FILE *file = cbm_fopen(output.path, "rb");
|
||||
bool read = file && fgets(out, (int)out_size, file) != NULL;
|
||||
if (file) {
|
||||
(void)fclose(file);
|
||||
}
|
||||
watcher_git_output_cleanup(&output);
|
||||
if (!read) {
|
||||
/* At the repository root git prints an empty line; that is success. */
|
||||
out[0] = '\0';
|
||||
return WATCHER_GIT_OK;
|
||||
}
|
||||
size_t len = strlen(out);
|
||||
while (len > 0 && (out[len - 1] == '\n' || out[len - 1] == '\r')) {
|
||||
out[--len] = '\0';
|
||||
}
|
||||
return WATCHER_GIT_OK;
|
||||
}
|
||||
|
||||
static watcher_git_status_t git_head(cbm_watcher_t *w, project_state_t *state, char *out,
|
||||
size_t out_size) {
|
||||
if (!out || out_size < 2) {
|
||||
@@ -511,9 +595,14 @@ static int64_t sig_stat_mtime_ns(const struct stat *st) {
|
||||
* of an already-dirty file still produces a new signature. A failed stat
|
||||
* (deleted file, quoting artifact) degrades to the entry text alone — the
|
||||
* deletion itself is represented by the porcelain status. */
|
||||
static uint64_t sig_fold_path_stat(uint64_t h, const char *root_path, const char *rel) {
|
||||
/* `rel` is repository-relative (that is what porcelain prints), so it is joined
|
||||
* through `cdup` — the hop from root_path up to the repository root — rather than
|
||||
* onto root_path directly. cdup is "" when the project IS the repository root,
|
||||
* which reduces this to the original join. */
|
||||
static uint64_t sig_fold_path_stat(uint64_t h, const char *root_path, const char *cdup,
|
||||
const char *rel) {
|
||||
char abs[CBM_SZ_4K];
|
||||
snprintf(abs, sizeof(abs), "%s/%s", root_path, rel);
|
||||
snprintf(abs, sizeof(abs), "%s/%s%s", root_path, cdup ? cdup : "", rel);
|
||||
struct stat st;
|
||||
if (stat(abs, &st) == 0) {
|
||||
int64_t mt = sig_stat_mtime_ns(&st);
|
||||
@@ -538,8 +627,16 @@ static watcher_git_status_t git_dirty_signature(cbm_watcher_t *w, project_state_
|
||||
return WATCHER_GIT_SUPERVISION_FAILED;
|
||||
}
|
||||
*signature_out = 0;
|
||||
const char *status_argv[] = {"git", "--no-optional-locks", "-C", state->root_path,
|
||||
"status", "--porcelain", "-uall", "-z",
|
||||
/* `-- .` scopes the report to the watched directory. Without it a project
|
||||
* watched at a sub-package of a monorepo reindexes whenever any SIBLING
|
||||
* package changes, because git reports the whole repository's dirty state
|
||||
* regardless of -C. Paths stay repository-relative either way, which is
|
||||
* what repo_cdup is for. */
|
||||
const char *status_argv[] = {"git", "--no-optional-locks",
|
||||
"-C", state->root_path,
|
||||
"status", "--porcelain",
|
||||
"-uall", "-z",
|
||||
"--", ".",
|
||||
NULL};
|
||||
watcher_git_output_t output;
|
||||
watcher_git_status_t status =
|
||||
@@ -585,7 +682,7 @@ static watcher_git_status_t git_dirty_signature(cbm_watcher_t *w, project_state_
|
||||
if (entry[0] == 'R' || entry[0] == 'C') {
|
||||
origin_token = true;
|
||||
}
|
||||
h = sig_fold_path_stat(h, state->root_path, entry + 3);
|
||||
h = sig_fold_path_stat(h, state->root_path, state->repo_cdup, entry + 3);
|
||||
}
|
||||
}
|
||||
elen = 0;
|
||||
@@ -638,7 +735,7 @@ static watcher_git_status_t git_dirty_signature(cbm_watcher_t *w, project_state_
|
||||
h = sig_fold(h, line, len);
|
||||
h = sig_fold(h, "", 1);
|
||||
if (len > 3 && line[2] == ' ') {
|
||||
h = sig_fold_path_stat(h, state->root_path, line + 3);
|
||||
h = sig_fold_path_stat(h, state->root_path, state->repo_cdup, line + 3);
|
||||
}
|
||||
}
|
||||
parsed = !ferror(fp) && fclose(fp) == 0;
|
||||
@@ -1060,9 +1157,39 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) {
|
||||
return false;
|
||||
}
|
||||
s->is_git = repository_status == WATCHER_GIT_OK;
|
||||
|
||||
/* `rev-parse --git-dir` walks UP, so an ordinary folder that merely happens
|
||||
* to live under some unrelated repository answers yes. Treating it as a git
|
||||
* project is what produced the runaway churn: it inherits the ancestor's
|
||||
* dirty state, which is permanently non-empty and has nothing to do with
|
||||
* this directory, so every poll looked like a change.
|
||||
*
|
||||
* A directory is only really git-managed here if it carries its own .git,
|
||||
* or the ancestor repository actually tracks something inside it. A
|
||||
* genuine sub-package of a monorepo passes the second test; a scratch or
|
||||
* gitignored folder sitting under a repo fails both and is polled as a
|
||||
* plain directory instead. */
|
||||
if (s->is_git && !git_has_own_dot_git(s->root_path)) {
|
||||
bool tracked = false;
|
||||
watcher_git_status_t tracked_status = git_tracks_anything_here(w, s, &tracked);
|
||||
if (tracked_status != WATCHER_GIT_OK && tracked_status != WATCHER_GIT_COMMAND_FAILED) {
|
||||
return false;
|
||||
}
|
||||
if (!tracked) {
|
||||
s->is_git = false;
|
||||
cbm_log_info("watcher.nested_non_git", "project", s->project_name, "path",
|
||||
s->root_path);
|
||||
}
|
||||
}
|
||||
|
||||
s->baseline_done = true;
|
||||
|
||||
if (s->is_git) {
|
||||
watcher_git_status_t cdup_status = git_repo_cdup(w, s, s->repo_cdup, sizeof(s->repo_cdup));
|
||||
if (cdup_status != WATCHER_GIT_OK && cdup_status != WATCHER_GIT_COMMAND_FAILED) {
|
||||
s->baseline_done = false;
|
||||
return false;
|
||||
}
|
||||
watcher_git_status_t head_status = git_head(w, s, s->last_head, sizeof(s->last_head));
|
||||
if (head_status != WATCHER_GIT_OK && head_status != WATCHER_GIT_COMMAND_FAILED) {
|
||||
s->baseline_done = false;
|
||||
|
||||
+40
-9
@@ -4447,24 +4447,55 @@ TEST(cli_install_plan_receipt_no_mutation_issue388) {
|
||||
test_mkdirp(dir);
|
||||
|
||||
char *json = cbm_build_install_plan_json(tmpdir, "/usr/local/bin/codebase-memory-mcp");
|
||||
ASSERT_NOT_NULL(json);
|
||||
ASSERT(strstr(json, "agent.install.plan.v1") != NULL);
|
||||
ASSERT(strstr(json, "writes_started") != NULL);
|
||||
ASSERT(strstr(json, "next_safe_command") != NULL);
|
||||
ASSERT(strstr(json, "cursor") != NULL);
|
||||
ASSERT(strstr(json, ".cursor/mcp.json") != NULL);
|
||||
ASSERT(strstr(json, ".codex/config.toml") != NULL);
|
||||
|
||||
/* WHY the failures are deferred: asserting inline returns before the free
|
||||
* and the rmdir below, so every red run leaked the receipt and left a
|
||||
* stray /tmp/cli-plan-* directory behind. That makes the next debugging
|
||||
* session harder than the failure it is reporting. Record what went wrong,
|
||||
* release everything, then fail -- and name the specific marker, because
|
||||
* "a marker was missing" costs a reader a bisect that "next_safe_command
|
||||
* was missing" does not. */
|
||||
const char *missing = NULL;
|
||||
if (!json) {
|
||||
missing = "receipt was NULL";
|
||||
} else {
|
||||
static const char *const required[] = {
|
||||
"agent.install.plan.v1", "writes_started", "next_safe_command", "cursor",
|
||||
".cursor/mcp.json", ".codex/config.toml",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) {
|
||||
if (!strstr(json, required[i])) {
|
||||
missing = required[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
free(json);
|
||||
|
||||
/* Critical: building the plan must NOT have created any config file. */
|
||||
char cfg[512];
|
||||
struct stat st;
|
||||
const char *created = NULL;
|
||||
snprintf(cfg, sizeof(cfg), "%s/.cursor/mcp.json", tmpdir);
|
||||
ASSERT(stat(cfg, &st) != 0); /* must not exist */
|
||||
if (stat(cfg, &st) == 0) {
|
||||
created = ".cursor/mcp.json";
|
||||
}
|
||||
snprintf(cfg, sizeof(cfg), "%s/.codex/config.toml", tmpdir);
|
||||
ASSERT(stat(cfg, &st) != 0); /* must not exist */
|
||||
if (!created && stat(cfg, &st) == 0) {
|
||||
created = ".codex/config.toml";
|
||||
}
|
||||
|
||||
test_rmdir_r(tmpdir);
|
||||
|
||||
char reason[256];
|
||||
if (missing) {
|
||||
snprintf(reason, sizeof(reason), "install plan receipt is missing %s", missing);
|
||||
FAIL(reason);
|
||||
}
|
||||
if (created) {
|
||||
snprintf(reason, sizeof(reason), "building an install plan created %s", created);
|
||||
FAIL(reason);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
|
||||
@@ -472,6 +472,15 @@ TEST(java_enum_dedup_preserves_calls_issue1234) {
|
||||
ASSERT(has_def(r, "Enum", "Day"));
|
||||
ASSERT(has_def(r, "Method", "isWeekend"));
|
||||
ASSERT(has_def(r, "Method", "label"));
|
||||
/* The enum CONSTANTS must survive alongside the methods. Reaching the
|
||||
* methods means descending into enum_body_declarations, and the tempting
|
||||
* way to do that -- redirecting the shared find_class_body -- also makes
|
||||
* the constants unreachable, because they are siblings of that node rather
|
||||
* than children. find_class_member_body exists to descend for members only
|
||||
* and leave find_class_body (which extract_enum_members uses) alone; these
|
||||
* two assertions are what stop that distinction being collapsed again. */
|
||||
ASSERT(has_def(r, "Variable", "MON"));
|
||||
ASSERT(has_def(r, "Variable", "SUN"));
|
||||
ASSERT(has_def(r, "Class", "DayUtil"));
|
||||
ASSERT(has_def(r, "Method", "describe"));
|
||||
ASSERT_EQ(count_defs_with_label(r, "Function"), 0);
|
||||
@@ -655,6 +664,21 @@ TEST(swift_class) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(swift_protocol) {
|
||||
/* A protocol requirement is a bodyless func inside a protocol body. Swift
|
||||
* codebases are heavily protocol-driven, so the requirement is very often
|
||||
* the declaration a reader is actually looking for — before this it was
|
||||
* absent from the graph entirely. */
|
||||
CBMFileResult *r = extract("protocol StudyRunning {\n func generate() -> String\n}\n",
|
||||
CBM_LANG_SWIFT, "t", "StudyRunning.swift");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(r->has_error);
|
||||
ASSERT(has_def(r, "Interface", "StudyRunning"));
|
||||
ASSERT(has_def(r, "Method", "generate"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* --- Kotlin --- */
|
||||
TEST(kotlin_function) {
|
||||
CBMFileResult *r = extract("fun greet(name: String): String = \"Hello $name\"\nfun main() { "
|
||||
@@ -1868,6 +1892,100 @@ TEST(sql_schema_qualified_name) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* --- dbt Jinja lineage --- */
|
||||
|
||||
/* Helper: does the file's usage list carry `name`? */
|
||||
static int has_usage(CBMFileResult *r, const char *name) {
|
||||
for (int i = 0; i < r->usages.count; i++) {
|
||||
if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, name) == 0) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
TEST(dbt_model_and_ref_lineage) {
|
||||
/* A dbt model: the file stem is the model identity, and each ref() is a
|
||||
* dependency on another model. The SQL grammar cannot read `{{ ref(..) }}`
|
||||
* at all, so without the dbt pass this file yields no lineage whatsoever. */
|
||||
CBMFileResult *r = extract("SELECT o.id, c.name\n"
|
||||
"FROM {{ ref('stg_orders') }} o\n"
|
||||
"JOIN {{ ref('stg_customers') }} c ON c.id = o.customer_id\n",
|
||||
CBM_LANG_SQL, "t", "models/marts/orders_enriched.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT(has_def(r, "Model", "orders_enriched"));
|
||||
ASSERT(has_usage(r, "stg_orders"));
|
||||
ASSERT(has_usage(r, "stg_customers"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(dbt_source_and_two_arg_ref) {
|
||||
/* Both dbt builtins name the relation in their LAST string argument:
|
||||
* source('group','table') -> table, and the two-argument
|
||||
* ref('package','model') form -> model. */
|
||||
CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n"
|
||||
"UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n",
|
||||
CBM_LANG_SQL, "t", "models/stg_customers.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT(has_def(r, "Model", "stg_customers"));
|
||||
ASSERT(has_usage(r, "customers"));
|
||||
ASSERT(has_usage(r, "legacy_customers"));
|
||||
/* the group/package argument is not the relation */
|
||||
ASSERT_FALSE(has_usage(r, "raw"));
|
||||
ASSERT_FALSE(has_usage(r, "analytics"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(dbt_ignores_non_dbt_jinja) {
|
||||
/* Templated SQL is not dbt SQL. An Airflow-style parameter substitution has
|
||||
* Jinja but no dbt builtin, so the dbt pass must contribute NOTHING — no
|
||||
* Model node named after the file, and no usage minted from the template
|
||||
* variables. This is the gate that keeps every non-dbt repository free of
|
||||
* fabricated data-lineage vocabulary.
|
||||
*
|
||||
* The ordinary SQL identifier path is unaffected and still sees the literal
|
||||
* `FROM events`; the second extraction below is the control proving that
|
||||
* usage is pre-existing SQL behaviour rather than anything dbt added. */
|
||||
CBMFileResult *r = extract("SELECT * FROM events WHERE day = '{{ ds }}'\n"
|
||||
" AND region = '{{ params.region_code }}'\n",
|
||||
CBM_LANG_SQL, "t", "queries/daily_events.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(has_def(r, "Model", "daily_events"));
|
||||
|
||||
/* Control: the same statement with the templates replaced by plain string
|
||||
* literals. Both parse as SQL identically, so an equal usage count is the
|
||||
* precise statement of "the dbt pass contributed nothing here" — stronger
|
||||
* than naming individual identifiers, and immune to how SQL happens to
|
||||
* tokenize the template text. */
|
||||
CBMFileResult *plain = extract("SELECT * FROM events WHERE day = '2026-01-01'\n"
|
||||
" AND region = 'eu-west'\n",
|
||||
CBM_LANG_SQL, "t", "queries/daily_events.sql");
|
||||
ASSERT_NOT_NULL(plain);
|
||||
ASSERT_FALSE(has_def(plain, "Model", "daily_events"));
|
||||
ASSERT_EQ(r->usages.count, plain->usages.count);
|
||||
ASSERT_EQ(r->defs.count, plain->defs.count);
|
||||
cbm_free_result(plain);
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(dbt_plain_sql_untouched) {
|
||||
/* Plain DDL keeps producing exactly the Table/View relations it did before
|
||||
* the dbt pass existed — no Model node, and the FROM lineage is unchanged. */
|
||||
CBMFileResult *r = extract("CREATE TABLE users (id INTEGER);\n"
|
||||
"CREATE VIEW active_users AS SELECT * FROM users;\n",
|
||||
CBM_LANG_SQL, "t", "schema.sql");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(r->has_error);
|
||||
ASSERT(has_def(r, "Table", "users"));
|
||||
ASSERT(has_def(r, "View", "active_users"));
|
||||
ASSERT_FALSE(has_def(r, "Model", "schema"));
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* --- Meson project --- */
|
||||
TEST(meson_project) {
|
||||
CBMFileResult *r = extract(
|
||||
@@ -5451,6 +5569,7 @@ SUITE(extraction) {
|
||||
RUN_TEST(csharp_class);
|
||||
RUN_TEST(csharp_interface);
|
||||
RUN_TEST(swift_class);
|
||||
RUN_TEST(swift_protocol);
|
||||
RUN_TEST(kotlin_function);
|
||||
RUN_TEST(kotlin_class);
|
||||
RUN_TEST(scala_function);
|
||||
@@ -5555,6 +5674,10 @@ SUITE(extraction) {
|
||||
RUN_TEST(sql_ddl_node_labels);
|
||||
RUN_TEST(sql_view_lineage_usages);
|
||||
RUN_TEST(sql_schema_qualified_name);
|
||||
RUN_TEST(dbt_model_and_ref_lineage);
|
||||
RUN_TEST(dbt_source_and_two_arg_ref);
|
||||
RUN_TEST(dbt_ignores_non_dbt_jinja);
|
||||
RUN_TEST(dbt_plain_sql_untouched);
|
||||
RUN_TEST(meson_project);
|
||||
RUN_TEST(css_rules);
|
||||
RUN_TEST(scss_rules);
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
#include <store/store.h>
|
||||
#include <pipeline/pipeline.h>
|
||||
#include <foundation/log.h>
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
@@ -1656,7 +1657,9 @@ TEST(contract_edge_parallel_service_edges) {
|
||||
{"gql.py", "def gql(query_string):\n return query_string\n"},
|
||||
{"client.py",
|
||||
"from gql import gql\n\n\ndef fetch_user():\n"
|
||||
" return gql(\"query GetUser { user { id name } }\")\n\n\n"
|
||||
/* The embedded double quote survives extraction from this single-quoted
|
||||
* Python string and breaks GRAPHQL_CALLS JSON unless operation is escaped. */
|
||||
" return gql('query GetUser { user(name: \"quoted\") { id name } }')\n\n\n"
|
||||
"def create_user():\n"
|
||||
" return gql(\"mutation CreateUser { addUser(name: \\\"x\\\") { id } }\")\n"},
|
||||
/* TRPC_CALLS: local createTRPCProxyClient (same-module resolution). */
|
||||
@@ -1697,6 +1700,19 @@ TEST(contract_edge_parallel_service_edges) {
|
||||
int grpc = store ? cbm_store_count_edges_by_type(store, lp.project, "GRPC_CALLS") : -1;
|
||||
int trpc = store ? cbm_store_count_edges_by_type(store, lp.project, "TRPC_CALLS") : -1;
|
||||
int infra = store ? cbm_store_count_edges_by_type(store, lp.project, "INFRA_MAPS") : -1;
|
||||
int invalid_props = -1;
|
||||
if (store) {
|
||||
sqlite3_stmt *stmt = NULL;
|
||||
sqlite3 *db = cbm_store_get_db(store);
|
||||
if (db && sqlite3_prepare_v2(db,
|
||||
"SELECT count(*) FROM edges WHERE properties IS NOT NULL "
|
||||
"AND properties != '' AND json_valid(properties)=0;",
|
||||
-1, &stmt, NULL) == SQLITE_OK &&
|
||||
sqlite3_step(stmt) == SQLITE_ROW) {
|
||||
invalid_props = sqlite3_column_int(stmt, 0);
|
||||
}
|
||||
sqlite3_finalize(stmt);
|
||||
}
|
||||
if (graphql < 1 || grpc < 1 || trpc < 1 || infra < 1) {
|
||||
fprintf(stderr,
|
||||
" [EDGE] parallel-service: GRAPHQL_CALLS=%d GRPC_CALLS=%d TRPC_CALLS=%d "
|
||||
@@ -1709,6 +1725,7 @@ TEST(contract_edge_parallel_service_edges) {
|
||||
ASSERT_TRUE(grpc >= 1);
|
||||
ASSERT_TRUE(trpc >= 1);
|
||||
ASSERT_TRUE(infra >= 1);
|
||||
ASSERT_EQ(invalid_props, 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Contract: scripts/extract_nomic_vectors.py must still emit the tracked
|
||||
# vendored/nomic/code_vectors_blob.S byte for byte.
|
||||
#
|
||||
# WHY this exists: the .S is a GENERATED file that was hand-edited to add the
|
||||
# ELF .note.GNU-stack section. Without that note GNU ld assumes the whole link
|
||||
# needs an executable stack, and every Linux release binary shipped GNU_STACK
|
||||
# RWE because of the omission. The generator kept emitting only the Mach-O
|
||||
# branch, so the next regeneration would have silently reverted the hardening
|
||||
# and reintroduced the defect. check-binary-composition.sh catches that at
|
||||
# release time; this catches it at edit time.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
GEN="$ROOT/scripts/extract_nomic_vectors.py"
|
||||
TRACKED="$ROOT/vendored/nomic/code_vectors_blob.S"
|
||||
|
||||
for f in "$GEN" "$TRACKED"; do
|
||||
if [ ! -f "$f" ]; then
|
||||
echo "FAIL: missing $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# The generator holds the .S as an f-string whose only placeholder is the
|
||||
# incbin path, so the tracked file with that path re-parameterized must appear
|
||||
# in it verbatim. Comparing the template rather than running the generator
|
||||
# keeps this test free of the torch/transformers import the script needs.
|
||||
python3 - "$GEN" "$TRACKED" <<'PYEOF'
|
||||
import sys
|
||||
|
||||
gen_path, tracked_path = sys.argv[1], sys.argv[2]
|
||||
|
||||
with open(gen_path, encoding="utf-8") as fh:
|
||||
generator = fh.read()
|
||||
with open(tracked_path, encoding="utf-8") as fh:
|
||||
tracked = fh.read()
|
||||
|
||||
INCBIN = "vendored/nomic/code_vectors.bin"
|
||||
if INCBIN not in tracked:
|
||||
sys.exit("FAIL: %s no longer references %s" % (tracked_path, INCBIN))
|
||||
if "{" in tracked or "}" in tracked:
|
||||
sys.exit("FAIL: %s gained a brace; the generator f-string would need "
|
||||
"escaping and this contract can no longer compare them directly"
|
||||
% tracked_path)
|
||||
|
||||
template = tracked.replace(INCBIN, "{incbin_path}")
|
||||
|
||||
if template not in generator:
|
||||
sys.exit(
|
||||
"FAIL: scripts/extract_nomic_vectors.py no longer emits the tracked\n"
|
||||
" vendored/nomic/code_vectors_blob.S.\n"
|
||||
" Regenerating the blob wrapper would overwrite the tracked file\n"
|
||||
" with different content. Update write_blob_s() so its f-string\n"
|
||||
" matches the tracked .S with the incbin path parameterized.")
|
||||
|
||||
# The note is the whole point; assert it explicitly so a future edit that keeps
|
||||
# the files in sync while dropping the note still fails loudly.
|
||||
NOTE = '.section .note.GNU-stack,"",@progbits'
|
||||
for label, blob in (("tracked .S", tracked), ("generator template", generator)):
|
||||
if NOTE not in blob:
|
||||
sys.exit("FAIL: %s lost the ELF %s section -- Linux binaries would ship "
|
||||
"an executable stack again" % (label, NOTE))
|
||||
|
||||
print("ok: generator template is byte-identical to the tracked blob wrapper")
|
||||
PYEOF
|
||||
@@ -2250,6 +2250,48 @@ TEST(pipeline_sql_lineage_and_relation_isolation) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* dbt lineage end-to-end. A dbt project's dependency structure lives entirely
|
||||
* in Jinja ({{ ref('x') }}), which the SQL grammar cannot read, so this is the
|
||||
* whole value: model -> model edges across files, plus the join onto a Table
|
||||
* declared in ordinary DDL — Model and Table are both relation labels, so one
|
||||
* lineage layer spans both. The Python file is the isolation control: `stg_orders`
|
||||
* exists project-wide only as a dbt model, and the registry's relation veto must
|
||||
* keep a same-named call out of the lineage layer. */
|
||||
TEST(pipeline_dbt_jinja_lineage) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dbt_lineage_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmp)) {
|
||||
FAIL("tmpdir");
|
||||
}
|
||||
write_temp_file(tmp, "raw_schema.sql", "CREATE TABLE customers (id INTEGER, name TEXT);\n");
|
||||
write_temp_file(tmp, "stg_orders.sql",
|
||||
"SELECT id, customer_id FROM {{ source('raw', 'customers') }}\n");
|
||||
write_temp_file(tmp, "orders_enriched.sql",
|
||||
"SELECT o.id, c.name\n"
|
||||
"FROM {{ ref('stg_orders') }} o\n"
|
||||
"JOIN {{ ref('stg_orders') }} c ON c.id = o.customer_id\n");
|
||||
write_temp_file(tmp, "app.py", "def load():\n return stg_orders()\n");
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/dbt.db", tmp);
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
const char *project = cbm_pipeline_project_name(p);
|
||||
cbm_store_t *s = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(s);
|
||||
/* model -> model: the ref() lineage the SQL grammar cannot see */
|
||||
ASSERT_TRUE(named_edge_count(s, project, "USAGE", "orders_enriched", "stg_orders") >= 1);
|
||||
/* model -> table: source() joining dbt onto plain DDL in the same repo */
|
||||
ASSERT_EQ(named_edge_count(s, project, "USAGE", "stg_orders", "customers"), 1);
|
||||
/* isolation: the Python call must not reach the model */
|
||||
ASSERT_EQ(named_edge_count(s, project, "CALLS", "load", "stg_orders"), 0);
|
||||
ASSERT_EQ(named_edge_count(s, project, "USAGE", "load", "stg_orders"), 0);
|
||||
cbm_store_close(s);
|
||||
cbm_pipeline_free(p);
|
||||
th_rmtree(tmp);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Renaming a table must drop lineage from DEPENDENT (unchanged) SQL files on
|
||||
* the incremental path. Table/View participate in the per-file LSP surface
|
||||
* hash as registry-only labels (lsp_surface.c), so tables.sql's def change
|
||||
@@ -12348,6 +12390,7 @@ SUITE(pipeline_semantic_manifest_repro) {
|
||||
RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge);
|
||||
RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
|
||||
RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
|
||||
RUN_TEST(pipeline_dbt_jinja_lineage);
|
||||
RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold);
|
||||
RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full);
|
||||
RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge);
|
||||
|
||||
@@ -57,7 +57,7 @@ TEST(sql_label_allowlists_match_cbm_label_is_type_like) {
|
||||
* cbm_registry_resolve vetoes them, so a code identifier sharing a table's
|
||||
* name never binds into the lineage layer. */
|
||||
TEST(sql_relation_labels_match_cbm_label_is_relation) {
|
||||
static const char *const relations[] = {"Table", "View"};
|
||||
static const char *const relations[] = {"Table", "View", "Model"};
|
||||
for (size_t i = 0; i < sizeof(relations) / sizeof(relations[0]); i++) {
|
||||
ASSERT_TRUE(cbm_label_is_relation(relations[i]));
|
||||
ASSERT_TRUE(cbm_label_is_registry_symbol(relations[i]));
|
||||
|
||||
@@ -1226,6 +1226,134 @@ TEST(watcher_detects_git_commit) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A plain directory that merely SITS UNDER an unrelated repository is not a git
|
||||
* project. `git rev-parse --git-dir` walks up, so it answers yes for such a
|
||||
* folder, and the watcher then inherited the ancestor's dirty state — which is
|
||||
* permanently non-empty and has nothing to do with this directory — and
|
||||
* reindexed on every single poll forever (#841/#937: reporters measured this in
|
||||
* hundreds of GB of writes per day). */
|
||||
TEST(watcher_nested_non_git_dir_does_not_inherit_ancestor_dirt) {
|
||||
char tmpdir[256];
|
||||
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_nested_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmpdir))
|
||||
FAIL("cbm_mkdtemp failed");
|
||||
|
||||
if (wt_git(tmpdir, "init -q") != 0) {
|
||||
th_rmtree(tmpdir);
|
||||
FAIL("git init failed");
|
||||
}
|
||||
{
|
||||
char p[300];
|
||||
th_write_file(wt_path(p, sizeof(p), tmpdir, "tracked.txt"), "hello\n");
|
||||
}
|
||||
wt_git(tmpdir, "add tracked.txt");
|
||||
wt_git(tmpdir, "commit -q -m init");
|
||||
|
||||
/* Leave the ancestor permanently dirty — this is the condition that used to
|
||||
* retrigger indexing on every poll of the nested directory. */
|
||||
{
|
||||
char p[300];
|
||||
th_write_file(wt_path(p, sizeof(p), tmpdir, "dirty.txt"), "uncommitted\n");
|
||||
}
|
||||
|
||||
/* A scratch directory inside it, tracked by nothing. */
|
||||
char nested[400];
|
||||
snprintf(nested, sizeof(nested), "%s/scratch", tmpdir);
|
||||
if (!cbm_mkdir_p(nested, 0755)) {
|
||||
th_rmtree(tmpdir);
|
||||
FAIL("mkdir nested failed");
|
||||
}
|
||||
{
|
||||
char p[500];
|
||||
th_write_file(wt_path(p, sizeof(p), nested, "notes.md"), "scratch\n");
|
||||
}
|
||||
|
||||
cbm_store_t *store = cbm_store_open_memory();
|
||||
cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL);
|
||||
cbm_watcher_watch(w, "nested-scratch", nested);
|
||||
index_call_count = 0;
|
||||
|
||||
cbm_watcher_poll_once(w); /* baseline */
|
||||
int after_baseline = index_call_count;
|
||||
|
||||
/* Three polls with the ancestor still dirty and the nested dir untouched.
|
||||
* Under the old classification each of these reindexed. */
|
||||
for (int i = 0; i < 3; i++) {
|
||||
cbm_watcher_touch(w, "nested-scratch");
|
||||
cbm_watcher_poll_once(w);
|
||||
}
|
||||
ASSERT_EQ(index_call_count, after_baseline);
|
||||
|
||||
cbm_watcher_free(w);
|
||||
cbm_store_close(store);
|
||||
th_rmtree(tmpdir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A genuine sub-package of a monorepo IS git-managed, and must still be watched
|
||||
* as such — the nested-directory guard above must not disqualify it. It also
|
||||
* must not react to a sibling package's changes: `git status` reports the whole
|
||||
* repository regardless of -C, so without a `-- .` pathspec every package in a
|
||||
* monorepo reindexes whenever any other one is edited. */
|
||||
TEST(watcher_monorepo_subdir_ignores_sibling_changes) {
|
||||
char tmpdir[256];
|
||||
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_mono_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmpdir))
|
||||
FAIL("cbm_mkdtemp failed");
|
||||
|
||||
if (wt_git(tmpdir, "init -q") != 0) {
|
||||
th_rmtree(tmpdir);
|
||||
FAIL("git init failed");
|
||||
}
|
||||
char pkg_a[400];
|
||||
char pkg_b[400];
|
||||
snprintf(pkg_a, sizeof(pkg_a), "%s/pkg-a", tmpdir);
|
||||
snprintf(pkg_b, sizeof(pkg_b), "%s/pkg-b", tmpdir);
|
||||
if (!cbm_mkdir_p(pkg_a, 0755) || !cbm_mkdir_p(pkg_b, 0755)) {
|
||||
th_rmtree(tmpdir);
|
||||
FAIL("mkdir packages failed");
|
||||
}
|
||||
{
|
||||
char p[500];
|
||||
th_write_file(wt_path(p, sizeof(p), pkg_a, "a.txt"), "a\n");
|
||||
th_write_file(wt_path(p, sizeof(p), pkg_b, "b.txt"), "b\n");
|
||||
}
|
||||
wt_git(tmpdir, "add -A");
|
||||
wt_git(tmpdir, "commit -q -m init");
|
||||
|
||||
cbm_store_t *store = cbm_store_open_memory();
|
||||
cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL);
|
||||
cbm_watcher_watch(w, "pkg-a", pkg_a);
|
||||
index_call_count = 0;
|
||||
|
||||
cbm_watcher_poll_once(w); /* baseline */
|
||||
int after_baseline = index_call_count;
|
||||
|
||||
/* Edit the SIBLING package only. pkg-a is untouched. */
|
||||
{
|
||||
char p[500];
|
||||
th_append_file(wt_path(p, sizeof(p), pkg_b, "b.txt"), "sibling edit\n");
|
||||
}
|
||||
cbm_watcher_touch(w, "pkg-a");
|
||||
cbm_watcher_poll_once(w);
|
||||
ASSERT_EQ(index_call_count, after_baseline);
|
||||
|
||||
/* Editing pkg-a itself must still be seen — the scoping must not have
|
||||
* silenced real changes. */
|
||||
{
|
||||
char p[500];
|
||||
th_append_file(wt_path(p, sizeof(p), pkg_a, "a.txt"), "own edit\n");
|
||||
}
|
||||
cbm_watcher_touch(w, "pkg-a");
|
||||
cbm_watcher_poll_once(w);
|
||||
ASSERT_EQ(index_call_count, after_baseline + 1);
|
||||
|
||||
cbm_watcher_free(w);
|
||||
cbm_store_close(store);
|
||||
th_rmtree(tmpdir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* SHA-256 repositories emit a 64-hex-character HEAD. The watcher must retain
|
||||
* the complete object ID (plus line terminator/NUL while reading it), otherwise
|
||||
* baseline initialization silently retries forever and auto-refresh never runs. */
|
||||
@@ -3041,6 +3169,8 @@ SUITE(watcher) {
|
||||
|
||||
/* Git change detection */
|
||||
RUN_TEST(watcher_detects_git_commit);
|
||||
RUN_TEST(watcher_nested_non_git_dir_does_not_inherit_ancestor_dirt);
|
||||
RUN_TEST(watcher_monorepo_subdir_ignores_sibling_changes);
|
||||
RUN_TEST(watcher_detects_sha256_git_commit);
|
||||
RUN_TEST(watcher_detects_dirty_worktree);
|
||||
RUN_TEST(watcher_identical_watch_preserves_dirty_baseline);
|
||||
|
||||
Reference in New Issue
Block a user