github-featured/deusdata--codebase-memory-mcp
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add persistent artifact storage for team sharing

Browse Source 
Add .codebase-memory/graph.db.zst — a zstd-compressed knowledge graph
artifact that can be committed to the repo. Teammates bootstrap from
the artifact instead of running a full reindex from scratch.

- Vendor zstd 1.5.7 (amalgamated build) for 8-13:1 compression
- Two-tier export: zstd -9 + index stripping for explicit index,
  zstd -3 for watcher/incremental auto-updates
- Import: decompress → integrity check → auto-recreate indexes
- Bootstrap in handle_index_repository: when no local DB exists but
  artifact is present, import first then run incremental
- Auto-create .gitattributes with merge=ours to prevent conflicts
- Fix: add missing idx_edges_url_path to create_user_indexes and
  url_path_gen generated column to init_schema
- 13 new tests (5 zstd wrapper + 8 artifact round-trip/edge cases)
This commit is contained in:
test
2026-04-15 21:16:47 +02:00
parent 81385d08eb
commit 8babe67bea
18 changed files with 56740 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitleaksignore
+3
View File
@@ -0,0 +1,3 @@
# False positives: AVX-512 intrinsic variable names in vendored zstd (xxhash)
internal/cbm/vendored/zstd/zstd.c:generic-api-key:13192
internal/cbm/vendored/zstd/zstd.c:generic-api-key:13241
Makefile.cbm
+27 -11
View File
@@ -140,6 +140,9 @@ GRAMMAR_SRCS = $(wildcard $(CBM_DIR)/grammar_*.c)
# LZ4 + Aho-Corasick
AC_LZ4_SRCS = $(CBM_DIR)/ac.c $(CBM_DIR)/lz4_store.c
# Zstd compression (for persistent artifacts)
ZSTD_SRCS = $(CBM_DIR)/zstd_store.c
# Preprocessor (C++)
PREPROCESSOR_SRC = $(CBM_DIR)/preprocessor.cpp
@@ -190,7 +193,8 @@ PIPELINE_SRCS = \
src/pipeline/pass_k8s.c \
src/pipeline/pass_similarity.c \
src/pipeline/pass_semantic_edges.c \
src/pipeline/pass_cross_repo.c
src/pipeline/pass_cross_repo.c \
src/pipeline/artifact.c
# SimHash / MinHash module
SIMHASH_SRCS = src/simhash/minhash.c
@@ -251,7 +255,7 @@ YYJSON_SRC = vendored/yyjson/yyjson.c
# All production sources
PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
$(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC)
$(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC)
# ── Test sources ─────────────────────────────────────────────────
@@ -295,6 +299,8 @@ TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_fqn.
TEST_WATCHER_SRCS = tests/test_watcher.c
TEST_LZ4_SRCS = tests/test_lz4.c
TEST_ZSTD_SRCS = tests/test_zstd.c
TEST_ARTIFACT_SRCS = tests/test_artifact.c
TEST_SQLITE_WRITER_SRCS = tests/test_sqlite_writer.c
@@ -319,7 +325,7 @@ TEST_YAML_SRCS = tests/test_yaml.c
TEST_SIMHASH_SRCS = tests/test_simhash.c
ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_INTEGRATION_SRCS)
ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_INTEGRATION_SRCS)
# ── Build directories ────────────────────────────────────────────
@@ -423,17 +429,22 @@ $(BUILD_DIR)/test_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
$(BUILD_DIR)/test_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
# Vendored zstd (test build)
ZSTD_OBJ_TEST = $(BUILD_DIR)/test_zstd.o
$(BUILD_DIR)/test_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
# nomic-embed-code pretrained vector blob
UNIXCODER_OBJ = $(BUILD_DIR)/unixcoder_blob.o
$(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUILD_DIR)
$(CC) -c -o $@ $<
OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(MONGOOSE_OBJ_TEST) $(LZ4_OBJ_TEST) $(UNIXCODER_OBJ)
OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(MONGOOSE_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ)
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR)
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR)
$(CC) $(CFLAGS_TEST) -o $@ \
$(ALL_TEST_SRCS) $(PROD_SRCS) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
$(OBJS_VENDORED_TEST) \
$(LDFLAGS_TEST)
@@ -472,14 +483,19 @@ $(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
$(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(MONGOOSE_OBJ_PROD) $(LZ4_OBJ_PROD) $(UNIXCODER_OBJ)
# Vendored zstd (compiled separately, not unity-built via zstd_store.c)
ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o
$(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(MONGOOSE_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)
MAIN_SRC = src/main.c
$(BUILD_DIR)/codebase-memory-mcp: $(MAIN_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_PROD) | $(BUILD_DIR)
$(BUILD_DIR)/codebase-memory-mcp: $(MAIN_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_PROD) | $(BUILD_DIR)
$(CC) $(CFLAGS_PROD) -o $@ \
$(MAIN_SRC) $(PROD_SRCS) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
$(OBJS_VENDORED_PROD) \
$(LDFLAGS)
@@ -504,7 +520,7 @@ embed: frontend
cbm-with-ui: embed $(OBJS_VENDORED_PROD)
$(CC) $(CFLAGS_PROD) -o $(BUILD_DIR)/codebase-memory-mcp \
$(MAIN_SRC) $(PROD_SRCS_WITH_ASSETS) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC) \
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
$(OBJS_VENDORED_PROD) \
$(wildcard $(BUILD_DIR)/embedded/embed_*.o) \
$(LDFLAGS)
@@ -530,7 +546,7 @@ SYSROOT_FLAG = $(if $(SYSROOT),-isysroot $(SYSROOT),)
LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) \
$(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) \
$(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) \
$(SQLITE_WRITER_SRC) $(MAIN_SRC)
$(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(MAIN_SRC)
LINT_HDRS = $(wildcard src/**/*.h src/*.h $(CBM_DIR)/*.h)
LINT_TEST_SRCS = $(ALL_TEST_SRCS)
internal/cbm/vendored/zstd/zstd.c
+52256
View File
File diff suppressed because it is too large Load Diff
internal/cbm/vendored/zstd/zstd.h
+3198
View File
File diff suppressed because it is too large Load Diff
internal/cbm/vendored/zstd/zstd_errors.h
+107
View File
@@ -0,0 +1,107 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_ERRORS_H_398273423
#define ZSTD_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
#endif
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
#ifndef ZSTDERRORLIB_VISIBLE
/* Backwards compatibility with old macro name */
# ifdef ZSTDERRORLIB_VISIBILITY
# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default")))
# else
# define ZSTDERRORLIB_VISIBLE
# endif
#endif
#ifndef ZSTDERRORLIB_HIDDEN
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
# else
# define ZSTDERRORLIB_HIDDEN
# endif
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
#endif
/*-*********************************************
* Error codes list
*-*********************************************
* Error codes _values_ are pinned down since v1.3.1 only.
* Therefore, don't rely on values if you may link to any version < v1.3.1.
*
* Only values < 100 are considered stable.
*
* note 1 : this API shall be used with static linking only.
* dynamic linking is not yet officially supported.
* note 2 : Prefer relying on the enum than on its value whenever possible
* This is the only supported way to use the error list < v1.3.1
* note 3 : ZSTD_isError() is always correct, whatever the library version.
**********************************************/
typedef enum {
ZSTD_error_no_error = 0,
ZSTD_error_GENERIC = 1,
ZSTD_error_prefix_unknown = 10,
ZSTD_error_version_unsupported = 12,
ZSTD_error_frameParameter_unsupported = 14,
ZSTD_error_frameParameter_windowTooLarge = 16,
ZSTD_error_corruption_detected = 20,
ZSTD_error_checksum_wrong = 22,
ZSTD_error_literals_headerWrong = 24,
ZSTD_error_dictionary_corrupted = 30,
ZSTD_error_dictionary_wrong = 32,
ZSTD_error_dictionaryCreation_failed = 34,
ZSTD_error_parameter_unsupported = 40,
ZSTD_error_parameter_combination_unsupported = 41,
ZSTD_error_parameter_outOfBound = 42,
ZSTD_error_tableLog_tooLarge = 44,
ZSTD_error_maxSymbolValue_tooLarge = 46,
ZSTD_error_maxSymbolValue_tooSmall = 48,
ZSTD_error_cannotProduce_uncompressedBlock = 49,
ZSTD_error_stabilityCondition_notRespected = 50,
ZSTD_error_stage_wrong = 60,
ZSTD_error_init_missing = 62,
ZSTD_error_memory_allocation = 64,
ZSTD_error_workSpace_tooSmall= 66,
ZSTD_error_dstSize_tooSmall = 70,
ZSTD_error_srcSize_wrong = 72,
ZSTD_error_dstBuffer_null = 74,
ZSTD_error_noForwardProgress_destFull = 80,
ZSTD_error_noForwardProgress_inputEmpty = 82,
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
ZSTD_error_frameIndex_tooLarge = 100,
ZSTD_error_seekableIO = 102,
ZSTD_error_dstBuffer_wrong = 104,
ZSTD_error_srcBuffer_wrong = 105,
ZSTD_error_sequenceProducer_failed = 106,
ZSTD_error_externalSequences_invalid = 107,
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
} ZSTD_ErrorCode;
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_ERRORS_H_398273423 */
internal/cbm/zstd_store.c
+27
View File
@@ -0,0 +1,27 @@
// zstd_store.c — Thin C wrappers around Zstandard.
#include "vendored/zstd/zstd.h"
#include "zstd_store.h"
#include <stddef.h>
int cbm_zstd_compress(const char *src, int srcLen, char *dst, int dstCap, int level) {
size_t rc = ZSTD_compress(dst, (size_t)dstCap, src, (size_t)srcLen, level);
if (ZSTD_isError(rc)) {
return 0;
}
return (int)rc;
}
int cbm_zstd_decompress(const char *src, int srcLen, char *dst, int dstCap) {
size_t rc = ZSTD_decompress(dst, (size_t)dstCap, src, (size_t)srcLen);
if (ZSTD_isError(rc)) {
return 0;
}
return (int)rc;
}
size_t cbm_zstd_compress_bound(int inputSize) {
return ZSTD_compressBound((size_t)inputSize);
}
internal/cbm/zstd_store.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef CBM_ZSTD_STORE_H
#define CBM_ZSTD_STORE_H
#include <stddef.h>
// Zstd compression at specified level (1=fast .. 22=best).
// Returns compressed size on success, 0 on error.
int cbm_zstd_compress(const char *src, int srcLen, char *dst, int dstCap, int level);
// Zstd decompression.
// Returns decompressed size on success, 0 on error.
int cbm_zstd_decompress(const char *src, int srcLen, char *dst, int dstCap);
// Maximum compressed size bound for given input size.
size_t cbm_zstd_compress_bound(int inputSize);
#endif // CBM_ZSTD_STORE_H
scripts/security-allowlist.txt
+5
View File
@@ -33,6 +33,11 @@ src/mcp/mcp.c:popen:via cbm_popen wrapper calls
src/pipeline/pass_githistory.c:cbm_popen:git log for file history (path validated)
src/pipeline/pass_githistory.c:popen:via cbm_popen wrapper call
# ── Pipeline: artifact persistence (git HEAD hash, merge driver config) ────
src/pipeline/artifact.c:cbm_popen:git rev-parse HEAD for artifact metadata (hardcoded cmd)
src/pipeline/artifact.c:cbm_popen:git config merge.ours.driver for gitattributes (hardcoded cmd)
src/pipeline/artifact.c:popen:via cbm_popen wrapper calls
# ── UI: HTTP server process management ─────────────────────────────────────
src/ui/http_server.c:popen:ps process listing for metrics endpoint
src/ui/http_server.c:fork:spawn indexing subprocess
src/mcp/mcp.c
+58 -22
View File
@@ -55,6 +55,7 @@ enum {
#include "foundation/log.h"
#include "foundation/str_util.h"
#include "foundation/compat_regex.h"
#include "pipeline/artifact.h"
#ifdef _WIN32
#include <process.h> /* _getpid */
@@ -271,7 +272,10 @@ static const tool_def_t TOOLS[] = {
"fast: structure only. cross-repo-intelligence: match Routes/Channels across projects.\"},"
"\"target_projects\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},"
"\"description\":\"Projects to search for cross-repo links (cross-repo-intelligence mode). "
"Use [\\\"*\\\"] for all indexed projects. Run list_projects to see available projects.\"}"
"Use [\\\"*\\\"] for all indexed projects. Run list_projects to see available projects.\"},"
"\"persistence\":{\"type\":\"boolean\",\"default\":false,\"description\":"
"\"Write compressed artifact to .codebase-memory/graph.db.zst for team sharing. "
"Teammates can bootstrap from the artifact instead of full re-indexing.\"}"
"},\"required\":[\"repo_path\"]}"},
{"search_graph",
@@ -2137,6 +2141,52 @@ static char *handle_cross_repo_mode(const char *repo_path, const char *args) {
return out;
}
/* Bootstrap from artifact if no local DB exists for this project. */
static void try_artifact_bootstrap(const char *project_name, const char *repo_path) {
char db_buf[CBM_SZ_1K];
project_db_path(project_name, db_buf, sizeof(db_buf));
struct stat db_st;
if (stat(db_buf, &db_st) != 0 && cbm_artifact_exists(repo_path)) {
cbm_log_info("index.artifact_bootstrap", "project", project_name);
cbm_artifact_import(repo_path, db_buf);
}
}
/* Build the success portion of the index_repository response. */
static void build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc *doc,
yyjson_mut_val *root, const char *project_name,
const char *repo_path, bool persistence) {
cbm_store_t *store = resolve_store(srv, project_name);
if (!store) {
return;
}
int nodes = cbm_store_count_nodes(store, project_name);
int edges = cbm_store_count_edges(store, project_name);
yyjson_mut_obj_add_int(doc, root, "nodes", nodes);
yyjson_mut_obj_add_int(doc, root, "edges", edges);
char adr_path[CBM_SZ_4K];
snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", repo_path);
struct stat adr_st;
bool adr_exists = (stat(adr_path, &adr_st) == 0);
yyjson_mut_obj_add_bool(doc, root, "adr_present", adr_exists);
if (!adr_exists) {
yyjson_mut_obj_add_str(
doc, root, "adr_hint",
"Project indexed. Consider creating an Architecture Decision Record: "
"explore the codebase with get_architecture(aspects=['all']), then use "
"manage_adr(mode='store') to persist architectural insights across sessions.");
}
bool has_artifact = cbm_artifact_exists(repo_path);
yyjson_mut_obj_add_bool(doc, root, "artifact_present", has_artifact);
if (persistence && has_artifact) {
yyjson_mut_obj_add_str(doc, root, "artifact_hint",
"Persistent artifact written to .codebase-memory/graph.db.zst. "
"Commit this file to share the index with teammates.");
}
}
static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
char *mode_str = cbm_mcp_get_string_arg(args, "mode");
@@ -2162,14 +2212,20 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
}
free(mode_str);
bool persistence = cbm_mcp_get_bool_arg(args, "persistence");
cbm_pipeline_t *p = cbm_pipeline_new(repo_path, NULL, mode);
if (!p) {
free(repo_path);
return cbm_mcp_text_result("failed to create pipeline", true);
}
cbm_pipeline_set_persistence(p, persistence);
char *project_name = heap_strdup(cbm_pipeline_project_name(p));
/* Bootstrap from artifact if no local DB exists */
try_artifact_bootstrap(project_name, repo_path);
/* Close cached store — pipeline will delete + recreate the .db file */
if (srv->owns_store && srv->store) {
cbm_store_close(srv->store);
@@ -2212,27 +2268,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
}
if (rc == 0) {
cbm_store_t *store = resolve_store(srv, project_name);
if (store) {
int nodes = cbm_store_count_nodes(store, project_name);
int edges = cbm_store_count_edges(store, project_name);
yyjson_mut_obj_add_int(doc, root, "nodes", nodes);
yyjson_mut_obj_add_int(doc, root, "edges", edges);
/* Check ADR presence and suggest creation if missing */
char adr_path[CBM_SZ_4K];
snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", repo_path);
struct stat adr_st;
bool adr_exists = (stat(adr_path, &adr_st) == 0);
yyjson_mut_obj_add_bool(doc, root, "adr_present", adr_exists);
if (!adr_exists) {
yyjson_mut_obj_add_str(
doc, root, "adr_hint",
"Project indexed. Consider creating an Architecture Decision Record: "
"explore the codebase with get_architecture(aspects=['all']), then use "
"manage_adr(mode='store') to persist architectural insights across sessions.");
}
}
build_index_success_response(srv, doc, root, project_name, repo_path, persistence);
}
char *json = yy_doc_to_str(doc);
src/pipeline/artifact.c
+568
View File
@@ -0,0 +1,568 @@
/*
* artifact.c — Persistent artifact export/import for team sharing.
*
* Export: strip indexes → VACUUM INTO temp → zstd compress → write .zst + metadata
* Import: decompress → write to cache → open (auto-creates indexes) → integrity check
*/
#include "foundation/constants.h"
enum {
ART_DIR_PERMS = 0755,
ART_ZSTD_FAST = 3,
ART_ZSTD_BEST = 9,
ART_RATIO_SCALE = 10, /* multiply ratio by 10 for integer logging */
ART_NUL = 1, /* NUL terminator byte */
};
#define ART_BYTES_PER_MB ((size_t)1024 * 1024)
#include "pipeline/artifact.h"
#include "store/store.h"
#include "foundation/platform.h"
#include "foundation/compat_fs.h"
#include "foundation/compat.h"
#include "foundation/log.h"
#include "zstd_store.h"
#include <sqlite3.h>
#include <yyjson/yyjson.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
/* ── Helpers ──────────────────────────────────────────────────────── */
/* Thread-local rotating buffers for small int→string conversions (logging).
* Rotating allows multiple itoa_buf() calls in a single log statement. */
enum { ART_RING = 4, ART_RING_MASK = 3 };
static const char *itoa_buf(int v) {
static _Thread_local char bufs[ART_RING][CBM_SZ_32];
static _Thread_local int idx = 0;
int i = idx;
idx = (idx + ART_NUL) & ART_RING_MASK;
snprintf(bufs[i], sizeof(bufs[i]), "%d", v);
return bufs[i];
}
/* Build path: <repo>/.codebase-memory/<name> into caller-owned buf. */
static void artifact_path(char *buf, size_t bufsz, const char *repo_path, const char *name) {
snprintf(buf, bufsz, "%s/%s/%s", repo_path, CBM_ARTIFACT_DIR, name);
}
/* Read entire file into malloc'd buffer. Sets *out_len. Returns NULL on error. */
static char *read_file_alloc(const char *path, size_t *out_len) {
FILE *fp = fopen(path, "rb");
if (!fp) {
return NULL;
}
(void)fseek(fp, 0, SEEK_END);
long sz = ftell(fp);
if (sz <= 0) {
(void)fclose(fp);
return NULL;
}
(void)fseek(fp, 0, SEEK_SET);
char *buf = malloc((size_t)sz);
if (!buf) {
(void)fclose(fp);
return NULL;
}
size_t rd = fread(buf, ART_NUL, (size_t)sz, fp);
(void)fclose(fp);
if ((long)rd != sz) {
free(buf);
return NULL;
}
*out_len = (size_t)sz;
return buf;
}
/* Write buffer to file atomically (write to tmp, rename). Returns 0 on success. */
static int write_file_atomic(const char *path, const char *data, size_t len) {
char tmp[CBM_SZ_4K];
snprintf(tmp, sizeof(tmp), "%s.tmp", path);
FILE *fp = fopen(tmp, "wb");
if (!fp) {
return CBM_NOT_FOUND;
}
size_t wr = fwrite(data, ART_NUL, len, fp);
(void)fclose(fp);
if (wr != len) {
cbm_unlink(tmp);
return CBM_NOT_FOUND;
}
if (rename(tmp, path) != 0) {
cbm_unlink(tmp);
return CBM_NOT_FOUND;
}
return 0;
}
/* Get current git HEAD hash. buf must be >= CBM_SZ_64. Returns false on error. */
static bool git_head_hash(const char *repo_path, char *buf, size_t bufsz) {
char cmd[CBM_SZ_1K];
snprintf(cmd, sizeof(cmd), "git -C '%s' rev-parse HEAD 2>/dev/null", repo_path);
FILE *fp = cbm_popen(cmd, "r");
if (!fp) {
buf[0] = '\0';
return false;
}
buf[0] = '\0';
if (fgets(buf, (int)bufsz, fp)) {
/* Strip trailing newline */
size_t len = strlen(buf);
while (len > 0 && (buf[len - ART_NUL] == '\n' || buf[len - ART_NUL] == '\r')) {
buf[--len] = '\0';
}
}
(void)cbm_pclose(fp);
return buf[0] != '\0';
}
/* Generate ISO 8601 timestamp into buf. */
static void iso_timestamp(char *buf, size_t bufsz) {
time_t now = time(NULL);
struct tm tm;
#ifdef _WIN32
gmtime_s(&tm, &now);
#else
gmtime_r(&now, &tm);
#endif
(void)strftime(buf, bufsz, "%Y-%m-%dT%H:%M:%SZ", &tm);
}
/* ── Metadata read/write ─────────────────────────────────────────── */
/* Read schema_version from artifact.json. Returns -1 if missing/invalid. */
static int read_metadata_version(const char *repo_path) {
char meta_path[CBM_SZ_4K];
artifact_path(meta_path, sizeof(meta_path), repo_path, CBM_ARTIFACT_META);
size_t len = 0;
char *json = read_file_alloc(meta_path, &len);
if (!json) {
return CBM_NOT_FOUND;
}
yyjson_doc *doc = yyjson_read(json, len, 0);
free(json);
if (!doc) {
return CBM_NOT_FOUND;
}
yyjson_val *root = yyjson_doc_get_root(doc);
yyjson_val *ver = yyjson_obj_get(root, "schema_version");
int version = ver ? yyjson_get_int(ver) : CBM_NOT_FOUND;
yyjson_doc_free(doc);
return version;
}
/* Read original_size from artifact.json. Returns 0 on error. */
static size_t read_metadata_original_size(const char *repo_path) {
char meta_path[CBM_SZ_4K];
artifact_path(meta_path, sizeof(meta_path), repo_path, CBM_ARTIFACT_META);
size_t len = 0;
char *json = read_file_alloc(meta_path, &len);
if (!json) {
return 0;
}
yyjson_doc *doc = yyjson_read(json, len, 0);
free(json);
if (!doc) {
return 0;
}
yyjson_val *root = yyjson_doc_get_root(doc);
yyjson_val *val = yyjson_obj_get(root, "original_size");
size_t result = val ? (size_t)yyjson_get_uint(val) : 0;
yyjson_doc_free(doc);
return result;
}
/* Write artifact.json metadata. */
static int write_metadata(const char *repo_path, const char *project_name, int nodes, int edges,
size_t original_size, size_t compressed_size, int compression_level) {
char commit[CBM_SZ_64] = "";
git_head_hash(repo_path, commit, sizeof(commit));
char ts[CBM_SZ_64];
iso_timestamp(ts, sizeof(ts));
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
yyjson_mut_obj_add_int(doc, root, "schema_version", CBM_ARTIFACT_SCHEMA_VERSION);
yyjson_mut_obj_add_str(doc, root, "commit", commit);
yyjson_mut_obj_add_str(doc, root, "indexed_at", ts);
yyjson_mut_obj_add_str(doc, root, "project", project_name);
yyjson_mut_obj_add_int(doc, root, "nodes", nodes);
yyjson_mut_obj_add_int(doc, root, "edges", edges);
yyjson_mut_obj_add_uint(doc, root, "original_size", (uint64_t)original_size);
yyjson_mut_obj_add_uint(doc, root, "compressed_size", (uint64_t)compressed_size);
yyjson_mut_obj_add_int(doc, root, "compression_level", compression_level);
size_t json_len = 0;
char *json = yyjson_mut_write(doc, YYJSON_WRITE_PRETTY, &json_len);
yyjson_mut_doc_free(doc);
if (!json) {
return CBM_NOT_FOUND;
}
char meta_path[CBM_SZ_4K];
artifact_path(meta_path, sizeof(meta_path), repo_path, CBM_ARTIFACT_META);
int rc = write_file_atomic(meta_path, json, json_len);
free(json);
return rc;
}
/* ── .gitattributes setup ────────────────────────────────────────── */
static void ensure_gitattributes(const char *repo_path) {
char ga_path[CBM_SZ_4K];
artifact_path(ga_path, sizeof(ga_path), repo_path, ".gitattributes");
struct stat st;
if (stat(ga_path, &st) == 0) {
return; /* already exists */
}
FILE *fp = fopen(ga_path, "w");
if (fp) {
(void)fputs("# Auto-generated by codebase-memory-mcp\n"
"# Prevent merge conflicts on compressed artifact\n" CBM_ARTIFACT_FILENAME
" merge=ours binary\n",
fp);
(void)fclose(fp);
}
/* Best-effort: configure merge driver */
char cmd[CBM_SZ_1K];
snprintf(cmd, sizeof(cmd), "git -C '%s' config merge.ours.driver true 2>/dev/null", repo_path);
FILE *p = cbm_popen(cmd, "r");
if (p) {
(void)cbm_pclose(p);
}
}
/* ── Index stripping ─────────────────────────────────────────────── */
/* SQL to drop all user-created indexes (not autoindexes, not FTS5). */
static const char *DROP_INDEXES_SQL = "DROP INDEX IF EXISTS idx_nodes_label;"
"DROP INDEX IF EXISTS idx_nodes_name;"
"DROP INDEX IF EXISTS idx_nodes_file;"
"DROP INDEX IF EXISTS idx_edges_source;"
"DROP INDEX IF EXISTS idx_edges_target;"
"DROP INDEX IF EXISTS idx_edges_type;"
"DROP INDEX IF EXISTS idx_edges_target_type;"
"DROP INDEX IF EXISTS idx_edges_source_type;"
"DROP INDEX IF EXISTS idx_edges_url_path;";
/* ── Export helpers ───────────────────────────────────────────────── */
/* Prepare a stripped DB copy for best-quality export.
* VACUUM INTO → drop indexes → VACUUM. Returns malloc'd buffer or NULL. */
static char *prepare_stripped_db(const char *db_path, size_t *out_size) {
char tmp_path[CBM_SZ_4K];
snprintf(tmp_path, sizeof(tmp_path), "%s/cbm_artifact_tmp.db", cbm_tmpdir());
cbm_unlink(tmp_path);
/* VACUUM INTO: clean compacted copy. Use raw sqlite3 to bypass store authorizer
* (which blocks ATTACH, used internally by VACUUM INTO). */
sqlite3 *raw_db = NULL;
if (sqlite3_open_v2(db_path, &raw_db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) {
cbm_log_error("artifact.export", "err", "open_source_db");
sqlite3_close(raw_db);
return NULL;
}
char vacuum_sql[CBM_SZ_4K];
snprintf(vacuum_sql, sizeof(vacuum_sql), "VACUUM INTO '%s';", tmp_path);
char *errmsg = NULL;
int vrc = sqlite3_exec(raw_db, vacuum_sql, NULL, NULL, &errmsg);
sqlite3_close(raw_db);
if (vrc != SQLITE_OK) {
cbm_log_error("artifact.export", "err", "vacuum_into");
sqlite3_free(errmsg);
cbm_unlink(tmp_path);
return NULL;
}
/* Strip indexes from the copy for better compression. */
sqlite3 *tmp_db = NULL;
if (sqlite3_open_v2(tmp_path, &tmp_db, SQLITE_OPEN_READWRITE, NULL) == SQLITE_OK) {
sqlite3_exec(tmp_db, DROP_INDEXES_SQL, NULL, NULL, NULL);
sqlite3_exec(tmp_db, "VACUUM;", NULL, NULL, NULL);
sqlite3_close(tmp_db);
}
char *data = read_file_alloc(tmp_path, out_size);
cbm_unlink(tmp_path);
/* Clean up WAL/SHM from temp */
char wal[CBM_SZ_4K];
char shm[CBM_SZ_4K];
snprintf(wal, sizeof(wal), "%s-wal", tmp_path);
snprintf(shm, sizeof(shm), "%s-shm", tmp_path);
cbm_unlink(wal);
cbm_unlink(shm);
return data;
}
/* ── Export ───────────────────────────────────────────────────────── */
int cbm_artifact_export(const char *db_path, const char *repo_path, const char *project_name,
int quality) {
if (!db_path || !repo_path || !project_name) {
return CBM_NOT_FOUND;
}
/* Ensure .codebase-memory/ directory exists */
char art_dir[CBM_SZ_4K];
snprintf(art_dir, sizeof(art_dir), "%s/%s", repo_path, CBM_ARTIFACT_DIR);
cbm_mkdir_p(art_dir, ART_DIR_PERMS);
size_t db_size = 0;
char *db_data = NULL;
int compression_level = ART_ZSTD_FAST;
if (quality == CBM_ARTIFACT_BEST) {
compression_level = ART_ZSTD_BEST;
db_data = prepare_stripped_db(db_path, &db_size);
} else {
db_data = read_file_alloc(db_path, &db_size);
}
if (!db_data || db_size == 0) {
free(db_data);
cbm_log_error("artifact.export", "err", "read_db");
return CBM_NOT_FOUND;
}
/* Compress with zstd */
size_t bound = cbm_zstd_compress_bound((int)db_size);
char *compressed = malloc(bound);
if (!compressed) {
free(db_data);
return CBM_NOT_FOUND;
}
int clen = cbm_zstd_compress(db_data, (int)db_size, compressed, (int)bound, compression_level);
free(db_data);
if (clen <= 0) {
free(compressed);
cbm_log_error("artifact.export", "err", "zstd_compress");
return CBM_NOT_FOUND;
}
/* Write compressed artifact */
char zst_path[CBM_SZ_4K];
artifact_path(zst_path, sizeof(zst_path), repo_path, CBM_ARTIFACT_FILENAME);
int wrc = write_file_atomic(zst_path, compressed, (size_t)clen);
free(compressed);
if (wrc != 0) {
cbm_log_error("artifact.export", "err", "write_artifact");
return CBM_NOT_FOUND;
}
/* Get node/edge counts for metadata */
int nodes = 0;
int edges = 0;
cbm_store_t *count_store = cbm_store_open_path(db_path);
if (count_store) {
nodes = cbm_store_count_nodes(count_store, project_name);
edges = cbm_store_count_edges(count_store, project_name);
cbm_store_close(count_store);
}
/* Write metadata */
write_metadata(repo_path, project_name, nodes, edges, db_size, (size_t)clen, compression_level);
/* Ensure .gitattributes for merge conflict prevention */
ensure_gitattributes(repo_path);
double ratio = db_size > 0 ? (double)db_size / (double)clen : 0.0;
cbm_log_info("artifact.export", "quality", quality == CBM_ARTIFACT_BEST ? "best" : "fast",
"original_mb", itoa_buf((int)(db_size / ART_BYTES_PER_MB)), "compressed_mb",
itoa_buf((int)((size_t)clen / ART_BYTES_PER_MB)), "ratio",
itoa_buf((int)(ratio * ART_RATIO_SCALE)));
return 0;
}
/* ── Import ──────────────────────────────────────────────────────── */
int cbm_artifact_import(const char *repo_path, const char *cache_db_path) {
if (!repo_path || !cache_db_path) {
return CBM_NOT_FOUND;
}
/* Check schema version compatibility */
int version = read_metadata_version(repo_path);
if (version < 0 || version > CBM_ARTIFACT_SCHEMA_VERSION) {
cbm_log_info("artifact.import", "skip", "schema_version_mismatch", "artifact_ver",
itoa_buf(version), "current_ver", itoa_buf(CBM_ARTIFACT_SCHEMA_VERSION));
return CBM_NOT_FOUND;
}
/* Get original_size for decompression buffer */
size_t original_size = read_metadata_original_size(repo_path);
if (original_size == 0) {
cbm_log_error("artifact.import", "err", "missing_original_size");
return CBM_NOT_FOUND;
}
/* Read compressed artifact */
char zst_path[CBM_SZ_4K];
artifact_path(zst_path, sizeof(zst_path), repo_path, CBM_ARTIFACT_FILENAME);
size_t clen = 0;
char *compressed = read_file_alloc(zst_path, &clen);
if (!compressed) {
cbm_log_error("artifact.import", "err", "read_artifact");
return CBM_NOT_FOUND;
}
/* Decompress */
char *decompressed = malloc(original_size);
if (!decompressed) {
free(compressed);
return CBM_NOT_FOUND;
}
int dlen = cbm_zstd_decompress(compressed, (int)clen, decompressed, (int)original_size);
free(compressed);
if (dlen <= 0) {
free(decompressed);
cbm_log_error("artifact.import", "err", "zstd_decompress");
return CBM_NOT_FOUND;
}
/* Write to temp file, then rename for atomicity */
char tmp_path[CBM_SZ_4K];
snprintf(tmp_path, sizeof(tmp_path), "%s.import_tmp", cache_db_path);
/* Ensure cache directory exists */
char cache_dir[CBM_SZ_1K];
snprintf(cache_dir, sizeof(cache_dir), "%s", cache_db_path);
char *last_slash = strrchr(cache_dir, '/');
if (last_slash) {
*last_slash = '\0';
cbm_mkdir_p(cache_dir, ART_DIR_PERMS);
}
int wrc = write_file_atomic(tmp_path, decompressed, (size_t)dlen);
free(decompressed);
if (wrc != 0) {
cbm_log_error("artifact.import", "err", "write_temp_db");
return CBM_NOT_FOUND;
}
/* Open with cbm_store_open_path to auto-create missing indexes + FTS5 */
cbm_store_t *store = cbm_store_open_path(tmp_path);
if (!store) {
cbm_log_error("artifact.import", "err", "open_imported_db");
cbm_unlink(tmp_path);
return CBM_NOT_FOUND;
}
/* Integrity check — refuse corrupted artifacts */
if (!cbm_store_check_integrity(store)) {
cbm_log_error("artifact.import", "err", "integrity_check_failed");
cbm_store_close(store);
cbm_unlink(tmp_path);
return CBM_NOT_FOUND;
}
cbm_store_close(store);
/* Atomic rename to final path */
if (rename(tmp_path, cache_db_path) != 0) {
cbm_log_error("artifact.import", "err", "rename_to_cache");
cbm_unlink(tmp_path);
return CBM_NOT_FOUND;
}
/* Clean up any stale WAL/SHM from the temp open */
char wal[CBM_SZ_4K];
char shm[CBM_SZ_4K];
snprintf(wal, sizeof(wal), "%s-wal", tmp_path);
snprintf(shm, sizeof(shm), "%s-shm", tmp_path);
cbm_unlink(wal);
cbm_unlink(shm);
cbm_log_info("artifact.import", "db", cache_db_path, "size_mb",
itoa_buf((int)((size_t)dlen / ART_BYTES_PER_MB)));
return 0;
}
/* ── Existence check ─────────────────────────────────────────────── */
bool cbm_artifact_exists(const char *repo_path) {
if (!repo_path) {
return false;
}
char zst_path[CBM_SZ_4K];
artifact_path(zst_path, sizeof(zst_path), repo_path, CBM_ARTIFACT_FILENAME);
struct stat st;
if (stat(zst_path, &st) != 0 || st.st_size == 0) {
return false;
}
/* Check schema version is compatible */
int version = read_metadata_version(repo_path);
return version >= 0 && version <= CBM_ARTIFACT_SCHEMA_VERSION;
}
/* ── Commit hash extraction ──────────────────────────────────────── */
char *cbm_artifact_commit(const char *repo_path) {
if (!repo_path) {
return NULL;
}
char meta_path[CBM_SZ_4K];
artifact_path(meta_path, sizeof(meta_path), repo_path, CBM_ARTIFACT_META);
size_t len = 0;
char *json = read_file_alloc(meta_path, &len);
if (!json) {
return NULL;
}
yyjson_doc *doc = yyjson_read(json, len, 0);
free(json);
if (!doc) {
return NULL;
}
yyjson_val *root = yyjson_doc_get_root(doc);
yyjson_val *val = yyjson_obj_get(root, "commit");
char *result = NULL;
if (val) {
const char *s = yyjson_get_str(val);
if (s && s[0]) {
size_t slen = strlen(s);
result = malloc(slen + ART_NUL);
if (result) {
memcpy(result, s, slen + ART_NUL);
}
}
}
yyjson_doc_free(doc);
return result;
}
src/pipeline/artifact.h
+49
View File
@@ -0,0 +1,49 @@
/*
* artifact.h — Persistent artifact export/import for team sharing.
*
* Exports the SQLite knowledge graph as a zstd-compressed artifact
* to .codebase-memory/graph.db.zst in the repository. Teammates
* can import the artifact to bootstrap their local index instead
* of running a full pipeline from scratch.
*/
#ifndef CBM_ARTIFACT_H
#define CBM_ARTIFACT_H
#include <stdbool.h>
/* Schema version — increment when DB schema changes (new tables/indexes).
* Import refuses artifacts with schema_version > current. */
#define CBM_ARTIFACT_SCHEMA_VERSION 1
#define CBM_ARTIFACT_FILENAME "graph.db.zst"
#define CBM_ARTIFACT_META "artifact.json"
#define CBM_ARTIFACT_DIR ".codebase-memory"
/* Export quality levels */
enum {
CBM_ARTIFACT_FAST = 0, /* zstd -3, no index stripping (watcher path) */
CBM_ARTIFACT_BEST = 1, /* zstd -9 + drop indexes + VACUUM INTO (explicit index) */
};
/* Export DB to .codebase-memory/graph.db.zst artifact.
* quality: CBM_ARTIFACT_FAST or CBM_ARTIFACT_BEST.
* Creates .codebase-memory/ dir, .gitattributes, and artifact.json.
* Returns 0 on success, -1 on error. */
int cbm_artifact_export(const char *db_path, const char *repo_path, const char *project_name,
int quality);
/* Import artifact from .codebase-memory/graph.db.zst to cache_db_path.
* Decompresses, runs integrity check, recreates indexes.
* Returns 0 on success, -1 on error. */
int cbm_artifact_import(const char *repo_path, const char *cache_db_path);
/* Check if a compatible artifact exists in repo_path/.codebase-memory/.
* Returns true only if both graph.db.zst and artifact.json exist
* and schema_version is compatible. */
bool cbm_artifact_exists(const char *repo_path);
/* Get the git commit hash from artifact metadata. Caller must free().
* Returns NULL if artifact doesn't exist or has no commit field. */
char *cbm_artifact_commit(const char *repo_path);
#endif /* CBM_ARTIFACT_H */
src/pipeline/pipeline.c
+15
View File
@@ -15,6 +15,7 @@
enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 5, PL_WAL_BUF = 1040 };
#define PL_NSEC_PER_SEC 1000000000LL
#include "pipeline/pipeline.h"
#include "pipeline/artifact.h"
#include "pipeline/pipeline_internal.h"
#include "pipeline/worker_pool.h"
#include "graph_buffer/graph_buffer.h"
@@ -73,6 +74,7 @@ struct cbm_pipeline {
char *project_name;
cbm_index_mode_t mode;
atomic_int cancelled;
bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
/* Indexing state (set during run) */
cbm_gbuf_t *gbuf;
@@ -118,11 +120,18 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path,
p->db_path = db_path ? strdup(db_path) : NULL;
p->project_name = cbm_project_name_from_path(repo_path);
p->mode = mode;
p->persistence = false;
atomic_init(&p->cancelled, 0);
return p;
}
void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) {
if (p) {
p->persistence = enabled;
}
}
void cbm_pipeline_free(cbm_pipeline_t *p) {
if (!p) {
return;
@@ -666,6 +675,12 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
cbm_store_close(hash_store);
cbm_log_info("pass.timing", "pass", "persist_hashes", "files", itoa_buf(file_count));
}
/* Export persistent artifact if enabled */
if (p->persistence) {
cbm_artifact_export(db_path, p->repo_path, p->project_name, CBM_ARTIFACT_BEST);
}
return 0;
}
src/pipeline/pipeline.h
+4
View File
@@ -42,6 +42,10 @@ typedef enum {
/* Create a new pipeline. Caller owns the result. */
cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm_index_mode_t mode);
/* Enable persistent artifact export (.codebase-memory/graph.db.zst).
* When enabled, the pipeline writes a compressed artifact after indexing. */
void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled);
/* Free a pipeline and all its internal state. NULL-safe. */
void cbm_pipeline_free(cbm_pipeline_t *p);
src/pipeline/pipeline_incremental.c
+8 -2
View File
@@ -13,6 +13,7 @@
enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24, INCR_WAL_BUF = 1040 };
#include "pipeline/pipeline.h"
#include "pipeline/artifact.h"
#include <stdio.h>
#include <time.h>
#include "pipeline/pipeline_internal.h"
@@ -258,7 +259,7 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil
}
/* Delete old DB and dump merged graph + hashes to disk. */
static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project,
cbm_file_info_t *files, int file_count) {
cbm_file_info_t *files, int file_count, const char *repo_path) {
struct timespec t;
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
@@ -294,6 +295,11 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *
cbm_store_close(hash_store);
}
/* Auto-update artifact if one already exists (persistence was enabled previously) */
if (repo_path && cbm_artifact_exists(repo_path)) {
cbm_artifact_export(db_path, repo_path, project, CBM_ARTIFACT_FAST);
}
}
/* ── Incremental pipeline entry point ────────────────────────────── */
@@ -425,7 +431,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
cbm_registry_free(registry);
/* Step 7: Dump to disk */
dump_and_persist(existing, db_path, project, files, file_count);
dump_and_persist(existing, db_path, project, files, file_count, cbm_pipeline_repo_path(p));
cbm_gbuf_free(existing);
cbm_log_info("incremental.done", "elapsed_ms", itoa_buf((int)elapsed_ms(t0)));
src/store/store.c
+45 -42
View File
@@ -212,47 +212,49 @@ static void iso_now(char *buf, size_t sz) {
/* ── Schema ─────────────────────────────────────────────────────── */
static int init_schema(cbm_store_t *s) {
const char *ddl = "CREATE TABLE IF NOT EXISTS projects ("
" name TEXT PRIMARY KEY,"
" indexed_at TEXT NOT NULL,"
" root_path TEXT NOT NULL"
");"
"CREATE TABLE IF NOT EXISTS file_hashes ("
" project TEXT NOT NULL REFERENCES projects(name) ON DELETE CASCADE,"
" rel_path TEXT NOT NULL,"
" sha256 TEXT NOT NULL,"
" mtime_ns INTEGER NOT NULL DEFAULT 0,"
" size INTEGER NOT NULL DEFAULT 0,"
" PRIMARY KEY (project, rel_path)"
");"
"CREATE TABLE IF NOT EXISTS nodes ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" project TEXT NOT NULL REFERENCES projects(name) ON DELETE CASCADE,"
" label TEXT NOT NULL,"
" name TEXT NOT NULL,"
" qualified_name TEXT NOT NULL,"
" file_path TEXT DEFAULT '',"
" start_line INTEGER DEFAULT 0,"
" end_line INTEGER DEFAULT 0,"
" properties TEXT DEFAULT '{}',"
" UNIQUE(project, qualified_name)"
");"
"CREATE TABLE IF NOT EXISTS edges ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" project TEXT NOT NULL REFERENCES projects(name) ON DELETE CASCADE,"
" source_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,"
" target_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,"
" type TEXT NOT NULL,"
" properties TEXT DEFAULT '{}',"
" UNIQUE(source_id, target_id, type)"
");"
"CREATE TABLE IF NOT EXISTS project_summaries ("
" project TEXT PRIMARY KEY,"
" summary TEXT NOT NULL,"
" source_hash TEXT NOT NULL,"
" created_at TEXT NOT NULL,"
" updated_at TEXT NOT NULL"
");";
const char *ddl =
"CREATE TABLE IF NOT EXISTS projects ("
" name TEXT PRIMARY KEY,"
" indexed_at TEXT NOT NULL,"
" root_path TEXT NOT NULL"
");"
"CREATE TABLE IF NOT EXISTS file_hashes ("
" project TEXT NOT NULL REFERENCES projects(name) ON DELETE CASCADE,"
" rel_path TEXT NOT NULL,"
" sha256 TEXT NOT NULL,"
" mtime_ns INTEGER NOT NULL DEFAULT 0,"
" size INTEGER NOT NULL DEFAULT 0,"
" PRIMARY KEY (project, rel_path)"
");"
"CREATE TABLE IF NOT EXISTS nodes ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" project TEXT NOT NULL REFERENCES projects(name) ON DELETE CASCADE,"
" label TEXT NOT NULL,"
" name TEXT NOT NULL,"
" qualified_name TEXT NOT NULL,"
" file_path TEXT DEFAULT '',"
" start_line INTEGER DEFAULT 0,"
" end_line INTEGER DEFAULT 0,"
" properties TEXT DEFAULT '{}',"
" UNIQUE(project, qualified_name)"
");"
"CREATE TABLE IF NOT EXISTS edges ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" project TEXT NOT NULL REFERENCES projects(name) ON DELETE CASCADE,"
" source_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,"
" target_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,"
" type TEXT NOT NULL,"
" properties TEXT DEFAULT '{}',"
" url_path_gen TEXT GENERATED ALWAYS AS (json_extract(properties,'$.url_path')),"
" UNIQUE(source_id, target_id, type)"
");"
"CREATE TABLE IF NOT EXISTS project_summaries ("
" project TEXT PRIMARY KEY,"
" summary TEXT NOT NULL,"
" source_hash TEXT NOT NULL,"
" created_at TEXT NOT NULL,"
" updated_at TEXT NOT NULL"
");";
int rc = exec_sql(s, ddl);
if (rc != CBM_STORE_OK) {
@@ -290,7 +292,8 @@ static int create_user_indexes(cbm_store_t *s) {
"CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id, type);"
"CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(project, type);"
"CREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(project, target_id, type);"
"CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(project, source_id, type);";
"CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(project, source_id, type);"
"CREATE INDEX IF NOT EXISTS idx_edges_url_path ON edges(project, url_path_gen);";
return exec_sql(s, sql);
}
tests/test_artifact.c
+229
View File
@@ -0,0 +1,229 @@
/*
* test_artifact.c — Tests for persistent artifact export/import.
*/
#include "test_framework.h"
#include "store/store.h"
#include "pipeline/artifact.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include <sys/stat.h>
#include <stdio.h>
/* ── Helpers ─────────────────────────────────────────────────────── */
static char g_tmpdir[1024];
static char g_repo[1024];
static char g_db[1024];
static void setup_artifact_test(void) {
snprintf(g_tmpdir, sizeof(g_tmpdir), "%s/cbm_test_artifact_XXXXXX", cbm_tmpdir());
cbm_mkdtemp(g_tmpdir);
snprintf(g_repo, sizeof(g_repo), "%s/repo", g_tmpdir);
cbm_mkdir_p(g_repo, 0755);
snprintf(g_db, sizeof(g_db), "%s/test.db", g_tmpdir);
}
/* Create a minimal but valid DB with some nodes and edges. */
static void create_test_db(const char *path) {
cbm_store_t *s = cbm_store_open_path(path);
if (!s) {
return;
}
cbm_store_exec(s, "INSERT OR IGNORE INTO projects(name, indexed_at, root_path) "
"VALUES('test-proj', '2026-01-01', '/tmp/test');");
cbm_store_exec(s, "INSERT INTO nodes(project, label, name, qualified_name, file_path) "
"VALUES('test-proj', 'Function', 'foo', 'test-proj.foo', 'main.c');");
cbm_store_exec(s, "INSERT INTO nodes(project, label, name, qualified_name, file_path) "
"VALUES('test-proj', 'Function', 'bar', 'test-proj.bar', 'main.c');");
cbm_store_exec(s, "INSERT INTO edges(project, source_id, target_id, type) "
"VALUES('test-proj', 1, 2, 'CALLS');");
cbm_store_close(s);
}
static void cleanup_dir(const char *path) {
char cmd[2048];
snprintf(cmd, sizeof(cmd), "rm -rf '%s'", path);
(void)system(cmd);
}
/* ── Tests ───────────────────────────────────────────────────────── */
TEST(artifact_export_fast_roundtrip) {
setup_artifact_test();
create_test_db(g_db);
/* Export with fast quality (zstd -3, no index stripping) */
int rc = cbm_artifact_export(g_db, g_repo, "test-proj", CBM_ARTIFACT_FAST);
ASSERT_EQ(rc, 0);
/* Verify artifact files exist */
char zst[1024];
snprintf(zst, sizeof(zst), "%s/.codebase-memory/graph.db.zst", g_repo);
struct stat st;
ASSERT_EQ(stat(zst, &st), 0);
ASSERT_GT((int)st.st_size, 0);
char meta[1024];
snprintf(meta, sizeof(meta), "%s/.codebase-memory/artifact.json", g_repo);
ASSERT_EQ(stat(meta, &st), 0);
/* Import to a new path */
char import_db[1024];
snprintf(import_db, sizeof(import_db), "%s/imported.db", g_tmpdir);
rc = cbm_artifact_import(g_repo, import_db);
ASSERT_EQ(rc, 0);
/* Verify imported DB has correct data */
cbm_store_t *s = cbm_store_open_path(import_db);
ASSERT_NOT_NULL(s);
int nodes = cbm_store_count_nodes(s, "test-proj");
int edges = cbm_store_count_edges(s, "test-proj");
ASSERT_EQ(nodes, 2);
ASSERT_EQ(edges, 1);
cbm_store_close(s);
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_export_best_roundtrip) {
setup_artifact_test();
create_test_db(g_db);
/* Export with best quality (zstd -9, index stripping + VACUUM) */
int rc = cbm_artifact_export(g_db, g_repo, "test-proj", CBM_ARTIFACT_BEST);
ASSERT_EQ(rc, 0);
/* Source DB should be untouched (VACUUM INTO doesn't modify source) */
cbm_store_t *src = cbm_store_open_path(g_db);
ASSERT_NOT_NULL(src);
ASSERT_EQ(cbm_store_count_nodes(src, "test-proj"), 2);
cbm_store_close(src);
/* Import and verify */
char import_db[1024];
snprintf(import_db, sizeof(import_db), "%s/imported.db", g_tmpdir);
rc = cbm_artifact_import(g_repo, import_db);
ASSERT_EQ(rc, 0);
cbm_store_t *s = cbm_store_open_path(import_db);
ASSERT_NOT_NULL(s);
ASSERT_EQ(cbm_store_count_nodes(s, "test-proj"), 2);
ASSERT_EQ(cbm_store_count_edges(s, "test-proj"), 1);
cbm_store_close(s);
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_exists_check) {
setup_artifact_test();
create_test_db(g_db);
/* No artifact yet */
ASSERT_FALSE(cbm_artifact_exists(g_repo));
/* Export creates the artifact */
cbm_artifact_export(g_db, g_repo, "test-proj", CBM_ARTIFACT_FAST);
ASSERT_TRUE(cbm_artifact_exists(g_repo));
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_commit_hash) {
setup_artifact_test();
create_test_db(g_db);
cbm_artifact_export(g_db, g_repo, "test-proj", CBM_ARTIFACT_FAST);
/* commit hash may be empty if repo is not a git repo, but should not crash */
char *commit = cbm_artifact_commit(g_repo);
/* For a non-git directory, commit will be NULL (git rev-parse HEAD fails) */
free(commit);
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_schema_version_mismatch) {
setup_artifact_test();
create_test_db(g_db);
cbm_artifact_export(g_db, g_repo, "test-proj", CBM_ARTIFACT_FAST);
/* Overwrite artifact.json with incompatible schema version */
char meta[1024];
snprintf(meta, sizeof(meta), "%s/.codebase-memory/artifact.json", g_repo);
FILE *fp = fopen(meta, "w");
ASSERT_NOT_NULL(fp);
fprintf(fp, "{\"schema_version\": 999, \"original_size\": 1000}");
fclose(fp);
/* exists should return false for incompatible version */
ASSERT_FALSE(cbm_artifact_exists(g_repo));
/* Import should fail */
char import_db[1024];
snprintf(import_db, sizeof(import_db), "%s/imported.db", g_tmpdir);
int rc = cbm_artifact_import(g_repo, import_db);
ASSERT_NEQ(rc, 0);
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_import_missing) {
setup_artifact_test();
/* Import from repo without artifact should fail gracefully */
char import_db[1024];
snprintf(import_db, sizeof(import_db), "%s/imported.db", g_tmpdir);
int rc = cbm_artifact_import(g_repo, import_db);
ASSERT_NEQ(rc, 0);
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_gitattributes_created) {
setup_artifact_test();
create_test_db(g_db);
cbm_artifact_export(g_db, g_repo, "test-proj", CBM_ARTIFACT_FAST);
char ga[1024];
snprintf(ga, sizeof(ga), "%s/.codebase-memory/.gitattributes", g_repo);
struct stat st;
ASSERT_EQ(stat(ga, &st), 0);
cleanup_dir(g_tmpdir);
PASS();
}
TEST(artifact_null_safety) {
ASSERT_NEQ(cbm_artifact_export(NULL, "/tmp", "p", 0), 0);
ASSERT_NEQ(cbm_artifact_export("/tmp/x.db", NULL, "p", 0), 0);
ASSERT_NEQ(cbm_artifact_import(NULL, "/tmp/x.db"), 0);
ASSERT_NEQ(cbm_artifact_import("/tmp", NULL), 0);
ASSERT_FALSE(cbm_artifact_exists(NULL));
ASSERT_NULL(cbm_artifact_commit(NULL));
PASS();
}
SUITE(artifact) {
RUN_TEST(artifact_export_fast_roundtrip);
RUN_TEST(artifact_export_best_roundtrip);
RUN_TEST(artifact_exists_check);
RUN_TEST(artifact_commit_hash);
RUN_TEST(artifact_schema_version_mismatch);
RUN_TEST(artifact_import_missing);
RUN_TEST(artifact_gitattributes_created);
RUN_TEST(artifact_null_safety);
}
tests/test_main.c
+7 -1
View File
@@ -36,6 +36,8 @@ extern void suite_pipeline(void);
extern void suite_fqn(void);
extern void suite_watcher(void);
extern void suite_lz4(void);
extern void suite_zstd(void);
extern void suite_artifact(void);
extern void suite_sqlite_writer(void);
extern void suite_go_lsp(void);
extern void suite_c_lsp(void);
@@ -101,10 +103,14 @@ int main(void) {
/* Watcher (M10) */
RUN_SUITE(watcher);
/* LZ4 + SQLite writer */
/* LZ4 + zstd + SQLite writer */
RUN_SUITE(lz4);
RUN_SUITE(zstd);
RUN_SUITE(sqlite_writer);
/* Persistent artifact export/import */
RUN_SUITE(artifact);
/* LSP resolvers */
RUN_SUITE(go_lsp);
RUN_SUITE(c_lsp);
tests/test_zstd.c
+117
View File
@@ -0,0 +1,117 @@
/*
* test_zstd.c — Tests for zstd compression wrappers.
*/
#include "test_framework.h"
extern int cbm_zstd_compress(const char *src, int srcLen, char *dst, int dstCap, int level);
extern int cbm_zstd_decompress(const char *src, int srcLen, char *dst, int dstCap);
extern size_t cbm_zstd_compress_bound(int inputSize);
TEST(zstd_roundtrip) {
const char *data = "Hello, zstd compression roundtrip test!";
int len = (int)strlen(data);
size_t bound = cbm_zstd_compress_bound(len);
ASSERT_GT((int)bound, 0);
char *cbuf = malloc(bound);
ASSERT_NOT_NULL(cbuf);
int clen = cbm_zstd_compress(data, len, cbuf, (int)bound, 3);
ASSERT_GT(clen, 0);
char *dbuf = malloc(len);
ASSERT_NOT_NULL(dbuf);
int dlen = cbm_zstd_decompress(cbuf, clen, dbuf, len);
ASSERT_EQ(dlen, len);
ASSERT_MEM_EQ(dbuf, data, len);
free(cbuf);
free(dbuf);
PASS();
}
TEST(zstd_roundtrip_large) {
int len = 100000;
char *data = malloc(len);
ASSERT_NOT_NULL(data);
/* Repetitive data — should compress well */
for (int i = 0; i < len; i++) {
data[i] = "function_name_pattern_abcdef"[i % 28];
}
size_t bound = cbm_zstd_compress_bound(len);
char *cbuf = malloc(bound);
ASSERT_NOT_NULL(cbuf);
int clen = cbm_zstd_compress(data, len, cbuf, (int)bound, 9);
ASSERT_GT(clen, 0);
/* Repetitive data should compress at least 2:1 */
ASSERT_LT(clen, len / 2);
char *dbuf = malloc(len);
ASSERT_NOT_NULL(dbuf);
int dlen = cbm_zstd_decompress(cbuf, clen, dbuf, len);
ASSERT_EQ(dlen, len);
ASSERT_MEM_EQ(dbuf, data, len);
free(data);
free(cbuf);
free(dbuf);
PASS();
}
TEST(zstd_compress_levels) {
const char *data = "test data for different compression levels";
int len = (int)strlen(data);
size_t bound = cbm_zstd_compress_bound(len);
char *cbuf = malloc(bound);
ASSERT_NOT_NULL(cbuf);
/* Both level 3 (fast) and level 9 (best) should produce valid output */
int clen3 = cbm_zstd_compress(data, len, cbuf, (int)bound, 3);
ASSERT_GT(clen3, 0);
int clen9 = cbm_zstd_compress(data, len, cbuf, (int)bound, 9);
ASSERT_GT(clen9, 0);
free(cbuf);
PASS();
}
TEST(zstd_decompress_too_small_output) {
const char *data = "this is test data that will be compressed";
int len = (int)strlen(data);
size_t bound = cbm_zstd_compress_bound(len);
char *cbuf = malloc(bound);
ASSERT_NOT_NULL(cbuf);
int clen = cbm_zstd_compress(data, len, cbuf, (int)bound, 3);
ASSERT_GT(clen, 0);
/* Try decompressing with too-small output buffer — should return 0 (error) */
char small[4];
int dlen = cbm_zstd_decompress(cbuf, clen, small, 4);
ASSERT_EQ(dlen, 0);
free(cbuf);
PASS();
}
TEST(zstd_bound_positive) {
ASSERT_GT((int)cbm_zstd_compress_bound(1), 0);
ASSERT_GT((int)cbm_zstd_compress_bound(100), 0);
ASSERT_GT((int)cbm_zstd_compress_bound(1000000), 0);
PASS();
}
SUITE(zstd) {
RUN_TEST(zstd_roundtrip);
RUN_TEST(zstd_roundtrip_large);
RUN_TEST(zstd_compress_levels);
RUN_TEST(zstd_decompress_too_small_output);
RUN_TEST(zstd_bound_positive);
}