Files
deusdata--codebase-memory-mcp/scripts/embed-frontend.sh
T
Martin Vogel 18fa9979ee Replace vmem with mimalloc global allocator, add extraction-phase prescan, fix __init__.py QN collision
Memory management:
- Vendor mimalloc v2.1.9 as global allocator (MI_OVERRIDE=1 in prod)
- New mem.h/mem.c: RSS-based budget tracking via mi_process_info()
- Remove vmem.c/vmem.h (mmap-based budget tracking)
- Remove slab tier2 bump allocator (~300 LOC); >64B goes to mimalloc
- Slab tier1 pages from malloc (= mimalloc) instead of vmem
- Arena blocks from malloc instead of vmem
- Budget raised from 35% to 50% RAM (no more untracked C++ heap)

Extraction-phase prescan (eliminates disk re-reads):
- HTTP call sites: keyword check + URL extraction during extraction
- HTTP routes: decorator + source-based extraction during extraction
- Config file refs: regex scan during extraction
- httplinks: 41.8s → 13ms on Linux kernel (3,212x faster)
- configlink: 41.4s → 0.8s on Linux kernel (54x faster)
- Linux kernel fast-mode total: 2m38s → 1m18s

Bug fixes:
- __init__.py Module QN no longer collides with Folder QN
- index.ts same fix for JS/TS packages
- 13 regression tests for QN collision at FQN + extraction layers
- search_graph/search_code default limit raised from 10 to 500k
- Resolve all clang-tidy, cppcheck, and clang-format warnings

Repo cleanup:
- tree-sitter-form, tree-sitter-magma moved to tools/
- .gitignore: build/, node_modules/, graph-ui/dist/, TEST_PLAN.md
2026-03-17 22:05:57 +01:00

195 lines
5.6 KiB
Bash
Executable File

#!/bin/bash
# embed-frontend.sh — Convert built frontend assets into linkable object files.
#
# Usage: scripts/embed-frontend.sh <dist_dir> <output_dir>
#
# For each file in dist_dir, creates:
# 1. An object file via `ld -r -b binary` (raw bytes, zero bloat)
# 2. A generated embedded_assets.c with a lookup table
#
# Symbols created per file: _binary_<mangled_name>_start, _binary_<mangled_name>_end
# On macOS, these get a leading underscore: __binary_<mangled_name>_start
set -euo pipefail
DIST_DIR="${1:?Usage: embed-frontend.sh <dist_dir> <output_dir>}"
OUTPUT_DIR="${2:?Usage: embed-frontend.sh <dist_dir> <output_dir>}"
# Clean old embedded objects (asset hashes change on each build)
rm -rf "$OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR"
# Detect platform
IS_MACOS=false
if [[ "$(uname -s)" == "Darwin" ]]; then
IS_MACOS=true
fi
# Content-type detection
content_type_for() {
local f="$1"
case "$f" in
*.html) echo "text/html" ;;
*.js) echo "application/javascript" ;;
*.css) echo "text/css" ;;
*.json) echo "application/json" ;;
*.svg) echo "image/svg+xml" ;;
*.png) echo "image/png" ;;
*.ico) echo "image/x-icon" ;;
*.woff2) echo "font/woff2" ;;
*.woff) echo "font/woff" ;;
*.map) echo "application/json" ;;
*) echo "application/octet-stream" ;;
esac
}
# Mangle filename to valid C symbol: replace non-alnum with _
mangle() {
echo "$1" | sed 's/[^a-zA-Z0-9]/_/g'
}
# Collect all files
FILES=()
while IFS= read -r -d '' file; do
FILES+=("$file")
done < <(find "$DIST_DIR" -type f -print0 | sort -z)
if [[ ${#FILES[@]} -eq 0 ]]; then
echo "Error: no files found in $DIST_DIR" >&2
exit 1
fi
echo "Embedding ${#FILES[@]} files from $DIST_DIR"
# Generate object files
OBJ_FILES=()
for file in "${FILES[@]}"; do
rel="${file#$DIST_DIR/}"
mangled=$(mangle "$rel")
obj="$OUTPUT_DIR/embed_${mangled}.o"
if $IS_MACOS; then
# macOS: use custom section + assembly approach
# ld -r -b binary not available on macOS default ld
# Instead, use xxd to create a C array (for macOS compatibility)
local_c="$OUTPUT_DIR/embed_${mangled}.c"
local_sym="_binary_${mangled}"
echo "/* Generated from $rel */" > "$local_c"
echo "const unsigned char ${local_sym}_data[] = {" >> "$local_c"
xxd -i < "$file" >> "$local_c"
echo "};" >> "$local_c"
echo "const unsigned int ${local_sym}_size = sizeof(${local_sym}_data);" >> "$local_c"
cc -c -O2 -o "$obj" "$local_c"
rm -f "$local_c"
else
# Linux: ld -r -b binary (zero bloat)
(cd "$DIST_DIR" && ld -r -b binary -o "$obj" "$rel")
fi
OBJ_FILES+=("$obj")
echo " $rel -> $obj"
done
# Generate embedded_assets.c
ASSETS_C="src/ui/embedded_assets.c"
cat > "$ASSETS_C" <<'HEADER'
/*
* embedded_assets.c — Generated file mapping URL paths to embedded bytes.
* DO NOT EDIT — regenerated by scripts/embed-frontend.sh
*/
#include "ui/embedded_assets.h"
#include <string.h>
HEADER
# Declare extern symbols
for file in "${FILES[@]}"; do
rel="${file#$DIST_DIR/}"
mangled=$(mangle "$rel")
sym="_binary_${mangled}"
if $IS_MACOS; then
echo "extern const unsigned char ${sym}_data[];" >> "$ASSETS_C"
echo "extern const unsigned int ${sym}_size;" >> "$ASSETS_C"
else
echo "extern const unsigned char ${sym}_start[];" >> "$ASSETS_C"
echo "extern const unsigned char ${sym}_end[];" >> "$ASSETS_C"
fi
done
echo "" >> "$ASSETS_C"
echo "cbm_embedded_file_t CBM_EMBEDDED_FILES[] = {" >> "$ASSETS_C"
for file in "${FILES[@]}"; do
rel="${file#$DIST_DIR/}"
mangled=$(mangle "$rel")
sym="_binary_${mangled}"
ct=$(content_type_for "$rel")
# URL path: /index.html for root, /assets/... for assets
url_path="/$rel"
if $IS_MACOS; then
echo " {\"$url_path\", ${sym}_data, 0, \"$ct\"}," >> "$ASSETS_C"
else
echo " {\"$url_path\", ${sym}_start, 0, \"$ct\"}," >> "$ASSETS_C"
fi
done
echo "};" >> "$ASSETS_C"
echo "const int CBM_EMBEDDED_FILE_COUNT = ${#FILES[@]};" >> "$ASSETS_C"
# Generate size fixup (for macOS where we have explicit size vars)
if $IS_MACOS; then
cat >> "$ASSETS_C" <<'SIZEINIT'
/* Initialize sizes — called once or we use the _size vars directly */
static void __attribute__((constructor)) init_embedded_sizes(void) {
cbm_embedded_file_t *files = CBM_EMBEDDED_FILES;
SIZEINIT
for i in "${!FILES[@]}"; do
rel="${FILES[$i]#$DIST_DIR/}"
mangled=$(mangle "$rel")
sym="_binary_${mangled}"
echo " files[$i].size = ${sym}_size;" >> "$ASSETS_C"
done
echo "}" >> "$ASSETS_C"
else
# Linux: compute size from start/end pointers
cat >> "$ASSETS_C" <<'SIZEINIT'
static void __attribute__((constructor)) init_embedded_sizes(void) {
cbm_embedded_file_t *files = CBM_EMBEDDED_FILES;
SIZEINIT
for i in "${!FILES[@]}"; do
rel="${FILES[$i]#$DIST_DIR/}"
mangled=$(mangle "$rel")
sym="_binary_${mangled}"
echo " files[$i].size = (unsigned int)(${sym}_end - ${sym}_start);" >> "$ASSETS_C"
done
echo "}" >> "$ASSETS_C"
fi
# Add lookup function
cat >> "$ASSETS_C" <<'LOOKUP'
const cbm_embedded_file_t *cbm_embedded_lookup(const char *path) {
for (int i = 0; i < CBM_EMBEDDED_FILE_COUNT; i++) {
if (strcmp(CBM_EMBEDDED_FILES[i].path, path) == 0) {
return &CBM_EMBEDDED_FILES[i];
}
}
return NULL;
}
LOOKUP
echo "Generated $ASSETS_C with ${#FILES[@]} embedded files"
echo "Object files in $OUTPUT_DIR:"
printf ' %s\n' "${OBJ_FILES[@]}"