github-featured/deusdata--codebase-memory-mcp
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix(ci): scan extracted UI release files only

Browse Source 
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel
2026-08-09 02:39:09 +02:00
parent 1ad53bb13d
commit 23e4fb0b4e
17 changed files with 204 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/README.md
+1 -1
View File
@@ -15,4 +15,4 @@ CI and the local infrastructure — both of which the venue-parity contract
| `generate-sbom.py` | The release SPDX SBOM (vendored versions reviewable here, diffable by vendoring PRs — was inline YAML). | `release.yml` |
| `require-all-green.sh` | The aggregate gate: fail unless every needed job succeeded or legitimately skipped (was inline YAML). | `pr.yml ci-ok` |
| `verify-shard-union.sh` | Prove sharded test legs lost nothing: shard count agreement, indices 1..n, identical suite lists, union of slices == full list (was inline YAML). | `_test.yml` shard-completeness |
| `check-virustotal.sh` | Release-asset VirusTotal lookups. | `release.yml` |
| `check-virustotal.sh` | Exact extracted-file VirusTotal lookups. | `release.yml` |
scripts/ci/append-vt-notes.sh
+33 -27
View File
@@ -106,7 +106,7 @@ asset_base = (
)
association_meta, associations = read_tsv(
association_path,
"cbm-release-scan-associations-v2",
"cbm-release-scan-associations-v3",
ASSOCIATION_FIELDS,
)
result_meta, results = read_tsv(
@@ -171,19 +171,22 @@ for association in associations:
):
fail(f"result hash/size differs from association: {association['scan_path']}")
archive_rows = [row for row in associations if row["association_type"] == "archive"]
member_rows = [row for row in associations if row["association_type"] == "member"]
asset_rows = [row for row in associations if row["association_type"] == "pack_asset"]
if len(archive_rows) != association_meta.get("archives"):
fail("downloadable archive association count is inconsistent")
archive_names = [row["archive"] for row in archive_rows]
if len(set(archive_names)) != len(archive_names):
fail("downloadable archive association is duplicated")
archive_hashes = {row["archive"]: row["archive_sha256"] for row in archive_rows}
if any(row["archive_sha256"] != row["object_sha256"] for row in archive_rows):
fail("downloadable archive SHA is not bound to its scan object")
if any(archive_hashes.get(row["archive"]) != row["archive_sha256"] for row in associations):
fail("member/asset association is not bound to its downloadable archive")
if len(member_rows) + len(asset_rows) != len(associations):
fail("association manifest contains a non-extracted scan target")
archive_hashes: Dict[str, str] = {}
for association in associations:
archive = association["archive"]
archive_sha256 = association["archive_sha256"]
if not archive or re.fullmatch(r"[0-9a-f]{64}", archive_sha256) is None:
fail("member/asset association lacks valid archive SHA provenance")
previous_sha256 = archive_hashes.get(archive)
if previous_sha256 is not None and previous_sha256 != archive_sha256:
fail(f"conflicting archive SHA provenance: {archive}")
archive_hashes[archive] = archive_sha256
if len(archive_hashes) != association_meta.get("archives"):
fail("distinct archive provenance count is inconsistent")
engine_counts = [int(row["completed_engines"]) for row in results]
minimum = min(engine_counts)
@@ -198,14 +201,20 @@ section = [
"## Security Verification",
"",
(
f"VirusTotal completed **{len(results)} distinct byte objects** covering "
f"**{len(associations)} exact release associations**: "
f"{len(archive_rows)} downloadable archives, {len(member_rows)} archive members, "
f"and {len(asset_rows)} independently extracted UI-pack assets."
f"VirusTotal completed **{len(results)} distinct extracted byte objects** covering "
f"**{len(associations)} exact extracted-file associations**: "
f"{len(member_rows)} archive members and "
f"{len(asset_rows)} independently extracted UI-pack assets."
),
(
f"The extraction manifest binds those associations to "
f"**{len(archive_hashes)} downloadable "
f"{'archive' if len(archive_hashes) == 1 else 'archives'}** by SHA-256 provenance. "
"Downloadable .tar.gz/.zip release containers were not submitted to VirusTotal."
),
"",
(
f"Every object returned **0 malicious and 0 suspicious** verdicts with "
f"Every scanned object returned **0 malicious and 0 suspicious** verdicts with "
f"{engine_range} decisive engine results (required minimum: {policy})."
),
(
@@ -217,20 +226,17 @@ section = [
"Durable public evidence: "
f"[associations]({asset_base}/virustotal-associations.tsv), "
f"[exact scan set]({asset_base}/virustotal-scan-set.tsv), "
f"[per-object results and report links]({asset_base}/virustotal-results.tsv), "
f"[per-extracted-object results and report links]({asset_base}/virustotal-results.tsv), "
f"[evidence checksums]({asset_base}/virustotal-evidence-checksums.txt)."
),
"",
"| Downloadable archive | SHA-256 | Engines | VirusTotal |",
"|---|---|---:|---|",
"Archive SHA-256 provenance (from the extraction manifest):",
"",
"| Downloadable archive | SHA-256 provenance |",
"|---|---|",
]
for association in sorted(archive_rows, key=lambda row: row["archive"]):
result = results_by_path[association["scan_path"]]
sha256 = result["sha256"]
section.append(
f"| `{association['archive']}` | `{sha256}` | "
f"{result['completed_engines']} | [0 detections]({result['virustotal_url']}) |"
)
for archive, archive_sha256 in sorted(archive_hashes.items()):
section.append(f"| `{archive}` | `{archive_sha256}` |")
section.append(END)
replacement = "\n".join(section)
scripts/ci/check-virustotal.sh
+14 -17
View File
@@ -186,7 +186,7 @@ def parse_versioned_tsv(
def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
metadata, rows = parse_versioned_tsv(
path,
marker="cbm-release-scan-set-v1",
marker="cbm-release-scan-set-v2",
fields=SCAN_FIELDS,
)
expected_count = metadata.get("scan_objects")
@@ -217,7 +217,7 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
size = int(row["size"])
association_count = int(row["association_count"])
kinds = row["association_kinds"].split(",")
allowed_kinds = {"archive", "binary", "pack", "runtime", "ui_asset"}
allowed_kinds = {"binary", "pack", "runtime", "ui_asset"}
if (
association_count < 1
or any(not kind or kind not in allowed_kinds for kind in kinds)
@@ -258,7 +258,7 @@ def validate_associations(
) -> None:
metadata, rows = parse_versioned_tsv(
path,
marker="cbm-release-scan-associations-v2",
marker="cbm-release-scan-associations-v3",
fields=ASSOCIATION_FIELDS,
)
if (
@@ -273,29 +273,26 @@ def validate_associations(
seen: set[Tuple[str, str, str, str]] = set()
archive_hashes: Dict[str, str] = {}
for row in rows:
if row["association_type"] != "archive":
continue
archive = row["archive"]
if not archive or archive in archive_hashes or row["archive_sha256"] != row["object_sha256"]:
raise GateError(f"malformed or duplicate archive association: {archive}")
archive_hashes[archive] = row["archive_sha256"]
archive_sha256 = row["archive_sha256"]
if not archive or SHA256_RE.fullmatch(archive_sha256) is None:
raise GateError(f"malformed archive provenance: {archive}")
previous = archive_hashes.setdefault(archive, archive_sha256)
if previous != archive_sha256:
raise GateError(f"conflicting archive provenance: {archive}")
if metadata.get("archives") != len(archive_hashes):
raise GateError("archive provenance count does not match the association manifest")
for row in rows:
association_type = row["association_type"]
key = (association_type, row["archive"], row["member"], row["asset_path"])
if key in seen:
raise GateError(f"duplicate release association: {key}")
seen.add(key)
if association_type not in {"archive", "member", "pack_asset"} or not row["archive"]:
if association_type not in {"member", "pack_asset"} or not row["archive"]:
raise GateError(f"malformed release association: {key}")
if archive_hashes.get(row["archive"]) != row["archive_sha256"]:
raise GateError(f"association is not bound to its archive object: {key}")
if association_type == "archive":
semantic_valid = (
not row["member"]
and not row["asset_path"]
and row["kind"] == "archive"
)
elif association_type == "member":
raise GateError(f"association is not bound to its archive provenance: {key}")
if association_type == "member":
semantic_valid = bool(row["member"]) and not row["asset_path"]
else:
semantic_valid = bool(row["member"] and row["asset_path"] and row["mime"]) and row["kind"] == "ui_asset"
scripts/ci/extract-release-archives.sh
+12 -18
View File
@@ -9,12 +9,14 @@
#
# The output directory is published as one atomic bundle:
# objects/ one file per distinct byte sequence
# associations.tsv every archive, member and CBMUIPK asset -> scan object
# associations.tsv every extracted member and CBMUIPK asset -> scan object
# scan-set.tsv the exact path/hash/size set the VT action must return
#
# The release archives themselves, their exact members, and the uncompressed
# HTML/JavaScript/CSS/etc payloads inside every CBMUIPK v1 pack are all covered.
# Identical bytes are uploaded once, but no shipped association is discarded.
# Every archive is validated and hashed for provenance, but downloadable
# .tar.gz/.zip release containers are not scanned. Their exact members and the
# uncompressed HTML/JavaScript/CSS/etc payloads inside every CBMUIPK v1 pack are
# covered. Identical bytes are uploaded once, but no extracted member/asset
# association is discarded.
set -euo pipefail
if [ "$#" -lt 2 ]; then
@@ -733,24 +735,16 @@ def main(argv: Sequence[str]) -> None:
staged_output = pathlib.Path(temporary) / "bundle"
staged_output.mkdir(mode=0o700)
store = ObjectStore(staged_output / "objects")
archive_store = ObjectStore(pathlib.Path(temporary) / "archives")
for archive_path in archive_paths:
archive_name = archive_path.name
variant = "ui" if archive_name.startswith("codebase-memory-mcp-ui-") else "standard"
archive_object = store.ingest_path(
archive_object = archive_store.ingest_path(
archive_path,
ceiling=MAX_ARCHIVE_BYTES,
label=archive_name,
)
archive_sha256 = archive_object.sha256
add_association(
rows,
archive_object,
association_type="archive",
archive=archive_name,
archive_sha256=archive_sha256,
variant=variant,
kind="archive",
)
_, kinds, member_total = (
process_tar(
archive_object.path,
@@ -792,7 +786,7 @@ def main(argv: Sequence[str]) -> None:
rows.sort(
key=lambda row: (
str(row["archive"]),
{"archive": 0, "member": 1, "pack_asset": 2}[str(row["association_type"])],
{"member": 0, "pack_asset": 1}[str(row["association_type"])],
str(row["member"]),
str(row["asset_path"]),
)
@@ -808,7 +802,7 @@ def main(argv: Sequence[str]) -> None:
)
write_tsv(
staged_output / "associations.tsv",
marker="cbm-release-scan-associations-v2",
marker="cbm-release-scan-associations-v3",
metadata=((key, counts[key]) for key in metadata_order),
fields=ASSOCIATION_FIELDS,
rows=rows,
@@ -825,7 +819,7 @@ def main(argv: Sequence[str]) -> None:
]
write_tsv(
staged_output / "scan-set.tsv",
marker="cbm-release-scan-set-v1",
marker="cbm-release-scan-set-v2",
metadata=((key, counts[key]) for key in ("scan_objects", "associations")),
fields=SCAN_SET_FIELDS,
rows=scan_rows,
@@ -848,7 +842,7 @@ def main(argv: Sequence[str]) -> None:
)
print(
f"scan bundle: {counts['scan_objects']} distinct byte objects cover "
f"{counts['associations']} archive/member/asset associations"
f"{counts['associations']} extracted member/asset associations"
)
print(f"associations: {output_dir / 'associations.tsv'}")
print(f"expected scan set: {output_dir / 'scan-set.tsv'}")
scripts/ci/publish-vt-evidence.sh
+2 -2
View File
@@ -38,8 +38,8 @@ publish_copy() {
chmod 0644 "$WORK/$destination"
}
publish_copy "$VT_ASSOCIATIONS" cbm-release-scan-associations-v2 virustotal-associations.tsv
publish_copy "$VT_EXPECTED_SCAN_SET" cbm-release-scan-set-v1 virustotal-scan-set.tsv
publish_copy "$VT_ASSOCIATIONS" cbm-release-scan-associations-v3 virustotal-associations.tsv
publish_copy "$VT_EXPECTED_SCAN_SET" cbm-release-scan-set-v2 virustotal-scan-set.tsv
publish_copy "$VT_RESULTS_PATH" cbm-virustotal-results-v1 virustotal-results.tsv
for name in virustotal-associations.tsv virustotal-results.tsv virustotal-scan-set.tsv; do