From ca9818b47018a8af2ebe6f892b32f9ee1bbb6863 Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Fri, 21 Aug 2026 15:27:52 +0200
Subject: [PATCH] security: catch prose smuggled into generated parser symbol
 tables

Tier 2 of the hidden-instruction gate. A generated LR parser's string
table holds grammar symbol names and punctuation terminals; an English
sentence in there is anomalous by construction. That is the check which
cleared a 353,744-line vendored parser by hand during review, and this
mechanises it.

The threshold was MEASURED rather than guessed. Across all 159 vendored
grammars: 48,271 string literals, of which 63 contain a space, because
multi-word keywords are real -- "is not", "not in", "static get". The
longest legitimate literal is three words ("hide empty description"), so
four is the tightest threshold with zero false positives, and it still
catches a four-word instruction. Current tree: 0 findings.

Deliberately NOT shipped: a tree-wide scan for hiding constructs. The
measurement did not support it. `<!--` and `<script` have 61 legitimate
uses across PR templates, docs and HTML-parsing tests, and `display:none`
is ordinary styling in docs/index.html, the project website. The real
concern is a hiding construct inside AGENT-FACING content, which is a
location question rather than a pattern question and belongs with the
Tier 3 location rule. A rule that needs excuses gets switched off, so it
is left out with the reasoning recorded at the point where it would have
gone.

Verified: injecting a six-word literal into a vendored parser's symbol
table is refused with the file, line and literal quoted; reverting
restores green.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 SECURITY.md                   |  3 +-
 scripts/security-injection.py | 69 +++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/SECURITY.md b/SECURITY.md
index fd2d99ea..94498b8b 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -100,7 +100,8 @@ This project implements multiple layers of security verification. Every release
 ### Build-Time (CI — every commit)
 
 - **9-layer security audit suite** runs on every build:
-  - Layer 0: Hidden-instruction audit (invisible/bidi/tag Unicode across the whole tree)
+  - Layer 0: Hidden-instruction audit (invisible/bidi/tag Unicode tree-wide;
+    prose smuggled into generated parser symbol tables)
   - Layer 1: Static allow-list for dangerous calls (`system`/`popen`/`fork`) + hardcoded URLs
   - Layer 2: Binary string audit (URLs, credentials, dangerous commands)
   - Layer 3: Network egress monitoring via strace (Linux)
diff --git a/scripts/security-injection.py b/scripts/security-injection.py
index 2db198ac..cb1c94a4 100644
--- a/scripts/security-injection.py
+++ b/scripts/security-injection.py
@@ -44,6 +44,7 @@ this gate rejects, so an allowlist entry cannot be produced mechanically.
 """
 
 import hashlib
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -116,6 +117,68 @@ def scan(root):
                 yield rel, line_no, line, digest, found
 
 
+# ── Tier 2: scoped structural checks ───────────────────────────────────
+#
+# Every threshold here was MEASURED against this tree before being gated, not
+# guessed. A rule with a false-positive rate becomes noise, gets whitelisted,
+# and then gets ignored -- so a rule that cannot be made clean is left out
+# rather than shipped loose.
+
+# A generated LR parser's string table holds grammar symbol names and
+# punctuation terminals. Multi-word keywords are real ("is not", "not in",
+# "static get"), so a bare space is NOT a signal. Measured across all 159
+# vendored grammars: 48,271 literals, 63 contain a space, and the longest
+# legitimate one is three words ("hide empty description"). Prose needs more.
+# Four is therefore the tightest threshold with zero false positives today, and
+# it still catches a four-word instruction like "ignore all previous
+# instructions".
+PARSER_PROSE_WORDS = 4
+
+# DEFERRED -- hiding constructs (`display:none`, `visibility:hidden`, HTML
+# comments, `<script`) are NOT checked here, because measurement showed the
+# rule cannot be made clean at tree scope. `<!--` and `<script` have 61
+# legitimate uses across PR templates, docs and HTML-parsing tests, and
+# `display:none` is ordinary styling in docs/index.html, the project website.
+# The real concern is a hiding construct inside AGENT-FACING content, which is
+# a location question rather than a pattern question -- it belongs with the
+# Tier 3 location rule, once the set of places we deliberately instruct agents
+# is enumerated. Shipping it loose here would produce a rule that needs
+# excuses, and a rule that needs excuses gets switched off.
+
+_LITERAL = re.compile(r'"((?:[^"\\\n]|\\.)*)"')
+
+
+def tier2_findings(root):
+    """Yield (path, line_no, detail) for prose smuggled into generated parsers."""
+    for rel in tracked_files(root):
+        full = root / rel
+        is_parser = rel.startswith("internal/cbm/vendored/grammars/") and rel.endswith(
+            "parser.c"
+        )
+        try:
+            raw = full.read_bytes()
+        except (OSError, ValueError):
+            continue
+        try:
+            text = raw.decode("utf-8")
+        except UnicodeDecodeError:
+            continue
+
+        if is_parser:
+            for m in _LITERAL.finditer(text):
+                lit = m.group(1)
+                words = [w for w in lit.split(" ") if w]
+                if len(words) >= PARSER_PROSE_WORDS:
+                    line_no = text.count("\n", 0, m.start()) + 1
+                    yield rel, line_no, (
+                        f"generated parser holds a {len(words)}-word string "
+                        f"literal (prose does not belong in a symbol table): "
+                        f"{lit[:80]!r}"
+                    )
+            continue
+
+
+
 def load_allowlist(root):
     """Return {(sha256, path): why}. Entries without a real why are dropped."""
     path = root / ALLOWLIST
@@ -263,6 +326,12 @@ def main(argv):
             print(f"    {digest}  {rel}  # <why this is safe>")
         problems += len(unexplained)
 
+    for rel, line_no, detail in tier2_findings(root):
+        if problems == 0:
+            print("=== HIDDEN-INSTRUCTION AUDIT: REFUSED ===\n")
+        print(f"{rel}:{line_no}: {detail}\n")
+        problems += 1
+
     stale = set(allowed) - {(d, r) for r, _n, _l, d, _f in hits}
     for digest, rel in sorted(stale):
         print(f"FAIL: stale allowlist entry (line no longer present): {digest}  {rel}")