From ca9a74e84abdf9483c234e82dc54b9ec2c00d8c0 Mon Sep 17 00:00:00 2001 From: Wes Mason Date: Tue, 4 Aug 2026 15:33:32 +0100 Subject: [PATCH] feat(observability-map): static observability scorer for webapp route entry points (#4455) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A static observability scorer for the webapp's route entry points, Lighthouse-style. The idea comes from evlog's `map` command, but that tool has no Remix adapter and checks for its own logging API, so the idea is ported rather than the tool. It scans all 427 loader/action entry points in `apps/webapp/app/routes` with the TypeScript compiler API and scores each against five checks: error-classification, auth-boundary, auth-scope, request-context and audit-trail. Current output on the real tree is **19/100** over 412 measured entry points. ``` cd internal-packages/observability-map pnpm exec tsx src/cli.ts # terminal report pnpm exec tsx src/cli.ts --json # machine output pnpm exec tsx src/cli.ts api/v1/token # one entry, per-check detail ``` The two findings at the top of the fix list are real: `/auth/sso` and `/api/v1/authorization-code` mint or exchange credentials unauthenticated, and `/_app/orgs/:organizationSlug/settings/team` resolves its org from a URL slug and gates each mutating branch on an RBAC check alone, which per `apps/webapp/CLAUDE.md` is not the tenant floor on self-hosted. Decisions worth knowing, all with the reasoning in the README: - The score started at 83 during development and fell to 19. Every drop was a perverse incentive being removed, not a regression: routes were being paid for having no error handling, two checks were reading the same fact, suppressing a failure raised the score, and a no-op `catch (e) { throw e }` was worth 50 points a route. - **A mutation corpus is the tool's main defence.** 44 entries apply semantics-preserving edits to a copy of the real route tree and assert the score cannot rise, per route as well as globally, because a mean can hide one route going up by taking another down. One entry runs as a live expected failure: `try { String(0); }` with a deciding catch is a known open hole worth 19 to 44, and it is disclosed rather than quietly excluded. - `audit-trail` and `request-context` are reported as headline figures rather than one finding repeated hundreds of times. Both still count in full where they should. - A cohort change moves the number without anything in the codebase getting better. Widening the sensitive cohort from 26 to 67 took the global from 15 to 19 with no webapp change at all, so the report prints per-check applicability and what the global would be without each one. CI: a report-only job posts a sticky comment when a PR moves the report, and says nothing when it does not. The package's own tests gate through `pr_checks.yml`. The diff-scoped merge gate is still deferred until the report has been used in anger. 524 tests plus the corpus. No runtime or dependency changes to anything that ships. --- This is **part 1 of 4 in a stack** made with GitButler: -  4  #4485 -  3  #4484 -  2  #4483 -  1  #4455 πŸ‘ˆ --- .github/workflows/observability-map.yml | 375 +++ .github/workflows/pr_checks.yml | 37 + .../unit-tests-observability-map.yml | 43 + .gitignore | 3 + .../observability-map/INTERNALS.md | 500 +++ internal-packages/observability-map/README.md | 336 ++ .../observability-map/package.json | 25 + .../src/adapters/remix.test.ts | 57 + .../observability-map/src/adapters/remix.ts | 40 + .../src/checks/auditTrail.ts | 54 + .../src/checks/authBoundary.ts | 137 + .../observability-map/src/checks/authScope.ts | 68 + .../src/checks/errorClassification.ts | 155 + .../src/checks/index.test.ts | 2198 ++++++++++++ .../observability-map/src/checks/index.ts | 23 + .../src/checks/requestContext.ts | 67 + .../observability-map/src/cli.test.ts | 221 ++ .../observability-map/src/cli.ts | 151 + .../src/docstringReferences.test.ts | 176 + .../observability-map/src/index.ts | 6 + .../observability-map/src/integration.test.ts | 405 +++ .../src/mutationCorpus.test.ts | 393 +++ .../src/mutationTrivia.test.ts | 81 + .../observability-map/src/mutations.ts | 1063 ++++++ .../observability-map/src/report/json.ts | 5 + .../src/report/prComment.test.ts | 733 ++++ .../observability-map/src/report/prComment.ts | 339 ++ .../src/report/prCommentCli.test.ts | 244 ++ .../src/report/prCommentCli.ts | 134 + .../src/report/terminal.test.ts | 401 +++ .../observability-map/src/report/terminal.ts | 218 ++ .../observability-map/src/routeExports.ts | 61 + .../observability-map/src/scan.test.ts | 2983 +++++++++++++++++ .../observability-map/src/scan.ts | 1648 +++++++++ .../observability-map/src/score.test.ts | 617 ++++ .../observability-map/src/score.ts | 249 ++ .../observability-map/src/sensitivity.test.ts | 236 ++ .../observability-map/src/sensitivity.ts | 137 + .../observability-map/src/suppression.test.ts | 296 ++ .../observability-map/src/suppression.ts | 145 + .../observability-map/src/triviality.test.ts | 167 + .../observability-map/src/triviality.ts | 80 + .../observability-map/src/types.ts | 132 + .../src/webappSymbols.test.ts | 179 + .../observability-map/tsconfig.build.json | 15 + .../observability-map/tsconfig.json | 21 + .../observability-map/turbo.json | 31 + .../observability-map/vitest.config.ts | 5 + package.json | 1 + pnpm-lock.yaml | 19 + 50 files changed, 15710 insertions(+) create mode 100644 .github/workflows/observability-map.yml create mode 100644 .github/workflows/unit-tests-observability-map.yml create mode 100644 internal-packages/observability-map/INTERNALS.md create mode 100644 internal-packages/observability-map/README.md create mode 100644 internal-packages/observability-map/package.json create mode 100644 internal-packages/observability-map/src/adapters/remix.test.ts create mode 100644 internal-packages/observability-map/src/adapters/remix.ts create mode 100644 internal-packages/observability-map/src/checks/auditTrail.ts create mode 100644 internal-packages/observability-map/src/checks/authBoundary.ts create mode 100644 internal-packages/observability-map/src/checks/authScope.ts create mode 100644 internal-packages/observability-map/src/checks/errorClassification.ts create mode 100644 internal-packages/observability-map/src/checks/index.test.ts create mode 100644 internal-packages/observability-map/src/checks/index.ts create mode 100644 internal-packages/observability-map/src/checks/requestContext.ts create mode 100644 internal-packages/observability-map/src/cli.test.ts create mode 100644 internal-packages/observability-map/src/cli.ts create mode 100644 internal-packages/observability-map/src/docstringReferences.test.ts create mode 100644 internal-packages/observability-map/src/index.ts create mode 100644 internal-packages/observability-map/src/integration.test.ts create mode 100644 internal-packages/observability-map/src/mutationCorpus.test.ts create mode 100644 internal-packages/observability-map/src/mutationTrivia.test.ts create mode 100644 internal-packages/observability-map/src/mutations.ts create mode 100644 internal-packages/observability-map/src/report/json.ts create mode 100644 internal-packages/observability-map/src/report/prComment.test.ts create mode 100644 internal-packages/observability-map/src/report/prComment.ts create mode 100644 internal-packages/observability-map/src/report/prCommentCli.test.ts create mode 100644 internal-packages/observability-map/src/report/prCommentCli.ts create mode 100644 internal-packages/observability-map/src/report/terminal.test.ts create mode 100644 internal-packages/observability-map/src/report/terminal.ts create mode 100644 internal-packages/observability-map/src/routeExports.ts create mode 100644 internal-packages/observability-map/src/scan.test.ts create mode 100644 internal-packages/observability-map/src/scan.ts create mode 100644 internal-packages/observability-map/src/score.test.ts create mode 100644 internal-packages/observability-map/src/score.ts create mode 100644 internal-packages/observability-map/src/sensitivity.test.ts create mode 100644 internal-packages/observability-map/src/sensitivity.ts create mode 100644 internal-packages/observability-map/src/suppression.test.ts create mode 100644 internal-packages/observability-map/src/suppression.ts create mode 100644 internal-packages/observability-map/src/triviality.test.ts create mode 100644 internal-packages/observability-map/src/triviality.ts create mode 100644 internal-packages/observability-map/src/types.ts create mode 100644 internal-packages/observability-map/src/webappSymbols.test.ts create mode 100644 internal-packages/observability-map/tsconfig.build.json create mode 100644 internal-packages/observability-map/tsconfig.json create mode 100644 internal-packages/observability-map/turbo.json create mode 100644 internal-packages/observability-map/vitest.config.ts diff --git a/.github/workflows/observability-map.yml b/.github/workflows/observability-map.yml new file mode 100644 index 000000000..3336492d5 --- /dev/null +++ b/.github/workflows/observability-map.yml @@ -0,0 +1,375 @@ +name: πŸ—ΊοΈ Observability Map + +on: + # No paths filter, deliberately. GitHub evaluates one per workflow, so a pull request whose diff + # stops matching does not start the workflow at all: the resolved state cannot fire and a comment + # from an earlier push stands for ever showing findings that are no longer in the diff. Verified on + # a throwaway pull request whose only route change was reverted, and the realistic case is worse + # than that empty diff, because a pull request touching a route and other files, whose author + # reverts the route change and keeps the rest, still has a non-empty diff that no longer matches. + # The gating moved into the jobs below instead, where it can read whether a comment exists. + pull_request: + types: [opened, synchronize, reopened] + # The corpus job below is gated to this package's own paths, so a scheduled run is what still + # scans the tree as it drifts. Nightly rather than per route pull request: a new route can make a + # known laundering shape start paying, but that is a property of the tree accumulating, not of any + # one pull request, and it does not need catching within five minutes of the merge. + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +concurrency: + group: observability-map-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # The whole cost of a pull request that touches nothing this workflow watches: a checkout, a paths + # filter and one comment lookup. Everything expensive is gated on this job's outputs, and the + # lookup is here rather than in the report job so that gate can read it and the report job need + # never start. + changes: + name: πŸ” What moved + # Only the pull request path reads this job's output. On a schedule the action has no base to + # diff, warns that `before` is missing and reports the files in the last commit on main, which + # nothing then consults. Skipping it there keeps the nightly off a job it does not need. + if: github.event_name == 'pull_request' + runs-on: warp-ubuntu-latest-x64-2x + permissions: + contents: read + # Reading the pull request's comments, to find one an earlier push left. Read only: the write + # stays on the report job, which is the only job that posts. + pull-requests: read + outputs: + # The corpus job's gate. Narrower than the report's on purpose: what the corpus measures is + # the tool's resistance to laundering, which only an edit to the tool can weaken. + package: ${{ steps.filter.outputs.package }} + # The report job's gate, the union: a route change moves the report as well. + report: ${{ steps.filter.outputs.package == 'true' || steps.filter.outputs.routes == 'true' }} + # The id of a marker comment an earlier push left, empty if there is none, and the one source + # both the render and upsert steps read it from. + comment: ${{ steps.comment.outputs.id }} + # Set only by a lookup that finished cleanly, so anything else, retries exhausted or the step + # dying somewhere unforeseen, reads as "do not touch this pull request's comments". + lookup: ${{ steps.comment.outputs.ok }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: filter + with: + filters: | + package: + - 'internal-packages/observability-map/**' + - '.github/workflows/observability-map.yml' + routes: + - 'apps/webapp/app/routes/**' + + # Looked up here because the report job's gate needs it: with the watched paths unmoved, a + # pull request that already has a comment gets a resolved state rather than being left with + # findings that no longer exist, and one that does not gets no job at all. + # + # On a failure that outlasts the retries this reports nothing, and the report job's gate reads + # that as "post nothing this run". Guessing is worse than silence: this step is the only thing + # that knows which comment to PATCH, so a guess of "no comment exists" POSTs, which either + # adds a second marker comment beside the stale one or says "the findings an earlier push + # reported are gone" on a pull request that never had findings. Worst case now is no comment + # this run, which the next push fixes. + - name: πŸ” Look for a comment from an earlier push + id: comment + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + found="" + ok="" + for attempt in 1 2 3; do + if found=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq '[.[] | select((.body // "") | startswith(""))][0].id // empty'); then + ok=1 + break + fi + echo "comment lookup attempt ${attempt} failed" >&2 + sleep $((attempt * 5)) + done + + if [ -z "$ok" ]; then + echo "comment lookup failed after 3 attempts; this run posts nothing" >&2 + exit 0 + fi + + # --paginate runs the jq once per page, so a marker comment on more than one page yields + # one id per page. Unhandled, that puts a newline in the PATCH url and the step dies under + # continue-on-error. The oldest wins: it is the one the upsert has been updating. + count=$(printf '%s\n' "$found" | grep -c '[0-9]' || true) + if [ "$count" -gt 1 ]; then + echo "warning: ${count} marker comments on this pull request; updating the oldest" >&2 + fi + { + echo "id=$(printf '%s\n' "$found" | awk 'NF { print $1; exit }')" + echo "ok=ok" + } >> "$GITHUB_OUTPUT" + + # The tree-scale mutation corpus: every known laundering shape applied to the whole route tree, + # asserting the score does not rise. Roughly four and a half minutes for 45 entries, which is why + # it is gated out of the package's default `pnpm test` and run here instead. Unlike the report + # job below it has no token to lose, so it runs for fork PRs too, and unlike the report job it is + # allowed to fail the build. + # + # Gated to this package's own paths rather than running on every route pull request. What the + # corpus measures is the TOOL's resistance to laundering, and only an edit to the tool can weaken + # that, so a routes-only change was paying four and a half minutes of a 4x runner for a result + # that could not differ from the last one. It was also the worst kind of job to spend that on: a + # red x that fires on a large share of webapp pull requests, is allowed to fail, and gates + # nothing, which is the shape people learn to scroll past. + # + # What this gives up is real and small. A route landing a shape no corpus entry has seen can make + # a known laundering mutation start paying, and that is now caught by the nightly rather than by + # the pull request that caused it. Tree drift accrues over months, so a day is the right + # granularity for it; the tool's own regressions, which are the ones a single commit can cause, + # still gate per pull request. + mutation-corpus: + name: 🧬 Mutation corpus + needs: changes + # `!cancelled()` is here for the nightly, not for tidiness. `needs` carries an implicit + # success() on the job it names, and that implicit test outranks the `||` below: with a plain + # condition, a `changes` job that failed or was skipped skips this one, so the nightly would + # stop scanning for tree drift and report nothing about having stopped. A status-check function + # in the `if` is what drops the implicit success(), so the event test below decides alone. + # `!cancelled()` rather than `always()` because `cancel-in-progress` above is a real path and a + # superseded run should not finish this job. + # + # Pull request behaviour is deliberately unchanged: on a PR a failed `changes` leaves + # `needs.changes.outputs.package` empty, so the corpus still skips. The nightly is the backstop + # for that, which is the same trade the paths gate already makes for routes-only pull requests. + if: >- + !cancelled() && + (github.event_name != 'pull_request' || needs.changes.outputs.package == 'true') + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: βŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: βŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: πŸ“₯ Download deps + run: pnpm install --frozen-lockfile + + - name: 🧬 Run the corpus + env: + OBS_MAP_MUTATION_CORPUS: "1" + run: | + pnpm --filter @internal/observability-map exec vitest run \ + src/mutationCorpus.test.ts --disable-console-intercept + + # The package's own tests are NOT run here. They gate through pr_checks.yml, which is the only + # workflow the all-checks aggregate can see, so a job in this file would report a result nobody + # is required to wait for. See unit-tests-observability-map.yml and the obsmap filter. + report: + needs: changes + runs-on: warp-ubuntu-latest-x64-4x + # Only this job comments, so only this job gets the write. + permissions: + contents: read + pull-requests: write + # Fork PRs get a read-only token, so the comment cannot post. Skipping the job beats a red x. + # The event test is what keeps this job off the nightly, which has no pull request to comment on + # and only exists for the corpus job above. + # + # The two output tests are what the workflow-level paths filter used to do, plus the thing it + # could not do. The report has to run when the watched paths moved, and ALSO when they did not + # but a marker comment is already on the pull request, because that comment is the one showing + # findings that have left the diff. Reconciling it needs no scan, so the steps below are gated + # again on the same output. + # + # `needs` carries an implicit success() and that is wanted here: a `changes` job that failed + # knows neither which paths moved nor whether a comment exists, and a report job that ran anyway + # could only guess. Same reason the lookup test is positive rather than a check for a failure + # sentinel: retries exhausted, or the lookup step dying anywhere unforeseen, both leave the + # output unset and both mean the same thing, so neither can be read as "no comment exists" by + # one step and "a comment exists" by another. That disagreement is what the sentinel pair this + # replaces got wrong once already. + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + needs.changes.outputs.lookup == 'ok' && + (needs.changes.outputs.report == 'true' || needs.changes.outputs.comment != '') + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: βŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: βŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: πŸ“₯ Download deps + run: pnpm install --frozen-lockfile + + # Guarded rather than allowed to fail: this job must never block a pull request. The failure + # is not swallowed either, the render step below turns a missing head report into a comment + # saying so, because a swallowed failure with no comment is the outcome nobody wants. + # + # `--out` rather than a stdout redirect, so nothing a tool decides to print can end up inside + # the document `prCommentCli` parses. `pnpm --filter` takes its recursive path and some + # versions announce `Scope: N of M workspace projects` on the way; that line landing in + # head.json would fail the parse and degrade every run to the stale-report comment, which is + # a permanent quiet failure rather than a loud one. It does not reproduce on the 10.33.2 + # pinned above, so this closes the class rather than a reproduction: the file is written by + # the process that owns it and stdout is left to be log output. Held by + # `it("let the scanner write its own report rather than capturing stdout")` in + # `internal-packages/observability-map/src/integration.test.ts`. + # + # `-s` keeps the partial dance honest now the redirect no longer creates the file: a scanner + # that exits 0 without writing takes the else branch and the stale-report comment, instead of + # failing the `mv` and turning the job red. + # + # Gated: this is the expensive half, and the reconcile run has nothing to compare. The steps + # above it are not gated because the renderer is TypeScript in this repo, so reconciling still + # needs the checkout and the install. That is the cost of the reconcile run and it is paid only + # by a pull request that has a comment and no longer matches the paths. + - name: πŸ”Ž Scan head + if: needs.changes.outputs.report == 'true' + run: | + if pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --out=/tmp/head.json.partial && [ -s /tmp/head.json.partial ]; then + mv /tmp/head.json.partial /tmp/head.json + else + rm -f /tmp/head.json /tmp/head.json.partial + echo "head scan failed; the comment will say the report is stale for this run" >&2 + fi + + # base.sha, not a merge base, and two reviewers have now read that as a bug. The checkout + # above is the default for a pull_request event, so the working tree is GitHub's test merge + # commit, whose parents are base.sha and the PR head. The head tree therefore already contains + # the base branch up to base.sha, and diffing it against base.sha is what isolates this pull + # request's own work. A merge base would leave the intervening base-branch commits in the head + # tree and out of the base tree, and blame the pull request for all of them. + - name: πŸ”Ž Scan base with the head's scanner + if: needs.changes.outputs.report == 'true' + run: | + if git worktree add /tmp/base-tree ${{ github.event.pull_request.base.sha }} \ + && pnpm --filter @internal/observability-map exec tsx src/cli.ts \ + --routes=/tmp/base-tree/apps/webapp/app/routes --out=/tmp/base.json \ + && [ -s /tmp/base.json ]; then + : + else + echo "-" > /tmp/base.json || true + echo "base scan failed or the worktree could not be added; falling back to no base" >&2 + fi + + # continue-on-error for the same reason as the scan: a rendering bug must not turn the job + # red. An empty /tmp/comment.md means there is nothing to post, which is a decision + # prCommentCli makes, not this shell. + # + # Both shas are forwarded so every comment this job posts says which commit it was rendered + # for, which a sticky comment edited in place across pushes otherwise never tells you. They go + # through the CLI as data: the renderer builds no URL and reads no environment. + - name: πŸ“ Render comment + continue-on-error: true + env: + SCANNED: ${{ needs.changes.outputs.report }} + EXISTING_COMMENT: ${{ needs.changes.outputs.comment }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + COMPARE_URL: ${{ github.server_url }}/${{ github.repository }}/compare/${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} + run: | + rm -f /tmp/comment.md + # `--out` rather than a stdout redirect, for the reason the scan steps above give, and with a + # worse failure mode than theirs: the marker has to be the comment's first line for the + # lookup to find it, so a line printed ahead of the document makes every push post a new + # comment instead of updating the one already there. Held by + # `it("let the renderer write its own comment rather than capturing stdout")`. + render() { + pnpm --filter @internal/observability-map exec tsx src/report/prCommentCli.ts \ + --commit-sha="$HEAD_SHA" --commit-url="$COMPARE_URL" --out=/tmp/comment.md.partial "$@" + } + + # Every write goes through this, so a renderer that exits non-zero never leaves a 0-byte + # comment.md for the upsert to skip in silence. + emit() { + rm -f /tmp/comment.md.partial + if render "$@"; then + mv /tmp/comment.md.partial /tmp/comment.md + return 0 + fi + rm -f /tmp/comment.md.partial + return 1 + } + + # Nothing this workflow watches moved, so nothing was scanned and there is no delta to + # compute. The job's gate only lets that case through when a comment from an earlier push + # is on the pull request, so there is exactly one thing left to say: what it shows is not + # in this diff any more. + if [ "$SCANNED" != "true" ]; then + emit --resolved || echo "could not render the resolved comment" >&2 + exit 0 + fi + + if [ ! -s /tmp/head.json ]; then + emit --scan-failed || echo "could not render the stale-report comment either" >&2 + exit 0 + fi + + base=/tmp/base.json + if [ ! -s /tmp/base.json ] || [ "$(cat /tmp/base.json)" = "-" ]; then + base="-" + fi + + flags=() + if [ -n "$EXISTING_COMMENT" ]; then + flags=(--existing-comment) + fi + + if ! emit /tmp/head.json "$base" "${flags[@]}"; then + echo "render failed; falling back to the stale-report comment" >&2 + emit --scan-failed || echo "could not render the stale-report comment either" >&2 + fi + + # continue-on-error for the same reason: a transient gh api failure (rate limit, network) + # must not fail the job either. Worst case, the PR gets no comment this run. + # + # The id comes from the same job output the render step read, so the two cannot disagree about + # whether a comment exists. A lookup that did not finish cleanly never reaches either of them: + # the job's gate stops it. + - name: πŸ’¬ Upsert PR comment + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EXISTING_COMMENT: ${{ needs.changes.outputs.comment }} + run: | + if [ ! -s /tmp/comment.md ]; then + echo "nothing to post: this pull request does not move the report" + exit 0 + fi + if [ -n "$EXISTING_COMMENT" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${EXISTING_COMMENT}" -F body=@/tmp/comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@/tmp/comment.md + fi diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b0fbd6ac0..e3df9b416 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -22,6 +22,7 @@ jobs: webapp: ${{ steps.filter.outputs.webapp }} packages: ${{ steps.filter.outputs.packages }} internal: ${{ steps.filter.outputs.internal }} + obsmap: ${{ steps.filter.outputs.obsmap }} cli: ${{ steps.filter.outputs.cli }} sdk: ${{ steps.filter.outputs.sdk }} steps: @@ -81,6 +82,36 @@ jobs: - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' - 'turbo.json' + # The whole webapp app tree, not just its routes, and that is the whole reason this + # filter exists. Two tests in @internal/observability-map read it: integration.test.ts + # scans the live route tree, and webappSymbols.test.ts walks all of apps/webapp/app and + # fails when a guard, sensitive or audit symbol stops resolving. Routes-only was this + # filter's own bug: renaming e.g. requireUserId in app/services/session.server.ts + # matched `webapp` and nothing else, so no job ran the suite and the break landed on + # main, or on the next unrelated internal-packages PR. + # + # The cost of the wider set, measured over the last 400 commits on main: 31% touch + # routes, 52% touch apps/webapp/app, so the job goes from firing on roughly a third of + # PRs to roughly a half. It is the cheap one -- a single 4x runner, no containers, no + # database, no prisma generate -- which is what makes that affordable. + # + # observability-map.yml is here because integration.test.ts asserts on its text and no + # other filter watches it, so editing the report workflow alone ran nothing at all. + # + # Deliberately NOT here: this package's own paths, and packages/plugins/src and + # internal-packages/rbac/src, the other two trees webappSymbols.test.ts reads. + # `internal` above already matches `internal-packages/**` and `packages/**`, and + # `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, which picks up + # @internal/observability-map and runs the same vitest suite. Listing them here as well + # ran the suite twice on every PR touching them, which was this filter's own doing. + obsmap: + - 'apps/webapp/app/**' + - '.github/workflows/pr_checks.yml' + - '.github/workflows/unit-tests-observability-map.yml' + - '.github/workflows/observability-map.yml' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' cli: - 'packages/cli-v3/**' - 'packages/build/**' @@ -149,6 +180,11 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + obsmap: + needs: changes + if: needs.changes.outputs.obsmap == 'true' + uses: ./.github/workflows/unit-tests-observability-map.yml + e2e: needs: changes if: needs.changes.outputs.cli == 'true' @@ -172,6 +208,7 @@ jobs: - e2e-webapp - packages - internal + - obsmap - e2e - sdk-compat if: always() diff --git a/.github/workflows/unit-tests-observability-map.yml b/.github/workflows/unit-tests-observability-map.yml new file mode 100644 index 000000000..eebf1ef97 --- /dev/null +++ b/.github/workflows/unit-tests-observability-map.yml @@ -0,0 +1,43 @@ +name: "πŸ§ͺ Unit Tests: Observability Map" + +permissions: + contents: read + +# Its own workflow rather than a job inside observability-map.yml, because that workflow is not +# reachable from pr_checks.yml's all-checks aggregate and so gates nothing. Called from there +# instead, behind a paths filter, which is how every other test suite in this repo is gated. +on: + workflow_call: + +jobs: + unitTests: + name: "πŸ§ͺ Unit Tests: Observability Map" + # No containers and no database: the package is a static analyser over source text, so the + # suite is CPU bound on parsing the route tree and needs nothing the runner does not have. + runs-on: warp-ubuntu-latest-x64-4x + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: βŽ” Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 10.33.2 + + - name: βŽ” Setup node + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 + with: + node-version: 24.18.0 + cache: "pnpm" + + - name: πŸ“₯ Download deps + run: pnpm install --frozen-lockfile + + # This suite reads apps/webapp/app (the route tree for the scan, the whole app tree for the + # symbol check) and the report workflow's text, which is why the filter that gates this + # workflow watches all of those and not only the routes folder. + - name: πŸ§ͺ Run tests + run: pnpm --filter @internal/observability-map run test diff --git a/.gitignore b/.gitignore index 7d9dc1690..f540927e3 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ ailogger-output.log # local planning/design docs, not committed **/docs/superpowers/ + +# observability-map CLI output artifact, not committed +observability-map.json diff --git a/internal-packages/observability-map/INTERNALS.md b/internal-packages/observability-map/INTERNALS.md new file mode 100644 index 000000000..c1dd52e2f --- /dev/null +++ b/internal-packages/observability-map/INTERNALS.md @@ -0,0 +1,500 @@ +# Internals + +How the scanner decides what it decides. Read [README.md](./README.md) first for what the tool +measures and how to run it; this file is for changing it. + +Every rule here refuses a shape that would otherwise mint free points, and every rejected +alternative written down is one somebody has proposed. + +## How the scanner reads a route + +`scan.ts` produces one `EntryPoint` per route module, carrying only body-scoped evidence. Three +rules decide what "the body" means, and every finding rests on them. + +**One hop, same file only.** A loader that delegates to a helper declared in the same file has that +helper's statements, try/catch and callees counted as its own. A helper's own helpers are not +followed, the visited set stops a cycle, and nothing imported from another module is ever opened. + +**Nested functions count as work.** A statement inside a callback written in the body is still a +statement the route runs. Leaving them out lets `trace("x", async () => { whole body })` collapse a +route to one statement, inside the triviality limit, so every check reports not-applicable for it +(`wrap-body-in-trace`). + +**Per export, not per file.** Six fields come in `loaderX`/`actionX` pairs, and the union is only +offered where the question itself is file-wide. The split refuses a family of false passes: a file +whose loader calls `requireUser` and whose action calls nothing reading as "guarded in the body", or +a file whose loader is `createLoaderApiRoute(...)` crediting its hand-written action with the +builder's authentication. `routeExports.ts` is the one enumeration both per-export checks read, +because `auth-scope` and `auth-boundary` each grew their own `[loader, action]` literal and only one +of them got each fix. + +`calleeNames` is the union and stays entry-point wide because the three questions that read it are +file-wide: what the file touches, how much it does, whether it records anything. There is +deliberately no entry-point-wide `checkedCallees`, so no check can reach for a union that would say +a loader's reading of `getUser` speaks for the action beside it. One push site in `scanFile` fills +the whole-entry list and each owning export's list, so the two cannot drift (`every callee name is +attributed to an export that exists`, pinned on fixtures and again over the real tree). + +Two fields exist because the bare callee name is not enough. `calleeName` keeps only the last +segment, so `prisma.organization.findFirst` arrives as `findFirst` with the receiver gone; +`calleeTexts` keeps the whole dotted path, which is how the per-export triviality rule knows a +three-statement body reaches the datastore. `auth-boundary` matches the bare name on purpose, so a +guard call cannot be hidden by its receiver. + +### Catch evidence, per clause + +`CatchEvidence` is one record per catch clause rather than a set of booleans per entry point, +because 39 routes have more than one catch and 17 mix a narrow parse guard with a broad handler, and +an aggregate lets the well-behaved clause speak for the swallow beside it. + +- `rethrows`: throwing is the clause's only way out, i.e. a throw is reached on the clause's + guaranteed path AND the clause contains no live `return` anywhere. +- `throws`: a throw is reached on that path, whether or not it is the only way out. Kept separately + so a verdict can say what is true of a clause that both throws and returns; the detail line "takes + one way out regardless of what was thrown" is true only of a clause that never throws. +- `branches`: the clause picks what to do from what it caught. An `if` or `switch` whose condition + references the caught binding and at least one of whose arms returns or throws, or a conditional + that is the whole value of a `return`/`throw`. `if (retries > 0)` does not count, + `if (e instanceof Error) { }` does not count, a bindingless `catch { }` cannot count at all, and an + `instanceof` used only to word a message does not count either, because every error still leaves + by the same path. +- `guardsParse`: the guarded region parses something. `JSON.parse`, `request.json()`, a zod + `parse`/`safeParse`, a `decode`, or a `new URL`/`URLSearchParams`/`RegExp`. Those three + constructors are read as `ts.isNewExpression` because a `new` expression is not a call and the + call-callee scan never sees them. Crediting any constructor would let `new BranchesPresenter()` + excuse a catch guarding ordinary work, true of 77 try blocks in the tree. +- `guardCanRaise`: the region does anything that could reach the clause. False means `try { 0; }` and + little else, because any call counts, including one that cannot throw. +- `guardMayRaise`: the containment twin, false only when the region provably cannot raise. Everything + `canRaise`'s whitelist misses stays true here, so `guardCanRaise` implies `guardMayRaise`. +- `awaitsOnlyParse`: everything the region waits for is one of those parses, or a read of the body it + parses. +- `tryStatementCount`: statements in the guarded block, counted as `statementCount` counts them. + +`canRaise` is a whitelist and it misses real raising code, which is the safe direction but does +matter: a destructuring declaration (`const { a } = undefined` throws), a temporal-dead-zone read, a +coercion that raises, and a `delete` on a frozen object all read as unable to raise. So the +refused-swallow arm of `error-classification` reads the route's own deciding catches through +`guardMayRaise` and never through `guardCanRaise`, ordering it off can-raise being what accuses a +route that owns a real classifying catch of owning none. + +"Does this route catch anything" is `catches.length`, never `hasTryCatch`. A `try`/`finally` with no +catch leaves `hasTryCatch` true and `catches` empty, and nothing is swallowed there: the error +propagates once the cleanup has run. + +## The dead-code defence + +Both of the catch-clause answers are read off the clause's guaranteed path. The governing rule: the +walk may enter a construct exactly where the entered statements are guaranteed to execute whenever +the clause body runs, so no credit can come from code a semantics-preserving edit could have added +dead. + +Entered on those terms: a bare nested block, a `do` body, the tryBlock of a `try` that has no catch +clause and whose finally contains no jump out of itself, the sole clause of a single-default +`switch`, the then-arm of an `if` whose condition is exactly the literal `true` keyword, and both +arms of an `if`/`else` with per-arm states merged by intersection. + +Not entered, deliberately: a bare `if` without an else, loops other than `do`, labelled statements, +function-like nodes, nested catch clauses, finally blocks, and the tryBlock of a `try` that has a +catch clause, where a throw is intercepted by the nested catch rather than escaping. + +Do not replace the rule with a list of statically-false shapes to refuse. Asking for the throw to be +unconditional refuses eleven spellings, from `if (false)` and `for (;false;)` through +`switch (1) { case 2: }` and `for (const k in {})` to `if (1 === 2)`, each worth 50 points a route, +without naming any of them. `dead-*` in the corpus is the tree-scale proof, one entry per shape. + +`rethrows` asks for one thing more: no `return` anywhere in the clause, or a `throw error;` written +after a statement that already exited reads as a rethrow, in seven spellings (`dead-throw-after-*`). +The cost is real, since `catch (e) { if (transient) throw e; return null; }` no longer reads as a +rethrow and so fails rather than sitting out. That is the direction to be wrong in, the reverse +handing out points. + +### Two folds, pointing opposite ways + +There are two literal folds in `scan.ts` and unifying them would be a bug. + +`containsLiveWhere` folds any literal guard `literalTruth` can decide, and it is strictly subtractive +against a plain containment read: wherever the truth cannot be decided, every hit containment would +have found is still found. That is what lets its two callers read it for opposite purposes. In +`catchClauseEvidence`'s `exited` flag a hit BLINDS the walk to whatever follows, and containment +blinds it on a provably dead statement, so prepending one to a deciding clause turns its pass into a +swallow verdict on 78 routes. In `selectsADistinctPath` a hit GRANTS a branch, and containment grants +one for an arm whose only exit is dead (`dead-armed-instanceof-if`, 80 routes and the tree from 19 to +27). Subtracting dead hits only ever un-blinds in the first case and only ever withholds in the +second. + +The walk's own entry tickets fold nothing but the literal `true` keyword. `!!1`, `1` and `!false` are +deliberately not entry tickets, because entry GRANTS credit and a wrong grant pays, where +`literalTruth`'s wider folding only ever withholds blindness. Do not unify the two. + +`literalTruth` treats `&&`, `||`, an identifier, a call, a bigint and a template with substitutions +as undecidable on purpose, so a live guard can never be read as dead. The cost is +`dead-conjunction-instanceof-if`, a corpus expected failure: `e instanceof Error && false` both +references the caught binding and can never be true, and no fold in the file can see it. Widening the +fold is a different rule with its own measurement. + +The `exited` flag is raised at the END of each statement, after that statement's own branch check. A +deciding statement contains an exit by definition, so raising it first makes every such statement +refuse itself, measured at 78 routes losing their pass. The ordering leaves the real-tree report and +all 240 clauses' evidence byte-identical. + +### A finally that cancels the try + +A finally block that completes abruptly supersedes the try's and the catch's completion, so an exit +written in either never leaves the statement. Two places read that, in opposite directions. + +`catchClauseEvidence` refuses to enter a catchless try whose finally holds a jump out of itself, +because entry grants rethrow credit and the throw would never escape the clause. The refusal is a +containment read, over-approximate on purpose: a jump that only may run still refuses (`refuses the +tryBlock when the finally only may break`, and `dead-throw-in-cancelled-try` at tree scale, worth 80 +routes and 8 global points). + +`containsLiveWhere` then folds the same statement to its finally's own statements, so a refused +statement cannot blind the walk to the real classification below it (`keeps the classification after +a finally-break no-op`). A finally holding a `return` is covered by the explicit `containsLiveReturn` +read instead, because `try { throw e; } finally { return null; }` genuinely swallows. + +### The residual both branch tests share + +Two arms that produce the same outcome by different spellings still read as a real decision. +`if (e instanceof Error) { return json(x); } return Response.json(x);` counts and decides nothing, as +does the `if` with no `else` whose arm returns what the statement after it returns. Telling those +apart needs the produced values compared for meaning rather than for text. The textual comparison is +the cheapest thing that catches the copy-paste form, which is the one a mutation produces. + +## Parse guards, and the narrow-try count + +A catch clause counts as a parse guard, rather than as the route's error handling, when the try block +parses, waits for nothing except that parse, and is short. All three conditions are load bearing. + +`awaitsOnlyParse` is what a statement count cannot express. +`try { const body = await request.json(); return await handleEverything(body); } catch { 500 }` is +two statements, one of them a parse, and the whole handler inside it: the count reads it as narrow +and it is the `otel.v1.logs.ts` swallow written compactly. Asking what the block waits for separates +them, and unlike the count it does not care how the statements are punctuated or how deeply the work +is nested. Awaiting is the signal rather than calling, because the calls that prepare a parse's input +are ordinary synchronous string work (`matchPattern.slice(4)` before a `new RegExp`), and requiring +every CALL to be a parse refuses four of the tree's clearest guards. Two residuals follow: a block +that does its non-parse work synchronously still reads as a guard, and `guardedWork` looks for a +`ts.AwaitExpression`, which `for await (...)` and `await using` are not. + +`NARROW_TRY_STATEMENTS` is 2, so the guarded operation can bind its result +(`const stripped = ...; new RegExp(stripped);`) and a third statement means the try has started to +cover the handler. It is an absolute count and not a ratio against the enclosing body, because a +ratio is diluted by anything else in the same body: padding the action with unrelated statements +after the try relabels the same broad swallow as a narrow guard, moving the denominator without +touching the clause (`inert-statements-after-try`). + +The count is paddable, which is why it is one condition of three rather than the load-bearing one. +`countStatement` counts declarators and comma operands rather than semicolons, so +`const a = f(), b = g(), c = h();` is three and `a(), b(), c()` is three (`merge-declarations`, +`merge-comma-expressions`), and a third way nobody has written down would work. + +Two rejected alternatives, both measured. Requiring the clause to answer with a 4xx credits, on its +own, the 11 widest swallows in the tree, including `admin.api.v1.workers.ts`, whose 28-statement try +answers every failure with a 400 carrying the internal error message; added on top of the rest it +costs three narrow guards their pass for computing a fallback value rather than answering a request. +And a narrow guard is not a way to qualify as classification on its own: the eleven entry points that +limb would clear hold six real swallows, including a silent run cancellation and two credential paths +reporting a database failure to the browser as a 400 with an internal message in it. + +## The iteration-callback boundary + +`items.map((item) => { try {...} })` is a fresh catch per element, so its clause is not the route's +own error handling. `trace(async () => {...})`, `mutateWithFallback({ pgMutation: ... })` and +`new ReadableStream({ start: ... })` all invoke their callback exactly once, so theirs is. The +structural signal is the method name, a list of eight, because nothing in a syntactic scan can tell +`users.map` from `Result.map`. + +Three rules keep the cheap direction from paying. A refused catch is kept WITH its evidence, built by +the same machinery as an own catch, and judged on what it does rather than on where it sits. A +refused swallow fails the route whenever nothing the route owns decides, and that arm is deliberately +not conditioned on the route owning no catches, so an own inert rethrow catch cannot lift a refused +swallow out of the verdict. A route whose only catches are refused and none of them swallows sits out +at not-applicable and never passes, which keeps a prepended dead deciding `.map` from minting a pass +on the 261 catchless routes (`dead-deciding-map`). + +That is what makes the name list survivable. Relocating a swallow behind the boundary still fails +(`still fails a swallow wrapped in a non-array receiver's .map(...)`), and relocating a decision +earns at most the route's exit from the denominator. A receiver that is an array literal of one +element or none is refused outright, since it cannot iterate. + +The other direction costs precision. A per-item callback under a callee the list does not know, +`pMap(items, cb)` or `Array.prototype.map.call(items, cb)`, is attributed to the route, so a +per-element catch that decides can carry it to a pass. No mutation of a real route produces it, since +a route has to already be iterating for the shape to exist, which makes it a wrong verdict waiting +for a route rather than a laundering path, and is why the list is worth extending when a new +iteration helper shows up. + +## What auth-scope reads as scoping + +Three conditions, all load bearing. With only the middle one, prepending +`const __unused = { anything: user.id };` to every body raises `settings.sso` and `settings.team`, +the only two findings `auth-scope` has ever produced and both confirmed cross-org exposures +(`dead-caller-scope-object`, `dead-caller-scope-userid`). + +- The value has to be the caller's own id, anchored at both ends: the root is one of the auth + bindings a builder hands the handler and the last segment is an identity field, so `user.name` is + not a scope and neither is `run.userId`, which is a resource's owner. +- The property NAME has to be an identity field. Of the ten names that take a caller-id value in the + route tree, `sub`, `value` and `consumerId` are the three that are not, and `anything: user.id` is + what a mutation writes. +- The object has to be handed, through any depth of nesting, to a call that could narrow a read with + it. Arrays count, so `{ OR: [{ userId }] }` still reaches its call. + +The third condition is a denylist of sinks rather than an allowlist of query callees, and that is a +measurement: 72 distinct callees are handed a caller id across the route tree, from +`prisma.project.findFirst` through `presenter.call` to bare `regenerateApiKey`, and no name pattern +separates those from `sendToPlain`. An allowlist would accuse whichever route named its helper next, +which is the failure this check cannot afford. The sinks refused are the log line and the response +body, both of which take the very `{ userId: user.id }` object a query filter takes: loggers account +for 13 of the caller-id sites and the two response serializers for 2 more. The shape is in the tree +already, in `engine.v1.dev.runs...attempts.start`, which logs `{ environmentId: ... }` beside the +`runStore.findRun` that earns its credit honestly (`log-caller-scope-userid`). + +A callee with no readable name of its own is credited, because refusing it would ACCUSE the route and +under-crediting beats accusing a route that is fine. `String({ userId: user.id })` therefore reads as +scoping, the same way `try { String(0); }` reads as error handling and for the same reason. + +`authorization: undefined`, `null` and `false` are read as not declared, because +`apiBuilder.server.ts` gates every option behind `if (option)` and declaring one is what the check +credits. + +An `ability.can(...)` call in the handler is deliberately not a third way to be scoped. +`apps/webapp/CLAUDE.md` says why: the OSS fallback ability is permissive +(`internal-packages/rbac/src/fallback.ts` returns `permissiveAbility` for a PAT and +`buildFallbackAbility(user.admin)` for a session, neither of which reads org membership), so an +ability check enforces the role while the membership-scoped query is the tenant floor. + +## Sensitivity, and the names the tool matches on + +Two rules hold the vocabulary honest, and tests rather than convention enforce both. + +Calling a guard can never be what makes a route sensitive, because a mitigation cannot be the hazard +and the reading is circular: a guard name on the symbol list marks every route that calls it +sensitive, and `auth-boundary` then passes all of them for calling it (`does not treat calling the +admin guard as what makes a route sensitive`). + +Every name and every segment has to exist. `src/webappSymbols.test.ts` resolves every sensitive +symbol, every path segment and every entry in `auth-boundary`'s guard list against `apps/webapp/app` +and the two packages the webapp authenticates through, and fails if one stops resolving, half the +symbol list having named nothing at all without it. The one exception is `ANTICIPATED_SEGMENTS`, +three words that name no route yet and are held to naming none. + +That test is also why `auth-boundary`'s guard list is names rather than the patterns it replaced. +`/^(require|authenticate)/` cleared a sensitive route on any callee beginning `require`, so +`requireSsoEntitlement`, a plan check, cleared the org SSO settings page; `/Authenticated/` passed +`resolveAuthenticatedEnv` on ten routes, a `findFirst` by environment id that authenticates nothing. +Both are corpus entries (`fake-require-guard`, `fake-authenticated-lookup`): under the patterns they +took the tree from 18 to 19 and raised five routes, and under the accept-list they raise nothing. + +## Triviality, in detail + +Trivial means a body of three statements or fewer, three or fewer calls, no try/catch, no builder +wrapping it, and nothing in the calls or the hint text naming a datastore or a service. + +Both limits are 3 because both real shapes need three: parse the params, build a path, redirect, or +an environment guard and two returns. A fourth call admits +`_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls; a fourth statement +admits the routes that authenticate and hand off to a presenter; a fifth admits an admin route that +calls a service and hand-rolls its own error responses. + +The rule is deliberately reluctant, because a route wrongly called trivial is exempted and never +shows up in the report again. So `calleeNames` descends into the callee of every call at any depth +while `statementCount` stops at a nested function, which means the call count still catches bodies +the statement count reads as short. A builder means the config passed to it (`findResource`, +`authorization`) is work the scanner never walks, so the visible body is not the whole route. And a +try/catch is exactly what `error-classification` reads, so a body with one has an error path worth +reporting on however short it is. + +One rule, two views, so the entry-point-wide answer and a single export's answer cannot drift. The +per-export view exists because a file-wide triviality rule accuses the wrong half of a file: +`auth.github.ts` is `export let loader = () => redirect("/login")` beside an action that calls +`authenticator.authenticate`, so a file-wide rule calls it non-trivial for the ACTION and +`auth-boundary` accuses a one-line redirect stub of missing an auth guard. `checks/index.test.ts` +pins both directions (`reports not-applicable for a redirect-stub loader beside a guarded action`, +`fails an export whose own body does real work unguarded`). + +The two views differ in one term, measured both ways. The entry-point-wide view matches the +side-effect hints against the whole file, so an import of `prisma` disqualifies it even when the +query sits somewhere the scanner does not walk. The per-export view matches that export's own callee +PATHS instead, because matching the file's text is defeatable: `log-caller-scope-userid` prepends a +`logger.error(...)` to every body, which with a file-wide term puts the word `logger` in +`auth.github.ts` and turns its untouched redirect loader from excused into accused. Emptying the term +is not the answer either, since `calleeNames` keeps only a call's last segment, so +`prisma.orgMember.findMany` reads as `findMany` and a three-statement body that queries the datastore +matches no hint at all, which takes five `auth-boundary` fixtures from `fail` to `not-applicable`. +The callee paths are body-scoped and name the receiver, which is what both readings needed. + +## Reading the directive out of the source + +The suppression directive is read from a real parsed `ts.SourceFile`, and then filtered against the +spans the parser has already claimed as content. Both halves are needed. + +Parsing rather than scanning is what stops a template literal with a substitution being rescanned as +ordinary code after `${x}`, and what makes JSX text a node at all. Filtering by span is what stops +the two comment-range lexers reading the start of such a node as a comment anyway, which they do +because `getLeadingCommentRanges` and `getTrailingCommentRanges` are raw lexers over source text from +an offset and consult no parse tree. A JSX text node that BEGINS with `//` or `/*` is the shape that +reached the real tree, in `resources.branches.create.tsx`'s `//`. + +The filter is on the range's start offset falling inside a claimed span, not on the gap between a +token's full start and its start, a gap filter losing a same-line trailing comment and a comment +inside a JSX expression container. Both lexers are called at every token boundary, because which one +returns a given comment depends on whether it shares a line with the token before it. Leaf tokens are +walked through `.getChildren()` rather than `ts.forEachChild`, which skips bare punctuation and +keyword tokens, and a comment can sit directly before one of those as the last line inside a block. + +The mutation corpus cannot cover any of this: a suppression can only lower an entry's score, because +`scoreEntry` caps it at the pre-suppression ratio, so suppression bugs are invisible to a harness +watching for the score rising. They need ordinary unit tests. `jsx text is content, not a comment` is +the four cases that fail without the JSX filter, and the positive control beside it, `still reads a +directive from a comment in a JSX expression container`, is what stops the filter being widened until +it eats real comments. + +## The mutation harness + +Every mutation is a TEXT rewrite driven by AST positions, never a reprint. A reprint would change +formatting everywhere and make a failure impossible to read; splicing at node positions leaves the +rest of the file byte-identical, so a corpus failure can be diffed down to the one construct that +moved. Overlapping edits are dropped inner-first, which is what "the outer rewrite won" means. + +Nothing is ever executed. Semantics-preserving means preserving the observable behaviour of the route +as written, which is what the scanner claims to measure, not that the mutated tree compiles against +its real types. + +**Splices go at the HEAD of a catch clause, not the tail.** 234 of the tree's 260 clauses end in a +`return` or a `throw`, so an appended shape is dead by ordering before the rule under test ever looks +at it. At the head every clause is reachable, and the shapes spliced this way are dead wherever they +sit, so moving them does not make the rewrite any less preserving. + +**The harness's population is asserted against the scanner's.** A mutation reaching fewer routes +lowers the score rather than raising it, so no invariant here can notice the harness missing an +export form. `wraps a body in every non-delegating entry point the scanner finds` pins it, with +`admin.tsx` the one named exclusion, its handler being a concise arrow with no block for a block +wrapper to wrap. The same failure mode is why the registry assertion and the additive-class +assertion are ungated while everything else in the file needs `OBS_MAP_MUTATION_CORPUS=1`: omitting a +check from a sweep leaves its failures in place, which lowers the score, so the corpus cannot catch +its own omission by failing. + +**The corpus deliberately disagrees with the scanner about where a handler sits.** `mutations.ts` +keeps its own copy of the builder handler shapes rather than importing them, because sharing the +scanner's notion would let a bug in that notion hide a laundering shape. Which exports exist is not a +judgement, though, which is the distinction above. + +**The anti-vacuity threshold is on sites, not only files,** a file count saying a rewrite touched a +file rather than that it reached anything inside it. Verdict movement cannot be the guard instead: +the IDEAL defended shape is one the scanner is blind to, so `dead-if-false` and the ten entries +beside it are defended precisely because the tree comes out identical, and requiring movement would +fail exactly the entries that work best. + +**A `lowers` exemption is a per-entry field with a reason, not a skip list.** An exempted entry must +still be falling, or the exemption is stale, and its falls must have exactly the measured residual +shape it was granted for: `error-classification` moving pass to not-applicable, every other check +unchanged, nothing moving to fail. Exactly two entries carry one, both non-array-receiver iteration +wrappers. + +**A `KNOWN_GAPS` entry runs as `it.fails`,** so closing the hole later turns the file red until the +entry is moved out deliberately. Two are open, both described above: +`dead-classifying-try-with-call` and `dead-conjunction-instanceof-if`. + +## Reporting + +Every denominator reads `rawChecks`, pre-suppression; `checks` is the display view. Suppressing the +one `request-context` or `audit-trail` finding on an entry must not shrink the gap denominators and +raise the printed percentage, on the same screen as a claim that suppression cannot do that. An +entry's score is capped by what it would have scored unsuppressed. + +`score` is 100 for an entry no scored check applied to, a placeholder rather than a verdict. +Rendering it as a figure turns a route refactored down to a trivial body into a 67-point improvement, +and a trivial route gaining real work into the pull request's worst regression, so the PR comment's +cell says "not measured" instead. `globalWithout` recomputes from `rawChecks` minus the suppression +cap, because lowering both figures by the same rule would leave the difference between them saying +something about suppressions rather than about the check. + +`hasDelta` has to be true whenever `renderPrComment` would say something different, because anything +it misses is a change the pull request silently does not report. It covers the global, the per-entry +score, measured state and suppression set, an entry added or removed, a check failing at head that +did not at base, the parse failure count, the unknown suppression warnings, the audit and context +gaps, `delegating` and `checkContributions`. The per-entry suppression set and the two gaps are the +terms that run the dangerous way, since suppressing an already-failing check moves no score, no +measured flag and no new failure, so without them a pull request whose entire purpose was silencing +findings posts nothing. What is defended is that the union is complete, not that each term is load +bearing: three terms are shadowed by another today and kept because which shadows which depends on +the shape of the change. `MapReport.suppressions` is deliberately left out, its totals being summed +from the very per-entry arrays the loop compares. + +Two sections of the PR comment grow with the tree and both are capped, because GitHub's comment limit +is 65,536 characters and a 422 loses the whole comment to the section warning about a typo: a +mistyped directive applied tree wide renders 87,938 characters. The delegated list is capped at +fifteen rather than ten because a file name is one comma-separated item rather than a line naming +every known check, and the longest route file name in the tree is 130 characters. + +The `AUDIT` line has one shape for every count and no branch on the count, the count being correct +and the branch being where a false sentence gets written. A suppression whose id names no check is +carried through to both renderers rather than dropped, because dropping it silently makes a typo look +like an acknowledgement. + +## Tests, timeouts and CI + +**The docstring checker.** `docstringReferences.test.ts` enforces that every test a docstring in +`src/` names exists, the rule having been asked for six times in prose and broken six times. It reads +every backticked kebab-case token, every backticked glob against the corpus ids by prefix, and every +backticked prose phrase of five words or more with no code punctuation. It does not read a reference +written without backticks, a title of fewer than five words, a comment with no node after it (leading +ranges only, so a comment on the last line of a block is never scanned), or a `.test.ts` file or +`mutations.ts`, both exempted by name. Three negative controls run the same predicates over an +invented docstring, so the guarantee does not rest on `src/` happening to contain a bad reference. + +**`TREE_SCAN_TIMEOUT` is a hang detector, not a budget.** Neither real-tree test asserts anything +about how long a scan takes, so a number tight enough to be a performance budget is only a way to +fail on a busy runner. Measured on an 8-core box: 6.3s for the scan and 10.8s for the sweep +uncontended, rising to 34.0s and 39.7s under twelve concurrent copies, with one run dying on a 30s +timeout. `unit-tests-internal.yml` runs twelve concurrent shard processes on one runner, so that +contention is what CI does. 120s is 3x the worst contended run measured; 60s is not enough. + +**Parse failures come from a `ts.Program`,** not from the diagnostics array the parser hangs on the +source file, which is internal and which the compiler is free to rename. An undetected parse failure +shrinks the denominator and inflates the score, so it must not be the kind of thing a compiler +upgrade can switch off silently. The host hands the program the source file we already have, so +nothing is parsed twice; the cost is the program machinery, about 850ms to about 1450ms on a full +scan of the real tree. + +**The turbo task is uncacheable,** because its real inputs are mostly not its own files: they are +`apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and the workflow files, so +turbo replays a pass recorded before a route changed and caches a failure as a success. `turbo.json` +carries the reasoning and the rejected `inputs` alternative beside the config. + +Three roads reach the suite and all three are asserted. `pr_checks.yml` calls +`unit-tests-observability-map.yml` behind an `obsmap` paths filter and lists it in the `all-checks` +aggregate, without which the job gates nothing, `all-checks` needing an explicit list of jobs and +being unable to see another workflow. The filter watches all of `apps/webapp/app` plus the report +workflow, because the suite reads more than the routes folder and a rename outside it matched only +`webapp`, ran no job, and broke the build for whoever pushed next. It deliberately does NOT name this +package or the two non-webapp roots, since `internal` already matches `internal-packages/**` and +`packages/**` and `unit-tests-internal.yml` runs `turbo run test --filter "@internal/*"`, so naming +them here runs the suite twice on every pull request touching the package. Widening `internal` to the +route paths instead runs all eighteen internal packages with postgres, clickhouse, redis and electric +to protect one test. + +The report workflow's own text is asserted from `integration.test.ts`, because it is the one thing +the docstring checker cannot reach. What those checks pin: the comment lookup sits in the cheap +`changes` job so the report job's gate can read it, the report job does not start unless the lookup +finished cleanly, and the id both steps use is one job output, so no two steps can disagree about +what a missing id means and turn a transient lookup failure into a duplicate comment or a false +all-clear. Both scan steps write their own file through `--out` rather than capturing stdout, +because +`pnpm --filter` takes its recursive path and some versions announce +`Scope: N of M workspace projects` on it, and a single line of that in head.json fails the renderer's +`JSON.parse` and degrades the workflow to the stale-report comment permanently. What is asserted is +the shape that cannot have the bug rather than the pinned 10.33.2 that happens not to. + +The render step writes through `--out` for the same reason, and its failure mode is the worse of the +two. `renderPrComment` puts the marker on the first line and the lookup finds the comment with +`startswith` on it, so a line printed ahead of the document does not degrade the comment, it hides it: +the next push finds no id and posts a second comment, and no later run can reconcile either. The scan +steps degrade to a stale report, which at least stays one comment. + +The corpus runs on the package's own paths and on a schedule rather than on every route pull request, +because it measures the tool's resistance to laundering, which only an edit to the tool can weaken, +and it costs four and a half minutes. The nightly covers tree drift late rather than not at all. diff --git a/internal-packages/observability-map/README.md b/internal-packages/observability-map/README.md new file mode 100644 index 000000000..fe337cfc1 --- /dev/null +++ b/internal-packages/observability-map/README.md @@ -0,0 +1,336 @@ +# @internal/observability-map + +Scores every webapp entry point on whether it could explain itself during an incident, and prints +the ones worth fixing. An entry point is a Remix `loader` or `action` under +`apps/webapp/app/routes`, 427 of them today. + +The score is 19 out of 100. It is low because the webapp does not attach tenant identity to its +failures: **11 of the 412 measured entry points name an environment, project, organization or user +on a failure path.** Everything else, when it breaks at 3am, gives you the route and the request id +and nothing about whose request it was. + +Every figure here comes from a run against the tree as it stands. How the scanner reaches a verdict, +and what it refuses to decide, is in [INTERNALS.md](./INTERNALS.md). + +## Running it + +```bash +pnpm --filter @internal/observability-map run map # the whole tree +pnpm --filter @internal/observability-map run map --json # same, as JSON on stdout +pnpm --filter @internal/observability-map run map /api/v1/token # one route, with its check results +``` + +The whole-tree run also writes `observability-map.json` at the repo root, which `--no-write` +suppresses. Single-route mode takes either the route path the report prints (`/api/v1/token`) or the +file name (`api.v1.token.ts`). An exact match wins over the routes it is a prefix of, and an +ambiguous prefix warns and names the alternatives rather than picking one. + +## CI + +A pull request touching `apps/webapp/app/routes` or this package gets a sticky comment scanning head +against the tip of the base branch, with the score, what changed, and the current fix list. Every +comment names the head commit it was rendered for, as a link to the compare range, because the +comment is edited in place across pushes and otherwise says nothing about which push it reflects. It +is report-only: nothing here fails the build or blocks a merge. See +`.github/workflows/observability-map.yml`. + +The workflow runs on every pull request and applies the path list as a gate inside the job rather +than as a `paths:` filter on the trigger. GitHub evaluates one of those per workflow, so a pull +request whose diff stops matching never starts the workflow at all, and the comment an earlier push +left then stands for ever showing findings that are no longer in the diff. The case that matters is +a pull request touching a route and other files whose author reverts the route change and keeps the +rest, which still has a diff and still does not match. So a pull request with a comment and nothing +left to compare gets the comment reconciled to its resolved state without scanning anything, and one +with neither pays for a single cheap job that reads the paths and looks for a comment. + +The base is `github.event.pull_request.base.sha` rather than a merge base, which reviewers have +reported as a bug twice. `actions/checkout` on a `pull_request` event checks out GitHub's test merge +commit, whose two parents are `base.sha` and the PR head, so the head tree being scanned already +contains everything on the base branch up to `base.sha`. Diffing that against `base.sha` isolates +the pull request's own work. A real merge base would leave the intervening base-branch commits in +the head tree and out of the base tree, and attribute all of them to the pull request. + +## What the score means + +It is the mean score of the 412 entry points that had at least one applicable check, where an +entry's score is the share of its applicable checks that passed. + +One property of that definition will mislead you otherwise. **Changing which routes a check applies +to moves the score without anything in the webapp changing.** Widening the sensitive cohort from 26 +routes to 67 gave `auth-boundary` 39 more routes to look at, 36 of which already passed it, and the +global went from 15 to 19 without a line of `apps/webapp` changing. Narrowing a check, or a refactor +that takes routes out of the denominator, runs the same way in reverse. A movement is evidence about +the codebase only once you have checked the CHECKS block below for an applicability change. Compare +fix lists, not scores. + +The number is deliberately unflattering, and one platform change would move most of it. Nothing +central attaches a tenant: `logger` pushes `{ requestId, path, host, method }` onto every line +through AsyncLocalStorage and forwards errors to Sentry, and the route builders log +`logBoundaryError(message, error, url)`. If the auth path pushed `environmentId` through +`trace(...)`, several hundred entry points would flip at once, and `request-context` would want +rethinking rather than celebrating. + +### What the number cannot tell you + +`request-context` checks that a failure-path log names a tenant field. It does not check that the +value is real. Adding a synthetic `environmentId: "obs-map"` field to the first object argument of +all 139 in-catch log calls, with no other change, takes the global from 19 to 29 and the CONTEXT +figure from 11 to 98. + +That is the tool verifying presence, not meaning, and it is not a bug to fix. Every check reads +syntax: a field name, a call, a binding reference. None can tell a genuine tenant id from a +hardcoded string with the right key. A reviewer owns whether the value behind the field is real, the +same way a Lighthouse accessibility score checks that an `alt` attribute exists and not that its +text describes the image. The number tells you where to look. It does not tell you what you will +find there. + +### What stops it being gamed + +`src/mutationCorpus.test.ts` applies 44 semantics-preserving or handling-deleting rewrites to the +whole route tree in a temp copy and asserts three things for each: the published global does not +rise, the mean over the routes measured in both runs does not rise, and for a semantics-preserving +rewrite no individual route's score rises or drops out of the measured set. Every laundering shape a +reviewer has found is an entry in `src/mutations.ts`, and each entry says which kind it is. + +Two entries are the ones the design turns on. Deleting every catch clause in the tree drops the +score from 19 to 8, so the metric does not pay you for removing error handling. Wrapping every body +in `try { ... } catch (e) { throw e }` leaves the global at 19 and raises no route, so it does not +pay you for adding error handling that does nothing either. + +One hole is open and the corpus says so. A catch over `try { 0; }` is refused, but `canRaise` +accepts any call, so `try { String(0); }` reads as real error handling: it takes the tree from 19 to +44 and raises 224 routes. Telling an inert call from one that can throw needs types the scanner does +not have. That entry, `dead-classifying-try-with-call`, runs as an expected failure with the +residual written out beside it, so the claim is "43 rewrites are defended and here is the one that +is not", never "unpaddable". + +The corpus takes about four and a half minutes, so it is gated behind `OBS_MAP_MUTATION_CORPUS=1` +and runs as its own CI job rather than in `pnpm test`. Run it if you change this package: + +```bash +OBS_MAP_MUTATION_CORPUS=1 pnpm --filter @internal/observability-map exec vitest run \ + src/mutationCorpus.test.ts --disable-console-intercept +``` + +## The five checks + +- **error-classification**: does every catch clause decide what it caught, by branching on the error + or by guarding a parse it can answer for. A clause that only rethrows decides nothing and is read + as though there were no catch, so it neither passes nor fails. +- **auth-boundary**: does a route handling credentials, access control, sessions, billing or + impersonation check who is asking. +- **auth-scope**: does a sensitive builder-wrapped route also narrow itself to the caller, in every + export, by declaring `authorization` or by filtering on the caller's own id. All nine route + builders authenticate the request, but their `authorization` option is optional and + `apiBuilder.server.ts` runs the RBAC gate inside `if (authorization)`, so a route can be + authenticated and scoped to nobody. That is the cross-org IDOR class `apps/webapp/CLAUDE.md` + names. It applies to 19 routes and 17 pass; both failures resolve their target organization from + the URL slug with no membership filter and put nothing but an ability check in front of it + (`_app.orgs.$organizationSlug.settings.sso/route.tsx` in its loader, + `_app.orgs.$organizationSlug.settings.team/route.tsx` in its action). The fix in each is + `members: { some: { userId } }` on the lookup. +- **request-context**: when this entry point's failure is reported, is the tenant named. +- **audit-trail**: does a sensitive mutation leave a record of who did it. Three routes do, all of + them impersonation paths reaching `prisma.impersonationAuditLog.create` in + `models/admin.server.ts`; the other 46 do not. + +`audit-trail` is excluded from the score. The other four are in it. + +`auth-boundary`, `auth-scope` and `audit-trail` only look at routes `src/sensitivity.ts` calls +sensitive, and that cohort is the fix list's primary sort key, so what goes in it decides what a +reader sees first. 67 routes are in it today: credentials and tokens, envvars, billing and the two +billing settings the bare `billing` segment does not match, impersonation, membership and invites +and roles and the team page, the login surface, API keys, and org or project deletion. Calling a +guard never makes a route sensitive, and `src/webappSymbols.test.ts` fails if a symbol or path +segment in the vocabulary stops resolving in the webapp. + +## What the score is made of + +The check list describes a composite the number mostly is not, so the report discloses the shape +instead of hiding it behind a weight. Today: + +```text +CHECKS + error-classification 166 applicable, 94 pass, 0 sole, global without it 10 + auth-boundary 62 applicable, 59 pass, 0 sole, global without it 15 + auth-scope 19 applicable, 17 pass, 0 sole, global without it 18 + request-context 412 applicable, 11 pass, 223 sole, global without it 65 + audit-trail 49 applicable, 3 pass, 0 sole, not in the score +``` + +`sole` says the most: 223 of the 412 measured entry points have exactly one applicable scored check, +so their score is 0 or 100 on a single boolean. Read the family bars with that in mind. They do not +compare families on observability in general; they mostly compare them on whether someone wrote a +tenant field into a catch log. + +Weighting was considered and rejected, because a coefficient nobody can explain invites argument +about the number instead of about the finding. The block above is in the terminal report and in the +JSON as `checkContributions`. + +## Two findings are headlines, not list entries + +`audit-trail` fails 46 of 49 and `request-context` fails 401 of 412. Printing either one per route +would bury the route-specific findings under the same sentence repeated hundreds of times, so both +are reported as a figure: the `AUDIT` and `CONTEXT` lines. 328 entry points fail nothing except +`request-context` and appear only in that figure, which leaves 76 in the fix list. An entry that +fails `request-context` *and* another scored check keeps both findings and stays in the list, so +`/account/tokens` still shows the whole picture. `audit-trail` does not count as "another" for this +purpose, being already a headline, so a route failing only `request-context` and `audit-trail` +collapses too (28 do today, all of them sensitive). + +42 of those 328 are sensitive, so the `CONTEXT` line says how many. Read them out of +`observability-map.json`, where every entry keeps its full check results, rather than assuming the +fix list is the whole story. + +`request-context` is still scored, unlike `audit-trail`. The gap it measures is real and the score +is meant to show it. Only the presentation collapses. + +## When a check declines to judge + +The rule every applicability decision follows: **would this evidence necessarily be visible in the +body if it existed?** + +A log call inside a catch would be, because the catch is right there in the body being read, so its +absence is evidence of absence and `request-context` fails the route. A guard on work that happens +inside an imported helper would not be, because neither the work nor the guard is in the body, so +`auth-boundary` reports not-applicable with a detail saying it could not verify rather than accusing +the route of being unguarded. `resources.impersonation.ts` is the worked example: it calls +`clearImpersonation`, which authenticates and writes an audit row in `app/models/admin.server.ts`, a +file this tool never opens. Five of the 67 sensitive routes sit out for this reason. + +The failure mode the rule exists to prevent is a fix list whose top three entries are all wrong, +which is what a check asserting more than its evidence supports produces. + +## Not applicable is not a pass + +An entry with no applicable scored check is `measured: false`, and it is left out of every mean the +report computes (`src/score.test.ts`: `excludes an unmeasured entry point from the global mean`, +`excludes an unmeasured entry point from its family mean too`). Its `score` field reads 100, a +placeholder for "nothing was measured here" that nothing averages, because the alternative of +letting unmeasured entries into the mean at 100 would let the tool look better the less it +understood. The header prints every count (`412 measured, 15 unmeasured`) so the denominator is +never hidden, and a family with nothing measured renders as `not measured` rather than as a full +green bar. + +15 of the 427 routes are unmeasured because `isTrivial` rules them out before any check runs. +Trivial means a body of three statements or fewer, three or fewer calls, no try/catch, no builder +wrapping it, and nothing in the calls or the source naming a datastore or a service (`prisma`, +`logger`, `fetch`, `redis`, and the like). Parse the params, build a path, redirect: nothing there +for a check to find evidence in either way. Exclusion is a denominator exit, not a credit. + +A route whose body is somewhere else is a different case. `export { action } from "./handler.server"` +and `export const action = handleWebhook` are not trivial: a redirect stub genuinely has nothing to +instrument, while a delegating route has work the scanner cannot see, and treating them alike would +delete a route from the metric whenever someone moved a body into a `.server.ts` file. Delegating +routes are counted apart from the unmeasured ones, listed on a `DELEGATED` line and carried in the +JSON as `delegating`, the same treatment a parse failure gets and for the same reason. There are +none in the tree today, which is exactly why the case needed writing down before someone wrote one. + +## Suppression + +```ts +// obs-map-disable auth-boundary -- public by design, see ADR 12 +``` + +The reason is mandatory: a suppression without one is ignored. The directive is read from comments +only, so a string literal quoting it does not switch a check off. + +It applies to the whole entry point, not to the line under it. Line scoping is not available, +because a finding is attached to an entry point and carries no line number to match against, which +is why the old `obs-map-disable-next-line` spelling is not honoured. + +A suppression cannot raise a score. The suppressed check leaves the numerator and the denominator, +and the result is capped by what the entry would have scored unsuppressed, so suppressing a failing +check holds the number still rather than improving it (`src/score.test.ts`: `does not raise the +score when a failing check is suppressed`). What you buy is removal from the worklist with a reason +on the record. The report prints how many suppressions are in force so the practice stays visible. + +## Known limits + +Read these before trusting a specific verdict. + +- **One hop, same file only.** If a loader delegates to a helper in the same file, that helper's + statements, catches and calls count as the route's. A helper's own helpers do not, and nothing + imported from another module is ever opened. `auth-boundary` applies to 62 of the 67 sensitive + entry points; the other 5 hand their work to an imported helper and are reported as unverified + rather than unguarded. +- **A guard is matched by name, not by what it does.** The accept-list is 29 names read off the + webapp, plus two `SOFT_GUARDS`. `src/webappSymbols.test.ts` proves each one is declared somewhere; + nothing proves the declaration it found is the guard we meant. `authenticateAdmin` and + `authenticatePlainRequest` are local helpers inside one route file each, so a second route + declaring its own no-op function of either name would be credited. +- **Two guard names are only checked as far as being read.** `getUser` and `getUserId` answer with + null instead of throwing, so calling one is not a boundary. They are credited only when the body + binds the result and some condition reads it (`EntryPoint.checkedCallees`). What that cannot see + is whether the test guards anything: `if (!user) { logger.warn("anonymous"); }` followed by the + work reads the same as returning. +- **`authenticate` and `isAuthenticated` are unresolved on purpose.** They are remix-auth's, and + resolving them means reading a path inside `apps/webapp/node_modules`, which fails confusingly on + an install-layout change. They are listed in `EXTERNAL_GUARDS` instead, so the resolution test + still rejects a name that is neither first-party nor listed. +- **`auth-scope` applicable structurally implies `auth-boundary` pass.** The check only applies to a + builder-wrapped route, and `auth-boundary` passes any builder-wrapped route, so all 19 carry the + same `auth-boundary` detail, "authenticated by the builder". That free point is a third or a + quarter of each of their scores: the 19 average 59.7 as scored and 44.6 with `auth-boundary` taken + out, and `settings.team`, a confirmed cross-org exposure, scores 25 rather than 0. Read the + finding rather than the score. +- **`auth-scope` cannot tell a caller-id filter from a caller-id actor argument.** + `presenter.call({ userId: user.id })` narrows the query; `generatePortalLink({ organizationId, + userId: user.id })` records who asked. Both read as scoping. Separating them means following the + argument into the callee, so the four helpers credited this way (`ApiKeysPresenter`, + `TeamPresenter`, `regenerateApiKey`, `DeleteOrganizationService`, all of which do + `members: { some: { userId } }` and throw) were hand-read instead. No route in the tree passes on + an actor argument alone. +- **`auth-scope` reads property assignments in that export's own handler.** A handler that pulls the + id into a local first, `const userId = user.id; ... { userId }`, or that builds its filter in a + same-file helper, scopes itself and is not seen, so it would be reported as unscoped. +- **`auth-scope` reads the builder-wrapped exports and says nothing about the rest of the file.** A + route whose action is builder-wrapped and whose loader is a plain `export async function loader` + is judged on the action alone, and the pass detail, "every builder-wrapped export has an + authorization gate", is true of what it read while reading as a claim about the whole route. Ten + routes in the tree mix the two, and the one sensitive enough for the check to run on is + `_app.orgs.$organizationSlug.settings._index/route.tsx`, whose builder-wrapped action carries the + pass and whose plain loader filters on `members: { some: { userId } }`. That was hand-read. +- **Three login-flow routes fail `auth-boundary` correctly and unhelpfully.** `/auth/sso`, + `/api/v1/authorization-code` and `/api/v1/token` are unauthenticated by design: the caller is + anonymous at that point, which is the whole purpose. The check's statement about them is true and + there is nothing to fix, so they are candidates for a suppression comment with the reason on the + record. +- **Loggers are matched by spelling.** A call counts as logging when the callee reads `logger.*` or + `log.*`. An aliased logger, one wrapped in a helper, or `console.error` is invisible, so a route + can be reported as recording nothing while it records plenty. +- **A catch that logs and rethrows reads as though it only rethrows.** The clause evidence cannot say + whether a clause does anything besides rethrow, so `error-classification` withholds credit rather + than granting it and reopening the free-points path a single `logger.error` line wide. +- **Only the first object-literal argument is read** for identifier fields, and only its property + names. `logger.error("failed", ctx)` where `ctx` is a variable contributes nothing, and neither + does a second object. +- **A catch inside a per-item callback is not the route's.** `items.map((item) => { try {...} })` is + a fresh boundary per element, so its clause is not read as the route's own error handling. The + test is the method name, which cannot tell `users.map` from `Result.map`. Being wrong there costs + precision rather than points: a refused catch fails the route rather than excusing it. +- **A route that delegates only one of its two exports is judged on the other.** + `export { action } from "./x"` beside a loader written in the file is not counted as delegating, + so half the route is scored and half is invisible. +- **`try { String(0); }` still buys a pass.** The open corpus entry above, and the largest single + hole known in the tool: measured live, it takes the tree from 19 to 44 and raises 224 routes. +- **A forged tenant field buys a pass too.** `request-context` reads the field name, never the + value, so a codemod writing `environmentId: "obs-map"` into every in-catch log call takes the + global from 19 to 29. Unlike the entry above this one is not a bug to fix, since no syntactic + check can tell a real tenant id from a constant, but it bounds what the number can mean either + way. +- **The score is a mean of means over a heuristic.** Read the fix list, the two headline figures and + the CHECKS block. Watching the single number for small movements will mislead you. + +## Layout + +`scan.ts` walks the routes directory and produces an `EntryPoint` per module, carrying only +body-scoped evidence. `checks/` holds the five checks, each a pure function of an `EntryPoint`. +`score.ts` turns checks into an entry score and a report, `report/` renders it, `cli.ts` is the +entry point. `sensitivity.ts`, `triviality.ts` and `suppression.ts` are the three inputs the checks +share. + +Tests sit next to their subject in `src/`. Every check has a false-positive fixture, something it +must not flag, alongside the positive one. Keep that: most of the bugs this package has had were +checks that fired on the wrong thing, and a test that only proves the heuristic fires would have +caught none of them. diff --git a/internal-packages/observability-map/package.json b/internal-packages/observability-map/package.json new file mode 100644 index 000000000..cd8c3fc14 --- /dev/null +++ b/internal-packages/observability-map/package.json @@ -0,0 +1,25 @@ +{ + "name": "@internal/observability-map", + "private": true, + "version": "0.0.1", + "type": "module", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "typescript": "catalog:" + }, + "devDependencies": { + "@types/node": "^24.13.3", + "rimraf": "6.0.1", + "tsx": "^4.19.2", + "vitest": "4.1.7" + }, + "scripts": { + "clean": "rimraf dist", + "typecheck": "tsc --noEmit", + "build": "pnpm run clean && tsc -p tsconfig.build.json", + "test": "vitest run", + "test:watch": "vitest", + "map": "tsx src/cli.ts" + } +} diff --git a/internal-packages/observability-map/src/adapters/remix.test.ts b/internal-packages/observability-map/src/adapters/remix.test.ts new file mode 100644 index 000000000..be45ce6c6 --- /dev/null +++ b/internal-packages/observability-map/src/adapters/remix.test.ts @@ -0,0 +1,57 @@ +import { familyOf, routePathOf } from "./remix.js"; + +describe("familyOf", () => { + it("classifies each family from the flat-route filename", () => { + expect(familyOf("api.v1.runs.$runId.ts")).toBe("api.v1"); + expect(familyOf("api.something.ts")).toBe("api.other"); + expect(familyOf("admin.api.v1.gc.ts")).toBe("admin"); + expect(familyOf("resources.queues.ts")).toBe("resources"); + expect(familyOf("_app.orgs.$slug.ts")).toBe("dashboard"); + expect(familyOf("otel.v1.logs.ts")).toBe("ingest"); + expect(familyOf("@.ts")).toBe("other"); + }); + + it("prefers admin over api.v1 for admin-prefixed api routes", () => { + expect(familyOf("admin.api.v1.environments.$id.ts")).toBe("admin"); + }); +}); + +describe("routePathOf", () => { + it("turns a flat-route filename into a path", () => { + expect(routePathOf("api.v1.runs.$runId.ts")).toBe("/api/v1/runs/:runId"); + }); + + it("strips a trailing method suffix", () => { + expect(routePathOf("api.v1.runs.ts")).toBe("/api/v1/runs"); + }); +}); + +// Directory routes: the scanner recurses into Remix directory routes, so `fileName` can be a +// relative path like `_app.orgs.$organizationSlug.projects.$projectParam/route.tsx` rather than a +// flat dot-separated name. The directory name (not `route.tsx`) carries the meaning. +describe("familyOf: directory routes", () => { + it("classifies a directory route by its directory name, not the flat rules", () => { + expect(familyOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx")).toBe( + "dashboard" + ); + }); + + it("classifies each family the same whether the route is flat or a directory", () => { + expect(familyOf("api.v1.runs.$runId/route.tsx")).toBe("api.v1"); + expect(familyOf("admin.api.v1.environments.$id/route.tsx")).toBe("admin"); + expect(familyOf("resources.queues/route.tsx")).toBe("resources"); + expect(familyOf("storybook.callout/route.tsx")).toBe("other"); + }); +}); + +describe("routePathOf: directory routes", () => { + it("does not emit a literal 'route' segment for a directory route", () => { + const path = routePathOf("_app.orgs.$organizationSlug.projects.$projectParam/route.tsx"); + expect(path).not.toContain("route"); + expect(path).toBe("/_app/orgs/:organizationSlug/projects/:projectParam"); + }); + + it("produces the same path shape for a directory route as its flat equivalent", () => { + expect(routePathOf("api.v1.runs.$runId/route.tsx")).toBe(routePathOf("api.v1.runs.$runId.ts")); + }); +}); diff --git a/internal-packages/observability-map/src/adapters/remix.ts b/internal-packages/observability-map/src/adapters/remix.ts new file mode 100644 index 000000000..00dcc76e4 --- /dev/null +++ b/internal-packages/observability-map/src/adapters/remix.ts @@ -0,0 +1,40 @@ +export type Family = + | "api.v1" + | "api.other" + | "webhooks" + | "admin" + | "resources" + | "dashboard" + | "ingest" + | "other"; + +/** + * The name that carries routing meaning for a given `fileName`. A directory route holds its module in + * a fixed `route.ts`/`route.tsx`, so the segment before the slash is the meaningful name. + */ +function routeName(fileName: string): string { + const slashIndex = fileName.indexOf("/"); + return slashIndex === -1 ? fileName : fileName.slice(0, slashIndex); +} + +export function familyOf(fileName: string): Family { + const name = routeName(fileName); + // admin is checked first: admin.api.v1.* is an admin route, not an api.v1 one. + if (name.startsWith("admin.")) return "admin"; + if (name.startsWith("api.v1.")) return "api.v1"; + if (name.startsWith("api.")) return "api.other"; + if (name.startsWith("webhooks.")) return "webhooks"; + if (name.startsWith("resources.")) return "resources"; + if (name.startsWith("_app.")) return "dashboard"; + if (name.startsWith("otel.") || name.startsWith("engine.")) return "ingest"; + return "other"; +} + +export function routePathOf(fileName: string): string { + const withoutExt = routeName(fileName).replace(/\.(ts|tsx)$/, ""); + const segments = withoutExt + .split(".") + .filter((s) => s.length > 0) + .map((s) => (s.startsWith("$") ? `:${s.slice(1)}` : s)); + return `/${segments.join("/")}`; +} diff --git a/internal-packages/observability-map/src/checks/auditTrail.ts b/internal-packages/observability-map/src/checks/auditTrail.ts new file mode 100644 index 000000000..d1a0c06ec --- /dev/null +++ b/internal-packages/observability-map/src/checks/auditTrail.ts @@ -0,0 +1,54 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "audit-trail"; + +/** + * Calls that write a record of who did something. All three reach + * `prisma.impersonationAuditLog.create` in `models/admin.server.ts`, which nothing here matches + * directly, because `calleeNames` records `create` for a member call and no route writes the row + * itself. Matched against `importedNames` too, so an import counts. + * + * Two of these are also in `SENSITIVE_SYMBOLS`, so a route made sensitive only by one of them cannot + * fail this check: the call that put it in the cohort is the call that satisfies it. That is not the + * circularity the sensitivity list was cleaned up to remove, since impersonation genuinely is the + * hazard AND genuinely writes the record, but the consequence is real either way. + */ +export const AUDIT_SYMBOLS = [ + "redirectWithImpersonation", + "clearImpersonation", + "startImpersonation", +]; + +/** + * Whether a sensitive mutation leaves a record of who did it. Applicability follows the same rule as + * every other check, which it did not before: gating on sensitivity and `hasAction` alone had this + * check accusing `resources.impersonation.ts` over an audit write behind the very import + * `auth-boundary` declined to judge it for. + * + * The order below matters and mirrors `auth-boundary`: a known audit call is read BEFORE the + * triviality exemption, because presence is evidence even where absence is not. + */ +export const auditTrail = { + id: ID, + run(ep: EntryPoint): CheckResult { + // Mutations only: a sensitive read does not need an actor record. + if (!classifySensitivity(ep).sensitive || !ep.hasAction) { + return { id: ID, status: "not-applicable", detail: "not a sensitive mutation" }; + } + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + if (AUDIT_SYMBOLS.some((s) => symbols.has(s))) { + return { id: ID, status: "pass", detail: "records an audit event" }; + } + if (isTrivial(ep)) { + return { + id: ID, + status: "not-applicable", + detail: + "cannot verify: no privileged work in the body, any audit write is behind an import", + }; + } + return { id: ID, status: "fail", detail: "sensitive mutation with no audit record" }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/authBoundary.ts b/internal-packages/observability-map/src/checks/authBoundary.ts new file mode 100644 index 000000000..ee5b014a4 --- /dev/null +++ b/internal-packages/observability-map/src/checks/authBoundary.ts @@ -0,0 +1,137 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { routeExports, type ExportName, type RouteExport } from "../routeExports.js"; +import { isTrivialExport } from "../triviality.js"; +import { BUILDERS } from "./errorClassification.js"; + +const ID = "auth-boundary"; + +/** + * The guard helpers this webapp actually has, matched against the calling export's own callee names. + * A guard the route only imports and never calls does not count, and neither does one the OTHER + * export calls. + * + * Names rather than the three patterns it replaces, all of which over-matched: INTERNALS.md, + * "Sensitivity, and the names the tool matches on". `webappSymbols.test.ts` fails if a name stops + * resolving, but + * cannot check that a declaration with the right name is the guard we meant, which is the residual + * the two local helpers below carry. + */ +export const GUARDS = new Set([ + // Session and PAT identity, `apps/webapp/app/services/session.server.ts` and friends. + "requireUser", + "requireUserId", + "requireOrganization", + "requireAdminApiRequest", + "authenticateApiRequest", + "authenticateApiRequestWithFailure", + "authenticateApiRequestWithPersonalAccessToken", + "authenticateApiRequestWithOrganizationAccessToken", + "authenticateApiKey", + "authenticateAuthorizationHeader", + "authenticateOrganizationAccessToken", + "authenticatePersonalAccessToken", + "authenticateRequest", + "authenticateAdminRequest", + "authenticatedEnvironmentForAuthentication", + "authenticateAndAuthorize", + // The RBAC controller, `packages/plugins/src/rbac.ts`, reached as `rbac.authenticateSession(...)`. + // `calleeName` records the property for a member call, so these arrive here unqualified. + "authenticateSession", + "authenticatePat", + "authenticateBearer", + "authenticateUserActor", + "authenticateAuthorizeSession", + "authenticateAuthorizeBearer", + // remix-auth, reached as `authenticator.authenticate(...)`. The login surface is sensitive and + // cannot require an already authenticated caller, so establishing identity from the credential + // presented is what a guard means there. + "authenticate", + "isAuthenticated", + // Local helpers, each declared inside the one route that uses it. + "authenticateAdmin", + "authenticatePlainRequest", + // Proof of possession: an HMAC on a callback URL, or a login-surface second factor. Same + // reasoning as `authenticate` above for the two `login.mfa` names. + "verifyHttpCallbackHash", + "verifyWebhook", + "verifyUserActorToken", + "verifyTotpForLogin", + "verifyRecoveryCodeForLogin", +]); + +/** + * Guards that answer with null instead of throwing, so calling one is not itself a boundary: these + * are credited only when THAT EXPORT's handlers read what they returned + * (`EntryPoint.loaderCheckedCallees`, which also carries the residual, that the test reading the + * answer need not guard anything). + */ +export const SOFT_GUARDS = new Set(["getUser", "getUserId"]); + +type GuardedExport = { name: ExportName; guarded: boolean; how: string; export: RouteExport }; + +/** + * The exports this file declares, each with its own verdict. Per export because the exposure is per + * export, and every input here used to be entry-point-wide, which is a false PASS on the one check + * where that hides a security gap. + * + * `routeExports` lists only the exports the file declares, so an export that calls nothing at all is + * judged rather than skipped: an empty body is exactly the unguarded case. + */ +function guardedExports(ep: EntryPoint): GuardedExport[] { + return routeExports(ep).map((e) => { + const verdict = (guarded: boolean, how: string) => ({ name: e.name, guarded, how, export: e }); + if (e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) { + return verdict(true, "authenticated by the builder"); + } + if (e.calleeNames.some((n) => GUARDS.has(n))) { + return verdict(true, "guarded in the body"); + } + if (e.checkedCallees.some((n) => SOFT_GUARDS.has(n))) { + return verdict(true, "resolves the caller and reads the answer"); + } + return verdict(false, ""); + }); +} + +/** + * Whether a route that handles credentials, tokens or money checks who is asking. + * + * A fail here is an accusation, so it is only made when the body is the place a guard would have to + * be. A trivial body cannot contain a visible privileged operation, so any guard is behind the same + * import as the work and its absence proves nothing. That is the applicability rule in README, "When + * a check declines to judge", and it is deliberately not the rule `request-context` uses. + */ +export const authBoundary = { + id: ID, + run(ep: EntryPoint): CheckResult { + const sensitivity = classifySensitivity(ep); + if (!sensitivity.sensitive) { + return { id: ID, status: "not-applicable", detail: "not sensitive" }; + } + // Never empty: `scanFile` returns null unless the file declares a loader or an action. + const exports = guardedExports(ep); + const guarded = exports.filter((e) => e.guarded); + // Triviality excuses per export, matching the attribution: read entry-point-wide, a busy action + // made a redirect-stub loader answerable for a guard it has nothing to guard. + const accused = exports.filter((e) => !e.guarded && !isTrivialExport(e.export)); + if (accused.length > 0) { + return { + id: ID, + status: "fail", + detail: `sensitive (${sensitivity.reasons.join(", ")}) with no auth guard in the body: ${accused + .map((e) => e.name) + .join(", ")}`, + }; + } + if (guarded.length > 0) { + const how = [...new Set(guarded.map((e) => e.how))].join(" and "); + return { id: ID, status: "pass", detail: how }; + } + return { + id: ID, + status: "not-applicable", + detail: "cannot verify: no privileged work in the body, any guard is behind an import", + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/authScope.ts b/internal-packages/observability-map/src/checks/authScope.ts new file mode 100644 index 000000000..c4d42d16f --- /dev/null +++ b/internal-packages/observability-map/src/checks/authScope.ts @@ -0,0 +1,68 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { classifySensitivity } from "../sensitivity.js"; +import { routeExports } from "../routeExports.js"; +import { BUILDERS } from "./errorClassification.js"; + +const ID = "auth-scope"; + +type BuilderExport = { name: string; callee: string; scoped: boolean; why: string }; + +/** + * The builder-wrapped exports of an entry point, each with its own verdict. Per export because the + * exposure is per export: `authorization` is declared on the builder call one export made, and a + * caller filter is written in the handler one export runs. + */ +function builderExports(ep: EntryPoint): BuilderExport[] { + return routeExports(ep) + .filter((e) => e.initializerCallee !== null && BUILDERS.has(e.initializerCallee)) + .map((e) => { + const authorization = e.builderOptions.includes("authorization"); + return { + name: e.name, + callee: e.initializerCallee!, + scoped: authorization || e.scopesByCaller, + why: authorization ? "an authorization gate" : "a filter on the caller's identity", + }; + }); +} + +/** + * Whether a route the builder authenticated is also narrowed to the caller. The IDOR class it + * measures: README, "The five checks". The two ways to be scoped, and why `ability.can(...)` is + * deliberately not a third: INTERNALS.md, "What auth-scope reads as scoping". + * + * There is no triviality test here because `isTrivial` answers false for any route with an + * initializer callee, and no delegating test because `scoreEntry` answers for every check before any + * of them runs. One WAS here saying otherwise. + * + * Three residuals, running both ways. Accusing: `scopesByCallerIn` reads property assignments in that + * export's own handlers only, so a handler pulling the id into a local first + * (`const userId = user.id; ... { userId }`) is not seen. Crediting: a caller id passed as an ACTOR + * argument rather than as a filter still counts, e.g. `generatePortalLink({ organizationId, userId })` + * records who asked without constraining which org is read. Crediting: a helper running the membership + * query for you is credited through the caller id handed to it, which is the same syntax; the four + * helpers this applies to were hand-read and are correct. + */ +export const authScope = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (!classifySensitivity(ep).sensitive) { + return { id: ID, status: "not-applicable", detail: "not sensitive" }; + } + const builders = builderExports(ep); + if (builders.length === 0) { + return { id: ID, status: "not-applicable", detail: "no route builder to read options from" }; + } + const unscoped = builders.filter((b) => !b.scoped); + if (unscoped.length === 0) { + const how = [...new Set(builders.map((b) => b.why))].join(" and "); + return { id: ID, status: "pass", detail: `every builder-wrapped export has ${how}` }; + } + const which = unscoped.map((b) => `${b.name} (${b.callee})`).join(", "); + return { + id: ID, + status: "fail", + detail: `authenticated but not scoped to the caller: ${which} declares no authorization gate and does not filter by the caller's identity`, + }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/errorClassification.ts b/internal-packages/observability-map/src/checks/errorClassification.ts new file mode 100644 index 000000000..8a6dcb30c --- /dev/null +++ b/internal-packages/observability-map/src/checks/errorClassification.ts @@ -0,0 +1,155 @@ +import type { CatchEvidence, CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "error-classification"; + +/** + * The route builders that authenticate the request, which is what `auth-boundary` reads them for. + * They also catch and classify, but `error-classification` does not credit that: a route with no + * catch of its own is judged on nothing, wrapper or not. + * + * `createSSELoader` is deliberately absent. It turns a non-Response error into a 500 but does not + * authenticate, so listing it would hand two routes a free `auth-boundary` pass. + * `createHybridActionApiRoute`, which the design named, exists nowhere in the tree. + */ +export const BUILDERS = new Set([ + "createLoaderApiRoute", + "createActionApiRoute", + "createLoaderPATApiRoute", + "createActionPATApiRoute", + "createMultiMethodApiRoute", + "createLoaderWorkerApiRoute", + "createActionWorkerApiRoute", + "dashboardLoader", + "dashboardAction", +]); + +/** + * How much a try block may guard and still count as narrow. Two, so the guarded operation can bind + * its result (`const stripped = ...; new RegExp(stripped);`). + * + * An absolute count and not a ratio against the enclosing body, or padding the body relabels the + * same broad swallow as a narrow guard (`inert-statements-after-try`). The count is NOT unpaddable, + * which an earlier docstring claimed: it is one condition of three and no longer the load-bearing + * one. See INTERNALS.md, "Parse guards, and the narrow-try count". + */ +const NARROW_TRY_STATEMENTS = 2; + +/** + * Whether a catch clause is a guard rather than the route's error handling: the try block parses, + * waits for nothing except that parse, and is short. + * + * Two residuals, since awaiting is the signal rather than calling. A block that does its non-parse + * work synchronously still reads as a guard, and `guardedWork` looks for a `ts.AwaitExpression`, + * which `for await (...)` and `await using` are not. Neither occurs in the tree and neither is + * reachable by rewriting a real route. Both are in the round A fix 3 report. + */ +function isParseGuard(clause: CatchEvidence): boolean { + return ( + clause.guardsParse && + clause.awaitsOnlyParse && + clause.tryStatementCount <= NARROW_TRY_STATEMENTS + ); +} + +/** + * Whether a clause decides anything about the error it caught. Two ways to qualify: it branches, or + * it guards a parse it can answer for. Rethrowing is not a third way, and neither is being narrow + * without parsing. + * + * The cost is real and worth stating here: `catch (e) { logger.error(...); throw e }` reads as + * inert too, because `CatchEvidence` cannot say whether a clause does anything besides rethrow. + * That is the safe direction, since crediting it reopens the hole a bare `logger.error` line wide, + * and `request-context` still reads that log and asks whether it names a tenant. + */ +function decides(clause: CatchEvidence): boolean { + return clause.branches || isParseGuard(clause); +} + +/** Passes the error through unchanged, which is the same outcome as not catching it. */ +function inert(clause: CatchEvidence): boolean { + return clause.rethrows && !decides(clause); +} + +/** The error stops here and nothing chose what it meant. */ +function swallows(clause: CatchEvidence): boolean { + return !decides(clause) && !inert(clause); +} + +/** + * Who decides what a failure means, and on what evidence. Judged per catch clause, so an entry point + * is only as good as its worst one. + * + * A route with no catch is not-applicable rather than a pass, and that takes the builder credit with + * it. "Does this route catch anything" is `catches.length`, never `hasTryCatch`, because a + * try/finally leaves the flag true and the list empty. + * + * The open hole a reader of this function has to know about: `guardCanRaise` refuses `try { 0; }` + * and is defeated by one inert call, so `try { String(0); }` reads as classification and takes the + * tree from 19 to 44. `dead-classifying-try-with-call` in the mutation corpus is that shape, + * running as an expected failure. Read the rule as "refuses `try { 0; }`", never as "an unreachable + * catch cannot be credited". Everything else here: INTERNALS.md, "The dead-code defence". + */ +export const errorClassification = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + const reachable = ep.catches.filter((c) => c.guardCanRaise); + const swallowed = reachable.filter(swallows); + if (swallowed.length > 0) { + const which = + reachable.length > 1 ? ` (${swallowed.length} of ${reachable.length} catches)` : ""; + // "One way out" is only true of a clause that never throws, so this reads `throws` and not + // `rethrows`. 16 clauses in the tree would otherwise carry the wrong detail line. + const everyWayOut = swallowed.every((c) => !c.throws); + return { + id: ID, + status: "fail", + detail: everyWayOut + ? `catches its errors and takes one way out regardless of what was thrown${which}` + : `catches its errors and chooses what to do without looking at what was thrown${which}`, + }; + } + // Deliberately NOT conditioned on `ep.catches.length === 0`: an own inert catch, which + // `wrap-body-in-rethrow` adds to every route, must not lift a refused swallow out of the verdict + // (`fails a per-item swallow even when the route owns an inert rethrow catch`). And read off + // `ep.catches` under `guardMayRaise`, never `reachable`, since a deciding catch `canRaise` + // cannot see still decides (`does not accuse a route that owns a catch of owning none`). + const reachableCb = ep.callbackCatches.filter((c) => c.guardCanRaise); + const ownDecides = ep.catches.some((c) => decides(c) && c.guardMayRaise); + if (!ownDecides && reachableCb.some(swallows)) { + return { + id: ID, + status: "fail", + detail: + "a catch inside an iteration callback swallows what it caught, and nothing the route owns decides", + }; + } + // The ceiling that keeps `dead-deciding-map` from minting a pass on the 261 catchless routes: + // refused catches never reach the pass arm. Read off `ep.catches` and not `reachable`, since a + // route that owns a catch owns one whether or not `canRaise` could see what it guarded. + if (ep.catches.length === 0 && ep.callbackCatches.length > 0) { + return { + id: ID, + status: "not-applicable", + detail: + "its only catches sit in iteration callbacks and none swallows, so the route itself classifies nothing", + }; + } + if (!reachable.some(decides)) { + return { + id: ID, + status: "not-applicable", + detail: + reachable.length === 0 + ? ep.catches.length === 0 + ? "catches nothing, so it classifies nothing" + : "guards nothing that can throw, so it classifies nothing" + : "every catch rethrows and nothing else, so it classifies nothing", + }; + } + return { id: ID, status: "pass", detail: "every catch decides what it caught" }; + }, +}; diff --git a/internal-packages/observability-map/src/checks/index.test.ts b/internal-packages/observability-map/src/checks/index.test.ts new file mode 100644 index 000000000..2f6e4e0e3 --- /dev/null +++ b/internal-packages/observability-map/src/checks/index.test.ts @@ -0,0 +1,2198 @@ +import { CHECKS, SCORED_CHECK_IDS } from "./index.js"; +import { scanFile } from "../scan.js"; + +const run = (id: string, fileName: string, source: string) => { + const ep = scanFile(fileName, source)!; + return CHECKS.find((c) => c.id === id)!.run(ep); +}; + +/** + * The React component that lives alongside the loader in a `.tsx` route. Every check reads + * body-scoped evidence, so nothing in here may change a verdict: it try/catches, it logs, it names + * every request identifier the checks look for, and it calls an auth helper. + */ +const COMPONENT = ` + export default function Page() { + const { environmentId, organizationId, projectId, runId } = useTypedLoaderData(); + useEffect(() => { + try { + requireUserId(environmentId); + logger.error("render failed", { environmentId, organizationId, projectId, runId }); + } catch (e) { + if (e instanceof Error) return; + throw e; + } + }, [environmentId]); + return
{runId}
; + } +`; + +describe("registry", () => { + it("holds the five checks, with audit-trail left out of the score", () => { + expect(CHECKS.map((c) => c.id)).toEqual([ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", + "audit-trail", + ]); + expect(SCORED_CHECK_IDS).toEqual([ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", + ]); + }); +}); + +describe("error-classification", () => { + // C1. A route with no catch makes no classification decision, so there is nothing here to judge + // and nothing to credit. Crediting it made deleting error handling raise the score. + it("is not applicable to a builder-wrapped route with no local try/catch", () => { + const r = run( + "error-classification", + "api.v1.x.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + ); + expect(r.status).toBe("not-applicable"); + }); + + it("fails a raw route whose catch swallows every error identically", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // Restored from the brief: the clause branches, on the `instanceof` and the `if`. + it("passes a raw route whose catch branches on the error", () => { + const r = run( + "error-classification", + "api.v1.z.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { if (e instanceof NotFound) return null; throw e; } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A clause that only rethrows makes no classification decision: the error propagates exactly as + // it would with no catch at all, so it is read as no catch at all. Scoring the two differently + // paid 50 points a route for wrapping a body in `try { ... } catch (e) { throw e }`. + it("is not applicable to a raw route whose catch only rethrows", () => { + const r = run( + "error-classification", + "api.v1.v.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { logger.error("thing lookup failed", { error: e }); throw e; } + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The builder only classifies what reaches it. A swallow inside the handler never does, so the + // swallow is read before the builder is credited. + it("fails a builder-wrapped route whose handler swallows", () => { + const r = run( + "error-classification", + "api.v2.runs.$runParam.cancel.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { CancelTaskRunService } from "~/services/cancelTaskRun.server"; + const { action } = createActionApiRoute({}, async ({ params }) => { + const service = new CancelTaskRunService(); + try { await service.call(params.runParam); } + catch { return json({ error: "Internal Server Error" }, { status: 500 }); } + return json({ ok: true }); + }); + export { action };` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a raw route that lets its errors propagate", () => { + const r = run( + "error-classification", + "api.v1.w.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "error-classification", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // A narrow guard around one operation classifies an expected failure without needing to branch + // or rethrow. Guarding a parse over a small part of the body is what tells it apart from a + // handler-wide catch. + it("passes a narrow guard around a single parse", () => { + const r = run( + "error-classification", + "resources.timezone.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + const saved = await prisma.preference.create({ data }); + return json({ saved }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // The exact boundary, held regardless of how big the rest of the function is: two statements binds + // the parsed result and passes, a third means the try has started to cover the handler. + it("passes a parse guard that binds its result in exactly two statements", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + parsed = new RegExp(raw); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // C5. The count is not the only condition any more, and this is the shape that showed why: two + // statements, one of them a parse, and the whole handler inside the try. Before `awaitsOnlyParse` + // the count read it as a narrow guard and passed it, which is the `otel.v1.logs.ts` swallow + // written compactly. Three spellings of the same thing, all of which the count reads as narrow. + const COMPACT_SWALLOWS: Array<[string, string]> = [ + [ + "two statements", + `try { const body = await request.json(); return await handleEverything(body); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, the parse nested inside the call", + `try { return await handleEverything(await request.json()); } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + [ + "one statement, merged into a declaration list", + `try { const body = await request.json(), out = await handleEverything(body); return out; } + catch (error) { return new Response("Internal Server Error", { status: 500 }); }`, + ], + ]; + + for (const [label, body] of COMPACT_SWALLOWS) { + it(`fails a whole handler wrapped in a parse-guard-shaped try (${label})`, () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) {\n${body}\n}` + ); + expect(r.status).toBe("fail"); + }); + } + + // The counterpart: the same route with the handler moved out of the try is a real guard and + // still passes, so the rule above is not just "any try containing an await fails". + it("passes the same route once the handler moves out of the try", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `export async function action({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "bad json" }, { status: 400 }); } + return await handleEverything(body); + }` + ); + expect(r.status).toBe("pass"); + }); + + // Synchronous string work preparing a parse's input is not what `awaitsOnlyParse` refuses. Four + // real routes are this shape, `admin.llm-models.new.tsx` among them. + it("passes a guard that prepares its input synchronously before parsing", () => { + const r = run( + "error-classification", + "admin.llm-models.new.tsx", + `export async function action({ request }) { + const matchPattern = String(await request.text()); + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return await save(matchPattern); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a parse guard that takes a third statement beyond binding the result", () => { + const r = run( + "error-classification", + "resources.pattern.ts", + `export async function action({ request }) { + let parsed; + try { + const raw = await request.text(); + const trimmed = raw.trim(); + parsed = new RegExp(trimmed); + } catch { + return json({ error: "Invalid pattern" }, { status: 400 }); + } + return json({ parsed: parsed.source }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // False positive fixture for the narrow rule: a narrow parse guard must not launder the broad + // handler catch sitting next to it. Clauses are judged one at a time, so the broad one still + // counts against the entry point. + it("still fails when a narrow guard sits beside a handler-wide swallow", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + return json({ thing, audit }); + } catch (error) { + return json({ error: "Something went wrong" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // try/finally with no catch clause. `hasTryCatch` is true here and `catches` is empty, and it is + // `catches` that answers "does this route catch anything". Nothing is swallowed: the error + // propagates once the connection is closed. + it("is not applicable to a try/finally that catches nothing", () => { + const r = run( + "error-classification", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + const other = await redis.exists("other-key"); + return json({ exists, other }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // Multi-catch, the case the aggregate booleans could not describe. Judged per clause: the parse + // guard is a guard, the handler catch rethrows, so both are accounted for. + it("passes a parse guard sitting beside a handler catch that rethrows", () => { + const r = run( + "error-classification", + "api.v1.thing.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + let data; + try { data = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const thing = await prisma.thing.create({ data }); + const audit = await prisma.audit.create({ data: { thing: thing.id } }); + const count = await prisma.thing.count(); + return json({ thing, audit, count }); + } catch (error) { + logger.error("create failed", { error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A parse guard that has grown to cover the handler is not a guard any more. `otel.v1.logs.ts` + // catches 15 of its 18 statements around a `request.json()` and answers 500 for all of them. + it("fails a parse guard that covers most of the body", () => { + const r = run( + "error-classification", + "otel.v1.logs.ts", + `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // Against the entry-point statement count, a sibling handler or a fat helper in the same file + // diluted the denominator and relabelled the same broad swallow as a narrow parse guard. + it("gives the same verdict to a byte-identical swallow whether or not an unrelated sibling and helper share the file", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const withUnrelatedSiblingAndHelper = `${action} + function unrelatedHelper() { + let total = 0; + total += 1; + total += 2; + total += 3; + total += 4; + total += 5; + total += 6; + total += 7; + total += 8; + total += 9; + total += 10; + total += 11; + return total; + } + export async function loader() { + const helperTotal = unrelatedHelper(); + return new Response(String(helperTotal)); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const withSiblingAndHelper = run( + "error-classification", + "otel.v1.logs.ts", + withUnrelatedSiblingAndHelper + ); + + expect(alone.status).toBe("fail"); + expect(withSiblingAndHelper.status).toBe("fail"); + }); + + // Same-body dilution, which moving the denominator to the enclosing body did not close: as a + // ratio, padding the SAME action with 11 inert statements took the identical swallow to pass. + it("gives the same verdict to a byte-identical swallow whether or not it is padded with inert statements in the same body", () => { + const action = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + }`; + + const padding = Array.from({ length: 11 }, (_, i) => `const pad${i} = ${i};`).join("\n"); + const paddedInSameBody = `import { otlpExporter } from "~/v3/otlpExporter.server"; + export async function action({ request }) { + try { + const exporter = await otlpExporter; + const contentType = request.headers.get("content-type"); + const body = await request.json(); + const result = await exporter.exportLogs(body); + const encoded = encodeResponse(result); + const headers = buildHeaders(contentType); + return new Response(encoded, { status: 200, headers }); + } catch (error) { + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } + ${padding} + return new Response("unreachable", { status: 200 }); + }`; + + const alone = run("error-classification", "otel.v1.logs.ts", action); + const padded = run("error-classification", "otel.v1.logs.ts", paddedInSameBody); + + expect(alone.status).toBe("fail"); + expect(padded.status).toBe("fail"); + }); + + // A3. `referencesBinding` used to match any identifier with the binding's text, including a + // property name in a member expression. A catch whose only `if` tests `fallback.error`, never the + // caught binding itself, was credited with classifying an error it never inspected. + it("fails a catch whose only if tests a same-named property, not the caught error", () => { + const r = run( + "error-classification", + "api.v1.y.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // False positive fixture: the only try/catch in the file belongs to the component. + it("does not judge a route whose try/catch is in the React component", () => { + const r = run( + "error-classification", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return typedjson({ rows }); + } + ${COMPONENT}` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The anti-laundering half of the boundary rule: judging refused catches on their evidence must + // not stop failing a swallow relocated behind one. + it("fails a route whose only catch is inside a Promise.all(items.map(...)) callback", () => { + const source = `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + }`; + const ep = scanFile("batch.process.ts", source)!; + expect(ep.catches).toEqual([]); + expect(ep.callbackCatches).toHaveLength(1); + const r = run("error-classification", "batch.process.ts", source); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("a catch inside an iteration callback swallows"); + }); + + // The evidence half of the mechanism-C rule: a refused catch that DECIDES caps at not-applicable + // rather than failing (the old placement rule) or passing (the crediting rule `dead-deciding-map` + // exists to refuse). The route's error handling is real and per item; the route itself decides + // nothing, so out of the denominator is the honest place for it. + it("sits out a route whose only catch is a deciding per-item boundary", () => { + const r = run( + "error-classification", + "batch.decide.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const results = await stream.map(async (item) => { + try { + await service.call(item); + } catch (e) { + if (e instanceof KnownError) { return new Response(e.code, { status: 400 }); } + throw e; + } + }); + return json({ results }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("its only catches sit in iteration callbacks"); + }); + + // Same shape with an inert per-item rethrow: not a swallow, so it sits out too. + it("sits out a route whose only catch is an inert per-item rethrow", () => { + const r = run( + "error-classification", + "batch.rethrow.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const results = await stream.map(async (item) => { + try { + await service.call(item); + } catch (e) { + throw e; + } + }); + return json({ results }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("its only catches sit in iteration callbacks"); + }); + + // Why the refused-swallow arm is not conditioned on the route owning no catches: an own inert catch + // is exactly what `wrap-body-in-rethrow` adds to every route, and the gate would lift this fail to + // not-applicable. + it("fails a per-item swallow even when the route owns an inert rethrow catch", () => { + const r = run( + "error-classification", + "batch.wrapped.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } catch (e) { + throw e; + } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("a catch inside an iteration callback swallows"); + }); + + // The no-pass ceiling at the fixture scale: a catchless route with a prepended dead deciding + // map sits out. A crediting rule would read pass here, which is 50 free points on the tree's + // 261 catchless routes; the old placement rule read fail, a false accusation on a preserving + // prepend. `dead-deciding-map` in the mutation corpus is the tree-scale version. + it("sits out a catchless route with a prepended dead deciding map", () => { + const r = run( + "error-classification", + "prepended-map.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + [0, 1].map((v) => { try { JSON.parse("0"); } catch (e) { if (e instanceof SyntaxError) { return null; } throw e; } return v; }); + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("its only catches sit in iteration callbacks"); + }); + + // The same route with nothing caught anywhere stays not-applicable, so the fail above is + // attributable to the refused catch and not to the check having stopped excusing anything. + it("is not applicable to a route that catches nothing at all", () => { + const r = run( + "error-classification", + "batch.process.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all(items.map(async (item) => processItem(item))); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("catches nothing"); + }); + + // S2 at the check level. The evidence tests in `scan.test.ts` pin `guardCanRaise` itself; these + // pin the check reading it, which is where the 50 points were. Prepending this to a route that + // catches nothing took it from not-applicable to pass, and 224 routes were in exactly that state. + it("is not applicable to a route whose only catch guards a try that cannot throw", () => { + const r = run( + "error-classification", + "prepended.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + const rows = await prisma.thing.findMany(); + return json({ rows }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("guards nothing that can throw"); + }); + + it("still fails a swallow that a dead classifying catch was prepended to", () => { + const r = run( + "error-classification", + "prepended-swallow.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + try { + return json(await prisma.thing.findMany()); + } catch (error) { + return new Response(null, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes the same classifying catch once its try does real work", () => { + const r = run( + "error-classification", + "live.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + try { await prisma.thing.findMany(); } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // "Takes one way out regardless of what was thrown" is false of a clause that throws for some + // errors, and 16 clauses in the tree were eligible for the wording. Whether `fail` is the right + // verdict for them at all is a separate question, parked. + it("does not accuse a clause that throws of taking one way out", () => { + const r = run( + "error-classification", + "mixed.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { if (rare) { return null; } throw e; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("without looking at what was thrown"); + expect(r.detail).not.toContain("one way out"); + }); + + it("still says one way out for a clause that never throws", () => { + const r = run( + "error-classification", + "swallow.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return json(await prisma.thing.findMany()); } + catch (e) { logger.error(e); return null; } + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("one way out"); + }); + + // `canRaise` does not list destructuring, and `const { a } = undefined` throws, so an owned + // classifying catch drops out of `reachable`; the refused-swallow arm reads `guardMayRaise` instead. + // Asserted on `status`, since an earlier version asserted the absence of a detail string no arm + // emits, which could not fail. + it("does not accuse a route that owns a catch of owning none", () => { + const r = run( + "error-classification", + "owned.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { const { a } = undefined; } catch (e) { + if (e instanceof TypeError) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("guards nothing that can throw"); + }); + + // I4 sibling: the same shape through `.filter` and a different `canRaise` miss (a plain + // declaration is not on the whitelist either), so the fix is the rule and not the fixture. + it("does not accuse a route whose deciding catch guards a declaration beside a filter swallow", () => { + const r = run( + "error-classification", + "owned-filter.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + const kept = items.filter((item) => { + try { return check(item); } catch { return false; } + }); + try { const parsed = { ...raw }; } catch (e) { + if (e instanceof TypeError) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ kept }); + }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The blocking catch has to DECIDE: an own inert rethrow catch over the same invisible guard + // still leaves the refused swallow in the verdict, or `wrap-body-in-rethrow` spelled with a + // destructuring guard would lift every per-item swallow out of it. + it("still fails a per-item swallow beside an inert catch over an invisible guard", () => { + const r = run( + "error-classification", + "owned-inert.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { const { a } = undefined; } catch (e) { throw e; } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("nothing the route owns decides"); + }); + + // And the blocking catch has to guard something that MAY raise: the provably-inert `try { 0; }` + // clause `dead-classifying-try` prepends decides and must still block nothing, or the prepend + // would lift a refused-swallow fail to not-applicable at tree scale. + it("still fails a per-item swallow beside a deciding catch over a dead guard", () => { + const r = run( + "error-classification", + "owned-dead.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const items = await prisma.item.findMany(); + await Promise.all( + items.map(async (item) => { + try { await processItem(item); } catch { return null; } + }) + ); + try { 0; } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("nothing the route owns decides"); + }); + + // C2. A single-element array cannot iterate, so `[0].map(async () => { whole body })` is not a + // per-item boundary and the route's own catch is found where it always was. Before this, the + // wrapper deleted the route's catches and took a swallow from fail to not-applicable. + it("still fails a swallow wrapped in Promise.all([0].map(...))", () => { + const r = run( + "error-classification", + "wrapped.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const [result] = await Promise.all([0].map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + })); + return result; + }` + ); + expect(r.status).toBe("fail"); + }); + + // A second receiver exercising the same mechanism: an empty array literal, which no name list + // would treat differently from a populated one. + it("still fails a swallow wrapped in [].flatMap(...)", () => { + const r = run( + "error-classification", + "wrapped-empty.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return [].flatMap(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // A third, where the name list cannot help at all: a non-array receiver whose method is called + // `map`. The boundary rule still refuses the callback, so the route has no catch of its own, and + // the refusal now fails rather than excusing. This is the shape the name list cannot tell from + // `users.map(...)`, and it is why the refusal had to stop paying. + it("still fails a swallow wrapped in a non-array receiver's .map(...)", () => { + const r = run( + "error-classification", + "wrapped-result.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + return await Result.map(async () => { + try { + const body = await request.json(); + const a = await stepOne(body); + const b = await stepTwo(a); + return json({ b }); + } catch (e) { + return new Response("nope", { status: 500 }); + } + }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // The verdict end of the `break and continue inside the construct they target` finding. A clause + // that sorts the error by code and then rethrows was failed, with a detail line asserting it + // takes one way out regardless of what was thrown, which is the opposite of what it does. Both + // spellings are here because the pair is the evidence: the switch must not change the verdict. + const SORTED_RETHROW = (sorter: string) => `import { prisma } from "~/db.server"; + export async function action({ request, params }) { + try { + return json(await prisma.thing.update({ where: { id: params.id }, data: {} })); + } catch (e) { + ${sorter} + throw e; + } + }`; + + it("does not accuse a clause that sorts the error by code and rethrows", () => { + const r = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW( + 'switch (e.code) { case "P2025": handleNotFound(e); break; default: handleOther(e); break; }' + ) + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).not.toContain("one way out"); + }); + + it("reads the same clause written without the switch identically", () => { + const withSwitch = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW( + 'switch (e.code) { case "P2025": handleNotFound(e); break; default: handleOther(e); break; }' + ) + ); + const without = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW("handleOther(e);") + ); + expect(withSwitch).toEqual(without); + }); + + // The other direction, so the rule above is not just "a switch is ignored": the same sorter with + // a clause that answers the request is a decision, and still passes. + it("still passes a clause whose switch on the error code answers the request", () => { + const r = run( + "error-classification", + "api.v1.sorted.ts", + SORTED_RETHROW( + 'switch (e.code) { case "P2025": return new Response(null, { status: 404 }); default: break; }' + ) + ); + expect(r.status).toBe("pass"); + }); + + // The verdict end of the walk's guaranteed-execution entries, one pair per entered construct. The + // evidence end is `the walk enters exactly the positions guaranteed to execute` in scan.test.ts. + const CLAUSE_WRAPPED = (clauseBody: string) => `import { prisma } from "~/db.server"; + export async function loader() { + try { + return json(await prisma.thing.findMany()); + } catch (e) { + ${clauseBody} + } + }`; + + const DECIDING_CLAUSE = + "if (e instanceof Error) { return new Response(null, { status: 400 }); }\n" + + "return new Response(null, { status: 500 });"; + + const GUARANTEED_WRAPPERS: Array<[string, (body: string) => string]> = [ + ["a catchless try/finally", (body) => `try {\n${body}\n} finally { }`], + ["a single-default switch", (body) => `switch (pick()) { default: {\n${body}\n} }`], + ["an if (true)", (body) => `if (true) {\n${body}\n}`], + [ + "an if/else with the body in both arms", + (body) => `if (pick()) {\n${body}\n} else {\n${body}\n}`, + ], + ]; + + for (const [label, wrap] of GUARANTEED_WRAPPERS) { + it(`reads a deciding clause relocated into ${label} with the same verdict`, () => { + const wrapped = run( + "error-classification", + "api.v1.wrapped.ts", + CLAUSE_WRAPPED(wrap(DECIDING_CLAUSE)) + ); + const bare = run( + "error-classification", + "api.v1.wrapped.ts", + CLAUSE_WRAPPED(DECIDING_CLAUSE) + ); + expect(bare.status).toBe("pass"); + expect(wrapped).toEqual(bare); + }); + } +}); + +describe("auth-boundary", () => { + it("passes a sensitive route guarded by a require helper", () => { + // Sensitive on the impersonation call, not on the guard: calling a guard is not what makes a + // route sensitive, see sensitivity.test.ts. + const r = run( + "auth-boundary", + "admin.api.v1.impersonate.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { setImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + await requireAdminApiRequest(request); + return setImpersonation(request, "user_1"); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a sensitive route guarded by an authenticate helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + if (!auth) throw new Response(null, { status: 401 }); + return prisma.token.findMany(); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a sensitive route with no guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a non-sensitive route", () => { + const r = run( + "auth-boundary", + "api.v1.timezones.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.tz.findMany(); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // False positive fixture for the delegated guard. `clearImpersonation` authenticates and writes + // an audit row, in `app/models/admin.server.ts`, which the scanner cannot open. The body shows no + // privileged work either, so there is nothing here to accuse: absence of evidence, not evidence + // of absence. + it("does not flag a sensitive route that hands its work to an imported helper", () => { + const r = run( + "auth-boundary", + "resources.impersonation.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + return clearImpersonation(request, "/admin"); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toMatch(/verif/i); + }); + + it("does not flag a sensitive redirect stub", () => { + const r = run( + "auth-boundary", + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The gate must not swallow the real thing: a body doing its own privileged work, unguarded. + it("still fails a sensitive route whose visible body does the work unguarded", () => { + const r = run( + "auth-boundary", + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export async function action({ request }) { + const body = await request.json(); + const code = await prisma.authorizationCode.findFirst({ where: { code: body.code } }); + if (!code) return json({ error: "Not found" }, { status: 404 }); + const token = await createPersonalAccessToken(code.userId); + return json({ token }); + }` + ); + expect(r.status).toBe("fail"); + }); + + // Possession of a valid signature is the auth boundary for a callback URL. + const signatureRoute = (guard: string) => + run( + "auth-boundary", + "webhooks.v1.billing.$hash.ts", + `import { ${guard} } from "~/services/webhooks.server"; + import { prisma } from "~/db.server"; + export async function action({ request, params }) { + const invoice = await prisma.invoice.findFirst({ where: { id: params.id } }); + if (!${guard}(params.hash, invoice)) { + return json({ error: "Invalid" }, { status: 401 }); + } + return json({ ok: true }); + }` + ); + + it("passes a sensitive callback guarded by a signature check", () => { + expect(signatureRoute("verifyWebhook").status).toBe("pass"); + }); + + // C1a. The accept-list is derived from the helpers this webapp has. `verifyWebhookSignature` is + // a plausible name that exists nowhere in it, and the pattern this list replaced passed it. + it("does not pass a signature guard the webapp does not have", () => { + expect(signatureRoute("verifyWebhookSignature").status).toBe("fail"); + }); + + // False positive fixture: the guard sits one hop away, in a same-file helper. + it("does not flag a sensitive route whose guard is in a same-file helper", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); }` + ); + expect(r.status).toBe("pass"); + }); + + // The guard has to be called, not merely imported: importedNames is file-wide. + it("fails a sensitive route that imports a guard it never calls", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader() { + const tokens = await prisma.token.findMany(); + return json({ tokens }); + } + export function meta() { return requireUserId; }` + ); + expect(r.status).toBe("fail"); + }); +}); + +describe("request-context", () => { + it("passes a route whose failure log names an identifier", () => { + const r = run( + "request-context", + "engine.v1.dev.config.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("dev config failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes on a route param, which names the tenant just as well", () => { + const r = run( + "request-context", + "resources.things.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { organizationSlug: params.organizationSlug, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("fails a failure log that carries only the error", () => { + const r = run( + "request-context", + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { error }); throw error; } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("fails a bare failure log with no object argument at all", () => { + const r = run( + "request-context", + "api.v1.r.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { logger.error("failed"); throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The builder logs `{ error, url }` at its boundary and nothing that names a tenant, so being + // wrapped in one earns no pass here. This is what stops the check echoing `auth-boundary`. + it("fails a builder-wrapped route whose own failure log names nobody", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async () => { + try { return json(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } + });` + ); + expect(r.status).toBe("fail"); + }); + + // C1. The global handler carries requestId, path, host and method, and no tenant. A route that + // never catches cannot name one, so it fails rather than being credited or excused. + it("fails a route that leaves everything to the central handler", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { authenticateApiRequest } from "~/services/apiAuth.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const auth = await authenticateApiRequest(request); + return json(await prisma.thing.findMany({ where: { environmentId: auth.environment.id } })); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("fails a route that catches but only names an identifier outside the catch", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + logger.info("starting", { environmentId: "env_1" }); + try { return await prisma.thing.findMany(); } catch (e) { throw e; } + }` + ); + expect(r.status).toBe("fail"); + }); + + // A route that catches and reports nothing at all is the case the log-based applicability gate + // used to excuse. It is a finding, not an exemption. + it("fails a route that catches and reports nothing", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { return json({ error: "Internal Server Error" }, { status: 500 }); } + }` + ); + expect(r.status).toBe("fail"); + }); + + // A guard around a parse is not the route taking over its failure path: whatever its real work + // throws still reaches the central handler. Same reading error-classification gives the field. + it("fails a route whose only catch guards a parse", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + return json(await prisma.thing.findMany({ where: body })); + }` + ); + expect(r.status).toBe("fail"); + }); + + // Was a known false positive: `new URL()` is a constructor, so the parse was invisible to the + // call-callee scan the evidence used to come from. `CatchEvidence.guardsParse` covers + // constructors, so the guard is legible now and the route is no longer judged as though it kept + // its failures. + it("fails a route whose only catch guards a constructor parse", () => { + const r = run( + "request-context", + "_app.@.orgs.$organizationSlug.$.tsx", + `import { prisma } from "~/db.server"; + function refererOrigin(request) { + const referer = request.headers.get("referer"); + try { return new URL(referer).origin; } + catch { return undefined; } + } + export async function loader({ request }) { + const origin = refererOrigin(request); + return typedjson({ origin, things: await prisma.thing.findMany() }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("fails a try/finally that catches nothing", () => { + const r = run( + "request-context", + "admin.api.v1.runs-replication.status.ts", + `import Redis from "ioredis"; + export async function loader() { + const redis = new Redis({ host: "localhost" }); + try { + const exists = await redis.exists("some-key"); + return json({ exists }); + } finally { + await redis.quit(); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still judges a route with a handler-wide catch beside a narrow guard", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + let body; + try { body = await request.json(); } + catch { return json({ error: "Invalid JSON" }, { status: 400 }); } + try { + const rows = await prisma.thing.findMany({ where: body }); + const count = await prisma.thing.count(); + return json({ rows, count }); + } catch (error) { + logger.error("failed", { error }); + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // The incentive fixture pair. The two routes differ by one line, the log call, and nothing else. + // Deleting that line must never improve the verdict or drop the route out of the report. + it("never improves a verdict when the log call is deleted", () => { + const body = (log: string) => + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { ${log} throw error; } + }`; + + const withLog = run( + "request-context", + "api.v1.q.ts", + body(`logger.error("failed", { environmentId: params.envId, error });`) + ); + const withoutLog = run("request-context", "api.v1.q.ts", body("")); + + expect(withLog.status).toBe("pass"); + expect(withoutLog.status).toBe("fail"); + expect(withoutLog.status).not.toBe("not-applicable"); + }); + + // False positive fixture: the component logs every identifier there is, inside its own catch. + // Only the loader's own failure log may decide this. + it("does not read the React component's log calls", () => { + const bare = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { error }); throw error; } + } + ${COMPONENT}` + ); + expect(bare.status).toBe("fail"); + + const attributed = run( + "request-context", + "_app.orgs.$organizationSlug.things/route.tsx", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return typedjson(await prisma.thing.findMany()); } + catch (error) { logger.error("failed", { projectParam: params.projectParam, error }); throw error; } + } + ${COMPONENT}` + ); + expect(attributed.status).toBe("pass"); + }); + + it("is not applicable to a trivial redirect", () => { + const r = run( + "request-context", + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // A5. IDENTIFIER_FIELD matched any suffix, so a resource id (a run, a batch, a notification, a + // chat, a span) that shares the same `Id`/`Param` shape as a tenant field passed, and a bare `id` + // passed too. TENANT_FIELD requires the root word itself to be environment, organization, project + // or user. + it("does not pass a failure log that only names a resource, not a tenant", () => { + const r = run( + "request-context", + "api.v3.batches.$batchId.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("batch lookup failed", { batchId: params.batchId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass a failure log that only names a bare id", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("cache miss", { id: 1 }); + logger.error("lookup failed", { id: 1, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes on the abbreviated envId/orgId forms the webapp also writes", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, orgId: params.orgId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // A5. request-context filtered failure-path logs on inCatch only, not on level, so a debug log + // naming a tenant field passed the check even though debug lines are routinely dropped or + // sampled out before an incident is read. + it("does not pass a debug-level failure log, even one that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.debug("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass an info-level failure log either", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.info("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes a warn-level failure log that names a tenant", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.warn("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + // M7. The real logger (packages/core/src/logger.ts) has log/error/warn/info/debug/verbose, no + // fatal and no trace, and log is level 0, never filtered by TRIGGER_LOG_LEVEL, so it must + // qualify. verbose is the actual noisiest level this codebase has, not trace. + it("passes a log-level failure log, which the real logger never filters", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.log("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); + + it("does not pass a verbose-level failure log", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.verbose("lookup failed", { environmentId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + // M8. A bare `env` field is ambiguous with a deployment environment name + // (`{ env: process.env.NODE_ENV }`), not a tenant, so it must not qualify on its own; the + // abbreviated root still works with a real suffix. + it("does not pass a failure log that only names a bare env field", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { env: process.env.NODE_ENV, error }); + throw error; + } + }` + ); + expect(r.status).toBe("fail"); + }); + + it("still passes envId, the abbreviated root with a real suffix", () => { + const r = run( + "request-context", + "api.v1.q.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { + logger.error("lookup failed", { envId: params.envId, error }); + throw error; + } + }` + ); + expect(r.status).toBe("pass"); + }); +}); + +describe("audit-trail", () => { + it("is applicable only to sensitive mutations", () => { + const readOnly = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `export async function loader() { return 1; }` + ); + expect(readOnly.status).toBe("not-applicable"); + + const mutation = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ); + expect(mutation.status).toBe("fail"); + }); + + // False positive fixture: an ordinary mutation is not an audit target. + it("is not applicable to a non-sensitive mutation", () => { + const r = run( + "audit-trail", + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.thing.create({ data: {} }); }` + ); + expect(r.status).toBe("not-applicable"); + }); + + // The pass branch had never been exercised against a real name: the fixture imported `auditLog` + // from `~/services/audit.server`, a helper and a module that both exist nowhere. All three names + // on the list now reach `prisma.impersonationAuditLog.create` in `models/admin.server.ts`. + it.each(["redirectWithImpersonation", "clearImpersonation", "startImpersonation"])( + "passes a sensitive mutation that records an audit event through %s", + (writer) => { + const r = run( + "audit-trail", + "admin.impersonate.tsx", + `import { ${writer} } from "~/models/admin.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const target = await prisma.user.findFirst({ where: { admin: false } }); + const session = await ${writer}(request, target.id, "/"); + return session; + }` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toBe("records an audit event"); + } + ); + + it("still fails a sensitive mutation that writes no record", () => { + const r = run( + "audit-trail", + "api.v1.auth.jwt.ts", + `import { prisma } from "~/db.server"; + export async function action({ request }) { + const token = await prisma.token.create({ data: { name: request.url } }); + return json(token); + }` + ); + expect(r.status).toBe("fail"); + }); + + // The coherence fix. `auth-boundary` declines to judge a trivial body because a guard would be + // behind the import; this check accused the same body over an audit write behind the same + // import. Same rule now, and presence is still read before the exemption, so a trivial body that + // does call a writer passes rather than sitting out. + it("declines to judge a trivial sensitive mutation, as auth-boundary does", () => { + const source = `import { doTheThing } from "~/models/admin.server"; + export async function action({ request }) { return doTheThing(request, "/admin"); }`; + expect(run("audit-trail", "resources.impersonation.ts", source).status).toBe("not-applicable"); + expect(run("auth-boundary", "resources.impersonation.ts", source).status).toBe( + "not-applicable" + ); + }); + + it("still passes a trivial sensitive mutation that calls a writer", () => { + const r = run( + "audit-trail", + "resources.impersonation.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function action({ request }) { return clearImpersonation(request, "/admin"); }` + ); + expect(r.status).toBe("pass"); + }); +}); + +// C1a. The guard list is names now, not a five-character prefix. Both directions matter: a real +// helper must still clear a sensitive route, and a callee that merely starts the right way must +// not. +describe("auth-boundary: the guard accept-list", () => { + const sensitiveRoute = (guard: string) => + run( + "auth-boundary", + "api.v1.orgs.$orgParam.members.ts", + `import { prisma } from "~/db.server"; + export async function loader({ request, params }) { + const caller = await ${guard}(request); + const members = await prisma.orgMember.findMany({ where: { orgId: params.orgParam } }); + return json({ members, caller }); + }` + ); + + it.each(["requireUserId", "requireUser", "authenticateApiRequest", "authenticateSession"])( + "passes a sensitive route guarded by %s", + (guard) => { + expect(sensitiveRoute(guard).status).toBe("pass"); + } + ); + + // The live case. `requireSsoEntitlement` is a plan check inside one route file, and the prefix + // pattern cleared the org SSO settings route on it. + it("does not pass a plan check that happens to start with require", () => { + const r = sensitiveRoute("requireSsoEntitlement"); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("no auth guard in the body"); + }); + + // `getUser` and `getUserId` answer with null instead of throwing, so being called is not evidence + // of a boundary: they are credited only when the body reads the answer. + it("does not pass a route that resolves the caller and ignores the answer", () => { + const r = run( + "auth-boundary", + "invite-accept.tsx", + `import { getUser } from "~/services/session.server"; + import { getInviteFromToken } from "~/models/member.server"; + export async function loader({ request }) { + const user = await getUser(request); + const token = new URL(request.url).searchParams.get("token"); + const invite = await getInviteFromToken({ token }); + return json({ invite, email: user.email }); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("does not pass a route that drops the caller entirely", () => { + const r = run( + "auth-boundary", + "invite-accept.tsx", + `import { getUserId } from "~/services/session.server"; + import { getInviteFromToken } from "~/models/member.server"; + export async function loader({ request }) { + await getUserId(request); + const token = new URL(request.url).searchParams.get("token"); + return json(await getInviteFromToken({ token })); + }` + ); + expect(r.status).toBe("fail"); + }); + + it("passes an invite acceptance that resolves the caller and refuses a mismatch", () => { + const r = run( + "auth-boundary", + "invite-accept.tsx", + `import { getUser } from "~/services/session.server"; + import { getInviteFromToken } from "~/models/member.server"; + export async function loader({ request }) { + const user = await getUser(request); + const token = new URL(request.url).searchParams.get("token"); + if (!user) return redirect("/login"); + const invite = await getInviteFromToken({ token }); + if (invite.email !== user.email) return redirect("/"); + return redirect("/"); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("passes a login page that sends an already authenticated caller away", () => { + const r = run( + "auth-boundary", + "login._index/route.tsx", + `import { getUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (userId) return redirect("/"); + const flags = await prisma.featureFlag.findMany(); + return typedjson({ flags }); + }` + ); + expect(r.status).toBe("pass"); + }); + + it("does not pass an invented require helper", () => { + expect(sensitiveRoute("requireValidParams").status).toBe("fail"); + expect(sensitiveRoute("requireQueryParam").status).toBe("fail"); + }); + + // `resolveAuthenticatedEnv` hydrates an environment record from its id. Ten routes call it and + // the /Authenticated/ pattern read every one of them as guarded. + it("does not pass a lookup whose name merely contains Authenticated", () => { + expect(sensitiveRoute("resolveAuthenticatedEnv").status).toBe("fail"); + expect(sensitiveRoute("commitAuthenticatedSession").status).toBe("fail"); + }); +}); + +/** + * Per-export attribution. Each `it` here goes green on the entry-point-wide version of exactly one of + * the three inputs, which is why they are separate cases rather than one. + */ +describe("auth-boundary: a guard credits only the export that calls it", () => { + const TOKENS = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server";`; + + it("fails an unguarded action beside a loader that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("fails an unguarded loader beside an action that calls a guard", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ params }) { + const tokens = await prisma.token.findMany({ where: { orgId: params.orgId } }); + return json({ tokens }); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); + + // The soft-guard arm reads its own export's checked-callee list for the same reason. + it("fails an unguarded action beside a loader that reads what getUserId returned", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { getUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (!userId) return redirect("/login"); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + // The builder arm. `usesBuilder` was an OR over both initializer callees, so a builder on one + // export authenticated a hand-written handler on the other. + it("fails a hand-written action beside a builder-wrapped loader", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderApiRoute({}, async ({ authentication }) => { + return json(await prisma.token.findMany({ where: { userId: authentication.userId } })); + }); + export async function action({ request }) { + const body = await request.json(); + await prisma.token.deleteMany({ where: { id: body.id } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action"); + }); + + it("passes when both exports call a guard of their own", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `${TOKENS} + export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + const userId = await requireUserId(request); + await prisma.token.deleteMany({ where: { userId } }); + return json({ ok: true }); + }` + ); + expect(r.status).toBe("pass"); + }); + + // One handler serving both exports guards both, which is the dominant API-route shape. + it("passes a shared builder handler that both exports resolve to", () => { + const r = run( + "auth-boundary", + "api.v1.tokens.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json({ userId: authentication.userId }); + });` + ); + expect(r.status).toBe("pass"); + }); + + /** The damper on the attribution: `auth.github.ts` and `auth.google.ts` went pass to fail on the + * real tree until `isTrivialExport` existed. */ + it("reports not-applicable for a redirect-stub loader beside a guarded action", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + const url = new URL(request.url); + const safeRedirect = sanitizeRedirectPath(url.searchParams.get("redirectTo"), "/"); + return await authenticator.authenticate("github", request, { + successRedirect: safeRedirect, + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toBe("guarded in the body"); + }); + + /** The per-export excuse must read the export's own body and not the file's text, or + * `log-caller-scope-userid` puts the word `logger` in this file and accuses the untouched loader. */ + it("does not un-excuse a redirect-stub loader because the file mentions a logger", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { logger } from "~/services/logger.server"; + export let loader = () => redirect("/login"); + export let action = async ({ request }) => { + logger.error("obs-map", { userId: request.userId }); + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("pass"); + }); + + it("fails an export whose own body does real work unguarded", () => { + const r = run( + "auth-boundary", + "auth.github.ts", + `import { authenticator } from "~/services/auth.server"; + import { prisma } from "~/db.server"; + export let loader = async ({ params }) => { + const org = await prisma.organization.findFirst({ where: { slug: params.slug } }); + const members = await prisma.orgMember.findMany({ where: { orgId: org.id } }); + return json({ org, members }); + }; + export let action = async ({ request }) => { + return await authenticator.authenticate("github", request, { + successRedirect: "/", + failureRedirect: "/login", + }); + };` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader"); + }); +}); + +// C1b. A builder authenticates the request; it does not necessarily scope it. `authorization` is +// optional on every one of them and the RBAC gate only runs when it is declared. +describe("auth-scope", () => { + const patRoute = (options: string, body: string) => + run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionPATApiRoute( + { method: "POST"${options}}, + async ({ authentication, params }) => { + ${body} + return json({ members }); + } + );` + ); + + const UNSCOPED = `const members = await prisma.orgMember.findMany({ + where: { organization: { slug: params.orgParam } }, + });`; + + it("fails a sensitive PAT route that authenticates and scopes nothing", () => { + const r = patRoute("", UNSCOPED); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("authenticated but not scoped to the caller"); + expect(r.detail).toContain("createActionPATApiRoute"); + }); + + it("passes when the builder declares the authorization gate", () => { + const r = patRoute( + `, authorization: { action: "read", resource: { type: "members" } }`, + UNSCOPED + ); + expect(r.status).toBe("pass"); + expect(r.detail).toContain("authorization gate"); + }); + + it("passes when the handler filters by the caller's membership instead", () => { + const r = patRoute( + "", + `const members = await prisma.orgMember.findMany({ + where: { organization: { slug: params.orgParam, members: { some: { userId: authentication.userId } } } }, + });` + ); + expect(r.status).toBe("pass"); + expect(r.detail).toContain("caller's identity"); + }); + + // Round C ruling 1. apps/webapp/CLAUDE.md: the OSS fallback ability is permissive, so + // `ability.can(...)` enforces the role and the membership-scoped query is the tenant floor. + // Crediting it made this check agree with `_app.orgs.$organizationSlug.settings.sso/route.tsx`, + // which resolves its target org from the URL slug and puts nothing else in front of it. + it("does not accept an ability gate in the handler as scoping", () => { + const r = run( + "auth-scope", + "_app.orgs.$slug.settings.team/route.tsx", + `import { dashboardAction } from "~/services/routeBuilders/dashboardBuilder"; + import { prisma } from "~/db.server"; + export const action = dashboardAction({ params: Params }, async ({ ability, params }) => { + if (!ability.can("manage", { type: "members" })) throw new Response(null, { status: 403 }); + const members = await prisma.orgMember.findMany({ where: { slug: params.slug } }); + return json({ members }); + });` + ); + expect(r.status).toBe("fail"); + }); + + // The two shapes on the real tree, both hand-read for round C. + it("fails when the loader is unscoped and only the action filters by the caller", () => { + const r = run( + "auth-scope", + "_app.orgs.$slug.settings.sso/route.tsx", + `import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + import { prisma } from "~/db.server"; + export const loader = dashboardLoader({ params: Params }, async ({ context, ability }) => { + if (!ability.can("manage", { type: "sso" })) throwPermissionDenied(); + return json(await prisma.ssoConnection.findMany({ where: { organizationId: context.organizationId } })); + }); + export const action = dashboardAction({ params: Params }, async ({ context, user }) => + json(await ssoController.generatePortalLink({ organizationId: context.organizationId, userId: user.id })) + );` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("loader (dashboardLoader)"); + expect(r.detail).not.toContain("action"); + }); + + it("fails when the action is unscoped and only the loader filters by the caller", () => { + const r = run( + "auth-scope", + "_app.orgs.$slug.settings.team/route.tsx", + `import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + import { prisma } from "~/db.server"; + export const loader = dashboardLoader({ params: Params }, async ({ user }) => + json(await new TeamPresenter().call({ userId: user.id })) + ); + export const action = dashboardAction({ params: Params }, async ({ context, ability }) => { + if (!ability.can("manage", { type: "members" })) throwPermissionDenied(); + return json(await prisma.orgMember.deleteMany({ where: { organizationId: context.organizationId } })); + });` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action (dashboardAction)"); + }); + + // A file whose loader is gated and whose action is not is not a gated route. + it("fails when only one of two builder exports declares the gate", () => { + const r = run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute, createLoaderPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const loader = createLoaderPATApiRoute( + { authorization: { action: "read", resource: { type: "members" } } }, + async ({ params }) => json(await prisma.orgMember.findMany({ where: { slug: params.orgParam } })) + ); + export const action = createActionPATApiRoute({ method: "POST" }, async ({ params }) => + json(await prisma.orgMember.deleteMany({ where: { slug: params.orgParam } })) + );` + ); + expect(r.status).toBe("fail"); + expect(r.detail).toContain("action (createActionPATApiRoute)"); + }); + + // Round D item 3, at the check level: the shape that cleared both real findings. + it("is not cleared by a dead object holding the caller id", () => { + const r = run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionPATApiRoute({ method: "POST" }, async ({ user, params }) => { + const unused = { userId: user.id }; + return json(await prisma.orgMember.deleteMany({ where: { slug: params.orgParam } })); + });` + ); + expect(r.status).toBe("fail"); + }); + + it("is not applicable to a route that is not sensitive", () => { + const r = run( + "auth-scope", + "api.v1.runs.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionApiRoute({ method: "POST" }, async ({ params }) => + json(await prisma.taskRun.findMany({ where: { id: params.id } })) + );` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toBe("not sensitive"); + }); + + it("is not applicable to a sensitive route with no builder to read options from", () => { + const r = run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function action({ request, params }) { + await requireUserId(request); + return json(await prisma.orgMember.deleteMany({ where: { slug: params.orgParam } })); + }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("no route builder"); + }); + + // A builder-wrapped route is never trivial, so the sensitive routes that sit out for having + // nothing in the body sit out here for having no builder instead. + it("is not applicable to a trivial sensitive route with no builder", () => { + const r = run( + "auth-scope", + "orgs.$organizationSlug.team.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader({ params }) { return redirect(teamPath(params.slug)); }` + ); + expect(r.status).toBe("not-applicable"); + expect(r.detail).toContain("no route builder"); + }); +}); + +// The cheapest way to launder auth-scope would be to write the option and give it nothing, which +// the builder's own `if (authorization)` treats as absent. +describe("auth-scope: an option declared as nothing is not declared", () => { + const withValue = (value: string) => + run( + "auth-scope", + "api.v1.orgs.$orgParam.members.ts", + `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { prisma } from "~/db.server"; + export const action = createActionPATApiRoute( + { method: "POST", authorization: ${value} }, + async ({ params }) => json(await prisma.orgMember.findMany({ where: { slug: params.orgParam } })) + );` + ); + + it.each(["undefined", "null", "false"])("fails on authorization: %s", (value) => { + expect(withValue(value).status).toBe("fail"); + }); + + it("passes on a real one, so the rule is about the value and not the key", () => { + expect(withValue(`{ action: "read", resource: { type: "members" } }`).status).toBe("pass"); + }); +}); diff --git a/internal-packages/observability-map/src/checks/index.ts b/internal-packages/observability-map/src/checks/index.ts new file mode 100644 index 000000000..6dd1fbc48 --- /dev/null +++ b/internal-packages/observability-map/src/checks/index.ts @@ -0,0 +1,23 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { errorClassification } from "./errorClassification.js"; +import { authBoundary } from "./authBoundary.js"; +import { authScope } from "./authScope.js"; +import { requestContext } from "./requestContext.js"; +import { auditTrail } from "./auditTrail.js"; + +export type Check = { id: string; run: (ep: EntryPoint) => CheckResult }; + +/** audit-trail is scored separately, see score.ts. */ +export const CHECKS: Check[] = [ + errorClassification, + authBoundary, + authScope, + requestContext, + auditTrail, +]; +export const SCORED_CHECK_IDS = [ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", +]; diff --git a/internal-packages/observability-map/src/checks/requestContext.ts b/internal-packages/observability-map/src/checks/requestContext.ts new file mode 100644 index 000000000..f0e0087b2 --- /dev/null +++ b/internal-packages/observability-map/src/checks/requestContext.ts @@ -0,0 +1,67 @@ +import type { CheckResult, EntryPoint } from "../types.js"; +import { isTrivial } from "../triviality.js"; + +const ID = "request-context"; + +/** + * A field name that plausibly names a TENANT: environment, organization, project or user. Anchored on + * the root word and not just the suffix, so `batchId` and `spanParam` do not qualify: those name a + * resource the failure touched rather than who it happened to. The abbreviated roots require a + * suffix, because a bare `env` is ambiguous with `{ env: process.env.NODE_ENV }`. + */ +const TENANT_FIELD = + /^(environment|organization|project|user)(Id|Ids|Slug|Ref|Param|Identifier)?$|^(env|org)(Id|Ids|Slug|Ref|Param|Identifier)$/; + +/** + * Read against the real logger (`packages/core/src/logger.ts`), whose levels are `log`, `error`, + * `warn`, `info`, `debug`, `verbose`, with no `fatal` and no `trace`. `log` is level 0, which + * `TRIGGER_LOG_LEVEL` never filters out. `info` is not reserved for failure reporting and + * `debug`/`verbose` are routinely sampled out before anyone reads an incident, so neither qualifies. + */ +const QUALIFYING_LEVELS = new Set(["log", "error", "warn"]); + +/** The level a `LogCall`'s callee was made at, e.g. `"error"` from `logger.error`. */ +function logLevel(callee: string): string { + return callee.slice(callee.lastIndexOf(".") + 1); +} + +/** + * Whether a failure here can be traced to whoever it happened to. + * + * Every non-trivial entry point is judged, and a route that never catches fails like any other. That + * is the whole point rather than an oversight: its failures go to the global handler, which names no + * tenant. Passing those routes, as this check used to, meant deleting every catch clause in the tree + * scored it 100, and excusing them as not-applicable is the same mistake in quieter clothes. What the + * platform attaches centrally and why none of it is a tenant: README, "What the score means". + */ +export const requestContext = { + id: ID, + run(ep: EntryPoint): CheckResult { + if (isTrivial(ep)) { + return { id: ID, status: "not-applicable", detail: "trivial route" }; + } + const failurePathLogs = ep.logCalls.filter( + (l) => l.inCatch && QUALIFYING_LEVELS.has(logLevel(l.callee)) + ); + const named = failurePathLogs.find((l) => l.fields.some((f) => TENANT_FIELD.test(f))); + if (named) { + const fields = named.fields.filter((f) => TENANT_FIELD.test(f)); + return { id: ID, status: "pass", detail: `failure log names ${fields.join(", ")}` }; + } + if (failurePathLogs.length > 0) { + return { + id: ID, + status: "fail", + detail: "logs its failure without naming an environment, organization, project or user", + }; + } + return { + id: ID, + status: "fail", + detail: + ep.catches.length > 0 + ? "keeps its failures and records nothing about whose they were" + : "leaves its failures to the central handler, which names no tenant", + }; + }, +}; diff --git a/internal-packages/observability-map/src/cli.test.ts b/internal-packages/observability-map/src/cli.test.ts new file mode 100644 index 000000000..04d8f9121 --- /dev/null +++ b/internal-packages/observability-map/src/cli.test.ts @@ -0,0 +1,221 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { main, type Io } from "./cli.js"; + +/** + * A routes tree of this package's own making. Run against `apps/webapp/app/routes`, these asserted the + * webapp's contents rather than this CLI, and a route rename broke them for whoever pushed next. The + * one deliberate real-tree test lives in `integration.test.ts`. + */ +const ROUTES = mkdtempSync(join(tmpdir(), "obs-map-fixture-routes-")); + +const withWork = (name: string) => `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.${name}.findMany(); } catch (e) { return null; } + }`; + +const FIXTURES: Record = { + // Exact target, and also the prefix of the two below it. + "api.v1.runs.ts": withWork("run"), + "api.v1.runs.$runId.ts": withWork("run"), + "api.v1.runs.$runId.cancel.ts": withWork("run"), + "api.v1.token.ts": withWork("token"), + // Five routes sharing a prefix that is not itself a route, for the ambiguity warning and for the + // "and N more" tail it grows past four matches. + "admin.api.v1.runs-replication.start.ts": withWork("replication"), + "admin.api.v1.runs-replication.stop.ts": withWork("replication"), + "admin.api.v1.runs-replication.status.ts": withWork("replication"), + "admin.api.v1.runs-replication.retry.ts": withWork("replication"), + "admin.api.v1.runs-replication.purge.ts": withWork("replication"), + // Nothing applicable: exercises the not-measured note. + "resources.health.ts": `export const loader = () => new Response("ok");`, + // A directive whose id names no check, for the warning it has to produce. + "api.v1.typo.ts": `// obs-map-disable eror-classification -- typo\n${withWork("thing")}`, +}; + +beforeAll(() => { + for (const [name, source] of Object.entries(FIXTURES)) { + writeFileSync(join(ROUTES, name), source); + } + // A directory route, so the fixture covers both shapes `scanDirectory` walks. + mkdirSync(join(ROUTES, "_app.orgs.$slug")); + writeFileSync(join(ROUTES, "_app.orgs.$slug", "route.tsx"), withWork("organization")); +}); + +afterAll(() => rmSync(ROUTES, { recursive: true, force: true })); + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "cli.js", `--routes=${ROUTES}`, ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +describe("map ", () => { + // I8. The report prints route paths, so the identifier on screen has to be one you can paste + // back in. Matching file names only meant `map /api/v1/token` exited 1. + it("accepts the route path the report prints", () => { + const r = run("/api/v1/token"); + expect(r.code).toBe(0); + expect(r.out).toContain("/api/v1/token"); + expect(r.out).toContain("CHECKS"); + }); + + it("accepts a file name too", () => { + const r = run("api.v1.token.ts"); + expect(r.code).toBe(0); + expect(r.out).toContain("api.v1.token.ts"); + }); + + it("accepts a directory route by the path its directory segment spells", () => { + const r = run("/_app/orgs/:slug"); + expect(r.code).toBe(0); + expect(r.out).toContain("_app.orgs.$slug/route.tsx"); + }); + + it("exits 1 with a message when nothing matches", () => { + const r = run("/api/v1/does-not-exist"); + expect(r.code).toBe(1); + expect(r.err).toContain("no entry point matching"); + }); + + it("warns when a prefix matches more than one route rather than silently taking the first", () => { + const r = run("/admin/api/v1/runs-replication"); + expect(r.code).toBe(0); + expect(r.err).toMatch(/matches 5 entry points, showing the first/); + expect(r.err).toContain("Others:"); + expect(r.err).toContain("and 1 more"); + }); + + // `/api/v1/runs` is a prefix of two others in the fixture, and also a route in its own right. + it("prefers an exact match over the routes it is a prefix of", () => { + const r = run("/api/v1/runs"); + expect(r.err).toBe(""); + expect(r.out.split("\n")[1]).toBe("api.v1.runs.ts"); + }); + + it("says so rather than printing a bare 100 when nothing applied", () => { + const r = run("/resources/health"); + expect(r.code).toBe(0); + expect(r.out).toContain("not measured"); + }); + + // B7. `map /api/v1/token --json` printed the text format and dropped the flag on the floor. + it("honours --json for a single route instead of printing the text format", () => { + const r = run("/api/v1/token", "--json"); + expect(r.code).toBe(0); + expect(r.out).not.toContain("CHECKS"); + const parsed = JSON.parse(r.out); + expect(parsed.fileName).toBe("api.v1.token.ts"); + expect(parsed.routePath).toBe("/api/v1/token"); + expect(Array.isArray(parsed.checks)).toBe(true); + }); +}); + +describe("map", () => { + // The flag is the only thing standing between a run and a written report, so the test has to + // check the file, not just the exit code. `--out` keeps that file in a temp directory: this test + // used to delete `observability-map.json` from the repo root and never put it back. + it("renders the whole report without writing when asked not to", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); + + const r = run("--out=" + out, "--no-write"); + + expect(r.code).toBe(0); + expect(r.out).toContain("COVERAGE"); + expect(r.out).toContain("FIX FIRST"); + expect(existsSync(out)).toBe(false); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("writes the report where --out names it when not asked to skip the write", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-out-")); + const out = join(dir, "report.json"); + + const r = run("--out=" + out); + + expect(r.code).toBe(0); + expect(existsSync(out)).toBe(true); + const parsed = JSON.parse(readFileSync(out, "utf8")); + expect(parsed.entries.length).toBe(Object.keys(FIXTURES).length + 1); + + rmSync(dir, { recursive: true, force: true }); + }); +}); + +// B6. Stdout can be JSON a caller parses, so the warning goes to stderr whenever stdout is JSON. +// The terminal report carries it in its body instead (`src/report/terminal.test.ts`), and printing +// it on both streams meant a plain run showed every warning twice. +describe("warning about a suppression that names no check", () => { + it("names the file and the bad id in the whole report", () => { + const r = run("--no-write"); + expect(r.code).toBe(0); + expect(r.out).toContain("api.v1.typo.ts"); + expect(r.out).toContain("eror-classification"); + }); + + it("prints the warning once for a terminal run of the whole report", () => { + const r = run("--no-write"); + const lines = `${r.out}${r.err}`.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION")); + expect(lines).toHaveLength(1); + }); + + it("names the file and the bad id on stderr when the whole report is json", () => { + const r = run("--no-write", "--json"); + expect(r.code).toBe(0); + expect(r.err).toContain("api.v1.typo.ts"); + expect(r.err).toContain("eror-classification"); + expect(r.out).not.toContain("UNKNOWN SUPPRESSION"); + }); + + it("warns for a single route without putting the warning in the json", () => { + const r = run("/api/v1/typo", "--json"); + expect(r.code).toBe(0); + expect(r.err).toContain("eror-classification"); + expect(JSON.parse(r.out).fileName).toBe("api.v1.typo.ts"); + }); + + it("says nothing on stderr for a route whose directives all name a check", () => { + const r = run("/api/v1/token"); + expect(r.err).toBe(""); + }); +}); + +describe("map --routes=", () => { + it("scans the directory it names instead of the repo's routes tree", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-routes-")); + writeFileSync( + join(dir, "resources.only.ts"), + `export const loader = () => new Response("ok");` + ); + + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir, "--json", "--no-write"], c.io); + + expect(code).toBe(0); + const parsed = JSON.parse(c.out()); + expect(parsed.entries).toHaveLength(1); + expect(parsed.entries[0].fileName).toBe("resources.only.ts"); + + rmSync(dir, { recursive: true, force: true }); + }); + + it("exits 1 with a message when the directory does not exist", () => { + const dir = join(tmpdir(), "obs-map-routes-does-not-exist"); + const c = capture(); + const code = main(["node", "cli.js", "--routes=" + dir], c.io); + + expect(code).toBe(1); + expect(c.err()).toContain("not a readable directory"); + expect(c.out()).toBe(""); + }); +}); diff --git a/internal-packages/observability-map/src/cli.ts b/internal-packages/observability-map/src/cli.ts new file mode 100644 index 000000000..a54995e19 --- /dev/null +++ b/internal-packages/observability-map/src/cli.ts @@ -0,0 +1,151 @@ +import { existsSync, statSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { EntryPoint } from "./types.js"; +import { scanDirectory } from "./scan.js"; +import { buildReport, scoreEntry } from "./score.js"; +import { routePathOf } from "./adapters/remix.js"; +import { + renderTerminal, + unknownSuppressionLine, + unknownSuppressionLines, +} from "./report/terminal.js"; +import { renderJson } from "./report/json.js"; + +const DEFAULT_ROUTES = "apps/webapp/app/routes"; + +/** + * Walks up looking for `pnpm-workspace.yaml`, so the routes directory resolves whether `map` runs from + * the repo root or from the package directory, where `pnpm --filter` puts you. + */ +function findRepoRoot(startDir: string): string { + let dir = startDir; + for (let i = 0; i < 10; i++) { + if (existsSync(resolve(dir, "pnpm-workspace.yaml"))) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + throw new Error("could not find repo root (no pnpm-workspace.yaml in any parent directory)"); +} + +/** Where output goes. Injectable so the tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** + * Entry points matching what the user typed, by file name or by route path, exact first. Route paths + * matter because they are what the report prints, so the identifier on screen is one you can paste + * back in. + */ +function findMatches(entryPoints: EntryPoint[], target: string): EntryPoint[] { + const asPath = target.startsWith("/") ? target : `/${target}`; + const asFile = target.replace(/^\//, ""); + + const exact = entryPoints.filter( + (e) => e.fileName === target || routePathOf(e.fileName) === asPath + ); + if (exact.length > 0) return exact; + + return entryPoints.filter( + (e) => e.fileName.startsWith(asFile) || routePathOf(e.fileName).startsWith(asPath) + ); +} + +/** Value of a `--flag=value` argument, or null when the flag is absent. */ +function flagValue(args: string[], flag: string): string | null { + const prefix = `${flag}=`; + const found = args.find((a) => a.startsWith(prefix)); + return found === undefined ? null : found.slice(prefix.length); +} + +export function main(argv: string[], io: Io = processIo): number { + const args = argv.slice(2); + const asJson = args.includes("--json"); + const noWrite = args.includes("--no-write"); + const target = args.find((a) => !a.startsWith("--")); + + const repoRoot = findRepoRoot(dirname(fileURLToPath(import.meta.url))); + const routesFlag = flagValue(args, "--routes"); + + let routesDir: string; + if (routesFlag !== null) { + routesDir = resolve(process.cwd(), routesFlag); + let isDir = false; + try { + isDir = statSync(routesDir).isDirectory(); + } catch { + isDir = false; + } + if (!isDir) { + io.err(`--routes: not a readable directory: ${routesDir}\n`); + return 1; + } + } else { + routesDir = resolve(repoRoot, DEFAULT_ROUTES); + } + + const { entryPoints, parseFailures } = scanDirectory(routesDir); + + if (target) { + const matches = findMatches(entryPoints, target); + if (matches.length === 0) { + io.err(`no entry point matching "${target}"\n`); + return 1; + } + if (matches.length > 1) { + const others = matches.slice(1, 4).map((m) => routePathOf(m.fileName)); + const rest = matches.length - 1 - others.length; + io.err( + `"${target}" matches ${matches.length} entry points, showing the first. ` + + `Others: ${others.join(", ")}${rest > 0 ? `, and ${rest} more` : ""}\n` + ); + } + const scored = scoreEntry(matches[0]!); + // On stderr in both formats: a warning on stdout would be inside the JSON a caller parses. + if (scored.unknownSuppressions.length > 0) { + io.err(`${unknownSuppressionLine(scored.fileName, scored.unknownSuppressions)}\n`); + } + if (asJson) { + io.out(`${JSON.stringify(scored, null, 2)}\n`); + return 0; + } + const measuredNote = scored.measured ? "" : " (not measured: no applicable checks)"; + io.out( + `${scored.routePath} ${scored.score}/100${measuredNote}\n${scored.fileName}\n\nCHECKS\n` + ); + for (const c of scored.checks) { + const mark = c.status === "pass" ? "PASS" : c.status === "fail" ? "FAIL" : "n/a "; + io.out(` ${mark} ${c.id}${c.detail ? ` (${c.detail})` : ""}\n`); + } + return 0; + } + + const report = buildReport(entryPoints, parseFailures); + // JSON only: `renderTerminal` already puts these lines in the report body, and a warning on stdout + // would be inside the document a caller parses. + if (asJson) for (const line of unknownSuppressionLines(report)) io.err(`${line}\n`); + io.out(asJson ? renderJson(report) : renderTerminal(report)); + io.out("\n"); + if (!noWrite) { + // `--out` exists so a test can point the write somewhere disposable rather than creating and + // deleting a file in the repo root. + const outFlag = flagValue(args, "--out"); + const outPath = + outFlag === null + ? resolve(repoRoot, "observability-map.json") + : resolve(process.cwd(), outFlag); + writeFileSync(outPath, renderJson(report)); + } + return 0; +} + +// Only when run as a program. Importing the module, which the tests do, must not scan the tree or +// write a report. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/docstringReferences.test.ts b/internal-packages/observability-map/src/docstringReferences.test.ts new file mode 100644 index 000000000..403850477 --- /dev/null +++ b/internal-packages/observability-map/src/docstringReferences.test.ts @@ -0,0 +1,176 @@ +import ts from "typescript"; +import { readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { CHECKS } from "./checks/index.js"; +import { MUTATIONS } from "./mutations.js"; + +/** + * Every test name a docstring in `src/` claims to be covered by must exist. The rule was asked for six + * times in prose and broken six times, so prose does not enforce itself. Exactly what is and is not + * checked, and the coverage holes that leaves: INTERNALS.md, "Tests, timeouts and CI". + */ + +const SRC = resolve(__dirname); +const TESTS = resolve(__dirname); +/** Excluded from the `files` scan below. Both live in `src/` for colocation, so the exclusion has to + * be by name rather than by directory. */ +const MUTATIONS_HELPER = resolve(SRC, "mutations.ts"); + +/** Kebab-case tokens that are domain vocabulary rather than a test or corpus name. Anything added here + * is a deliberate statement that the token names no test, and shows up in review as such. */ +const NOT_A_TEST_NAME = new Set([ + // A `CheckStatus` value. + "not-applicable", + // The directive spelling that was retired, named in `suppression.ts` to say it is not honoured. + "obs-map-disable-next-line", + // The worked example of a mistyped check id, which names no test by construction. + "eror-classification", + // A route path segment quoted in `sensitivity.ts`, part of the vocabulary that file is about. + "session-duration", +]); + +/** A backticked phrase this long or longer, with no code punctuation, is read as a test title. */ +const MINIMUM_TITLE_WORDS = 5; + +/** Characters that mean a backticked phrase is a code sample rather than a test title. */ +const CODE_PUNCTUATION = /[{}()[\];=<>"'`|&$/\\]|\.\.\.|\.tsx?\b/; + +function walkFiles(dir: string, suffix: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walkFiles(path, suffix, out); + else if (entry.name.endsWith(suffix)) out.push(path); + } + return out; +} + +/** Comment text with jsdoc line prefixes removed, so a backticked phrase that wrapped across two lines + * reads as one phrase rather than one with a stray asterisk in it. */ +function commentText(file: string): string { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const seen = new Set(); + const parts: string[] = []; + const visit = (node: ts.Node) => { + for (const range of ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + parts.push(source.slice(range.pos, range.end)); + } + ts.forEachChild(node, visit); + }; + visit(sf); + return parts.join("\n").replace(/\n\s*\*\s?/g, " "); +} + +/** Static titles from every `it`/`test`/`describe` call, including the literal chunks of a + * template-literal title, so a reference to part of a generated name still resolves. */ +function testTitles(): Set { + const titles = new Set(); + const add = (value: string) => { + const trimmed = value.replace(/\s+/g, " ").trim(); + if (trimmed.length > 0) titles.add(trimmed); + }; + for (const file of walkFiles(TESTS, ".test.ts")) { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const visit = (node: ts.Node) => { + if (ts.isCallExpression(node)) { + const callee = node.expression; + const root = ts.isPropertyAccessExpression(callee) ? callee.expression : callee; + if (ts.isIdentifier(root) && ["it", "test", "describe"].includes(root.text)) { + const first = node.arguments[0]; + if (first) { + if (ts.isStringLiteralLike(first)) add(first.text); + if (ts.isTemplateExpression(first)) { + add(first.head.text); + for (const span of first.templateSpans) add(span.literal.text); + } + } + } + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return titles; +} + +describe("docstrings in src name things that exist", () => { + const known = new Set([ + ...CHECKS.map((c) => c.id), + ...MUTATIONS.map((m) => m.id), + ...NOT_A_TEST_NAME, + ]); + const titles = testTitles(); + const corpusIds = MUTATIONS.map((m) => m.id); + const files = walkFiles(SRC, ".ts").filter( + (f) => !f.endsWith(".test.ts") && f !== MUTATIONS_HELPER + ); + + it("finds source files and test titles to check against", () => { + expect(files.length).toBeGreaterThan(5); + expect(titles.size).toBeGreaterThan(50); + expect(corpusIds.length).toBeGreaterThan(20); + }); + + it("every backticked kebab-case token names a check, a corpus entry or a test", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/g)) { + const token = match[1]!; + if (known.has(token)) continue; + if ([...titles].some((t) => t.includes(token))) continue; + unknown.push(`${file}: ${token}`); + } + } + expect(unknown).toEqual([]); + }); + + it("every backticked glob matches at least one corpus entry", () => { + const unmatched: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][a-z0-9-]*)-\*`/g)) { + const prefix = `${match[1]!}-`; + if (corpusIds.some((id) => id.startsWith(prefix))) continue; + unmatched.push(`${file}: ${prefix}*`); + } + } + expect(unmatched).toEqual([]); + }); + + it("every backticked prose phrase long enough to be a test title is one", () => { + const unknown: string[] = []; + for (const file of files) { + for (const match of commentText(file).matchAll(/`([a-z][^`\n]*)`/g)) { + const phrase = match[1]!.replace(/\s+/g, " ").trim(); + if (phrase.split(" ").length < MINIMUM_TITLE_WORDS) continue; + if (CODE_PUNCTUATION.test(phrase)) continue; + if (titles.has(phrase)) continue; + unknown.push(`${file}: ${phrase}`); + } + } + expect(unknown).toEqual([]); + }); + + // The checker has to be able to fail, or it is decoration. These run the same predicates over an + // invented docstring, so the guarantee does not rest on `src/` happening to contain a bad reference. + it("would reject a docstring naming a test that does not exist", () => { + const invented = "see `content-is-not-a-comment` for the proof"; + const token = /`([a-z][a-z0-9]*(?:-[a-z0-9]+)+)`/.exec(invented)![1]!; + expect(known.has(token)).toBe(false); + expect([...titles].some((t) => t.includes(token))).toBe(false); + }); + + it("would reject a docstring naming a corpus glob that matches nothing", () => { + const prefix = /`([a-z][a-z0-9-]*)-\*`/.exec("covered by `no-such-family-*` above")![1]!; + expect(corpusIds.some((id) => id.startsWith(`${prefix}-`))).toBe(false); + }); + + it("would reject a docstring naming a prose test title that does not exist", () => { + const phrase = "reads a directive that nobody ever wrote down anywhere"; + expect(phrase.split(" ").length).toBeGreaterThanOrEqual(MINIMUM_TITLE_WORDS); + expect(CODE_PUNCTUATION.test(phrase)).toBe(false); + expect(titles.has(phrase)).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/src/index.ts b/internal-packages/observability-map/src/index.ts new file mode 100644 index 000000000..1a8e99049 --- /dev/null +++ b/internal-packages/observability-map/src/index.ts @@ -0,0 +1,6 @@ +export { scanDirectory, scanFile } from "./scan.js"; +export { buildReport, scoreEntry } from "./score.js"; +export { renderTerminal } from "./report/terminal.js"; +export { renderJson } from "./report/json.js"; +export type { MapReport, ScoredEntry } from "./score.js"; +export type { EntryPoint, CheckResult, CheckStatus } from "./types.js"; diff --git a/internal-packages/observability-map/src/integration.test.ts b/internal-packages/observability-map/src/integration.test.ts new file mode 100644 index 000000000..de2147505 --- /dev/null +++ b/internal-packages/observability-map/src/integration.test.ts @@ -0,0 +1,405 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { isScannableFile, scanDirectory, scanFile } from "./scan.js"; +import { buildReport } from "./score.js"; +import { SCORED_CHECK_IDS } from "./checks/index.js"; + +/** + * This file's deliberate coupling to `apps/webapp/app/routes`, and not the suite's only one: + * `webappSymbols.test.ts` walks all of `apps/webapp/app` and two more trees, and + * `mutationCorpus.test.ts` scans the route tree behind an env gate. + * + * The coupling is acceptable because nothing here names a route or a count: the scan must not crash, + * the entry point count must sit inside a wide band, and parse failures must be zero. Those are the + * only things a fixture tree cannot tell us, since a fixture only contains shapes somebody thought to + * write down. How the whole suite is gated in CI: INTERNALS.md, "Tests, timeouts and CI". + */ +const ROUTES = resolve(__dirname, "../../../apps/webapp/app/routes"); + +/** + * Every `.ts`/`.tsx` file under the tree, at any depth. Deliberately not the scanner's walk: as a copy + * of it, `entryPoints.length < countCandidates()` was a tautology that could not fail for any route + * shape both of them missed. + */ +function countRouteModuleFiles(dir: string): number { + let count = 0; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + count += countRouteModuleFiles(join(dir, entry.name)); + continue; + } + if (entry.isFile() && isScannableFile(entry.name)) count++; + } + return count; +} + +beforeAll(() => { + // A hard failure rather than a silent skip: if this package moves relative to apps/webapp, the + // real-tree coverage must disappear loudly. + if (!existsSync(ROUTES)) { + throw new Error(`the webapp routes directory is missing: ${ROUTES}`); + } +}); + +// The build emits ESM while `main` and `types` advertised it to a package.json with no module type, +// which on node 24 loads with a MODULE_TYPELESS_PACKAGE_JSON warning and a reparse. +describe("the package it advertises", () => { + const manifest = JSON.parse( + readFileSync(resolve(__dirname, "../package.json"), "utf8") + ) as Record; + + // Asserted flat rather than behind a skip, so dropping the entry point later has to edit this + // rather than slip past it. + it("declares the module type its build emits alongside the entry point it advertises", () => { + expect(manifest.main).toBe("./dist/src/index.js"); + expect(manifest.types).toBe("./dist/src/index.d.ts"); + expect(manifest.type).toBe("module"); + }); +}); + +const WORKFLOWS = resolve(__dirname, "../../../.github/workflows"); +const REPORT = resolve(WORKFLOWS, "observability-map.yml"); + +function read(path: string): string { + if (!existsSync(path)) throw new Error(`workflow is missing: ${path}`); + return readFileSync(path, "utf8"); +} + +/** Comment lines dropped, so an assertion is about what the YAML says rather than what its prose + * mentions. */ +const withoutComments = (text: string) => text.replace(/^\s*#.*$/gm, ""); + +/** One job's block, from its key to the next key at job indent. */ +function job(name: string): string { + const parts = read(REPORT).split(new RegExp(`^ {2}${name}:$`, "m")); + expect(parts).toHaveLength(2); + return parts[1]!.split(/^ {2}[a-z][a-z-]*:$/m)[0]!; +} + +/** A job's condition and everything else it declares before its steps. */ +const gate = (name: string) => job(name).split(" steps:")[0]!; + +/** Step bodies, split on the `- name:` lines, which is all the structure this needs. */ +const steps = (block: string) => block.split(/^ {6}- name: /m).slice(1); + +/** + * The one thing the docstring checker cannot reach, since it walks `src/` only. What the C1 defect was + * and what replaced it: INTERNALS.md, "Tests, timeouts and CI". These are text checks over the + * workflow + * rather than a parse of its semantics, so they catch the wiring coming apart and nothing about + * whether GitHub agrees. + */ +describe("the report workflow's one source of the comment id", () => { + it("does not start the report job at all unless the lookup finished cleanly", () => { + expect(gate("report")).toContain("needs.changes.outputs.lookup == 'ok'"); + }); + + it("gives the render and the upsert step the same output to read", () => { + const readers = steps(job("report")).filter((step) => step.includes("EXISTING_COMMENT")); + expect(readers.length).toBeGreaterThanOrEqual(2); + expect( + readers.filter( + (step) => !step.includes("EXISTING_COMMENT: ${{ needs.changes.outputs.comment }}") + ) + ).toEqual([]); + }); + + // The lookup is what lets the report job be gated, so it happens in the cheap job an unrelated pull + // request pays for anyway. + it("looks the comment up in the cheap job and not again in the report job", () => { + expect(job("changes")).toContain("issues/${PR_NUMBER}/comments"); + expect(job("changes")).toContain("pull-requests: read"); + expect(job("report")).not.toContain("--paginate"); + }); + + it("takes one id from a lookup that paginates rather than passing every line on", () => { + const lookup = steps(job("changes")).find((step) => + step.includes('startswith(""); + expect(renderPrComment(head, head).split("\n")[0]).toBe(""); + }); + + it("says the comparison is unavailable when base is null", () => { + const head = buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Base comparison unavailable."); + expect(out).not.toContain("no change"); + }); + + it("reports a score drop and the newly failing checks", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\(base \d+, down \d+\)/); + expect(out).toContain("| /api/v1/auth/tokens |"); + // request-context and error-classification regress; auth-boundary is not applicable here + // (no sensitivity signal on this route), so it must not show up as newly failing. + expect(out).toMatch(/\| \/api\/v1\/auth\/tokens \| \d+ \| \d+ \|[^|]*error-classification/); + }); + + it("reports a score improvement the other way round", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + expect(out).toMatch(/\(base \d+, up \d+\)/); + }); + + it("says nothing changed when every score matches", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + expect(out).toContain("No entry point this PR touches changed its score."); + expect(out).toContain("(base 100, no change)"); + }); + + it("shows a new entry with its base column as 'new' and lists its failing checks", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", brokenSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).toMatch(/\| \/api\/v1\/new \| new \| \d+ \|/); + }); + + // Mirrors the guard report.test.ts has for the terminal renderer: audit-trail fails almost + // every sensitive mutation today (no audit helper exists), so it is a headline figure, not a + // per-route nag. A regression here previously let it leak into the "now failing" column. + it("does not list audit-trail among a new sensitive entry's failing checks", () => { + const sensitiveMutation = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + const head = buildReport([sensitiveMutation], []); + const base = buildReport([], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/envvars |"))!; + expect(row).toBeDefined(); + expect(row).toContain("new"); + expect(row).not.toContain("audit-trail"); + expect(row).toMatch(/error-classification|auth-boundary|request-context/); + }); + + it("skips a new entry that passes every check it was measured against", () => { + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.new.ts", cleanSource)!], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + expect(out).not.toContain("/api/v1/new"); + expect(out).toContain("No entry point this PR touches changed its score."); + }); + + // B3. `score` is 100 for an entry no scored check applied to, and the table read that placeholder + // as a figure: a route refactored down to a trivial body rendered as a 67-point improvement. + describe("an unmeasured entry", () => { + const trivial = `export const loader = () => new Response("ok");`; + + it("renders as not measured rather than as 100 when the head stopped being measurable", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toContain("not measured"); + expect(row).not.toMatch(/\|\s*100\s*\|/); + }); + + it("names the base score when it is the head that has none, not the base", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + expect(head.global).toBeNull(); + expect(base.global).not.toBeNull(); + + const out = renderPrComment(head, base); + expect(out).toContain(`(base ${base.global})`); + expect(out).not.toContain("(base not measured)"); + }); + + it("still says the base is the missing one when it really is", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + expect(base.global).toBeNull(); + + expect(renderPrComment(head, base)).toContain("(base not measured)"); + }); + + it("renders as not measured in the base column when the head gained real work", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| not measured \| \d+ \|/); + }); + + // The early-out compared scores only, so a measured 100 turning into an unmeasured placeholder + // 100 produced no row at all: the table said nothing happened. + it("still produces a row when a measured 100 becomes an unmeasured placeholder 100", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", trivial)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + expect(base.entries[0]!.score).toBe(100); + expect(head.entries[0]!.score).toBe(100); + + const out = renderPrComment(head, base); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/auth/tokens |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| 100 \| not measured \|/); + }); + + // Not the same statement as a new entry that passes everything, which is skipped above. + it("still gets a row when it is new, since its 100 is a placeholder and not a pass", () => { + const head = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", cleanSource)!, + scanFile("resources.health.ts", trivial)!, + ], + [] + ); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const out = renderPrComment(head, base); + + const row = out.split("\n").find((l) => l.startsWith("| /resources/health |"))!; + expect(row).toBeDefined(); + expect(row).toMatch(/\| new \| not measured \|/); + }); + + it("does not sort an unmeasured transition above a real regression", () => { + const head = buildReport( + [scanFile("resources.gone.ts", trivial)!, scanFile("resources.busy.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("resources.gone.ts", brokenSource)!, scanFile("resources.busy.ts", cleanSource)!], + [] + ); + const out = renderPrComment(head, base); + + const busy = out.indexOf("/resources/busy"); + const gone = out.indexOf("/resources/gone"); + expect(busy).toBeGreaterThan(-1); + expect(gone).toBeGreaterThan(-1); + expect(busy).toBeLessThan(gone); + }); + }); + + // I4. A suppression added to a check that was PASSING drops the score by round A's cap and + // produces a row with an empty "now failing" column, sorted among the real regressions. On the + // real tree `_app.@.orgs.$organizationSlug.$.tsx` renders 67 to 50 exactly that way. + describe("a row a suppression caused", () => { + // Two of three applicable scored checks pass, so suppressing one of the passes takes the + // visible ratio from 2/3 to 1/2, which is the 67 to 50 the real tree renders on + // `_app.@.orgs.$organizationSlug.$.tsx`. The catch has to decide something for + // error-classification to apply at all, and nothing may name a tenant, or the ratio is 3/3. + const twoOfThree = `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } + }`; + const silence = (id: string, source: string) => + `// obs-map-disable ${id} -- silenced\n${source}`; + + it("says so on the route, so it is not read as a regression", () => { + const base = buildReport([scanFile("api.v1.auth.tokens.ts", twoOfThree)!], []); + const head = buildReport( + [scanFile("api.v1.auth.tokens.ts", silence("error-classification", twoOfThree))!], + [] + ); + expect(base.entries[0]!.score).toBe(67); + expect(head.entries[0]!.score).toBe(50); + + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + // The column that would otherwise explain the drop is empty, which is the whole problem. + expect(row.split("|")[4]!.trim()).toBe(""); + }); + + // I3. This one moves no score at all, so before the suppression set was compared it produced + // no row and no comment: a pull request whose whole purpose is to silence findings was silent. + it("appears even when suppressing an already-failing check moved no score", () => { + const base = buildReport([scanFile("api.v1.t.ts", brokenSource)!], []); + const head = buildReport( + [scanFile("api.v1.t.ts", silence("error-classification", brokenSource))!], + [] + ); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.global).toBe(base.global); + + const out = renderPrComment(head, base); + expect(out).not.toContain("No entry point this PR touches changed its score."); + const row = out.split("\n").find((l) => l.startsWith("| /api/v1/t "))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: error-classification)"); + }); + + it("says nothing about suppression on a row that has none", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", brokenSource)!], []); + const base = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/auth/tokens"))!; + expect(row).not.toContain("suppressed:"); + }); + + it("gives a new entry that lands at 100 only because of a suppression a row", () => { + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const head = buildReport( + [ + scanFile("api.v1.a.ts", cleanSource)!, + scanFile("api.v1.new.ts", silence("request-context", cleanSource))!, + ], + [] + ); + const row = renderPrComment(head, base) + .split("\n") + .find((l) => l.startsWith("| /api/v1/new"))!; + expect(row).toBeDefined(); + expect(row).toContain("(suppressed: request-context)"); + }); + }); + + // M5. This section rendered one line per file, and a tree-wide typo took the comment past + // GitHub's 65,536 character limit for a 422 nobody sees. + it("caps the unknown-suppression lines instead of running past the comment size limit", () => { + const entries = []; + for (let i = 0; i < 40; i++) { + entries.push( + scanFile( + `api.v1.route${i}.ts`, + `// obs-map-disable eror-classification -- typo\n${brokenSource}` + )! + ); + } + const head = buildReport(entries, []); + const out = renderPrComment(head, null); + + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(10); + expect(out).toContain("and 30 more files with unknown ids"); + expect(out.length).toBeLessThan(65536); + }); + + // Round E item 1. The same unbounded-section failure, in the one other section that grows with + // the tree. A codemod moving route bodies into `.server.ts` modules is the refactor `delegating` + // exists to notice, and it is what makes this list tree-sized. + it("caps the delegated route list instead of running past the comment size limit", () => { + const entries = []; + for (let i = 0; i < 400; i++) { + const padding = `route-with-a-realistically-long-name-${String(i).padStart(4, "0")}`; + entries.push( + scanFile( + `_app.orgs.$organizationSlug.projects.$projectParam.${padding}/route.tsx`, + `export { action } from "./handler.server";` + )! + ); + } + const head = buildReport(entries, []); + const line = renderPrComment(head, null) + .split("\n") + .find((l) => l.startsWith("DELEGATED"))!; + + expect(line).toContain("400 routes"); + expect(line).toContain(", and 385 more"); + expect(line.match(/\/route\.tsx/g)).toHaveLength(15); + expect(renderPrComment(head, null).length).toBeLessThan(65536); + }); + + // The terminal has no size limit to respect, so the cap must not reach it. + it("leaves the terminal report naming every delegating route", () => { + const entries = []; + for (let i = 0; i < 40; i++) { + entries.push(scanFile(`webhooks.v1.hook${i}.ts`, `export { action } from "./h.server";`)!); + } + const line = renderTerminal(buildReport(entries, [])) + .split("\n") + .find((l) => l.startsWith("DELEGATED"))!; + expect(line).toContain("webhooks.v1.hook39.ts"); + expect(line).not.toContain("more"); + }); + + it("sorts a sensitive entry with a small drop above a non-sensitive entry with a large drop", () => { + const sensitiveSmallDropBase = scanFile("api.v1.auth.tokens.ts", cleanSource)!; + const sensitiveSmallDropHead = scanFile( + "api.v1.auth.tokens.ts", + `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { error }); throw error; } + }` + )!; + + const notSensitiveLargeDropBase = scanFile( + "resources.busy.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const notSensitiveLargeDropHead = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + const head = buildReport([sensitiveSmallDropHead, notSensitiveLargeDropHead], []); + const base = buildReport([sensitiveSmallDropBase, notSensitiveLargeDropBase], []); + const out = renderPrComment(head, base); + + const sensitiveIndex = out.indexOf("/api/v1/auth/tokens"); + const notSensitiveIndex = out.indexOf("/resources/busy"); + expect(sensitiveIndex).toBeGreaterThan(-1); + expect(notSensitiveIndex).toBeGreaterThan(-1); + expect(sensitiveIndex).toBeLessThan(notSensitiveIndex); + }); + + it("reports a removed entry as a count line, not a row", () => { + const head = buildReport([scanFile("api.v1.auth.tokens.ts", cleanSource)!], []); + const base = buildReport( + [scanFile("api.v1.auth.tokens.ts", cleanSource)!, scanFile("api.v1.gone.ts", brokenSource)!], + [] + ); + const out = renderPrComment(head, base); + + expect(out).toContain("1 entries removed"); + expect(out).not.toContain("/api/v1/gone"); + }); + + it("caps the changed-entries table at 15 rows and says how many more", () => { + const headEntries = []; + const baseEntries = []; + for (let i = 0; i < 20; i++) { + headEntries.push(scanFile(`api.v1.route${i}.ts`, brokenSource)!); + baseEntries.push(scanFile(`api.v1.route${i}.ts`, cleanSource)!); + } + const head = buildReport(headEntries, []); + const base = buildReport(baseEntries, []); + const out = renderPrComment(head, base); + + const rows = out.split("\n").filter((l) => l.startsWith("| /api/v1/route")); + expect(rows).toHaveLength(15); + expect(out).toContain("and 5 more"); + }); + + it("warns about parse failures in either report, since they shrink the denominator", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-head.ts"]); + const base = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken-base.ts"]); + const out = renderPrComment(head, base); + expect(out).toMatch(/Warning: parse failures \(1 at head, 1 at base\)/); + }); + + it("does not warn about parse failures when there are none", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + expect(renderPrComment(head, null)).not.toContain("Warning: parse failures"); + }); + + it("footer names the report-only rule and the readme", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], []); + const out = renderPrComment(head, null); + expect(out).toContain("Report only, nothing here gates the merge."); + expect(out).toContain("internal-packages/observability-map/README.md"); + }); +}); + +/** + * The comment is edited in place across pushes, so every comment the job posts carries the head sha as + * a link to the compare range. The commit arrives as data, so these renderers stay pure. + */ +describe("the commit stamp", () => { + const COMMIT = { + sha: "0123456789abcdef0123456789abcdef01234567", + url: "https://github.com/triggerdotdev/trigger.dev/compare/1111111...2222222", + }; + const STAMP = + "As of [`0123456`](https://github.com/triggerdotdev/trigger.dev/compare/1111111...2222222)."; + const head = () => buildReport([scanFile("api.v1.a.ts", brokenSource)!], []); + + it("stamps the report comment under the heading, before anything the report says", () => { + const lines = renderPrComment(head(), null, COMMIT).split("\n"); + expect(lines[2]).toBe("## Observability map"); + expect(lines[4]).toBe(STAMP); + }); + + it("stamps the resolved comment, which is the one a stale report gets replaced with", () => { + const lines = renderResolvedComment(COMMIT).split("\n"); + expect(lines[4]).toBe(STAMP); + expect(lines.join("\n")).toContain("Nothing in this pull request moves the report any more."); + }); + + it("stamps the stale-report comment", () => { + expect(renderScanFailedComment(COMMIT).split("\n")[4]).toBe(STAMP); + }); + + it("shortens the sha to seven characters for the link text, not the target", () => { + expect(renderPrComment(head(), null, COMMIT)).toContain("[`0123456`]("); + expect(renderPrComment(head(), null, COMMIT)).not.toContain("[`01234567`]"); + }); + + // The optional half. Unit tests and a local CLI run have no commit context, and rendering has to + // work without one rather than printing a link to nowhere. + it("renders every comment without a stamp when there is no commit context", () => { + for (const out of [ + renderPrComment(head(), null), + renderResolvedComment(), + renderScanFailedComment(), + ]) { + expect(out).not.toContain("As of ["); + expect(out.split("\n")[2]).toBe("## Observability map"); + } + }); +}); + +// B4. The job posts only when the pull request moves the report, so the decision has to be a +// tested function of the two reports rather than shell logic in the workflow. +/** + * `hasDelta` compares no `sensitive` field, and does not need one. Sensitivity reaches the rendered + * comment only through `fixFirstSection`, whose primary sort key it is, and a route can only appear + * there with a scored failure. A scored failure needs a try/catch in the body, `isTrivialExport` + * rejects any export that has one, and a sensitive non-trivial export is therefore either accused + * (`fail`) or guarded (`pass`) on `auth-boundary`, never `not-applicable`. So a flip that could move + * the fix list always moves `auth-boundary`, which moves `checkContributions`, which is compared. + * + * Both halves are pinned here because the argument rests on that coupling: make a trivial route + * capable of a scored failure and the first case below starts rendering a difference `hasDelta` cannot + * see. + */ +describe("a sensitivity flip", () => { + const stub = `import { redirect } from "@remix-run/server-runtime"; + export const loader = () => redirect("/");`; + const stubSensitive = `import { redirect } from "@remix-run/server-runtime"; + import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export const loader = () => redirect("/");`; + + it("renders nothing different on a route too trivial to reach the fix list", () => { + const base = buildReport([scanFile("resources.stub.ts", stub)!], []); + const head = buildReport([scanFile("resources.stub.ts", stubSensitive)!], []); + expect(base.entries[0]!.sensitive).toBe(false); + expect(head.entries[0]!.sensitive).toBe(true); + + expect(renderPrComment(head, base)).toBe(renderPrComment(base, base)); + expect(hasDelta(head, base)).toBe(false); + }); + + it("moves auth-boundary, and so is caught, on a route that does real work", () => { + const working = `export async function loader() { + try { compute(); } catch (e) { return null; } + }`; + const workingSensitive = `import { createPersonalAccessToken } from "~/services/personalAccessToken.server"; + export async function loader() { + try { compute(); } catch (e) { return null; } + }`; + const base = buildReport([scanFile("resources.work.ts", working)!], []); + const head = buildReport([scanFile("resources.work.ts", workingSensitive)!], []); + + const status = (r: typeof base) => + r.entries[0]!.checks.find((c) => c.id === "auth-boundary")!.status; + expect(status(base)).toBe("not-applicable"); + expect(status(head)).toBe("fail"); + expect(hasDelta(head, base)).toBe(true); + }); +}); + +describe("hasDelta", () => { + const trivial = `export const loader = () => new Response("ok");`; + const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []); + + it("is true when there is no base to compare against", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), null)).toBe(true); + }); + + it("is false for two identical reports", () => { + expect(hasDelta(one("api.v1.a.ts", cleanSource), one("api.v1.a.ts", cleanSource))).toBe(false); + }); + + it("is true when the global score moved", () => { + expect(hasDelta(one("api.v1.a.ts", brokenSource), one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was added", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + it("is true when an entry was removed", () => { + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + expect(hasDelta(one("api.v1.a.ts", cleanSource), base)).toBe(true); + }); + + // The global is a mean over measured entries, so two entries moving in opposite directions can + // leave it where it was. The per-entry comparison is what catches that. + it("is true when an entry's score moved but the global mean did not", () => { + const head = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", cleanSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", cleanSource)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + // audit-trail does not feed the score, so it can start failing without moving a single figure. + it("is true when an unscored check started failing and no score moved", () => { + const head = one("api.v1.envvars.ts", cleanSource); + const base = one( + "api.v1.envvars.ts", + `// obs-map-disable audit-trail -- no helper exists yet\n${cleanSource}` + ); + expect(head.global).toBe(base.global); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when an entry stopped being measured at the same placeholder score", () => { + const head = one("api.v1.a.ts", trivial); + const base = one("api.v1.a.ts", cleanSource); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when a parse failure appeared, since the comment warns about it", () => { + const head = buildReport([scanFile("api.v1.a.ts", cleanSource)!], ["broken.ts"]); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); + + // I3. These three are the half that was missing, and it ran the dangerous way: a pull request + // that only silences findings posted nothing, while a mistyped directive did post. + it("is true when a suppression was added to a check that was already failing", () => { + const base = one("api.v1.t.ts", brokenSource); + const head = one( + "api.v1.t.ts", + `// obs-map-disable error-classification -- silenced\n${brokenSource}` + ); + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.measured).toBe(base.measured); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when the audit gap closed, which no score reports", () => { + const audited = `import { clearImpersonation } from "~/models/admin.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await clearImpersonation(request, "/admin"); + return json(token); + }`; + const unaudited = `import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + return json(token); + }`; + const head = one("api.v1.auth.tokens.ts", audited); + const base = one("api.v1.auth.tokens.ts", unaudited); + expect(head.auditGap).not.toEqual(base.auditGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // The CONTEXT line reads pre-suppression data, so with request-context suppressed its figure can + // move while the post-suppression checks, the score and the global all stay put. The comment + // says "0 of 1" and then "1 of 1"; nothing else in the report moves at all. + it("is true when the context figure moved behind a suppression", () => { + const silence = "// obs-map-disable request-context -- reported as a figure\n"; + const namesNobody = `${silence}import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.envVar.update({ where: {}, data: {} }); } catch (e) { return null; } + }`; + const namesTenant = `${silence}import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ params }) { + try { return await prisma.envVar.update({ where: {}, data: {} }); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); return null; } + }`; + const head = one("api.v1.envvars.ts", namesTenant); + const base = one("api.v1.envvars.ts", namesNobody); + + expect(head.global).toBe(base.global); + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(head.entries[0]!.suppressed).toEqual(base.entries[0]!.suppressed); + expect(head.contextGap).not.toEqual(base.contextGap); + expect(hasDelta(head, base)).toBe(true); + }); + + // Suppressing a check that was not applicable anyway changes no status and no score, and moving + // that suppression between two entries leaves the report-level totals identical too. The + // per-entry suppression comparison is the only thing left that can see it. + it("is true when a suppression of an inapplicable check moved between entries", () => { + const silenced = `// obs-map-disable auth-boundary -- not a user-facing route\n${brokenSource}`; + const head = buildReport( + [scanFile("api.v1.a.ts", silenced)!, scanFile("api.v1.b.ts", brokenSource)!], + [] + ); + const base = buildReport( + [scanFile("api.v1.a.ts", brokenSource)!, scanFile("api.v1.b.ts", silenced)!], + [] + ); + expect(head.suppressions).toEqual(base.suppressions); + expect(head.global).toBe(base.global); + expect(head.entries.map((e) => e.score)).toEqual(base.entries.map((e) => e.score)); + const statuses = (r: typeof head) => + r.entries.map((e) => e.checks.map((c) => `${c.id}=${c.status}`).join(" ")); + expect(statuses(head)).toEqual(statuses(base)); + expect(hasDelta(head, base)).toBe(true); + }); + + it("is true when a suppression names a check that does not exist", () => { + const head = one( + "api.v1.a.ts", + `// obs-map-disable eror-classification -- typo\n${cleanSource}` + ); + expect(hasDelta(head, one("api.v1.a.ts", cleanSource))).toBe(true); + }); +}); + +// C4b and C5. Both are rendered, so both have to move `hasDelta`, and neither is reachable through +// the per-entry score comparison the loop makes. +describe("hasDelta: the figures round C added", () => { + const one = (name: string, source: string) => buildReport([scanFile(name, source)!], []); + const delegated = `export { action } from "./handler.server";`; + + it("is true when a route started delegating its body", () => { + const head = one("webhooks.v1.stripe.ts", delegated); + const base = one("webhooks.v1.stripe.ts", brokenSource); + expect(head.delegating).toEqual(["webhooks.v1.stripe.ts"]); + expect(hasDelta(head, base)).toBe(true); + }); + + // The shape the per-entry loop cannot see: the score stays where it was and what the score is + // made of changed underneath it. + it("is true when a check stopped applying without moving any entry's score", () => { + const head = one("api.v1.auth.tokens.ts", cleanSource); + const base = one("api.v1.auth.tokens.ts", cleanSource); + const contribution = base.checkContributions.find((c) => c.id === "auth-boundary")!; + contribution.applicable = contribution.applicable + 1; + expect(head.entries[0]!.score).toBe(base.entries[0]!.score); + expect(hasDelta(head, base)).toBe(true); + }); + + it("says the comment renders the delegated line it is comparing", () => { + const head = one("webhooks.v1.stripe.ts", delegated); + expect(renderPrComment(head, null)).toContain("DELEGATED"); + }); + + it("says the comment renders the check contributions it is comparing", () => { + const head = one("api.v1.auth.tokens.ts", cleanSource); + expect(renderPrComment(head, null)).toContain("What the score is made of"); + }); +}); diff --git a/internal-packages/observability-map/src/report/prComment.ts b/internal-packages/observability-map/src/report/prComment.ts new file mode 100644 index 000000000..beef3818f --- /dev/null +++ b/internal-packages/observability-map/src/report/prComment.ts @@ -0,0 +1,339 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { + auditLine, + checkContributionLines, + contextLine, + fixFirst, + delegatedLines, + scoredFailures, + unknownSuppressionLines, +} from "./terminal.js"; + +/** First line of every comment this job posts, so the upsert step can find its own comment again. */ +export const MARKER = ""; + +/** + * The commit a comment was rendered for. Data rather than something the renderers read for themselves, + * so they stay pure and a run with no commit context renders the same comment without the line. + */ +export type CommitContext = { + /** The head sha, full. Shortened for the link text here rather than by the caller. */ + sha: string; + /** Compare URL for the pull request's range, base to head. */ + url: string; +}; + +const SHORT_SHA_LENGTH = 7; + +/** Directly under the heading, because the comment is edited in place across pushes and the first + * question about it is which push it reflects. */ +function commitLines(commit: CommitContext | undefined): string[] { + if (!commit) return []; + return [`As of [\`${commit.sha.slice(0, SHORT_SHA_LENGTH)}\`](${commit.url}).`, ""]; +} + +const MAX_CHANGED_ROWS = 15; + +/** A mistyped directive applied tree wide rendered 87,938 characters against GitHub's 65,536 limit, + * so the whole comment was lost to the section warning about a typo. */ +const MAX_UNKNOWN_SUPPRESSION_LINES = 10; + +/** + * The same failure in the other section that grows with the tree, since a codemod moving route bodies + * into `.server.ts` modules is both the refactor this feature exists to notice and the one that makes + * the list tree-sized. Fifteen rather than the ten above because a file name is one comma-separated + * item: the longest in the tree is 130 characters, so fifteen of those is under 2kB. + */ +const MAX_DELEGATED_ROUTES = 15; + +// Scored checks only, the same exclusion `scoredFailures` makes. +const failingIds = (e: ScoredEntry) => scoredFailures(e).map((c) => c.id); + +function scoreLine(head: MapReport, base: MapReport | null): string { + const headline = + head.global === null + ? `not measured over ${head.measured} measured of ${head.entries.length} entry points` + : `**${head.global}/100** over ${head.measured} measured of ${head.entries.length} entry points`; + + if (!base) return headline; + // Head first: when this side has no score the headline already says so, and naming the base as the + // missing one sends the reader to the wrong commit. + if (head.global === null) return `${headline} (base ${base.global ?? "not measured"})`; + if (base.global === null) return `${headline} (base not measured)`; + + const diff = head.global - base.global; + const comparison = + diff === 0 + ? `(base ${base.global}, no change)` + : diff > 0 + ? `(base ${base.global}, up ${diff})` + : `(base ${base.global}, down ${-diff})`; + return `${headline} ${comparison}`; +} + +/** What goes in a score column: a figure, an absence, or no prior entry to compare against. */ +const NOT_MEASURED = "not measured"; + +/** + * `score` is a placeholder 100 for an entry no scored check applied to. Rendering that as a figure + * turned a route refactored down to a trivial body into a 67-point improvement, and a trivial route + * gaining real work into the PR's worst regression. + */ +const scoreCell = (e: ScoredEntry): number | string => (e.measured ? e.score : NOT_MEASURED); + +/** Ids suppressed at head that were not suppressed at base. `[]` covers both "no change" and a + * suppression being removed, which shows up as the score going back up. */ +const newlySuppressed = (head: ScoredEntry, base: ScoredEntry | undefined): string[] => + head.suppressed.filter((id) => !(base?.suppressed ?? []).includes(id)); + +type ChangedRow = { + routePath: string; + sensitive: boolean; + baseScore: number | string; + headScore: number | string; + nowFailing: string[]; + /** Ids this pull request newly suppressed. Rendered on the route cell, because a suppression added + * to a passing check drops the score by the cap and produces a row with an empty "now failing" + * column, indistinguishable from a real regression. */ + suppressed: string[]; + /** How much the entry got worse, used to sort the table. A new entry is scored against a perfect + * 100, so one landing at 60 sorts with an existing one that dropped 40. Zero whenever either side + * is unmeasured, since there is no arithmetic to do between a figure and an absence. */ + drop: number; +}; + +function changedRows(head: MapReport, base: MapReport): { rows: ChangedRow[]; removed: number } { + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + + const rows: ChangedRow[] = []; + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) { + // A new entry passing every check it was measured against has nothing to fix. One nothing + // applied to still gets a row, since its 100 is a placeholder rather than a pass. + if (h.measured && h.score === 100 && h.suppressed.length === 0) continue; + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: "new", + headScore: scoreCell(h), + nowFailing: failingIds(h), + suppressed: newlySuppressed(h, b), + drop: h.measured ? 100 - h.score : 0, + }); + continue; + } + // Measured state and the suppression set are both part of what changed: a measured-to-unmeasured + // transition can leave the score at its placeholder, and suppressing an already-failing check + // moves no score at all, so skipping on the number alone hid both. + const suppressed = newlySuppressed(h, b); + const suppressionChanged = suppressed.length > 0 || h.suppressed.length !== b.suppressed.length; + if (b.measured === h.measured && b.score === h.score && !suppressionChanged) continue; + const baseFailing = new Set(failingIds(b)); + rows.push({ + routePath: h.routePath, + sensitive: h.sensitive, + baseScore: scoreCell(b), + headScore: scoreCell(h), + nowFailing: failingIds(h).filter((id) => !baseFailing.has(id)), + suppressed, + drop: b.measured && h.measured ? b.score - h.score : 0, + }); + } + + rows.sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + b.drop - a.drop || + a.routePath.localeCompare(b.routePath) + ); + + const removed = base.entries.filter((e) => !headFiles.has(e.fileName)).length; + return { rows, removed }; +} + +function whatChangedSection(head: MapReport, base: MapReport | null): string[] { + const lines = ["**What this PR changed**"]; + + if (!base) { + lines.push("Base comparison unavailable."); + return lines; + } + + const { rows, removed } = changedRows(head, base); + + if (rows.length === 0 && removed === 0) { + lines.push("No entry point this PR touches changed its score."); + return lines; + } + + if (rows.length > 0) { + lines.push(""); + lines.push("| route | base | head | now failing |"); + lines.push("| --- | --- | --- | --- |"); + for (const row of rows.slice(0, MAX_CHANGED_ROWS)) { + const note = row.suppressed.length > 0 ? ` (suppressed: ${row.suppressed.join(", ")})` : ""; + lines.push( + `| ${row.routePath}${note} | ${row.baseScore} | ${row.headScore} | ${row.nowFailing.join(", ")} |` + ); + } + if (rows.length > MAX_CHANGED_ROWS) { + lines.push(""); + lines.push(`and ${rows.length - MAX_CHANGED_ROWS} more`); + } + } + + if (removed > 0) { + lines.push(""); + lines.push(`${removed} entries removed`); + } + + return lines; +} + +function fixFirstSection(head: MapReport): string[] { + const lines = ["FIX FIRST"]; + const worst = fixFirst(head.entries); + + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + `- ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + } + return lines; +} + +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); + +/** + * Whether this pull request moves the report at all, so the job can stay quiet when it does not. + * + * This has to be true whenever `renderPrComment` would say something different, because anything it + * misses is a change the pull request silently does not report. The terms overlap on purpose, and + * what is defended is that their union is complete rather than that each one is load bearing. + * `MapReport.suppressions` is the one term deliberately left out, because its totals are summed from + * the very per-entry arrays the loop below compares. See INTERNALS.md, "Reporting". + */ +export function hasDelta(head: MapReport, base: MapReport | null): boolean { + if (!base) return true; + if (head.global !== base.global) return true; + if (head.parseFailures.length !== base.parseFailures.length) return true; + if (!same(head.unknownSuppressions, base.unknownSuppressions)) return true; + if (!same(head.auditGap, base.auditGap)) return true; + if (!same(head.contextGap, base.contextGap)) return true; + if (!same(head.delegating, base.delegating)) return true; + if (!same(head.checkContributions, base.checkContributions)) return true; + + const baseByFile = new Map(base.entries.map((e) => [e.fileName, e])); + const headFiles = new Set(head.entries.map((e) => e.fileName)); + if (base.entries.some((e) => !headFiles.has(e.fileName))) return true; + + for (const h of head.entries) { + const b = baseByFile.get(h.fileName); + if (!b) return true; + if (b.measured !== h.measured || b.score !== h.score) return true; + if (!same(h.suppressed, b.suppressed)) return true; + const baseFailing = new Set(b.checks.filter((c) => c.status === "fail").map((c) => c.id)); + if (h.checks.some((c) => c.status === "fail" && !baseFailing.has(c.id))) return true; + } + return false; +} + +/** What replaces a comment whose findings a later push fixed. Going silent leaves the earlier comment + * standing with findings that no longer exist. */ +export function renderResolvedComment(commit?: CommitContext): string { + return [ + MARKER, + "", + "## Observability map", + "", + ...commitLines(commit), + "Nothing in this pull request moves the report any more. The findings an earlier push " + + "reported are gone.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + +/** + * What the job posts when the head scan did not produce a report. The alternatives were a red x on a + * job that must never block a pull request, or swallowing the failure so the only signal was a comment + * that never appeared. + */ +export function renderScanFailedComment(commit?: CommitContext): string { + return [ + MARKER, + "", + "## Observability map", + "", + ...commitLines(commit), + "The scan failed for this run, so there is no report. Anything above is from an earlier push " + + "and is stale. The workflow log has the error.", + "", + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md.", + ].join("\n"); +} + +/** Pure function, no I/O: `head` and `base` are already-built reports, matched by `fileName`. */ +export function renderPrComment( + head: MapReport, + base: MapReport | null, + commit?: CommitContext +): string { + const lines = [ + MARKER, + "", + "## Observability map", + "", + ...commitLines(commit), + scoreLine(head, base), + "", + ]; + + lines.push(...whatChangedSection(head, base), ""); + lines.push(...fixFirstSection(head), ""); + + const audit = auditLine(head); + if (audit) lines.push(audit); + const context = contextLine(head); + if (context) lines.push(context); + const delegated = delegatedLines(head, MAX_DELEGATED_ROUTES); + lines.push(...delegated); + const unknown = unknownSuppressionLines(head); + lines.push(...unknown.slice(0, MAX_UNKNOWN_SUPPRESSION_LINES)); + if (unknown.length > MAX_UNKNOWN_SUPPRESSION_LINES) { + lines.push(`and ${unknown.length - MAX_UNKNOWN_SUPPRESSION_LINES} more files with unknown ids`); + } + if (audit || context || delegated.length > 0 || unknown.length > 0) lines.push(""); + + const contributions = checkContributionLines(head); + if (contributions.length > 0) { + lines.push("
What the score is made of", "", "```"); + lines.push(...contributions); + lines.push("```", "", "
", ""); + } + + lines.push( + "Report only, nothing here gates the merge. The rules and their reasons: " + + "internal-packages/observability-map/README.md." + ); + + const headFailures = head.parseFailures.length; + const baseFailures = base?.parseFailures.length ?? 0; + if (headFailures > 0 || baseFailures > 0) { + const parts: string[] = []; + if (headFailures > 0) parts.push(`${headFailures} at head`); + if (baseFailures > 0) parts.push(`${baseFailures} at base`); + lines.push( + `Warning: parse failures (${parts.join(", ")}) are excluded from the score, shrinking the denominator.` + ); + } + + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/src/report/prCommentCli.test.ts b/internal-packages/observability-map/src/report/prCommentCli.test.ts new file mode 100644 index 000000000..58bf07d6b --- /dev/null +++ b/internal-packages/observability-map/src/report/prCommentCli.test.ts @@ -0,0 +1,244 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { main, type Io } from "./prCommentCli.js"; +import { buildReport } from "../score.js"; +import { renderJson } from "./json.js"; +import { scanFile } from "../scan.js"; + +const capture = () => { + const out: string[] = []; + const err: string[] = []; + const io: Io = { out: (s) => out.push(s), err: (s) => err.push(s) }; + return { io, out: () => out.join(""), err: () => err.join("") }; +}; + +const run = (...args: string[]) => { + const c = capture(); + const code = main(["node", "prCommentCli.js", ...args], c.io); + return { code, out: c.out(), err: c.err() }; +}; + +const swallows = `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }`; + +const handles = `import { requireUserId } from "~/services/session.server"; + import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { logger.error("token create failed", { userId, error }); throw error; } + }`; + +describe("prCommentCli", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-cli-")); + const headPath = join(dir, "head.json"); + const basePath = join(dir, "base.json"); + const unchangedPath = join(dir, "unchanged.json"); + + const source = `export const loader = () => new Response("ok");`; + const report = buildReport([scanFile("resources.a.ts", source)!], []); + writeFileSync(headPath, renderJson(buildReport([scanFile("api.v1.t.ts", swallows)!], []))); + writeFileSync(basePath, renderJson(buildReport([scanFile("api.v1.t.ts", handles)!], []))); + writeFileSync(unchangedPath, renderJson(report)); + + afterAll(() => rmSync(dir, { recursive: true })); + + it("renders the markdown comment for head and base file arguments", () => { + const r = run(headPath, basePath); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + }); + + // B4. The job used to comment on every push, including one that moved nothing. + it("posts nothing when the report did not move and no comment exists yet", () => { + const r = run(unchangedPath, unchangedPath); + expect(r.code).toBe(0); + expect(r.out).toBe(""); + }); + + it("replaces an existing comment with a resolved state when the delta has gone", () => { + const r = run(unchangedPath, unchangedPath, "--existing-comment"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("Nothing in this pull request moves the report any more."); + expect(r.out).not.toContain("FIX FIRST"); + }); + + it("posts the full comment when there is a delta, existing comment or not", () => { + for (const args of [ + [headPath, basePath], + [headPath, basePath, "--existing-comment"], + ]) { + const r = run(...args); + expect(r.code).toBe(0); + expect(r.out).toContain("FIX FIRST"); + expect(r.out).not.toContain("moves the report any more"); + } + }); + + it("prints the stale-report comment for --scan-failed without reading any file", () => { + const r = run("--scan-failed"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("The scan failed for this run"); + }); + + // The reconcile path. A pull request whose diff stops matching the paths the workflow watches + // scans nothing, so there are no reports to compare and no delta to compute, and the comment an + // earlier push left still shows findings that have gone. This is how the workflow says so. + it("prints the resolved comment for --resolved without reading any file", () => { + const r = run("--resolved"); + expect(r.code).toBe(0); + expect(r.out.split("\n")[0]).toBe(""); + expect(r.out).toContain("Nothing in this pull request moves the report any more."); + expect(r.out).not.toContain("FIX FIRST"); + }); + + describe("the commit the comment is rendered for", () => { + const sha = "0123456789abcdef0123456789abcdef01234567"; + const url = "https://github.com/triggerdotdev/trigger.dev/compare/1111111...2222222"; + const stamp = `As of [\`0123456\`](${url}).`; + + it("stamps the report comment from the two commit flags", () => { + const r = run(headPath, basePath, `--commit-sha=${sha}`, `--commit-url=${url}`); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + it("stamps the resolved comment the reconcile path posts", () => { + const r = run("--resolved", `--commit-sha=${sha}`, `--commit-url=${url}`); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + it("stamps the resolved comment the delta path posts when the delta has gone", () => { + const r = run( + unchangedPath, + unchangedPath, + "--existing-comment", + `--commit-sha=${sha}`, + `--commit-url=${url}` + ); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + it("stamps the stale-report comment", () => { + const r = run("--scan-failed", `--commit-sha=${sha}`, `--commit-url=${url}`); + expect(r.code).toBe(0); + expect(r.out).toContain(stamp); + }); + + // Half a pair can only come from an edit to the workflow, and a comment quietly missing the + // line it was supposed to gain is the failure nobody would ever notice. + it("exits 1 rather than dropping the stamp when only one of the two flags is given", () => { + for (const half of [`--commit-sha=${sha}`, `--commit-url=${url}`]) { + const r = run(headPath, basePath, half); + expect(r.code).toBe(1); + expect(r.err).toContain("have to be given together"); + expect(r.out).toBe(""); + } + }); + + // What an unset workflow expression interpolates to. Both empty is the local run. + it("reads an empty flag value as no commit context rather than as half a pair", () => { + const r = run(headPath, basePath, "--commit-sha=", "--commit-url="); + expect(r.code).toBe(0); + expect(r.out).toContain("FIX FIRST"); + expect(r.out).not.toContain("As of ["); + }); + }); + + it("treats '-' as no base", () => { + const r = run(headPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("treats a missing second argument as no base", () => { + const r = run(headPath); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + // Without a base there is no delta to compute, so silence would be a guess. Posting is the + // honest answer even for a report identical to one nobody can see. + it("posts even for an unmoved report when the base is unavailable", () => { + const r = run(unchangedPath, "-"); + expect(r.code).toBe(0); + expect(r.out).toContain("Base comparison unavailable."); + }); + + it("exits 1 with a usage message when head.json is missing", () => { + const r = run(); + expect(r.code).toBe(1); + expect(r.err).toContain("usage:"); + }); + + it("exits 1 with a one-line message, not a stack trace, when head.json does not exist", () => { + const r = run(join(dir, "does-not-exist.json")); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("cannot read head report"); + expect(r.err).not.toContain(" at "); + }); + + it("exits 1 with a one-line message, not a stack trace, when head.json is malformed", () => { + const malformedPath = join(dir, "malformed.json"); + writeFileSync(malformedPath, "{ not json"); + const r = run(malformedPath); + expect(r.code).toBe(1); + expect(r.err.split("\n").filter(Boolean)).toHaveLength(1); + expect(r.err).toContain("head report is not valid JSON"); + expect(r.err).not.toContain(" at "); + }); + + describe("--out", () => { + it("writes the comment to the file and leaves stdout empty", () => { + const outPath = join(dir, "out-delta.md"); + const r = run(headPath, basePath, `--out=${outPath}`); + expect(r.code).toBe(0); + expect(r.out).toBe(""); + + const written = readFileSync(outPath, "utf8"); + expect(written.split("\n")[0]).toBe(""); + }); + + it("writes an empty file when there is nothing to post, rather than no file", () => { + const outPath = join(dir, "out-nothing.md"); + const r = run(unchangedPath, unchangedPath, `--out=${outPath}`); + expect(r.code).toBe(0); + expect(existsSync(outPath)).toBe(true); + expect(readFileSync(outPath, "utf8")).toBe(""); + }); + + it("writes the resolved and scan-failed comments too", () => { + for (const mode of ["--resolved", "--scan-failed"]) { + const outPath = join(dir, `out${mode}.md`); + expect(run(mode, `--out=${outPath}`).code).toBe(0); + expect(readFileSync(outPath, "utf8").split("\n")[0]).toBe( + "" + ); + } + }); + + it("truncates a file left by an earlier run", () => { + const outPath = join(dir, "out-stale.md"); + writeFileSync(outPath, "stale content from a previous push\n"); + expect(run(unchangedPath, unchangedPath, `--out=${outPath}`).code).toBe(0); + expect(readFileSync(outPath, "utf8")).toBe(""); + }); + }); + + it("exits 1 with a one-line message when base.json is malformed", () => { + const malformedBasePath = join(dir, "malformed-base.json"); + writeFileSync(malformedBasePath, "not json at all"); + const r = run(headPath, malformedBasePath); + expect(r.code).toBe(1); + expect(r.err).toContain("base report is not valid JSON"); + }); +}); diff --git a/internal-packages/observability-map/src/report/prCommentCli.ts b/internal-packages/observability-map/src/report/prCommentCli.ts new file mode 100644 index 000000000..e0ad53c91 --- /dev/null +++ b/internal-packages/observability-map/src/report/prCommentCli.ts @@ -0,0 +1,134 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { MapReport } from "../score.js"; +import { + hasDelta, + renderPrComment, + renderResolvedComment, + renderScanFailedComment, + type CommitContext, +} from "./prComment.js"; + +/** Where output goes. Injectable so tests can read it without spawning a process. */ +export type Io = { out: (s: string) => void; err: (s: string) => void }; + +const processIo: Io = { + out: (s) => process.stdout.write(s), + err: (s) => process.stderr.write(s), +}; + +/** Raises a message naming the file rather than letting an unreadable path or malformed JSON surface + * as a stack trace. */ +function readReport(path: string, label: string): MapReport { + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch { + throw new Error(`cannot read ${label}: ${path}`); + } + try { + return JSON.parse(raw) as MapReport; + } catch { + throw new Error(`${label} is not valid JSON: ${path}`); + } +} + +/** An `--opt=value` argument, last one winning. An empty value reads as absent, since that is what an + * unset workflow expression interpolates to. */ +function flag(args: string[], name: string): string | undefined { + const prefix = `--${name}=`; + const values = args.filter((a) => a.startsWith(prefix)).map((a) => a.slice(prefix.length)); + return values.filter(Boolean).pop(); +} + +/** + * The commit the caller says this comment is for. Half a pair is rejected rather than dropped, since a + * comment silently missing the line it was supposed to gain is the failure nobody would notice. + */ +function commitFrom(args: string[]): CommitContext | undefined { + const sha = flag(args, "commit-sha"); + const url = flag(args, "commit-url"); + if (sha && url) return { sha, url }; + if (sha || url) throw new Error("--commit-sha and --commit-url have to be given together"); + return undefined; +} + +/** + * `-` or a missing second arg means no base, which is what the CI job falls back to when the base scan + * failed, so the comment still renders rather than the job going red. + * + * Empty output means "post nothing". `--existing-comment` is how the workflow says a comment from an + * earlier push is already there, so with the delta gone that comment is replaced with a resolved state + * rather than left standing. `--scan-failed` and `--resolved` take no report at all, for a head scan + * that produced nothing to read and for a run where the watched paths did not move. + */ +export function main(argv: string[], io: Io = processIo): number { + const args = argv.slice(2); + const scanFailed = args.includes("--scan-failed"); + const resolved = args.includes("--resolved"); + const existingComment = args.includes("--existing-comment"); + const positional = args.filter((a) => !a.startsWith("--")); + const headPath = positional[0]; + const basePath = positional[1]; + + /** + * The marker has to be the document's first line or the workflow's lookup cannot find the comment it + * left, and every later push posts a new one instead of updating it. `--out` keeps stdout free for + * whatever a tool decides to announce, the same reason `src/cli.ts` has it. An empty write is a real + * outcome and not a failure: it is how "post nothing" reaches the workflow's `-s` check. + */ + const outPath = flag(args, "out"); + const write = (text: string) => { + if (outPath === undefined) { + if (text) io.out(text); + return; + } + writeFileSync(outPath, text); + }; + + let commit: CommitContext | undefined; + try { + commit = commitFrom(args); + } catch (error) { + io.err(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } + + if (scanFailed) { + write(`${renderScanFailedComment(commit)}\n`); + return 0; + } + + if (resolved) { + write(`${renderResolvedComment(commit)}\n`); + return 0; + } + + if (!headPath) { + io.err("usage: prCommentCli.ts [base.json|-] [--existing-comment]\n"); + return 1; + } + + let head: MapReport; + let base: MapReport | null; + try { + head = readReport(headPath, "head report"); + base = !basePath || basePath === "-" ? null : readReport(basePath, "base report"); + } catch (error) { + io.err(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } + + if (hasDelta(head, base)) { + write(`${renderPrComment(head, base, commit)}\n`); + return 0; + } + write(existingComment ? `${renderResolvedComment(commit)}\n` : ""); + return 0; +} + +// Only when run as a program. Importing the module, which the tests do, must not read a file. +const invokedDirectly = + process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) process.exitCode = main(process.argv); diff --git a/internal-packages/observability-map/src/report/terminal.test.ts b/internal-packages/observability-map/src/report/terminal.test.ts new file mode 100644 index 000000000..34c3e81b0 --- /dev/null +++ b/internal-packages/observability-map/src/report/terminal.test.ts @@ -0,0 +1,401 @@ +import { renderTerminal } from "./terminal.js"; +import { renderJson } from "./json.js"; +import { buildReport } from "../score.js"; +import { scanFile } from "../scan.js"; + +const report = () => + buildReport( + [ + scanFile( + "api.v1.a.ts", + `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute({}, async () => new Response("ok"));` + )!, + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + )!, + ], + ["broken.ts"] + ); + +/** The FIX FIRST section, with the index asserted rather than assumed. `indexOf` returns -1 for a + * section that is not there, and `slice(-1)` is the last character of the report, which every + * `not.toContain` below would pass against. */ +function sliceFixFirst(out: string): string { + const start = out.indexOf("FIX FIRST"); + expect(start).toBeGreaterThan(-1); + const end = out.indexOf("no findings:"); + expect(end).toBeGreaterThan(start); + return out.slice(start, end); +} + +describe("renderTerminal", () => { + // The guard has to be able to fail, or it is decoration in a file whose own comment warns about + // exactly this trap. + it("refuses to slice a report with no fix list rather than returning its last character", () => { + expect(() => sliceFixFirst("a report with neither section in it")).toThrow(); + }); + + it("shows the global score, the audit gap and the fix list", () => { + const out = renderTerminal(report()); + expect(out).toContain("COVERAGE"); + expect(out).toContain("FIX FIRST"); + expect(out).toContain("audit"); + expect(out).toContain("/api/v1/auth/tokens"); + }); + + it("surfaces suppressions so laundering is visible rather than silent", () => { + const suppressed = scanFile( + "api.v1.d.ts", + `// obs-map-disable error-classification -- deliberate, see ticket + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + const out = renderTerminal(buildReport([suppressed], [])); + expect(out).toMatch(/SUPPRESSED\s+1 check across 1 entry point/); + }); + + it("does not mention suppressions when there are none", () => { + expect(renderTerminal(report())).not.toMatch(/suppress/i); + }); + + it("surfaces parse failures so the denominator is not silently wrong", () => { + expect(renderTerminal(report())).toContain("broken.ts"); + }); + + it("shows the unmeasured count so 415 vs 427 is not silently confusing", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + expect(out).toContain("1 unmeasured"); + }); + + it("orders FIX FIRST by sensitivity first, then ascending score, and excludes audit-only gaps", () => { + // Sensitive, score 0: both applicable scored checks fail (auth-boundary, error-classification, + // request-context all fail because the catch swallows without naming who it happened to). + const sensitiveZero = scanFile( + "api.v1.envvars.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { + return await prisma.envVar.update({ where: {}, data: {} }); + } catch (e) { + return null; + } + }` + )!; + + // Sensitive, score 33: its catch decides something, telling a bad request apart from the rest, + // but it has no guard and names nobody. Fails more than request-context, so it stays in the + // list rather than collapsing into the figure. + const sensitiveThirtyThree = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } + catch (error) { + if (error instanceof BadRequest) return json({ error: "bad" }, { status: 400 }); + throw error; + } + }` + )!; + + // Not sensitive, score 0: worse score than sensitiveThirtyThree, but must still sort after both + // sensitive entries because sensitivity outranks raw score. + const notSensitiveZero = scanFile( + "resources.busy.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + // Sensitive mutation, scored checks all pass (guarded, no try/catch): the only gap is + // audit-trail, which is a headline figure, not a per-route fix-list item. Must not appear. + const sensitiveAuditOnly = scanFile( + "api.v1.billing.ts", + `import { prisma } from "~/db.server"; + import { logger } from "~/services/logger.server"; + import { requireUserId } from "~/services/session.server"; + export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.billing.update({ where: { userId }, data: {} }); } + catch (error) { logger.error("billing update failed", { userId, error }); throw error; } + }` + )!; + + const out = renderTerminal( + buildReport([sensitiveThirtyThree, sensitiveZero, notSensitiveZero, sensitiveAuditOnly], []) + ); + + // Slice to the end of the list, not to a string the I5 fix deleted: `indexOf` returned -1 for + // "already solid" and the assertions were quietly running against the whole tail. + const fixFirst = sliceFixFirst(out); + const idxZero = fixFirst.indexOf("api.v1.envvars.ts"); + const idxThirtyThree = fixFirst.indexOf("api.v1.auth.tokens.ts"); + const idxNotSensitive = fixFirst.indexOf("resources.busy.ts"); + + expect(idxZero).toBeGreaterThan(-1); + expect(idxThirtyThree).toBeGreaterThan(idxZero); + expect(idxNotSensitive).toBeGreaterThan(idxThirtyThree); + expect(fixFirst).not.toContain("api.v1.billing.ts"); + }); +}); + +describe("renderJson", () => { + it("round-trips to an object carrying the score and entries", () => { + const parsed = JSON.parse(renderJson(report())); + expect(typeof parsed.global).toBe("number"); + expect(Array.isArray(parsed.entries)).toBe(true); + }); +}); + +describe("rendering honestly when there is nothing to say", () => { + // I4. mean([]) returned 100, so a family with nothing measured rendered a full green bar. + it("renders a family with nothing measured as not measured, not as 100", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([trivial], [])); + const line = out.split("\n").find((l) => l.includes("resources"))!; + expect(line).not.toMatch(/100/); + expect(line).toMatch(/not measured/i); + }); + + it("renders the global score as not measured when nothing was measured", () => { + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + expect(renderTerminal(buildReport([trivial], []))).toMatch(/score not measured/i); + }); + + // I11. The audit sentence was printed unconditionally, including when the figure said otherwise. + it("does not claim no audit helper exists when one is in use", () => { + const audited = scanFile( + "api.v1.auth.tokens.ts", + `import { clearImpersonation } from "~/models/admin.server"; + import { prisma } from "~/db.server"; + export async function action() { + const token = await prisma.token.create({ data: {} }); + await clearImpersonation(request, "/admin"); + return json(token); + }` + )!; + const out = renderTerminal(buildReport([audited], [])); + expect(out).toContain("1 of 1"); + expect(out).not.toContain("No audit helper exists"); + }); + + // The other half of the branch. A zero used to print "No audit helper exists in the webapp", which + // is false, and a scan of any subset with no impersonation route in it brings it straight back. + it("does not claim the webapp has no audit helper when nothing reached one", () => { + const unaudited = scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + return json(await prisma.token.create({ data: {} })); + }` + )!; + const out = renderTerminal(buildReport([unaudited], [])); + expect(out).toContain("AUDIT 0 of 1 sensitive mutations record an actor. 1 without one."); + expect(out).not.toContain("No audit helper exists"); + }); + + it("says nothing about audit when no sensitive mutation was found", () => { + const plain = scanFile( + "resources.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + expect(renderTerminal(buildReport([plain], []))).not.toMatch(/AUDIT/); + }); + + // I5. "already solid" counted entries that are clean because they do nothing, alongside entries + // nothing applied to, in one flattering number. + it("separates entries that passed from entries nothing applied to", () => { + const clean = scanFile( + "api.v1.clean.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + const out = renderTerminal(buildReport([clean, trivial], [])); + expect(out).not.toMatch(/already solid/i); + expect(out).toMatch(/1 passed every applicable check/i); + expect(out).toMatch(/1 had none to apply/i); + }); +}); + +describe("collapsing the house-style finding", () => { + const namesNobody = () => + scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + + const namesNobodyAndSwallows = () => + scanFile( + "api.v1.auth.tokens.ts", + `import { prisma } from "~/db.server"; + export async function action() { + try { return await prisma.token.create({ data: {} }); } catch (e) { return null; } + }` + )!; + + // request-context fails 401 of 412 entry points, so listing each one turns the fix list into a + // single finding repeated. Same reasoning that keeps audit-trail out of the list. + it("keeps an entry whose only finding is request-context out of the fix list", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + // Guarded for the same reason as the slice above: an absent section makes `indexOf` return -1, + // `slice(-1)` yields the last character, and the negative assertion passes for the wrong reason. + const fixFirst = sliceFixFirst(out); + expect(fixFirst).not.toContain("api.v1.silent.ts"); + }); + + it("reports the gap as a headline figure instead", () => { + const out = renderTerminal(buildReport([namesNobody()], [])); + expect(out).toMatch(/CONTEXT\s+0 of 1 entry points name a tenant on a failure path/); + }); + + // NEW-3. 18 of the collapsed entries are sensitive, including /admin/impersonate and the envvars + // routes, so the line has to say a reader should go and look at them. + it("says how many of the collapsed entries are sensitive", () => { + const sensitiveAndSilent = scanFile( + "api.v1.auth.jwt.ts", + `import { requireUserId } from "~/services/session.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + }` + )!; + const out = renderTerminal(buildReport([sensitiveAndSilent], [])); + expect(out).toMatch(/1 appears? only here[^\n]*1 of them sensitive/i); + }); + + it("says how many entries the collapse took out of the list", () => { + const out = renderTerminal(buildReport([namesNobody(), namesNobodyAndSwallows()], [])); + expect(out).toMatch(/1 appears? only here/i); + }); + + it("still lists request-context when the entry fails something else too", () => { + const out = renderTerminal(buildReport([namesNobodyAndSwallows()], [])); + const fixFirst = sliceFixFirst(out); + expect(fixFirst).toContain("api.v1.auth.tokens.ts"); + expect(fixFirst).toContain("request-context"); + expect(fixFirst).toContain("error-classification"); + }); +}); + +// B6. A stderr warning is the minimum; the terminal report is where someone would notice. +describe("reporting a suppression that names no check", () => { + const typo = (fileName: string) => + scanFile( + fileName, + `// obs-map-disable eror-classification -- typo + import { prisma } from "~/db.server"; + export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } + }` + )!; + + it("names the file and the bad id", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain("UNKNOWN SUPPRESSION"); + expect(out).toContain("api.v1.a.ts"); + expect(out).toContain("eror-classification"); + }); + + it("lists the ids that would have worked", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).toContain( + "error-classification, auth-boundary, auth-scope, request-context, audit-trail" + ); + }); + + it("does not report the finding as suppressed", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts")], [])); + expect(out).not.toMatch(/SUPPRESSED\s+\d/); + }); + + it("reports one line per file rather than one for the run", () => { + const out = renderTerminal(buildReport([typo("api.v1.a.ts"), typo("api.v1.b.ts")], [])); + expect(out.split("\n").filter((l) => l.startsWith("UNKNOWN SUPPRESSION"))).toHaveLength(2); + }); + + it("says nothing when every directive named a real check", () => { + expect(renderTerminal(report())).not.toContain("UNKNOWN SUPPRESSION"); + }); +}); + +// C4b. A delegating route must not read as a clean bill of health. It is surfaced the way a parse +// failure is, because it shrinks the denominator the same way. +describe("reporting a route whose body is in another module", () => { + const delegated = () => + buildReport( + [ + scanFile("webhooks.v1.stripe.ts", `export { action } from "./handler.server";`)!, + scanFile( + "api.v1.a.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!, + ], + [] + ); + + it("names the route and says nothing was checked", () => { + const out = renderTerminal(delegated()); + expect(out).toContain("DELEGATED"); + expect(out).toContain("webhooks.v1.stripe.ts"); + expect(out).toContain("nothing here was checked"); + }); + + it("separates it from the entries nothing applied to, in the headline", () => { + expect(renderTerminal(delegated())).toContain("1 measured, 0 unmeasured, 1 delegated of 2"); + }); + + it("carries the file names into the JSON", () => { + expect(JSON.parse(renderJson(delegated())).delegating).toEqual(["webhooks.v1.stripe.ts"]); + }); + + it("says nothing when every route keeps its body", () => { + expect(renderTerminal(report())).not.toContain("DELEGATED"); + }); +}); + +// C5. What the composite is made of, disclosed on screen rather than folded into a weight. +describe("reporting what the score is made of", () => { + it("gives a line per check with applicability, passes and worth", () => { + const out = renderTerminal(report()); + expect(out).toContain("CHECKS"); + expect(out).toMatch( + /request-context\s+\d+ applicable,\s+\d+ pass,\s+\d+ sole, global without it/ + ); + }); + + it("marks the check that does not feed the score", () => { + expect(renderTerminal(report())).toMatch(/audit-trail\s+.*not in the score/); + }); + + it("carries the same figures into the JSON", () => { + const parsed = JSON.parse(renderJson(report())); + expect(parsed.checkContributions.map((c: { id: string }) => c.id)).toContain("auth-scope"); + }); +}); diff --git a/internal-packages/observability-map/src/report/terminal.ts b/internal-packages/observability-map/src/report/terminal.ts new file mode 100644 index 000000000..5b0df7849 --- /dev/null +++ b/internal-packages/observability-map/src/report/terminal.ts @@ -0,0 +1,218 @@ +import type { MapReport, ScoredEntry } from "../score.js"; +import { CHECKS, SCORED_CHECK_IDS } from "../checks/index.js"; + +const NOT_MEASURED = "not measured".padEnd(15); + +/** A bar and a figure, or a plain "not measured" where there is no figure to draw. */ +const gauge = (score: number | null) => { + if (score === null) return NOT_MEASURED; + const filled = Math.round(score / 10); + return `${"β–°".repeat(filled)}${"β–±".repeat(10 - filled)} ${String(score).padStart(3)}`; +}; + +/** Failing checks that feed `score`, so never `audit-trail`: every sensitive mutation fails that + * today, and it is reported once as the `AUDIT` line instead. */ +export const scoredFailures = (e: ScoredEntry) => + e.checks.filter((c) => SCORED_CHECK_IDS.includes(c.id) && c.status === "fail"); + +/** + * The routes the FIX FIRST list is drawn from, worst first: sensitive before not, then by score, then + * by name. Exported because `prComment.ts` renders the same list with different bullets and had a + * byte-identical copy of this filter and sort. + */ +export const fixFirst = (entries: ScoredEntry[]): ScoredEntry[] => + entries + .filter((e) => scoredFailures(e).length > 0 && !contextOnly(e)) + .sort( + (a, b) => + Number(b.sensitive) - Number(a.sensitive) || + a.score - b.score || + a.fileName.localeCompare(b.fileName) + ); + +/** An entry whose only finding is `request-context`, which fails almost everything, so it is + * collapsed into the `CONTEXT` figure rather than listed. An entry that fails something else as well + * keeps all of its findings and stays in the list. */ +export const contextOnly = (e: ScoredEntry) => { + const failures = scoredFailures(e); + return failures.length === 1 && failures[0]!.id === "request-context"; +}; + +/** + * The UNKNOWN SUPPRESSION lines, one per file, shared with `prComment.ts`. A typo suppresses nothing, + * so without this the author reads the finding as acknowledged and the tool goes on reporting it. + */ +export function unknownSuppressionLine(fileName: string, ids: string[]): string { + return ( + `UNKNOWN SUPPRESSION ${fileName}: ${ids.join(", ")} ` + + `(no such check, nothing suppressed). Known: ${CHECKS.map((c) => c.id).join(", ")}.` + ); +} + +export function unknownSuppressionLines(report: MapReport): string[] { + return report.unknownSuppressions.map(({ fileName, ids }) => + unknownSuppressionLine(fileName, ids) + ); +} + +/** + * The AUDIT figure, shared with `prComment.ts` so both renderers say the same thing. Null when no + * sensitive mutation exists. One shape for every count and no branch on the count, because the branch + * is what carried the bug: a zero used to print "No audit helper exists in the webapp", which is + * false. "0 of N record an actor" already says what a zero means. + */ +export function auditLine(report: MapReport): string | null { + const { sensitiveMutations, withAudit } = report.auditGap; + if (sensitiveMutations === 0) return null; + return ( + `AUDIT ${withAudit} of ${sensitiveMutations} sensitive mutations record an actor. ` + + `${sensitiveMutations - withAudit} without one.` + ); +} + +/** The CONTEXT figure, shared with `prComment.ts`. Null when nothing is applicable. */ +export function contextLine(report: MapReport): string | null { + const { applicable, naming } = report.contextGap; + if (applicable === 0) return null; + const collapsed = report.entries.filter(contextOnly); + const sensitive = collapsed.filter((e) => e.sensitive).length; + return ( + `CONTEXT ${naming} of ${applicable} entry points name a tenant on a failure path.` + + (collapsed.length > 0 + ? ` ${collapsed.length} appear${collapsed.length === 1 ? "s" : ""} only here, ` + + `${sensitive} of them sensitive, in the JSON rather than the fix list.` + : "") + ); +} + +/** + * The DELEGATED lines, shared with `prComment.ts`. Worded as a shortfall rather than a note: these + * routes left the denominator and no check looked at any of them. `limit` shortens the evidence for a + * caller with a size limit; the count in front of the list is always the full one. + */ +export function delegatedLines(report: MapReport, limit = Infinity): string[] { + if (report.delegating.length === 0) return []; + const n = report.delegating.length; + const shown = report.delegating.slice(0, limit); + const tail = n > shown.length ? `, and ${n - shown.length} more` : ""; + return [ + `DELEGATED ${n} route${n === 1 ? "" : "s"} keep${n === 1 ? "s" : ""} the body in another ` + + `module, so nothing here was checked and ${n === 1 ? "it is" : "they are"} out of the score: ` + + shown.join(", ") + + tail, + ]; +} + +/** The CHECKS block: what the composite is made of. `sole` is the figure that says most, since an + * entry only one scored check applies to scores 0 or 100 on that one boolean. */ +export function checkContributionLines(report: MapReport): string[] { + if (report.checkContributions.every((c) => c.applicable === 0)) return []; + const width = Math.max(...report.checkContributions.map((c) => c.id.length)); + return [ + "CHECKS", + ...report.checkContributions.map((c) => { + const worth = !c.scored + ? "not in the score" + : c.globalWithout === null + ? "nothing left measured without it" + : `global without it ${c.globalWithout}`; + return ( + ` ${c.id.padEnd(width)} ${String(c.applicable).padStart(3)} applicable, ` + + `${String(c.passed).padStart(3)} pass, ${String(c.sole).padStart(3)} sole, ${worth}` + ); + }), + ]; +} + +export function renderTerminal(report: MapReport): string { + const lines: string[] = []; + + const headline = report.global === null ? "score not measured" : `score ${report.global}/100`; + const delegatedCount = + report.delegating.length > 0 ? `, ${report.delegating.length} delegated` : ""; + lines.push( + `${headline} ${report.measured} measured, ${report.unmeasured} unmeasured${delegatedCount} of ${report.entries.length} entry points` + ); + lines.push(""); + lines.push("COVERAGE"); + for (const [family, stats] of Object.entries(report.byFamily).sort((a, b) => b[1].n - a[1].n)) { + lines.push( + ` ${family.padEnd(12)} ${gauge(stats.mean)} ${stats.measured}/${stats.n} entry points` + ); + } + lines.push( + ` ${"sensitive".padEnd(12)} ${gauge(report.sensitiveCohort.mean)} ${ + report.sensitiveCohort.measured + }/${report.sensitiveCohort.n} entry points` + ); + + const contributions = checkContributionLines(report); + if (contributions.length > 0) { + lines.push(""); + lines.push(...contributions); + } + + const audit = auditLine(report); + if (audit) { + lines.push(""); + lines.push(audit); + } + + const context = contextLine(report); + if (context) { + lines.push(""); + lines.push(context); + } + + const delegated = delegatedLines(report); + if (delegated.length > 0) { + lines.push(""); + lines.push(...delegated); + } + + const unknown = unknownSuppressionLines(report); + if (unknown.length > 0) { + lines.push(""); + lines.push(...unknown); + } + + if (report.suppressions.checks > 0) { + const { entries, checks } = report.suppressions; + lines.push( + `SUPPRESSED ${checks} check${checks === 1 ? "" : "s"} across ${entries} entry point${ + entries === 1 ? "" : "s" + }, each with a reason on the record. A suppression removes a finding from this list, ` + + `it does not raise a score.` + ); + } + + const worst = fixFirst(report.entries); + + lines.push(""); + lines.push("FIX FIRST"); + for (const e of worst.slice(0, 3)) { + const marks = e.sensitive ? " (sensitive)" : ""; + lines.push( + ` ${e.routePath}${marks} - ${scoredFailures(e) + .map((c) => c.id) + .join(", ")}` + ); + lines.push(` ${e.fileName}`); + } + if (worst.length > 3) { + lines.push(""); + lines.push(`THEN ${worst.length - 3} more with gaps`); + } + + lines.push(""); + // Two figures rather than one, because lumping "nothing applicable" in with "passed" counted routes + // as solid for doing nothing. + const clean = report.entries.filter((e) => e.measured && scoredFailures(e).length === 0).length; + lines.push( + `no findings: ${clean} passed every applicable check, ${report.unmeasured} had none to apply` + ); + if (report.parseFailures.length > 0) { + lines.push(`parse failures (excluded from the score): ${report.parseFailures.join(", ")}`); + } + return lines.join("\n"); +} diff --git a/internal-packages/observability-map/src/routeExports.ts b/internal-packages/observability-map/src/routeExports.ts new file mode 100644 index 000000000..588c8fc89 --- /dev/null +++ b/internal-packages/observability-map/src/routeExports.ts @@ -0,0 +1,61 @@ +import type { EntryPoint } from "./types.js"; + +export type ExportName = "loader" | "action"; + +/** + * One export of a route file, carrying that export's own evidence and nothing from the other one. + * Every field has an entry-point-wide twin on `EntryPoint`, and reaching for the twin is the mistake + * this type exists to make hard. + */ +export type RouteExport = { + name: ExportName; + /** Callee of the initializer call this export is assigned from, if any. */ + initializerCallee: string | null; + /** Top-level keys of the object literal passed to that call. */ + builderOptions: string[]; + /** Callees inside this export's handlers, and inside same-file helpers they call. */ + calleeNames: string[]; + /** The same calls as whole dotted paths, so `prisma.thing.findFirst` keeps its receiver. */ + calleeTexts: string[]; + /** Callees whose answer those handlers demonstrably read. */ + checkedCallees: string[]; + /** Whether those handlers narrow a query by the caller's own id. */ + scopesByCaller: boolean; + statementCount: number; + hasTryCatch: boolean; +}; + +/** + * The exports this route file declares, in `loader`, `action` order. One enumeration for the whole + * package, because the two per-export checks each grew their own hand-maintained `[loader, action]` + * literal, disagreeing about how to tell whether an export was there at all. + * + * Absent exports are not returned, so a caller never has to remember to filter them. + */ +export function routeExports(ep: EntryPoint): RouteExport[] { + const all: RouteExport[] = [ + { + name: "loader", + initializerCallee: ep.loaderInitializerCallee, + builderOptions: ep.loaderBuilderOptions, + calleeNames: ep.loaderCalleeNames, + calleeTexts: ep.loaderCalleeTexts, + checkedCallees: ep.loaderCheckedCallees, + scopesByCaller: ep.loaderScopesByCaller, + statementCount: ep.loaderStatementCount, + hasTryCatch: ep.loaderHasTryCatch, + }, + { + name: "action", + initializerCallee: ep.actionInitializerCallee, + builderOptions: ep.actionBuilderOptions, + calleeNames: ep.actionCalleeNames, + calleeTexts: ep.actionCalleeTexts, + checkedCallees: ep.actionCheckedCallees, + scopesByCaller: ep.actionScopesByCaller, + statementCount: ep.actionStatementCount, + hasTryCatch: ep.actionHasTryCatch, + }, + ]; + return all.filter((e) => (e.name === "loader" ? ep.hasLoader : ep.hasAction)); +} diff --git a/internal-packages/observability-map/src/scan.test.ts b/internal-packages/observability-map/src/scan.test.ts new file mode 100644 index 000000000..d65d1398a --- /dev/null +++ b/internal-packages/observability-map/src/scan.test.ts @@ -0,0 +1,2983 @@ +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { ParseFailureError, routeModuleFiles, scanDirectory, scanFile } from "./scan.js"; + +const LOADER = ` +import { json } from "@remix-run/server-runtime"; +export async function loader() { return json({}); } +`; + +const COMPONENT_ONLY = ` +export default function Page() { return null; } +`; + +describe("scanFile", () => { + it("detects an exported loader as a server entry point", () => { + const ep = scanFile("api.v1.things.ts", LOADER); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(false); + }); + + it("ignores a route that only exports a component", () => { + expect(scanFile("_app.things.tsx", COMPONENT_ONLY)).toBeNull(); + }); + + it("detects a loader assigned from a call expression", () => { + const ep = scanFile("api.v1.x.ts", `export const loader = createLoaderApiRoute({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); +}); + +describe("scanFile: named export clauses", () => { + it("detects `export { loader }` and resolves the builder callee from the local declaration", () => { + const ep = scanFile( + "api.v1.query.ts", + ` + const { loader } = createLoaderApiRoute({ findResource: async () => 1 }, async () => { + const a = 1; + return json({ a }); + }); + export { loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(2); + }); + + it("detects an aliased named export `export { h as loader }`", () => { + const ep = scanFile( + "api.v1.aliased.ts", + ` + async function h() { return json({}); } + export { h as loader }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.statementCount).toBe(1); + }); + + it("reports both `export const loader` and a separate `export { action }`", () => { + const ep = scanFile( + "api.v1.both.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})); + const { action } = createActionApiRoute({}, async ({ body }) => { + const x = body.x; + return json({ x }); + }); + export { action }; + ` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + it("resolves an export assigned from a property of a local builder result", () => { + const ep = scanFile( + "api.v1.errors.$errorId.ignore.ts", + ` + const route = createActionApiRoute({ method: "POST" }, async ({ body }) => { + const a = 1; + const b = 2; + return json({ a, b }); + }); + export const action = route.action; + export const loader = route.loader; + ` + ); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + // Both exports share one handler, so its statements are counted once. + expect(ep!.statementCount).toBe(3); + }); + + it("counts a `handler` property body but not the surrounding builder config lambdas", () => { + const ep = scanFile( + "engine.v1.dev.presence.ts", + ` + export const loader = createSSELoader({ + timeout: 1000, + findResource: async (params) => { + const a = 1; + const b = 2; + const c = 3; + return lookup(a, b, c); + }, + handler: async ({ request }) => { + const auth = await authenticate(request); + return stream(auth); + }, + }); + ` + ); + expect(ep!.statementCount).toBe(2); + expect(ep!.calleeNames).not.toContain("lookup"); + }); + + it("counts the per-method `methods.POST.handler` bodies", () => { + const ep = scanFile( + "api.v1.prompts.$slug.override.ts", + ` + const { action, loader } = createMultiMethodApiRoute({ + params: ParamsSchema, + methods: { + POST: { + body: CreateBody, + handler: async ({ body }) => { + const created = await create(body); + return json(created); + }, + }, + DELETE: { + handler: async ({ params }) => { + return json({ ok: true }); + }, + }, + }, + }); + export { action, loader }; + ` + ); + expect(ep!.statementCount).toBe(3); + expect(ep!.calleeNames).toContain("create"); + }); + + it("ignores a nested config callback that happens to be named `handler`", () => { + const ep = scanFile( + "api.v1.named-collision.ts", + ` + export const loader = build({ + onError: { + handler: async () => { + const a = 1; + const b = 2; + const c = 3; + return null; + }, + }, + handler: async () => json({}), + }); + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("ignores a callback passed to a decorator further along the builder chain", () => { + const ep = scanFile( + "api.v1.chained.ts", + ` + export const loader = createLoaderApiRoute({}, async () => json({})).withCors(async () => { + const a = 1; + const b = 2; + return a + b; + }); + ` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + expect(ep!.statementCount).toBe(1); + }); + + it("detects an action-only route", () => { + const ep = scanFile( + "api.v1.action-only.ts", + `export async function action() { return json({}); }` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasAction).toBe(true); + expect(ep!.hasLoader).toBe(false); + }); + + it("does not crash on a re-export or a star export", () => { + expect(() => scanFile("re-export.ts", `export { loader } from "./other";`)).not.toThrow(); + expect(() => scanFile("star.ts", `export * from "./other";`)).not.toThrow(); + const ep = scanFile("re-export.ts", `export { loader } from "./other";`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); +}); + +describe("scanFile: statement counting", () => { + it("counts only the loader's statements, not a fat exported component's", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + const a = 1; + const b = 2; + const c = 3; + const d = 4; + return null; + } + export function ErrorBoundary() { + const e = 1; + return null; + } + ` + ); + expect(ep!.statementCount).toBe(1); + }); + + it("counts through a try/catch wrapper rather than reporting 1", () => { + const ep = scanFile( + "otel.v1.traces.ts", + ` + export async function action({ request }) { + try { + const body = await request.arrayBuffer(); + const result = await process(body); + return json(result); + } catch (e) { + logger.error(e); + return json({ error: true }, { status: 500 }); + } + } + ` + ); + // try (1) + 3 in the try block + 2 in the catch block + expect(ep!.statementCount).toBe(6); + }); + + it("counts a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxyToPostHog(request) { + const url = new URL(request.url); + try { + const upstream = await fetch(url); + return new Response(upstream.body); + } catch (e) { + logger.error(e); + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxyToPostHog(request); + } + export async function action({ request }) { + return proxyToPostHog(request); + } + ` + ); + // loader (1) + action (1) + helper: const url, try (1) + 2 + 2 + expect(ep!.statementCount).toBe(8); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.calleeNames).toContain("proxyToPostHog"); + expect(ep!.calleeNames).toContain("fetch"); + }); + + it("counts a shared helper once when both the loader and the action delegate to it", () => { + const ep = scanFile( + "shared.ts", + ` + function work() { + const a = 1; + const b = 2; + return a + b; + } + export async function loader() { return work(); } + export async function action() { return work(); } + ` + ); + // loader (1) + action (1) + helper (3), the helper counted once + expect(ep!.statementCount).toBe(5); + }); + + it("follows a delegating helper one hop only", () => { + const ep = scanFile( + "two-hop.ts", + ` + function deep() { + const a = 1; + const b = 2; + const c = 3; + return a + b + c; + } + function shallow() { + return deep(); + } + export async function loader() { return shallow(); } + ` + ); + // loader (1) + shallow (1). `deep` is a second hop and is not counted. + expect(ep!.statementCount).toBe(2); + }); + + it("terminates on a recursive helper", () => { + const ep = scanFile( + "recursive.ts", + ` + function recurse(n) { + if (n <= 0) return 0; + return recurse(n - 1); + } + export async function loader() { return recurse(3); } + ` + ); + // loader (1) + recurse: if (1) + return (1) + return (1) + expect(ep!.statementCount).toBe(4); + }); + + it("does not count an imported helper it cannot resolve", () => { + const ep = scanFile( + "imported.ts", + ` + import { proxy } from "./proxy.server"; + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("does not count a same-file function the body never calls", () => { + const ep = scanFile( + "unused-helper.ts", + ` + function unrelated() { + try { + const a = 1; + const b = 2; + return a + b; + } catch (e) { + return null; + } + } + export async function loader() { + return json({}); + } + ` + ); + expect(ep!.statementCount).toBe(1); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("counts statements nested in if/for/while/switch blocks", () => { + const ep = scanFile( + "nested.ts", + ` + export async function loader() { + if (a) { + const x = 1; + doThing(x); + } + for (const i of list) { + use(i); + } + return json({}); + } + ` + ); + // if (1) + 2 nested + for (1) + 1 nested + return (1) + expect(ep!.statementCount).toBe(6); + }); +}); + +describe("scanFile: entry-point scoping", () => { + it("reports hasTryCatch false when the only try is in a non-entry-point export", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + report(e); + } + return null; + } + ` + ); + expect(ep!.hasLoader).toBe(true); + expect(ep!.hasTryCatch).toBe(false); + }); + + it("reports hasTryCatch true when the try is inside the loader", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + try { + return json(await load()); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + }); + + it("excludes callees invoked outside the loader/action body", () => { + const ep = scanFile( + "route.tsx", + ` + const schema = z.object({}); + export async function loader() { + const data = await fetchThings(); + return json(data); + } + export default function Page() { + useFancyHook(); + return null; + } + ` + ); + expect(ep!.calleeNames).toContain("fetchThings"); + expect(ep!.calleeNames).toContain("json"); + expect(ep!.calleeNames).not.toContain("useFancyHook"); + expect(ep!.calleeNames).not.toContain("object"); + }); +}); + +describe("scanFile: callee resolution", () => { + it("records the root callee of a chained builder call", () => { + const ep = scanFile( + "api.v1.cors.ts", + `export const loader = createLoaderApiRoute({}).withCors();` + ); + expect(ep!.loaderInitializerCallee).toBe("createLoaderApiRoute"); + }); + + it("leaves the callee null for a shape that cannot be named", () => { + const ep = scanFile("api.v1.anon.ts", `export const loader = async () => json({});`); + expect(ep!.hasLoader).toBe(true); + expect(ep!.loaderInitializerCallee).toBeNull(); + }); + + it("resolves a multi-level call and falls back to the bare name past an unnameable one", () => { + const ep = scanFile( + "api.v1.things.ts", + ` + export async function loader({ request }) { + try { + const org = await prisma.organization.findFirst({ where: { id: 1 } }); + return json(await new PromptService().createOverride(org)); + } catch (e) { + logger.error("nope", { error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + // A three-level property chain still lands on its bare method name. + expect(ep!.calleeNames).toContain("findFirst"); + // A chain through a `new` expression has no name of its own, so this also falls back to the + // bare name rather than losing the call. + expect(ep!.calleeNames).toContain("createOverride"); + // The full path still builds where nothing unnameable sits in it, which is what `LogCall.callee` + // depends on. + expect(ep!.logCalls[0]!.callee).toBe("logger.error"); + }); +}); + +describe("scanFile: parse failures", () => { + it("throws on a malformed source rather than returning a clean entry point", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { const a = ; return json(`) + ).toThrow(ParseFailureError); + }); + + // The shapes that prove the public route through `ts.Program` still sees a malformed file. + it("throws on an unclosed jsx element in a tsx route", () => { + expect(() => + scanFile( + "broken.route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi; }` + ) + ).toThrow(ParseFailureError); + }); + + it("throws on an unterminated template literal", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return \`unterminated; }`) + ).toThrow(ParseFailureError); + }); + + it("throws on a stray closing brace after a complete function", () => { + expect(() => + scanFile("broken.ts", `export async function loader() { return json({}); } }`) + ).toThrow(ParseFailureError); + }); + + it("names the diagnostic rather than reporting a bare failure", () => { + try { + scanFile("broken.ts", `export async function loader() { const a = ; }`); + expect.unreachable("scanFile should have thrown"); + } catch (error) { + expect(error).toBeInstanceOf(ParseFailureError); + expect((error as ParseFailureError).diagnostic.length).toBeGreaterThan(0); + } + }); + + // Both spellings find the same malformed files today, so the tests above cannot tell them apart. + // What a compiler upgrade can break is the private one, and only a source-level guard fails for it. + it("reads its diagnostics through public typescript api rather than a private field", () => { + const source = readFileSync(resolve(__dirname, "./scan.ts"), "utf8"); + expect(source).not.toContain("parseDiagnostics"); + expect(source).toContain("getSyntacticDiagnostics"); + }); + + it("does not throw on a well-formed tsx route", () => { + expect(() => + scanFile( + "route.tsx", + `export async function loader() { return json({}); } + export default function Page() { return
hi
; }` + ) + ).not.toThrow(); + }); +}); + +describe("scanDirectory", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "obs-map-scan-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("recurses into route directories and keeps file names distinct", () => { + writeFileSync(join(dir, "a.ts"), `export async function loader() { return json({}); }`); + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + mkdirSync(join(dir, "other.route")); + writeFileSync( + join(dir, "other.route", "route.tsx"), + `export async function action() { return json({}); }` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + const names = entryPoints.map((ep) => ep.fileName).sort(); + + expect(parseFailures).toEqual([]); + expect(names).toEqual(["a.ts", "nested.route/route.tsx", "other.route/route.tsx"]); + }); + + it("records a malformed file as a parse failure instead of an entry point", () => { + writeFileSync( + join(dir, "broken.ts"), + `export async function loader() { const a = ; return json(` + ); + + const { entryPoints, parseFailures } = scanDirectory(dir); + + expect(entryPoints).toEqual([]); + expect(parseFailures).toHaveLength(1); + // The file name, then the diagnostic that made it a failure. + expect(parseFailures[0]).toMatch(/^broken\.ts: \S/); + }); + + // root ignores the mode bits, so the unreadable file would read fine. + it.skipIf(process.getuid?.() === 0)( + "rethrows an error that is not a parse failure instead of counting it as one", + () => { + const unreadable = join(dir, "unreadable.ts"); + writeFileSync(unreadable, `export async function loader() { return json({}); }`); + chmodSync(unreadable, 0o000); + + try { + expect(() => scanDirectory(dir)).toThrow(/EACCES|EPERM/); + } finally { + chmodSync(unreadable, 0o600); + } + } + ); + + it("skips a non-route file inside a route directory", () => { + mkdirSync(join(dir, "nested.route")); + writeFileSync( + join(dir, "nested.route", "route.tsx"), + `export async function loader() { return json({}); }` + ); + writeFileSync( + join(dir, "nested.route", "loaders.server.ts"), + `export async function loader() { return json({}); }` + ); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["nested.route/route.tsx"]); + }); + + it("scans a route file whose name ends in .test.ts but skips .d.ts", () => { + writeFileSync( + join(dir, "projects.v3.$projectRef.test.ts"), + `export async function loader() { return json({}); }` + ); + writeFileSync(join(dir, "types.d.ts"), `export declare const x: number;`); + + const { entryPoints } = scanDirectory(dir); + + expect(entryPoints.map((ep) => ep.fileName)).toEqual(["projects.v3.$projectRef.test.ts"]); + }); +}); + +describe("scanFile: catch clause evidence", () => { + it("sets rethrows on the clause when a catch rethrows", () => { + const ep = scanFile( + "rethrow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error(e); + throw e; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(true); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // `rethrows` was once set by any `ThrowStatement` at all, so a `throw e;` appended after a `return` + // flipped a swallowing catch to inert with no behavioural change. + it("does not set rethrows for a throw that is dead code after a return", () => { + const ep = scanFile( + "dead-throw.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + throw e; + } + } + ` + ); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // Dead code from a statically-false condition, and a throw merely REGISTERED in a callback the + // clause constructs. All four must leave a plain swallow inert. + describe("dead and deferred code inside a catch does not count as evidence", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${mutation} + return null; + } + } + `; + + it("is inert as a baseline with no mutation", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Eleven shapes that put a `throw` somewhere it can never run, none of them named in the rule: a + // throw counts when it is unconditional. No claim the list is complete, and a twelfth family + // arrived the round after, see `dead throw written after something that already exited`. + const DEAD_SHAPES: Array<[string, string]> = [ + ["if (false)", "if (false) { throw e; }"], + ["while (false)", "while (false) { throw e; }"], + ["for (;false;)", "for (;false;) { throw e; }"], + ["if (true) else", "if (true) { doThing(); } else { throw e; }"], + ["switch with no matching case", "switch (1) { case 2: throw e; }"], + ["inner try/catch", "try { doThing(); } catch { throw e; }"], + ["for...of an empty array", "for (const item of []) { throw e; }"], + ["for...in an empty object", "for (const key in {}) { throw e; }"], + ["if on an empty string", 'if ("") { throw e; }'], + ["if on a negated literal", "if (!true) { throw e; }"], + ["if on a constant comparison", "if (1 === 2) { throw e; }"], + ]; + + for (const [label, shape] of DEAD_SHAPES) { + it(`does not set rethrows for a throw inside ${label}`, () => { + const ep = scanFile("x.ts", swallow(shape)); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + } + + it("does not set rethrows for a throw merely registered in a constructed callback", () => { + const ep = scanFile("x.ts", swallow("queue.push(() => { throw e; });")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + it("does not set branches for an error test merely registered in a constructed callback", () => { + const ep = scanFile( + "x.ts", + swallow("queue.push(() => { if (e instanceof Error) { doThing(); } });") + ); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // The same "merely registered" mechanism under a different callback-taking call, so the fix is + // not scoped to `.push`. + it("does not set rethrows for a throw registered in a setTimeout callback", () => { + const ep = scanFile("x.ts", swallow("setTimeout(() => { throw e; }, 0);")); + expect(ep!.catches[0]).toMatchObject({ rethrows: false, branches: false }); + }); + + // Positive control: a do/while runs its body at least once, so a throw in one is genuinely + // unconditional and the while(false) fix must not have swallowed it too. + it("still sets rethrows for a throw in a do/while, which runs its body once regardless", () => { + const ep = scanFile("x.ts", swallow("do { throw e; } while (false);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + + // The mirror of the family above: each dead spelling earns nothing and must also COST nothing, + // since a containment read of the dead statement raised `exited` and blinded the walk to the real + // classification below it, on 78 real routes. The spellings are the CORPUS ones from `dead-*`, not + // the `DEAD_SHAPES` table's, whose inner-try twin is not provably dead and gets no twin here. + describe("dead and deferred code prepended to a deciding catch does not blind it", () => { + const deciding = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${mutation} + if (e instanceof Error) { return new Response(null, { status: 400 }); } + return new Response(null, { status: 500 }); + } + } + `; + + it("decides as a baseline with no mutation", () => { + const ep = scanFile("x.ts", deciding("")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + const DEAD_PREPENDS: Array<[string, string]> = [ + ["if (false)", "if (false) { throw e; }"], + ["while (false)", "while (false) { throw e; }"], + ["for (;false;)", "for (;false;) { throw e; }"], + ["if (true) else", "if (true) { 0; } else { throw e; }"], + ["switch with no matching case", "switch (1) { case 2: throw e; }"], + ["an inner try over a literal", "try { 0; } catch { throw e; }"], + ["for...of an empty array", "for (const obsMapItem of []) { throw e; }"], + ["for...in an empty object", "for (const obsMapKey in {}) { throw e; }"], + ["if on an empty string", 'if ("") { throw e; }'], + ["if on a negated literal", "if (!true) { throw e; }"], + ["if on a constant comparison", "if (1 === 2) { throw e; }"], + ]; + + for (const [label, shape] of DEAD_PREPENDS) { + it(`keeps branches true past a dead throw inside ${label}`, () => { + const ep = scanFile("x.ts", deciding(shape)); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } + + // The dead if wrapped in a bare block, which the walk enters: the block's own live-exit read has + // to fold too, or entering it re-raises the flag the fold lowered. + it("keeps branches true past a dead throw in a block around an if (false)", () => { + const ep = scanFile("x.ts", deciding("{ if (false) { throw e; } }")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // The returns half: a dead `return null;` must not veto the rethrow, which regressed 11 real + // routes from not-applicable to fail. `dead-if-false-return` is the tree-scale version. + it("still sets rethrows past a dead return in an if (false) arm", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (false) { return null; } + throw e; + } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + // Negative controls: the fold withholds blindness and never refusal, so an always-true guard's + // throw still kills the error test after it. + it("still refuses an error test after an always-true spelling that throws", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (!false) { throw e; } + if (e instanceof Error) { return new Response(null, { status: 400 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // `case 1` matches and runs on into `case 2`, so the return is live and vetoes the rethrow. + // Misreading the slice as dead blinds the veto, which is the direction that hands out credit. + it("reads a switch fall-through onto a live return as live", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + switch (1) { case 1: 0; case 2: return null; } + throw e; + } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + }); + + // The walk enters a construct exactly where the entered statements are guaranteed to execute. + // Without these entries, relocating a clause's own statements inside a wrapper put the branch + // evidence out of reach while the returns veto still saw the return: 83 real routes per corpus + // entry. Each pair holds the wrapped and unwrapped spellings to the same evidence. + describe("the walk enters exactly the positions guaranteed to execute", () => { + const clauseEvidence = (body: string) => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + ${body} + } + }` + ); + return ep!.catches[0]!; + }; + + const DECIDING = + "if (e instanceof KnownError) { return new Response(e.code, { status: 400 }); }\n" + + "return new Response(null, { status: 500 });"; + const RETHROW = "logger.error(e);\nthrow e;"; + + const WRAPPERS: Array<[string, (body: string) => string]> = [ + ["a catchless try/finally", (body) => `try {\n${body}\n} finally { }`], + ["a single-default switch", (body) => `switch (pick()) { default: {\n${body}\n} }`], + ["an if (true)", (body) => `if (true) {\n${body}\n}`], + [ + "an if/else with the body in both arms", + (body) => `if (pick()) {\n${body}\n} else {\n${body}\n}`, + ], + ]; + + for (const [label, wrap] of WRAPPERS) { + it(`reads a deciding clause wrapped in ${label} identically`, () => { + expect(clauseEvidence(wrap(DECIDING))).toEqual(clauseEvidence(DECIDING)); + }); + + it(`reads a rethrowing clause wrapped in ${label} identically`, () => { + expect(clauseEvidence(wrap(RETHROW))).toEqual(clauseEvidence(RETHROW)); + }); + } + + // The switch entry is exact: one clause and it is a default. Anything else is not entered and + // keeps its top-level treatment, which is what the identity pair below and the `break and + // continue inside the construct they target` family already pin for the multi-clause shapes. + it("reads a clause wrapped in a single-default switch as the bare clause", () => { + expect(clauseEvidence(`switch (1) { default: {\n${DECIDING}\n} }`)).toEqual( + clauseEvidence(DECIDING) + ); + }); + + // Intersection, not union. One arm running is a condition, not a guarantee, so evidence in a + // single arm earns nothing; `dead-classifier-one-arm` in the mutation corpus is the tree-scale + // twin with the arm provably dead. + it("does not credit a classifier that sits in one arm only", () => { + const evidence = clauseEvidence( + "if (pick()) { if (e instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }\n" + + "return null;" + ); + expect(evidence.branches).toBe(false); + }); + + it("does not credit a classifier in a dead arm beside an inert arm", () => { + const evidence = clauseEvidence( + "if (false) { if (e instanceof Error) { return new Response(null, { status: 400 }); } } else { 0; }\n" + + "return null;" + ); + expect(evidence).toMatchObject({ branches: false, rethrows: false, throws: false }); + }); + + // The else-arm under a literal-true guard can never run, so nothing in it is evidence: no + // rethrow minted, and the deciding statements after the wrapper keep their credit. + it("reads a dead else arm under if true as contributing nothing", () => { + const evidence = clauseEvidence(`if (true) { 0; } else { throw e; }\n${DECIDING}`); + expect(evidence).toMatchObject({ throws: false, branches: true }); + }); + + // A throw in the tryBlock of a CAUGHT try never escapes the clause: the nested catch takes + // it. Crediting it as a rethrow would launder a returnless swallow into not-applicable. + it("does not read the tryBlock of a caught try as this clause's rethrow", () => { + const evidence = clauseEvidence("try { throw e; } catch {}\nlogger.error(e);"); + expect(evidence).toMatchObject({ rethrows: false, throws: false }); + }); + + // The walk does not enter a finally block, so the returns veto must still read it off the + // whole statement: this clause's finally return eats the throw, and the error never leaves. + it("reads a try whose finally returns as swallowing, not rethrowing", () => { + const evidence = clauseEvidence("try { throw e; } finally { return null; }"); + expect(evidence.rethrows).toBe(false); + }); + + // A finally leaving itself by `break` cancels the try's completion the same way a finally return + // does, so a throw in that tryBlock never escapes. Crediting it minted branches on 80 real + // routes; `dead-throw-in-cancelled-try` is the tree-scale twin. + it("reads a throw a finally break discards as no rethrow", () => { + const evidence = clauseEvidence( + "do { try { throw e; } finally { break; } } while (false);\nlogger.error(e);" + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false, branches: false }); + }); + + it("reads a throw a finally continue discards as no rethrow", () => { + const evidence = clauseEvidence( + 'do { try { throw e; } finally { continue; } } while (false);\nlogger.error("x", { e });' + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false, branches: false }); + }); + + it("reads a throw a switch-hosted finally break discards as no rethrow", () => { + const evidence = clauseEvidence( + "switch (0) { default: try { throw e; } finally { break; } }\nlogger.error(e);" + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false, branches: false }); + }); + + // The refusal is a containment read: a jump that only MAY run still cancels entry, because + // entry grants credit and a wrong grant pays. + it("refuses the tryBlock when the finally only may break", () => { + const evidence = clauseEvidence( + "do { try { throw e; } finally { if (pick()) { break; } } } while (false);\nlogger.error(e);" + ); + expect(evidence).toMatchObject({ rethrows: false, throws: false }); + }); + + // A loop inside the finally captures its own bare jumps, so nothing there leaves the finally + // and the try's completion stands: the rethrow is genuine. + it("does not refuse a finally whose loop captures its own break", () => { + const evidence = clauseEvidence("try { throw e; } finally { while (pick()) { break; } }"); + expect(evidence).toMatchObject({ rethrows: true, throws: true }); + }); + + // The cancelled statement contributes nothing, in either direction: no credit from inside it, + // and no blinding of the real classification after it. Same evidence as the bare clause. + it("keeps the classification after a finally-break no-op", () => { + expect( + clauseEvidence( + `do { try { if (e instanceof Error) { throw e; } } finally { break; } } while (false);\n${DECIDING}` + ) + ).toEqual(clauseEvidence(DECIDING)); + }); + + // The `definitelyExits` fold: `if (true) { X }` definitely exits iff X does, so the trailing + // throw is cut rather than read. Without the fold the throw still walks and mints `throws`. + it("cuts a dead trailing statement after an if true that exits", () => { + const evidence = clauseEvidence("if (true) { return null; }\nthrow e;"); + expect(evidence.throws).toBe(false); + }); + }); + + // The other end of the same problem: cutting the statement list only on a BARE `return`/`throw` + // left a `throw e;` after a nested construct that had already returned reading as a rethrow, worth + // 50 points a route. Two rules answer them together, `definitelyExits` seeing through the wrappers + // and `rethrows` requiring no reachable `return`. `dead-throw-after-*` is the tree-scale family. + describe("dead throw written after something that already exited", () => { + const exiting = (wrapped: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${wrapped} + throw e; + } + } + `; + + const EXITED: Array<[string, string]> = [ + ["a bare block", "{ logger.error(e); return null; }"], + ["a do body", "do { return null; } while (false);"], + ["an if (true)", "if (true) { return null; }"], + ["an if/else where both arms return", "if (pick()) { return null; } else { return 0; }"], + ["a switch with a returning default", "switch (1) { default: return null; }"], + ["a try/finally that returns", "try { return null; } finally { }"], + ]; + + for (const [label, wrapped] of EXITED) { + it(`does not set rethrows for a throw after ${label}`, () => { + const ep = scanFile("x.ts", exiting(wrapped)); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + } + + // The same wrappers on the branches side, plus three the rethrow list has no use for. This asks a + // weaker question than `definitelyExits` does, and on purpose: "could this have exited", not + // "must it have", which is why a labelled block, a `for...of` and a `while` are all on the list. + // + // The ordering is the whole trick and it is easy to get backwards. The flag is raised at the END + // of each statement, after that statement's own branch check, or every deciding statement refuses + // itself: measured at 78 routes accused. + const BRANCH_EXITED: Array<[string, string]> = [ + ...EXITED, + ["a labelled block", "outer: { return null; }"], + ["a for...of that returns", "for (const q of items) { return q; }"], + ["a while that returns", "while (go) { return null; }"], + ]; + + for (const [label, wrapped] of BRANCH_EXITED) { + it(`does not credit an error test written after ${label}`, () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + ${wrapped} + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + // The precision this gives up, pinned so it is a decision and not a surprise: a conditional exit + // before the error test also stops the credit. No clause in the tree is this shape today. + it("does not credit an error test written after a conditional return", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + if (rare) { return null; } + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still credits an error test with nothing exiting before it", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { + logger.error(e); + if (e instanceof Error) { return json({ a: 1 }); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // Positive control: nothing before the throw exits, so the clause has no other way out. + it("still sets rethrows when nothing before the throw returns", () => { + const ep = scanFile("x.ts", exiting("logger.error(e);")); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + // Second positive control, for the no-return half specifically: a `return` the walk has already + // cut as dead must not count against the rethrow. + it("still sets rethrows when the only return is dead code after the throw", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (e) { throw e; return null; } + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + }); + + // A bare `break` or `continue` targets the nearest enclosing construct of its kind rather than the + // list the question is about, so counting one wherever it appeared accused a route of swallowing an + // error it rethrows. The false-accusation direction, so both halves are pinned: what must stay + // reachable, and what must still be cut. + describe("break and continue inside the construct they target", () => { + const clause = (body: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (e) { + ${body} + } + } + `; + + // Reachable, so the throw after them is a real rethrow. Each jump targets the construct it is + // written in, and every one of these constructs falls through to the next statement. + const FALLS_THROUGH: Array<[string, string]> = [ + [ + "a switch whose clauses all break", + 'switch (e.code) { case "P2025": handleNotFound(); break; default: handleOther(); break; }', + ], + ["a switch whose default is a bare break", "switch (e.code) { default: break; }"], + ["a do body that breaks", "do { break; } while (false);"], + ["a do body that continues", "do { continue; } while (false);"], + [ + "a do body whose if/else both break", + "do { if (pick()) { break; } else { break; } } while (false);", + ], + ]; + + for (const [label, wrapped] of FALLS_THROUGH) { + it(`still sets rethrows for a throw written after ${label}`, () => { + const ep = scanFile("x.ts", clause(`${wrapped}\nthrow e;`)); + expect(ep!.catches[0]!.rethrows).toBe(true); + }); + + it(`still credits an error test written after ${label}`, () => { + const ep = scanFile( + "x.ts", + clause(`${wrapped}\nif (e instanceof Error) { return json({ a: 1 }); }\nthrow e;`) + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } + + // The clause the whole finding was about, end to end: sorting the error by code and then + // rethrowing is a rethrow, which is `not-applicable`, and never a swallow. + it("reads a switch on the error code followed by a rethrow as a rethrow", () => { + const ep = scanFile( + "x.ts", + clause( + 'switch (e.code) { case "P2025": handleNotFound(); break; default: handleOther(); break; }\nthrow e;' + ) + ); + expect(ep!.catches[0]).toMatchObject({ rethrows: true, throws: true, branches: false }); + }); + + // Cut, so the throw after them is dead and must not be credited. The first two are the + // over-correction control: a clause that returns and also breaks still exits, and reading the + // break as "no exit" would take the whole `dead-throw-after-*` family back. + const EXITS: Array<[string, string]> = [ + [ + "a switch clause that returns before it breaks", + "switch (1) { default: { return null; } break; }", + ], + [ + "a switch whose every clause returns", + "switch (e.code) { case 1: return null; default: return 0; }", + ], + ["a do body that returns before it breaks", "do { return null; break; } while (false);"], + ]; + + for (const [label, wrapped] of EXITS) { + it(`does not set rethrows for a throw written after ${label}`, () => { + const ep = scanFile("x.ts", clause(`${wrapped}\nthrow e;`)); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + } + + // A `continue` in a switch clause targets the enclosing loop, so it is inherited rather than + // dropped with the `break`. The labelled jumps beside it leave the `for` entirely, and the bare + // `break` is the control that separates the three. + const IN_LOOP: Array<[string, boolean]> = [ + ["break outer", false], + ["continue outer", false], + ["continue", false], + ["break", true], + ]; + + for (const [jump, rethrows] of IN_LOOP) { + it(`reads a switch clause that says ${jump} inside a labelled loop as rethrows=${rethrows}`, () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + outer: for (const x of items) { + try { await service.call(x); } + catch (e) { + switch (e.code) { case 1: ${jump}; default: ${jump}; } + throw e; + } + } + return null; + }` + ); + expect(ep!.catches[0]!.rethrows).toBe(rethrows); + }); + } + }); + + // A clause whose try block cannot throw is not error handling. Crediting one was the largest hole + // ever found here: 19 to 44 on the real tree and 224 routes raised. `dead-classifying-try` is the + // tree-scale version. + describe("a catch over a try block that cannot throw", () => { + const guarding = (guarded: string) => ` + export async function loader({ request }) { + try { ${guarded} } catch (e) { + if (e instanceof Error) { return new Response(null, { status: 400 }); } + throw e; + } + return await prisma.thing.findMany(); + } + `; + + const INERT: Array<[string, string]> = [ + ["an empty block", ""], + ["a literal expression statement", "0;"], + ["a literal declaration", "const x = 1;"], + ["arithmetic on literals", "const x = 1 + 2 * 3;"], + ["a bare identifier read", "const x = someLocal;"], + ]; + + for (const [label, guarded] of INERT) { + it(`is not read as error handling when the try holds only ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(false); + }); + } + + // One per reason `canRaise` recognises, so the predicate is not passing the cases above by being + // false for everything. + const LIVE: Array<[string, string]> = [ + ["a call", "doThing();"], + ["a construction", "new Thing();"], + ["an await", "await later;"], + ["a member access", "const x = thing.value;"], + ["an element access", "const x = thing[0];"], + ["a throw", "throw new Error('x');"], + ["an iteration", "for (const item of items) { }"], + ["an instanceof", "const x = thing instanceof Error;"], + ]; + + for (const [label, guarded] of LIVE) { + it(`is read as error handling when the try holds ${label}`, () => { + const ep = scanFile("x.ts", guarding(guarded)); + expect(ep!.catches[0]!.guardCanRaise).toBe(true); + }); + } + }); + + it("leaves both flags false when the catch only returns", () => { + const ep = scanFile( + "swallow.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("sets branches for an `if` on the error", () => { + const ep = scanFile( + "branch-if.ts", + ` + export async function loader({ request }) { + try { + return json(await request.json()); + } catch (e) { + if (e instanceof SyntaxError) { + return json({ error: "bad json" }, { status: 400 }); + } + return json({ error: "failed" }, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + }); + + it("sets branches for an instanceof conditional that is the whole returned expression", () => { + const ep = scanFile( + "branch-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + return e instanceof Response ? e : json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("sets branches for a switch on the error", () => { + const ep = scanFile( + "branch-switch.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + switch (e.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("ignores a catch that lives in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json({}); + } + export default function Page() { + try { + render(); + } catch (e) { + if (e instanceof RenderError) throw e; + return null; + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catches).toEqual([]); + }); + + it("reads a catch inside a same-file helper the body delegates to", () => { + const ep = scanFile( + "ph.$.ts", + ` + async function proxy(request) { + try { + return await fetch(request.url); + } catch (e) { + if (e.name === "AbortError") throw e; + return new Response(null, { status: 502 }); + } + } + export async function loader({ request }) { + return proxy(request); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + // The `throw e` is guarded by an `if`, so it is not on the straight-line path and does not read as + // a rethrow. The `if` itself decides, so the verdict the checks care about is unchanged. + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("leaves catches empty for a try with no catch", () => { + const ep = scanFile( + "finally-only.ts", + ` + export async function loader() { + try { + return json(await load()); + } finally { + release(); + } + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches).toEqual([]); + }); + + it("flags a try that guards a single request.json()", () => { + const ep = scanFile( + "admin.api.v1.platform-notifications.ts", + ` + export async function action({ request }) { + const user = await requireUser(request); + let body; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const result = await createPlatformNotification(body); + return json(result); + } + ` + ); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.catches[0]!.rethrows).toBe(false); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("is empty when there is no try at all", () => { + const ep = scanFile("plain.ts", `export async function loader() { return json({}); }`); + expect(ep!.hasTryCatch).toBe(false); + expect(ep!.catches).toEqual([]); + }); + + // Stopping at ANY function-like node excluded a per-item `.map()` boundary correctly and also + // deleted the route's own catch whenever the body was wrapped in a single-shot callback. The real + // distinction is per-item iteration versus everything else. + describe("inline single-shot wrappers are attributed to the route", () => { + it("attributes a catch wrapped in trace(async () => {...})", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + return trace("update", async () => { + try { + await doWork(request); + return json({ ok: true }); + } catch { + return json({ error: "failed" }, { status: 500 }); + } + }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside a pgMutation callback passed as an object property", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const outcome = await mutateWithFallback({ + pgMutation: async (taskRun) => { + try { + await doWork(taskRun); + } catch { + return json({ error: "Internal Server Error" }, { status: 500 }); + } + }, + }); + return outcome; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + + it("attributes a catch inside new ReadableStream({ start })", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const stream = new ReadableStream({ + async start(controller) { + try { + await doWork(controller); + } catch { + controller.error("failed"); + } + }, + }); + return new Response(stream); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + }); + }); + + describe("per-item iteration callbacks are not attributed to the route", () => { + it("does not attribute a catch inside items.map(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map((item) => { + try { + return process(item); + } catch { + return null; + } + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); + + it("does not attribute a catch inside Promise.all(items.map(...))", () => { + const ep = scanFile( + "batch.process.ts", + ` + export async function action({ request }) { + const items = await loadItems(request); + await Promise.all( + items.map(async (item) => { + try { + await processItem(item); + } catch { + return null; + } + }) + ); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + // A try/catch appeared somewhere in the body, which is still a real signal for triviality. + expect(ep!.hasTryCatch).toBe(true); + }); + + it("does not attribute a catch inside items.forEach(...)", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + items.forEach((item) => { + try { + process(item); + } catch { + return; + } + }); + return json({ ok: true }); + } + ` + ); + expect(ep!.catches).toEqual([]); + }); + + // A refused catch keeps its evidence, so `error-classification` can judge what it does rather + // than where it sits. Both flavours are pinned, the deciding per-item catch and the inert one. + it("populates evidence for a refused per-item catch that decides", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map(async (item) => { + try { + return await process(item); + } catch (e) { + if (e instanceof KnownError) { return new Response(e.code, { status: 400 }); } + throw e; + } + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + expect(ep!.callbackCatches).toHaveLength(1); + expect(ep!.callbackCatches[0]).toMatchObject({ branches: true, guardCanRaise: true }); + }); + + it("populates evidence for a refused per-item catch that only rethrows", () => { + const ep = scanFile( + "x.ts", + ` + export async function action({ request }) { + const items = await load(request); + return items.map(async (item) => { + try { + return await process(item); + } catch (e) { + throw e; + } + }); + } + ` + ); + expect(ep!.catches).toEqual([]); + expect(ep!.callbackCatches).toHaveLength(1); + expect(ep!.callbackCatches[0]).toMatchObject({ rethrows: true, branches: false }); + }); + }); +}); + +describe("scanFile: log calls", () => { + it("records the fields of a log call's object argument", () => { + const ep = scanFile( + "logging.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + logger.error("load failed", { environmentId: env.id, error: e }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.logCalls).toHaveLength(1); + expect(ep!.logCalls[0]).toEqual({ + callee: "logger.error", + fields: ["environmentId", "error"], + inCatch: true, + }); + }); + + it("records a log call with no object argument, outside a catch", () => { + const ep = scanFile( + "logging-plain.ts", + ` + export async function loader() { + log.info("starting"); + return json({}); + } + ` + ); + expect(ep!.logCalls).toEqual([{ callee: "log.info", fields: [], inCatch: false }]); + }); + + it("ignores a non-logger call and a log call in the React component", () => { + const ep = scanFile( + "route.tsx", + ` + export async function loader() { + return json(await load()); + } + export default function Page() { + logger.debug("rendered", { runId: 1 }); + return null; + } + ` + ); + expect(ep!.logCalls).toEqual([]); + }); +}); + +describe("scanFile: per-catch evidence", () => { + it("records one entry per catch clause, keeping a parse guard distinct from a broad catch", () => { + const ep = scanFile( + "two-catches.ts", + ` + export async function action({ request }) { + let body; + try { + body = await request.json(); + } catch { + return json({}, { status: 400 }); + } + try { + const run = await find(body.id); + const updated = await update(run); + await notify(updated); + return json(updated); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + expect(ep!.catches[0]).toEqual({ + rethrows: false, + branches: false, + throws: false, + guardsParse: true, + awaitsOnlyParse: true, + guardCanRaise: true, + guardMayRaise: true, + tryStatementCount: 1, + }); + expect(ep!.catches[1]).toMatchObject({ + guardsParse: false, + tryStatementCount: 4, + }); + }); + + it("leaves catches empty for a try/finally with no catch clause", () => { + const ep = scanFile( + "runs-replication.status.ts", + ` + export async function loader() { + const redis = createRedis(); + try { + for (const source of sources) { + const exists = await redis.exists(source.slotName); + leaders.set(source.id, exists === 1); + } + } finally { + await redis.quit(); + } + return json({}); + } + ` + ); + expect(ep!.catches).toEqual([]); + // hasTryCatch keeps its meaning: a `try` appears. Nothing is caught here. + expect(ep!.hasTryCatch).toBe(true); + }); + + it("sees a URL constructor as a guarded parse", () => { + const ep = scanFile( + "_app.@.orgs.$organizationSlug.$.tsx", + ` + function refererOrigin(request) { + const referer = request.headers.get("referer"); + if (!referer) return undefined; + try { + return new URL(referer).origin; + } catch { + return undefined; + } + } + export async function action({ request }) { + const origin = refererOrigin(request); + return json({ origin }); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); + + it("sees a parse in a try that outgrows a single statement", () => { + const ep = scanFile( + "admin.api.v1.orgs.$organizationId.stream-basin.ts", + ` + export async function action({ request }) { + let parsed; + try { + const text = await request.text(); + const raw = text.length > 0 ? JSON.parse(text) : {}; + const result = BodySchema.safeParse(raw); + if (!result.success) { + return json({ ok: false }, { status: 400 }); + } + parsed = result.data; + } catch { + return json({ ok: false, error: "Invalid JSON body" }, { status: 400 }); + } + return json(parsed); + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + expect(ep!.catches[0]!.tryStatementCount).toBe(6); + }); + + it("does not call a broad catch over database work a parse guard", () => { + const ep = scanFile( + "broad.ts", + ` + export async function loader({ params }) { + try { + const run = await prisma.run.findFirst({ where: { id: params.id } }); + const events = await prisma.event.findMany({ where: { runId: run.id } }); + await touch(run); + return json({ run, events }); + } catch (e) { + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]).toEqual({ + rethrows: false, + branches: false, + throws: false, + guardsParse: false, + awaitsOnlyParse: false, + guardCanRaise: true, + guardMayRaise: true, + tryStatementCount: 4, + }); + }); + + it("keeps rethrow and branch evidence per clause", () => { + const ep = scanFile( + "mixed-clauses.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) throw e; + return json({}, { status: 500 }); + } + } + export async function action({ request }) { + try { + return json(await save(request)); + } catch (e) { + return null; + } + } + ` + ); + expect(ep!.catches).toHaveLength(2); + // The loader's clause branches and does not rethrow; the action's does neither. What this test is + // for is that the two clauses stay separate. + expect(ep!.catches.filter((c) => !c.rethrows && c.branches)).toHaveLength(1); + expect(ep!.catches.filter((c) => !c.rethrows && !c.branches)).toHaveLength(1); + }); + + it("includes a catch from a same-file helper and excludes one from the React component", () => { + const ep = scanFile( + "route.tsx", + ` + function parseTags(payload) { + try { + return JSON.parse(payload); + } catch { + return null; + } + } + export async function loader({ params }) { + return json(parseTags(params.payload)); + } + export default function Page() { + try { + render(); + } catch (e) { + throw e; + } + return null; + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); +}); + +describe("scanFile: guardsParse is limited to parsing constructors", () => { + it("does not treat a presenter construction as a parse guard", () => { + const ep = scanFile( + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.branches/route.tsx", + ` + export async function loader({ request, params }) { + try { + const presenter = new BranchesPresenter(); + const result = await presenter.call({ userId: 1, projectSlug: params.projectParam }); + return typedjson(result); + } catch (error) { + logger.error("Error loading preview branches page", { error }); + throw new Response(undefined, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("does not treat a collection construction as a parse guard", () => { + const ep = scanFile( + "account.tokens/route.tsx", + ` + export async function action({ request }) { + try { + const roles = await loadRoles(request); + const names = new Set(roles.map((r) => r.name)); + return json({ names: [...names] }); + } catch (error) { + return json({ error: "failed" }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(false); + }); + + it("treats RegExp and URLSearchParams construction as a parse guard", () => { + const regexp = scanFile( + "regexp.ts", + ` + export async function action({ request }) { + const pattern = await patternFrom(request); + try { + new RegExp(pattern); + } catch { + return json({ error: "Invalid regex" }, { status: 400 }); + } + return json({ ok: true }); + } + ` + ); + expect(regexp!.catches[0]!.guardsParse).toBe(true); + + const search = scanFile( + "search-params.ts", + ` + export async function loader({ request }) { + try { + return json(Object.fromEntries(new URLSearchParams(request.url))); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(search!.catches[0]!.guardsParse).toBe(true); + }); + + it("still sees a parse call in a try that also constructs something ordinary", () => { + const ep = scanFile( + "parse-and-construct.ts", + ` + export async function action({ request }) { + try { + const body = JSON.parse(await request.text()); + const service = new PromptService(); + return json(await service.create(body)); + } catch { + return json({}, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.guardsParse).toBe(true); + }); +}); + +describe("scanFile: branches ignores the error-stringifying ternary", () => { + it("does not count an instanceof nested inside a returned call argument", () => { + const ep = scanFile( + "admin.api.v1.runs-replication.start.ts", + ` + export async function action({ request }) { + try { + return json(await start(request)); + } catch (error) { + return json({ error: error instanceof Error ? error.message : error }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches).toHaveLength(1); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not count an instanceof used to build a logged message", () => { + const ep = scanFile( + "admin.api.v1.feature-flags.ts", + ` + export async function action({ request }) { + try { + return json(await setFlag(request)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.error("flag update failed", { message }); + return json({ error: message }, { status: 400 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still counts an instanceof in an `if`", () => { + const ep = scanFile( + "branch-if-instanceof.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof Response) return e; + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); + +describe("scanFile: branches requires the if/switch condition to examine the error", () => { + it("does not set branches for an `if` on an unrelated variable", () => { + const ep = scanFile( + "retry-count.ts", + ` + export async function action({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch (e) { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("sets branches for an `if` whose condition references the caught error", () => { + const ep = scanFile( + "branch-if-e.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (e) { + if (e instanceof ApiError) return json({}, { status: e.status }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("sets branches for a `switch` on a property of the caught error", () => { + const ep = scanFile( + "branch-switch-error-code.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + switch (error.code) { + case "P2025": + return json({}, { status: 404 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("does not set branches for a `switch` on an unrelated discriminant", () => { + const ep = scanFile( + "branch-switch-unrelated.ts", + ` + export async function action({ request }) { + const mode = "strict"; + try { + return json(await load(request)); + } catch (error) { + switch (mode) { + case "strict": + return json({}, { status: 400 }); + default: + return json({}, { status: 500 }); + } + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("cannot set branches for a bindingless catch, even with an `if` inside it", () => { + const ep = scanFile( + "bindingless-if.ts", + ` + export async function loader({ request }) { + let attempt = 0; + try { + return json(await load(request)); + } catch { + if (attempt > 3) return json({}, { status: 503 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); + +// `referencesBinding` once matched any identifier with the binding's text, including a property name, +// an object literal key and a name re-declared in a nested scope. +describe("scanFile: branches requires a genuine read of the binding, not a lookalike", () => { + it("does not set branches for an `if` that only reads a same-named property", () => { + const ep = scanFile( + "branch-property-name.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` that only reads a same-named object literal key", () => { + const ep = scanFile( + "branch-object-key.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (buildOptions({ error: false }).ok) return json({}, { status: 500 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a callback that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-param.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (items.some(function (error) { return error.code === 1; })) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not set branches for an `if` whose only reference is inside a block that re-declares the name", () => { + const ep = scanFile( + "branch-shadowed-block.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if ( + (() => { + const error = 1; + return error > 0; + })() + ) { + return json({}, { status: 500 }); + } + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still sets branches for an `if` that genuinely reads the binding beside a lookalike", () => { + const ep = scanFile( + "branch-genuine-and-lookalike.ts", + ` + export async function loader({ request }) { + try { + return json(await load(request)); + } catch (error) { + if (fallback.error || error instanceof NotFound) return json({}, { status: 404 }); + return json({}, { status: 500 }); + } + } + ` + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); +}); + +// The shadow check once only caught shadowing NESTED inside the condition, so a scope that WRAPS the +// if was invisible. `catchClauseEvidence` tracks shadowing as it descends instead: once a scope +// re-declares the binding, everything nested inside stays shadowed. +describe("scanFile: a binding shadowed by an enclosing scope, not just a nested one", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return null; + } + } + `; + + it("does not credit an if inside a for...of loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error of errors) { if (error.code === 1) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a for...in loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (const error in errorsByKey) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a classic for loop that re-declares the binding", () => { + const ep = scanFile( + "x.ts", + swallow("for (let error = 0; error < 10; error++) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a nested catch clause with the same binding name", () => { + const ep = scanFile( + "x.ts", + swallow("try { doWork(); } catch (error) { if (error) { doThing(); } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if inside a block whose own destructured const shadows the binding", () => { + const ep = scanFile( + "x.ts", + swallow(` + if (attempt > 0) { + const { error } = computeSomething(); + if (error) { doThing(); } + } + `) + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if reading a destructured parameter inside the if's own condition", () => { + const ep = scanFile( + "x.ts", + swallow("if (items.some(({ error }) => error > 0)) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if shadowed by an array-destructured declaration", () => { + const ep = scanFile( + "x.ts", + swallow("const [error] = getErrors();\n if (error) { doThing(); }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // Positive control: a genuine reference to the real binding, on the clause's own path, with an + // arm that takes the error somewhere the other arm does not go. + it("still credits an if that genuinely reads the outer binding directly", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { return badRequest(); }")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // Two shapes that read the real binding and are still not credited, for reasons that are not + // shadowing: precision the straight-line rule gives up on purpose. + it("does not credit an if whose arm does not take the error anywhere", () => { + const ep = scanFile("x.ts", swallow("if (error instanceof Error) { doThing(); }")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit an if nested inside a loop, even with a different loop variable", () => { + const ep = scanFile( + "x.ts", + swallow("for (const item of items) { if (error.code === item) { return item; } }") + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); + +// The ternary path once checked only that the condition tested the error, so rewriting `return X;` as +// `return e instanceof Error ? (X) : (X)` was worth 50 points a route for a change that decides +// nothing. `same-arms-ternary` is the tree-scale version. +describe("a ternary on the error has to send its arms somewhere different", () => { + const returning = (value: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + return ${value}; + } + } + `; + + it("does not credit a ternary whose arms are identical", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json({})) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in whitespace", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? (json( {} )) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a ternary whose arms differ only in parentheses", () => { + const ep = scanFile("x.ts", returning("error instanceof Error ? ((json({}))) : (json({}))")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("still credits a ternary whose arms go somewhere different", () => { + const ep = scanFile( + "x.ts", + returning("error instanceof Response ? error : json({}, { status: 500 })") + ); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + // The same three cases on the throw path, which read none of them: the walk sets `rethrows` and + // cuts the path first, so a thrown ternary was never offered to `selectsAnErrorPath`. + // `wrap-body-in-same-arms-throw-ternary` is the tree-scale version of the refusal. + const throwing = (value: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + throw ${value}; + } + } + `; + + it("credits a thrown ternary whose arms go somewhere different", () => { + const ep = scanFile("x.ts", throwing("error instanceof Response ? error : new Error('x')")); + expect(ep!.catches[0]!.branches).toBe(true); + }); + + it("does not credit a thrown ternary whose arms are identical", () => { + const ep = scanFile("x.ts", throwing("error instanceof Error ? error : error")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a thrown ternary whose arms differ only in parentheses", () => { + const ep = scanFile("x.ts", throwing("error instanceof Error ? ((error)) : (error)")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + it("does not credit a thrown ternary that never reads the caught binding", () => { + const ep = scanFile("x.ts", throwing("other instanceof Error ? error : new Error('x')")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The throw still cuts the path, so a decision written after it is dead and stays uncredited. + it("does not credit a thrown ternary written after the clause already threw", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + throw error; + throw error instanceof Response ? error : new Error('x'); + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The same comparison on the `if` path, which had the exit test but not the arm test. + it("does not credit an if/else whose two arms are identical", () => { + const ep = scanFile( + "x.ts", + `export async function loader() { + try { return await prisma.thing.findMany(); } + catch (error) { + if (error instanceof Error) { return json({}); } else { return json({}); } + } + }` + ); + expect(ep!.catches[0]!.branches).toBe(false); + }); +}); + +// The liveness gap in the branch predicate: under a plain containment read, an arm holding an exit +// that can never run read as an arm that takes the error somewhere, worth 50 points a route. +// `dead-armed-instanceof-if` is the tree-scale version, global 19 to 27 and 80 routes raised. +describe("an arm whose only exit is dead decides nothing", () => { + const swallow = (mutation: string) => ` + export async function loader() { + try { + return await prisma.thing.findMany(); + } catch (error) { + ${mutation} + return json({ error: "generic" }, { status: 500 }); + } + } + `; + + it("reads the unmutated clause as a swallow, as a baseline", () => { + const ep = scanFile("x.ts", swallow("")); + expect(ep!.catches[0]!.branches).toBe(false); + }); + + // The reported shape, plus three further spellings of the same no-op reaching the same + // predicate by different routes: a dead loop body, a dead else arm, and a switch clause. + const DEAD_ARMS: Array<[string, string]> = [ + ["an if (false) arm", "if (error instanceof Error) { if (false) { return null; } }"], + ["a while (false) body", "if (error instanceof Error) { while (false) { throw error; } }"], + [ + "a dead else arm beside an arm that goes nowhere", + "if (error instanceof Error) { doThing(); } else { for (const k in {}) { return null; } }", + ], + [ + "a switch clause whose exit is dead", + "switch (error.code) { case 'x': if (1 === 2) { throw error; } }", + ], + ]; + + for (const [label, mutation] of DEAD_ARMS) { + it(`does not credit ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(false); + }); + } + + // Positive controls: without these the fix could be "always false" and the four cases above would + // still pass. `an arm guarded by a condition that does not fold` is also the pin on the rejected + // alternative, asking the arm to `definitelyExits` rather than to hold a live exit, which refuses + // all four shapes above and falsely accuses a real classifying clause in + // `admin.api.v1.orgs.$organizationId.environments.staging.ts`, taking the global from 19 to 18. + const LIVE_ARMS: Array<[string, string]> = [ + ["a plain returning arm", "if (error instanceof Error) { return badRequest(); }"], + [ + "an arm guarded by a condition that does not fold", + "if (error instanceof Error) { if (error.code === 'P2002') { return conflict(); } }", + ], + [ + "a live exit in the else arm only", + "if (error instanceof Error) { doThing(); } else { return badRequest(); }", + ], + ["a switch clause that returns", "switch (error.code) { case 'P2002': return conflict(); }"], + ]; + + for (const [label, mutation] of LIVE_ARMS) { + it(`still credits ${label}`, () => { + const ep = scanFile("x.ts", swallow(mutation)); + expect(ep!.catches[0]!.branches).toBe(true); + }); + } +}); + +// `export const { action, loader } = createActionApiRoute(...)` produced no entry point at all, so +// the route was absent from the denominator rather than parsed, failed or unmeasured. The two-step +// spelling already worked, so exactly half the shape was wired. +describe("scanFile: a destructured export declaration", () => { + const BUILDER = `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";`; + + it("finds an action destructured straight out of a builder call", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action } = createActionApiRoute({ method: "POST" }, async () => json({}));` + ); + expect(ep).not.toBeNull(); + expect(ep!.hasAction).toBe(true); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + it("finds both halves of a destructured builder export", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action, loader } = createActionApiRoute({}, async () => json({}));` + ); + expect(ep!.hasAction).toBe(true); + expect(ep!.hasLoader).toBe(true); + }); + + it("finds an action destructured from a local the builder produced", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + const route = createActionApiRoute({}, async () => json({})); + export const { action } = route;` + ); + expect(ep).not.toBeNull(); + expect(ep!.actionInitializerCallee).toBe("createActionApiRoute"); + }); + + // The exported name is the element name, so a rename decides what this file exports. + it("reads the exported name rather than the property it came from", () => { + const renamed = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { loader: action } = createActionApiRoute({}, async () => json({}));` + ); + expect(renamed!.hasAction).toBe(true); + expect(renamed!.hasLoader).toBe(false); + + const hidden = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action: internal } = createActionApiRoute({}, async () => json({}));` + ); + expect(hidden).toBeNull(); + }); + + it("counts the statements of a handler reached through the binding pattern", () => { + const ep = scanFile( + "api.v1.things.ts", + `${BUILDER} + export const { action } = createActionApiRoute({}, async ({ request }) => { + const body = await request.json(); + const saved = await prisma.thing.create({ data: body }); + return json(saved); + });` + ); + expect(ep!.statementCount).toBe(3); + }); +}); + +// C4b. A route whose body is in another module gives zero statements and zero callees, so the +// triviality rule read it as a redirect stub and every check reported not-applicable for it. The +// scan says so directly instead, and `score.ts` counts it apart from the routes nothing applied to. +describe("scanFile: a route that delegates its body to another module", () => { + it("marks a re-export of an action from another module", () => { + const ep = scanFile("webhooks.v1.stripe.ts", `export { action } from "./handler.server";`); + expect(ep!.delegating).toBe(true); + }); + + it("marks an action aliased to an imported function", () => { + const ep = scanFile( + "webhooks.v1.stripe.ts", + `import { handleWebhook } from "./handler.server"; + export const action = handleWebhook;` + ); + expect(ep!.delegating).toBe(true); + }); + + it("marks a renamed re-export", () => { + const ep = scanFile( + "webhooks.v1.stripe.ts", + `export { handleWebhook as action } from "./handler.server";` + ); + expect(ep!.delegating).toBe(true); + }); + + it("does not mark a route whose body is in the file", () => { + const ep = scanFile("api.v1.things.ts", LOADER); + expect(ep!.delegating).toBe(false); + }); + + it("does not mark a route wrapped in a builder, whose options are still readable", () => { + const ep = scanFile( + "api.v1.things.ts", + `import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + import { handler } from "./handler.server"; + export const action = createActionApiRoute({ method: "POST" }, handler);` + ); + expect(ep!.delegating).toBe(false); + }); + + // Half a route in the file is still half a route to judge. + it("does not mark a route that delegates one export and writes the other", () => { + const ep = scanFile( + "api.v1.things.ts", + `export { action } from "./handler.server"; + export async function loader() { return json({}); }` + ); + expect(ep!.delegating).toBe(false); + }); +}); + +// C1b. What `auth-scope` reads: the options a builder was given, whether the handler filters by +// the caller's own id, and whether it runs the ability gate itself. +describe("scanFile: the signals auth-scope reads", () => { + const PAT = `import { createActionPATApiRoute } from "~/services/routeBuilders/apiBuilder.server";`; + + it("records the top-level option keys of the builder call", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const action = createActionPATApiRoute( + { method: "POST", authorization: { action: "manage", resource: { type: "org" } } }, + async () => json({}) + );` + ); + expect(ep!.actionBuilderOptions).toEqual(["method", "authorization"]); + }); + + it("keeps the loader's options and the action's options apart", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({ authorization: {} }, async () => json({})); + export const action = createActionPATApiRoute({ method: "POST" }, async () => json({}));` + ); + expect(ep!.loaderBuilderOptions).toEqual(["authorization"]); + expect(ep!.actionBuilderOptions).toEqual(["method"]); + }); + + it("sees a query filtered by the caller's id, in both builders' spellings", () => { + const api = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ authentication }) => { + return json(await prisma.org.findMany({ + where: { members: { some: { userId: authentication.userId } } }, + })); + });` + ); + expect(api!.loaderScopesByCaller).toBe(true); + + const dashboard = scanFile( + "_app.orgs.$slug.apikeys/route.tsx", + `import { dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + export const loader = dashboardLoader({}, async ({ user }) => { + return json(await presenter.call({ userId: user.id, slug: "x" })); + });` + ); + expect(dashboard!.loaderScopesByCaller).toBe(true); + }); + + // Round C ruling 1. The signal is per export because the exposure is per export. Entry-point-wide + // it passed `_app.orgs.$organizationSlug.settings.team/route.tsx`, whose loader narrows itself to + // the caller and whose action resolves the target org from the URL slug. + it("attributes the caller filter to the export it was written in", () => { + const ep = scanFile( + "_app.orgs.$slug.settings.team/route.tsx", + `import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; + export const loader = dashboardLoader({}, async ({ user }) => + json(await presenter.call({ userId: user.id })) + ); + export const action = dashboardAction({}, async ({ context, request }) => + json(await prisma.orgMember.deleteMany({ where: { organizationId: context.organizationId } })) + );` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + expect(ep!.actionScopesByCaller).toBe(false); + }); + + // Round D item 3. The predicate fired on any property at all whose value was a caller id, so one + // dead statement cleared the check. Both halves are now required: an identity property name, and + // an object that is handed to a call. + it("does not read a dead object holding the caller id as a scope", () => { + const arbitraryKey = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + const unused = { anything: user.id }; + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(arbitraryKey!.loaderScopesByCaller).toBe(false); + + const identityKey = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + const unused = { userId: user.id }; + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(identityKey!.loaderScopesByCaller).toBe(false); + }); + + it("reads the caller id through any depth of nesting inside a call argument", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return json(await prisma.org.findMany({ + where: { OR: [{ members: { some: { userId: user.id } } }] }, + })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + }); + + it("does not read a non-identity field off the caller as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return json(await prisma.org.findMany({ where: { title: user.name } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // Round E item 3. Neither of the two conditions above constrains the CALLEE, so a log line + // carrying the caller's id satisfied a tenant-scoping security check. Cheaper to write than the + // dead object, and unlike the dead object it survives review, because a log line is real code + // somebody wants. + it("does not read a caller id handed to a log call as a scope", () => { + const logger = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.error("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(logger!.loaderScopesByCaller).toBe(false); + + // A different logger family: `LOGGER_CALLEE` does not match `console.warn`, so this is the + // second sink rather than a restatement of the first. + const console_ = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + console.warn("create failed", { userId: user.id }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(console_!.loaderScopesByCaller).toBe(false); + }); + + // The refusal is made at the call, not at the property, so burying the id under the depth of + // nesting a real filter has does not get it past. + it("does not read a caller id nested inside a log call's payload as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + logger.info("looking up", { where: { OR: [{ userId: user.id }] } }); + return json(await prisma.org.findMany({ where: { slug: "x" } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The response body is the other sink that takes the same object and cannot narrow a read. + it("does not read a caller id handed to a response serializer as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ user }) => { + return typedjson({ userId: user.id }); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); + + // The direction that matters more than any of the above: refusing the log line must not accuse a + // handler that also runs the query. This is the shape on the real tree today, + // `engine.v1.dev.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts`, which logs + // the environment id and reads with it, and which must keep its pass. + it("still reads a real query filter written beside a log call", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async ({ authentication }) => { + const run = await runStore.findRun({ runtimeEnvironmentId: authentication.environment.id }); + if (!run) logger.error("no run", { environmentId: authentication.environment.id }); + return json(run); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(true); + }); + + // A resource's owner is not the caller. + it("does not read another object's userId as a scope", () => { + const ep = scanFile( + "api.v1.orgs.ts", + `${PAT} + export const loader = createActionPATApiRoute({}, async () => { + return json(await prisma.org.findMany({ where: { userId: run.userId } })); + });` + ); + expect(ep!.loaderScopesByCaller).toBe(false); + }); +}); + +/** + * The per-export split of `calleeNames`. `auth-boundary` reads it, so a name landing in the wrong + * half is a wrong verdict on the one check where a false pass hides a security gap. + */ +describe("scanFile: callee names attributed per export", () => { + it("keeps each export's callees out of the other's list", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export async function loader({ request }) { + const userId = await requireUserId(request); + return json(await prisma.token.findMany({ where: { userId } })); + } + export async function action({ request }) { + await deleteEverything(request); + return json({ ok: true }); + }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.loaderCalleeNames).not.toContain("deleteEverything"); + expect(ep!.actionCalleeNames).toContain("deleteEverything"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a same-file helper to the export that calls it", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function loadTokens(request) { + const userId = await requireUserId(request); + return prisma.token.findMany({ where: { userId } }); + } + export async function loader({ request }) { return json(await loadTokens(request)); } + export async function action({ request }) { return json(await request.json()); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).not.toContain("requireUserId"); + }); + + it("attributes a helper both exports call to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function guarded(request) { return requireUserId(request); } + export async function loader({ request }) { return json(await guarded(request)); } + export async function action({ request }) { return json(await guarded(request)); }` + ); + expect(ep!.loaderCalleeNames).toContain("requireUserId"); + expect(ep!.actionCalleeNames).toContain("requireUserId"); + }); + + // One handler, both exports: `const { loader, action } = createActionApiRoute({ handler })`. + it("attributes a handler serving both exports to both of them", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const { action, loader } = createActionApiRoute({}, async ({ authentication }) => { + return json(await authenticateApiRequest(authentication)); + });` + ); + expect(ep!.loaderCalleeNames).toContain("authenticateApiRequest"); + expect(ep!.actionCalleeNames).toContain("authenticateApiRequest"); + }); + + // The union the split came from. `calleeNames` stays entry-point-wide for `sensitivity.ts`, + // `triviality.ts` and `audit-trail`, so the two representations have to agree. + it("every callee name is attributed to an export that exists", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `async function shared(request) { return audit(request); } + export async function loader({ request }) { return json(await shared(request)); } + export async function action({ request }) { return json(await mutate(request)); }` + ); + const attributed = new Set([...ep!.loaderCalleeNames, ...ep!.actionCalleeNames]); + expect([...new Set(ep!.calleeNames)].filter((n) => !attributed.has(n))).toEqual([]); + for (const name of attributed) expect(ep!.calleeNames).toContain(name); + }); + + it("counts statements and a try per export as well as entry-point-wide", () => { + const ep = scanFile( + "api.v1.tokens.ts", + `export const loader = () => redirect("/login"); + export async function action({ request }) { + try { return json(await request.json()); } catch (e) { throw e; } + }` + ); + expect(ep!.loaderStatementCount).toBe(1); + expect(ep!.loaderHasTryCatch).toBe(false); + expect(ep!.actionHasTryCatch).toBe(true); + expect(ep!.hasTryCatch).toBe(true); + expect(ep!.statementCount).toBe(ep!.loaderStatementCount + ep!.actionStatementCount); + }); +}); + +// Round C ruling 2. A guard that answers with null instead of throwing is only a boundary if the +// route reads the answer, so the scan records which callees' results a condition looked at. +describe("scanFile: callees whose answer the body read", () => { + it("records a resolver whose result a negated if tests", () => { + const ep = scanFile( + "invite-accept.tsx", + `import { getUser } from "~/services/session.server"; + export async function loader({ request }) { + const user = await getUser(request); + if (!user) return redirect("/login"); + return json({ email: user.email }); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("getUser"); + }); + + it("records one whose result a plain if tests", () => { + const ep = scanFile( + "login._index/route.tsx", + `import { getUserId } from "~/services/session.server"; + export async function loader({ request }) { + const userId = await getUserId(request); + if (userId) throw redirect("/"); + return typedjson({}); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("getUserId"); + }); + + it("records one whose result a ternary tests", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + const user = await getUser(request); + return user ? json({ ok: true }) : redirect("/login"); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("getUser"); + }); + + it("does not record a result that is bound and never tested", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + const user = await getUser(request); + return json(await prisma.invite.findMany({ where: { email: user.email } })); + }` + ); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); + }); + + it("does not record a result that is dropped entirely", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + await getUser(request); + return json(await prisma.invite.findMany()); + }` + ); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); + }); + + it("does not record a callee just because a same-named local is tested elsewhere", () => { + const ep = scanFile( + "x.ts", + `export async function loader({ request }) { + const rows = await prisma.invite.findMany(); + if (rows.length === 0) return json([]); + return json(rows); + }` + ); + expect(ep!.loaderCheckedCallees).toContain("findMany"); + expect(ep!.loaderCheckedCallees).not.toContain("getUser"); + }); +}); + +/** + * Contract only. Enumeration order is not controllable from a test: ext4 returns name-hash order, so + * every name set constructed here already comes back sorted and the assertion cannot be made to fail + * locally. It still holds the contract for a filesystem that enumerates in creation order. + */ +describe("routeModuleFiles", () => { + it("returns the tree in name order", () => { + const dir = mkdtempSync(join(tmpdir(), "obs-map-order-")); + try { + for (const name of ["zeta.ts", "alpha.ts", "middle.ts"]) { + writeFileSync(join(dir, name), LOADER); + } + mkdirSync(join(dir, "beta")); + writeFileSync(join(dir, "beta", "route.ts"), LOADER); + + const names = routeModuleFiles(dir).map((f) => f.relativeName); + expect(names).toEqual([...names].sort()); + expect(names).toEqual(["alpha.ts", "beta/route.ts", "middle.ts", "zeta.ts"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/internal-packages/observability-map/src/scan.ts b/internal-packages/observability-map/src/scan.ts new file mode 100644 index 000000000..c80861f77 --- /dev/null +++ b/internal-packages/observability-map/src/scan.ts @@ -0,0 +1,1648 @@ +import ts from "typescript"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { CatchEvidence, EntryPoint, LogCall } from "./types.js"; + +/** Thrown by `scanFile` when the source does not parse cleanly. */ +export class ParseFailureError extends Error { + constructor( + readonly fileName: string, + readonly diagnostic: string + ) { + super(`${fileName}: ${diagnostic}`); + this.name = "ParseFailureError"; + } +} + +type EntryFunction = ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; + +function isEntryFunction(node: ts.Node): node is EntryFunction { + return ( + ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node) + ); +} + +/** Strip wrappers that do not change which expression is really being referred to. */ +function unwrap(expr: ts.Expression): ts.Expression { + let current = expr; + for (;;) { + if ( + ts.isParenthesizedExpression(current) || + ts.isAwaitExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + continue; + } + return current; + } +} + +/** + * Root callee of a call, unwrapping chains: `createLoaderApiRoute({}).withCors()` + * resolves to `createLoaderApiRoute`, not `withCors`. + */ +function rootCalleeName(call: ts.CallExpression): string | null { + let current: ts.Expression = unwrap(call.expression); + for (;;) { + if (ts.isIdentifier(current)) return current.text; + if ( + ts.isCallExpression(current) || + ts.isPropertyAccessExpression(current) || + ts.isElementAccessExpression(current) + ) { + current = unwrap(current.expression); + continue; + } + return null; + } +} + +/** + * A dotted property path, `authentication.userId`, or null when the chain does not start from a + * plain identifier. Property accesses only: a call or an index anywhere in the chain gives null. + */ +function propertyPath(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (!ts.isPropertyAccessExpression(target)) return null; + const base = propertyPath(target.expression); + return base === null ? null : `${base}.${target.name.text}`; +} + +/** + * Expressions that are the caller's own id, in the spellings the route tree uses. Anchored at both + * ends: the root is one of the auth bindings a builder hands the handler and the last segment is an + * identity field, so `user.name` is not a scope and neither is `run.userId`, which is a resource's + * owner rather than the caller. + */ +const CALLER_ID_PATH = + /^(authentication|authenticationResult|auth|sessionAuth|user)(\.[A-Za-z0-9_$]+)*\.(userId|id|actor)$/; + +/** + * Property names that mean the value says WHOSE rather than merely carrying the caller's id around. + * Read off the tree: of the ten names taking a caller-id value, `sub`, `value` and `consumerId` are + * the three left out. + */ +const CALLER_ID_FIELD = + /^(id|userId|user|memberId|orgMemberId|createdBy|createdByUserId|environmentId|runtimeEnvironmentId|organizationId|orgId|projectId)$/; + +/** + * Callees handed the caller's id that cannot narrow a read with it: the log line and the response + * body. A denylist of sinks rather than an allowlist of query callees, which is a measurement and not + * a preference. See INTERNALS.md, "What auth-scope reads as scoping". + */ +const NON_SCOPING_CALLEE = /(^|\.)console\.[A-Za-z_$][\w$]*$|^(json|typedjson|defer)$/; + +/** + * Whether a callee could plausibly narrow a read with the object it is handed. A callee with no + * readable name of its own is credited, because refusing it would ACCUSE the route. + */ +function couldScopeAQuery(callee: ts.Expression): boolean { + const text = calleeText(callee); + if (text === null) return true; + return !LOGGER_CALLEE.test(text) && !NON_SCOPING_CALLEE.test(text); +} + +/** + * Whether the object literal holding this property is handed to a call that could scope a query, + * through any depth of nesting. Arrays count, so `{ OR: [{ userId }] }` still reaches its call. + * + * Refuses a filter built and dropped (`dead-caller-scope-object`, `dead-caller-scope-userid`) and + * one handed to a `NON_SCOPING_CALLEE` (`log-caller-scope-userid`). Does NOT refuse a filter handed + * to a named call that ignores it: `String({ userId: user.id })` reads as scoping, the same way + * `try { String(0); }` reads as error handling. + */ +function isHandedToAScopingCall(property: ts.PropertyAssignment): boolean { + let node: ts.Node = property; + for (let parent = node.parent; parent; node = parent, parent = node.parent) { + if (ts.isCallExpression(parent) || ts.isNewExpression(parent)) { + if (parent.arguments?.some((a) => a === node) !== true) return false; + return couldScopeAQuery(parent.expression); + } + if ( + !ts.isObjectLiteralExpression(parent) && + !ts.isPropertyAssignment(parent) && + !ts.isArrayLiteralExpression(parent) + ) { + return false; + } + } + return false; +} + +/** + * Whether any handler in `fns` assigns the caller's own id to an object-literal property. Three + * conditions, all load bearing: INTERNALS.md, "What auth-scope reads as scoping". + * + * Per export rather than per entry point, which is why it is computed here rather than in the main + * body walk. Nested functions are walked, since a filter built inside a callback still filters. + * Same-file helpers are NOT followed, unlike the main walk, so a route computing its filter in a + * helper is reported as unscoped. Nothing in the tree does. + */ +function scopesByCallerIn(fns: Iterable): boolean { + let found = false; + const visit = (node: ts.Node) => { + if (found) return; + if ( + ts.isPropertyAssignment(node) && + ts.isIdentifier(node.name) && + CALLER_ID_FIELD.test(node.name.text) + ) { + const path = propertyPath(node.initializer); + if (path !== null && CALLER_ID_PATH.test(path) && isHandedToAScopingCall(node)) { + found = true; + return; + } + } + ts.forEachChild(node, visit); + }; + for (const fn of fns) { + if (fn.body) visit(fn.body); + } + return found; +} + +/** Callee as recorded in `calleeNames`: the identifier, or the property for a member call. */ +function calleeName(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (ts.isPropertyAccessExpression(target)) return target.name.text; + return null; +} + +/** + * The whole callee path of a call, `prisma.organization.findFirst` rather than `findFirst`. Used to + * match a call against `LOGGER_CALLEE` and `PARSE_CALLEE`. Null when the path runs through something + * with no name of its own, e.g. `new PromptService().createOverride`, where the caller falls back to + * the bare name. + */ +function calleeText(expr: ts.Expression): string | null { + const target = unwrap(expr); + if (ts.isIdentifier(target)) return target.text; + if (target.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isPropertyAccessExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}.${target.name.text}`; + } + if (ts.isCallExpression(target)) { + const base = calleeText(target.expression); + return base === null ? null : `${base}()`; + } + return null; +} + +/** `logger.error`, `log.info`, `this.logger.debug`. */ +const LOGGER_CALLEE = /(^|\.)(logger|log)\.[A-Za-z_$][\w$]*$/; + +/** Property names on the first object-literal argument, e.g. `{ environmentId, error }`. */ +function objectArgumentFields(call: ts.CallExpression): string[] { + for (const arg of call.arguments) { + const target = unwrap(arg); + if (!ts.isObjectLiteralExpression(target)) continue; + const fields: string[] = []; + for (const property of target.properties) { + const name = propertyName(property); + if (name) fields.push(name); + } + return fields; + } + return []; +} + +/** + * Calls that turn input into a value and throw when it is malformed. `parse`/`safeParse` cover + * `JSON.parse` and the zod schemas. `.json` has to be a member call, because a bare `json(...)` is + * the remix response helper, which every route calls and which parses nothing. + */ +const PARSE_CALLEE = /(^|\.)(parse|safeParse|parseAsync|safeParseAsync|decode)$|\.json$/; + +/** + * Constructors that parse untrusted input and throw when it is malformed. Deliberately short: any + * constructor at all excuses a catch guarding ordinary work, which was true of 77 try blocks. + */ +const PARSE_CONSTRUCTORS = new Set(["URL", "URLSearchParams", "RegExp"]); + +function isParseCall(node: ts.Node): boolean { + if (ts.isNewExpression(node)) { + return ts.isIdentifier(node.expression) && PARSE_CONSTRUCTORS.has(node.expression.text); + } + if (!ts.isCallExpression(node)) return false; + const text = calleeText(node.expression) ?? calleeName(node.expression); + return text !== null && PARSE_CALLEE.test(text); +} + +/** + * Body reads: what a parse guard waits for before it parses, when the parse is written separately + * from the read. Only consulted for `awaitsOnlyParse`, never for `guardsParse`, which bounds it: a + * body read on its own does not make a try block a parse guard. + */ +const BODY_READ_METHODS = new Set(["text", "formData", "arrayBuffer", "blob", "bytes"]); + +function isBodyRead(node: ts.Node): boolean { + if (!ts.isCallExpression(node)) return false; + const callee = unwrap(node.expression); + return ts.isPropertyAccessExpression(callee) && BODY_READ_METHODS.has(callee.name.text); +} + +/** + * Syntax that can raise, i.e. anything a try block might do that produces something for a catch + * clause to catch. A whitelist, so it also misses real raising code; `guardedWork` has both + * directions of that residual. + */ +function canRaise(node: ts.Node): boolean { + return ( + ts.isCallExpression(node) || + ts.isNewExpression(node) || + ts.isTaggedTemplateExpression(node) || + ts.isAwaitExpression(node) || + ts.isYieldExpression(node) || + ts.isPropertyAccessExpression(node) || + ts.isElementAccessExpression(node) || + ts.isThrowStatement(node) || + ts.isForOfStatement(node) || + ts.isForInStatement(node) || + (ts.isBinaryExpression(node) && + (node.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword || + node.operatorToken.kind === ts.SyntaxKind.InKeyword)) + ); +} + +/** + * What the guarded region does, in the three terms `error-classification` needs. Each term's rule + * and measurement: INTERNALS.md, "Catch evidence, per clause". + * + * Two residuals in opposite directions, both live. `guardCanRaise` refuses `try { 0; }` and nothing + * cleverer, because `canRaise` accepts any call at all, so `try { String(0); }` reads as + * classification and takes the tree from 19 to 44: `dead-classifying-try-with-call`, the corpus's + * expected failure. And `canRaise` misses code that CAN raise, a destructuring declaration and a + * temporal-dead-zone read among them, which is why `guardMayRaise` exists beside it. + * + * Nested function bodies are skipped throughout: a callback written inside the try is not work the + * try is guarding on this pass through, and a `throw` inside one is not either. + */ +function guardedWork(tryBlock: ts.Block): { + guardsParse: boolean; + awaitsOnlyParse: boolean; + guardCanRaise: boolean; +} { + let guardsParse = false; + let awaitsOnlyParse = true; + let guardCanRaise = false; + const visit = (node: ts.Node) => { + if (ts.isFunctionLike(node)) return; + if (isParseCall(node)) guardsParse = true; + if (canRaise(node)) guardCanRaise = true; + if (ts.isAwaitExpression(node)) { + const awaited = unwrap(node.expression); + if (!isParseCall(awaited) && !isBodyRead(awaited)) awaitsOnlyParse = false; + } + ts.forEachChild(node, visit); + }; + visit(tryBlock); + return { guardsParse, awaitsOnlyParse, guardCanRaise }; +} + +/** Whether some node in the tree rooted at `node` matches `predicate`. */ +function someNode(node: ts.Node, predicate: (n: ts.Node) => boolean): boolean { + if (predicate(node)) return true; + return ts.forEachChild(node, (child) => someNode(child, predicate)) === true; +} + +function containsInstanceOf(node: ts.Node): boolean { + return someNode( + node, + (n) => ts.isBinaryExpression(n) && n.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword + ); +} + +/** + * Whether a binding name pattern declares `target`, recursively, including every destructured shape: + * a shadow check that only recognised `ts.isIdentifier` missed all of them. + */ +function bindingDeclares(name: ts.BindingName, target: string): boolean { + if (ts.isIdentifier(name)) return name.text === target; + for (const element of name.elements) { + if (!ts.isOmittedExpression(element) && bindingDeclares(element.name, target)) return true; + } + return false; +} + +/** Whether `name` is declared by a var/let/const, function or class statement directly in this + * statement list. Not recursive: a nested block's own declarations are handled when the walk + * reaches that block. */ +function declaresInScope(statements: readonly ts.Statement[], name: string): boolean { + for (const statement of statements) { + if (ts.isFunctionDeclaration(statement) && statement.name?.text === name) return true; + if (ts.isClassDeclaration(statement) && statement.name?.text === name) return true; + if ( + ts.isVariableStatement(statement) && + statement.declarationList.declarations.some((d) => bindingDeclares(d.name, name)) + ) { + return true; + } + } + return false; +} + +/** + * Whether `node` contains a genuine read of the given catch binding. Two shapes share the binding's + * text without reading it, the property side of a member expression and an object literal key, and a + * name re-declared in a nested scope refers to that declaration instead, so the walk stops at the + * boundary that re-declares it. + */ +function referencesBinding(node: ts.Node, bindingName: string): boolean { + if (ts.isIdentifier(node) && node.text === bindingName) { + const parent = node.parent; + if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false; + if (ts.isPropertyAssignment(parent) && parent.name === node) return false; + return true; + } + + if ( + ts.isFunctionLike(node) && + node.parameters.some((p) => bindingDeclares(p.name, bindingName)) + ) { + return false; + } + + if (ts.isCatchClause(node)) { + const decl = node.variableDeclaration; + if (decl && bindingDeclares(decl.name, bindingName)) return false; + } + + if (ts.isBlock(node) && declaresInScope(node.statements, bindingName)) return false; + + return ts.forEachChild(node, (child) => referencesBinding(child, bindingName)) === true; +} + +/** The catch binding's name, or null for a bindingless `catch { ... }` or a destructured one. */ +function catchBindingName(clause: ts.CatchClause): string | null { + const decl = clause.variableDeclaration; + return decl && ts.isIdentifier(decl.name) ? decl.name.text : null; +} + +/** A node's source text with all whitespace removed, for comparing two branch arms. */ +function normalizedText(node: ts.Node): string { + return node.getText().replace(/\s+/g, ""); +} + +/** + * Whether a conditional expression tests the error to pick what the clause does, rather than to word + * what it says. The caller only offers it the whole value of a `return`/`throw`, so + * `return json({ error: e instanceof Error ? e.message : String(e) })` does not reach here: that is + * message formatting and every error leaves by the same path. + * + * The two arms have to differ, the same requirement `selectsADistinctPath` makes of an `if`, with + * parentheses and whitespace stripped. `same-arms-ternary` and + * `wrap-body-in-same-arms-throw-ternary` are the tree-scale versions, the second of which would take + * every route in the tree to a pass. The residual both branch tests share is on + * `selectsADistinctPath`. + */ +function selectsAnErrorPath(node: ts.ConditionalExpression, bindingName: string | null): boolean { + if (bindingName === null) return false; + if (!containsInstanceOf(node.condition)) return false; + if (!referencesBinding(node.condition, bindingName)) return false; + return normalizedText(unwrap(node.whenTrue)) !== normalizedText(unwrap(node.whenFalse)); +} + +/** + * Which bare (unlabelled) jumps, at this point in the recursion, leave the statement list the + * question is being asked about. A bare jump targets the nearest enclosing construct of its kind, so + * descending past one of those changes the answer for the jumps it captures. + */ +type BareJumps = { break: boolean; continue: boolean }; + +/** A jump written directly in the list under question always leaves it: whatever it targets + * encloses the list. */ +const ESCAPES: BareJumps = { break: true, continue: true }; + +/** A `do` body, asked about from the list the `do` sits in. Both jumps reach the statement written + * after the `do`, so neither leaves that list. */ +const IN_DO_BODY: BareJumps = { break: false, continue: false }; + +/** + * A statement that leaves the statement list it sits in on every path through itself, so anything + * after it in the same list never runs. Recognises a nested construct and not only the bare jump + * forms, which is what stopped a dead `throw error;` counting as a rethrow (`dead-throw-after-*`). + * + * A bare `break` or `continue` only counts where it actually leaves the list, which is what `jumps` + * carries: a `break` in a switch clause targets the switch, a `continue` targets an enclosing loop + * that the switch cannot be. `break and continue inside the construct they target` holds both + * halves. A labelled jump always counts, since nothing between the list and the jump can carry the + * label. + * + * A sound under-approximation: everything not listed answers false, including a `do` that never + * falls through, since separating that from `do { continue; } while (c)` means folding a loop + * condition. Saying false when the truth is true only leaves a later statement in the list, which + * withholds evidence rather than inventing it. + */ +function definitelyExits(statement: ts.Statement, jumps: BareJumps = ESCAPES): boolean { + if (ts.isReturnStatement(statement) || ts.isThrowStatement(statement)) return true; + if (ts.isBreakStatement(statement)) return statement.label !== undefined || jumps.break; + if (ts.isContinueStatement(statement)) return statement.label !== undefined || jumps.continue; + if (ts.isBlock(statement)) { + return statement.statements.some((s) => definitelyExits(s, jumps)); + } + // A `do` body runs before its condition is ever read. + if (ts.isDoStatement(statement)) return definitelyExits(statement.statement, IN_DO_BODY); + if (ts.isIfStatement(statement)) { + // Keyword-exact, the same spelling rule as the walk's `if (true)` entry and for the same + // reason: this GRANTS a reachability cut and a wrong grant pays. + // `cuts a dead trailing statement after an if true that exits` is the pin. + if (unwrap(statement.expression).kind === ts.SyntaxKind.TrueKeyword) { + return definitelyExits(statement.thenStatement, jumps); + } + return ( + statement.elseStatement !== undefined && + definitelyExits(statement.thenStatement, jumps) && + definitelyExits(statement.elseStatement, jumps) + ); + } + if (ts.isTryStatement(statement)) { + if (statement.finallyBlock && definitelyExits(statement.finallyBlock, jumps)) return true; + if (!definitelyExits(statement.tryBlock, jumps)) return false; + return ( + statement.catchClause === undefined || definitelyExits(statement.catchClause.block, jumps) + ); + } + if (ts.isSwitchStatement(statement)) { + const inClause: BareJumps = { break: false, continue: jumps.continue }; + const clauses = statement.caseBlock.clauses; + const last = clauses[clauses.length - 1]; + if (!clauses.some(ts.isDefaultClause) || last === undefined) return false; + // An empty clause falls through to the next one, so it does not have to exit itself; the last + // clause has nothing to fall through to and does. + return ( + clauses.every( + (c) => c.statements.length === 0 || c.statements.some((s) => definitelyExits(s, inClause)) + ) && last.statements.some((s) => definitelyExits(s, inClause)) + ); + } + return false; +} + +/** `statements` up to and including the first one that definitely exits. */ +function reachableStatements(statements: readonly ts.Statement[]): readonly ts.Statement[] { + // The arrow matters: `findIndex` passes an index as the second argument, which `definitelyExits` + // would read as its `jumps` record. + const index = statements.findIndex((s) => definitelyExits(s)); + return index === -1 ? statements : statements.slice(0, index + 1); +} + +/** + * Whether the tree rooted at `node` contains a `break` or `continue` that would leave it. What the + * catch walk asks of a finally block before entering the tryBlock beside it, since a finally that + * completes abruptly cancels the try's completion (`dead-throw-in-cancelled-try`). + * + * A containment read and not a liveness one, on purpose: the caller is deciding whether to GRANT + * credit, so a jump that only may run still refuses + * (`refuses the tryBlock when the finally only may break`). A `return` is not looked for here + * because the returns veto already reads it off the whole statement. + */ +function containsEscapingJump(node: ts.Node, jumps: BareJumps = ESCAPES): boolean { + if (ts.isFunctionLike(node)) return false; + if (ts.isBreakStatement(node)) return node.label !== undefined || jumps.break; + if (ts.isContinueStatement(node)) return node.label !== undefined || jumps.continue; + if (ts.isSwitchStatement(node)) { + const inClause: BareJumps = { break: false, continue: jumps.continue }; + return node.caseBlock.clauses.some((c) => + c.statements.some((s) => containsEscapingJump(s, inClause)) + ); + } + // Any loop captures both bare jumps, so nothing inside one can leave `node` + // (`does not refuse a finally whose loop captures its own break`). + if (ts.isIterationStatement(node, false)) { + return ( + ts.forEachChild(node, (child) => + containsEscapingJump(child, { break: false, continue: false }) + ) === true + ); + } + return ts.forEachChild(node, (child) => containsEscapingJump(child, jumps)) === true; +} + +/** + * Literal truthiness of a guard expression: true, false, or null when not decidable from the token + * alone. An identifier, call, bigint, `&&`, `||` or a template with substitutions is always null, so + * a live guard can never be read as dead. That is deliberate and it is what leaves + * `dead-conjunction-instanceof-if` open. Pinned by `still refuses an error test after an always-true + * spelling that throws`. + */ +function literalTruth(expr: ts.Expression): boolean | null { + const target = unwrap(expr); + const literal = literalValue(target); + if (literal !== undefined) return Boolean(literal); + if (ts.isPrefixUnaryExpression(target) && target.operator === ts.SyntaxKind.ExclamationToken) { + const inner = literalTruth(target.operand); + return inner === null ? null : !inner; + } + if (ts.isBinaryExpression(target)) { + const op = target.operatorToken.kind; + if ( + op === ts.SyntaxKind.EqualsEqualsEqualsToken || + op === ts.SyntaxKind.ExclamationEqualsEqualsToken + ) { + const left = literalValue(target.left); + const right = literalValue(target.right); + if (left === undefined || right === undefined) return null; + const equal = left === right; + return op === ts.SyntaxKind.EqualsEqualsEqualsToken ? equal : !equal; + } + } + return null; +} + +/** The value of a literal token, or undefined when the expression is not a bare literal. */ +function literalValue(expr: ts.Expression): string | number | boolean | null | undefined { + const target = unwrap(expr); + if (target.kind === ts.SyntaxKind.TrueKeyword) return true; + if (target.kind === ts.SyntaxKind.FalseKeyword) return false; + if (target.kind === ts.SyntaxKind.NullKeyword) return null; + if (ts.isStringLiteral(target) || ts.isNoSubstitutionTemplateLiteral(target)) return target.text; + if (ts.isNumericLiteral(target)) return Number(target.text); + return undefined; +} + +/** Whether a try block could throw at all: false only when every statement is an expression + * statement over a bare literal, the one shape that provably cannot raise. */ +function tryBlockMayThrow(block: ts.Block): boolean { + return !block.statements.every( + (s) => ts.isExpressionStatement(s) && literalValue(s.expression) !== undefined + ); +} + +/** + * Whether the tree rooted at `root` contains a node `hit` accepts that a provably-untaken branch does + * not already rule out. Strictly subtractive against a plain containment read, which is what lets its + * two callers read it for opposite purposes: see INTERNALS.md, "Two folds, pointing opposite ways". + * + * The `exited` half is pinned by `dead and deferred code prepended to a deciding catch does not blind + * it` and the `BRANCH_EXITED` family; the `selectsADistinctPath` half by `an arm whose only exit is + * dead decides nothing` and `dead-armed-instanceof-if`. + */ +function containsLiveWhere(root: ts.Node, hit: (n: ts.Node) => boolean): boolean { + const walk = (node: ts.Node): boolean => { + if (ts.isFunctionLike(node)) return false; + if (hit(node)) return true; + if (ts.isIfStatement(node)) { + const truth = literalTruth(node.expression); + if (truth === true) return walk(node.thenStatement); + if (truth === false) { + return node.elseStatement !== undefined && walk(node.elseStatement); + } + return ( + walk(node.thenStatement) || (node.elseStatement !== undefined && walk(node.elseStatement)) + ); + } + if (ts.isWhileStatement(node)) { + if (literalTruth(node.expression) === false) return false; + return walk(node.statement); + } + if (ts.isForStatement(node)) { + if (node.condition !== undefined && literalTruth(node.condition) === false) return false; + return ts.forEachChild(node, walk) === true; + } + if (ts.isForOfStatement(node) || ts.isForInStatement(node)) { + const iterable = unwrap(node.expression); + const emptyArray = ts.isArrayLiteralExpression(iterable) && iterable.elements.length === 0; + const emptyObject = + ts.isForInStatement(node) && + ts.isObjectLiteralExpression(iterable) && + iterable.properties.length === 0; + if (emptyArray || emptyObject) return false; + return ts.forEachChild(node, walk) === true; + } + if (ts.isSwitchStatement(node)) { + const disc = literalValue(node.expression); + const clauses = node.caseBlock.clauses; + const allLiteral = + disc !== undefined && + clauses.every((c) => ts.isDefaultClause(c) || literalValue(c.expression) !== undefined); + if (!allLiteral) return clauses.some((c) => c.statements.some(walk)); + // Fall-through: from the first matching (or default) clause, every later clause is reachable. + let matched = clauses.findIndex( + (c) => !ts.isDefaultClause(c) && literalValue(c.expression) === disc + ); + if (matched === -1) matched = clauses.findIndex(ts.isDefaultClause); + if (matched === -1) return false; + return clauses.slice(matched).some((c) => c.statements.some(walk)); + } + if (ts.isTryStatement(node)) { + // A finally that always completes abruptly supersedes the try's and the catch's completion, so + // only its own statements stay live. Folded only where `definitelyExits` can prove it; a + // conditional jump keeps the containment answer, which refuses credit rather than inventing + // it. Without this the `dead-throw-in-cancelled-try` prepend blinded the walk to every real + // classification below it (`keeps the classification after a finally-break no-op`). + if (node.finallyBlock !== undefined && definitelyExits(node.finallyBlock)) { + return walk(node.finallyBlock); + } + if (walk(node.tryBlock)) return true; + if (node.finallyBlock !== undefined && walk(node.finallyBlock)) return true; + if (node.catchClause !== undefined && tryBlockMayThrow(node.tryBlock)) { + return walk(node.catchClause.block); + } + return false; + } + return ts.forEachChild(node, walk) === true; + }; + return walk(root); +} + +function containsLiveExit(node: ts.Node): boolean { + return containsLiveWhere(node, (n) => ts.isReturnStatement(n) || ts.isThrowStatement(n)); +} + +function containsLiveReturn(node: ts.Node): boolean { + return containsLiveWhere(node, ts.isReturnStatement); +} + +/** + * Whether an `if`/`switch` sends at least one arm somewhere the others do not go, by returning or + * throwing from inside it. `if (e instanceof Error) { }` fails this, since every error still leaves + * by the same path afterwards (`empty-instanceof-if`). Two textually identical arms do not count, + * the same comparison `selectsAnErrorPath` makes of a ternary. + * + * The exit an arm is credited for has to be a LIVE one, never a plain containment read + * (`an arm whose only exit is dead decides nothing`, `dead-armed-instanceof-if`). + * + * The residual both branch tests share, stated here once for both: two arms that produce the same + * outcome by different spellings still read as a real decision, e.g. + * `if (e instanceof Error) { return json(x); } return Response.json(x);`. Telling those apart needs + * the produced values compared for meaning rather than for text, which is a different kind of + * analysis from anything else in this file. + */ +function selectsADistinctPath(statement: ts.IfStatement | ts.SwitchStatement): boolean { + if (ts.isIfStatement(statement)) { + const otherwise = statement.elseStatement; + if (otherwise !== undefined) { + if (normalizedText(statement.thenStatement) === normalizedText(otherwise)) return false; + return containsLiveExit(statement.thenStatement) || containsLiveExit(otherwise); + } + return containsLiveExit(statement.thenStatement); + } + // Per clause statement rather than over the whole switch: reading the switch as one node would + // hand `containsLiveWhere`'s discriminant fold a `switch (1)` it can decide, which is not this + // predicate's business, since an unreachable CLAUSE is caught by the same fold one level down. + return statement.caseBlock.clauses.some((clause) => clause.statements.some(containsLiveExit)); +} + +/** + * What a catch clause does with the error, beyond the fact that it caught one. + * + * Both answers are read off the clause's own guaranteed path: the walk enters a construct exactly + * where the entered statements are guaranteed to execute whenever the clause body runs, so no credit + * can ever come from code a semantics-preserving edit could have added dead. Which constructs are + * entered and which are refused, and the eleven dead spellings this replaced: INTERNALS.md, + * "The dead-code defence". `dead-*` in the mutation corpus is the tree-scale proof, one entry per + * shape. + * + * The cost is real, in both rules. `catch (e) { if (transient) throw e; return null; }` no longer + * reads as a rethrow, so it fails rather than sitting out. That is the direction to be wrong in. + */ +function catchClauseEvidence(clause: ts.CatchClause): { + rethrows: boolean; + throws: boolean; + branches: boolean; +} { + // The state record travels so an if/else arm can be walked against an isolated copy. `returns` + // stays a single shared flag, because it is a clause-wide veto and never per-arm evidence. + // + // `exited` is raised at the END of each statement, after that statement's own branch check: a + // deciding statement contains an exit by definition, so raising it first makes every such + // statement refuse itself, measured at 78 routes losing their pass. + // + // `vetoReturns` is false only in the arm walks. `returns` is read at the PARENT level over the + // whole statement, so an arm walk re-reading its own statements adds a false veto for a + // folded-dead arm (`dead-classifier-one-arm`). + type ClauseState = { + rethrows: boolean; + branches: boolean; + exited: boolean; + vetoReturns: boolean; + }; + let returns = false; + const state: ClauseState = { rethrows: false, branches: false, exited: false, vetoReturns: true }; + const bindingName = catchBindingName(clause); + + const walk = (statements: readonly ts.Statement[], state: ClauseState) => { + // A block re-declaring the binding name means an `if` below it references the shadowing + // declaration, not this clause's error, so the whole list is skipped for branch purposes. + const shadowed = bindingName !== null && declaresInScope(statements, bindingName); + + for (const statement of reachableStatements(statements)) { + if (ts.isThrowStatement(statement)) { + state.rethrows = true; + // Read the branch check here, before the path is cut. A thrown ternary picks WHICH error + // leaves, which is a classification, and the shared check below is unreachable from this arm + // because it always continues first. + if ( + bindingName !== null && + !shadowed && + !state.exited && + statement.expression !== undefined + ) { + const thrown = unwrap(statement.expression); + if (ts.isConditionalExpression(thrown) && selectsAnErrorPath(thrown, bindingName)) { + state.branches = true; + } + } + state.exited = true; + continue; + } + if (ts.isBlock(statement)) { + walk(statement.statements, state); + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // A `do` body runs before its condition is ever read, so it is on the straight-line path + // whatever the condition says. The only loop form that is. + if (ts.isDoStatement(statement)) { + const body = statement.statement; + walk(ts.isBlock(body) ? body.statements : [body], state); + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // The three handlers below share one template: walk the inner list with the SAME shared state, + // then read `returns` and `exited` off the whole statement and continue. The explicit + // `containsLiveReturn` read is load-bearing, because the `continue` skips the shared read below + // and `try { throw e; } finally { return null; }` genuinely swallows (`reads a try whose + // finally returns as swallowing, not rethrowing`). + // + // A catchless `try` is entered only when its finally cannot complete abruptly, since a finally + // that does cancels the try's completion and the throw never escapes the clause + // (`reads a throw a finally break discards as no rethrow`, `dead-throw-in-cancelled-try`). The + // finallyBlock itself is NOT walked, so classification living only there is under-credited. A + // `try` WITH a catch clause is not entered at all, since a throw in its tryBlock is + // intercepted by the nested catch, which is judged separately as its own `ep.catches` entry + // (`does not read the tryBlock of a caught try as this clause's rethrow`). + if ( + ts.isTryStatement(statement) && + statement.catchClause === undefined && + (statement.finallyBlock === undefined || !containsEscapingJump(statement.finallyBlock)) + ) { + walk(statement.tryBlock.statements, state); + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // A `switch` whose caseBlock is exactly one DefaultClause: those statements always run, as a + // bare list. Any other switch shape falls through to the branch gate below, so a real + // `switch (e.code)` keeps its top-level credit. `reads a clause wrapped in a single-default + // switch as the bare clause` is the pin. + if (ts.isSwitchStatement(statement)) { + const clauses = statement.caseBlock.clauses; + const only = clauses.length === 1 ? clauses[0] : undefined; + if (only !== undefined && ts.isDefaultClause(only)) { + walk(only.statements, state); + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + if (containsLiveExit(statement)) state.exited = true; + continue; + } + } + // An `if` whose condition is exactly the `true` keyword: the then-arm always runs, the else-arm + // is NEVER walked (`reads a dead else arm under if true as contributing nothing`). + // Keyword-exact on purpose, because entry GRANTS credit where `literalTruth`'s wider folding + // only withholds blindness. Do not unify the two folds. Takes precedence over the arm walk. + if ( + ts.isIfStatement(statement) && + unwrap(statement.expression).kind === ts.SyntaxKind.TrueKeyword + ) { + const arm = statement.thenStatement; + walk(ts.isBlock(arm) ? arm.statements : [arm], state); + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + if (containsLiveExit(statement)) state.exited = true; + continue; + } + // Any other reachable statement that could return means throwing is not the only way out. + // Read per statement rather than over the whole clause, so a `return` the walk has already cut + // as dead does not count, and read LIVE rather than by containment, since vetoing on + // `if (false) { return null; }` regressed a rethrow-only clause from not-applicable to fail on + // 11 real routes (`still sets rethrows past a dead return in an if (false) arm`). + if (state.vetoReturns && containsLiveReturn(statement)) returns = true; + + if (bindingName !== null && !shadowed && !state.exited) { + if ( + (ts.isIfStatement(statement) || ts.isSwitchStatement(statement)) && + referencesBinding(statement.expression, bindingName) && + selectsADistinctPath(statement) + ) { + state.branches = true; + } else if (ts.isReturnStatement(statement) && statement.expression !== undefined) { + const value = unwrap(statement.expression); + if (ts.isConditionalExpression(value) && selectsAnErrorPath(value, bindingName)) { + state.branches = true; + } + } + } + + // An `if` WITH an else: one arm always runs, so evidence in BOTH arms is unconditional and + // evidence in one arm only earns nothing. The arms merge by INTERSECTION, because union is the + // laundering direction (`dead-classifier-one-arm`, `does not credit a classifier that sits in + // one arm only`). `returns` is never intersected, since narrowing a veto per arm is unsafe. + if (ts.isIfStatement(statement) && statement.elseStatement !== undefined) { + const armWalk = (arm: ts.Statement): ClauseState => { + const armState: ClauseState = { + rethrows: false, + branches: false, + exited: state.exited, + vetoReturns: false, + }; + walk(ts.isBlock(arm) ? arm.statements : [arm], armState); + return armState; + }; + const thenArm = armWalk(statement.thenStatement); + const elseArm = armWalk(statement.elseStatement); + state.rethrows ||= thenArm.rethrows && elseArm.rethrows; + state.branches ||= thenArm.branches && elseArm.branches; + } + + if (containsLiveExit(statement)) state.exited = true; + } + }; + walk(clause.block.statements, state); + + return { + rethrows: state.rethrows && !returns, + throws: state.rethrows, + branches: state.branches, + }; +} + +/** + * Method names that invoke their callback once per element, never once as a whole. The structural + * signal that separates a per-item boundary from a route's own body expressed through one more layer + * of function nesting (`trace(async () => {...})` and friends, which run their callback exactly + * once). A name list, because nothing in a syntactic scan can tell `users.map` from `Result.map`. + */ +const ITERATION_METHODS = new Set([ + "map", + "forEach", + "filter", + "reduce", + "reduceRight", + "flatMap", + "some", + "every", +]); + +/** Whether an expression is an array literal of fewer than two elements, the one receiver shape + * that cannot be a per-item iteration however the method is named. */ +function isAtMostSingletonArray(expr: ts.Expression): boolean { + const target = unwrap(expr); + return ts.isArrayLiteralExpression(target) && target.elements.length < 2; +} + +/** + * Whether the function-like `node` is the callback argument of a per-item iteration. Both directions + * of being wrong, and what makes the name list survivable: INTERNALS.md, "The iteration-callback + * boundary". + * + * The residual a reader here needs: a per-item callback under a callee this list does not know, + * `pMap(items, cb)`, is attributed to the route, so a per-element catch that decides can carry it to + * a pass. That is a wrong verdict waiting for a route to be written that way rather than a laundering + * path, and it is why the list is worth extending when a new iteration helper shows up in the tree. + */ +function isIterationCallback(node: ts.Node): boolean { + const parent = node.parent; + if (!parent || !ts.isCallExpression(parent)) return false; + if (!parent.arguments.includes(node as ts.Expression)) return false; + const callee = unwrap(parent.expression); + if (!ts.isPropertyAccessExpression(callee)) return false; + if (!ITERATION_METHODS.has(callee.name.text)) return false; + return !isAtMostSingletonArray(callee.expression); +} + +const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]); + +function propertyName(property: ts.ObjectLiteralElementLike): string | null { + if (!property.name) return null; + if (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name)) + return property.name.text; + return null; +} + +/** `methods: { POST: { handler } }`, the per-method shape of `createMultiMethodApiRoute`. */ +function collectMethodHandlers(methods: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const method of methods.properties) { + const name = propertyName(method); + if (!name || !HTTP_METHODS.has(name)) continue; + if (!ts.isPropertyAssignment(method)) continue; + const config = unwrap(method.initializer); + if (!ts.isObjectLiteralExpression(config)) continue; + for (const property of config.properties) { + if (!ts.isPropertyAssignment(property) || propertyName(property) !== "handler") continue; + const value = unwrap(property.initializer); + if (isEntryFunction(value)) out.push(value); + } + } +} + +/** + * The handler on an object argument, in the two shapes the route builders use: `handler` at the top + * level of the config and `methods.POST.handler`. Matching by name at any depth would pick up the + * sibling lambdas (`findResource`, `authorization.resource`) that are not the entry-point body. + */ +function collectNamedHandlers(object: ts.ObjectLiteralExpression, out: EntryFunction[]): void { + for (const property of object.properties) { + if (!ts.isPropertyAssignment(property)) continue; + const name = propertyName(property); + const value = unwrap(property.initializer); + if (name === "handler" && isEntryFunction(value)) out.push(value); + if (name === "methods" && ts.isObjectLiteralExpression(value)) { + collectMethodHandlers(value, out); + } + } +} + +/** The innermost call of a chain: the `createLoaderApiRoute(...)` in `createLoaderApiRoute(...).withCors(...)`. */ +function rootCall(call: ts.CallExpression): ts.CallExpression { + let current = call; + for (;;) { + let next = unwrap(current.expression); + while (ts.isPropertyAccessExpression(next) || ts.isElementAccessExpression(next)) { + next = unwrap(next.expression); + } + if (ts.isCallExpression(next)) { + current = next; + continue; + } + return current; + } +} + +/** + * The handler functions passed to a builder call. Only the root call of a chain is read: a + * callback given to a decorator further along the chain (`.withCors(cb)`) is not the route body. + */ +function collectHandlerFunctions(call: ts.CallExpression, out: EntryFunction[]): void { + for (const arg of rootCall(call).arguments) { + const unwrapped = unwrap(arg); + if (isEntryFunction(unwrapped)) out.push(unwrapped); + else if (ts.isObjectLiteralExpression(unwrapped)) collectNamedHandlers(unwrapped, out); + } +} + +/** Literals a builder option can be given that mean it was not given: `apiBuilder.server.ts` gates + * every option behind `if (option)`, and declaring one is what `auth-scope` credits. */ +function isDeclaredValue(property: ts.ObjectLiteralElementLike): boolean { + if (!ts.isPropertyAssignment(property)) return true; + const value = unwrap(property.initializer); + if (ts.isIdentifier(value) && value.text === "undefined") return false; + return value.kind !== ts.SyntaxKind.NullKeyword && value.kind !== ts.SyntaxKind.FalseKeyword; +} + +/** + * Top-level property names of every object-literal argument to the root call. Only the top level, + * because `authorization` on `createMultiMethodApiRoute` is declared once beside `methods` rather + * than per method (`apiBuilder.server.ts`). + */ +function collectOptionKeys(call: ts.CallExpression): string[] { + const keys: string[] = []; + for (const arg of rootCall(call).arguments) { + const target = unwrap(arg); + if (!ts.isObjectLiteralExpression(target)) continue; + for (const property of target.properties) { + const name = propertyName(property); + if (name && isDeclaredValue(property)) keys.push(name); + } + } + return keys; +} + +type Initializer = { callee: string | null; functions: EntryFunction[]; optionKeys: string[] }; + +const NO_INITIALIZER: Initializer = { callee: null, functions: [], optionKeys: [] }; + +/** Top-level `function x` / `const x = ...` declarations, keyed by binding name. */ +type LocalDeclarations = Map; + +function analyzeInitializer( + expr: ts.Expression | undefined, + locals: LocalDeclarations, + seen: Set +): Initializer { + if (!expr) return NO_INITIALIZER; + const target = unwrap(expr); + + if (isEntryFunction(target)) return { callee: null, functions: [target], optionKeys: [] }; + + if (ts.isCallExpression(target)) { + const functions: EntryFunction[] = []; + collectHandlerFunctions(target, functions); + return { + callee: rootCalleeName(target), + functions, + optionKeys: collectOptionKeys(target), + }; + } + + // `export const action = route.action` where `const route = createActionApiRoute(...)`, and the + // plain alias `export const loader = h`. Resolve back to the declaration the name came from. + if (ts.isIdentifier(target)) return resolveLocal(target.text, locals, seen); + if (ts.isPropertyAccessExpression(target)) { + const root = unwrap(target.expression); + if (ts.isIdentifier(root)) return resolveLocal(root.text, locals, seen); + } + + return NO_INITIALIZER; +} + +function resolveLocal(name: string, locals: LocalDeclarations, seen: Set): Initializer { + if (seen.has(name)) return NO_INITIALIZER; + seen.add(name); + const local = locals.get(name); + if (!local) return NO_INITIALIZER; + if (ts.isFunctionDeclaration(local)) return { callee: null, functions: [local], optionKeys: [] }; + return analyzeInitializer(local, locals, seen); +} + +/** + * How many operands a comma expression has, so `a(), b(), c()` is three and not one. Anything else + * is one. + */ +function commaOperands(expr: ts.Expression): number { + const target = unwrap(expr); + if (ts.isBinaryExpression(target) && target.operatorToken.kind === ts.SyntaxKind.CommaToken) { + return commaOperands(target.left) + commaOperands(target.right); + } + return 1; +} + +/** + * Statements in a statement, counting through block-bearing statements so a body wrapped in a + * single `try` reports its real size. Does not descend into nested function bodies. + * + * Counts bindings and comma operands rather than semicolons, which is what makes the number mean + * something. `const a = f(), b = g(), c = h();` is three initializers however it is punctuated, and + * `a(), b(), c()` is three calls: scoring either as one let a seven-statement try be rewritten into + * a two-statement one with no change to what it runs, which took `error-classification` from fail + * to pass. `merge-declarations` and `merge-comma-expressions` in the mutation corpus are + * the tree-scale versions. + */ +function countStatement(statement: ts.Statement): number { + if (ts.isBlock(statement)) { + return countStatements(statement.statements); + } + + if (ts.isVariableStatement(statement)) { + return statement.declarationList.declarations.length; + } + + if (ts.isExpressionStatement(statement)) { + return commaOperands(statement.expression); + } + + let count = 1; + + if (ts.isTryStatement(statement)) { + count += countStatements(statement.tryBlock.statements); + if (statement.catchClause) count += countStatements(statement.catchClause.block.statements); + if (statement.finallyBlock) count += countStatements(statement.finallyBlock.statements); + return count; + } + + if (ts.isIfStatement(statement)) { + count += countStatement(statement.thenStatement); + if (statement.elseStatement) count += countStatement(statement.elseStatement); + return count; + } + + if ( + ts.isForStatement(statement) || + ts.isForInStatement(statement) || + ts.isForOfStatement(statement) || + ts.isWhileStatement(statement) || + ts.isDoStatement(statement) || + ts.isLabeledStatement(statement) || + ts.isWithStatement(statement) + ) { + count += countStatement(statement.statement); + return count; + } + + if (ts.isSwitchStatement(statement)) { + for (const clause of statement.caseBlock.clauses) { + count += countStatements(clause.statements); + } + return count; + } + + return count; +} + +function countStatements(statements: ts.NodeArray): number { + let count = 0; + for (const statement of statements) count += countStatement(statement); + return count; +} + +function countFunctionStatements(fn: EntryFunction): number { + if (!fn.body) return 0; + // A concise arrow body (`() => json({})`) is one expression, so one statement. + if (!ts.isBlock(fn.body)) return 1; + return countStatements(fn.body.statements); +} + +type ExportName = "loader" | "action"; + +/** + * The call-site facts a body walk accumulates, kept in one shape so the entry-point-wide totals and + * each export's own totals are filled by the same code rather than by two similar loops. + */ +type BodyFacts = { + calleeNames: string[]; + /** + * The same calls as `calleeNames`, each as its whole dotted path. `calleeName` keeps only the + * last segment, so `prisma.organization.findFirst` arrives in `calleeNames` as `findFirst` and + * the receiver that says WHAT is being called is gone. The per-export triviality rule needs it + * back: `prisma` in the path is how a short body is known to touch the datastore. + */ + calleeTexts: string[]; + /** Locals initialised from a call, by local name. */ + declaredFrom: Map; + /** Every identifier read by an `if`, `while`, `switch` or conditional condition. */ + testedNames: Set; + statementCount: number; + hasTryCatch: boolean; +}; + +function newBodyFacts(): BodyFacts { + return { + calleeNames: [], + calleeTexts: [], + declaredFrom: new Map(), + testedNames: new Set(), + statementCount: 0, + hasTryCatch: false, + }; +} + +/** Callees whose answer these bodies looked at: declared from a call AND read by a condition. */ +function checkedCalleesOf(facts: BodyFacts): string[] { + return [ + ...new Set( + [...facts.declaredFrom] + .filter(([local]) => facts.testedNames.has(local)) + .flatMap(([, callees]) => callees) + ), + ]; +} + +type EntryTarget = { + hasLoader: boolean; + hasAction: boolean; + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + loaderBuilderOptions: string[]; + actionBuilderOptions: string[]; + /** Handler functions per export, so a scope signal can be attributed to the half of the file it + * was found in. `functions` is their union, which is what every entry-point-wide field reads. */ + loaderFunctions: Set; + actionFunctions: Set; + functions: Set; +}; + +/** + * Top-level `function x` / `const x = ...` declarations by binding name, so a named export clause + * (`export { action }`) can be resolved back to the initializer it came from. + */ +function collectLocalDeclarations(sf: ts.SourceFile): LocalDeclarations { + const locals: LocalDeclarations = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name) { + locals.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer) continue; + if (ts.isIdentifier(decl.name)) { + locals.set(decl.name.text, decl.initializer); + continue; + } + if (ts.isObjectBindingPattern(decl.name)) { + for (const element of decl.name.elements) { + if (ts.isIdentifier(element.name)) locals.set(element.name.text, decl.initializer); + } + } + } + } + + return locals; +} + +/** + * Top-level functions by name, for resolving a body that delegates its work to a same-file helper + * (`export async function loader({ request }) { return proxyToPostHog(request); }`). + */ +function collectLocalFunctions(sf: ts.SourceFile): Map { + const functions = new Map(); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && statement.body) { + functions.set(statement.name.text, statement); + continue; + } + if (!ts.isVariableStatement(statement)) continue; + + for (const decl of statement.declarationList.declarations) { + if (!decl.initializer || !ts.isIdentifier(decl.name)) continue; + const value = unwrap(decl.initializer); + if (isEntryFunction(value)) functions.set(decl.name.text, value); + } + } + + return functions; +} + +/** `noLib` and `noResolve` keep the throwaway program below off the disk: nothing here needs a type, + * only the syntax the parser already produced. */ +const SYNTAX_ONLY_OPTIONS: ts.CompilerOptions = { noLib: true, noResolve: true, allowJs: true }; + +/** + * Syntactic diagnostics for an already-parsed source file, through `ts.Program` rather than off the + * internal diagnostics array the parser hangs on the source file, which a compiler upgrade could + * rename out from under us. The host hands the program the `sf` we already have, so nothing is parsed + * twice. Costs and reasoning: INTERNALS.md, "Tests, timeouts and CI". + */ +function syntacticDiagnostics(sf: ts.SourceFile): readonly ts.Diagnostic[] { + const host: ts.CompilerHost = { + getSourceFile: (name) => (name === sf.fileName ? sf : undefined), + getDefaultLibFileName: () => "lib.d.ts", + writeFile: () => {}, + getCurrentDirectory: () => "", + getCanonicalFileName: (name) => name, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + fileExists: (name) => name === sf.fileName, + readFile: () => undefined, + }; + return ts.createProgram([sf.fileName], SYNTAX_ONLY_OPTIONS, host).getSyntacticDiagnostics(sf); +} + +export function scanFile(fileName: string, source: string): EntryPoint | null { + const sf = ts.createSourceFile(fileName, source, ts.ScriptTarget.Latest, true); + + // `createSourceFile` recovers from malformed input instead of throwing, so the diagnostics are + // the only signal that a file did not parse. + const diagnostics = syntacticDiagnostics(sf); + if (diagnostics.length > 0) { + const first = diagnostics[0]!; + throw new ParseFailureError(fileName, ts.flattenDiagnosticMessageText(first.messageText, " ")); + } + + const importedNames: string[] = []; + for (const statement of sf.statements) { + if (!ts.isImportDeclaration(statement) || !statement.importClause) continue; + const bindings = statement.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const el of bindings.elements) importedNames.push(el.name.text); + } + if (statement.importClause.name) importedNames.push(statement.importClause.name.text); + } + + const target: EntryTarget = { + hasLoader: false, + hasAction: false, + loaderInitializerCallee: null, + actionInitializerCallee: null, + loaderBuilderOptions: [], + actionBuilderOptions: [], + loaderFunctions: new Set(), + actionFunctions: new Set(), + functions: new Set(), + }; + + // The option keys travel with the callee they came from, so a second declaration cannot lend its + // options to the first one's builder. + const record = (name: string, initializer: Initializer) => { + if (name === "loader") { + target.hasLoader = true; + if (target.loaderInitializerCallee === null) { + target.loaderInitializerCallee = initializer.callee; + target.loaderBuilderOptions = initializer.optionKeys; + } + } else { + target.hasAction = true; + if (target.actionInitializerCallee === null) { + target.actionInitializerCallee = initializer.callee; + target.actionBuilderOptions = initializer.optionKeys; + } + } + const perExport = name === "loader" ? target.loaderFunctions : target.actionFunctions; + for (const fn of initializer.functions) { + target.functions.add(fn); + perExport.add(fn); + } + }; + + const isExported = (n: ts.Node) => + ts.canHaveModifiers(n) && + ts.getModifiers(n)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) === true; + + const locals = collectLocalDeclarations(sf); + + for (const statement of sf.statements) { + if (ts.isFunctionDeclaration(statement) && statement.name && isExported(statement)) { + const name = statement.name.text; + if (name === "loader" || name === "action") { + record(name, { callee: null, functions: [statement], optionKeys: [] }); + } + continue; + } + + if (ts.isVariableStatement(statement) && isExported(statement)) { + for (const decl of statement.declarationList.declarations) { + if (ts.isIdentifier(decl.name)) { + const name = decl.name.text; + if (name === "loader" || name === "action") { + record(name, analyzeInitializer(decl.initializer, locals, new Set())); + } + continue; + } + // `export const { action, loader } = createActionApiRoute(...)`. Skipping a non-identifier + // binding name here left this shape absent from the denominator entirely, neither a parse + // failure nor unmeasured. The exported name is the ELEMENT name, so `{ loader: action }` + // exports an action and `{ action: internal }` exports neither. + if (!ts.isObjectBindingPattern(decl.name)) continue; + for (const element of decl.name.elements) { + if (!ts.isIdentifier(element.name)) continue; + const name = element.name.text; + if (name !== "loader" && name !== "action") continue; + record(name, analyzeInitializer(decl.initializer, locals, new Set())); + } + } + continue; + } + + if (ts.isExportDeclaration(statement) && statement.exportClause) { + // `export * from "./x"` has no clause and reaches nothing here; `export * as ns from "./x"` + // is a namespace clause, which cannot name a loader or action either. + if (!ts.isNamedExports(statement.exportClause)) continue; + + for (const element of statement.exportClause.elements) { + const exportedName = element.name.text; + if (exportedName !== "loader" && exportedName !== "action") continue; + // A re-export (`export { loader } from "./x"`) has no local binding to resolve. + if (statement.moduleSpecifier) { + record(exportedName, NO_INITIALIZER); + continue; + } + const localName = element.propertyName?.text ?? exportedName; + record(exportedName, resolveLocal(localName, locals, new Set())); + } + } + } + + if (!target.hasLoader && !target.hasAction) return null; + + const callbackCatches: CatchEvidence[] = []; + const catches: CatchEvidence[] = []; + const logCalls: LogCall[] = []; + + const wholeEntry = newBodyFacts(); + const byExport: Record = { + loader: newBodyFacts(), + action: newBodyFacts(), + }; + + const localFunctions = collectLocalFunctions(sf); + // One hop into a same-file helper, whose statements, try/catch and callees belong to the entry + // point. `visited` stops a cycle and any double counting; `helperOwners` carries which EXPORTS + // reach the helper, taking the union on a second discovery because `visited` has already queued it. + const visited = new Set(target.functions); + const helpers: EntryFunction[] = []; + const helperOwners = new Map>(); + + const walkBody = (fn: EntryFunction, followHelpers: boolean, owners: ReadonlySet) => { + // One push site feeds the entry-point-wide list and each owning export's list, so they cannot + // drift apart. See `EntryPoint.loaderCalleeNames`. + const sinks: BodyFacts[] = [wholeEntry]; + for (const owner of owners) sinks.push(byExport[owner]); + const collectTested = (node: ts.Node) => { + if (ts.isIdentifier(node)) for (const s of sinks) s.testedNames.add(node.text); + ts.forEachChild(node, collectTested); + }; + + const addStatements = (n: number) => { + for (const sink of sinks) sink.statementCount += n; + }; + addStatements(countFunctionStatements(fn)); + + if (!fn.body) return; + // `inCallback` is true once the walk has entered a per-item iteration callback and is never reset, + // since nesting deeper inside one is still inside it. `calleeNames`, `logCalls` and the statement + // count keep descending regardless; a catch does not, and is kept in `callbackCatches` with its + // evidence rather than dropped. Only an iteration callback is a boundary, not every function-like + // node. See INTERNALS.md, "The iteration-callback boundary". + const visit = (node: ts.Node, inCatch: boolean, inCallback: boolean) => { + if (ts.isFunctionLike(node)) { + if (isEntryFunction(node)) addStatements(countFunctionStatements(node)); + const entersIterationCallback = inCallback || isIterationCallback(node); + ts.forEachChild(node, (child) => visit(child, inCatch, entersIterationCallback)); + return; + } + if (ts.isTryStatement(node)) { + for (const sink of sinks) sink.hasTryCatch = true; + if (node.catchClause) { + // Built the same way for a refused catch as for an own one, so the dead-code defence + // applies to both. Which list it lands in is this walk's attribution decision alone. + const tryStatementCount = countStatements(node.tryBlock.statements); + const clause = catchClauseEvidence(node.catchClause); + (inCallback ? callbackCatches : catches).push({ + rethrows: clause.rethrows, + throws: clause.throws, + branches: clause.branches, + ...guardedWork(node.tryBlock), + guardMayRaise: tryBlockMayThrow(node.tryBlock), + tryStatementCount, + }); + } + } + + if (ts.isCatchClause(node)) { + ts.forEachChild(node, (child) => visit(child, true, inCallback)); + return; + } + + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { + const initializer = unwrap(node.initializer); + if (ts.isCallExpression(initializer)) { + const cn = calleeName(initializer.expression); + if (cn) { + for (const sink of sinks) { + const existing = sink.declaredFrom.get(node.name.text); + if (existing) existing.push(cn); + else sink.declaredFrom.set(node.name.text, [cn]); + } + } + } + } + + if (ts.isIfStatement(node) || ts.isWhileStatement(node) || ts.isSwitchStatement(node)) { + collectTested(node.expression); + } + if (ts.isConditionalExpression(node)) collectTested(node.condition); + + if (ts.isCallExpression(node)) { + const cn = calleeName(node.expression); + if (cn) { + const text = calleeText(node.expression) ?? cn; + for (const sink of sinks) { + sink.calleeNames.push(cn); + sink.calleeTexts.push(text); + } + + if (LOGGER_CALLEE.test(text)) { + logCalls.push({ + callee: text, + fields: objectArgumentFields(node), + inCatch, + }); + } + } + + if (followHelpers) { + const callee = unwrap(node.expression); + if (ts.isIdentifier(callee)) { + const helper = localFunctions.get(callee.text); + if (helper && !visited.has(helper)) { + visited.add(helper); + helpers.push(helper); + helperOwners.set(helper, new Set(owners)); + } else if (helper) { + const already = helperOwners.get(helper); + if (already) for (const owner of owners) already.add(owner); + } + } + } + } + ts.forEachChild(node, (child) => visit(child, inCatch, inCallback)); + }; + visit(fn.body, false, false); + }; + + // Every handler is walked exactly once, whichever exports own it, so the entry-point-wide + // statement count and catch list stay single while the per-export lists see it from both sides. + const ownersOf = (fn: EntryFunction): Set => { + const owners = new Set(); + if (target.loaderFunctions.has(fn)) owners.add("loader"); + if (target.actionFunctions.has(fn)) owners.add("action"); + return owners; + }; + for (const fn of target.functions) walkBody(fn, true, ownersOf(fn)); + for (const helper of helpers) walkBody(helper, false, helperOwners.get(helper) ?? new Set()); + + return { + fileName, + source, + hasLoader: target.hasLoader, + hasAction: target.hasAction, + loaderInitializerCallee: target.loaderInitializerCallee, + actionInitializerCallee: target.actionInitializerCallee, + loaderBuilderOptions: target.loaderBuilderOptions, + actionBuilderOptions: target.actionBuilderOptions, + // No handler function and no builder call: `export { action } from "./handler.server"` and + // `export const action = handleWebhook`. See `EntryPoint.delegating`. + delegating: + target.functions.size === 0 && + target.loaderInitializerCallee === null && + target.actionInitializerCallee === null, + loaderScopesByCaller: scopesByCallerIn(target.loaderFunctions), + actionScopesByCaller: scopesByCallerIn(target.actionFunctions), + loaderCheckedCallees: checkedCalleesOf(byExport.loader), + actionCheckedCallees: checkedCalleesOf(byExport.action), + importedNames, + calleeNames: wholeEntry.calleeNames, + loaderCalleeNames: byExport.loader.calleeNames, + actionCalleeNames: byExport.action.calleeNames, + loaderCalleeTexts: byExport.loader.calleeTexts, + actionCalleeTexts: byExport.action.calleeTexts, + hasTryCatch: wholeEntry.hasTryCatch, + loaderHasTryCatch: byExport.loader.hasTryCatch, + actionHasTryCatch: byExport.action.hasTryCatch, + catches, + callbackCatches, + logCalls, + statementCount: wholeEntry.statementCount, + loaderStatementCount: byExport.loader.statementCount, + actionStatementCount: byExport.action.statementCount, + }; +} + +const SOURCE_FILE = /\.tsx?$/; + +/** + * Whether a file name is one the scanner reads at all. Exported because three test files ask the same + * question and each had written its own copy, and a predicate that drifts makes the corpus's + * anti-vacuity thresholds count files the scan never saw. + */ +export function isScannableFile(fileName: string): boolean { + return SOURCE_FILE.test(fileName) && !fileName.endsWith(".d.ts"); +} + +/** One route module: where to read it, and the name the report and the scan record it under. */ +export type RouteModuleFile = { absolutePath: string; relativeName: string }; + +/** + * The route modules under `dir`: every scannable flat file, plus the `route.ts`/`route.tsx` of each + * immediate subdirectory. Exported so `mutationCorpus.test.ts` materializes exactly this set rather + * than re-deriving it, for the same reason as `isScannableFile` above. + */ +export function routeModuleFiles(dir: string): RouteModuleFile[] { + const files: RouteModuleFile[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + // Flat-route directories hold the route module in `route.ts`/`route.tsx`. + for (const child of readdirSync(join(dir, entry.name), { withFileTypes: true })) { + if (!child.isFile() || (child.name !== "route.ts" && child.name !== "route.tsx")) continue; + files.push({ + absolutePath: join(dir, entry.name, child.name), + relativeName: `${entry.name}/${child.name}`, + }); + } + continue; + } + if (!entry.isFile() || !isScannableFile(entry.name)) continue; + files.push({ absolutePath: join(dir, entry.name), relativeName: entry.name }); + } + // `readdirSync` order is filesystem-defined, and it reaches the report: head and base are scanned + // from two different directories, so an order difference alone would read as a delta and post a + // comment claiming a change no score made. + return files.sort((a, b) => + a.relativeName < b.relativeName ? -1 : a.relativeName > b.relativeName ? 1 : 0 + ); +} + +export function scanDirectory(dir: string): { + entryPoints: EntryPoint[]; + parseFailures: string[]; +} { + const entryPoints: EntryPoint[] = []; + const parseFailures: string[] = []; + + const scan = (absolutePath: string, relativeName: string) => { + let ep: EntryPoint | null; + try { + ep = scanFile(relativeName, readFileSync(absolutePath, "utf8")); + } catch (error) { + // Only a genuinely malformed source is a parse failure. An unreadable file or a bug in the + // scanner must not be laundered into the same bucket, or a non-zero count means nothing. + if (error instanceof ParseFailureError) { + parseFailures.push(`${relativeName}: ${error.diagnostic}`); + return; + } + throw error; + } + if (ep) entryPoints.push(ep); + }; + + for (const file of routeModuleFiles(dir)) scan(file.absolutePath, file.relativeName); + + return { entryPoints, parseFailures }; +} diff --git a/internal-packages/observability-map/src/score.test.ts b/internal-packages/observability-map/src/score.test.ts new file mode 100644 index 000000000..0691f103a --- /dev/null +++ b/internal-packages/observability-map/src/score.test.ts @@ -0,0 +1,617 @@ +import { scoreEntry, buildReport } from "./score.js"; +import { scanFile } from "./scan.js"; + +const BUILDER = `import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +export const loader = createLoaderApiRoute({}, async () => new Response("ok"));`; + +const RAW = `import { prisma } from "~/db.server"; +export async function loader() { return prisma.thing.findMany(); }`; + +/** Trivial and not sensitive: every scored check reports not-applicable. */ +const TRIVIAL = `export const loader = () => new Response("ok");`; + +/** Not trivial (touches prisma, has a try/catch) and not sensitive: swallows every error and + * records nothing about whose failure it was, so both applicable scored checks fail. */ +const BUSY_AND_FAILING = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + +/** Guarded, classifies what it catches, and names the tenant on the failure path. */ +const CLEAN = `import { requireUserId } from "~/services/session.server"; +import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function action({ request, params }) { + const userId = await requireUserId(request); + try { + return await prisma.token.create({ data: { userId } }); + } catch (error) { + logger.error("token create failed", { userId, environmentId: params.envId, error }); + throw error; + } +}`; + +describe("scoreEntry", () => { + it("scores an entry that passes every applicable check 100", () => { + expect(scoreEntry(scanFile("api.v1.auth.tokens.ts", CLEAN)!).score).toBe(100); + }); + + // A builder wrapper classifies errors for the route, but the route itself catches nothing and + // names nobody on its failure path, so there is one applicable check and it fails. + it("does not credit a builder route for the error handling it does not do", () => { + const scored = scoreEntry(scanFile("api.v1.a.ts", BUILDER)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.score).toBe(0); + }); + + it("excludes audit-trail from the per-entry score", () => { + const scored = scoreEntry(scanFile("api.v1.auth.jwt.ts", CLEAN)!); + expect(scored.checks.find((c) => c.id === "audit-trail")!.status).toBe("fail"); + expect(scored.score).toBe(100); + }); + + it("counts a suppressed check as not-applicable", () => { + const suppressed = `// obs-map-disable error-classification -- health probe +${RAW}`; + const scored = scoreEntry(scanFile("api.v1.b.ts", suppressed)!); + const ec = scored.checks.find((c) => c.id === "error-classification")!; + expect(ec.status).toBe("not-applicable"); + }); + + it("a suppressed-to-passing entry point does not read as unmeasured", () => { + // error-classification would fail here; auth-boundary is not-applicable (not sensitive). + // Suppressing the only applicable scored check must not be indistinguishable from an entry + // point nothing applies to: it is still reported, just not scored on that axis. + const suppressed = `// obs-map-disable error-classification -- health probe +${BUSY_AND_FAILING}`; + const scored = scoreEntry(scanFile("api.v1.c.ts", suppressed)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + // The point of the test, which it did not previously assert: request-context still applies, so + // the entry is still measured and still counted in the mean. + expect(scored.checks.find((c) => c.id === "request-context")!.status).toBe("fail"); + expect(scored.measured).toBe(true); + }); + + // I1. `score = passed / applicable` meant removing a failing check from the denominator raised + // the entry's score, so suppression laundered findings into points. A suppression buys removal + // from the worklist, never a better number. + it("does not raise the score when a failing check is suppressed", () => { + const source = `import { requireUserId } from "~/services/session.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const userId = await requireUserId(request); + try { return await prisma.token.create({ data: { userId } }); } + catch (error) { return null; } +}`; + const plain = scoreEntry(scanFile("api.v1.auth.tokens.ts", source)!); + const suppressed = scoreEntry( + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable error-classification -- deliberate, see ticket +${source}` + )! + ); + + expect(plain.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(suppressed.checks.find((c) => c.id === "error-classification")!.status).toBe( + "not-applicable" + ); + expect(suppressed.score).toBeLessThanOrEqual(plain.score); + }); + + it("records which scored checks were suppressed", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.suppressed).toEqual(["error-classification", "request-context"]); + }); + + // A1. `measured` reads pre-suppression applicability. Before the fix it read `visible` + // (post-suppression) applicability, so suppressing an entry's only applicable checks flipped + // `measured` to false and dropped the entry, still failing, out of the global mean, every family + // mean and the sensitive cohort. `unmeasured` stays for entries nothing was ever applicable to. + it("still reads as measured when every applicable scored check is suppressed", () => { + const suppressed = scoreEntry( + scanFile( + "api.v1.b.ts", + `// obs-map-disable error-classification -- health probe +// obs-map-disable request-context -- nothing to name here +${BUSY_AND_FAILING}` + )! + ); + expect(suppressed.measured).toBe(true); + expect(suppressed.score).toBe(0); + }); + + it("marks an entry point with nothing applicable as unmeasured, scored 100", () => { + const scored = scoreEntry(scanFile("resources.health.ts", TRIVIAL)!); + expect(scored.checks.every((c) => c.status === "not-applicable")).toBe(true); + expect(scored.measured).toBe(false); + expect(scored.score).toBe(100); + }); + + it("marks an entry point with at least one applicable scored check as measured", () => { + const scored = scoreEntry(scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!); + expect(scored.measured).toBe(true); + }); +}); + +describe("buildReport", () => { + it("reports the request-context gap as a figure, like the audit gap", () => { + const naming = scanFile( + "api.v1.named.ts", + `import { logger } from "~/services/logger.server"; + import { prisma } from "~/db.server"; + export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } + }` + )!; + const silent = scanFile( + "api.v1.silent.ts", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.thing.findMany(); }` + )!; + const trivial = scanFile( + "resources.health.ts", + `export const loader = () => new Response("ok");` + )!; + + const report = buildReport([naming, silent, trivial], []); + expect(report.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + + it("counts suppressions so laundering is visible in the report", () => { + const report = buildReport( + [ + scanFile( + "api.v1.b.ts", + `// obs-map-disable error-classification -- health probe +${BUSY_AND_FAILING}` + )!, + scanFile("api.v1.c.ts", BUSY_AND_FAILING)!, + ], + [] + ); + expect(report.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + + // I4. contextGap and auditGap read `checks` (post-suppression), so suppressing the one failing + // request-context on an entry removed it from the denominator too, moving CONTEXT from 1 of 2 + // (50%) to 1 of 1 (100%) printed on the same screen as "a suppression does not raise a score". + // Both gaps now read `rawChecks`, pre-suppression, exactly like `measured`. + it("does not let a suppressed request-context finding shrink the context gap's denominator", () => { + const failing = `import { prisma } from "~/db.server"; +export async function loader() { + try { return await prisma.thing.findMany(); } catch (e) { return null; } +}`; + const passing = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { return await prisma.thing.findMany(); } + catch (error) { logger.error("failed", { environmentId: params.envId, error }); throw error; } +}`; + + const before = buildReport([scanFile("a.ts", failing)!, scanFile("b.ts", passing)!], []); + expect(before.contextGap).toEqual({ applicable: 2, naming: 1 }); + + const after = buildReport( + [ + scanFile( + "a.ts", + `// obs-map-disable request-context -- silence +${failing}` + )!, + scanFile("b.ts", passing)!, + ], + [] + ); + expect(after.contextGap).toEqual({ applicable: 2, naming: 1 }); + }); + + // I4, second half. A suppressed audit-trail directive never showed up in `suppressed` because + // audit-trail is not in SCORED_CHECK_IDS, so the SUPPRESSED line undercounted while the audit + // denominator silently shrank underneath it. Every suppression is now counted, scored or not. + it("counts an audit-trail suppression and does not let it shrink the audit gap's denominator", () => { + const missingAudit = `import { prisma } from "~/db.server"; +export async function action() { return prisma.token.create({ data: {} }); }`; + const withAudit = `import { clearImpersonation } from "~/models/admin.server"; +import { prisma } from "~/db.server"; +export async function action({ request }) { + const token = await prisma.token.create({ data: {} }); + await clearImpersonation(request, "/admin"); + return json(token); +}`; + + const before = buildReport( + [ + scanFile("api.v1.auth.tokens.ts", missingAudit)!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(before.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + + const after = buildReport( + [ + scanFile( + "api.v1.auth.tokens.ts", + `// obs-map-disable audit-trail -- accepted risk +${missingAudit}` + )!, + scanFile("api.v1.auth.jwt.ts", withAudit)!, + ], + [] + ); + expect(after.auditGap).toEqual({ sensitiveMutations: 2, withAudit: 1 }); + expect(after.suppressions).toEqual({ entries: 1, checks: 1 }); + }); + + it("reports the audit gap separately from the score", () => { + // A sensitive mutation with no audit record, but nothing else wrong: the audit gap is reported + // as its own figure and must not pull the score down with it. + const report = buildReport([scanFile("api.v1.auth.tokens.ts", CLEAN)!], []); + expect(report.auditGap.sensitiveMutations).toBe(1); + expect(report.auditGap.withAudit).toBe(0); + expect(report.global).toBe(100); + }); + + it("records parse failures", () => { + const report = buildReport([scanFile("api.v1.a.ts", BUILDER)!], ["broken.ts"]); + expect(report.parseFailures).toEqual(["broken.ts"]); + }); + + it("excludes an unmeasured entry point from the global mean", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + expect(report.measured).toBe(1); + expect(report.unmeasured).toBe(1); + // If the trivial entry's vacuous 100 counted toward the mean, the global score would be 50 + // instead of matching the one entry that was actually measured. + expect(report.global).toBe(scoreEntry(busy).score); + }); + + it("excludes an unmeasured entry point from its family mean too", () => { + const trivial = scanFile("resources.health.ts", TRIVIAL)!; + const busy = scanFile("resources.busy.ts", BUSY_AND_FAILING)!; + + const report = buildReport([trivial, busy], []); + + const family = report.byFamily["resources"]!; + expect(family.n).toBe(2); + expect(family.measured).toBe(1); + expect(family.mean).toBe(scoreEntry(busy).score); + }); +}); + +// A1. `measured` used to read post-suppression (`visible`) applicability, so suppressing an +// entry's only applicable checks removed it from the global mean, every family mean and the +// sensitive cohort, rather than keeping it at its capped score. That is how a suppression with no +// behavioural change moved the global from 17 to 33 tree-wide. +describe("A1: a suppression cannot raise the global", () => { + it("scoring 100 and 0, suppressing every check on the failing entry leaves the global at 50", () => { + const passing = scanFile("api.v1.auth.tokens.ts", CLEAN)!; + const failing = scanFile("api.v1.busy.ts", BUSY_AND_FAILING)!; + + const before = buildReport([passing, failing], []); + expect(before.global).toBe(50); + + const failingSuppressed = scanFile( + "api.v1.busy.ts", + `// obs-map-disable error-classification -- silence +// obs-map-disable request-context -- silence +${BUSY_AND_FAILING}` + )!; + const after = buildReport([passing, failingSuppressed], []); + + expect(after.global).toBe(50); + expect(after.measured).toBe(2); + }); + + // A test suppressing a PASSING check used to sit here and was deleted rather than kept as + // decoration: the per-entry cap guarantees it on its own, so it passed against the pre-fix code. +}); + +/** + * The invariant both ways round. Removing error handling must lower the score, and that direction + * alone could not see the free-points path: adding a catch that only rethrows used to move a route + * from not-applicable to pass, worth 27 points across the tree. + */ +describe("no-op error handling must not pay", () => { + const BODY = `const rows = await prisma.thing.findMany(); + return json({ rows });`; + + const plain = `import { prisma } from "~/db.server"; +export async function loader() { + ${BODY} +}`; + + const wrappedInARethrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + throw e; + } +}`; + + const handled = `import { logger } from "~/services/logger.server"; +import { prisma } from "~/db.server"; +export async function loader({ params }) { + try { + ${BODY} + } catch (error) { + if (error instanceof NotFoundError) return json({ error: "not found" }, { status: 404 }); + logger.error("thing lookup failed", { environmentId: params.envId, error }); + throw error; + } +}`; + + it("does not pay for wrapping a body in a catch that only rethrows", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", plain)!); + const after = scoreEntry(scanFile("api.v1.x.ts", wrappedInARethrow)!); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + + // A4. rethrows used to be set by any ThrowStatement anywhere in the clause, dead code included, + // so appending `throw e;` after a `return` in a swallowing catch flipped error-classification + // from fail to not-applicable: a mutation with no behavioural effect that hid the swallow. + it("does not improve the verdict when a dead throw follows a return in a swallowing catch", () => { + const swallows = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + } +}`; + const swallowsWithDeadThrow = `import { prisma } from "~/db.server"; +export async function loader() { + try { + ${BODY} + } catch (e) { + return null; + throw e; + } +}`; + + const before = scoreEntry(scanFile("api.v1.x.ts", swallows)!); + const after = scoreEntry(scanFile("api.v1.x.ts", swallowsWithDeadThrow)!); + + expect(before.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.checks.find((c) => c.id === "error-classification")!.status).toBe("fail"); + expect(after.score).toBeLessThanOrEqual(before.score); + }); + + it("does not pay for wrapping a whole tree in catches that only rethrow", () => { + const before = buildReport( + [scanFile("api.v1.x.ts", plain)!, scanFile("api.v1.y.ts", plain)!], + [] + ); + const after = buildReport( + [scanFile("api.v1.x.ts", wrappedInARethrow)!, scanFile("api.v1.y.ts", wrappedInARethrow)!], + [] + ); + expect(after.global!).toBeLessThanOrEqual(before.global!); + }); + + it("does not pay for deleting error handling either", () => { + const before = scoreEntry(scanFile("api.v1.x.ts", handled)!); + const after = scoreEntry(scanFile("api.v1.x.ts", plain)!); + expect(after.score).toBeLessThanOrEqual(before.score); + // And the handled version is genuinely better, so the invariant is not holding by both being 0. + expect(before.score).toBeGreaterThan(after.score); + }); + + it("still credits a catch that decides something on its way through", () => { + const scored = scoreEntry(scanFile("api.v1.x.ts", handled)!); + expect(scored.checks.find((c) => c.id === "error-classification")!.status).toBe("pass"); + }); +}); + +// C4b. A route whose body is in another module used to be scored as if it were a redirect stub: +// zero statements, zero callees, `isTrivial` true, every check not-applicable, and a placeholder +// 100 that no mean ever used. The tool said nothing about it at all, so moving a body into a +// `.server.ts` file silently deleted the route from the metric. +describe("a route that delegates its body to another module", () => { + const DELEGATED = `export { action } from "./handler.server";`; + const entry = () => scoreEntry(scanFile("webhooks.v1.stripe.ts", DELEGATED)!); + + it("reports every check as not-applicable for the reason that is true", () => { + const e = entry(); + expect(e.rawChecks.every((c) => c.status === "not-applicable")).toBe(true); + expect(new Set(e.rawChecks.map((c) => c.detail))).toEqual( + new Set(["delegates its body to another module"]) + ); + }); + + it("is not measured, and says so on the entry", () => { + const e = entry(); + expect(e.delegating).toBe(true); + expect(e.measured).toBe(false); + }); + + // request-context would otherwise fail it for leaving its failures to the central handler, which + // is an accusation about a body this file does not contain. + it("is not accused of anything the scanner cannot see", () => { + expect(entry().rawChecks.find((c) => c.id === "request-context")!.status).toBe( + "not-applicable" + ); + }); + + it("is counted apart from the entries nothing happened to apply to", () => { + const r = buildReport( + [scanFile("webhooks.v1.stripe.ts", DELEGATED)!, scanFile("@.ts", TRIVIAL)!], + [] + ); + expect(r.delegating).toEqual(["webhooks.v1.stripe.ts"]); + expect(r.unmeasured).toBe(1); + expect(r.measured).toBe(0); + }); + + it("cannot raise the global, since it is in no mean", () => { + const withDelegate = buildReport( + [scanFile("api.v1.b.ts", BUSY_AND_FAILING)!, scanFile("webhooks.v1.stripe.ts", DELEGATED)!], + [] + ); + const without = buildReport([scanFile("api.v1.b.ts", BUSY_AND_FAILING)!], []); + expect(withDelegate.global).toBe(without.global); + }); +}); + +// C5. The composite is disclosed rather than weighted: the reader gets applicability, pass rate, +// how many entries rest on one check alone, and what the global would be without each check. +describe("per-check contribution", () => { + const r = () => + buildReport( + [ + scanFile("api.v1.a.ts", BUSY_AND_FAILING)!, + scanFile("api.v1.b.ts", RAW)!, + scanFile("@.ts", TRIVIAL)!, + ], + [] + ); + + it("has one row per check, in registry order", () => { + expect(r().checkContributions.map((c) => c.id)).toEqual([ + "error-classification", + "auth-boundary", + "auth-scope", + "request-context", + "audit-trail", + ]); + }); + + it("counts applicability and passes off the pre-suppression results", () => { + const context = r().checkContributions.find((c) => c.id === "request-context")!; + expect(context.applicable).toBe(2); + expect(context.passed).toBe(0); + }); + + // `RAW` has no catch, so request-context is the only scored check that applies to it. + it("counts the entries that rest on one check alone", () => { + const rows = r().checkContributions; + expect(rows.find((c) => c.id === "request-context")!.sole).toBe(1); + expect(rows.find((c) => c.id === "error-classification")!.sole).toBe(0); + }); + + it("says what the global would be without each scored check", () => { + const report = r(); + expect(report.global).toBe(0); + const errors = report.checkContributions.find((c) => c.id === "error-classification")!; + expect(errors.scored).toBe(true); + expect(errors.globalWithout).toBe(0); + }); + + it("gives no without-figure for a check that is not in the score", () => { + const audit = r().checkContributions.find((c) => c.id === "audit-trail")!; + expect(audit.scored).toBe(false); + expect(audit.globalWithout).toBeNull(); + }); + + // Taking the only applicable check away leaves nothing measured, which is an absence and not a + // perfect score. + it("gives a null without-figure when nothing would be left measured", () => { + const only = buildReport([scanFile("api.v1.b.ts", RAW)!], []); + expect( + only.checkContributions.find((c) => c.id === "request-context")!.globalWithout + ).toBeNull(); + }); +}); + +// C4b, stated as the property rather than as a shape. Moving a body into a `.server.ts` file is an +// ordinary refactor: it must not delete the route from the metric silently. The corpus cannot hold +// this one, because its per-route assertion reads "dropped out of the measured set" as a rise, and +// dropping out is the correct outcome here. What is forbidden is dropping out QUIETLY. +describe("refactoring a body out of the route file", () => { + const BEFORE = `import { prisma } from "~/db.server"; +export async function action() { + try { return await prisma.thing.create({ data: {} }); } catch (e) { return null; } +}`; + const AFTER = `export { action } from "./handler.server";`; + + it("leaves the mean, and is reported instead of being dropped", () => { + const before = buildReport([scanFile("webhooks.v1.stripe.ts", BEFORE)!], []); + const after = buildReport([scanFile("webhooks.v1.stripe.ts", AFTER)!], []); + + expect(before.measured).toBe(1); + expect(before.global).toBe(0); + expect(after.measured).toBe(0); + expect(after.global).toBeNull(); + expect(after.delegating).toEqual(["webhooks.v1.stripe.ts"]); + expect(after.unmeasured).toBe(0); + }); +}); + +/** + * `contextGap` and `auditGap` are the arithmetic `checkContributions` already does, written out again + * by hand for two named ids, on the two figures the report puts in front of a reader as headlines. + * Pinned rather than shared: what matters is that three spellings of "applicable, and how many + * passed" cannot disagree, and an assertion says that without moving code the renderers read. + */ +describe("the hand-rolled gap figures agree with the per-check contributions", () => { + const SOURCE = `import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +export async function action({ params }) { + try { + return await prisma.apiKey.create({ data: { orgId: params.orgId } }); + } catch (e) { + logger.error("failed", { orgId: params.orgId }); + return null; + } +}`; + + // A sensitive mutation that DOES record an audit event, so `withAudit` is not simply + // `sensitiveMutations`. Without it the audit assertion held whatever the numerator counted. + const AUDITED = `import { prisma } from "~/db.server"; +import { startImpersonation } from "~/models/admin.server"; +export async function action({ request, params }) { + const session = await startImpersonation(request, params.userId); + await prisma.apiKey.create({ data: { orgId: params.orgId } }); + return redirect("/", { headers: session }); +}`; + + const report = buildReport( + [ + scanFile("api.v1.orgs.$orgId.apikeys.ts", SOURCE)!, + scanFile("api.v1.tokens.ts", SOURCE)!, + scanFile("resources.impersonation.ts", AUDITED)!, + scanFile("healthcheck.ts", `export const loader = () => new Response("ok");`)!, + ], + [] + ); + + const contribution = (id: string) => report.checkContributions.find((c) => c.id === id)!; + + it("reports the same request-context denominator and numerator", () => { + expect(report.contextGap.applicable).toBe(contribution("request-context").applicable); + expect(report.contextGap.naming).toBe(contribution("request-context").passed); + }); + + it("reports the same audit-trail denominator and numerator", () => { + expect(report.auditGap.sensitiveMutations).toBe(contribution("audit-trail").applicable); + expect(report.auditGap.withAudit).toBe(contribution("audit-trail").passed); + }); + + // A denominator of zero would make both assertions above hold vacuously. + // Both assertions above hold vacuously on a zero denominator, and the audit one holds vacuously + // whenever every applicable route fails, since the two counts coincide. + it("measured something for both of them, with the audit numerator strictly between", () => { + expect(report.contextGap.applicable).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeGreaterThan(0); + expect(report.auditGap.withAudit).toBeLessThan(report.auditGap.sensitiveMutations); + }); +}); diff --git a/internal-packages/observability-map/src/score.ts b/internal-packages/observability-map/src/score.ts new file mode 100644 index 000000000..c045e216f --- /dev/null +++ b/internal-packages/observability-map/src/score.ts @@ -0,0 +1,249 @@ +import type { CheckResult, EntryPoint } from "./types.js"; +import { CHECKS, SCORED_CHECK_IDS } from "./checks/index.js"; +import { parseSuppressions } from "./suppression.js"; +import { familyOf, routePathOf, type Family } from "./adapters/remix.js"; +import { classifySensitivity } from "./sensitivity.js"; + +export type ScoredEntry = { + fileName: string; + routePath: string; + family: Family; + sensitive: boolean; + /** The route's body is in another module, so every check reads not-applicable for that reason. + * Carried apart from `measured` because "we could not see it" and "nothing happened to apply" are + * different facts and the report has to be able to say which. */ + delegating: boolean; + /** Post-suppression display view. Every denominator reads `rawChecks` instead. */ + checks: CheckResult[]; + /** Every check exactly as it ran. The one true source for any figure that counts applicability. */ + rawChecks: CheckResult[]; + /** Whether at least one scored check was applicable BEFORE suppression, so a fully suppressed + * entry stays measured at its capped score. False means the 100 in `score` is a vacuous default + * rather than a finding, and every mean excludes it. */ + measured: boolean; + /** Every check a comment in the source suppressed, scored or not, in `CHECKS` order. */ + suppressed: string[]; + /** Ids in a directive that name no check, so they suppress nothing. Carried so the renderers can + * say so: dropping them silently made a typo look like an acknowledgement. */ + unknownSuppressions: string[]; + /** Passed over applicable, across scored checks only. 100 when nothing applies. */ + score: number; +}; + +/** + * What one check contributes to the composite. Disclosed rather than weighted, deliberately: see + * README, "What the score is made of". + */ +export type CheckContribution = { + id: string; + /** Entry points the check was applicable to, pre-suppression. */ + applicable: number; + /** Of those, how many passed. */ + passed: number; + /** Whether the check feeds `global` at all. `audit-trail` does not, see `buildReport`. */ + scored: boolean; + /** Entry points this was the ONLY applicable scored check for, so their score is its verdict and + * nothing else. Zero for a check that is not scored. */ + sole: number; + /** The global recomputed with this check out of the scored set, so the difference from `global` is + * what the check is worth. Null when it is not scored, or when nothing would be left measured. */ + globalWithout: number | null; +}; + +export type MapReport = { + /** Null when no entry point had an applicable scored check: an absent figure, not a perfect one. */ + global: number | null; + /** Entry points with at least one applicable scored check, i.e. those `global` is averaged over. */ + measured: number; + /** Entry points every scored check reported not-applicable for; excluded from `global`. Counts + * only routes the scanner could read: a delegating one is in `delegating` instead. */ + unmeasured: number; + /** Routes whose body is in another module, by file name. Excluded from `global` for the same reason + * a parse failure is, and reported for the same reason: the denominator is smaller than the entry + * point count and nothing about these routes has been checked. */ + delegating: string[]; + /** Per-check applicability, pass rate and worth, in `CHECKS` order. */ + checkContributions: CheckContribution[]; + /** Suppressions in force: how many entry points carry one, and how many scored checks in total. */ + suppressions: { entries: number; checks: number }; + /** Suppression directives naming no check, per file, so a typo is reported rather than dropped. */ + unknownSuppressions: { fileName: string; ids: string[] }[]; + byFamily: Record; + sensitiveCohort: { n: number; measured: number; mean: number | null }; + auditGap: { sensitiveMutations: number; withAudit: number }; + /** Reported as a figure rather than as hundreds of identical list entries, the same treatment + * `audit-trail` gets, while staying fully in the score. */ + contextGap: { applicable: number; naming: number }; + entries: ScoredEntry[]; + parseFailures: string[]; +}; + +/** + * What every check reports for a route whose body is in another module. Answered here rather than in + * each check, because it is a fact about what the scan could see and not about any one question, so + * no check tests `ep.delegating` itself. Two did, and both branches were unreachable. + */ +const DELEGATED_CHECKS = (): CheckResult[] => + CHECKS.map((c) => ({ + id: c.id, + status: "not-applicable" as const, + detail: "delegates its body to another module", + })); + +export function scoreEntry(ep: EntryPoint): ScoredEntry { + const { byId: suppressed, unknown } = parseSuppressions(ep.source, ep.fileName); + const raw = ep.delegating ? DELEGATED_CHECKS() : CHECKS.map((c) => c.run(ep)); + const checks = raw.map((result) => { + const reason = suppressed.get(result.id); + return reason + ? { id: result.id, status: "not-applicable" as const, detail: `suppressed: ${reason}` } + : result; + }); + + const scored = raw.filter((c) => SCORED_CHECK_IDS.includes(c.id)); + const ratio = (of: CheckResult[]) => { + const applicable = of.filter((c) => c.status !== "not-applicable"); + if (applicable.length === 0) return 100; + return Math.round( + (applicable.filter((c) => c.status === "pass").length / applicable.length) * 100 + ); + }; + + const visible = scored.filter((c) => !suppressed.has(c.id)); + const scoredApplicable = scored.filter((c) => c.status !== "not-applicable"); + + return { + fileName: ep.fileName, + routePath: routePathOf(ep.fileName), + family: familyOf(ep.fileName), + sensitive: classifySensitivity(ep).sensitive, + delegating: ep.delegating, + checks, + rawChecks: raw, + suppressed: raw.filter((c) => suppressed.has(c.id)).map((c) => c.id), + unknownSuppressions: unknown, + measured: scoredApplicable.length > 0, + // Capped by the pre-suppression ratio, or removing a failing check from both numerator and + // denominator raises it, which is how 33 became 50 became 100 before the cap existed. + score: Math.min(ratio(visible), ratio(scored)), + }; +} + +/** Null for an empty group rather than 100: rendering the absence as a full green bar said the + * opposite of what the data said. */ +const mean = (xs: number[]): number | null => + xs.length === 0 ? null : Math.round(xs.reduce((a, b) => a + b, 0) / xs.length); + +/** + * `n` is every entry point in the group; `mean` is over the measured subset only. `measured` is + * reported alongside so a reader can tell a family scoring high because it is clean from one scoring + * high because most of it was never measured. + */ +function groupStats(entries: ScoredEntry[]): { + n: number; + measured: number; + mean: number | null; +} { + const measuredEntries = entries.filter((e) => e.measured); + return { + n: entries.length, + measured: measuredEntries.length, + mean: mean(measuredEntries.map((e) => e.score)), + }; +} + +/** + * The global as it would read with `omitted` out of the scored set. Recomputed from `rawChecks` minus + * the suppression cap, because lowering both figures by the same rule would leave the difference + * saying something about suppressions rather than about the check. + */ +function globalWithout(entries: ScoredEntry[], omitted: string): number | null { + const scores: number[] = []; + for (const e of entries) { + const applicable = e.rawChecks.filter( + (c) => SCORED_CHECK_IDS.includes(c.id) && c.id !== omitted && c.status !== "not-applicable" + ); + if (applicable.length === 0) continue; + const passed = applicable.filter((c) => c.status === "pass").length; + scores.push(Math.round((passed / applicable.length) * 100)); + } + return mean(scores); +} + +function checkContributions(entries: ScoredEntry[]): CheckContribution[] { + return CHECKS.map((check) => { + const results = entries + .map((e) => e.rawChecks.find((c) => c.id === check.id)) + .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); + const scored = SCORED_CHECK_IDS.includes(check.id); + return { + id: check.id, + applicable: results.length, + passed: results.filter((c) => c.status === "pass").length, + scored, + sole: scored + ? entries.filter((e) => { + const applicable = e.rawChecks.filter( + (c) => SCORED_CHECK_IDS.includes(c.id) && c.status !== "not-applicable" + ); + return applicable.length === 1 && applicable[0]!.id === check.id; + }).length + : 0, + globalWithout: scored ? globalWithout(entries, check.id) : null, + }; + }); +} + +export function buildReport(eps: EntryPoint[], parseFailures: string[]): MapReport { + const entries = eps.map(scoreEntry); + const measuredEntries = entries.filter((e) => e.measured); + + const byFamily: MapReport["byFamily"] = {}; + for (const family of new Set(entries.map((e) => e.family))) { + byFamily[family] = groupStats(entries.filter((e) => e.family === family)); + } + + const sensitive = entries.filter((e) => e.sensitive); + + // Both gaps read `rawChecks`, pre-suppression: suppressing the one finding on an entry must not + // shrink these denominators and raise the printed percentage, on the same screen as a claim that + // suppression cannot do that. + const contextChecks = entries + .map((e) => e.rawChecks.find((c) => c.id === "request-context")) + .filter((c): c is CheckResult => c !== undefined && c.status !== "not-applicable"); + + const auditApplicable = entries.filter((e) => + e.rawChecks.some((c) => c.id === "audit-trail" && c.status !== "not-applicable") + ); + + const suppressing = entries.filter((e) => e.suppressed.length > 0); + + return { + global: mean(measuredEntries.map((e) => e.score)), + measured: measuredEntries.length, + unmeasured: entries.filter((e) => !e.measured && !e.delegating).length, + delegating: entries.filter((e) => e.delegating).map((e) => e.fileName), + checkContributions: checkContributions(entries), + suppressions: { + entries: suppressing.length, + checks: suppressing.reduce((n, e) => n + e.suppressed.length, 0), + }, + unknownSuppressions: entries + .filter((e) => e.unknownSuppressions.length > 0) + .map((e) => ({ fileName: e.fileName, ids: e.unknownSuppressions })), + byFamily, + sensitiveCohort: groupStats(sensitive), + auditGap: { + sensitiveMutations: auditApplicable.length, + withAudit: auditApplicable.filter((e) => + e.rawChecks.some((c) => c.id === "audit-trail" && c.status === "pass") + ).length, + }, + contextGap: { + applicable: contextChecks.length, + naming: contextChecks.filter((c) => c.status === "pass").length, + }, + entries, + parseFailures, + }; +} diff --git a/internal-packages/observability-map/src/sensitivity.test.ts b/internal-packages/observability-map/src/sensitivity.test.ts new file mode 100644 index 000000000..c54d4d957 --- /dev/null +++ b/internal-packages/observability-map/src/sensitivity.test.ts @@ -0,0 +1,236 @@ +import { classifySensitivity } from "./sensitivity.js"; +import { scanFile } from "./scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("classifySensitivity", () => { + it("flags a route whose filename says nothing but which calls a sensitive helper", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("clearImpersonation"))).toBe(true); + }); + + it("flags on the path when the filename is explicit", () => { + const e = ep("api.v1.projects.$ref.envvars.ts", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag an ordinary read route", () => { + const e = ep( + "api.v1.timezones.ts", + `import { json } from "@remix-run/server-runtime"; + export async function loader() { return json({ timezones: [] }); }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); + + it("does not flag on a substring that merely contains a sensitive word", () => { + const e = ep( + "api.v1.authorship.ts", + `export async function loader() { return { author: "x" }; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// Directory routes: `fileName` can be `dirName/route.tsx` rather than a flat dotted name. A naive +// `fileName.split(".")` treats "billing/route" as one non-matching segment because the slash never +// gets split, so it misses the directory name entirely. The classifier must derive real path +// segments (via the same route-path logic the remix adapter uses) so both shapes work alike. +describe("classifySensitivity: directory routes", () => { + it("flags a single-segment directory route by its directory name", () => { + const e = ep("billing/route.tsx", `export async function loader() { return 1; }`); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("billing"))).toBe(true); + }); + + it("flags a multi-segment directory route via a segment among dynamic params", () => { + const e = ep( + "_app.orgs.$slug.billing/route.tsx", + `export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a directory route on a substring that merely contains a sensitive word", () => { + const e = ep("api.v1.authorship/route.tsx", `export async function loader() { return 1; }`); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +// calleeNames is scoped to the loader/action body, unlike importedNames which is file-wide. A +// sensitive symbol called only at module scope is invisible to calleeNames, so it is only caught +// when it also shows up as an import. +describe("classifySensitivity: calleeNames is body-scoped, importedNames is file-wide", () => { + it("flags a sensitive symbol invoked only at module scope, via the import rather than the callee", () => { + const e = ep( + "api.v1.setup.ts", + `import { startImpersonation } from "~/models/admin.server"; + startImpersonation(globalThis, "seed"); + export async function loader() { return 1; }` + ); + const s = classifySensitivity(e); + expect(s.sensitive).toBe(true); + expect(s.reasons.some((r) => r.includes("startImpersonation"))).toBe(true); + }); + + it("flags a sensitive callee defined locally and invoked inside the loader, even without an import", () => { + const e = ep( + "api.v1.local-admin.ts", + `async function regenerateApiKey() { return "x"; } + export async function loader() { return regenerateApiKey(); }` + ); + expect(classifySensitivity(e).sensitive).toBe(true); + }); + + it("does not flag a sensitive-named call made only at module scope outside the loader/action body", () => { + const e = ep( + "api.v1.module-scope.ts", + `function regenerateApiKey() { return "x"; } + regenerateApiKey(); + export async function loader() { return 1; }` + ); + expect(classifySensitivity(e).sensitive).toBe(false); + }); +}); + +describe("what sensitivity must not mean", () => { + const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + + // C4. Calling the admin guard cannot be what makes a route risky: it is the mitigation, not the + // hazard. Counting it made 34 of 67 sensitive entries sensitive only because they were guarded, + // and `auth-boundary` then passed every one of them on the same call. Circular, and it was the + // fix list's primary sort key. + it("does not treat calling the admin guard as what makes a route sensitive", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.queue-metrics.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { prisma } from "~/db.server"; + export async function loader({ request }) { + await requireAdminApiRequest(request); + return json(await prisma.queueMetric.findMany()); + }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags an admin route that does something sensitive on its own account", () => { + const s = classifySensitivity( + ep( + "admin.api.v1.impersonate.ts", + `import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + import { startImpersonation } from "~/models/admin.server"; + export async function action({ request }) { + await requireAdminApiRequest(request); + return startImpersonation(request, "user_1"); + }` + ) + ); + expect(s.sensitive).toBe(true); + expect(s.reasons).toContain("calls startImpersonation"); + }); + + // A waitpoint token is a run coordination handle, not a credential. Seven of the eight routes + // matching the `tokens` segment were waitpoint routes. + it("does not treat a waitpoint token route as a credential route", () => { + const s = classifySensitivity( + ep( + "api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.waitpoint.update({ where: {}, data: {} }); }` + ) + ); + expect(s.sensitive).toBe(false); + }); + + it("still flags the personal access token routes", () => { + expect( + classifySensitivity( + ep( + "account.tokens/route.tsx", + `import { prisma } from "~/db.server"; + export async function loader() { return prisma.personalAccessToken.findMany(); }` + ) + ).reasons + ).toContain('path segment "tokens"'); + expect( + classifySensitivity( + ep( + "api.v1.token.ts", + `import { prisma } from "~/db.server"; + export async function action() { return prisma.token.create({ data: {} }); }` + ) + ).reasons + ).toContain('path segment "token"'); + }); +}); + +// The classifier once covered tokens, billing, impersonation and envvars and missed the entire +// surface where authorization bugs live. The vocabulary is read off `apps/webapp/app/routes`, and +// `webappSymbols.test.ts` holds it to that. +describe("classifySensitivity: the access-control surface", () => { + const flags = (fileName: string) => + classifySensitivity(ep(fileName, `export async function loader() { return 1; }`)).sensitive; + + it.each([ + ["api.v1.orgs.$orgParam.members.ts", "membership"], + ["api.v1.orgs.$orgParam.invites.ts", "invites"], + ["invite-accept.tsx", "an invite acceptance"], + ["_app.orgs.$organizationSlug.settings.roles/route.tsx", "roles"], + ["orgs.$organizationSlug.team.ts", "the team page"], + ["login._index/route.tsx", "the login page"], + ["login.mfa/route.tsx", "the mfa step"], + ["magic.tsx", "a magic link"], + ["auth.sso.ts", "sso"], + ["account.security/route.tsx", "account security"], + [ + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx", + "api keys", + ], + ["admin.api.v1.revoked-api-keys.$id.ts", "revoked api keys"], + ["_app.orgs.$organizationSlug.settings.billing-limits/route.tsx", "billing limits"], + ["_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx", "billing alerts"], + ["admin.api.v1.orgs.$organizationId.billing-limit.hit.ts", "an admin billing limit"], + ["resources.account.session-duration/route.tsx", "session duration"], + ])("flags %s (%s)", (fileName) => { + expect(flags(fileName)).toBe(true); + }); + + // Remix's trailing underscore opts a route out of its parent layout and says nothing about what + // the route does, so the segment vocabulary is written without it. + it("flags a route whose segment carries the no-layout marker", () => { + expect(flags("resources.impersonation_.view-as.ts")).toBe(true); + expect(flags("resources.impersonation.ts")).toBe(true); + }); + + // Measured and rejected. Every `sessions` route in this tree is the realtime agent-session + // product, sixteen files of it, and reading the word as the auth session surface would have put + // all sixteen in front of the routes that mint credentials. + it("does not flag the agent-session product on the word sessions", () => { + expect(flags("api.v1.sessions.$session.close.ts")).toBe(false); + expect( + flags( + "_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.sessions._index/route.tsx" + ) + ).toBe(false); + }); + + // Measured and rejected. Logging out destroys the caller's own session: no other party's + // credential to guard, no actor to record beyond the one already leaving. + it("does not flag the logout route", () => { + expect(flags("logout.tsx")).toBe(false); + }); + + it("still does not flag an ordinary route that shares a prefix with the new vocabulary", () => { + expect(flags("api.v1.teams-directory.ts")).toBe(false); + expect(flags("resources.logins.ts")).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/src/sensitivity.ts b/internal-packages/observability-map/src/sensitivity.ts new file mode 100644 index 000000000..5ef6582c8 --- /dev/null +++ b/internal-packages/observability-map/src/sensitivity.ts @@ -0,0 +1,137 @@ +import type { EntryPoint } from "./types.js"; +import { routePathOf } from "./adapters/remix.js"; + +/** + * Symbols whose presence says the route does something risky: minting or revoking a credential, + * escalating to another user, destroying a tenant. Calling a guard is never one of them, because a + * mitigation cannot be the hazard, and `webappSymbols.test.ts` fails if a name stops resolving in the + * webapp. Both rules and what they cost: INTERNALS.md, "Sensitivity, and the names the tool + * matches on". + */ +export const SENSITIVE_SYMBOLS = [ + // Escalation: acting as another user. + "startImpersonation", + "clearImpersonation", + "generateImpersonationToken", + // Minting and revoking credentials. + "createPersonalAccessToken", + "createPersonalAccessTokenFromAuthorizationCode", + "revokePersonalAccessToken", + "createOrganizationAccessToken", + "revokeOrganizationAccessToken", + "createAuthorizationCode", + "createApiKeyForEnv", + "createPkApiKeyForEnv", + "regenerateApiKey", + "generateJWTTokenForEnvironment", + "generateRegistryCredentials", + "mintRunToken", + "mintSessionToken", + "mintDashboardAgentToken", + "mintDashboardAgentUserActorToken", + // Access control and tenant destruction. `DeleteOrganizationService`/`DeleteProjectService` are + // classes, reached through `importedNames`: four routes import one and none is named for it. + "removeTeamMember", + "revokeInvite", + "DeleteOrganizationService", + "DeleteProjectService", +]; + +/** + * Segments in `SENSITIVE_SEGMENTS` that match no route in the tree today, kept apart because + * `webappSymbols.test.ts` holds the rest of the vocabulary to naming something. Adding a word here is + * a deliberate statement that it names nothing yet, and shows up in review as one. + */ +export const ANTICIPATED_SEGMENTS = ["payment", "invoices", "secrets"]; + +/** + * Whole path segments only, so "authorship" does not match "auth". Every entry exists in + * `apps/webapp/app/routes` today and `webappSymbols.test.ts` fails if one stops appearing. + * + * Two absences worth knowing rather than rediscovering. `logout` is left out because + * `logout.tsx` destroys the caller's own session, so there is no other party's credential to guard and + * no actor to record, and including it produced one permanent `auth-boundary` failure nothing could + * clear. `sessions` is left out because every `sessions` route in this tree is the realtime + * agent-session product rather than the auth surface, sixteen files of it; the genuine + * session-management surface is `session-duration` below, and the credential minting inside those + * routes is caught by `mintSessionToken` above. + */ +export const SENSITIVE_SEGMENTS = [ + // Credentials, tokens and money: the original vocabulary. + "auth", + "jwt", + "token", + "tokens", + "envvars", + "billing", + ...ANTICIPATED_SEGMENTS, + "impersonate", + "authorization-code", + "regenerate-api-key", + // Access control: who is in a tenant and what they may do. + "members", + "invites", + "invite", + "invite-accept", + "invite-resend", + "invite-revoke", + "roles", + "team", + // Authentication and session management. The login surface handles credentials even though it + // is, by design, the one surface an unauthenticated caller may reach. + "login", + "magic", + "mfa", + "sso", + "security", + "session-duration", + // Credentials again, in the spellings the tree actually uses. + "apikeys", + "revoked-api-keys", + "impersonation", + // The bare `billing` segment matches neither of these, and it matches no admin route at all. + "billing-limit", + "billing-limits", + "billing-alerts", +]; + +/** + * A route-name segment with Remix's layout markers taken off. A trailing underscore opts a route out + * of its parent layout and changes nothing about what the route does. + */ +export function normalizeSegment(segment: string): string { + // Trimmed by hand rather than with /_+$/, which backtracks polynomially and trips CodeQL. Nothing + // here is attacker-controlled, so this is about not spending a reviewer's attention on the alert. + let end = segment.length; + while (end > 0 && segment[end - 1] === "_") end--; + return segment.slice(0, end); +} + +export type Sensitivity = { sensitive: boolean; reasons: string[] }; + +export function classifySensitivity(ep: EntryPoint): Sensitivity { + const reasons: string[] = []; + + // `importedNames` is file-wide and `calleeNames` is body-scoped, so a sensitive symbol called only + // at module scope is caught here only if it is also imported. + const symbols = new Set([...ep.importedNames, ...ep.calleeNames]); + for (const s of SENSITIVE_SYMBOLS) { + if (symbols.has(s)) reasons.push(`calls ${s}`); + } + + // `routePathOf` normalises both flat and directory routes to `/`-separated segments, so this + // matches whole segments in either shape rather than splitting the raw fileName on ".". + const segments = routePathOf(ep.fileName) + .split("/") + .filter((s) => s.length > 0) + .map(normalizeSegment); + for (const [i, seg] of segments.entries()) { + if (!SENSITIVE_SEGMENTS.includes(seg)) continue; + // A waitpoint token is a handle for resuming a run rather than a credential, and seven of the + // eight `tokens` matches in the tree were waitpoint routes. + if ((seg === "token" || seg === "tokens") && segments[i - 1] === "waitpoints") continue; + reasons.push(`path segment "${seg}"`); + } + + return { sensitive: reasons.length > 0, reasons }; +} diff --git a/internal-packages/observability-map/src/suppression.test.ts b/internal-packages/observability-map/src/suppression.test.ts new file mode 100644 index 000000000..2ebe8150b --- /dev/null +++ b/internal-packages/observability-map/src/suppression.test.ts @@ -0,0 +1,296 @@ +import { parseSuppressions, suppressedChecks } from "./suppression.js"; + +describe("suppressedChecks", () => { + it("reads a suppression with its reason", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe, deliberately silent + export async function loader() { return { ok: true }; }` + ); + expect(m.get("error-classification")).toBe("liveness probe, deliberately silent"); + }); + + it("ignores a suppression with no reason", () => { + const m = suppressedChecks(`// obs-map-disable error-classification`); + expect(m.size).toBe(0); + }); + + it("ignores a suppression whose reason is only whitespace", () => { + const m = suppressedChecks(`// obs-map-disable error-classification -- `); + expect(m.size).toBe(0); + }); + + it("reads several suppressions in one file", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe + // obs-map-disable request-context -- no identifiers exist here + export async function loader() { return { ok: true }; }` + ); + expect(m.size).toBe(2); + expect(m.get("error-classification")).toBe("liveness probe"); + expect(m.get("request-context")).toBe("no identifiers exist here"); + }); + + it("returns nothing for a file with no suppressions", () => { + expect(suppressedChecks(`export async function loader() { return 1; }`).size).toBe(0); + }); + + it("does not carry a reason across lines", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification + // some other comment -- with a dash + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + // I2. The directive is a comment directive. Matching it file-wide meant a string literal that + // merely quotes it, in a test fixture or an error message, silently suppressed a real check. + it("ignores the directive inside a string literal", () => { + const m = suppressedChecks( + `const example = "obs-map-disable error-classification -- not a real suppression"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("reads the directive from a block comment", () => { + const m = suppressedChecks( + `/* obs-map-disable auth-boundary -- public by design, see ADR 12 */ + export async function loader() { return 1; }` + ); + expect(m.get("auth-boundary")).toBe("public by design, see ADR 12"); + }); + + it("reads the directive from a jsdoc line", () => { + const m = suppressedChecks( + `/** + * obs-map-disable request-context -- nothing tenant-scoped here + */ + export async function loader() { return 1; }` + ); + expect(m.get("request-context")).toBe("nothing tenant-scoped here"); + }); + + it("ignores code that happens to follow a comment on the same line", () => { + const m = suppressedChecks( + `const x = 1; // obs-map-disable error-classification -- fine + export async function loader() { return x; }` + ); + expect(m.get("error-classification")).toBe("fine"); + }); + + // The directive was called `-next-line` while applying to the whole entry point, so a comment on + // the last line of a file switched a check off for everything above it. Renamed rather than + // scoped, because a finding has no line number to scope it to. The old spelling is not honoured. + it("does not honour the old -next-line spelling", () => { + const m = suppressedChecks( + `// obs-map-disable-next-line error-classification -- stale directive + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + // A2. `indexOf("//")` against the raw text found the marker inside a string literal too, so a + // string that merely quotes the directive granted a suppression nobody wrote and silenced a real + // check. Reading comment ranges off the parsed source closes this: a string literal is one node + // with its own span, never trivia, so a directive inside it is content, not a comment. + it("does not suppress from a directive quoted inside a string literal", () => { + const m = suppressedChecks( + `const msg = "see // obs-map-disable error-classification -- because reasons"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive quoted inside a string with no real comment marker", () => { + const m = suppressedChecks( + `const u = "https://example.com obs-map-disable auth-boundary -- nope"; + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a template literal", () => { + const m = suppressedChecks( + "const msg = `see // obs-map-disable error-classification -- template literal`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + // A standalone `ts.createScanner` has no parser state, so it granted two suppressions nobody wrote: + // it never rescans a template as a continuation after a substitution, and it has no JSX context. + it("does not suppress from a directive after a template substitution", () => { + const m = suppressedChecks( + "const msg = `${name} // obs-map-disable error-classification -- via substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside JSX text", () => { + const m = suppressedChecks( + `export default function Page() { + return

docs at https://example.com obs-map-disable error-classification -- x

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // The case above passes without any JSX handling, because the lexers only find a comment at the + // exact offset they are asked about. These four are the shapes that needed the fix, JSX text that + // BEGINS with a marker, and removing `ts.isJsxText` fails all four and nothing else in the suite. + describe("jsx text is content, not a comment", () => { + const page = (body: string) => `const name = "x"; + export default function Page() { + return ${body}; + }`; + + it("does not suppress from JSX text beginning with a line comment marker", () => { + const m = suppressedChecks( + page(`

// obs-map-disable error-classification -- jsx line

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text beginning with a block comment marker", () => { + const m = suppressedChecks( + page(`

/* obs-map-disable audit-trail -- jsx block */

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from JSX text starting right after an expression container", () => { + const m = suppressedChecks( + page(`

{name}// obs-map-disable request-context -- after expression

`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // A fourth input, exercising the same mechanism at a different tree position: the text is not + // the first child of the outermost element, so the token boundary the lexer is pointed at is a + // different one again. + it("does not suppress from JSX text nested several elements deep", () => { + const m = suppressedChecks( + page(`
// obs-map-disable auth-boundary -- nested jsx
`), + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control for the same code path: a real comment inside a JSX expression container is + // not JSX text and must survive. A filter that dropped it would pass every test above for the + // wrong reason. + it("still reads a directive from a comment in a JSX expression container", () => { + const m = suppressedChecks( + page(`

{/* obs-map-disable audit-trail -- real comment */}

`), + "route.tsx" + ); + expect(m.get("audit-trail")).toBe("real comment"); + }); + }); + + // Extra inputs beyond the brief's two, exercising the same "not a real parse position" mechanism + // differently: a second substitution, and a string literal nested inside a JSX expression + // container, which is a different node kind again from either hole above. + it("does not suppress from a directive after a second template substitution", () => { + const m = suppressedChecks( + "const msg = `${a}${b} // obs-map-disable auth-boundary -- nested substitution`;\n" + + "export async function loader() { return 1; }" + ); + expect(m.size).toBe(0); + }); + + it("does not suppress from a directive inside a string literal nested in a JSX expression container", () => { + const m = suppressedChecks( + `export default function Page() { + return

{"see // obs-map-disable request-context -- nested string"}

; + }`, + "route.tsx" + ); + expect(m.size).toBe(0); + }); + + // Positive control: a genuine directive still works in a .tsx file, and a directive after a + // template with no substitution (already covered above) is not the only shape that must survive. + it("still reads a genuine directive in a .tsx file", () => { + const m = suppressedChecks( + `// obs-map-disable error-classification -- liveness probe + export default function Page() { return

hi

; }`, + "route.tsx" + ); + expect(m.get("error-classification")).toBe("liveness probe"); + }); + + // Regression control: a generic arrow function is only unambiguous when the file is parsed as + // plain TypeScript, not TSX (`` would otherwise start a JSX element). A .ts file must still + // parse sanely and keep reading a genuine trailing comment correctly. + it("still reads a genuine directive beside a generic arrow function in a .ts file", () => { + const m = suppressedChecks( + "const identity = (x: T): T => x; // obs-map-disable auth-boundary -- generic helper\n" + + "export async function loader() { return identity(1); }" + ); + expect(m.get("auth-boundary")).toBe("generic helper"); + }); +}); + +// B6. `// obs-map-disable eror-classification -- typo` used to parse, land in the map, match no +// check and appear nowhere, so the author read the finding as acknowledged. +describe("a suppression naming a check that does not exist", () => { + it("suppresses nothing and is reported as unknown", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(r.byId.size).toBe(0); + expect(r.unknown).toEqual(["eror-classification"]); + }); + + it("does not swallow the real suppressions beside it", () => { + const r = parseSuppressions( + `// obs-map-disable auth-boundry -- typo + // obs-map-disable auth-boundary -- public by design + export async function loader() { return 1; }` + ); + expect(r.byId.get("auth-boundary")).toBe("public by design"); + expect(r.unknown).toEqual(["auth-boundry"]); + }); + + it("reports each unknown id once however many times it appears", () => { + const r = parseSuppressions( + `// obs-map-disable request-contex -- typo + /* obs-map-disable request-contex -- typo again */ + // obs-map-disable audit-trial -- another typo + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual(["request-contex", "audit-trial"]); + }); + + it("is not reported when the directive had no reason, since it was never a suppression", () => { + const r = parseSuppressions( + `// obs-map-disable eror-classification + export async function loader() { return 1; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("is not read out of a string literal any more than a real one is", () => { + const r = parseSuppressions( + `const help = "// obs-map-disable eror-classification -- typo"; + export async function loader() { return help; }` + ); + expect(r.unknown).toEqual([]); + }); + + it("keeps suppressedChecks returning only the ids that name a check", () => { + const m = suppressedChecks( + `// obs-map-disable eror-classification -- typo + export async function loader() { return 1; }` + ); + expect(m.size).toBe(0); + }); +}); diff --git a/internal-packages/observability-map/src/suppression.ts b/internal-packages/observability-map/src/suppression.ts new file mode 100644 index 000000000..ee788a3c5 --- /dev/null +++ b/internal-packages/observability-map/src/suppression.ts @@ -0,0 +1,145 @@ +import ts from "typescript"; +import { CHECKS } from "./checks/index.js"; + +const KNOWN_CHECK_IDS = new Set(CHECKS.map((c) => c.id)); + +/** + * The directive, and the reason that must follow it. The reason runs to the end of the line: `.` does + * not match a newline, so a suppression cannot pick up a reason from the next line. The old + * `obs-map-disable-next-line` spelling and why it was renamed rather than scoped: README, + * "Suppression". + */ +const PATTERN = /obs-map-disable\s+([a-z-]+)\s+--\s+(.+)/; + +/** + * Every leaf token in the parsed source, through `.getChildren()` rather than `ts.forEachChild`, which + * silently skips a bare punctuation or keyword token. A comment can sit directly before one of those + * with nothing else following it, on the last line inside a block. + */ +function leafTokens(node: ts.Node): ts.Node[] { + const children = node.getChildren(); + return children.length === 0 ? [node] : children.flatMap(leafTokens); +} + +/** + * Node kinds whose text the parser has already claimed as content, so nothing inside their span can be + * trivia however it is spelled. `jsx text is content, not a comment` is the four cases that fail + * without `ts.isJsxText` here, and the positive control beside them, `still reads a directive from a + * comment in a JSX expression container`, is what stops the filter being widened until it eats real + * comments. The template and string kinds are covered by + * `does not suppress from a directive inside a template literal` and + * `ignores the directive inside a string literal`. + * + * The mutation corpus cannot cover any of this, because a suppression can only lower a score. See + * INTERNALS.md, "Reading the directive out of the source". + */ +function isClaimedContent(node: ts.Node): boolean { + return ( + ts.isJsxText(node) || + ts.isStringLiteral(node) || + ts.isNoSubstitutionTemplateLiteral(node) || + ts.isTemplateHead(node) || + ts.isTemplateMiddle(node) || + ts.isTemplateTail(node) || + ts.isRegularExpressionLiteral(node) + ); +} + +/** + * Every comment range in the source, read off a real parsed `ts.SourceFile` rather than a standalone + * `ts.createScanner`, and then filtered against the spans above. Both halves are needed, and the + * filter is on the range's start offset rather than on the gap between a token's full start and its + * start: INTERNALS.md, "Reading the directive out of the source". Both lexers are called at every + * token + * boundary, because which one returns a given comment depends on whether it shares a line with the + * token before it. + */ +function commentRanges(source: string, sf: ts.SourceFile): ts.CommentRange[] { + const claimed: ts.TextRange[] = []; + const collectClaimed = (node: ts.Node) => { + if (isClaimedContent(node)) claimed.push({ pos: node.getStart(sf), end: node.end }); + ts.forEachChild(node, collectClaimed); + }; + collectClaimed(sf); + const inClaimedSpan = (pos: number) => claimed.some((s) => pos >= s.pos && pos < s.end); + + const seen = new Set(); + const ranges: ts.CommentRange[] = []; + const add = (found: ts.CommentRange[] | undefined) => { + for (const range of found ?? []) { + if (seen.has(range.pos)) continue; + seen.add(range.pos); + if (inClaimedSpan(range.pos)) continue; + ranges.push(range); + } + }; + for (const token of leafTokens(sf)) { + add(ts.getLeadingCommentRanges(source, token.getFullStart())); + add(ts.getTrailingCommentRanges(source, token.getEnd())); + } + return ranges; +} + +/** One physical line of comment content per range, the `//`, `/*`, `*​/` and a jsdoc `*` prefix + * stripped, so a multi-line block comment still matches the directive one line at a time. */ +function commentLines(source: string, sf: ts.SourceFile): string[] { + const lines: string[] = []; + for (const range of commentRanges(source, sf)) { + const text = source.slice(range.pos, range.end); + if (range.kind === ts.SyntaxKind.SingleLineCommentTrivia) { + lines.push(text.slice(2)); + continue; + } + const body = text.slice(2, text.length - 2); // drop the leading /* and the closing */ + for (const rawLine of body.split("\n")) { + const trimmed = rawLine.trimStart(); + lines.push(trimmed.startsWith("*") ? trimmed.slice(1) : rawLine); + } + } + return lines; +} + +export type Suppressions = { + /** Check id to reason, for ids that name a check in `CHECKS`. */ + byId: Map; + /** Ids that parsed as a directive but name no check, deduplicated. A typo (`eror-classification`) + * used to land in the map, match nothing and appear nowhere, so the author read the finding as + * acknowledged while the tool kept reporting it. */ + unknown: string[]; +}; + +/** + * Every suppression directive in the source, split by whether its id names a real check. A directive + * without a reason, or outside a comment, is ignored either way. + * + * `fileName` picks the parser's script kind, because JSX is only distinguished from a generic type + * argument list (`(x) => x`) when the file is really a `.tsx`. + */ +export function parseSuppressions(source: string, fileName = "check.ts"): Suppressions { + const scriptKind = fileName.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sf = ts.createSourceFile( + fileName, + source, + ts.ScriptTarget.Latest, + /* setParentNodes */ true, + scriptKind + ); + + const byId = new Map(); + const unknown = new Set(); + for (const line of commentLines(source, sf)) { + const match = PATTERN.exec(line); + if (!match) continue; + const [, id, reason] = match; + const trimmedReason = reason?.trim(); + if (!id || !trimmedReason || trimmedReason.length === 0) continue; + if (KNOWN_CHECK_IDS.has(id)) byId.set(id, trimmedReason); + else unknown.add(id); + } + return { byId, unknown: [...unknown] }; +} + +/** The known half of `parseSuppressions`, for callers that only apply suppressions. */ +export function suppressedChecks(source: string, fileName = "check.ts"): Map { + return parseSuppressions(source, fileName).byId; +} diff --git a/internal-packages/observability-map/src/triviality.test.ts b/internal-packages/observability-map/src/triviality.test.ts new file mode 100644 index 000000000..5cdb62009 --- /dev/null +++ b/internal-packages/observability-map/src/triviality.test.ts @@ -0,0 +1,167 @@ +import { isTrivial } from "./triviality.js"; +import { scanFile } from "./scan.js"; + +const ep = (fileName: string, source: string) => scanFile(fileName, source)!; + +describe("isTrivial", () => { + it("treats a redirect-only route as trivial", () => { + const e = ep( + "@.ts", + `import { redirect } from "@remix-run/server-runtime"; + export async function loader() { return redirect("/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat a route that queries the database as trivial", () => { + const e = ep( + "api.v1.things.ts", + `import { prisma } from "~/db.server"; + export async function loader() { + const rows = await prisma.thing.findMany(); + return rows; + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + // The motivating case from the design: four lines, one delegating call, nothing to instrument. + it("treats the impersonation-clearing route as trivial", () => { + const e = ep( + "@.ts", + `import { clearImpersonation } from "~/models/admin.server"; + export async function loader({ request }) { return clearImpersonation(request, "/admin"); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a static-response route as trivial", () => { + const e = ep( + "internal.webhooks.slack.interactivity.ts", + `export function action() { return new Response(null, { status: 200 }); }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a guard and two fixed responses as trivial", () => { + const e = ep( + "api.v1.mock.ts", + `export async function action() { + if (process.env.NODE_ENV === "production") { + return new Response("Not found", { status: 404 }); + } + return new Response(JSON.stringify({ id: "123" }), { status: 200 }); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("treats a params-parse and redirect as trivial", () => { + const e = ep( + "orgs.$organizationSlug.billing.ts", + `import { redirect } from "@remix-run/server-runtime"; + import { OrganizationParamsSchema, v3BillingPath } from "~/utils/pathBuilder"; + export const loader = async ({ params }) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingPath({ slug: organizationSlug })); + };` + ); + expect(isTrivial(e)).toBe(true); + }); +}); + +// statementCount deliberately does not descend into inline callbacks, so a two-statement body can +// still hold a pile of work. calleeNames does descend, which is what catches these. +describe("isTrivial: work hidden from the statement count", () => { + it("does not treat a short body holding a busy callback as trivial", () => { + const e = ep( + "api.v1.remote-build-provider-status.ts", + `export async function loader() { + const result = await fromPromise( + (async () => { + const response = await callProvider(); + const parsed = ProviderStatus.safeParse(await response.json()); + if (!parsed.success) return err("bad-payload"); + return ok(parsed.data); + })() + ); + return result.match(toJson, toError); + }` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a builder-wrapped route with a one-line handler as trivial", () => { + const e = ep( + "api.v1.deployments.current.ts", + `import { json } from "@remix-run/server-runtime"; + import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; + export const loader = createLoaderApiRoute( + { findResource: async (_params, auth) => lookup(auth) }, + async ({ resource }) => { return json(resource); } + );` + ); + expect(isTrivial(e)).toBe(false); + }); + + it("does not treat a body that delegates to a same-file helper as trivial", () => { + const e = ep( + "api.v1.proxy.ts", + `export async function loader({ request }) { return proxy(request); } + async function proxy(request) { + const url = buildUrl(request); + const response = await send(url); + const body = await response.text(); + return new Response(body); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +// Calibrated against apps/webapp/app/routes: three statements is the widest window that holds only +// redirects, fixed responses and single hand-offs. The fourth statement is where routes start +// authenticating and then calling a presenter, which is work worth reporting on. +describe("isTrivial: the statement boundary", () => { + it("treats a three-statement redirect as trivial", () => { + const e = ep( + "schedules._index/route.tsx", + `import { redirect } from "@remix-run/server-runtime"; + import { EnvironmentParamSchema, v3EnvironmentPath } from "~/utils/pathBuilder"; + export async function loader({ params }) { + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const tasksPath = v3EnvironmentPath({ slug: organizationSlug }, { slug: projectParam }, { slug: envParam }); + return redirect(\`\${tasksPath}?types=SCHEDULED\`); + }` + ); + expect(isTrivial(e)).toBe(true); + }); + + it("does not treat an authenticated hand-off to a presenter as trivial", () => { + const e = ep( + "tasks.stream/route.tsx", + `import { TasksStreamPresenter } from "~/presenters/v3/TasksStreamPresenter.server"; + import { requireUserId } from "~/services/session.server"; + import { EnvironmentParamSchema } from "~/utils/pathBuilder"; + export async function loader({ request, params }) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const presenter = new TasksStreamPresenter(); + return presenter.call({ request, projectParam, envParam, organizationSlug, userId }); + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); + +describe("isTrivial: an error path is something to instrument", () => { + it("does not treat a short body with a try/catch as trivial", () => { + const e = ep( + "api.v1.ping.ts", + `export async function loader() { + try { return await ping(); } catch { return null; } + }` + ); + expect(isTrivial(e)).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/src/triviality.ts b/internal-packages/observability-map/src/triviality.ts new file mode 100644 index 000000000..5868558c3 --- /dev/null +++ b/internal-packages/observability-map/src/triviality.ts @@ -0,0 +1,80 @@ +import type { RouteExport } from "./routeExports.js"; +import type { EntryPoint } from "./types.js"; + +/** + * Substrings that say the route touches a service, a datastore or the network. Always matched + * against the callee names, and additionally against `TrivialityView.hintText`. + */ +const SIDE_EFFECT_HINTS = ["prisma", "logger", "fetch", "$transaction", "redis", "engine"]; + +/** Calls a genuinely trivial body makes. Three; a fourth admits + * `_app.orgs.$organizationSlug.settings/route.tsx`, which awaits two service calls. */ +const MAX_CALLS = 3; + +/** Statements a genuinely trivial body has. Three; a fourth admits the routes that authenticate and + * then hand off to a presenter, which have real work behind them. */ +const MAX_STATEMENTS = 3; + +/** + * What the rule reads, so the entry-point-wide answer and a single export's answer are the same rule + * over different bodies rather than two rules that can drift. Both limits, the reluctance and the + * measured `hintText` decision: INTERNALS.md, "Triviality, in detail". + */ +type TrivialityView = { + statementCount: number; + calleeNames: string[]; + hasTryCatch: boolean; + /** Every builder call in scope of this view. A view with one is never trivial. */ + initializerCallees: (string | null)[]; + /** Text to match the side-effect hints against besides the callee names: the whole file for the + * entry-point-wide view, that export's own callee PATHS for a per-export one. Reading the file's + * text into one export's verdict is defeatable by `log-caller-scope-userid`; emptying the term + * instead switches five `auth-boundary` fixtures off. */ + hintText: string; +}; + +/** + * Nothing to instrument: a body of a statement or two that only redirects, returns a fixed response, + * or hands off in a single call. Checks report not-applicable for these rather than failing. + * + * Deliberately reluctant, because a route wrongly called trivial is exempted and never shows up in + * the report again. + */ +function isTrivialView(view: TrivialityView): boolean { + if (view.statementCount > MAX_STATEMENTS) return false; + if (view.calleeNames.length > MAX_CALLS) return false; + if (view.hasTryCatch) return false; + if (view.initializerCallees.some((c) => c !== null)) return false; + + const callees = view.calleeNames.join(" ").toLowerCase(); + const hints = view.hintText.toLowerCase(); + return !SIDE_EFFECT_HINTS.some((h) => callees.includes(h) || hints.includes(h)); +} + +/** The rule over everything the entry point does, both exports and their same-file helpers. */ +export function isTrivial(ep: EntryPoint): boolean { + return isTrivialView({ + statementCount: ep.statementCount, + calleeNames: ep.calleeNames, + hasTryCatch: ep.hasTryCatch, + initializerCallees: [ep.loaderInitializerCallee, ep.actionInitializerCallee], + hintText: ep.source, + }); +} + +/** + * The same rule over ONE export's handlers. Needed because a per-export verdict judged against an + * entry-point-wide rule accuses the wrong half of a file: `auth-boundary` accused + * `auth.github.ts`'s one-line redirect loader of missing a guard because the ACTION is not trivial. + * `checks/index.test.ts` pins both directions (`reports not-applicable for a redirect-stub loader + * beside a guarded action`, `fails an export whose own body does real work unguarded`). + */ +export function isTrivialExport(e: RouteExport): boolean { + return isTrivialView({ + statementCount: e.statementCount, + calleeNames: e.calleeNames, + hasTryCatch: e.hasTryCatch, + initializerCallees: [e.initializerCallee], + hintText: e.calleeTexts.join(" "), + }); +} diff --git a/internal-packages/observability-map/src/types.ts b/internal-packages/observability-map/src/types.ts new file mode 100644 index 000000000..bee1aae1c --- /dev/null +++ b/internal-packages/observability-map/src/types.ts @@ -0,0 +1,132 @@ +export type CheckStatus = "pass" | "fail" | "not-applicable"; + +export type CheckResult = { + id: string; + status: CheckStatus; + detail?: string; +}; + +/** + * One catch clause in a loader/action body, or in a same-file helper the body calls. Per clause + * rather than per entry point, so a narrow parse guard sitting beside a broad handler catch stays + * legible. Every field's exact rule and its measured reasoning: INTERNALS.md, "Catch evidence, per + * clause". + */ +export type CatchEvidence = { + /** Throwing is the clause's only way out: a throw on the clause's guaranteed path (see + * `catchClauseEvidence`) and no live `return` anywhere. */ + rethrows: boolean; + /** A throw is reached on that path, whether or not it is the only way out. Kept apart from + * `rethrows` so a verdict can say what is true of a clause that both throws and returns. */ + throws: boolean; + /** The clause picks what to do from what it caught, on that same guaranteed path. */ + branches: boolean; + /** The guarded region parses something. Includes `new URL`/`URLSearchParams`/`RegExp`, which the + * call-callee scan cannot see because a `new` expression is not a call. */ + guardsParse: boolean; + /** The guarded region does anything that could raise. Any call counts, including one that cannot + * throw, so `try { String(0); }` is true here: `dead-classifying-try-with-call`. */ + guardCanRaise: boolean; + /** + * The containment twin of `guardCanRaise`: false only for the provably inert `try { 0; }`, so + * can-raise implies may-raise. Read by the refused-callback arm of `error-classification`, where + * a `canRaise` miss would otherwise accuse a route that owns a real classifying catch + * (`does not accuse a route that owns a catch of owning none`). + */ + guardMayRaise: boolean; + /** Everything the guarded region waits for is one of those parses. Synchronous work is not + * counted: preparing a parse's input is synchronous, and the swallows this separates out wait on + * a service. */ + awaitsOnlyParse: boolean; + /** Statements in the guarded try block, counted as `statementCount` counts them. */ + tryStatementCount: number; +}; + +/** A logging call made from a loader/action body, or from a same-file helper the body calls. */ +export type LogCall = { + /** Full callee path, e.g. `logger.error`. */ + callee: string; + /** Property names on the first object-literal argument, e.g. `["environmentId", "error"]`. */ + fields: string[]; + /** Whether the call sits inside a catch clause, i.e. on the failure path. */ + inCatch: boolean; +}; + +/** + * Body-scoped evidence for one route module. Which fields are per export and why, and what "the + * body" means: INTERNALS.md, "How the scanner reads a route". + */ +export type EntryPoint = { + fileName: string; + source: string; + hasLoader: boolean; + hasAction: boolean; + /** Callee name when `loader`/`action` is assigned from a call, e.g. a route builder. */ + loaderInitializerCallee: string | null; + actionInitializerCallee: string | null; + /** Top-level keys of the object literals passed to that call. Empty means "nothing declared + * here" rather than "no builder". */ + loaderBuilderOptions: string[]; + actionBuilderOptions: string[]; + /** + * The route declares a loader or an action and the scan resolved neither a handler function nor a + * builder call for any of them, so nothing about the request handling is in this file. A route + * that delegates ONE export and writes the other in the file is not delegating by this + * definition, and is judged on the half that is visible. + */ + delegating: boolean; + /** Whether THIS export's handler assigns the caller's own id to an object-literal property. + * Property assignments only, so a value read into a local first is not seen. */ + loaderScopesByCaller: boolean; + actionScopesByCaller: boolean; + /** + * Callees whose answer THIS export's handlers demonstrably looked at: bound to a local, and that + * local read by some condition. No entry-point-wide version exists on purpose, so no check can + * let a loader's reading of `getUser` speak for the action beside it. + * + * Deliberately coarse: it does not check that the test guards anything, so a route writing + * `if (!user) { logger.warn("anonymous"); }` and carrying on is credited. + */ + loaderCheckedCallees: string[]; + actionCheckedCallees: string[]; + /** Named and default imports, file-wide. */ + importedNames: string[]; + /** + * Names of functions called inside the loader/action bodies, or in a same-file helper they call. + * Entry-point-wide, and read only by the questions that are themselves entry-point-wide. A + * question about ONE export's exposure must read the pair below: `auth-boundary` read this and + * credited a file whose loader called a guard for an action that called none. + */ + calleeNames: string[]; + /** The same names attributed to the export whose handlers made the call. A handler serving both + * exports contributes to both. */ + loaderCalleeNames: string[]; + actionCalleeNames: string[]; + /** The same calls as whole dotted paths (`prisma.organization.findFirst`), which the per-export + * triviality rule needs to know a short body reaches the datastore. */ + loaderCalleeTexts: string[]; + actionCalleeTexts: string[]; + /** Whether a `try` appears in those bodies. A `try`/`finally` sets this while `catches` stays + * empty, so ask `catches.length` whether anything is caught. */ + hasTryCatch: boolean; + /** The same fact for one export's handlers alone. Read by the per-export triviality rule. */ + loaderHasTryCatch: boolean; + actionHasTryCatch: boolean; + /** One entry per catch clause in those bodies, in source order. */ + catches: CatchEvidence[]; + /** + * Catch clauses the scan refused to attribute to the route because they sit inside a per-item + * iteration callback. Never join `catches`, never speak for `tryStatementCount`, never reach a + * pass. Kept WITH their evidence so `error-classification` can judge what a refused catch does + * rather than where it sits. + */ + callbackCatches: CatchEvidence[]; + /** Calls to a `logger.*` or `log.*` callee in those bodies, in source order. */ + logCalls: LogCall[]; + /** Statement count across loader/action bodies, used by the triviality rule. */ + statementCount: number; + /** The same count for one export's handlers alone. A handler serving both exports is counted in + * each, so these do not sum to `statementCount`. */ + loaderStatementCount: number; + actionStatementCount: number; +}; diff --git a/internal-packages/observability-map/src/webappSymbols.test.ts b/internal-packages/observability-map/src/webappSymbols.test.ts new file mode 100644 index 000000000..7cc7ed72d --- /dev/null +++ b/internal-packages/observability-map/src/webappSymbols.test.ts @@ -0,0 +1,179 @@ +import ts from "typescript"; +import { readdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { AUDIT_SYMBOLS } from "./checks/auditTrail.js"; +import { GUARDS, SOFT_GUARDS } from "./checks/authBoundary.js"; +import { isScannableFile } from "./scan.js"; +import { + ANTICIPATED_SEGMENTS, + normalizeSegment, + SENSITIVE_SEGMENTS, + SENSITIVE_SYMBOLS, +} from "./sensitivity.js"; + +/** + * Every name and every path segment the tool matches on must exist in the codebase it is pointed at. + * Half of `SENSITIVE_SYMBOLS` named nothing before this test, and the guard list has the same failure + * mode with a worse consequence: a guard name resolving nowhere makes a route that can never pass. + * + * Checked: every guard name and sensitive symbol is DECLARED under one of `ROOTS`, as a function, + * class, interface, type, enum, variable or member name; members count because several guards are + * reached through an object and `calleeName` records the property. And every sensitive path segment + * appears as a segment of a real route file name. + * + * Not checked, each a place a wrong entry can hide: that the declaration found is the one meant, that + * a guard actually guards, and anything outside `ROOTS`. + */ + +const REPO = resolve(__dirname, "../../.."); + +/** + * Where a guard or a sensitive symbol may be declared. The webapp first, then the two packages it + * authenticates through: `packages/plugins` declares the RBAC controller interface the dashboard + * and PAT builders call, and `internal-packages/rbac` declares its fallback and the user-actor + * token verifier that three routes import directly. + */ +const ROOTS = [ + resolve(REPO, "apps/webapp/app"), + resolve(REPO, "packages/plugins/src"), + resolve(REPO, "internal-packages/rbac/src"), +]; + +/** + * Guard names declared by a dependency rather than by us, and therefore deliberately unchecked. Both + * are remix-auth's, and resolving them meant reading a path inside `apps/webapp/node_modules`, which + * an install-layout change turns into a confusing environmental failure. Listing them is a smaller + * claim honestly made: the test still fails on a name that is neither first-party nor listed. + */ +const EXTERNAL_GUARDS = new Set(["authenticate", "isAuthenticated"]); + +/** Two is the number of remix-auth methods on the guard list. A third entry means someone widened + * the unchecked set, which is the thing this bound exists to make visible in review. */ +const MAX_EXTERNAL_GUARDS = 2; + +const ROUTES = resolve(REPO, "apps/webapp/app/routes"); + +function walkFiles(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walkFiles(path, out); + else if (isScannableFile(entry.name)) out.push(path); + } + return out; +} + +function declaredNames(): Set { + const names = new Set(); + const addBinding = (name: ts.BindingName) => { + if (ts.isIdentifier(name)) { + names.add(name.text); + return; + } + for (const element of name.elements) { + if (!ts.isOmittedExpression(element)) addBinding(element.name); + } + }; + + const files = ROOTS.flatMap((root) => walkFiles(root)); + for (const file of files) { + const sf = ts.createSourceFile(file, readFileSync(file, "utf8"), ts.ScriptTarget.Latest, false); + const visit = (node: ts.Node) => { + if (ts.isVariableDeclaration(node) || ts.isParameter(node)) addBinding(node.name); + else if ( + (ts.isFunctionDeclaration(node) || + ts.isClassDeclaration(node) || + ts.isInterfaceDeclaration(node) || + ts.isTypeAliasDeclaration(node) || + ts.isEnumDeclaration(node)) && + node.name + ) { + names.add(node.name.text); + } else if ( + (ts.isMethodDeclaration(node) || + ts.isMethodSignature(node) || + ts.isPropertyDeclaration(node) || + ts.isPropertySignature(node) || + ts.isPropertyAssignment(node) || + ts.isShorthandPropertyAssignment(node)) && + node.name && + ts.isIdentifier(node.name) + ) { + names.add(node.name.text); + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return names; +} + +/** Every dot-separated piece of every route name, flat file or directory, e.g. `billing-limits`. */ +function routeSegments(): Set { + const segments = new Set(); + for (const entry of readdirSync(ROUTES, { withFileTypes: true })) { + for (const part of entry.name.replace(/\.tsx?$/, "").split(".")) { + // `sensitivity.ts`'s own normalizer. This validates the vocabulary that file matches on, so + // a segment has to be trimmed here exactly as it is trimmed there; the local `/_+$/` was + // also the regex `normalizeSegment`'s own comment says not to use. + segments.add(normalizeSegment(part)); + } + } + return segments; +} + +describe("the names the tool matches on exist in the webapp", () => { + const declared = declaredNames(); + const segments = routeSegments(); + + it("found a codebase to check against", () => { + expect(declared.size).toBeGreaterThan(5000); + expect(segments.size).toBeGreaterThan(100); + }); + + it("every sensitive symbol is declared somewhere", () => { + expect(SENSITIVE_SYMBOLS.filter((s) => !declared.has(s))).toEqual([]); + }); + + // The list this test did not cover, and it had rotted completely: all three of `auditLog`, + // `recordAudit` and `writeAuditEvent` were exported nowhere, so `audit-trail`'s pass branch could + // not fire and the report said "No audit helper exists in the webapp" while + // `models/admin.server.ts` was writing `impersonationAuditLog` rows on two paths. + it("every audit symbol is declared somewhere", () => { + expect(AUDIT_SYMBOLS.filter((s) => !declared.has(s))).toEqual([]); + }); + + it("every auth guard is declared somewhere, or is a listed dependency method", () => { + const names = [...GUARDS, ...SOFT_GUARDS]; + expect(names.filter((g) => !declared.has(g) && !EXTERNAL_GUARDS.has(g))).toEqual([]); + }); + + // The escape hatch is only worth having while it stays small. + it("keeps the unchecked guard names to the two remix-auth methods", () => { + expect(EXTERNAL_GUARDS.size).toBeLessThanOrEqual(MAX_EXTERNAL_GUARDS); + expect([...EXTERNAL_GUARDS].filter((g) => !GUARDS.has(g))).toEqual([]); + }); + + it("every sensitive path segment names a real route segment", () => { + const live = SENSITIVE_SEGMENTS.filter((s) => !ANTICIPATED_SEGMENTS.includes(s)); + expect(live.filter((s) => !segments.has(s))).toEqual([]); + }); + + // The escape hatch is only worth having while it is small and honest about itself. + it("every anticipated segment really does name nothing yet", () => { + expect(ANTICIPATED_SEGMENTS.filter((s) => segments.has(s))).toEqual([]); + }); + + // The checker has to be able to fail. These run the same predicates over the names the last round + // shipped, which is what the test exists to have caught. + it("would reject the symbols that named nothing", () => { + for (const dead of ["setImpersonation", "createJWT", "signJWT", "updateEnvVars"]) { + expect(declared.has(dead)).toBe(false); + } + }); + + it("would reject a guard name and a path segment that name nothing", () => { + expect(declared.has("requireNothingAtAll")).toBe(false); + expect(EXTERNAL_GUARDS.has("requireNothingAtAll")).toBe(false); + expect(segments.has("no-such-route-segment")).toBe(false); + }); +}); diff --git a/internal-packages/observability-map/tsconfig.build.json b/internal-packages/observability-map/tsconfig.build.json new file mode 100644 index 000000000..6e0d21f36 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts", "src/mutations.ts"], + "compilerOptions": { + "noEmit": false, + // A build that does not compile must not leave a dist behind: three type errors shipped in the + // last wave while `build` still emitted, so the failure was only visible in the exit code. + "noEmitOnError": true, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "types": ["node"] + } +} diff --git a/internal-packages/observability-map/tsconfig.json b/internal-packages/observability-map/tsconfig.json new file mode 100644 index 000000000..ea5663ae8 --- /dev/null +++ b/internal-packages/observability-map/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2019", + // ES2020 rather than the ES2019 most sibling packages use, because the tests call + // String.prototype.matchAll. It already resolved: @types/node carries a + // `/// `, so the program had es2020 whatever this line said. Stating + // it here stops the requirement resting on a transitive reference from a types package. + "lib": ["ES2020"], + "module": "ESNext", + "moduleResolution": "Bundler", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "types": ["vitest/globals", "node"], + "customConditions": ["@triggerdotdev/source"] + }, + "exclude": ["node_modules", "dist"] +} diff --git a/internal-packages/observability-map/turbo.json b/internal-packages/observability-map/turbo.json new file mode 100644 index 000000000..42fa06565 --- /dev/null +++ b/internal-packages/observability-map/turbo.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://turborepo.org/schema.json", + "extends": ["//"], + "pipeline": { + // Uncacheable on purpose. This suite's real inputs live outside the package: it scans + // `apps/webapp/app`, `packages/plugins/src`, `internal-packages/rbac/src` and four files under + // `.github/workflows`. The root `test` task keys the cache on this package's own files, so a + // cached pass replayed after a route change broke the scan, which is a guard that stops + // guarding while still reading green. + // + // `inputs` was tried and rejected rather than assumed unworkable. Turbo 1.x does accept `..` + // in an input glob, and `../../apps/webapp/app/**` did bust the cache on a route change. What + // it also does is replace the default file set rather than add to it, so the same config + // silently dropped this package's own `vitest.config.ts` from the hash: editing it replayed a + // cached pass. The `$TURBO_DEFAULT$` token that adds rather than replaces is turbo 2.x only + // and matches nothing on the 1.10.3 here. Trading a stale-on-routes hole for a + // stale-on-own-config hole is not a fix, and an inputs list mirroring what the tests read is + // one more thing that drifts out of sync without saying so. + // + // Cost is about 23s per run, and no CI job pays it: the dedicated workflow calls vitest + // without turbo, and `unit-tests-internal.yml` runs cold. + // + // `it("keeps its test task out of the turbo cache")` in `src/integration.test.ts` fails if + // this is removed. + "test": { + "dependsOn": ["^build"], + "outputs": [], + "cache": false + } + } +} diff --git a/internal-packages/observability-map/vitest.config.ts b/internal-packages/observability-map/vitest.config.ts new file mode 100644 index 000000000..c1680ce67 --- /dev/null +++ b/internal-packages/observability-map/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { include: ["**/*.test.ts"], globals: true, isolate: true, testTimeout: 10_000 }, +}); diff --git a/package.json b/package.json index 2708b06be..9fad94dd9 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "clean": "turbo run clean", "clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", "typecheck": "turbo run typecheck", + "map": "pnpm --filter @internal/observability-map run map", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", "test:e2e:dev": "turbo run test:e2e:dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45163c161..acbefa9c7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1139,6 +1139,25 @@ importers: specifier: 6.0.1 version: 6.0.1 + internal-packages/observability-map: + dependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 + devDependencies: + '@types/node': + specifier: 24.13.3 + version: 24.13.3 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + tsx: + specifier: ^4.19.2 + version: 4.22.4 + vitest: + specifier: 4.1.7 + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0)) + internal-packages/otlp-importer: dependencies: long: