Merge remote-tracking branch 'origin/main' into pr-628-fix-621-mcp-go-tests
# Conflicts: # CHANGELOG.md # CONTRIBUTORS.md
This commit is contained in:
Trevin Chow
@@ -11,7 +11,7 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.8.0",
|
||||
"version": "3.8.1",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.8.0",
|
||||
"version": "3.8.1",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
|
||||
+62
-12
@@ -1,18 +1,68 @@
|
||||
# Exclude binary assets and dev/test artifacts from ClawHub bundle
|
||||
assets/
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
release-notes.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
SKILL-original.md
|
||||
# ClawHub/Hermes packaging exclusions for repository-root scans.
|
||||
# Mirrors .skillignore so non-runtime docs/dev artifacts stay out of the
|
||||
# public bundle and install-time skill security scan.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
skills/last30days/assets/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
work/
|
||||
print/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
mcp/
|
||||
gemini-extension.json
|
||||
greptile.json
|
||||
pyproject.toml
|
||||
|
||||
# Non-runtime docs, plans, release notes, fixtures, and tests
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
media/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
CONCEPTS.md
|
||||
CONFIGURATION.md
|
||||
CONTRIBUTORS.md
|
||||
HERMES_SETUP.md
|
||||
release-notes.md
|
||||
SKILL-original.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
|
||||
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
|
||||
skills/last30days/scripts/build-skill.sh
|
||||
skills/last30days/scripts/compare.sh
|
||||
skills/last30days/scripts/evaluate_search_quality.py
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
skills/last30days/scripts/setup-pass.sh
|
||||
skills/last30days/scripts/test_device_auth.py
|
||||
skills/last30days/scripts/test-v1-vs-v2.sh
|
||||
skills/last30days/scripts/verify_v3.py
|
||||
|
||||
# Keep visible: optional runtime watchlist/store/briefing feature scripts
|
||||
# (`watchlist.py`, `store.py`, and `briefing.py`).
|
||||
|
||||
@@ -25,7 +25,6 @@ assets/ export-ignore
|
||||
# Historical + repo-only manifests
|
||||
SPEC.md export-ignore
|
||||
TASKS.md export-ignore
|
||||
test-run.log export-ignore
|
||||
CONTRIBUTORS.md export-ignore
|
||||
HERMES_SETUP.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
|
||||
@@ -13,3 +13,10 @@ updates:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: gomod
|
||||
directory: /mcp
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
@@ -5,10 +5,7 @@ on:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
attestations: write
|
||||
permissions: {}
|
||||
|
||||
|
||||
jobs:
|
||||
@@ -18,12 +15,12 @@ jobs:
|
||||
build-skill:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -34,12 +31,12 @@ jobs:
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Attest .skill artifact provenance
|
||||
uses: actions/attest-build-provenance@v4
|
||||
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
|
||||
with:
|
||||
subject-path: dist/last30days.skill
|
||||
|
||||
- name: Upload skill artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: last30days-skill
|
||||
path: dist/last30days.skill
|
||||
@@ -49,6 +46,13 @@ jobs:
|
||||
# zip layout; we only supply the pre-built binary via --skip-build.
|
||||
build-mcpb:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
env:
|
||||
MCPB_OUTPUT: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
MCPB_PLATFORM: ${{ matrix.platform }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -64,14 +68,15 @@ jobs:
|
||||
platform: linux/amd64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: stable
|
||||
go-version-file: mcp/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Install printing-press
|
||||
# Pin to a known-good PP release so the bundle command's behavior
|
||||
@@ -92,10 +97,11 @@ jobs:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: "0"
|
||||
RELEASE_VERSION: ${{ github.ref_name }}
|
||||
run: |
|
||||
mkdir -p mcp/build
|
||||
go -C mcp build \
|
||||
-ldflags "-X main.Version=${{ github.ref_name }}" \
|
||||
-ldflags "-X main.Version=${RELEASE_VERSION}" \
|
||||
-o build/last30days-pp-mcp \
|
||||
./cmd/last30days-pp-mcp
|
||||
|
||||
@@ -108,24 +114,30 @@ jobs:
|
||||
printing-press bundle mcp \
|
||||
--skip-build \
|
||||
--binary mcp/build/last30days-pp-mcp \
|
||||
--platform ${{ matrix.platform }} \
|
||||
--output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
--platform "${MCPB_PLATFORM}" \
|
||||
--output "${MCPB_OUTPUT}"
|
||||
|
||||
- name: Attest .mcpb artifact provenance
|
||||
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
|
||||
with:
|
||||
subject-path: ${{ env.MCPB_OUTPUT }}
|
||||
|
||||
- name: Upload .mcpb artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
path: ${{ env.MCPB_OUTPUT }}
|
||||
|
||||
# Gather every platform artifact and attach to one GitHub release.
|
||||
release:
|
||||
needs: [build-skill, build-mcpb]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
@@ -133,8 +145,9 @@ jobs:
|
||||
- name: Create GitHub release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
gh release create "${GITHUB_REF_NAME}" \
|
||||
gh release create "${RELEASE_TAG}" \
|
||||
dist/last30days.skill \
|
||||
dist/last30days-pp-mcp-*.mcpb \
|
||||
--generate-notes \
|
||||
|
||||
@@ -41,12 +41,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run OpenSSF Scorecard
|
||||
uses: ossf/scorecard-action@v2.4.3
|
||||
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
|
||||
with:
|
||||
results_file: scorecard.sarif
|
||||
results_format: sarif
|
||||
@@ -57,13 +57,13 @@ jobs:
|
||||
|
||||
# Retain the raw SARIF as a build artifact for offline inspection.
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: scorecard-sarif
|
||||
path: scorecard.sarif
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload SARIF to code-scanning
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
with:
|
||||
sarif_file: scorecard.sarif
|
||||
|
||||
@@ -17,19 +17,15 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
|
||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
||||
# visible in CI logs even before the project has a clean blocking baseline.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Block known vulnerabilities in the locked Python dependency graph.
|
||||
- name: Run uv audit against locked dependencies
|
||||
continue-on-error: true
|
||||
run: uv audit --locked
|
||||
|
||||
secret-scan:
|
||||
@@ -39,23 +35,16 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
||||
# The TruffleHog action automatically scans the PR range for pull_request
|
||||
# events and the pushed commit range for push events.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
||||
# The action derives the commit range from the GitHub event and fails on
|
||||
# verified secrets. Keep output limited to verified findings to avoid noisy
|
||||
# unverified annotations.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2
|
||||
continue-on-error: true
|
||||
uses: trufflesecurity/trufflehog@30d5bb91af1a771378349dbbb0c82129392acf70 # v3.95.6
|
||||
with:
|
||||
path: ./
|
||||
version: v3.95.2
|
||||
extra_args: --only-verified
|
||||
version: 3.95.5
|
||||
extra_args: --results=verified
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
@@ -34,12 +34,12 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.25.5"
|
||||
cache-dependency-path: mcp/go.sum
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
security-events: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# Private benchmark / evaluation artifacts — never push to upstream
|
||||
docs/comparison-results/
|
||||
test-run.log
|
||||
scripts/evaluate-synthesis.py
|
||||
scripts/generate-synthesis-inputs.py
|
||||
fixtures/polymarket_sample.json
|
||||
@@ -7,6 +8,7 @@ docs/v2.1-tweets.md
|
||||
docs/30-day-anniversary-thread.md
|
||||
docs/30-day-anniversary-tweets.md
|
||||
variants/open/references/research.md
|
||||
docs/investigations/
|
||||
|
||||
# OS / tool files
|
||||
.DS_Store
|
||||
@@ -42,6 +44,7 @@ htmlcov/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
docs/brainstorms/
|
||||
.context/
|
||||
|
||||
/work
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# Hermes install-time scanner/package exclusions for repository-root scans.
|
||||
# Keep the public bundle focused on the runtime skill under skills/last30days/.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
skills/last30days/assets/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
work/
|
||||
print/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
mcp/
|
||||
gemini-extension.json
|
||||
greptile.json
|
||||
pyproject.toml
|
||||
|
||||
# Non-runtime docs, plans, release notes, fixtures, and tests
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
media/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
CONCEPTS.md
|
||||
CONFIGURATION.md
|
||||
CONTRIBUTORS.md
|
||||
HERMES_SETUP.md
|
||||
release-notes.md
|
||||
SKILL-original.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
|
||||
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
|
||||
skills/last30days/scripts/build-skill.sh
|
||||
skills/last30days/scripts/compare.sh
|
||||
skills/last30days/scripts/evaluate_search_quality.py
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
skills/last30days/scripts/setup-pass.sh
|
||||
skills/last30days/scripts/test_device_auth.py
|
||||
skills/last30days/scripts/test-v1-vs-v2.sh
|
||||
skills/last30days/scripts/verify_v3.py
|
||||
|
||||
# Keep visible: optional runtime watchlist/store/briefing feature scripts
|
||||
# (`watchlist.py`, `store.py`, and `briefing.py`).
|
||||
@@ -42,6 +42,7 @@ Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
- Every `lib/*.py` call to `log.source_log(...)` must pass `tty_only=False`. The default is `True`, which silently drops every line when stderr isn't a TTY (Claude Code, Codex, CI, captured output) — turning source observability into invisible failure. Enforced by `tests/test_source_log_visibility.py`.
|
||||
- **CLI-gated optional sources** (Digg via `digg-pp-cli`, YouTube via `yt-dlp`) activate only when `shutil.which` resolves the binary on the **agent subprocess PATH** — not merely when the file exists on disk. First-run setup installs Digg through `@mvanhorn/printing-press-library` (default `$HOME/.local/bin`); Hermes/OpenClaw gateways often need that directory on PATH. Setup must distinguish PATH-visible installs from off-PATH binaries and must not claim "now active" unless the engine gate would pass. See `docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md`.
|
||||
- **First-run onboarding is consent-driven, model-led, and host-split.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth) — it cannot prompt, so consent lives in `SKILL.md` Step 0. Step 0 has TWO branches: a **Claude Code Modal Flow** (the restored v3.0.0 `AskUserQuestion`-driven NUX — welcome, Auto/Manual/Skip, cookie consent, ScrapeCreators offer, `INCLUDE_SOURCES` opt-in, first-topic picker) for hosts with modals, and a **Non-Modal Prose Flow** for hosts without (OpenClaw, Codex, Cursor, Gemini CLI). Both ask before reading cookies, surface the macOS Full Disk Access fix on permission-denied, and offer the ScrapeCreators GitHub signup (10,000 free calls) on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do NOT collapse the modal flow back into a bare silent `setup` call or flatten it to prose-only — the guided modals are the feature (they eroded once and were restored). The onboarding contract is locked by `tests/test_onboarding_contract.py`. Threads/Pinterest are intentionally not surfaced in onboarding (power-user `INCLUDE_SOURCES` only).
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
|
||||
@@ -11,6 +11,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
- MCP Go tests (`mcp/`) now run in CI on every push/PR alongside the Python test suite, so MCP server regressions are caught before merge ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
|
||||
## [3.8.1] - 2026-06-22
|
||||
|
||||
### Added
|
||||
- **Restored the v3.0.0 first-run NUX wizard (Claude Code Modal Flow).** Step 0 now restores the original guided, `AskUserQuestion`-driven onboarding that eroded over time: a welcome message, an Auto/Manual/Skip setup modal, a cookie-consent modal, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. It is gated to hosts with modals; hosts without (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent **Non-Modal Prose Flow**. Digg is threaded into the install messaging alongside yt-dlp everywhere it appears, the ScrapeCreators credit count is `10,000 free calls`, and the flow is locked against re-erosion by `tests/test_onboarding_contract.py`. Builds on the consent-driven foundation from #659/#660. Original wizard captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
- **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659).
|
||||
|
||||
### Fixed
|
||||
- **First-run setup no longer runs silently.** The prior Step 0 told the model to run `setup` and "follow the wizard's prompts end-to-end", but the wizard has no prompts — so onboarding extracted cookies, installed tools, and wrote `SETUP_COMPLETE` with zero interaction and never offered the ScrapeCreators signup. Reproduced 2026-06-22 (Fredy Montero, fresh macOS).
|
||||
|
||||
## [3.8.0] - 2026-06-21
|
||||
|
||||
### Added
|
||||
@@ -59,6 +68,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
- The X FROM lane (the subject's own timeline) now pulls up to 8 posts per handle (was 3); the about/related lanes stay modest.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Secrets `.env` and its parent config directory are now auto-tightened to `0o600`/`0o700` after creation, and `check-config.sh`'s `check_perms` now auto-fixes loose permissions with `chmod 600` instead of warning only ([#573](https://github.com/mvanhorn/last30days-skill/issues/573))
|
||||
|
||||
## [3.5.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
+32
-13
@@ -44,21 +44,39 @@ The engine's `.env` reader doesn't expand `$HOME` — only the tilde, via `Path(
|
||||
- `--save-dir <path>` - one-off output location. **Flag wins over env var.** If neither flag nor env var is set, the engine does not write a file (DB persistence is independent — see `LAST30DAYS_STORE` below).
|
||||
- `--output <file>` - write the rendered output to an exact file path, using the format selected by `--emit`.
|
||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||
- `--no-browser-cookies` - hard-disable browser-cookie extraction for this run, even when `FROM_BROWSER` is configured. MCP and folder-mode hosts use this for safe defaults.
|
||||
|
||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||
|
||||
---
|
||||
|
||||
## First-run onboarding
|
||||
|
||||
On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It takes one of two forms depending on the host:
|
||||
|
||||
- **Claude Code Modal Flow** - the restored v3.0.0 guided NUX, used on hosts with `AskUserQuestion` (Claude Code). A welcome message, then modals for Auto/Manual/Skip setup, cookie consent, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker.
|
||||
- **Non-Modal Prose Flow** - the same work done conversationally on hosts without modals (OpenClaw, Codex, Cursor, Gemini CLI, raw CLI).
|
||||
|
||||
Both share the same consent points:
|
||||
|
||||
1. **Browser cookies** - the model asks before reading anything. On yes it runs `setup --allow-browser-cookies`, which extracts Firefox/Safari cookies (never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser is explicitly configured) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup without `--allow-browser-cookies` (or with `FROM_BROWSER=off`), which skips all cookie reads and still installs the tools.
|
||||
2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry.
|
||||
3. **ScrapeCreators GitHub signup** - offered on every first run (10,000 free calls). On consent it runs `setup --github`, which opens a browser for GitHub device-auth (or registers instantly via the `gh` CLI when installed) and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, X, YouTube comments, and the SC Reddit/YouTube backups activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. (Threads and Pinterest are not surfaced in onboarding but remain available via `INCLUDE_SOURCES`.)
|
||||
|
||||
Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation and both host flows are specified in `skills/last30days/SKILL.md` Step 0. The original v3.0.0 wizard is captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
|
||||
---
|
||||
|
||||
## API keys (`.env`)
|
||||
|
||||
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
||||
The skill reads keys from a `.env` file. Two locations are supported:
|
||||
|
||||
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
||||
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
||||
1. **`~/.config/last30days/.env`** at the user level (global default) - loaded by default.
|
||||
2. **`.claude/last30days.env`** in the current project directory (project-scoped) - loaded only when trusted by setting `LAST30DAYS_TRUST_PROJECT_CONFIG=1` in the process environment or global config.
|
||||
|
||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||
|
||||
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
||||
The project-scoped file is useful for **intentional per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), then opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or `~/.config/last30days/.env`. Folder-mode hosts such as Codex desktop do not trust hidden project config by default, and discovery stops at the git root so unrelated parent folders cannot silently influence runs.
|
||||
|
||||
**Source-by-source** - what each key unlocks:
|
||||
|
||||
@@ -112,10 +130,11 @@ CT0=<your-ct0-token>
|
||||
# OR Xquik key-based X search
|
||||
# XQUIK_API_KEY=<your-xquik-key>
|
||||
# OR cookie-jar (free; logs in via your browser session).
|
||||
# Unset = Firefox + Safari (silent). FROM_BROWSER=auto also tries the Chromium
|
||||
# family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it only prompts
|
||||
# for macOS Keychain access on the browser that actually holds your X cookies.
|
||||
# Or name a single browser, e.g. brave/edge. On Windows only Firefox is supported.
|
||||
# Unset = no browser-cookie reads. FROM_BROWSER=auto tries Firefox/Safari and
|
||||
# the Chromium family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it
|
||||
# only prompts for macOS Keychain access on the browser that actually holds your
|
||||
# X cookies. Or name a single browser, e.g. brave/edge. On Windows only Firefox
|
||||
# is supported.
|
||||
# FROM_BROWSER=firefox
|
||||
|
||||
# Bluesky
|
||||
@@ -125,7 +144,7 @@ BSKY_APP_PASSWORD=<your-app-password>
|
||||
|
||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a safe preflight report for source availability, config source, browser-cookie plan, external command availability, write destinations, and ignored untrusted project config without reading browser cookies or running live provider probes.
|
||||
|
||||
### Perplexity source modes
|
||||
|
||||
@@ -219,7 +238,7 @@ Accepts the same comma-separated names and aliases as `--search` (`web` → grou
|
||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||
|
||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
||||
2. **OpenAI** - `OPENAI_API_KEY` only. Codex ChatGPT auth at `~/.codex/auth.json` is intentionally not used as an OpenAI provider credential.
|
||||
3. **xAI** - `XAI_API_KEY`
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (Sonar fallback for the Perplexity source / `--deep-research`; also usable as a reasoning provider)
|
||||
5. **Local / deterministic** - always available, lowest quality
|
||||
@@ -315,9 +334,9 @@ The schedule field stored on each topic is metadata - the actual cron / Task Sch
|
||||
|
||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||
|
||||
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
||||
### 1. Trusted per-client `.claude/last30days.env`
|
||||
|
||||
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
||||
When each client has its own working directory, drop a `.claude/last30days.env` into the client folder and opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or global `~/.config/last30days/.env`. The skill loads the project file only after that trust signal. Typical contents:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||
@@ -326,7 +345,7 @@ INCLUDE_SOURCES=tiktok,instagram
|
||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||
```
|
||||
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
|
||||
### 2. Per-client save dir + suffix wrapper
|
||||
|
||||
|
||||
+1
-2
@@ -54,8 +54,7 @@ v3 has durable watchlist with multi-source storage and extended time windows.
|
||||
|
||||
## Past Contributors
|
||||
|
||||
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)); MCP Go tests in CI ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)); .env permission auto-fix ([#573](https://github.com/mvanhorn/last30days-skill/pull/599)); MCP Go tests in CI ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
- [@JosephOIbrahim](https://github.com/JosephOIbrahim) - Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
|
||||
- [@levineam](https://github.com/levineam) - Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
|
||||
- [@jonthebeef](https://github.com/jonthebeef) - Early testing and feedback
|
||||
|
||||
|
||||
@@ -286,7 +286,7 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
| X / Twitter | Log into x.com in any browser, or set `XQUIK_API_KEY` / `XAI_API_KEY` | Browser cookies are free; keys are provider-specific |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls, then PAYG |
|
||||
| Perplexity Sonar / Search API / Deep Research | Perplexity key, or OpenRouter key as Sonar fallback | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
# last30days — X search + funny + laziness: consolidated bug inventory
|
||||
|
||||
**Date:** 2026-06-17
|
||||
**Engine version under test:** 3.4.0 (current `main`, commit fce934b)
|
||||
**Source evidence:** five live `/last30days` debug sessions, captured per-session in `sessions/`.
|
||||
**Every line/behavior claim below was re-verified against current `main`, not taken on faith from the transcripts.**
|
||||
|
||||
This is the grounding document for the fix plan. It examines each problem separately,
|
||||
states the verified root cause with file/line, and rates fleet impact (the fixes ship to
|
||||
100k+ users — most of whom do NOT have working browser-cookie X auth, so keyless paths and
|
||||
honesty matter most at scale).
|
||||
|
||||
---
|
||||
|
||||
## A. X search bugs
|
||||
|
||||
### A1 — `from:{handle}` AND-bug (the headline X bug) · fleet impact: HIGH (every person/entity topic with X auth)
|
||||
**Symptom:** Passing `--x-handle=mvanhorn` (or xuezhao, steipete, …) returns ~0 of the person's own tweets. The "X" column fills with unrelated keyword collisions instead.
|
||||
**Verified root cause:** `bird_x.search_handles._search_one_handle` builds
|
||||
`from:{handle} {core_topic} since:{from_date}` whenever `core_topic` is truthy
|
||||
(`bird_x.py` ~373). X search is literal AND — so it only matches the person's tweets that *also contain the topic words* (usually their own name), which they never tweet. The unfiltered `from:{handle} since:` branch only fires when `topic is None`, and the caller (`pipeline.py:861`) always passes a truthy `topic`. So the real-timeline path is effectively unreachable.
|
||||
**Proven fix exists in-code:** running `from:{handle} since:` (no AND) returned 40 real tweets in-session.
|
||||
**User requirement (2026-06-17):** X must surface tweets **FROM** the person (engagement-weighted) — this is the `from:` lane, fixed to drop the topic-AND for person/handle topics.
|
||||
|
||||
### A2 — No mention/“about” lane at all · fleet impact: HIGH
|
||||
**Symptom:** Tweets *mentioning* the person (`@handle`) or *to* them never get collected as a category.
|
||||
**Verified root cause:** the pipeline only runs `from:` (author) searches and keyword subqueries. There is no `@handle` / `to:handle` mention query anywhere (`bird_x.search_handles` does `from:` only; no other call site builds a mention query).
|
||||
**User requirement (2026-06-17):** X must ALSO surface tweets **TO/ABOUT** the person (engagement-weighted) — a new mention lane, weighted by likes/reposts, deduped against the `from:` lane.
|
||||
|
||||
### A3 — Handle search is silent on success · fleet impact: MED (observability; caused 3 wrong diagnoses in-session)
|
||||
**Verified root cause:** `_search_one_handle` only `_log()`s on timeout / OSError / non-zero / invalid-JSON. A successful OR empty handle search emits zero log lines (`bird_x.py` ~382-405). The only `[Bird] Searching:` lines in task logs are the Phase-1 keyword subqueries, so the `from:` search looks like it never ran.
|
||||
**Fix:** log the query + result count on success, like the keyword path.
|
||||
|
||||
### A4 — `--diagnose` false-green for X · fleet impact: HIGH (fleet-wide trust bug)
|
||||
**Symptom:** `--diagnose` reports `bird_authenticated: true` / `bird_username: "env AUTH_TOKEN"` even when X is effectively returning nothing, and labels the source `env AUTH_TOKEN` when the real lane is live browser-cookie extraction (`_AUTH_TOKEN_SOURCE: browser`).
|
||||
**Verified root cause:** `pipeline.diagnose` reads `env.get_x_source_status` → `bird_status["authenticated"]` (`env.py:864`), a static credential-presence check, not a runtime probe.
|
||||
**Fix:** diagnose should do a real 1-tweet probe and report the true auth lane (browser vs env vs keychain). At scale most users have NO working X auth; the footer/coverage must say "X: 0 — no working auth" honestly instead of showing green.
|
||||
|
||||
### A5 — Off-topic X pollution · **CORRECTED at execution (2026-06-17)**
|
||||
**Symptom:** "compound interest / compound nevus" junk filled the X column in the Matt-vs-Trevin run even though bird was failing the whole run.
|
||||
**Original (transcript) hypothesis — DISPROVEN against current code:** the debug session blamed Digg's X-post enrichment side channel. False: Digg-enriched X posts live in `metadata["posts"]` and render ONLY as Digg-cluster quotes via `render._digg_posts_for` (returns `[]` for non-digg sources). Nothing adds them to `items_by_source["x"]`. Digg does NOT pollute the X column — a red herring.
|
||||
**Actual verified root cause:** the X-column "compound" junk came from the A6 strongest-token fallback querying a bare generic token, whose results ARE parsed into X items. **A6 is the real fix; the planned "entity-filter Digg X posts" unit (U1) was dropped as a non-occurring path.**
|
||||
**Residual minor note:** Digg-cluster X-post *quotes* could be tangentially off-topic, but they render as Digg quotes (not X items) and come from already-topic-matched clusters — low-severity, defer.
|
||||
|
||||
### A6 — Strongest-token fallback collapses to a bare generic token · fleet impact: MED
|
||||
**Verified root cause:** the last-chance retry picks `strongest = max(candidates, key=len)` then queries `f"{strongest} since:{from_date}"` (`bird_x.py:339-341`). For "trevin chow ai agents compound" it picks the longest token — `compound` — and dumps generic "compound" results into the shared X pool.
|
||||
**Fix:** don't collapse to a single generic token; keep an entity anchor (handle/name) in the retry, or drop the subquery rather than over-broaden.
|
||||
|
||||
### A7 — Name-collision / disambiguation gap · fleet impact: MED-HIGH (every mid-profile person)
|
||||
**Symptom:** "Kevin Rose" pulled Kevin Warsh (Fed chair), Leon Rose (Knicks), Kevin Durant, Kevin Hart — 55 items, ~zero on-topic. "Lan Xuezhao" pulled Lanzhou noodle + cdrama edits.
|
||||
**Root cause:** bare-name keyword subqueries are too collision-prone for mid-profile people; the engine has no disambiguation anchor (handle/domain/context) baked into the subqueries by default.
|
||||
**Note:** larger relevance problem than the deterministic A1-A6 bugs. Candidate for a follow-up scope rather than the first PR. **Call out for the plan.**
|
||||
|
||||
### NOT bugs — stop re-chasing (verified):
|
||||
- **Bird is not broken.** Vendored `scripts/lib/vendor/bird-search/` v0.8.0, MIT, zero deps; works against live cookies. Peter deprecating the public `@steipete/bird` package is irrelevant — it's vendored.
|
||||
- **`@steipete/sweet-cookie` is optional**, never a required dep (the "not installed" line is from the optional browser-cookie lane).
|
||||
- **The parser is correct.** Metrics nest under `item["engagement"]` and `item["date"]`, not top-level — earlier "metrics = None" was a debug print-key mistake.
|
||||
|
||||
---
|
||||
|
||||
## B. Funny things not showing up
|
||||
|
||||
### B1 — Best Takes renders empty in normal use (the structural root cause) · fleet impact: HIGH
|
||||
**Symptom:** Across the Kanye and Steinberger runs, no `## Best Takes` section appeared; the funniest lines ("Is anyone surprised? It's called TurkiYe", "I bet one of his kids will be a bully") never reached the synthesis.
|
||||
**Verified root cause (two compounding):**
|
||||
1. `rerank.score_fun` LLM-scores only when a reasoning `provider` exists in the **engine subprocess** (`pipeline.py:534`, `provider=reasoning_provider`). In normal `/last30days` usage the engine subprocess has NO paid reasoning provider (the hosting model is the planner but can't be called back into the subprocess), so fun scores come from the heuristic fallback (~38). `_render_best_takes` requires `fun_score >= _BEST_TAKE_FUNNY_FLOOR (40)` AND effective ≥ threshold (70 at medium) — so Best Takes returns `[]`. The vote-weighting shipped in #592 is moot without LLM fun scoring.
|
||||
2. `_render_candidate` (the compact EVIDENCE block) renders top comments ONLY for the representative items of the top-`cluster_limit` (8) clusters (`render.py:166-171, 1208`). Funny comments on lower-ranked or non-representative items are never in the synthesis block at all.
|
||||
|
||||
### B2 — The fun judgment is in the wrong place · design finding
|
||||
The hosting model (Claude) is an excellent fun judge and *does* have an API. The engine subprocess is the one that can't LLM-score. So the fix direction is to **move fun SELECTION to the hosting model**: have the engine surface a comment-rich, vote-scored "Best Takes candidates" / "Top Community Comments" block (across more than the top-8 representative items) inside the EVIDENCE envelope, and have SKILL.md make weaving 2-3 of the funniest a hard gate. Vote scores (the #592 signal) become the ranking input the model selects from. **Soft vs hard fork — call out for the plan:** engine-surfaces-candidates + model-selects (structural) vs. just lowering the Best Takes heuristic floor (weak).
|
||||
|
||||
---
|
||||
|
||||
## C. The AI is lazy / not reading what it finds
|
||||
|
||||
### C1 — Compact stdout treated as the whole dataset · fleet impact: HIGH (every run)
|
||||
**Symptom:** The model synthesized a news-shaped report off the compact EVIDENCE block and never opened the saved raw `.md` until prodded — missing comments, the subject's own quotes, a BULLY DELUXE release two days out, a Dutch court win, an allegation.
|
||||
**Root cause (behavioral + structural):** the compact EVIDENCE block is a lossy index (top-8 clusters, representative items, truncated). The richer per-source comment/quote layer lives deeper in the saved raw file, which the model treats as optional. SKILL.md's "weave the funniest takes" / PRE-PRESENT SELF-CHECK exist but are skipped as formalities.
|
||||
**Fix direction:** (a) structural — get the high-value layer (top comments with votes, the subject's own posts) INTO the synthesis-facing block so the model can't miss it; (b) behavioral — turn the self-check into an enforced gate (e.g. require ≥2 verbatim attributed quotes, lead with most-recent dated/upcoming event, test the thesis against highest-engagement items).
|
||||
|
||||
### C2 — Fabricated / reconstructed citation URL · fleet impact: HIGH (correctness; a wrong link looks authoritative)
|
||||
**Symptom:** In the Steinberger run the model linked @OtsileKole to a status ID that belonged to a different account — reconstructed from memory instead of copied from the raw file (a LAW 8 violation).
|
||||
**Fix:** every URL in output copied verbatim from the raw data; plain-text fallback if not found; never reconstruct a status ID.
|
||||
|
||||
### C3 — Meta-commentary about tooling leaks into the deliverable · fleet impact: MED (output quality)
|
||||
**Symptom:** "the social-listening engine struck out … 'Kevin Rose' collided with Kevin Warsh …" — narrating the engine's own failure inside the user-facing report.
|
||||
**Fix:** the synthesis presents what's true about the subject and quietly drops junk; engine-health notes belong in the footer/diagnostics, not the prose.
|
||||
|
||||
---
|
||||
|
||||
## D. Formatting integrity (user constraint #4)
|
||||
|
||||
### D1 — Cascading-repeat mangling of the "What I learned" block · fleet impact: unknown
|
||||
**Symptom:** In the Kanye run the synthesis block repeated the same paragraphs many times with progressively growing left-indentation — a badly corrupted render.
|
||||
**Status:** likely a model-output / terminal-streaming artifact rather than an engine-code bug (it's in the model's emitted prose, not the engine stdout). **Needs reproduction before claiming an engine cause — open question.**
|
||||
**Hard constraint on ALL fixes:** changes to the engine's emitted output (especially adding a comments/Best-Takes-candidates block to the EVIDENCE envelope) must NOT break the envelope markers, the PASS-THROUGH FOOTER, the badge, or the `What I learned:` contract. Format is already fragile; every output-touching unit must preserve it and be diffed against a real run.
|
||||
|
||||
---
|
||||
|
||||
## Priority for the fleet (100k+ users, most without working X auth)
|
||||
|
||||
1. **A5 Digg X-pollution** + **A6 fallback drift** — hits everyone, no X auth needed.
|
||||
2. **B1/B2 funny (Best Takes dark) + C1 laziness** — the product's headline value ("funniest comments on the whole internet"), every run, keyless.
|
||||
3. **A4 diagnose false-green** — fleet-wide honesty.
|
||||
4. **A1 from:-AND-bug + A2 mention lane** — the explicit FROM+ABOUT-with-weight requirement; deterministic, high correctness win for the X-auth subset.
|
||||
5. **A3 silent logging** — cheap observability, do alongside.
|
||||
6. **A7 disambiguation** + **D1 formatting cascade** — likely follow-up scope (bigger/uncertain).
|
||||
@@ -1,19 +0,0 @@
|
||||
# Session 01 — `/last30days Matt Van Horn vs Trevin Chow`
|
||||
|
||||
**What this session proved:** the X handle search runs but is poisoned by the topic-AND, fails silently, and the X column is actually Digg-side-channel pollution.
|
||||
|
||||
## Key evidence (verbatim engine/log lines)
|
||||
- Engine fired keyword X searches only in the visible log: `[Bird] Searching: matt van horn printing press since:2026-05-19`. No `from:mvanhorn` line — because `search_handles` logs nothing on success (→ A3).
|
||||
- GitHub person-mode worked (`[GitHub] Person-mode search for @mvanhorn`), so per-entity targeting was wired; X was the broken lane.
|
||||
- The 13 "X posts" were `@imAbhishek9596`, `@lebojoycechauke`, etc. — none authored by @mvanhorn.
|
||||
- Final verified mechanism: handle search built `from:mvanhorn matt van horn since:...` (topic AND'd onto the timeline → ~0). The clean `from:mvanhorn since:...` returned 40 real tweets.
|
||||
- The off-topic "compound interest / compound nevus" X items came in through Digg's X-enrichment side channel (`[Digg] post-dedupe enriched ... clusters with X posts`), NOT bird — bird was failing the whole run under the 2-entity parallel fanout (`Bird search failed`).
|
||||
- Strongest-token fallback collapsed `trevin chow ai agents compound` → bare token `compound`.
|
||||
|
||||
## Red herrings burned (NOT bugs)
|
||||
- "sweet-cookie missing = broken" — optional browser-cookie helper, never required.
|
||||
- "auth dead" — standalone test failed only because `get_config()` + `set_credentials()` weren't called to inject cookies.
|
||||
- "metrics = None parser bug" — print read wrong keys; parser nests under `engagement`/`date`.
|
||||
|
||||
## Bugs surfaced → inventory IDs
|
||||
A1 (from-AND), A3 (silent log), A4 (diagnose false-green), A5 (Digg pollution), A6 (fallback drift), A2 (no mention lane).
|
||||
@@ -1,15 +0,0 @@
|
||||
# Session 02 — `/last30days Kanye West`
|
||||
|
||||
**What this session proved:** the funniest, highest-engagement comments never reach the synthesis; the model synthesizes a news report off the lossy compact block; and the output formatting can corrupt.
|
||||
|
||||
## Key evidence
|
||||
- The best line of the month — `"Is anyone surprised? It's called TurkiYe"` (u/Drekkful, 764 upvotes), under a 3,895-upvote r/hiphopheads thread — was never mentioned. The model only read the compact "Ranked Evidence Clusters" (title + one snippet/item); the comment lived deeper in the saved raw file's per-source section.
|
||||
- Also missed: `"I bet one of his kids will be a bully"` (BULLY album callback), `"anyone interested in my kidney? i need tickets"`, plus a 21,707-like TikTok comment.
|
||||
- Missed real NEWS too (not just jokes): BULLY DELUXE dropping in 2 days, a Dutch court win clearing Arnhem shows (contradicted the "Europe is collapsing" thesis), an Italy ban, a serious allegation.
|
||||
- Model's own root cause: *"I treated the compact stdout as the dataset. It is a lossy index … the actual comment text, full post bodies, and top-comment upvote counts only live in the raw file's All Items by Source section."*
|
||||
|
||||
## Formatting corruption (constraint #4)
|
||||
- The emitted `What I learned:` block repeated the same paragraphs many times with progressively growing left-indentation — a badly cascaded render (→ D1). Likely a model/terminal artifact; needs reproduction.
|
||||
|
||||
## Bugs surfaced → inventory IDs
|
||||
B1 (Best Takes empty / comments not in synthesis block), C1 (compact-as-dataset laziness), D1 (formatting cascade), plus contradiction-pass and recency-pass gaps in synthesis behavior.
|
||||
@@ -1,11 +0,0 @@
|
||||
# Session 03 — `/last30days Kevin Rose`
|
||||
|
||||
**What this session proved:** mid-profile names are swamped by collisions, and the model leaks tooling-failure meta-commentary into the deliverable.
|
||||
|
||||
## Key evidence
|
||||
- "Kevin Rose" pulled Kevin Warsh (new Fed chair), Leon Rose (Knicks), Kevin Durant, Kevin Hart — 55 engine items, ~zero genuinely about the Digg founder. The 123K-upvote r/technology haul was all other people.
|
||||
- The synthesis opened with a paragraph narrating the engine's own miss: *"the social-listening engine struck out on Kevin Rose … 'Kevin Rose' collided with much louder newsmakers…"* — clutter the user must read past (→ C3).
|
||||
- A disambiguated re-run (every subquery locked to "Digg founder" context) killed the Warsh/Durant noise and surfaced real signal — proving disambiguation is the lever, not a synthesis band-aid (→ A7).
|
||||
|
||||
## Bugs surfaced → inventory IDs
|
||||
A7 (name-collision / disambiguation), C3 (meta-commentary in output).
|
||||
@@ -1,12 +0,0 @@
|
||||
# Session 04 — `/last30days Peter Steinberger`
|
||||
|
||||
**What this session proved:** funny/community texture missed again (same root cause as Kanye), PLUS a fabricated citation URL and a pattern anchored on the weakest source.
|
||||
|
||||
## Key evidence
|
||||
- Genuinely funny skeptic comments existed and made ZERO of the report: `@theoldschooldk` (310 likes) *"Don't talk to me about the SPEND. Tell me what he BUILT"*, `u/alemorg` *"one of the most inefficient workflows on earth"*. Model: *"the compact emit didn't hand me a pre-built Best Takes section, so I didn't go dig for them."*
|
||||
- **Fabricated URL (LAW 8):** linked `@OtsileKole` to a status ID that actually belonged to `@sabir_huss50540` — reconstructed from memory instead of copied from the raw file (→ C2).
|
||||
- Dropped the subject's own first-person quote (`@OmarShahine` fireside: *"you cannot really automate taste or thoughtful design"*) — the highest-value citation tier.
|
||||
- Anchored "build loops, not prompts" on a 210-view YouTube video while the raw file showed a rich multi-voice cluster (Boris Cherny coined it). Weak-source anchoring; ignored that all YouTube transcripts 429-failed (0/6).
|
||||
|
||||
## Bugs surfaced → inventory IDs
|
||||
B1 (Best Takes empty / funny missed), C1 (compact-as-dataset), C2 (fabricated URL), plus weak-source-anchoring + source-health-weighting gaps.
|
||||
@@ -1,14 +0,0 @@
|
||||
# Session 05 — `/last30days Lan Xuezhao`
|
||||
|
||||
**What this session proved (cleanest):** the from:-AND-bug AND the total absence of a mention lane — directly motivating the "FROM + TO/ABOUT, with weight" requirement.
|
||||
|
||||
## Key evidence
|
||||
- Engine X column = finance/VC keyword collisions (`@TheValueist`, `@arnaudmercier`, `@pierskicks`) + one `@hardmaru` "Thanks, Lan!" reply. None authored by @xuezhao.
|
||||
- Verified mechanism: `--x-handle=xuezhao` → `from:xuezhao lan xuezhao since:...` (topic AND) → ~0. The unfiltered `from:xuezhao since:` branch is unreachable because topic is always truthy (→ A1).
|
||||
- Structural gap: *"the engine only runs from: (authored-by) and keyword searches. There is no @xuezhao / to:xuezhao mention query anywhere in the pipeline"* (→ A2).
|
||||
- Manual unfiltered `from:xuezhao` pulled 12 real posts (DeepSeek take, "new AI stack is global, sovereign and embedded", SpaceX grind) — genuinely interesting, all missed by the engine.
|
||||
- Mentions were rich too (practitioners asking about her transcription/Hermes-dashboard rig) — exactly the "TO/ABOUT the person, with weight" lane the user now wants.
|
||||
- Engagement counts came back 0 on the manual cookie-search field — note for weighting the mention/from lanes (resolve metrics per-tweet).
|
||||
|
||||
## Bugs surfaced → inventory IDs
|
||||
A1 (from-AND), A2 (no mention lane), A3 (silent log), plus the FROM+ABOUT-with-weight requirement.
|
||||
@@ -0,0 +1,205 @@
|
||||
# Original v3.0.0 First-Run NUX Wizard (reference capture)
|
||||
|
||||
Captured verbatim from `SKILL.md` at git commit `0a9ff16` (v3.0.0, 2026-04-08),
|
||||
the first-run setup wizard Matt built. Preserved here for provenance and as the
|
||||
source for the restored modal NUX (see docs/plans/2026-06-22-001-feat-restore-nux-wizard-plan.md).
|
||||
This is a historical snapshot - the live wizard in SKILL.md Step 0 uses the CURRENT
|
||||
source inventory (Digg, youtube_comments, SC backups) and omits Threads/Pinterest.
|
||||
|
||||
```markdown
|
||||
## Step 0: First-Run Setup Wizard
|
||||
|
||||
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever.
|
||||
|
||||
To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section **silently** and proceed to Step 1. **Do NOT say "Setup is complete" or any other status message — just move on.** The user doesn't need to be told setup is done every time they run the skill.
|
||||
|
||||
**When first run is detected, detect your platform first:**
|
||||
|
||||
**If you do NOT have WebSearch capability (OpenClaw, Codex, raw CLI):** Run the OpenClaw setup flow below.
|
||||
**If you DO have WebSearch (Claude Code):** Run the standard setup flow below.
|
||||
|
||||
---
|
||||
|
||||
### OpenClaw / Non-WebSearch Setup Flow
|
||||
|
||||
Run environment detection first:
|
||||
```bash
|
||||
python3 "${SKILL_ROOT}/scripts/last30days.py" setup --openclaw
|
||||
```
|
||||
|
||||
Read the JSON output. It tells you what's already configured. Display a status summary:
|
||||
|
||||
```
|
||||
👋 Welcome to /last30days!
|
||||
|
||||
Detected:
|
||||
{✅ or ❌} yt-dlp (YouTube search)
|
||||
{✅ or ❌} X/Twitter ({method} configured)
|
||||
{✅ or ❌} ScrapeCreators (TikTok, Instagram, Reddit backup)
|
||||
{✅ or ❌} Web search ({backend} configured)
|
||||
```
|
||||
|
||||
Then for each missing item, offer setup in priority order:
|
||||
|
||||
1. **ScrapeCreators** (if not configured): "ScrapeCreators adds TikTok and Instagram search (plus a Reddit backup if public Reddit gets rate-limited). 10,000 free calls, no credit card. (No referrals, no kickbacks - we don't get a cut.)"
|
||||
- Option A: "ScrapeCreators via GitHub (recommended)" -- Check if `gh` CLI was detected in the environment detection output above. If gh IS detected: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". Before running the command, display: "Registering via GitHub CLI..." If gh is NOT detected: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". Before running the command, display: "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 "${SKILL_ROOT}/scripts/last30days.py" setup --github`, parse JSON output. Tries PAT first (if `gh` is installed), falls back to device flow which copies a one-time code to your clipboard and opens your browser. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to .env.
|
||||
- Option B: "I have a key" -- accept paste, write to .env
|
||||
- Option C: "Skip for now"
|
||||
|
||||
2. **X/Twitter** (if not configured): "X search finds tweets and conversations. To unlock X: add FROM_BROWSER=auto (reads browser cookies, free), XAI_API_KEY (no browser access, api.x.ai), or AUTH_TOKEN+CT0 (manual cookies)."
|
||||
- Option A: "I have an xAI API key" (recommended for servers -- persistent, no expiry). Write XAI_API_KEY to .env.
|
||||
- Option B: "I have AUTH_TOKEN + CT0 from my browser" -- accept both, write to .env
|
||||
- Option C: "Skip for now"
|
||||
|
||||
3. **YouTube** (if yt-dlp not found): "YouTube search needs yt-dlp. Run: `pip install yt-dlp`"
|
||||
|
||||
4. **Web search** (if no Brave/Exa/Serper key): "A web search key enables smarter results. Brave Search is free for 2,000 queries/month at brave.com/search/api"
|
||||
|
||||
After setup, write `SETUP_COMPLETE=true` to .env and proceed to research.
|
||||
|
||||
**Skip to "END OF FIRST-RUN WIZARD" below after completing the OpenClaw flow.**
|
||||
|
||||
---
|
||||
|
||||
### Claude Code Setup Flow (Standard)
|
||||
|
||||
**You MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text -> (2) setup modal -> (3) run setup if chosen -> (4) optional ScrapeCreators modal -> (5) topic picker. You MUST start at step 1.**
|
||||
|
||||
**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion - do NOT repeat any of the welcome text inside the AskUserQuestion call.**
|
||||
|
||||
Welcome to /last30days!
|
||||
|
||||
I research any topic across Reddit, X, YouTube, and other sources - synthesizing what people are actually saying right now.
|
||||
|
||||
Auto setup gives you 5 core sources for free in 30 seconds:
|
||||
- X/Twitter - reads your x.com browser cookies to authenticate (not saved to disk). Chrome on macOS will prompt for Keychain access.
|
||||
- Reddit with comments - public JSON, no API key needed
|
||||
- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars)
|
||||
- Hacker News + Polymarket + GitHub (if `gh` CLI installed) - always on, zero config
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation.
|
||||
|
||||
**Then call AskUserQuestion with ONLY this question and these options - no additional text:**
|
||||
|
||||
Question: "How would you like to set up?"
|
||||
Options:
|
||||
- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp for YouTube"
|
||||
- "Manual setup - show me what to configure"
|
||||
- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if gh installed), Web"
|
||||
|
||||
**If the user picks 1 (Auto setup):**
|
||||
|
||||
**Before running the setup command, get cookie consent:**
|
||||
|
||||
Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`. If it does, skip the consent prompt and run setup directly.
|
||||
|
||||
If `BROWSER_CONSENT=true` is NOT present, **call AskUserQuestion:**
|
||||
Question: "Auto setup will scan your browser for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. Chrome on macOS will prompt for Keychain access. OK to proceed?"
|
||||
Options:
|
||||
- "Yes, scan my cookies for X" - Run setup as normal. Append `BROWSER_CONSENT=true` to .env after setup completes.
|
||||
- "Skip X, just set up YouTube" - Run setup with YouTube only (install yt-dlp). Do not scan cookies.
|
||||
- "I have an xAI API key instead" - Ask them to paste it, write XAI_API_KEY to .env. Then install yt-dlp.
|
||||
|
||||
Run the setup subcommand:
|
||||
```bash
|
||||
cd {SKILL_DIR} && python3 scripts/last30days.py setup
|
||||
```
|
||||
Show the user the results (what cookies were found, whether yt-dlp was installed).
|
||||
|
||||
**Then show the optional ScrapeCreators offer (plain text, then modal):**
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those platforms - 10,000 free calls, no credit card. It also serves as a Reddit backup if public Reddit ever gets rate-limited.
|
||||
|
||||
**Before showing the ScrapeCreators modal, check for `gh` CLI:** Run `which gh` via Bash silently. Store the result as gh_available (true if found, false if not).
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Want to add TikTok, Instagram, and Reddit backup via ScrapeCreators? (We don't get a cut.)"
|
||||
Options:
|
||||
- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". If NOT gh_available: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". After the user selects this option: If gh_available, display "Registering via GitHub CLI..." before running the command. If NOT gh_available, display "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V on Mac, Ctrl+V on Windows/Linux)." Then run `cd {SKILL_DIR} && python3 scripts/last30days.py setup --github` via Bash with a 5-minute timeout. This tries PAT auth first (if `gh` CLI is installed, zero browser needed), then falls back to GitHub device flow which copies a one-time code to your clipboard and opens GitHub in your browser. Parse the JSON stdout. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to `~/.config/last30days/.env`. If `method` is `pat`, show: "You're in! Registered via GitHub CLI - zero browser needed. 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is true, show: "You're in! (The authorization code was copied to your clipboard automatically.) 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is false, show: "You're in! 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `status` is `timeout` or `error`, show: "GitHub auth didn't complete. No worries - you can sign up at scrapecreators.com instead or try again later." Then offer the web signup option.
|
||||
- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env
|
||||
- "I have a key" - accept the key, write to .env
|
||||
- "Skip for now" - proceed without ScrapeCreators
|
||||
|
||||
**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:**
|
||||
|
||||
Your ScrapeCreators key powers TikTok, Instagram, Threads, Pinterest, and YouTube comments. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.)
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Which ScrapeCreators sources do you want on?"
|
||||
Options:
|
||||
- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "TikTok and Instagram are on, plus Reddit backup if public Reddit has issues. You can add threads, pinterest, youtube_comments to INCLUDE_SOURCES anytime."
|
||||
- "Everything - TikTok, Instagram, Threads, Pinterest, YouTube comments" - append `INCLUDE_SOURCES=tiktok,instagram,threads,pinterest,youtube_comments` to ~/.config/last30days/.env. Confirm: "All ScrapeCreators sources are on."
|
||||
- "Just the basics - let's run our first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will serve as Reddit backup. You can add sources to INCLUDE_SOURCES in your .env anytime."
|
||||
|
||||
**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:**
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "What do you want to research first?"
|
||||
Options:
|
||||
- "Claude Code vs Codex" - tech comparison
|
||||
- "Sam Altman" - person in the news
|
||||
- "Warriors Basketball" - sports
|
||||
- "AI Legal Prompting Techniques" - niche/professional
|
||||
- "Type my own topic"
|
||||
|
||||
If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly.
|
||||
|
||||
**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.**
|
||||
|
||||
**If the user picks 2 (Manual setup):**
|
||||
Show them this guide (present as plain text, not blockquoted):
|
||||
|
||||
The magic of /last30days is Reddit comments + X posts together - and both are free. Here's how to unlock each source.
|
||||
|
||||
Add these to `~/.config/last30days/.env`:
|
||||
|
||||
X/Twitter (pick one - this is the most important):
|
||||
- `FROM_BROWSER=auto` - free. Reads your x.com login cookies at search time to authenticate. Cookies are read live each run, not saved to disk. Chrome on macOS will prompt for Keychain access the first time. Firefox and Safari don't.
|
||||
- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers or if you don't want cookie scanning.
|
||||
- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com -> F12 -> Application -> Cookies)
|
||||
|
||||
Reddit (free, works out of the box):
|
||||
- Public JSON gives you threads + top comments with upvote counts. No setup required.
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - optional backup source if public Reddit gets rate-limited.
|
||||
- `OPENAI_API_KEY=xxx` - optional fallback if public Reddit search has trouble finding threads.
|
||||
|
||||
YouTube (free, open source):
|
||||
- Run `brew install yt-dlp` - free, open source, 190K+ GitHub stars. Enables YouTube search and transcripts.
|
||||
|
||||
Bonus: TikTok, Instagram, Threads, Pinterest, YouTube comments (ScrapeCreators):
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com.
|
||||
- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the most popular ones. Add threads, pinterest, youtube_comments for more.
|
||||
|
||||
GitHub Issues/PRs (free, no key needed):
|
||||
- If you have the `gh` CLI installed (`brew install gh`), GitHub search is automatic. No API key required.
|
||||
|
||||
Perplexity Sonar Pro (AI-synthesized research via OpenRouter):
|
||||
- `OPENROUTER_API_KEY=xxx` - adds AI-synthesized research with citations as an additive source alongside Reddit/X/YouTube. Returns structured narratives with specific dates, names, and numbers that social sources miss. ~$0.02/run.
|
||||
- After adding your key, set `INCLUDE_SOURCES=perplexity` (or append to existing, e.g. `INCLUDE_SOURCES=tiktok,instagram,perplexity`).
|
||||
- Use `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query) on topics that need serious investigation.
|
||||
- Bonus: also powers the planning and reranking engine if you don't have a Gemini/OpenAI/xAI key.
|
||||
|
||||
Other bonus sources (add anytime):
|
||||
- `EXA_API_KEY=xxx` - semantic web search, 1K free/month (exa.ai)
|
||||
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password)
|
||||
- `BRAVE_API_KEY=xxx` - Brave web search
|
||||
|
||||
Always add this last line: `SETUP_COMPLETE=true`
|
||||
|
||||
**CRITICAL: NEVER overwrite an existing .env file.** Before writing ANY key to `~/.config/last30days/.env`:
|
||||
1. Check if the file exists: `test -f ~/.config/last30days/.env`
|
||||
2. If it exists, READ it first, then APPEND only missing keys using `>>` (double redirect)
|
||||
3. NEVER use `>` (single redirect) which destroys existing content
|
||||
4. If it doesn't exist, create it: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env`
|
||||
|
||||
**Then call AskUserQuestion:**
|
||||
Question: "How do you want to add your keys?"
|
||||
Options:
|
||||
- "Open .env in my editor" - Creates the file with a commented template and opens it. You edit, save, and come back.
|
||||
- "Paste keys here" - Paste your API keys and I'll write the file for you.
|
||||
- "I'll do it myself" - I'll tell you the file path and you handle it.
|
||||
|
||||
**If the user picks "Open .env in editor":**
|
||||
Create `~/.config/last30days/.env` if it doesn't exist (check first!), pre-populated with this template:
|
||||
```
|
||||
```
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.8.0",
|
||||
"version": "3.8.1",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
|
||||
@@ -35,8 +35,7 @@ check_perms() {
|
||||
# every Linux session start and printed a false WARNING.
|
||||
perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
|
||||
if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
|
||||
echo "/last30days: WARNING — $file has permissions $perms (should be 600)."
|
||||
echo " Fix: chmod 600 $file"
|
||||
chmod 600 "$file" && echo "/last30days: WARNING — $file had permissions $perms — auto-fixed with chmod 600" || echo "/last30days: WARNING — $file has permissions $perms (should be 600). Fix: chmod 600 $file"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ module github.com/mvanhorn/last30days-skill/mcp
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/mark3labs/mcp-go v0.54.0
|
||||
require github.com/mark3labs/mcp-go v0.55.0
|
||||
|
||||
require (
|
||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mark3labs/mcp-go v0.54.0 h1:PZhQvd+5xrT43cUoiaKn/hDcvLUhcLc1twSEKYPTcTA=
|
||||
github.com/mark3labs/mcp-go v0.54.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/mark3labs/mcp-go v0.55.0 h1:lJfz2aoctiwK+sI991+uIYwmKNIBciI+O7zsyDsa4U8=
|
||||
github.com/mark3labs/mcp-go v0.55.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
@@ -36,7 +37,7 @@ func Register(s *server.MCPServer, cfg Config) {
|
||||
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
|
||||
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
|
||||
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
|
||||
mcplib.WithReadOnlyHintAnnotation(true),
|
||||
mcplib.WithReadOnlyHintAnnotation(false),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(true),
|
||||
),
|
||||
@@ -74,10 +75,7 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
)), nil
|
||||
}
|
||||
|
||||
runArgs := []string{topic, "--emit=" + emit}
|
||||
if save {
|
||||
runArgs = append(runArgs, "--save")
|
||||
}
|
||||
runArgs := researchRunArgs(topic, emit, save)
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
@@ -90,6 +88,18 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func researchRunArgs(topic, emit string, save bool) []string {
|
||||
runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"}
|
||||
if save {
|
||||
saveDir := os.Getenv("LAST30DAYS_MEMORY_DIR")
|
||||
if saveDir == "" {
|
||||
saveDir = "~/Documents/Last30Days"
|
||||
}
|
||||
runArgs = append(runArgs, "--save-dir", saveDir)
|
||||
}
|
||||
return runArgs
|
||||
}
|
||||
|
||||
func requireString(args map[string]any, name string) (string, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
|
||||
@@ -97,6 +97,36 @@ func TestBoolArgument(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) {
|
||||
args := researchRunArgs("OpenAI", "compact", false)
|
||||
want := []string{"OpenAI", "--emit=compact", "--no-browser-cookies"}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
|
||||
args := researchRunArgs("OpenAI", "html", true)
|
||||
got := strings.Join(args, "\x00")
|
||||
if strings.Contains(got, "--save\x00") || strings.HasSuffix(got, "--save") {
|
||||
t.Fatalf("args still include unsupported --save: %#v", args)
|
||||
}
|
||||
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "~/Documents/Last30Days"}
|
||||
if got != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsSaveUsesMemoryDirEnvOverride(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
|
||||
args := researchRunArgs("OpenAI", "html", true)
|
||||
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "/tmp/last30days-reports"}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
|
||||
// Validation failures are returned as MCP tool errors (not Go errors)
|
||||
// so Claude sees a structured failure with a readable message rather
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
"manifest_version": "0.3",
|
||||
"name": "last30days-pp-mcp",
|
||||
"display_name": "Last30Days",
|
||||
"version": "3.0.0",
|
||||
"version": "3.6.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.8.0"
|
||||
version = "3.8.1"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -8,7 +8,7 @@ dependencies = []
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3,<10",
|
||||
"pytest>=9.1.0,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
|
||||
+201
-16
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "3.8.0"
|
||||
version: "3.8.1"
|
||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
@@ -230,7 +230,20 @@ ToolSearch select:WebSearch
|
||||
|
||||
WebSearch is a **deferred tool** in Claude Code v2.1.114. The frontmatter of this file authorizes it (`allowed-tools: ... WebSearch`) but the runtime lists it as "schemas are NOT loaded." Calling WebSearch without `ToolSearch select:WebSearch` first will fail or do nothing. That friction is the documented cause of the second-most-common failure mode of this skill: the model sees "WebSearch is there but deferred," takes the low-friction path, skips Step 0.5 and 0.55, and runs the engine bare with only keyword search. The output looks fine but misses founder X timelines, GitHub repo activity, and subreddit-specific threads.
|
||||
|
||||
Load WebSearch first. No exceptions. Then proceed to the branching rule below.
|
||||
Load WebSearch first. No exceptions. Then run the first-run gate below before anything else.
|
||||
|
||||
**FIRST-RUN GATE — run this Bash command immediately after loading WebSearch, before reading the topic or doing any research:**
|
||||
|
||||
```bash
|
||||
grep -q "SETUP_COMPLETE=true" ~/.config/last30days/.env 2>/dev/null && echo "1" || echo "FIRST_RUN_DETECTED"
|
||||
```
|
||||
|
||||
This emits exactly one token: `1` or `FIRST_RUN_DETECTED`, never both.
|
||||
|
||||
- Output is `1` → setup is complete. Continue to the branching rule below.
|
||||
- Output is `FIRST_RUN_DETECTED` → this is a first run. Jump immediately to `## Step 0: First-Run Setup Wizard` and complete it **before doing any topic research**. Do NOT proceed to Step 0.5, do NOT load WebSearch supplements, do NOT synthesize anything. The wizard installs yt-dlp (YouTube), the Digg CLI (via `npx`), and extracts browser cookies for X/Twitter and other sources. Skipping it produces a degraded WebSearch-only result that misrepresents the skill's capability to the user.
|
||||
|
||||
**Named failure mode (2026-06-22, first-run setup skip - Fredy Montero run):** Model read "proceed to Step 0.5" in the branching rule and jumped there directly, bypassing `## Step 0: First-Run Setup Wizard` at line ~339. Result: no browser cookie extraction, no yt-dlp, no Digg CLI install, WebSearch-only synthesis with no X/YouTube/TikTok data. Root cause: the branching rule named Step 0.5 as the next step without mentioning the wizard. Fix: this gate and the updated branching rule below.
|
||||
|
||||
**STEP 1 - RUN THE ENGINE. You MUST run `scripts/last30days.py` via Bash. Do not produce output from WebSearch alone.**
|
||||
|
||||
@@ -238,7 +251,7 @@ The single most common failure mode of this skill is the model reading this file
|
||||
|
||||
Branching rule:
|
||||
|
||||
- **If the user provided a topic** (e.g. `/last30days Kanye West`, `/last30days nvidia earnings`): proceed to Step 0.5 / Step 0.55 / Step 0.75 / Research Execution below. Do not skip straight to WebSearch. WebSearch is a **supplement after** the Python engine runs (see Step 2). It is **not a substitute**.
|
||||
- **If the user provided a topic** (e.g. `/last30days Kanye West`, `/last30days nvidia earnings`): confirm the first-run gate above passed (output `1`), then proceed to `## Step 0: First-Run Setup Wizard` (or skip it if already confirmed complete), then continue to Step 0.45 / Step 0.5 / Step 0.55 / Step 0.75 / Research Execution below. Do not skip straight to WebSearch. WebSearch is a **supplement after** the Python engine runs (see Step 2). It is **not a substitute**.
|
||||
- **If the user provided no topic**: ask the user for a topic with a single short question. Do not run research. Do not run WebSearch. Wait.
|
||||
|
||||
If you are about to write a response without having run `scripts/last30days.py` at least once, stop. Return to Research Execution and run the engine. Every valid output from this skill includes the emoji-tree footer (`✅ All agents reported back!`) that the engine produces data for. No footer means you did not run the skill.
|
||||
@@ -251,7 +264,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
||||
|
||||
---
|
||||
|
||||
# last30days v3.8.0: Research Any Topic from the Last 30 Days
|
||||
# last30days v3.8.1: Research Any Topic from the Last 30 Days
|
||||
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
|
||||
@@ -322,6 +335,21 @@ fi
|
||||
LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}"
|
||||
```
|
||||
|
||||
**PYTHON VERSION GATE — when the Runtime Preflight Bash block above exits with a Python version error:**
|
||||
|
||||
If the preflight script emits `ERROR: last30days v3 requires Python 3.12+` (or `LAST30DAYS_PYTHON must point to Python 3.12+`) and exits, you MUST:
|
||||
|
||||
1. Display this message to the user:
|
||||
> "The last30days engine needs Python 3.12+. Your system has an older version. Install it with one command:
|
||||
> - **Mac:** `brew install python@3.12`
|
||||
> - **Windows:** `winget install Python.Python.3.12`
|
||||
> - **Linux:** `sudo apt install python3.12` (or `pyenv install 3.12`)
|
||||
>
|
||||
> Then re-run `/last30days <your topic>` and the setup wizard will configure everything automatically."
|
||||
2. **Stop.** Do not attempt research. Do not fall back to WebSearch-only synthesis.
|
||||
|
||||
WebSearch-only synthesis is not equivalent to running the engine — it misses Reddit community data, X/Twitter timelines, YouTube transcripts, TikTok, and Polymarket. Presenting it without disclosure misleads the user about what was actually searched. This is the same category of failure as a WebSearch-only run with no engine footer.
|
||||
|
||||
**Native-search signal (web coverage).** If you (the hosting model) have your own web-search tool available — e.g. Claude Code's `WebSearch`, which STEP 0 loads — export `LAST30DAYS_NATIVE_SEARCH=1` in the same shell before invoking the engine:
|
||||
|
||||
```bash
|
||||
@@ -338,18 +366,175 @@ The engine reads `LAST30DAYS_MEMORY_DIR` from either the process env or `~/.conf
|
||||
|
||||
## Step 0: First-Run Setup Wizard
|
||||
|
||||
Before proceeding to Step 1, handle first-run setup.
|
||||
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even when the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST run the wizard BEFORE any research. The topic is preserved - research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. It takes about 30 seconds and only runs once, ever.
|
||||
|
||||
**You are the conversational driver.** The Python setup script does only mechanical work (cookie reads, tool installs, the GitHub device-auth flow) - it CANNOT prompt the user, because it runs as a non-interactive subprocess. So consent happens HERE, in chat: you ask, the user answers, and you gate each subprocess call on the answer. Do NOT just run `setup` and report the result - that is the silent-onboarding regression this section exists to prevent.
|
||||
|
||||
**First-run detection (silent, no commands, no output to user):**
|
||||
- If `~/.config/last30days/.env` does NOT exist, this is a first run.
|
||||
- If the file exists and contains `SETUP_COMPLETE=true`, skip Step 0 entirely and go to Step 1 (CRITICAL: Parse User Intent below). Do NOT announce that setup is complete. The user does not need a status message on every run.
|
||||
|
||||
**If this IS a first run:**
|
||||
- Run `python3 skills/last30days/scripts/last30days.py setup` (relative to the skill root) to launch the setup wizard.
|
||||
- Follow the wizard's prompts end-to-end. The wizard handles platform detection (OpenClaw vs Claude Code), auto vs manual setup, browser cookie extraction, ScrapeCreators opt-in, a best-effort auto-install of the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only` — Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if the CLI is installed off-PATH; recommend-only if `npx` is unavailable), and the initial topic picker.
|
||||
- After the wizard writes `SETUP_COMPLETE=true` to `~/.config/last30days/.env`, proceed to research.
|
||||
**Named onboarding contracts:**
|
||||
- *(2026-06-22, silent-wizard regression - Fredy Montero run):* a prior version said "Run `setup` ... follow the wizard's prompts end-to-end." But `run_auto_setup()` has NO prompts - it extracts cookies, installs yt-dlp + Digg, and writes `SETUP_COMPLETE` with zero interaction. The model ran the silent path, never asked cookie consent, never surfaced the macOS Full Disk Access fix, and never offered the ScrapeCreators signup. Consent must be conversational.
|
||||
- *(2026-06-22, NUX restoration):* the original v3.0.0 Claude Code wizard was a guided, modal-driven flow (welcome → Auto/Manual/Skip → cookie consent → ScrapeCreators offer → source opt-in → first-topic picker) that eroded over time. It is restored below as the **Claude Code Modal Flow**. Do NOT collapse it back into a bare prose call - the guided modals are the feature. Reference capture: `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
|
||||
The setup wizard lives as a Python module so it works across all hosts (Claude Code, Codex, Cursor, etc.) and the common-case (already set up) path through this file stays short.
|
||||
**Platform split - run exactly ONE branch:**
|
||||
- **If you HAVE WebSearch and AskUserQuestion (Claude Code):** run the **Claude Code Modal Flow** immediately below.
|
||||
- **If you do NOT (OpenClaw, Codex, Cursor, Gemini CLI, raw CLI):** run the **Non-Modal Prose Flow** further down. It does the same work conversationally, without modals.
|
||||
|
||||
---
|
||||
|
||||
### Claude Code Modal Flow
|
||||
|
||||
**Follow these steps IN ORDER. Do NOT skip ahead to research. The sequence is: (1) welcome text → (2) setup modal → (3) run setup if chosen → (4) ScrapeCreators offer modal → (5) source opt-in modal → (6) first-topic picker. Start at step 1.**
|
||||
|
||||
**Step 1 - Welcome.** Display this welcome text ONCE as a normal message (not blockquoted).
|
||||
|
||||
Welcome to /last30days!
|
||||
|
||||
I research any topic across Reddit, X, YouTube, and more - synthesizing what people are actually saying right now.
|
||||
|
||||
Auto setup gives you the core sources free in about 30 seconds:
|
||||
- X/Twitter - reads your browser cookies to authenticate (read live each run, never saved to disk).
|
||||
- Reddit with comments - public JSON, no API key needed.
|
||||
- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars).
|
||||
- Digg - trending news, GitHub stars, and pipeline feeds - installs the free, keyless Digg CLI.
|
||||
- Hacker News + Polymarket + GitHub (auto-on if the `gh` CLI is installed) - always on, zero config.
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation.
|
||||
|
||||
**Step 2 - Setup choice.** Then IMMEDIATELY call AskUserQuestion with ONLY this question and these options (do not repeat the welcome text inside the modal):
|
||||
|
||||
Question: "How would you like to set up?"
|
||||
Options:
|
||||
- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp (YouTube) and the Digg CLI"
|
||||
- "Manual setup - show me what to configure"
|
||||
- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if `gh` installed), Web"
|
||||
|
||||
**Step 3 - Run setup based on the choice.**
|
||||
|
||||
**If the user picks Skip for now:** write `SETUP_COMPLETE=true` to `~/.config/last30days/.env` (append-only; run `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env` first if the file does not exist) so the wizard does NOT re-fire on every subsequent run, then skip straight to Step 6 (the topic picker). Do not run any `setup` command - the always-on sources (Reddit, HN, Polymarket, GitHub, Web) need no setup.
|
||||
|
||||
**If the user picks Auto setup:**
|
||||
|
||||
Get cookie consent first. Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`; if so, skip the consent prompt and run `setup --allow-browser-cookies` directly. Otherwise **call AskUserQuestion:**
|
||||
Question: "Auto setup will scan your browser (Firefox/Safari) for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. OK to proceed?"
|
||||
Options:
|
||||
- "Yes, scan my cookies for X" - run `python3 skills/last30days/scripts/last30days.py setup --allow-browser-cookies` (relative to the skill root). Append `BROWSER_CONSENT=true` to `.env` after setup completes.
|
||||
- "Skip X, just set up YouTube + Digg" - run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. Skips all cookie reads; still installs yt-dlp and Digg.
|
||||
- "I have an xAI API key instead" - ask them to paste it, write `XAI_API_KEY` to `.env`, then run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup` (installs yt-dlp + Digg, no cookie read).
|
||||
|
||||
The consented `setup --allow-browser-cookies` run extracts cookies (Firefox/Safari by default - never Chrome, to avoid a macOS Keychain prompt, unless `FROM_BROWSER=auto` or a named Chromium browser was explicitly configured) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if installed off-PATH; recommend-only if `npx` is unavailable). Show the user what was found and installed - including whether Digg landed on PATH (active) or off-PATH (installed but not yet active).
|
||||
|
||||
**macOS Full Disk Access remediation.** After the `setup` run, inspect its stderr. If it contains `Permission denied reading Cookies.binarycookies` and the platform is macOS, the OS blocked the read - surface the fix instead of swallowing it: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry of the `setup` command. If the user skips, continue.
|
||||
|
||||
**Step 4: ScrapeCreators offer (every first run).** Show this as plain text, then a modal:
|
||||
|
||||
ScrapeCreators adds TikTok and Instagram - 10,000 free calls, no credit card. Your key also powers YouTube comments, a YouTube transcript fallback (used only when yt-dlp gets rate-limited), and a Reddit backup (if public Reddit gets rate-limited). (We don't get a cut.)
|
||||
|
||||
Before the modal, run `which gh` via Bash silently; store as gh_available.
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Want to add TikTok, Instagram, and the ScrapeCreators backups? (We don't get a cut.)"
|
||||
Options:
|
||||
- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available, description: "Registers via GitHub CLI in ~2 seconds - no browser." If NOT gh_available, description: "Copies a one-time code to your clipboard and opens GitHub to authorize." After selection: if gh_available, display "Registering via GitHub CLI..."; if not, display "I'll copy a one-time code to your clipboard and open GitHub. When prompted for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 skills/last30days/scripts/last30days.py setup --github` with a 5-minute timeout. Parse the JSON. On `status == "success"` the engine persists the key automatically and returns `"persisted": true` with a MASKED `api_key` (the raw key never appears - do not ask for or echo it); confirm "You're in! 10,000 free calls. TikTok, Instagram, YouTube comments, and the Reddit/YouTube backups are now active." On `status == "success"` but `"persisted": false` (key write failed, e.g. a permissions error), do NOT claim sources are active - tell the user the signup worked but saving the key failed, and have them add `SCRAPECREATORS_API_KEY=<key>` to `~/.config/last30days/.env` manually (the raw key is masked in output, so re-run `setup --github` or retrieve it from scrapecreators.com to get the value). On `status` `timeout`/`error`, show "GitHub auth didn't complete - no worries, sign up at scrapecreators.com or try again later," then offer the web option.
|
||||
- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash, then ask them to paste the API key. Write `SCRAPECREATORS_API_KEY={key}` to `~/.config/last30days/.env`.
|
||||
- "I have a key" - accept the key, write to `.env`.
|
||||
- "Skip for now" - proceed without ScrapeCreators.
|
||||
|
||||
**Step 5: Source opt-in (only if a ScrapeCreators key was saved, not if skipped).** Plain text then modal:
|
||||
|
||||
Your ScrapeCreators key powers TikTok, Instagram, and YouTube comments. Want TikTok and Instagram on for every run? (Each adds one ScrapeCreators call per search.)
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Which ScrapeCreators sources do you want on?"
|
||||
Options:
|
||||
- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to `~/.config/last30days/.env`. Confirm: "TikTok and Instagram are on, plus the Reddit/YouTube backups if the free sources get rate-limited."
|
||||
- "Just the basics - let's run my first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will still serve as the Reddit and YouTube backups. You can add sources to `INCLUDE_SOURCES` in your `.env` anytime."
|
||||
|
||||
**Step 6: First-topic picker.** Once `SETUP_COMPLETE=true` is written, **call AskUserQuestion:**
|
||||
Question: "What do you want to research first?"
|
||||
Options:
|
||||
- "Claude Code vs Codex" - tech comparison
|
||||
- "Sam Altman" - person in the news
|
||||
- "Warriors Basketball" - sports
|
||||
- "AI Legal Prompting Techniques" - niche/professional
|
||||
- "Type my own topic"
|
||||
|
||||
If the user picks an example, run research with it. If "Type my own", ask what they want. **If the user already supplied a topic with the command (e.g. `/last30days Mercer Island`), SKIP this picker and use their topic directly.**
|
||||
|
||||
**END OF FIRST-RUN WIZARD.** Everything in the Modal Flow ONLY runs on first run. If `SETUP_COMPLETE=true` exists, skip ALL of it - no welcome, no modals, no topic picker - and go straight to research (Parse User Intent).
|
||||
|
||||
**If the user picked Manual setup** at Step 2, follow the **Manual Setup Guide** below instead of the Auto branch (the guide writes `SETUP_COMPLETE=true` itself), then continue to Step 6.
|
||||
|
||||
---
|
||||
|
||||
### Non-Modal Prose Flow
|
||||
|
||||
For hosts without interactive modal prompts (OpenClaw, Codex, Cursor, Gemini CLI, raw CLI). Same work, done conversationally. Run in order; wait where it says to wait.
|
||||
|
||||
**1. Welcome.** One short branded line, e.g.: `Welcome to /last30days - let me get you set up (about 30 seconds).`
|
||||
|
||||
**2. Cookie consent (ask BEFORE reading anything).** First check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env` (e.g. granted in a prior Claude Code session); if so, skip this prompt and run `setup --allow-browser-cookies` directly. Otherwise ask. Example: `I can read your browser cookies (Firefox/Safari) to unlock X/Twitter and other logged-in sources. Want me to? (yes / no)` **Wait for the answer.**
|
||||
- On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --allow-browser-cookies` (and append `BROWSER_CONSENT=true` to `.env` after it completes). Extracts cookies (Firefox/Safari, never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser was explicitly configured) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; activates only when on the agent subprocess PATH, typically `$HOME/.local/bin`; reports honestly if off-PATH; recommend-only if `npx` is unavailable).
|
||||
- On **no** → run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. Skips all cookie reads; still installs yt-dlp and Digg, still writes `SETUP_COMPLETE`.
|
||||
|
||||
**3. Full Disk Access remediation (macOS only).** After `setup`, inspect stderr. If it contains `Permission denied reading Cookies.binarycookies` on macOS, surface: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry. If skipped, continue.
|
||||
|
||||
**4. ScrapeCreators signup offer (every first run, consent BEFORE launching the browser).** Explain it grants 10,000 free calls that unlock TikTok, Instagram, YouTube comments, plus a Reddit backup and a YouTube transcript fallback, and that it opens a GitHub authorization page. Ask, e.g.: `Want to unlock TikTok, Instagram, and more? I can sign you up for ScrapeCreators with GitHub (10,000 free calls) - it opens a browser to authorize. (yes / no)` **Wait for the answer.**
|
||||
- On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --github`. A browser window opens; the user authorizes with the code shown. On success the engine persists the key automatically and returns `"persisted": true` with a MASKED `api_key` (never ask for or echo the raw key). Confirm the paid sources are active.
|
||||
- On **success but `"persisted": false`** (auth completed yet the key write failed) → do NOT claim sources are active. Tell the user signup worked but saving failed, and have them add `SCRAPECREATORS_API_KEY=<key>` to `~/.config/last30days/.env` manually (the raw key is masked in output, so re-run `setup --github` or retrieve it from scrapecreators.com to get the value).
|
||||
- On **timeout / denied** → tell the user it didn't complete and offer to retry or skip.
|
||||
- On **no** → note they can run it later by asking to set up ScrapeCreators, then continue.
|
||||
|
||||
**5. Complete.** Once `SETUP_COMPLETE=true` is written, briefly confirm which sources are now active (read the `setup --github` JSON `persisted` field, or re-run safe `--diagnose`) and proceed to research. For Codex desktop, Cursor, Gemini CLI, and raw folder-mode hosts, hidden `.claude/last30days.env` project config is ignored unless `LAST30DAYS_TRUST_PROJECT_CONFIG=1` is set from the process environment or global config; do not tell the user a project file is active unless diagnose reports it as the config source.
|
||||
|
||||
---
|
||||
|
||||
### Manual Setup Guide
|
||||
|
||||
Shown when a Claude Code user picks "Manual setup", or for anyone who wants to configure by hand. Present as plain text (not blockquoted).
|
||||
|
||||
The magic of /last30days is Reddit comments + X posts together - and both are free. Add these to `~/.config/last30days/.env`:
|
||||
|
||||
**X/Twitter (pick one - the most important source):**
|
||||
- `FROM_BROWSER=auto` - free. Reads your x.com login cookies live at search time (Firefox/Safari, never saved to disk).
|
||||
- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers.
|
||||
- `XQUIK_API_KEY=xxx` - keyless-style X via Xquik.
|
||||
- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com → F12 → Application → Cookies).
|
||||
|
||||
**Reddit (free, works out of the box):**
|
||||
- Public JSON gives threads + top comments with upvote counts. No setup required.
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - optional backup if public Reddit gets rate-limited.
|
||||
|
||||
**YouTube (free, open source):**
|
||||
- Run `brew install yt-dlp` (or `pip install yt-dlp`) - enables YouTube search + transcripts.
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - optional server-side transcript fallback, used only when yt-dlp is rate-limited/bot-gated.
|
||||
|
||||
**Digg (free, keyless):**
|
||||
- Run `npx @mvanhorn/printing-press-library install digg --cli-only` - installs the Digg CLI for trending news, GitHub stars, and pipeline feeds. Activates when `digg-pp-cli` is on your PATH (typically `$HOME/.local/bin`).
|
||||
|
||||
**GitHub Issues/PRs (free, no key needed):**
|
||||
- If the `gh` CLI is installed and authed (`brew install gh && gh auth login`), GitHub search is automatic. No API key required.
|
||||
|
||||
**Bonus: TikTok, Instagram, YouTube comments (ScrapeCreators):**
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com.
|
||||
- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the popular ones. (Threads and Pinterest are also available via `INCLUDE_SOURCES=threads,pinterest` for power users.)
|
||||
|
||||
**Other optional sources (add anytime):**
|
||||
- `PERPLEXITY_API_KEY=xxx` (or `OPENROUTER_API_KEY=xxx`) - AI-synthesized research with citations; set `INCLUDE_SOURCES=perplexity`.
|
||||
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password).
|
||||
- `BRAVE_API_KEY=xxx` or `EXA_API_KEY=xxx` - web search backends.
|
||||
|
||||
**CRITICAL: NEVER overwrite an existing `.env`.** Before writing ANY key:
|
||||
1. Check if the file exists: `test -f ~/.config/last30days/.env`
|
||||
2. If it exists, READ it, then APPEND only missing keys with `>>` (double redirect).
|
||||
3. NEVER use `>` (single redirect) - it destroys existing content.
|
||||
4. If it doesn't exist: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env`
|
||||
|
||||
Always add this last line: `SETUP_COMPLETE=true`. Then proceed to research.
|
||||
|
||||
The setup wizard's mechanical work lives in a Python module so it runs across all hosts (Claude Code, Codex, Cursor, etc.) while you drive the consent conversation above. The common-case (already set up) path through this file stays short.
|
||||
|
||||
---
|
||||
|
||||
@@ -671,8 +856,8 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
@@ -1011,8 +1196,8 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
@@ -1793,7 +1978,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (10,000 free calls, then PAYG)
|
||||
- Optionally sends search queries to Brave Search API, Parallel AI API, Perplexity API (`api.perplexity.ai`), or OpenRouter API for web search / synthesis
|
||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||
- Stores research findings in local SQLite database (watchlist mode only)
|
||||
@@ -1806,7 +1991,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Does not log, cache, or write API keys to output files
|
||||
- Does not send data to any endpoint not listed above
|
||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||
|
||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||
|
||||
@@ -188,7 +188,15 @@ def generate_weekly() -> dict:
|
||||
"this_week_engagement": this_engagement,
|
||||
"last_week_engagement": last_engagement,
|
||||
"engagement_change_pct": round(engagement_change, 1),
|
||||
"top_findings": this_week[:5], # Top 5 by engagement (already sorted)
|
||||
# get_new_findings returns first_seen DESC, so sort by engagement
|
||||
# before slicing — otherwise the digest headlines the most recent
|
||||
# items, not the highest-engagement ones (the daily path keys on
|
||||
# engagement too).
|
||||
"top_findings": sorted(
|
||||
this_week,
|
||||
key=lambda f: f.get("engagement_score", 0),
|
||||
reverse=True,
|
||||
)[:5],
|
||||
})
|
||||
|
||||
result = {
|
||||
|
||||
@@ -282,12 +282,30 @@ def get_judgments(
|
||||
) -> dict[str, int]:
|
||||
cache_file = output_dir / "judgments" / f"{slug}.json"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale_cache = False
|
||||
if cache_file.exists():
|
||||
payload = json.loads(cache_file.read_text())
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
# The cache key is the topic slug alone, but judgments are model-
|
||||
# specific. Only reuse the cache when it was produced by the same judge
|
||||
# model; otherwise re-judge, so a --judge-model change cannot return
|
||||
# stale grades that silently skew precision@k / nDCG. Caches written
|
||||
# before judge_model was recorded miss here and get refreshed once.
|
||||
if payload.get("judge_model") == judge_model:
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
stale_cache = True
|
||||
if not gemini_api_key or not items:
|
||||
if stale_cache:
|
||||
# Discarded a different-model cache but can't re-judge. Returning {}
|
||||
# scores every item as ungraded (zero precision@k / nDCG); say so
|
||||
# rather than letting the run report silently wrong numbers.
|
||||
sys.stderr.write(
|
||||
f"[Eval] Cached judgments for {slug!r} were graded by a different "
|
||||
f"judge model and no Gemini API key is set to re-judge; returning "
|
||||
f"no grades (metrics for this topic will be zero).\n"
|
||||
)
|
||||
return {}
|
||||
payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items))
|
||||
payload["judge_model"] = judge_model
|
||||
cache_file.write_text(json.dumps(payload, indent=2))
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
|
||||
|
||||
@@ -24,10 +24,15 @@ def ensure_supported_python(version_info: tuple[int, int, int] | object | None =
|
||||
major, minor, micro = tuple(version_info[:3])
|
||||
if (major, minor) >= MIN_PYTHON:
|
||||
return
|
||||
req = f"{MIN_PYTHON[0]}.{MIN_PYTHON[1]}"
|
||||
sys.stderr.write(
|
||||
"last30days v3 requires Python 3.12+.\n"
|
||||
f"last30days v3 requires Python {req}+.\n"
|
||||
f"Detected Python {major}.{minor}.{micro}.\n"
|
||||
"Install and use python3.12 or python3.13, then rerun this command.\n"
|
||||
f"Install with:\n"
|
||||
f" Mac: brew install python@{req}\n"
|
||||
f" Windows: winget install Python.Python.{req}\n"
|
||||
f" Linux: sudo apt install python{req} (or pyenv install {req})\n"
|
||||
f"Then rerun: python{req} <path-to-script> setup\n"
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
@@ -288,6 +293,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--debug", action="store_true", help="Enable HTTP debug logging")
|
||||
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||
parser.add_argument("--no-browser-cookies", action="store_true",
|
||||
help="Disable browser-cookie extraction even when FROM_BROWSER is configured")
|
||||
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||
parser.add_argument("--output", help="Optional exact file path for saving the rendered output")
|
||||
parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output")
|
||||
@@ -613,7 +620,7 @@ def _write_last_run(topic: str, report: "schema.Report") -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _propagate_config_to_environ() -> None:
|
||||
def _propagate_config_to_environ(config: dict[str, object]) -> None:
|
||||
"""Push relevant env keys to os.environ so provider modules can read them.
|
||||
|
||||
The env.get_config() function reads from a .env file, but providers.py
|
||||
@@ -621,17 +628,71 @@ def _propagate_config_to_environ() -> None:
|
||||
XAI_BASE_URL overrides are silently ignored. This is a no-op for
|
||||
keys that are already set in process env.
|
||||
"""
|
||||
try:
|
||||
config = env.get_config()
|
||||
except Exception:
|
||||
return
|
||||
for key in ("OPENAI_BASE_URL", "XAI_BASE_URL"):
|
||||
val = config.get(key)
|
||||
if val and not os.environ.get(key):
|
||||
os.environ[key] = val
|
||||
|
||||
|
||||
_propagate_config_to_environ()
|
||||
def _setup_allows_browser_cookies(args: argparse.Namespace, extra_argv: list[str]) -> bool:
|
||||
return (
|
||||
not args.no_browser_cookies
|
||||
and not args.diagnose
|
||||
and "--allow-browser-cookies" in extra_argv
|
||||
)
|
||||
|
||||
|
||||
SETUP_PASSTHROUGH_FLAGS = {
|
||||
"--allow-browser-cookies",
|
||||
"--device-auth",
|
||||
"--github",
|
||||
"--openclaw",
|
||||
}
|
||||
|
||||
SKILL_ONLY_FLAGS = {
|
||||
"--agent",
|
||||
}
|
||||
|
||||
|
||||
def _validate_extra_argv(parser: argparse.ArgumentParser, topic: str, extra_argv: list[str]) -> None:
|
||||
if not extra_argv:
|
||||
return
|
||||
if topic.lower() == "setup":
|
||||
unsupported = [arg for arg in extra_argv if arg not in SETUP_PASSTHROUGH_FLAGS]
|
||||
if unsupported:
|
||||
parser.error(
|
||||
"unsupported setup argument(s): "
|
||||
+ ", ".join(unsupported)
|
||||
+ f"; supported setup passthrough flags are {', '.join(sorted(SETUP_PASSTHROUGH_FLAGS))}"
|
||||
)
|
||||
return
|
||||
skill_only = [arg for arg in extra_argv if arg in SKILL_ONLY_FLAGS]
|
||||
other_unknown = [arg for arg in extra_argv if arg not in SKILL_ONLY_FLAGS]
|
||||
if skill_only:
|
||||
message = (
|
||||
"unsupported Python CLI argument(s): "
|
||||
+ ", ".join(skill_only)
|
||||
+ "; these are skill arguments and must not be forwarded to scripts/last30days.py"
|
||||
)
|
||||
if other_unknown:
|
||||
message += "; also unsupported: " + ", ".join(other_unknown)
|
||||
parser.error(message)
|
||||
parser.error("unsupported Python CLI argument(s): " + ", ".join(extra_argv))
|
||||
|
||||
|
||||
def _config_policy_for_args(args: argparse.Namespace, topic: str, extra_argv: list[str]) -> env.ConfigLoadPolicy:
|
||||
if args.no_browser_cookies:
|
||||
browser_mode = "off"
|
||||
elif args.diagnose:
|
||||
browser_mode = "plan_only"
|
||||
elif topic.lower() == "setup":
|
||||
browser_mode = "read" if _setup_allows_browser_cookies(args, extra_argv) else "off"
|
||||
else:
|
||||
browser_mode = "read"
|
||||
return env.ConfigLoadPolicy(
|
||||
browser_cookies=browser_mode,
|
||||
inspect_ignored_project_config=args.diagnose,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -642,7 +703,10 @@ def main() -> int:
|
||||
if args.debug:
|
||||
os.environ["LAST30DAYS_DEBUG"] = "1"
|
||||
|
||||
config = env.get_config()
|
||||
topic = " ".join(args.topic).strip()
|
||||
_validate_extra_argv(parser, topic, extra_argv)
|
||||
config = env.get_config(policy=_config_policy_for_args(args, topic, extra_argv))
|
||||
_propagate_config_to_environ(config)
|
||||
|
||||
# Env-var fallback for --save-dir, mirroring the LAST30DAYS_STORE pattern below.
|
||||
# Uses `is None` / `is not None` checks (not truthy `or`) at every layer so that
|
||||
@@ -661,30 +725,40 @@ def main() -> int:
|
||||
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"]
|
||||
|
||||
# Handle setup subcommand
|
||||
topic = " ".join(args.topic).strip()
|
||||
if topic.lower() == "setup":
|
||||
from lib import setup_wizard
|
||||
if "--openclaw" in extra_argv:
|
||||
results = setup_wizard.run_openclaw_setup(config)
|
||||
print(json.dumps(results))
|
||||
return 0
|
||||
if "--github" in extra_argv:
|
||||
results = setup_wizard.run_github_auth()
|
||||
print(json.dumps(results))
|
||||
return 0
|
||||
if "--device-auth" in extra_argv:
|
||||
results = setup_wizard.run_full_device_auth()
|
||||
if "--github" in extra_argv or "--device-auth" in extra_argv:
|
||||
if "--github" in extra_argv:
|
||||
results = setup_wizard.run_github_auth()
|
||||
else:
|
||||
results = setup_wizard.run_full_device_auth()
|
||||
# Persist the returned key so the paid sources activate on the next
|
||||
# run, and mask it in stdout so the secret never lands in the host
|
||||
# model's captured Bash output.
|
||||
api_key = results.get("api_key")
|
||||
if results.get("status") == "success" and api_key:
|
||||
results["persisted"] = setup_wizard.write_api_key(env.CONFIG_FILE, api_key)
|
||||
results["api_key"] = setup_wizard.mask_api_key(api_key)
|
||||
else:
|
||||
results["persisted"] = False
|
||||
print(json.dumps(results))
|
||||
return 0
|
||||
sys.stderr.write("Running auto-setup...\n")
|
||||
results = setup_wizard.run_auto_setup(config)
|
||||
results = setup_wizard.run_auto_setup(
|
||||
config,
|
||||
allow_browser_cookies=_setup_allows_browser_cookies(args, extra_argv),
|
||||
)
|
||||
# Persist FROM_BROWSER only when every service's cookies came from the
|
||||
# SAME single browser — then we can fast-path future runs to it. If
|
||||
# different services matched different browsers, or none matched, leave
|
||||
# FROM_BROWSER unset: the safe default (Firefox/Safari) then covers all
|
||||
# of them with no Keychain prompt. We deliberately do NOT pin "auto"
|
||||
# here (it would re-probe Chrome and re-trigger the prompt) nor a single
|
||||
# browser (it would silently skip the service that used the other one).
|
||||
# FROM_BROWSER unset so the safe default remains no browser-cookie
|
||||
# reads. We deliberately do NOT pin "auto" here (it would re-probe
|
||||
# Chrome and re-trigger the prompt) nor a single browser (it would
|
||||
# silently skip the service that used the other one).
|
||||
found_browsers = set(results.get("cookies_found", {}).values())
|
||||
from_browser = found_browsers.pop() if len(found_browsers) == 1 else None
|
||||
setup_wizard.write_setup_config(env.CONFIG_FILE, from_browser=from_browser)
|
||||
@@ -693,7 +767,7 @@ def main() -> int:
|
||||
return 0
|
||||
|
||||
requested_sources = resolve_requested_sources(args.search, config)
|
||||
diag = pipeline.diagnose(config, requested_sources)
|
||||
diag = pipeline.diagnose(config, requested_sources, safe=args.diagnose)
|
||||
|
||||
if args.diagnose:
|
||||
print(json.dumps(diag, indent=2, sort_keys=True))
|
||||
@@ -742,6 +816,10 @@ def main() -> int:
|
||||
external_plan = _json.loads(plan_str)
|
||||
except _json.JSONDecodeError as exc:
|
||||
sys.stderr.write(f"[Planner] Invalid --plan JSON: {exc}\n")
|
||||
# Fail fast instead of silently dropping to the internal planner
|
||||
# and burning a paid run the user did not ask for. Mirrors the
|
||||
# --plan file-read branch above and parse_competitors_plan.
|
||||
raise SystemExit(2)
|
||||
|
||||
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||
|
||||
@@ -157,7 +157,12 @@ def _query_cookies_db(
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
# mkstemp creates the file 0600. copy2 would copy the source's mode
|
||||
# (Firefox cookies.sqlite is commonly 0644, looser on WSL /mnt/c) onto
|
||||
# the temp file, leaving live session secrets world-readable in shared
|
||||
# /tmp until the chmod below runs. copyfile writes content only and
|
||||
# leaves the 0600 perms intact, closing that window.
|
||||
shutil.copyfile(str(db_path), tmp_path)
|
||||
_lock_temp_cookie_copy(tmp_path)
|
||||
|
||||
conn = sqlite3.connect(tmp_path)
|
||||
|
||||
@@ -77,6 +77,38 @@ class OpenAIAuth:
|
||||
codex_auth_file: str
|
||||
|
||||
|
||||
BrowserCookieMode = Literal["off", "read", "plan_only"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConfigLoadPolicy:
|
||||
"""Local-read gates for configuration loading.
|
||||
|
||||
Bare library calls use the safe default: no browser-cookie extraction and no
|
||||
project-scoped config. CLI entry points can opt into narrower behavior after
|
||||
parsing command intent.
|
||||
"""
|
||||
|
||||
browser_cookies: BrowserCookieMode = "off"
|
||||
allow_project_config: bool = False
|
||||
inspect_ignored_project_config: bool = False
|
||||
|
||||
|
||||
def _truthy(value: Any) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _project_config_trusted(policy: ConfigLoadPolicy, file_env: dict[str, Any]) -> bool:
|
||||
if policy.allow_project_config:
|
||||
return True
|
||||
process_value = os.environ.get("LAST30DAYS_TRUST_PROJECT_CONFIG")
|
||||
if process_value is not None:
|
||||
return _truthy(process_value)
|
||||
return _truthy(file_env.get("LAST30DAYS_TRUST_PROJECT_CONFIG"))
|
||||
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
||||
if os.name == "nt":
|
||||
@@ -315,34 +347,46 @@ def _find_project_env() -> Path | None:
|
||||
"""Find per-project .env by walking up from cwd.
|
||||
|
||||
Searches for .claude/last30days.env in each parent directory,
|
||||
stopping at the user's home directory or filesystem root.
|
||||
stopping at the git root, user's home directory, or filesystem root.
|
||||
"""
|
||||
cwd = Path.cwd()
|
||||
for parent in [cwd, *cwd.parents]:
|
||||
candidate = parent / '.claude' / 'last30days.env'
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
if (parent / ".git").exists():
|
||||
break
|
||||
# Stop at filesystem root or home
|
||||
if parent == Path.home() or parent == parent.parent:
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:
|
||||
"""Load configuration from multiple sources.
|
||||
|
||||
Priority (highest wins):
|
||||
1. Environment variables (os.environ)
|
||||
2. .claude/last30days.env (per-project config)
|
||||
2. Trusted .claude/last30days.env (per-project config)
|
||||
3. ~/.config/last30days/.env (global config)
|
||||
4. macOS Keychain items prefixed ``last30days-`` (Darwin only)
|
||||
"""
|
||||
policy = policy or ConfigLoadPolicy()
|
||||
# Load from global config file
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
|
||||
# Load from per-project config (overrides global)
|
||||
project_env_path = _find_project_env()
|
||||
# Load per-project config only when trust comes from process env, global
|
||||
# user config, or an explicit policy. A project file cannot grant trust to
|
||||
# itself because it is not parsed until after this decision.
|
||||
project_config_trusted = _project_config_trusted(policy, file_env)
|
||||
project_env_path = _find_project_env() if project_config_trusted else None
|
||||
project_env = load_env_file(project_env_path) if project_env_path else {}
|
||||
ignored_project_env_path = None
|
||||
ignored_project_keys: list[str] = []
|
||||
if not project_config_trusted and policy.inspect_ignored_project_config:
|
||||
ignored_project_env_path = _find_project_env()
|
||||
if ignored_project_env_path:
|
||||
ignored_project_keys = sorted(load_env_file(ignored_project_env_path).keys())
|
||||
|
||||
# Merge file sources: project > global
|
||||
merged_env = {**file_env, **project_env}
|
||||
@@ -429,6 +473,7 @@ def get_config() -> dict[str, Any]:
|
||||
# Optional SearXNG instance for the keyless-search fallback rung.
|
||||
('LAST30DAYS_SEARXNG_URL', None),
|
||||
('FROM_BROWSER', None),
|
||||
('LAST30DAYS_TRUST_PROJECT_CONFIG', None),
|
||||
('SETUP_COMPLETE', None),
|
||||
('INCLUDE_SOURCES', ''),
|
||||
('EXCLUDE_SOURCES', ''),
|
||||
@@ -476,13 +521,18 @@ def get_config() -> dict[str, Any]:
|
||||
config['_CONFIG_SOURCE'] = 'pass'
|
||||
else:
|
||||
config['_CONFIG_SOURCE'] = 'env_only'
|
||||
if ignored_project_env_path:
|
||||
config['_IGNORED_PROJECT_CONFIG'] = str(ignored_project_env_path)
|
||||
config['_IGNORED_PROJECT_CONFIG_KEYS'] = ignored_project_keys
|
||||
config['_BROWSER_COOKIE_MODE'] = policy.browser_cookies
|
||||
config['_BROWSER_COOKIE_BROWSERS'] = cookie_extraction_browsers(config)
|
||||
|
||||
# Extract browser credentials if configured
|
||||
browser_creds = extract_browser_credentials(config)
|
||||
for key, value in browser_creds.items():
|
||||
if not config.get(key):
|
||||
config[key] = value
|
||||
config[f"_{key}_SOURCE"] = "browser"
|
||||
if policy.browser_cookies == "read":
|
||||
browser_creds = extract_browser_credentials(config)
|
||||
for key, value in browser_creds.items():
|
||||
if not config.get(key):
|
||||
config[key] = value
|
||||
config[f"_{key}_SOURCE"] = "browser"
|
||||
|
||||
return config
|
||||
|
||||
@@ -508,16 +558,17 @@ COOKIE_DOMAINS: dict[str, dict[str, Any]] = {
|
||||
def cookie_extraction_browsers(config: dict[str, Any]) -> list[str]:
|
||||
"""Browsers to try for cookie extraction, honoring FROM_BROWSER.
|
||||
|
||||
Default (FROM_BROWSER unset): Firefox and Safari only. These read local
|
||||
files silently with no system dialogs. The Chromium family (Chrome, Brave,
|
||||
Edge, Vivaldi, Opera, Arc, Chromium) is skipped because reading their
|
||||
cookies on macOS requires the browser's Safe Storage Keychain key, which
|
||||
triggers a system password prompt that cannot be reliably suppressed. On
|
||||
Windows only Firefox cookie extraction is supported; Chrome and Edge use
|
||||
DPAPI-encrypted cookie stores that are not yet supported.
|
||||
Default (FROM_BROWSER unset): no browser-cookie reads. The Chromium family
|
||||
(Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium) is available only when
|
||||
explicitly selected because reading their cookies on macOS requires the
|
||||
browser's Safe Storage Keychain key, which triggers a system password prompt
|
||||
that cannot be reliably suppressed. On Windows only Firefox cookie
|
||||
extraction is supported; Chrome and Edge use DPAPI-encrypted cookie stores
|
||||
that are not yet supported.
|
||||
|
||||
- ``FROM_BROWSER=<name>`` - a single browser (e.g. ``firefox``, ``brave``,
|
||||
``edge``, ``arc``).
|
||||
- ``FROM_BROWSER=firefox,safari`` - a comma-separated explicit browser list.
|
||||
- ``FROM_BROWSER=auto`` - also try every Chromium browser (user accepts the
|
||||
Keychain dialog when needed).
|
||||
- ``FROM_BROWSER=off`` - returns [] (extraction disabled).
|
||||
@@ -528,14 +579,37 @@ def cookie_extraction_browsers(config: dict[str, Any]) -> list[str]:
|
||||
"""
|
||||
silent_browsers = ["firefox", "safari"]
|
||||
chromium_browsers = ["chrome", "brave", "edge", "vivaldi", "opera", "arc", "chromium"]
|
||||
known_browsers = silent_browsers + chromium_browsers
|
||||
from_browser = (config.get("FROM_BROWSER") or "").strip().lower()
|
||||
if not from_browser:
|
||||
return []
|
||||
if from_browser == "off":
|
||||
return []
|
||||
if from_browser in silent_browsers or from_browser in chromium_browsers:
|
||||
return [from_browser]
|
||||
if from_browser == "auto":
|
||||
return silent_browsers + chromium_browsers
|
||||
return list(silent_browsers)
|
||||
if "," in from_browser:
|
||||
requested = [b.strip() for b in from_browser.split(",") if b.strip()]
|
||||
resolved = [b for b in requested if b in known_browsers]
|
||||
unknown = [b for b in requested if b not in known_browsers]
|
||||
if unknown:
|
||||
sys.stderr.write(
|
||||
"[last30days] WARNING: FROM_BROWSER ignored unrecognized browser(s): "
|
||||
f"{', '.join(unknown)} (known: {', '.join(known_browsers)})\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
return resolved
|
||||
if from_browser in known_browsers:
|
||||
return [from_browser]
|
||||
# Non-empty, not off/auto, not a known browser, not a list: unrecognized.
|
||||
# Warn rather than fail silently so a typo (FROM_BROWSER=chrme) is visible
|
||||
# instead of looking like "no cookies found".
|
||||
sys.stderr.write(
|
||||
f"[last30days] WARNING: FROM_BROWSER='{from_browser}' is not a recognized "
|
||||
f"browser; no cookies will be read (known: {', '.join(known_browsers)}, "
|
||||
"or 'auto'/'off')\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
return []
|
||||
|
||||
|
||||
|
||||
@@ -584,9 +658,11 @@ def get_x_source_with_method(config: dict[str, Any]) -> tuple[str | None, str]:
|
||||
return None, "none"
|
||||
|
||||
|
||||
def config_exists() -> bool:
|
||||
def config_exists(policy: ConfigLoadPolicy | None = None) -> bool:
|
||||
"""Check if any configuration source exists."""
|
||||
if _find_project_env():
|
||||
policy = policy or ConfigLoadPolicy()
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE and CONFIG_FILE.exists() else {}
|
||||
if _project_config_trusted(policy, file_env) and _find_project_env():
|
||||
return True
|
||||
if CONFIG_FILE:
|
||||
return CONFIG_FILE.exists()
|
||||
|
||||
@@ -31,6 +31,9 @@ DEPTH_CONFIG = {
|
||||
"deep": 60,
|
||||
}
|
||||
|
||||
MIN_STORY_POINTS = 2
|
||||
HN_OVERFETCH_MULTIPLIER = 2
|
||||
|
||||
ENRICH_LIMITS = {
|
||||
"quick": 3,
|
||||
"default": 5,
|
||||
@@ -83,6 +86,7 @@ def search_hackernews(
|
||||
Dict with Algolia response (contains 'hits' list).
|
||||
"""
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
fetch_count = count * HN_OVERFETCH_MULTIPLIER
|
||||
from_ts = _date_to_unix(from_date)
|
||||
to_ts = _date_to_unix(to_date) + 86400 # Include the end date
|
||||
|
||||
@@ -97,15 +101,15 @@ def search_hackernews(
|
||||
# `created_at_i` in numericFilters; `points` is NOT in its
|
||||
# `numericAttributesForFiltering`, so a `points>N` clause makes the API
|
||||
# return HTTP 400 ("invalid numeric attribute(points)") and zero stories.
|
||||
# Low-engagement stories are demoted by parse-time relevance scoring
|
||||
# (rank + engagement_boost in parse_hackernews_response) instead.
|
||||
# Low-engagement stories are filtered client-side after overfetching so the
|
||||
# invalid numeric filter is not reintroduced.
|
||||
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
||||
# miss Ask HN/Show HN threads where the topic appears in the body.
|
||||
params = {
|
||||
"query": core_flat,
|
||||
"tags": "story",
|
||||
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts}",
|
||||
"hitsPerPage": str(count),
|
||||
"hitsPerPage": str(fetch_count),
|
||||
}
|
||||
# Algolia defaults to AND across query tokens, so a 4-5 word theme query
|
||||
# matches no stories. Mark all-but-the-first token as optional so Algolia
|
||||
@@ -126,7 +130,17 @@ def search_hackernews(
|
||||
_log(f"Search failed: {e}")
|
||||
return {"hits": [], "error": str(e)}
|
||||
|
||||
hits = response.get("hits", [])
|
||||
raw_hits = response.get("hits", [])
|
||||
qualifying_hits = [
|
||||
hit for hit in raw_hits
|
||||
if (hit.get("points") or 0) > MIN_STORY_POINTS
|
||||
]
|
||||
hits = qualifying_hits[:count]
|
||||
dropped_low_engagement = len(raw_hits) - len(qualifying_hits)
|
||||
if dropped_low_engagement:
|
||||
_log(f"Filtered {dropped_low_engagement}/{len(raw_hits)} low-engagement stories")
|
||||
if len(hits) != len(raw_hits):
|
||||
response = {**response, "hits": hits}
|
||||
_log(f"Found {len(hits)} stories")
|
||||
return response
|
||||
|
||||
|
||||
@@ -96,6 +96,14 @@ def analyze(
|
||||
|
||||
def infer_company_size(items: list[schema.SourceItem], *, topic: str = "") -> str:
|
||||
"""Infer a coarse company-size tier from jobs evidence."""
|
||||
topic_lower = topic.lower()
|
||||
firmographic_text = " ".join(
|
||||
" ".join([
|
||||
str(item.metadata.get("company_size") or ""),
|
||||
topic,
|
||||
])
|
||||
for item in items
|
||||
).lower()
|
||||
text = " ".join(
|
||||
" ".join([
|
||||
item.title,
|
||||
@@ -111,9 +119,9 @@ def infer_company_size(items: list[schema.SourceItem], *, topic: str = "") -> st
|
||||
# never the job-description body - JDs list enterprise customers (e.g.
|
||||
# "trusted by Microsoft, Google"), which would misclassify a startup as
|
||||
# mega-cap and suppress its real signals.
|
||||
if re.search(r"\b(apple|uber|google|microsoft|amazon|meta|netflix)\b", topic.lower()):
|
||||
if re.search(r"\b(apple|uber|google|microsoft|amazon|meta|netflix)\b", topic_lower):
|
||||
return "mega-cap"
|
||||
if count >= 200 or re.search(r"\b(fortune 500|thousands of employees)\b", text):
|
||||
if count >= 200 or re.search(r"\b(fortune 500|thousands of employees)\b", firmographic_text):
|
||||
return "large-enterprise"
|
||||
if count >= 35 or re.search(r"\b(series [cd]|public company)\b", text):
|
||||
return "growth"
|
||||
|
||||
@@ -574,7 +574,15 @@ def _promote_meta_marker(body: str) -> str:
|
||||
Both collapse to ``<div class="meta">TEXT</div>``.
|
||||
"""
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
text = match.group(1).strip()
|
||||
# The marker survives the comment-strip pass, and the markdown reaching
|
||||
# this point can include LLM-synthesized content derived from untrusted
|
||||
# web/social bodies. The escaped-form branches below carry text the
|
||||
# markdown pass already entity-escaped, while the raw-form fallbacks do
|
||||
# not — so normalize with unescape, then escape exactly once. A crafted
|
||||
# `<!-- META: <img src=x onerror=...> -->` thus cannot render as live
|
||||
# markup in the saved, shareable HTML artifact, and legitimate
|
||||
# date/source-name markers render unchanged.
|
||||
text = html.escape(html.unescape(match.group(1).strip()))
|
||||
return f'<div class="meta">{text}</div>'
|
||||
|
||||
# Escaped form (most common after markdown conversion)
|
||||
|
||||
@@ -162,10 +162,15 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
return available
|
||||
|
||||
|
||||
def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None) -> dict[str, Any]:
|
||||
def diagnose(
|
||||
config: dict[str, Any],
|
||||
requested_sources: list[str] | None = None,
|
||||
*,
|
||||
safe: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
requested_sources = normalize_requested_sources(requested_sources)
|
||||
google_key = _google_key(config)
|
||||
x_status = env.get_x_source_status(config, probe=True)
|
||||
x_status = env.get_x_source_status(config, probe=not safe)
|
||||
native_web_backend = None
|
||||
if config.get("BRAVE_API_KEY"):
|
||||
native_web_backend = "brave"
|
||||
@@ -185,6 +190,34 @@ def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None)
|
||||
reasoning_provider_available = any(
|
||||
providers_status[name] for name in ("google", "openai", "xai", "openrouter")
|
||||
)
|
||||
external_commands = {
|
||||
"yt-dlp": bool(which("yt-dlp")),
|
||||
"digg-pp-cli": bool(which("digg-pp-cli")),
|
||||
"gh": bool(which("gh")),
|
||||
}
|
||||
credential_destinations = {
|
||||
"global_env": str(env.CONFIG_FILE) if env.CONFIG_FILE else None,
|
||||
}
|
||||
browser_cookies = {
|
||||
"mode": config.get("_BROWSER_COOKIE_MODE", "off"),
|
||||
"browsers": list(config.get("_BROWSER_COOKIE_BROWSERS") or []),
|
||||
"reads_values": False if safe else config.get("_BROWSER_COOKIE_MODE") == "read",
|
||||
}
|
||||
ignored_project_keys = list(config.get("_IGNORED_PROJECT_CONFIG_KEYS") or [])
|
||||
endpoint_override_keys = {
|
||||
"BSKY_SEARCH_HOST",
|
||||
"LAST30DAYS_SEARXNG_URL",
|
||||
"LAST30DAYS_YOUTUBE_SSH_HOST",
|
||||
"OPENAI_BASE_URL",
|
||||
"XAI_BASE_URL",
|
||||
"XIAOHONGSHU_API_BASE",
|
||||
}
|
||||
ignored_endpoint_overrides = [
|
||||
key for key in ignored_project_keys if key in endpoint_override_keys
|
||||
]
|
||||
local_writes: list[dict[str, str]] = []
|
||||
if config.get("LAST30DAYS_MEMORY_DIR"):
|
||||
local_writes.append({"kind": "report", "path": str(config.get("LAST30DAYS_MEMORY_DIR"))})
|
||||
return {
|
||||
"providers": providers_status,
|
||||
"local_mode": not reasoning_provider_available,
|
||||
@@ -201,6 +234,15 @@ def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None)
|
||||
"has_scrapecreators": bool(config.get("SCRAPECREATORS_API_KEY")),
|
||||
"has_github": bool(config.get("GITHUB_TOKEN") or which("gh")),
|
||||
"available_sources": available_sources(config, requested_sources),
|
||||
"safe": safe,
|
||||
"config_source": config.get("_CONFIG_SOURCE"),
|
||||
"ignored_project_config": config.get("_IGNORED_PROJECT_CONFIG"),
|
||||
"ignored_project_config_keys": ignored_project_keys,
|
||||
"ignored_endpoint_overrides": ignored_endpoint_overrides,
|
||||
"browser_cookies": browser_cookies,
|
||||
"external_commands": external_commands,
|
||||
"credential_destinations": credential_destinations,
|
||||
"local_writes": local_writes,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -471,8 +471,9 @@ def _render_pre_research_warning(report: schema.Report) -> list[str]:
|
||||
"- Subreddit-specific threads on dedicated communities",
|
||||
"- Topic-specific TikTok and Instagram creators",
|
||||
"",
|
||||
"To fix: in a fresh Claude Code window, run `ToolSearch select:WebSearch` first,",
|
||||
f"then rerun `/last30days {report.topic}`. The skill will resolve handles",
|
||||
"To fix: in a fresh agent session (Claude Code, Codex, Hermes, Gemini, or any runtime),",
|
||||
"ensure your runtime's web-search tool is active, then",
|
||||
f"rerun `/last30days {report.topic}`. The skill will resolve handles",
|
||||
"and communities before calling the engine this time, producing richer results.",
|
||||
"",
|
||||
"If this topic really is abstract (e.g. \"AI regulation\") and doesn't need",
|
||||
|
||||
@@ -28,12 +28,12 @@ def is_first_run(config: Dict[str, Any]) -> bool:
|
||||
return not config.get("SETUP_COMPLETE")
|
||||
|
||||
|
||||
def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = False) -> Dict[str, Any]:
|
||||
"""Perform the auto-setup actions.
|
||||
|
||||
- Runs cookie extraction for all registered domains, trying the browsers
|
||||
from ``env.cookie_extraction_browsers()`` (honors ``FROM_BROWSER``;
|
||||
defaults to Firefox/Safari, so no Chrome Keychain prompt)
|
||||
- Optionally runs cookie extraction for all registered domains, trying the
|
||||
browsers from ``env.cookie_extraction_browsers()``. Browser reads are off
|
||||
unless ``allow_browser_cookies`` is true.
|
||||
- Checks if yt-dlp is installed
|
||||
- Best-effort install of digg-pp-cli (Printing Press library)
|
||||
|
||||
@@ -49,31 +49,31 @@ def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
digg_stderr: present when digg_action is install_failed
|
||||
digg_path: present when digg_action is installed_off_path (binary on disk, not on PATH)
|
||||
"""
|
||||
from . import cookie_extract
|
||||
from .env import COOKIE_DOMAINS, cookie_extraction_browsers
|
||||
|
||||
cookies_found: Dict[str, str] = {}
|
||||
|
||||
# Honor FROM_BROWSER and default to the silent browsers (Firefox/Safari).
|
||||
# Using "auto" here used to probe Chrome unconditionally, triggering a
|
||||
# "Chrome Safe Storage" Keychain prompt on first run that the steady-state
|
||||
# path deliberately avoids. Chrome is now opt-in via FROM_BROWSER=chrome|auto.
|
||||
# An empty list (FROM_BROWSER=off) makes the inner loop a no-op.
|
||||
browsers = cookie_extraction_browsers(config)
|
||||
if allow_browser_cookies:
|
||||
from . import cookie_extract
|
||||
|
||||
for source_name, spec in COOKIE_DOMAINS.items():
|
||||
domain = spec["domain"]
|
||||
cookie_names = spec["cookies"]
|
||||
cookie_config = dict(config)
|
||||
if not (cookie_config.get("FROM_BROWSER") or "").strip():
|
||||
cookie_config["FROM_BROWSER"] = "firefox,safari"
|
||||
browsers = cookie_extraction_browsers(cookie_config)
|
||||
|
||||
for browser in browsers:
|
||||
try:
|
||||
result = cookie_extract.extract_cookies_with_source(browser, domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("Cookie extraction failed for %s via %s: %s", source_name, browser, exc)
|
||||
continue
|
||||
if result is not None and result[0]:
|
||||
cookies_found[source_name] = result[1]
|
||||
break # Found cookies for this service, stop trying browsers
|
||||
for source_name, spec in COOKIE_DOMAINS.items():
|
||||
domain = spec["domain"]
|
||||
cookie_names = spec["cookies"]
|
||||
|
||||
for browser in browsers:
|
||||
try:
|
||||
result = cookie_extract.extract_cookies_with_source(browser, domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("Cookie extraction failed for %s via %s: %s", source_name, browser, exc)
|
||||
continue
|
||||
if result is not None and result[0]:
|
||||
cookies_found[source_name] = result[1]
|
||||
break # Found cookies for this service, stop trying browsers
|
||||
|
||||
# Check yt-dlp availability and install via Homebrew if missing
|
||||
ytdlp_action: str
|
||||
@@ -330,6 +330,63 @@ def write_setup_config(env_path: Path, from_browser: str | None = None) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def write_api_key(env_path: Path, api_key: str, key_name: str = "SCRAPECREATORS_API_KEY") -> bool:
|
||||
"""Append an API key to the .env file as a 0o600 secret.
|
||||
|
||||
Reuses the same secret-safe write path as ``write_setup_config`` so the
|
||||
value lands with restrictive permissions and round-trips through
|
||||
``env.load_env_file``. Idempotent: if ``key_name`` is already present in
|
||||
the file, nothing is written and the existing value is preserved (we never
|
||||
clobber a key the user may have set by hand).
|
||||
|
||||
Args:
|
||||
env_path: Path to the .env file (e.g. ~/.config/last30days/.env).
|
||||
api_key: The raw key value to persist.
|
||||
key_name: The env var name to write (default SCRAPECREATORS_API_KEY).
|
||||
|
||||
Returns:
|
||||
True if the key was written or already present, False on error or when
|
||||
``api_key`` is empty.
|
||||
"""
|
||||
if not api_key:
|
||||
return False
|
||||
try:
|
||||
env_path = Path(env_path)
|
||||
env_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
existing_content = ""
|
||||
if env_path.exists():
|
||||
existing_content = env_path.read_text(encoding="utf-8")
|
||||
for line in existing_content.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith("#") and "=" in stripped:
|
||||
if stripped.split("=", 1)[0].strip() == key_name:
|
||||
return True # Already configured; do not duplicate
|
||||
|
||||
line = f"{key_name}={_format_env_value(api_key)}\n"
|
||||
with _open_secret_append(env_path) as f:
|
||||
if existing_content and not existing_content.endswith("\n"):
|
||||
f.write("\n")
|
||||
f.write(line)
|
||||
|
||||
return True
|
||||
|
||||
except OSError as exc:
|
||||
logger.error("Failed to write API key to %s: %s", env_path, exc)
|
||||
return False
|
||||
|
||||
|
||||
def mask_api_key(api_key: str) -> str:
|
||||
"""Return a non-secret display form of an API key (prefix + last 4).
|
||||
|
||||
Used so the key never appears verbatim in stdout the host model captures.
|
||||
Short or empty keys collapse to a fixed placeholder.
|
||||
"""
|
||||
if not api_key or len(api_key) <= 8:
|
||||
return "sc_…"
|
||||
return f"{api_key[:3]}…{api_key[-4:]}"
|
||||
|
||||
|
||||
def get_setup_status_text(results: Dict[str, Any]) -> str:
|
||||
"""Return a human-readable summary of auto-setup results.
|
||||
|
||||
|
||||
@@ -103,7 +103,10 @@ def run_with_timeout(
|
||||
proc.kill()
|
||||
except (ProcessLookupError, PermissionError, OSError, AttributeError):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass # process unkillable (e.g. D-state); leave as zombie
|
||||
raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s")
|
||||
|
||||
return SubprocResult(
|
||||
|
||||
@@ -200,7 +200,7 @@ Just start with "last30" and talk to me like normal.
|
||||
|
||||
# Shorter promo for single missing key
|
||||
PROMO_SINGLE_KEY = {
|
||||
"reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 100 free credits, no CC - scrapecreators.com\n",
|
||||
"reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 10,000 free calls, no CC - scrapecreators.com\n",
|
||||
"x": "\n💡 Unlock X: log into x.com in your browser, then re-run. "
|
||||
"Firefox works on all platforms. Safari works on macOS (detected automatically). "
|
||||
"Chrome, Brave, Edge, Arc, Vivaldi, Opera, or Chromium on macOS require "
|
||||
@@ -214,7 +214,7 @@ BIRD_AUTH_HELP = f"""
|
||||
{Colors.YELLOW}Bird authentication failed.{Colors.RESET}
|
||||
|
||||
To fix this:
|
||||
1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env or .claude/last30days.env
|
||||
1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env, or to trusted .claude/last30days.env with LAST30DAYS_TRUST_PROJECT_CONFIG=1
|
||||
2. Or set XAI_API_KEY for the xAI fallback backend
|
||||
"""
|
||||
|
||||
@@ -222,7 +222,7 @@ BIRD_AUTH_HELP_PLAIN = """
|
||||
Bird authentication failed.
|
||||
|
||||
To fix this:
|
||||
1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env or .claude/last30days.env
|
||||
1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env, or to trusted .claude/last30days.env with LAST30DAYS_TRUST_PROJECT_CONFIG=1
|
||||
2. Or set XAI_API_KEY for the xAI fallback backend
|
||||
"""
|
||||
|
||||
|
||||
@@ -193,6 +193,10 @@ def _connect(db_path: Optional[Path] = None) -> sqlite3.Connection:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
# WAL lets readers coexist with one writer, but two writers (cron + user)
|
||||
# still contend for the write lock. Default busy_timeout is 0, so the loser
|
||||
# raises "database is locked" instantly; wait instead.
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
return conn
|
||||
|
||||
|
||||
@@ -452,16 +456,41 @@ def store_findings(
|
||||
update_rows,
|
||||
)
|
||||
if insert_rows:
|
||||
# source_url is UNIQUE. The SELECT above is not atomic with this
|
||||
# write, so a concurrent run (cron + user) can insert the same URL
|
||||
# between our read and write. Upsert on conflict instead of letting
|
||||
# IntegrityError abort the whole batch and lose every finding.
|
||||
conn.executemany(
|
||||
"""INSERT INTO findings
|
||||
(run_id, topic_id, source, source_url, source_title,
|
||||
author, content, summary, engagement_score, relevance_score)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(source_url) DO UPDATE SET
|
||||
last_seen = datetime('now'),
|
||||
sighting_count = sighting_count + 1,
|
||||
engagement_score = max(
|
||||
engagement_score, excluded.engagement_score),
|
||||
run_id = excluded.run_id""",
|
||||
insert_rows,
|
||||
)
|
||||
|
||||
new_count = len(insert_rows)
|
||||
updated_count = len(update_rows)
|
||||
if insert_rows:
|
||||
# A row whose URL was inserted by a concurrent run between our SELECT
|
||||
# and the upsert resolves via ON CONFLICT (an update, not a new row),
|
||||
# bumping its sighting_count above 1. Re-derive the split so
|
||||
# research_runs.findings_new isn't inflated by conflict-resolved rows
|
||||
# (source_url is field index 3 in each insert tuple).
|
||||
inserted_urls = [row[3] for row in insert_rows]
|
||||
placeholders = ",".join("?" for _ in inserted_urls)
|
||||
conflicted = conn.execute(
|
||||
f"SELECT COUNT(*) FROM findings "
|
||||
f"WHERE source_url IN ({placeholders}) AND sighting_count > 1",
|
||||
inserted_urls,
|
||||
).fetchone()[0]
|
||||
new_count -= conflicted
|
||||
updated_count += conflicted
|
||||
_record_sightings(conn, run_id, topic_id, with_urls, existing_by_url)
|
||||
conn.execute(
|
||||
"UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?",
|
||||
|
||||
@@ -8,6 +8,7 @@ import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
@@ -28,10 +29,22 @@ def _deliver_findings(topic_name: str, counts: dict) -> None:
|
||||
mode = store.get_setting("delivery_mode", "announce")
|
||||
message = _format_delivery_message(topic_name, counts, mode)
|
||||
|
||||
# Require https before routing. The old "hooks.slack.com" in channel
|
||||
# substring test ran before any scheme check, so a channel like
|
||||
# http://evil.example/hooks.slack.com was treated as Slack and POSTed in
|
||||
# cleartext to the wrong host. Match Slack on the exact hostname instead.
|
||||
parsed = urllib.parse.urlparse(channel)
|
||||
if parsed.scheme != "https":
|
||||
print(
|
||||
f"Delivery skipped: delivery_channel must be an https:// URL, got {channel!r}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
if "hooks.slack.com" in channel:
|
||||
if parsed.hostname == "hooks.slack.com":
|
||||
_send_slack_webhook(channel, message)
|
||||
elif channel.startswith("https://"):
|
||||
else:
|
||||
_send_generic_webhook(channel, message)
|
||||
except Exception as e:
|
||||
# Don't fail the research run if delivery fails
|
||||
@@ -171,6 +184,11 @@ def _run_topic(topic: dict) -> dict:
|
||||
"--quick",
|
||||
"--lookback-days",
|
||||
"90",
|
||||
# Watchlist is an unattended cron host: never probe browser
|
||||
# cookies (matches the MCP server). Avoids a silent Chromium
|
||||
# read / unattended macOS Keychain prompt when a user has set
|
||||
# FROM_BROWSER=auto for interactive use.
|
||||
"--no-browser-cookies",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
@@ -236,8 +254,15 @@ def cmd_config(args):
|
||||
print(json.dumps({"action": "config", "key": "daily_budget", "value": str(args.value)}))
|
||||
return
|
||||
if args.key == "delivery":
|
||||
store.set_setting("delivery_channel", str(args.value))
|
||||
print(json.dumps({"action": "config", "key": "delivery_channel", "value": str(args.value)}))
|
||||
value = str(args.value)
|
||||
# Reject a non-https channel at write time so the operator gets
|
||||
# immediate feedback, rather than discovering it via a stderr line
|
||||
# buried in a research run hours later. Matches the delivery-time guard
|
||||
# in _deliver_findings.
|
||||
if value and urllib.parse.urlparse(value).scheme != "https":
|
||||
raise SystemExit(f"delivery_channel must be an https:// URL, got {value!r}")
|
||||
store.set_setting("delivery_channel", value)
|
||||
print(json.dumps({"action": "config", "key": "delivery_channel", "value": value}))
|
||||
return
|
||||
raise SystemExit(f"Unknown config key: {args.key}")
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
📁 Output directory: /Users/mvanhorn/last30days-skill-private/docs/test-results/v1-vs-v2-20260206-231338
|
||||
|
||||
📦 Backing up current V2 SKILL.md...
|
||||
📥 Installing V1 SKILL.md from upstream...
|
||||
✅ V1 installed (stripped: context:fork, agent:Explore, disable-model-invocation)
|
||||
|
||||
==========================================
|
||||
Running V1 — 17 queries
|
||||
==========================================
|
||||
|
||||
[V1] (1/17) prompting techniques for chatgpt for legal questions [PROMPTING+TOOL]
|
||||
@@ -1,5 +1,6 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
@@ -28,6 +29,63 @@ class BriefingV3Tests(unittest.TestCase):
|
||||
store._db_override = old_db_override
|
||||
briefing.BRIEFS_DIR = old_briefs_dir
|
||||
|
||||
def test_generate_weekly_ranks_top_findings_by_engagement(self):
|
||||
"""Weekly digest top_findings must be the highest-engagement items, not
|
||||
the most recent. get_new_findings returns first_seen DESC, so without an
|
||||
explicit engagement sort the digest headlines recent low-engagement noise."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "research.db"
|
||||
briefs_dir = Path(tmpdir) / "briefs"
|
||||
old_db_override = store._db_override
|
||||
old_briefs_dir = briefing.BRIEFS_DIR
|
||||
try:
|
||||
store._db_override = db_path
|
||||
briefing.BRIEFS_DIR = briefs_dir
|
||||
topic = store.add_topic("test topic")
|
||||
run_id = store.record_run(
|
||||
topic["id"], source_mode="v3", status="completed"
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
# Finding i: i days ago, engagement i. The most recent (i=0) has
|
||||
# the lowest engagement, so a recency sort and an engagement sort
|
||||
# disagree on the top 5.
|
||||
conn = store._connect()
|
||||
try:
|
||||
for i in range(6):
|
||||
first_seen = (now - timedelta(days=i, hours=1)).strftime(
|
||||
"%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO findings
|
||||
(run_id, topic_id, source, source_url,
|
||||
source_title, engagement_score, first_seen)
|
||||
VALUES (?, ?, 'reddit', ?, ?, ?, ?)""",
|
||||
(
|
||||
run_id,
|
||||
topic["id"],
|
||||
f"https://example.com/{i}",
|
||||
f"finding-{i}",
|
||||
float(i),
|
||||
first_seen,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
result = briefing.generate_weekly()
|
||||
top = result["topics"][0]["top_findings"]
|
||||
scores = [f["engagement_score"] for f in top]
|
||||
|
||||
self.assertEqual(len(top), 5)
|
||||
self.assertEqual(scores, [5.0, 4.0, 3.0, 2.0, 1.0])
|
||||
# The most-recent, lowest-engagement finding is dropped, not kept.
|
||||
self.assertNotIn(0.0, scores)
|
||||
finally:
|
||||
store._db_override = old_db_override
|
||||
briefing.BRIEFS_DIR = old_briefs_dir
|
||||
|
||||
def test_save_briefing_uses_utf8_encoding(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
old_briefs_dir = briefing.BRIEFS_DIR
|
||||
|
||||
@@ -117,15 +117,14 @@ class TestEnvBrowserSelection:
|
||||
assert browser in tried, f"auto should try {browser}"
|
||||
|
||||
@patch("lib.cookie_extract.extract_cookies")
|
||||
def test_default_still_silent_only(self, mock_extract):
|
||||
"""Default (no FROM_BROWSER) stays Firefox+Safari - no Keychain prompt."""
|
||||
def test_default_skips_browser_cookie_reads(self, mock_extract):
|
||||
"""Default (no FROM_BROWSER) reads no local browser cookies."""
|
||||
mock_extract.return_value = None
|
||||
config = _base_config()
|
||||
|
||||
extract_browser_credentials(config)
|
||||
|
||||
tried = {call[0][0] for call in mock_extract.call_args_list}
|
||||
assert tried == {"firefox", "safari"}
|
||||
mock_extract.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -64,6 +64,28 @@ class CliV3Tests(unittest.TestCase):
|
||||
self.assertIn("ranked_candidates", payload)
|
||||
self.assertIn("clusters", payload)
|
||||
|
||||
def test_invalid_plan_json_exits_nonzero(self):
|
||||
"""Malformed --plan JSON must fail fast, not silently fall back to the
|
||||
internal planner and burn a paid run the user did not ask for."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"skills/last30days/scripts/last30days.py",
|
||||
"test topic",
|
||||
"--mock",
|
||||
"--emit=json",
|
||||
"--plan",
|
||||
"{not valid json",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(2, result.returncode, result.stderr)
|
||||
self.assertIn("Invalid --plan JSON", result.stderr)
|
||||
|
||||
def test_parse_search_flag_normalizes_aliases_and_dedupes(self):
|
||||
self.assertEqual(
|
||||
["grounding", "reddit", "hackernews"],
|
||||
@@ -147,6 +169,58 @@ class CliV3Tests(unittest.TestCase):
|
||||
self.assertEqual(["biosecurity"], args.topic)
|
||||
self.assertEqual([], extra)
|
||||
|
||||
def test_research_unknown_flag_fails_before_config_load(self):
|
||||
with mock.patch.object(
|
||||
cli.env, "get_config", side_effect=AssertionError("config should not load")
|
||||
), mock.patch.object(sys, "argv", ["last30days.py", "topic", "--save"]):
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc:
|
||||
cli.main()
|
||||
self.assertEqual(2, exc.exception.code)
|
||||
self.assertIn("--save", stderr.getvalue())
|
||||
|
||||
def test_agent_is_skill_argument_not_python_cli_flag(self):
|
||||
with mock.patch.object(
|
||||
cli.env, "get_config", side_effect=AssertionError("config should not load")
|
||||
), mock.patch.object(sys, "argv", ["last30days.py", "topic", "--agent"]):
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc:
|
||||
cli.main()
|
||||
self.assertEqual(2, exc.exception.code)
|
||||
self.assertIn("skill arguments", stderr.getvalue())
|
||||
|
||||
def test_agent_error_includes_other_unknown_flags(self):
|
||||
with mock.patch.object(
|
||||
cli.env, "get_config", side_effect=AssertionError("config should not load")
|
||||
), mock.patch.object(sys, "argv", ["last30days.py", "topic", "--agent", "--save"]):
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc:
|
||||
cli.main()
|
||||
self.assertEqual(2, exc.exception.code)
|
||||
message = stderr.getvalue()
|
||||
self.assertIn("--agent", message)
|
||||
self.assertIn("--save", message)
|
||||
|
||||
def test_setup_passthrough_flags_remain_scoped_to_setup(self):
|
||||
with mock.patch.object(cli.env, "get_config", return_value={}), \
|
||||
mock.patch("lib.setup_wizard.run_github_auth", return_value={"status": "cancelled"}), \
|
||||
mock.patch.object(sys, "argv", ["last30days.py", "setup", "--github"]):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
rc = cli.main()
|
||||
self.assertEqual(0, rc)
|
||||
|
||||
def test_setup_rejects_unknown_passthrough_flag_before_config_load(self):
|
||||
with mock.patch.object(
|
||||
cli.env, "get_config", side_effect=AssertionError("config should not load")
|
||||
), mock.patch.object(sys, "argv", ["last30days.py", "setup", "--bad"]):
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc:
|
||||
cli.main()
|
||||
self.assertEqual(2, exc.exception.code)
|
||||
self.assertIn("--bad", stderr.getvalue())
|
||||
|
||||
def test_ensure_supported_python_rejects_old_interpreter_with_actionable_error(self):
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr):
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Host-contract tests for non-modal agent runtimes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
|
||||
|
||||
|
||||
def _prose_flow() -> str:
|
||||
text = SKILL_MD.read_text(encoding="utf-8")
|
||||
start_marker = "### Non-Modal Prose Flow"
|
||||
end_marker = "### Manual Setup Guide"
|
||||
start = text.find(start_marker)
|
||||
assert start != -1, f"missing section marker: {start_marker}"
|
||||
end = text.find(end_marker, start)
|
||||
assert end != -1, f"missing section marker: {end_marker}"
|
||||
return text[start:end]
|
||||
|
||||
|
||||
def test_non_modal_hosts_are_named():
|
||||
prose = _prose_flow()
|
||||
for host in ("Codex", "Cursor", "Gemini CLI", "raw CLI"):
|
||||
assert host in prose
|
||||
|
||||
|
||||
def test_non_modal_cookie_consent_uses_engine_allow_flag():
|
||||
prose = _prose_flow()
|
||||
consent = prose.index("Cookie consent")
|
||||
allow = prose.index("setup --allow-browser-cookies")
|
||||
decline = prose.index("FROM_BROWSER=off")
|
||||
assert consent < allow
|
||||
assert consent < decline
|
||||
|
||||
|
||||
def test_non_modal_completion_mentions_safe_diagnose_and_project_trust():
|
||||
prose = _prose_flow()
|
||||
assert "safe `--diagnose`" in prose
|
||||
assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in prose
|
||||
assert "Codex desktop" in prose
|
||||
@@ -10,6 +10,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from lib import cookie_extract
|
||||
from lib.cookie_extract import (
|
||||
extract_cookies,
|
||||
extract_firefox_cookies,
|
||||
@@ -123,6 +124,38 @@ class TestExtractFirefoxCookies:
|
||||
|
||||
assert result == {"auth_token": "tok_abc123"}
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="POSIX permission model does not apply on Windows; mkstemp is 0o666 there")
|
||||
def test_temp_cookie_copy_never_world_readable(self, tmp_path):
|
||||
"""The temp copy must be private the instant it exists, not only after
|
||||
the lock chmod. Regression for the TOCTOU window where copy2 widened the
|
||||
0600 mkstemp file to the source's 0644 before _lock_temp_cookie_copy ran.
|
||||
"""
|
||||
db_path = tmp_path / "cookies.sqlite"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute(
|
||||
"CREATE TABLE moz_cookies (name TEXT NOT NULL, value TEXT NOT NULL, host TEXT NOT NULL)"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO moz_cookies (name, value, host) VALUES (?, ?, ?)",
|
||||
("auth_token", "tok_abc123", ".x.com"),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
os.chmod(db_path, 0o644) # loose source perms, as Firefox ships them
|
||||
|
||||
observed = {}
|
||||
real_lock = cookie_extract._lock_temp_cookie_copy
|
||||
|
||||
def spy(path):
|
||||
# Mode of the copy as it exists right after copyfile, before chmod.
|
||||
observed["mode_after_copy"] = os.stat(path).st_mode & 0o777
|
||||
return real_lock(path)
|
||||
|
||||
with patch.object(cookie_extract, "_lock_temp_cookie_copy", side_effect=spy):
|
||||
_query_cookies_db(db_path, ".x.com", ["auth_token"])
|
||||
|
||||
assert observed["mode_after_copy"] == 0o600
|
||||
|
||||
def test_valid_cookies_extracted(self, mock_firefox_env):
|
||||
"""Cookies for the target domain are returned correctly."""
|
||||
profiles_dir = mock_firefox_env()
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Documentation contract for Python CLI and wrapper-only flags."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import last30days as cli
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIGURATION = ROOT / "CONFIGURATION.md"
|
||||
SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
|
||||
|
||||
|
||||
def _parser_flags() -> set[str]:
|
||||
parser = cli.build_parser()
|
||||
flags: set[str] = set()
|
||||
for action in parser._actions:
|
||||
flags.update(action.option_strings)
|
||||
return flags
|
||||
|
||||
|
||||
def test_configuration_documents_new_safety_flags():
|
||||
text = CONFIGURATION.read_text(encoding="utf-8")
|
||||
flags = _parser_flags()
|
||||
assert "--no-browser-cookies" in flags
|
||||
assert "--no-browser-cookies" in text
|
||||
assert "--save-dir" in text
|
||||
assert "--output" in text
|
||||
|
||||
|
||||
def test_save_is_not_documented_as_python_cli_flag():
|
||||
text = CONFIGURATION.read_text(encoding="utf-8")
|
||||
assert "--save-dir <path>" in text
|
||||
assert "--save " not in text
|
||||
assert "`--save`" not in text
|
||||
|
||||
|
||||
def test_agent_is_documented_as_skill_argument_not_python_flag():
|
||||
text = SKILL_MD.read_text(encoding="utf-8")
|
||||
start = text.index("## Agent Mode (--agent flag)")
|
||||
agent_section = text[start:start + 2000]
|
||||
assert "If `--agent` appears in ARGUMENTS" in agent_section
|
||||
assert "Skill tool" in text
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Security-copy contract tests for local reads and credential destinations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIGURATION = ROOT / "CONFIGURATION.md"
|
||||
README = ROOT / "README.md"
|
||||
SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
|
||||
UI_PY = ROOT / "skills" / "last30days" / "scripts" / "lib" / "ui.py"
|
||||
|
||||
|
||||
def test_cookie_setup_requires_explicit_allow_flag_in_docs():
|
||||
config = CONFIGURATION.read_text(encoding="utf-8")
|
||||
skill = SKILL_MD.read_text(encoding="utf-8")
|
||||
assert "setup --allow-browser-cookies" in config
|
||||
assert "setup --allow-browser-cookies" in skill
|
||||
assert "Unset = no browser-cookie reads" in config
|
||||
|
||||
|
||||
def test_project_config_trust_is_documented():
|
||||
config = CONFIGURATION.read_text(encoding="utf-8")
|
||||
skill = SKILL_MD.read_text(encoding="utf-8")
|
||||
assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in config
|
||||
assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in skill
|
||||
assert "Folder-mode hosts such as Codex desktop do not trust hidden project config by default" in config
|
||||
|
||||
|
||||
def test_codex_auth_not_advertised_as_openai_fallback():
|
||||
config = CONFIGURATION.read_text(encoding="utf-8")
|
||||
assert "Codex ChatGPT auth" in config
|
||||
assert "intentionally not used" in config
|
||||
assert "or Codex auth" not in config
|
||||
|
||||
|
||||
def test_scrapecreators_copy_uses_canonical_free_call_count():
|
||||
text = "\n".join(
|
||||
[
|
||||
CONFIGURATION.read_text(encoding="utf-8"),
|
||||
README.read_text(encoding="utf-8"),
|
||||
SKILL_MD.read_text(encoding="utf-8"),
|
||||
UI_PY.read_text(encoding="utf-8"),
|
||||
]
|
||||
)
|
||||
assert "10,000 free calls" in text
|
||||
assert "100 free credits" not in text
|
||||
assert "1,000 free" not in text
|
||||
+31
-10
@@ -5,7 +5,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from lib.env import extract_browser_credentials, COOKIE_DOMAINS
|
||||
from lib.env import ConfigLoadPolicy, extract_browser_credentials, COOKIE_DOMAINS
|
||||
|
||||
|
||||
def _base_config(**overrides):
|
||||
@@ -55,17 +55,13 @@ class TestExtractBrowserCredentials:
|
||||
mock_extract.assert_not_called()
|
||||
|
||||
@patch("lib.cookie_extract.extract_cookies")
|
||||
def test_no_from_browser_defaults_to_silent(self, mock_extract):
|
||||
"""Default (no FROM_BROWSER): tries Firefox and Safari only, skips Chrome."""
|
||||
def test_no_from_browser_skips_all(self, mock_extract):
|
||||
"""Default (no FROM_BROWSER): reads no browser cookies."""
|
||||
mock_extract.return_value = None
|
||||
config = _base_config()
|
||||
result = extract_browser_credentials(config)
|
||||
assert result == {}
|
||||
# Should try firefox and safari but NOT chrome
|
||||
browser_args = [call[0][0] for call in mock_extract.call_args_list]
|
||||
assert "firefox" in browser_args
|
||||
assert "safari" in browser_args
|
||||
assert "chrome" not in browser_args
|
||||
mock_extract.assert_not_called()
|
||||
|
||||
@patch("lib.cookie_extract.extract_cookies")
|
||||
def test_from_browser_firefox_only(self, mock_extract):
|
||||
@@ -105,14 +101,14 @@ class TestExtractBrowserCredentials:
|
||||
|
||||
|
||||
class TestGetConfigCookieIntegration:
|
||||
"""Integration test: get_config() calls extract_browser_credentials."""
|
||||
"""Integration tests for policy-gated cookie extraction in get_config()."""
|
||||
|
||||
@patch("lib.cookie_extract.extract_cookies")
|
||||
@patch("lib.env._find_project_env", return_value=None)
|
||||
@patch("lib.env.load_env_file", return_value={})
|
||||
@patch("lib.env._load_keychain", return_value={})
|
||||
@patch("lib.env.get_openai_auth")
|
||||
def test_get_config_injects_cookies(
|
||||
def test_get_config_default_does_not_extract_cookies(
|
||||
self, mock_openai, mock_keychain, mock_load, mock_proj, mock_extract
|
||||
):
|
||||
from lib.env import get_config, OpenAIAuth
|
||||
@@ -128,5 +124,30 @@ class TestGetConfigCookieIntegration:
|
||||
}
|
||||
with patch.dict(os.environ, env_patch, clear=False):
|
||||
config = get_config()
|
||||
assert config["AUTH_TOKEN"] is None
|
||||
assert config["CT0"] is None
|
||||
mock_extract.assert_not_called()
|
||||
|
||||
@patch("lib.cookie_extract.extract_cookies")
|
||||
@patch("lib.env._find_project_env", return_value=None)
|
||||
@patch("lib.env.load_env_file", return_value={})
|
||||
@patch("lib.env._load_keychain", return_value={})
|
||||
@patch("lib.env.get_openai_auth")
|
||||
def test_get_config_with_cookie_policy_injects_cookies(
|
||||
self, mock_openai, mock_keychain, mock_load, mock_proj, mock_extract
|
||||
):
|
||||
from lib.env import get_config, OpenAIAuth
|
||||
mock_openai.return_value = OpenAIAuth(
|
||||
token=None, source="none", status="missing",
|
||||
account_id=None, codex_auth_file="/fake",
|
||||
)
|
||||
mock_extract.return_value = {"auth_token": "browser_tok", "ct0": "browser_ct0"}
|
||||
env_patch = {
|
||||
"SETUP_COMPLETE": "true",
|
||||
"FROM_BROWSER": "auto",
|
||||
"LAST30DAYS_CONFIG_DIR": "",
|
||||
}
|
||||
with patch.dict(os.environ, env_patch, clear=False):
|
||||
config = get_config(policy=ConfigLoadPolicy(browser_cookies="read"))
|
||||
assert config["AUTH_TOKEN"] == "browser_tok"
|
||||
assert config["CT0"] == "browser_ct0"
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
@@ -115,7 +117,14 @@ class EvaluatorV3Tests(unittest.TestCase):
|
||||
output_dir = Path(tmp)
|
||||
cache_dir = output_dir / "judgments"
|
||||
cache_dir.mkdir()
|
||||
(cache_dir / "topic.json").write_text(json.dumps({"judgments": [{"id": "a", "grade": 3}]}))
|
||||
(cache_dir / "topic.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"judge_model": "gemini-3.1-flash-lite",
|
||||
"judgments": [{"id": "a", "grade": 3}],
|
||||
}
|
||||
)
|
||||
)
|
||||
cached = evaluator.get_judgments(
|
||||
output_dir=output_dir,
|
||||
slug="topic",
|
||||
@@ -138,6 +147,39 @@ class EvaluatorV3Tests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual({}, skipped)
|
||||
|
||||
def test_get_judgments_remisses_on_judge_model_change(self):
|
||||
"""A cache written by a different judge model must not be reused; a
|
||||
--judge-model change forces a re-judge instead of returning stale grades."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output_dir = Path(tmp)
|
||||
cache_dir = output_dir / "judgments"
|
||||
cache_dir.mkdir()
|
||||
(cache_dir / "topic.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"judge_model": "gemini-3.1-flash-lite",
|
||||
"judgments": [{"id": "a", "grade": 3}],
|
||||
}
|
||||
)
|
||||
)
|
||||
# Same slug, different model, no API key to re-judge: the stale
|
||||
# grades must NOT come back — an empty result signals "re-judge
|
||||
# needed" rather than silently wrong numbers, and the discard is
|
||||
# announced on stderr instead of failing silently.
|
||||
stderr = io.StringIO()
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
result = evaluator.get_judgments(
|
||||
output_dir=output_dir,
|
||||
slug="topic",
|
||||
topic="test topic",
|
||||
query_type="general",
|
||||
items=[{"key": "a"}],
|
||||
judge_model="gemini-2.5-pro",
|
||||
gemini_api_key=None,
|
||||
)
|
||||
self.assertEqual({}, result)
|
||||
self.assertIn("different", stderr.getvalue())
|
||||
|
||||
def test_create_eval_env_and_run_last30days(self):
|
||||
credential_env = {
|
||||
key: ""
|
||||
|
||||
@@ -225,13 +225,17 @@ def test_search_hackernews_depth_config(mock_request):
|
||||
"""Test that depth parameter controls hit count."""
|
||||
mock_request.return_value = {"hits": [], "nbHits": 0}
|
||||
|
||||
# Quick mode should request 15 hits
|
||||
# Quick mode returns up to 15 hits, but overfetches before client-side
|
||||
# engagement filtering so low-point stories do not shrink result depth.
|
||||
hackernews.search_hackernews("test", "2026-01-01", "2026-01-31", depth="quick")
|
||||
|
||||
call_args = mock_request.call_args[0]
|
||||
url = call_args[1]
|
||||
|
||||
assert "hitsPerPage=15" in url
|
||||
expected_hits_per_page = (
|
||||
hackernews.DEPTH_CONFIG["quick"] * hackernews.HN_OVERFETCH_MULTIPLIER
|
||||
)
|
||||
assert f"hitsPerPage={expected_hits_per_page}" in url
|
||||
|
||||
@patch('lib.hackernews.http.request')
|
||||
|
||||
@@ -266,16 +270,34 @@ def test_search_hackernews_http_error_handling(mock_request):
|
||||
@patch('lib.hackernews.http.request')
|
||||
|
||||
|
||||
def test_search_hackernews_engagement_filter(mock_request):
|
||||
"""Test that low-engagement stories are filtered client-side."""
|
||||
mock_request.return_value = {
|
||||
"hits": [
|
||||
create_mock_hit(object_id="low", points=2),
|
||||
create_mock_hit(object_id="high", points=3),
|
||||
],
|
||||
"nbHits": 2,
|
||||
}
|
||||
|
||||
result = hackernews.search_hackernews("test", "2026-01-01", "2026-01-31")
|
||||
|
||||
call_args = mock_request.call_args[0]
|
||||
url = call_args[1]
|
||||
|
||||
# Algolia rejects points in numericFilters; keep only supported date filters.
|
||||
assert "points" not in url
|
||||
assert [hit["objectID"] for hit in result["hits"]] == ["high"]
|
||||
|
||||
|
||||
@patch('lib.hackernews.http.request')
|
||||
def test_search_hackernews_no_points_numericfilter(mock_request):
|
||||
"""numericFilters must NOT include a `points` clause.
|
||||
|
||||
`points` is not in the HN Algolia index's `numericAttributesForFiltering`,
|
||||
so a `points>2` clause returns HTTP 400 ("invalid numeric attribute(points)")
|
||||
and zero stories. Engagement is reflected in parse-time relevance scoring
|
||||
and zero stories. Engagement is filtered client-side after overfetching
|
||||
instead. This guards against the invalid filter being reintroduced.
|
||||
|
||||
(Previously this test asserted `points` + `%3E2` were present in the URL,
|
||||
pinning the very bug that broke every HN query — a bug-encoding test.)
|
||||
"""
|
||||
mock_request.return_value = {"hits": [], "nbHits": 0}
|
||||
|
||||
@@ -287,6 +309,23 @@ def test_search_hackernews_no_points_numericfilter(mock_request):
|
||||
assert "created_at_i" in url
|
||||
assert "points" not in url
|
||||
|
||||
|
||||
@patch('lib.hackernews.http.request')
|
||||
def test_search_hackernews_truncates_after_overfetch(mock_request):
|
||||
"""Test that overfetching does not return more than the requested depth."""
|
||||
mock_request.return_value = {
|
||||
"hits": [
|
||||
create_mock_hit(object_id=str(i), points=10)
|
||||
for i in range(20)
|
||||
],
|
||||
"nbHits": 20,
|
||||
}
|
||||
|
||||
result = hackernews.search_hackernews("test", "2026-01-01", "2026-01-31", depth="quick")
|
||||
|
||||
assert len(result["hits"]) == 15
|
||||
assert [hit["objectID"] for hit in result["hits"]] == [str(i) for i in range(15)]
|
||||
|
||||
# === Tests for parse_hackernews_response() ===
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,23 @@ class HiringSignalsTests(unittest.TestCase):
|
||||
self.assertEqual("mega-cap", summary["company_size_tier"])
|
||||
self.assertIn("too diffuse", summary["omitted_reason"])
|
||||
|
||||
def test_fortune_500_customer_boilerplate_does_not_make_startup_large_enterprise(self):
|
||||
items = [
|
||||
job(
|
||||
"Founding Enterprise Solutions Engineer",
|
||||
"Help Fortune 500 customers adopt SSO, SOC 2, and procurement workflows.",
|
||||
"Sales",
|
||||
),
|
||||
job(
|
||||
"Security Platform Engineer",
|
||||
"Build enterprise security, audit, and admin workflows for Fortune 500 customers.",
|
||||
"Engineering",
|
||||
),
|
||||
]
|
||||
summary = hiring_signals.analyze(items, explicit=False, topic="Listen Labs")
|
||||
self.assertEqual("startup", summary["company_size_tier"])
|
||||
self.assertTrue(summary["include"])
|
||||
|
||||
def test_explicit_mode_keeps_low_confidence_signal(self):
|
||||
items = [job("Customer Success Manager", "support enterprise customers", "Success")]
|
||||
summary = hiring_signals.analyze(items, explicit=True, topic="Acme")
|
||||
|
||||
@@ -238,6 +238,41 @@ class HtmlRenderBehaviorTests(unittest.TestCase):
|
||||
self.assertNotIn("<a ", rendered)
|
||||
self.assertNotIn("href=", rendered)
|
||||
|
||||
def test_meta_marker_escapes_payload_through_pipeline(self):
|
||||
"""A META marker carrying markup must not render as live HTML.
|
||||
|
||||
The marker is exempted from the comment-strip pass and promoted into a
|
||||
<div class="meta">. Its text can come from LLM-synthesized content
|
||||
derived from untrusted source bodies, so a crafted
|
||||
`<!-- META: <img src=x onerror=...> -->` must be escaped, not rendered.
|
||||
"""
|
||||
md = "intro\n\n<!-- META: <img src=x onerror=alert(1)> -->\n\nmore"
|
||||
body = html_render._markdown_to_html(md)
|
||||
body = html_render._wrap_engine_footer(body)
|
||||
body = html_render._promote_meta_marker(body)
|
||||
self.assertNotIn("<img", body)
|
||||
self.assertIn("<img src=x onerror=alert(1)>", body)
|
||||
|
||||
def test_meta_marker_escapes_raw_fallback(self):
|
||||
"""The raw (unescaped) META fallback path must also escape its payload."""
|
||||
body = html_render._promote_meta_marker(
|
||||
"<!-- META: <img src=x onerror=alert(1)> -->"
|
||||
)
|
||||
self.assertNotIn("<img", body)
|
||||
self.assertEqual(
|
||||
body, '<div class="meta"><img src=x onerror=alert(1)></div>'
|
||||
)
|
||||
|
||||
def test_meta_marker_preserves_plain_text(self):
|
||||
"""Legitimate date/source-name markers render unchanged (no double-escape)."""
|
||||
body = html_render._promote_meta_marker(
|
||||
"<!-- META: 2026-01-01 to 2026-01-31 · reddit, x -->"
|
||||
)
|
||||
self.assertEqual(
|
||||
body,
|
||||
'<div class="meta">2026-01-01 to 2026-01-31 · reddit, x</div>',
|
||||
)
|
||||
|
||||
def test_markdown_links_allow_relative_url(self):
|
||||
rendered = html_render._markdown_to_html("[home](/path?x=1#section)")
|
||||
self.assertIn(
|
||||
|
||||
@@ -2,9 +2,10 @@ import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
@@ -267,5 +268,34 @@ class TestSkillMdFirstRunReference(unittest.TestCase):
|
||||
mock_write.assert_called_once()
|
||||
|
||||
|
||||
class TestCheckPermsAutoFix(unittest.TestCase):
|
||||
"""check_perms should auto-fix loose .env permissions instead of warning only."""
|
||||
|
||||
def test_loose_env_is_tightened_by_check_perms(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
config_dir = Path(tmp) / ".config" / "last30days"
|
||||
config_dir.mkdir(parents=True)
|
||||
env_file = config_dir / ".env"
|
||||
env_file.write_text("SETUP_COMPLETE=true\n")
|
||||
os.chmod(env_file, 0o644)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["HOME"] = str(Path(tmp))
|
||||
env["LAST30DAYS_CONFIG_DIR"] = str(config_dir)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", "hooks/scripts/check-config.sh"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("auto-fixed", result.stdout.lower())
|
||||
self.assertEqual(stat.S_IMODE(os.stat(env_file).st_mode), 0o600)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Contract tests for the restored first-run NUX wizard in SKILL.md.
|
||||
|
||||
Step 0 has two branches: a **Claude Code Modal Flow** (AskUserQuestion-driven,
|
||||
the restored v3.0.0 NUX) and a **Non-Modal Prose Flow** for hosts without modals
|
||||
(OpenClaw, Codex, Cursor, Gemini CLI). These tests assert the structural
|
||||
guarantees of both branches, plus the cross-cutting copy rules: the hard
|
||||
"Step 0 before Step 1" gate, Digg threaded alongside yt-dlp, the 10,000-free-calls
|
||||
credit count, and Threads/Pinterest kept out of the onboarding offers. They read
|
||||
SKILL.md as text - the model's runtime contract - matching
|
||||
tests/test_runtime_preflight_contract.py.
|
||||
|
||||
These lock the flow against silent re-erosion (the failure mode that orphaned the
|
||||
wizard in PR #659 and flattened it before this restoration).
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
|
||||
|
||||
|
||||
class TestOnboardingContract(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.text = SKILL_MD.read_text(encoding="utf-8")
|
||||
# Scope assertions to Step 0 so generic substrings elsewhere in the file
|
||||
# do not satisfy ordering/presence checks.
|
||||
start = self.text.index("## Step 0: First-Run Setup Wizard")
|
||||
end = self.text.index("## CRITICAL: Parse User Intent", start)
|
||||
self.step0 = self.text[start:end]
|
||||
# Branch slices.
|
||||
modal_start = self.step0.index("### Claude Code Modal Flow")
|
||||
prose_start = self.step0.index("### Non-Modal Prose Flow")
|
||||
manual_start = self.step0.index("### Manual Setup Guide")
|
||||
self.modal = self.step0[modal_start:prose_start]
|
||||
self.prose = self.step0[prose_start:manual_start]
|
||||
self.manual = self.step0[manual_start:]
|
||||
|
||||
# --- Platform split + hard gate ---
|
||||
|
||||
def test_platform_split_present(self):
|
||||
"""Step 0 routes modal-capable hosts and prose hosts to distinct flows."""
|
||||
self.assertIn("Platform split", self.step0)
|
||||
self.assertIn("### Claude Code Modal Flow", self.step0)
|
||||
self.assertIn("### Non-Modal Prose Flow", self.step0)
|
||||
|
||||
def test_hard_gate_step0_before_step1(self):
|
||||
"""The erosion-resistant gate that orphaned the wizard in #659 is restored."""
|
||||
self.assertIn("ALWAYS execute Step 0 BEFORE Step 1", self.step0)
|
||||
|
||||
# --- Modal flow: the restored NUX, stages in order ---
|
||||
|
||||
def test_modal_flow_stage_order(self):
|
||||
"""Welcome -> setup modal -> cookie consent -> SC offer -> opt-in -> picker."""
|
||||
anchors = [
|
||||
"Welcome to /last30days!",
|
||||
"How would you like to set up?",
|
||||
"scan your browser", # cookie-consent modal
|
||||
"Want to add TikTok, Instagram, and the ScrapeCreators backups?", # SC offer
|
||||
"Which ScrapeCreators sources do you want on?", # source opt-in
|
||||
"What do you want to research first?", # topic picker
|
||||
]
|
||||
idxs = [self.modal.find(a) for a in anchors]
|
||||
for a, i in zip(anchors, idxs):
|
||||
self.assertGreater(i, -1, f"modal flow missing stage anchor: {a!r}")
|
||||
self.assertEqual(idxs, sorted(idxs), "modal flow stages are out of order")
|
||||
|
||||
def test_modal_uses_askuserquestion(self):
|
||||
self.assertIn("AskUserQuestion", self.modal)
|
||||
|
||||
def test_modal_cookie_consent_before_setup(self):
|
||||
consent = self.modal.find("scan your browser")
|
||||
setup = self.modal.find("last30days.py setup")
|
||||
self.assertGreater(consent, -1, "no cookie-consent modal in modal flow")
|
||||
self.assertGreater(setup, -1, "no setup invocation in modal flow")
|
||||
self.assertLess(consent, setup, "cookie consent must precede setup in modal flow")
|
||||
|
||||
def test_topic_picker_skips_when_topic_supplied(self):
|
||||
"""The picker documents skipping when the user already gave a topic."""
|
||||
self.assertIn("What do you want to research first?", self.modal)
|
||||
self.assertIn("SKIP this picker", self.modal)
|
||||
|
||||
# --- Prose flow: same work, modal-free ---
|
||||
|
||||
def test_prose_flow_has_no_modals(self):
|
||||
self.assertNotIn("AskUserQuestion", self.prose)
|
||||
|
||||
def test_prose_cookie_consent_before_setup(self):
|
||||
consent = self.prose.find("Cookie consent")
|
||||
setup = self.prose.find("last30days.py setup")
|
||||
self.assertGreater(consent, -1, "no cookie-consent step in prose flow")
|
||||
self.assertGreater(setup, -1, "no setup invocation in prose flow")
|
||||
self.assertLess(consent, setup, "cookie consent must precede setup in prose flow")
|
||||
|
||||
def test_prose_decline_uses_from_browser_off(self):
|
||||
self.assertIn("FROM_BROWSER=off", self.prose)
|
||||
|
||||
# --- Full Disk Access remediation (both branches) ---
|
||||
|
||||
def test_full_disk_access_remediation_present(self):
|
||||
self.assertIn("Permission denied reading Cookies.binarycookies", self.modal)
|
||||
self.assertIn("Full Disk Access", self.modal)
|
||||
self.assertIn("Permission denied reading Cookies.binarycookies", self.prose)
|
||||
self.assertIn("Full Disk Access", self.prose)
|
||||
|
||||
def test_skip_path_writes_setup_complete(self):
|
||||
"""The 'Skip for now' setup choice must write SETUP_COMPLETE or the wizard loops."""
|
||||
skip_idx = self.modal.find("If the user picks Skip for now")
|
||||
self.assertGreater(skip_idx, -1, "no Skip-for-now handling in modal flow")
|
||||
# The skip branch must persist the completion flag in its own paragraph.
|
||||
skip_para = self.modal[skip_idx:skip_idx + 400]
|
||||
self.assertIn("SETUP_COMPLETE=true", skip_para)
|
||||
|
||||
# --- ScrapeCreators signup + persisted edge case ---
|
||||
|
||||
def test_scrapecreators_signup_present_both_branches(self):
|
||||
self.assertIn("setup --github", self.modal)
|
||||
self.assertIn("setup --github", self.prose)
|
||||
|
||||
def test_persisted_false_edge_case_documented(self):
|
||||
self.assertIn('"persisted": false', self.step0)
|
||||
|
||||
# --- Digg threaded alongside yt-dlp everywhere it appears ---
|
||||
|
||||
def test_digg_threaded_with_ytdlp(self):
|
||||
self.assertIn("Digg", self.modal)
|
||||
self.assertIn("Digg", self.prose)
|
||||
self.assertIn("Digg", self.manual)
|
||||
# The Auto-setup modal option names both tools together.
|
||||
self.assertIn("yt-dlp (YouTube) and the Digg CLI", self.modal)
|
||||
|
||||
# --- Credit count = 10,000, no conflicting numbers in onboarding ---
|
||||
|
||||
def test_credit_count_is_10000(self):
|
||||
self.assertIn("10,000 free calls", self.step0)
|
||||
self.assertNotIn("1,000 free", self.step0)
|
||||
self.assertNotIn("1000 free credit", self.step0)
|
||||
self.assertNotIn("1000 credits", self.step0)
|
||||
self.assertNotIn("100 free call", self.step0)
|
||||
|
||||
# --- Threads/Pinterest kept out of the onboarding offers ---
|
||||
|
||||
def test_threads_pinterest_absent_from_modal_and_prose(self):
|
||||
"""They stay a power-user INCLUDE_SOURCES note in the manual guide only."""
|
||||
self.assertNotIn("Threads", self.modal)
|
||||
self.assertNotIn("Pinterest", self.modal)
|
||||
self.assertNotIn("Threads", self.prose)
|
||||
self.assertNotIn("Pinterest", self.prose)
|
||||
|
||||
# --- Legacy guarantees retained ---
|
||||
|
||||
def test_old_silent_wizard_instruction_removed(self):
|
||||
self.assertNotIn("Follow the wizard's prompts end-to-end", self.text)
|
||||
|
||||
def test_consent_is_conversational_contract_documented(self):
|
||||
self.assertIn("Named onboarding contract", self.step0)
|
||||
self.assertIn("non-interactive subprocess", self.step0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Tests for trusted project-scoped configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from lib import env, pipeline
|
||||
|
||||
|
||||
def _neutral_secret_sources():
|
||||
return (
|
||||
mock.patch.object(env, "_load_keychain", return_value={}),
|
||||
mock.patch.object(env, "_load_pass", return_value={}),
|
||||
)
|
||||
|
||||
|
||||
def test_untrusted_project_config_is_ignored_by_default(tmp_path, monkeypatch):
|
||||
project_env = tmp_path / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir()
|
||||
project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["XAI_API_KEY"] is None
|
||||
assert cfg["_CONFIG_SOURCE"] == "env_only"
|
||||
|
||||
|
||||
def test_project_config_loads_with_global_trust_signal(tmp_path, monkeypatch):
|
||||
global_env = tmp_path / "global.env"
|
||||
global_env.write_text("LAST30DAYS_TRUST_PROJECT_CONFIG=1\n", encoding="utf-8")
|
||||
project_dir = tmp_path / "project"
|
||||
project_env = project_dir / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir(parents=True)
|
||||
project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(project_dir)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", global_env)
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["XAI_API_KEY"] == "xai-project"
|
||||
assert cfg["_CONFIG_SOURCE"].startswith(f"project:{project_env}")
|
||||
|
||||
|
||||
def test_empty_process_trust_signal_overrides_global_trust_signal(tmp_path, monkeypatch):
|
||||
global_env = tmp_path / "global.env"
|
||||
global_env.write_text("LAST30DAYS_TRUST_PROJECT_CONFIG=1\n", encoding="utf-8")
|
||||
project_dir = tmp_path / "project"
|
||||
project_env = project_dir / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir(parents=True)
|
||||
project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(project_dir)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", global_env)
|
||||
monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "")
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["XAI_API_KEY"] is None
|
||||
assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}")
|
||||
|
||||
|
||||
def test_explicit_zero_process_trust_signal_overrides_global_trust_signal(tmp_path, monkeypatch):
|
||||
"""An explicit process `=0` is a deny and wins over a global `=1`."""
|
||||
global_env = tmp_path / "global.env"
|
||||
global_env.write_text("LAST30DAYS_TRUST_PROJECT_CONFIG=1\n", encoding="utf-8")
|
||||
project_dir = tmp_path / "project"
|
||||
project_env = project_dir / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir(parents=True)
|
||||
project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(project_dir)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", global_env)
|
||||
monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "0")
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["XAI_API_KEY"] is None
|
||||
assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}")
|
||||
|
||||
|
||||
def test_project_config_discovery_stops_at_git_root(tmp_path, monkeypatch):
|
||||
outside_env = tmp_path / ".claude" / "last30days.env"
|
||||
outside_env.parent.mkdir()
|
||||
outside_env.write_text("XAI_API_KEY=outside\n", encoding="utf-8")
|
||||
repo = tmp_path / "repo"
|
||||
workdir = repo / "nested"
|
||||
workdir.mkdir(parents=True)
|
||||
(repo / ".git").mkdir()
|
||||
monkeypatch.chdir(workdir)
|
||||
monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "1")
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["XAI_API_KEY"] is None
|
||||
assert cfg["_CONFIG_SOURCE"] == "env_only"
|
||||
|
||||
|
||||
def test_global_config_loads_when_project_config_is_untrusted(tmp_path, monkeypatch):
|
||||
global_env = tmp_path / "global.env"
|
||||
global_env.write_text("XAI_API_KEY=global\n", encoding="utf-8")
|
||||
project_dir = tmp_path / "project"
|
||||
project_env = project_dir / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir(parents=True)
|
||||
project_env.write_text("XAI_API_KEY=project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(project_dir)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", global_env)
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["XAI_API_KEY"] == "global"
|
||||
assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}")
|
||||
|
||||
|
||||
def test_config_exists_ignores_untrusted_project_config(tmp_path, monkeypatch):
|
||||
project_env = tmp_path / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir()
|
||||
project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
|
||||
assert env.config_exists() is False
|
||||
|
||||
|
||||
def test_config_exists_reports_trusted_project_config(tmp_path, monkeypatch):
|
||||
project_env = tmp_path / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir()
|
||||
project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "1")
|
||||
|
||||
assert env.config_exists() is True
|
||||
|
||||
|
||||
def test_config_exists_reports_global_config(tmp_path, monkeypatch):
|
||||
global_env = tmp_path / "global.env"
|
||||
global_env.write_text("XAI_API_KEY=xai-global\n", encoding="utf-8")
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", global_env)
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
|
||||
assert env.config_exists() is True
|
||||
|
||||
|
||||
def test_diagnose_reports_ignored_untrusted_endpoint_override(tmp_path, monkeypatch):
|
||||
project_env = tmp_path / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir()
|
||||
project_env.write_text(
|
||||
"BSKY_SEARCH_HOST=https://bsky-attacker.example\n"
|
||||
"LAST30DAYS_SEARXNG_URL=https://searxng-attacker.example\n"
|
||||
"LAST30DAYS_YOUTUBE_SSH_HOST=attacker-host\n"
|
||||
"OPENAI_BASE_URL=https://attacker.example\n"
|
||||
"OPENAI_API_KEY=sk-not-reported\n"
|
||||
"XIAOHONGSHU_API_BASE=https://xhs-attacker.example\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-global")
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
|
||||
keychain, pass_store = _neutral_secret_sources()
|
||||
with keychain, pass_store:
|
||||
cfg = env.get_config(
|
||||
policy=env.ConfigLoadPolicy(inspect_ignored_project_config=True)
|
||||
)
|
||||
diag = pipeline.diagnose(cfg, safe=True)
|
||||
|
||||
assert cfg["OPENAI_API_KEY"] == "sk-global"
|
||||
assert cfg["OPENAI_BASE_URL"] is None
|
||||
assert diag["ignored_project_config"] == str(project_env)
|
||||
assert sorted(diag["ignored_endpoint_overrides"]) == [
|
||||
"BSKY_SEARCH_HOST",
|
||||
"LAST30DAYS_SEARXNG_URL",
|
||||
"LAST30DAYS_YOUTUBE_SSH_HOST",
|
||||
"OPENAI_BASE_URL",
|
||||
"XIAOHONGSHU_API_BASE",
|
||||
]
|
||||
assert "sk-not-reported" not in str(diag)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Regression tests for agent-host local-read boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
from unittest import mock
|
||||
|
||||
import last30days as cli
|
||||
from lib import env
|
||||
|
||||
|
||||
def test_importing_cli_does_not_load_config_or_propagate_endpoints(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("XAI_BASE_URL", raising=False)
|
||||
with mock.patch("lib.env.get_config", side_effect=AssertionError("import loaded config")):
|
||||
importlib.reload(cli)
|
||||
assert os.environ.get("OPENAI_BASE_URL") is None
|
||||
assert os.environ.get("XAI_BASE_URL") is None
|
||||
|
||||
|
||||
def test_diagnose_uses_plan_only_cookie_policy_and_safe_pipeline(monkeypatch):
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_get_config(*, policy):
|
||||
seen["policy"] = policy
|
||||
return {"_BROWSER_COOKIE_MODE": policy.browser_cookies, "_BROWSER_COOKIE_BROWSERS": ["firefox"]}
|
||||
|
||||
with mock.patch.object(cli.env, "get_config", side_effect=fake_get_config), \
|
||||
mock.patch.object(cli.pipeline, "diagnose", return_value={"ok": True}) as diagnose, \
|
||||
mock.patch.object(sys, "argv", ["last30days.py", "--diagnose"]):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
assert cli.main() == 0
|
||||
|
||||
assert seen["policy"].browser_cookies == "plan_only"
|
||||
diagnose.assert_called_once_with(
|
||||
{"_BROWSER_COOKIE_MODE": "plan_only", "_BROWSER_COOKIE_BROWSERS": ["firefox"]},
|
||||
None,
|
||||
safe=True,
|
||||
)
|
||||
assert json.loads(stdout.getvalue()) == {"ok": True}
|
||||
|
||||
|
||||
def test_setup_without_cookie_flag_disables_browser_cookie_setup(monkeypatch):
|
||||
with mock.patch.object(cli.env, "get_config", return_value={}), \
|
||||
mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \
|
||||
mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \
|
||||
mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \
|
||||
mock.patch.object(sys, "argv", ["last30days.py", "setup"]):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
assert cli.main() == 0
|
||||
|
||||
assert setup.call_args.kwargs["allow_browser_cookies"] is False
|
||||
|
||||
|
||||
def test_setup_cookie_flag_allows_browser_cookie_setup(monkeypatch):
|
||||
with mock.patch.object(cli.env, "get_config", return_value={}), \
|
||||
mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \
|
||||
mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \
|
||||
mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \
|
||||
mock.patch.object(sys, "argv", ["last30days.py", "setup", "--allow-browser-cookies"]):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
assert cli.main() == 0
|
||||
|
||||
assert setup.call_args.kwargs["allow_browser_cookies"] is True
|
||||
|
||||
|
||||
def test_no_browser_cookies_overrides_setup_cookie_flag(monkeypatch):
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_get_config(*, policy):
|
||||
seen["policy"] = policy
|
||||
return {}
|
||||
|
||||
with mock.patch.object(cli.env, "get_config", side_effect=fake_get_config), \
|
||||
mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \
|
||||
mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \
|
||||
mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \
|
||||
mock.patch.object(
|
||||
sys,
|
||||
"argv",
|
||||
["last30days.py", "--no-browser-cookies", "setup", "--allow-browser-cookies"],
|
||||
):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
assert cli.main() == 0
|
||||
|
||||
assert seen["policy"].browser_cookies == "off"
|
||||
assert setup.call_args.kwargs["allow_browser_cookies"] is False
|
||||
|
||||
|
||||
def test_diagnose_overrides_setup_cookie_flag(monkeypatch):
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_get_config(*, policy):
|
||||
seen["policy"] = policy
|
||||
return {}
|
||||
|
||||
with mock.patch.object(cli.env, "get_config", side_effect=fake_get_config), \
|
||||
mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \
|
||||
mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \
|
||||
mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \
|
||||
mock.patch.object(
|
||||
sys,
|
||||
"argv",
|
||||
["last30days.py", "--diagnose", "setup", "--allow-browser-cookies"],
|
||||
):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
assert cli.main() == 0
|
||||
|
||||
assert seen["policy"].browser_cookies == "plan_only"
|
||||
assert setup.call_args.kwargs["allow_browser_cookies"] is False
|
||||
|
||||
|
||||
def test_research_run_defaults_to_browser_cookie_read():
|
||||
"""A plain research run reads cookies (the path that powers X auth)."""
|
||||
parser = cli.build_parser()
|
||||
args, extra = parser.parse_known_args(["some topic"])
|
||||
policy = cli._config_policy_for_args(args, "some topic", extra)
|
||||
assert policy.browser_cookies == "read"
|
||||
|
||||
|
||||
def test_no_browser_cookies_flag_disables_research_run_cookie_read():
|
||||
"""--no-browser-cookies flips a research run to the no-read policy."""
|
||||
parser = cli.build_parser()
|
||||
args, extra = parser.parse_known_args(["--no-browser-cookies", "some topic"])
|
||||
policy = cli._config_policy_for_args(args, "some topic", extra)
|
||||
assert policy.browser_cookies == "off"
|
||||
|
||||
|
||||
def test_watchlist_subprocess_disables_browser_cookies():
|
||||
"""The unattended watchlist cron must never probe browser cookies."""
|
||||
import watchlist
|
||||
|
||||
fake_result = mock.Mock(returncode=1, stdout="", stderr="boom")
|
||||
with mock.patch.object(watchlist, "store") as store, \
|
||||
mock.patch.object(watchlist.subprocess, "run", return_value=fake_result) as run:
|
||||
store.record_run.return_value = 1
|
||||
watchlist._run_topic({"id": 1, "name": "test topic", "search_queries": None})
|
||||
|
||||
argv = run.call_args.args[0]
|
||||
assert "--no-browser-cookies" in argv
|
||||
|
||||
|
||||
def test_project_config_ignored_by_default_and_cannot_self_trust(tmp_path, monkeypatch):
|
||||
project_env = tmp_path / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir()
|
||||
project_env.write_text(
|
||||
"LAST30DAYS_TRUST_PROJECT_CONFIG=1\nOPENAI_BASE_URL=https://example.invalid\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
|
||||
with mock.patch.object(env, "_load_keychain", return_value={}), \
|
||||
mock.patch.object(env, "_load_pass", return_value={}):
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["OPENAI_BASE_URL"] is None
|
||||
assert cfg["_CONFIG_SOURCE"] == "env_only"
|
||||
|
||||
|
||||
def test_project_config_loads_with_process_trust_signal(tmp_path, monkeypatch):
|
||||
project_env = tmp_path / ".claude" / "last30days.env"
|
||||
project_env.parent.mkdir()
|
||||
project_env.write_text("OPENAI_BASE_URL=https://trusted.example\n", encoding="utf-8")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", None)
|
||||
monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "1")
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
|
||||
with mock.patch.object(env, "_load_keychain", return_value={}), \
|
||||
mock.patch.object(env, "_load_pass", return_value={}):
|
||||
cfg = env.get_config()
|
||||
|
||||
assert cfg["OPENAI_BASE_URL"] == "https://trusted.example"
|
||||
assert cfg["_CONFIG_SOURCE"].startswith(f"project:{project_env}")
|
||||
@@ -16,32 +16,30 @@ def test_security_workflow_exists() -> None:
|
||||
assert WORKFLOW.is_file()
|
||||
|
||||
|
||||
def test_security_workflow_runs_dependency_audit_advisory_first() -> None:
|
||||
def test_security_workflow_runs_dependency_audit_as_blocking_check() -> None:
|
||||
text = _workflow_text()
|
||||
dependency_audit_job = text.split("dependency-audit:", 1)[1].split("secret-scan:", 1)[0]
|
||||
|
||||
assert "dependency-audit:" in text
|
||||
assert "uv audit --locked" in text
|
||||
assert "continue-on-error: true" in text
|
||||
assert "Set continue-on-error: false once a clean baseline run is confirmed" in text
|
||||
assert "uv audit --locked" in dependency_audit_job
|
||||
assert "continue-on-error: true" not in dependency_audit_job
|
||||
|
||||
|
||||
def test_security_workflow_runs_secret_scan_for_pull_requests_and_main_pushes() -> None:
|
||||
text = _workflow_text()
|
||||
secret_scan_job = text.split("secret-scan:", 1)[1]
|
||||
|
||||
assert "secret-scan:" in text
|
||||
assert "trufflesecurity/trufflehog" in text
|
||||
assert "github.event_name == 'pull_request'" in text
|
||||
assert "github.event_name == 'push'" in text
|
||||
assert "--only-verified" in text
|
||||
|
||||
|
||||
def test_security_workflow_documents_advisory_policy() -> None:
|
||||
text = _workflow_text()
|
||||
|
||||
assert "advisory-first" in text.lower()
|
||||
assert "does not block merges" in text.lower()
|
||||
assert "fixtures" in text.lower()
|
||||
assert "env-based auth" in text.lower()
|
||||
assert "pull_request:" in text
|
||||
assert "push:" in text
|
||||
assert "workflow_dispatch:" in text
|
||||
assert "branches:\n - main" in text
|
||||
assert "trufflesecurity/trufflehog" in secret_scan_job
|
||||
assert "version: 3.95.5" in secret_scan_job
|
||||
assert "extra_args: --results=verified" in secret_scan_job
|
||||
assert "continue-on-error: true" not in secret_scan_job
|
||||
assert "if: github.event_name" not in secret_scan_job
|
||||
assert "path: ./" not in secret_scan_job
|
||||
|
||||
|
||||
def test_agent_guidance_mentions_secret_hygiene() -> None:
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
"""Tests for OpenClaw setup and device auth functions."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
import last30days as cli
|
||||
from lib import setup_wizard
|
||||
|
||||
|
||||
@@ -504,3 +508,49 @@ class TestRunGithubAuth:
|
||||
result = setup_wizard.run_github_auth(timeout=1)
|
||||
assert result["status"] == "timeout"
|
||||
mock_subproc.assert_not_called()
|
||||
|
||||
|
||||
class TestSetupGithubCliWiring:
|
||||
"""Tests for the `setup --github` CLI branch: persist + mask the key."""
|
||||
|
||||
def _run_setup_github(self, tmp_path, monkeypatch):
|
||||
"""Invoke `setup --github` in-process, return (parsed_json, env_path)."""
|
||||
env_path = tmp_path / ".env"
|
||||
monkeypatch.setattr(cli.env, "CONFIG_FILE", env_path)
|
||||
monkeypatch.setattr(sys, "argv", ["last30days", "setup", "--github"])
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
rc = cli.main()
|
||||
assert rc == 0
|
||||
return json.loads(buf.getvalue()), env_path
|
||||
|
||||
@patch("lib.setup_wizard.run_github_auth")
|
||||
def test_success_persists_and_masks(self, mock_auth, tmp_path, monkeypatch):
|
||||
"""Success -> key written to .env, stdout JSON masked, persisted true."""
|
||||
mock_auth.return_value = {
|
||||
"status": "success", "method": "device",
|
||||
"api_key": "sc_live_supersecret9999", "user_code": "ABCD-1234",
|
||||
}
|
||||
|
||||
payload, env_path = self._run_setup_github(tmp_path, monkeypatch)
|
||||
|
||||
# Key persisted to disk with the real value
|
||||
assert "SCRAPECREATORS_API_KEY=sc_live_supersecret9999" in env_path.read_text()
|
||||
# JSON reports persistence and the raw secret never appears in stdout
|
||||
assert payload["persisted"] is True
|
||||
assert payload["status"] == "success"
|
||||
assert payload["api_key"] != "sc_live_supersecret9999"
|
||||
assert "supersecret9999" not in json.dumps(payload)
|
||||
# Useful non-secret fields survive
|
||||
assert payload["user_code"] == "ABCD-1234"
|
||||
|
||||
@patch("lib.setup_wizard.run_github_auth")
|
||||
def test_timeout_persists_nothing(self, mock_auth, tmp_path, monkeypatch):
|
||||
"""Timeout -> no key on disk, persisted false."""
|
||||
mock_auth.return_value = {"status": "timeout", "user_code": "WXYZ-5678"}
|
||||
|
||||
payload, env_path = self._run_setup_github(tmp_path, monkeypatch)
|
||||
|
||||
assert payload["persisted"] is False
|
||||
assert not env_path.exists()
|
||||
assert payload["status"] == "timeout"
|
||||
|
||||
+111
-10
@@ -50,7 +50,7 @@ class TestRunAutoSetup:
|
||||
mock_which.return_value = "/usr/local/bin/yt-dlp"
|
||||
|
||||
config = {}
|
||||
results = setup_wizard.run_auto_setup(config)
|
||||
results = setup_wizard.run_auto_setup(config, allow_browser_cookies=True)
|
||||
|
||||
assert "x" in results["cookies_found"]
|
||||
assert results["cookies_found"]["x"] == "chrome"
|
||||
@@ -69,6 +69,7 @@ class TestRunAutoSetup:
|
||||
results = setup_wizard.run_auto_setup(config)
|
||||
|
||||
assert results["cookies_found"] == {}
|
||||
mock_extract.assert_not_called()
|
||||
assert results["ytdlp_installed"] is False
|
||||
assert results["ytdlp_action"] == "no_homebrew"
|
||||
|
||||
@@ -80,7 +81,7 @@ class TestRunAutoSetup:
|
||||
mock_which.return_value = None
|
||||
|
||||
config = {}
|
||||
results = setup_wizard.run_auto_setup(config)
|
||||
results = setup_wizard.run_auto_setup(config, allow_browser_cookies=True)
|
||||
|
||||
assert results["cookies_found"] == {}
|
||||
|
||||
@@ -99,7 +100,7 @@ class TestRunAutoSetup:
|
||||
mock_which.return_value = None
|
||||
|
||||
config = {}
|
||||
results = setup_wizard.run_auto_setup(config)
|
||||
results = setup_wizard.run_auto_setup(config, allow_browser_cookies=True)
|
||||
|
||||
assert results["cookies_found"]["x"] == "firefox"
|
||||
assert results["cookies_found"]["truthsocial"] == "firefox"
|
||||
@@ -417,15 +418,116 @@ class TestWriteSetupConfig:
|
||||
assert "SETUP_COMPLETE=true" in lines[1]
|
||||
|
||||
|
||||
class TestWriteApiKey:
|
||||
"""Tests for write_api_key() — persisting the ScrapeCreators signup key."""
|
||||
|
||||
def test_writes_key_with_secret_permissions(self):
|
||||
"""Key is written and the file is 0o600 (owner read/write only)."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
env_path = Path(tmpdir) / "subdir" / ".env"
|
||||
|
||||
result = setup_wizard.write_api_key(env_path, "sc_live_abcdef123456")
|
||||
|
||||
assert result is True
|
||||
assert env_path.exists()
|
||||
assert "SCRAPECREATORS_API_KEY=sc_live_abcdef123456" in env_path.read_text()
|
||||
assert (env_path.stat().st_mode & 0o777) == 0o600
|
||||
|
||||
def test_value_round_trips_through_env_loader(self):
|
||||
"""Persisted key reloads to the exact original value."""
|
||||
from lib import env as env_mod
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
env_path = Path(tmpdir) / ".env"
|
||||
|
||||
setup_wizard.write_api_key(env_path, "sc_live_abcdef123456")
|
||||
|
||||
loaded = env_mod.load_env_file(env_path)
|
||||
assert loaded["SCRAPECREATORS_API_KEY"] == "sc_live_abcdef123456"
|
||||
|
||||
def test_idempotent_when_key_already_present(self):
|
||||
"""If the key already exists, do not duplicate or overwrite it."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
env_path = Path(tmpdir) / ".env"
|
||||
env_path.write_text("SCRAPECREATORS_API_KEY=existing_key\n")
|
||||
|
||||
result = setup_wizard.write_api_key(env_path, "sc_new_value")
|
||||
|
||||
assert result is True
|
||||
content = env_path.read_text()
|
||||
assert content.count("SCRAPECREATORS_API_KEY") == 1
|
||||
assert "existing_key" in content
|
||||
assert "sc_new_value" not in content
|
||||
|
||||
def test_appends_without_clobbering_other_keys(self):
|
||||
"""Existing unrelated keys are preserved."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
env_path = Path(tmpdir) / ".env"
|
||||
env_path.write_text("SETUP_COMPLETE=true\nFROM_BROWSER=firefox\n")
|
||||
|
||||
setup_wizard.write_api_key(env_path, "sc_key_xyz")
|
||||
|
||||
content = env_path.read_text()
|
||||
assert "SETUP_COMPLETE=true" in content
|
||||
assert "FROM_BROWSER=firefox" in content
|
||||
assert "SCRAPECREATORS_API_KEY=sc_key_xyz" in content
|
||||
|
||||
def test_value_with_whitespace_is_quoted(self):
|
||||
"""A pathological value with whitespace is quoted so it round-trips."""
|
||||
from lib import env as env_mod
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
env_path = Path(tmpdir) / ".env"
|
||||
|
||||
setup_wizard.write_api_key(env_path, "key with space")
|
||||
|
||||
content = env_path.read_text()
|
||||
assert 'SCRAPECREATORS_API_KEY="key with space"' in content
|
||||
assert env_mod.load_env_file(env_path)["SCRAPECREATORS_API_KEY"] == "key with space"
|
||||
|
||||
def test_empty_key_returns_false_and_writes_nothing(self):
|
||||
"""An empty api_key persists nothing and reports failure."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
env_path = Path(tmpdir) / ".env"
|
||||
|
||||
assert setup_wizard.write_api_key(env_path, "") is False
|
||||
assert not env_path.exists()
|
||||
|
||||
def test_unwritable_target_returns_false(self):
|
||||
"""Unwritable target dir -> False, no exception escapes."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ro_dir = Path(tmpdir) / "ro"
|
||||
ro_dir.mkdir()
|
||||
ro_dir.chmod(0o500) # no write
|
||||
try:
|
||||
result = setup_wizard.write_api_key(ro_dir / "sub" / ".env", "sc_key")
|
||||
assert result is False
|
||||
finally:
|
||||
ro_dir.chmod(0o700) # restore so tempdir cleanup succeeds
|
||||
|
||||
|
||||
class TestMaskApiKey:
|
||||
"""Tests for mask_api_key() — non-secret display form."""
|
||||
|
||||
def test_masks_long_key(self):
|
||||
masked = setup_wizard.mask_api_key("sc_live_abcdef123456")
|
||||
assert "abcdef" not in masked
|
||||
assert masked.endswith("3456")
|
||||
assert masked.startswith("sc_")
|
||||
|
||||
def test_short_key_collapses_to_placeholder(self):
|
||||
assert setup_wizard.mask_api_key("short") == "sc_…"
|
||||
|
||||
def test_empty_key_collapses_to_placeholder(self):
|
||||
assert setup_wizard.mask_api_key("") == "sc_…"
|
||||
|
||||
|
||||
class TestCookieExtractionBrowsers:
|
||||
"""Tests for env.cookie_extraction_browsers() — the shared browser policy."""
|
||||
|
||||
def test_default_excludes_chrome(self):
|
||||
"""FROM_BROWSER unset -> Firefox/Safari only, never Chrome (no prompt)."""
|
||||
def test_default_disables_extraction(self):
|
||||
"""FROM_BROWSER unset -> no browser-cookie reads."""
|
||||
from lib import env
|
||||
browsers = env.cookie_extraction_browsers({})
|
||||
assert "chrome" not in browsers
|
||||
assert browsers == ["firefox", "safari"]
|
||||
assert browsers == []
|
||||
|
||||
def test_off_disables_extraction(self):
|
||||
from lib import env
|
||||
@@ -448,13 +550,12 @@ class TestWizardDoesNotProbeChromeByDefault:
|
||||
def test_default_run_never_requests_chrome(self, _mock_which, mock_extract):
|
||||
setup_wizard.run_auto_setup({})
|
||||
requested_browsers = {call.args[0] for call in mock_extract.call_args_list}
|
||||
assert "chrome" not in requested_browsers
|
||||
assert requested_browsers <= {"firefox", "safari"}
|
||||
assert requested_browsers == set()
|
||||
|
||||
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
|
||||
@patch("shutil.which", return_value=None)
|
||||
def test_from_browser_auto_does_request_chrome(self, _mock_which, mock_extract):
|
||||
setup_wizard.run_auto_setup({"FROM_BROWSER": "auto"})
|
||||
setup_wizard.run_auto_setup({"FROM_BROWSER": "auto"}, allow_browser_cookies=True)
|
||||
requested_browsers = {call.args[0] for call in mock_extract.call_args_list}
|
||||
assert "chrome" in requested_browsers
|
||||
|
||||
|
||||
@@ -419,6 +419,74 @@ def test_store_findings_increments_sighting_count(temp_db, sample_report):
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_store_findings_upserts_on_concurrent_duplicate_url(temp_db, monkeypatch):
|
||||
"""A stale dedup read (a concurrent run that inserted the same URL between
|
||||
our SELECT and INSERT) must upsert, not raise IntegrityError and lose the
|
||||
whole batch. Regression for the SELECT-then-INSERT race in store_findings."""
|
||||
topic = store.add_topic("Test Topic")
|
||||
finding = {
|
||||
"source": "reddit",
|
||||
"source_url": "https://reddit.com/race",
|
||||
"source_title": "Race",
|
||||
"engagement_score": 5.0,
|
||||
}
|
||||
|
||||
# First run inserts the URL.
|
||||
run1 = store.record_run(topic["id"], source_mode="v3")
|
||||
store.store_findings(run1, topic["id"], [finding])
|
||||
|
||||
# Force the dedup lookup to miss the now-existing URL, so store_findings
|
||||
# takes the INSERT path for a row that is already present — exactly what a
|
||||
# racing writer's stale read produces.
|
||||
real_connect = store._connect
|
||||
dedup_prefix = (
|
||||
"SELECT id, source_url, engagement_score FROM findings WHERE source_url IN"
|
||||
)
|
||||
|
||||
class StaleReadConn:
|
||||
def __init__(self, conn):
|
||||
self._conn = conn
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if sql.strip().startswith(dedup_prefix):
|
||||
return self._conn.execute(
|
||||
"SELECT id, source_url, engagement_score FROM findings WHERE 0"
|
||||
)
|
||||
return self._conn.execute(sql, params)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._conn, name)
|
||||
|
||||
monkeypatch.setattr(
|
||||
store, "_connect", lambda *a, **k: StaleReadConn(real_connect(*a, **k))
|
||||
)
|
||||
|
||||
run2 = store.record_run(topic["id"], source_mode="v3")
|
||||
# Without ON CONFLICT this raises sqlite3.IntegrityError on the UNIQUE
|
||||
# source_url and rolls back the batch.
|
||||
counts = store.store_findings(run2, topic["id"], [{**finding, "engagement_score": 9.0}])
|
||||
|
||||
# The conflict-resolved row is an update, not a new finding. The counters
|
||||
# must reflect that, not inflate findings_new.
|
||||
assert counts == {"new": 0, "updated": 1}
|
||||
|
||||
conn = sqlite3.connect(str(temp_db))
|
||||
rows = conn.execute(
|
||||
"SELECT engagement_score, sighting_count FROM findings WHERE source_url = ?",
|
||||
("https://reddit.com/race",),
|
||||
).fetchall()
|
||||
run_counts = conn.execute(
|
||||
"SELECT findings_new, findings_updated FROM research_runs WHERE id = ?",
|
||||
(run2,),
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
assert len(rows) == 1 # not duplicated, not crashed
|
||||
assert rows[0][0] == 9.0 # engagement upgraded via max()
|
||||
assert rows[0][1] == 2 # sighting_count bumped by the conflict update
|
||||
assert run_counts == (0, 1) # research_runs counters not inflated
|
||||
|
||||
|
||||
def test_store_findings_skips_items_without_url(temp_db):
|
||||
"""Test that findings without URLs are skipped."""
|
||||
topic = store.add_topic("Test Topic")
|
||||
|
||||
@@ -176,7 +176,8 @@ class TestRunWithTimeout(unittest.TestCase):
|
||||
|
||||
def wait(self, timeout=None):
|
||||
# First wait (timeout=5) forces the SIGKILL escalation branch;
|
||||
# the final wait() (no timeout) returns.
|
||||
# the final bounded wait is swallowed if the process still
|
||||
# refuses to exit.
|
||||
if timeout is not None:
|
||||
raise TimeoutExpired(cmd="x", timeout=timeout)
|
||||
return 0
|
||||
|
||||
@@ -252,6 +252,18 @@ def test_cmd_config_delivery(temp_db, capsys):
|
||||
assert output["key"] == "delivery_channel"
|
||||
|
||||
|
||||
def test_cmd_config_delivery_rejects_non_https(temp_db):
|
||||
"""A non-https delivery channel is rejected at write time, not stored."""
|
||||
args = Mock()
|
||||
args.key = "delivery"
|
||||
args.value = "http://evil.example/hooks.slack.com"
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
watchlist.cmd_config(args)
|
||||
|
||||
assert not store.get_setting("delivery_channel")
|
||||
|
||||
|
||||
def test_cmd_config_budget(temp_db, capsys):
|
||||
"""Test configuring daily budget."""
|
||||
args = Mock()
|
||||
|
||||
@@ -225,6 +225,42 @@ def test_deliver_findings_handles_failure_gracefully(mock_post, mock_get_setting
|
||||
@patch('watchlist.http.post')
|
||||
|
||||
|
||||
def test_deliver_findings_rejects_non_https_slack_substring(mock_post, mock_get_setting, capsys):
|
||||
"""A non-https channel that merely contains 'hooks.slack.com' must not be
|
||||
sent. The old substring match POSTed it in cleartext to whatever host the
|
||||
URL actually named."""
|
||||
mock_get_setting.side_effect = lambda key, default="": {
|
||||
"delivery_channel": "http://evil.example/hooks.slack.com",
|
||||
"delivery_mode": "announce",
|
||||
}.get(key, default)
|
||||
|
||||
watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
|
||||
|
||||
assert not mock_post.called
|
||||
assert "https://" in capsys.readouterr().err
|
||||
|
||||
@patch('watchlist.store.get_setting')
|
||||
@patch('watchlist.http.post')
|
||||
|
||||
|
||||
def test_deliver_findings_slack_match_is_exact_host(mock_post, mock_get_setting):
|
||||
"""An https URL with 'hooks.slack.com' only in the path routes as generic,
|
||||
not Slack — the match is on the exact hostname, not a substring."""
|
||||
mock_get_setting.side_effect = lambda key, default="": {
|
||||
"delivery_channel": "https://webhook.example.com/hooks.slack.com/x",
|
||||
"delivery_mode": "announce",
|
||||
}.get(key, default)
|
||||
|
||||
watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
|
||||
|
||||
json_data = mock_post.call_args[1]["json_data"]
|
||||
assert "message" in json_data # generic payload shape
|
||||
assert "text" not in json_data # not the Slack {"text": ...} shape
|
||||
|
||||
@patch('watchlist.store.get_setting')
|
||||
@patch('watchlist.http.post')
|
||||
|
||||
|
||||
def test_deliver_findings_respects_delivery_mode(mock_post, mock_get_setting):
|
||||
"""Test that different delivery modes produce different messages."""
|
||||
mock_get_setting.side_effect = lambda key, default="": {
|
||||
|
||||
@@ -106,7 +106,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "last30days-skill"
|
||||
version = "3.8.0"
|
||||
version = "3.8.1"
|
||||
source = { virtual = "." }
|
||||
|
||||
[package.dev-dependencies]
|
||||
@@ -119,7 +119,7 @@ dev = [
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
{ name = "pytest", specifier = ">=9.0.3,<10" },
|
||||
{ name = "pytest", specifier = ">=9.1.0,<10" },
|
||||
{ name = "pytest-cov", specifier = ">=7,<8" },
|
||||
]
|
||||
|
||||
@@ -152,7 +152,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.0.3"
|
||||
version = "9.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
@@ -161,9 +161,9 @@ dependencies = [
|
||||
{ name = "pluggy" },
|
||||
{ name = "pygments" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user