From 6ef466d43f1d089c5ccc664cc9f02c5f9bba8110 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 17 Jun 2026 21:02:45 -0400 Subject: [PATCH 01/50] fix(store): serialize concurrent writers and upsert on URL conflict store.py advertised WAL-mode "safe concurrent access (cron + user)" but two real failure modes broke that promise: - _connect never set busy_timeout, so the default 0ms made a contending writer raise "database is locked" instantly instead of waiting. Set busy_timeout=5000. - store_findings did a dedup SELECT then a plain INSERT in a separate step. source_url is UNIQUE and the read is not atomic with the write, so two concurrent runs could both see a URL as missing and both insert it; the second commit hit IntegrityError and rolled back the entire batch, losing every finding. The insert is now an ON CONFLICT(source_url) DO UPDATE that mirrors the re-sighting path (bump last_seen/sighting_count, keep the max engagement). Adds a regression test that forces a stale dedup read and asserts the upsert path instead of a crash. --- skills/last30days/scripts/store.py | 16 +++++++- tests/test_store.py | 59 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.py index ddf3366..22e1f7f 100644 --- a/skills/last30days/scripts/store.py +++ b/skills/last30days/scripts/store.py @@ -193,6 +193,10 @@ def _connect(db_path: Optional[Path] = None) -> sqlite3.Connection: conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA synchronous=NORMAL") conn.execute("PRAGMA foreign_keys=ON") + # WAL lets readers coexist with one writer, but two writers (cron + user) + # still contend for the write lock. Default busy_timeout is 0, so the loser + # raises "database is locked" instantly; wait instead. + conn.execute("PRAGMA busy_timeout=5000") return conn @@ -452,11 +456,21 @@ def store_findings( update_rows, ) if insert_rows: + # source_url is UNIQUE. The SELECT above is not atomic with this + # write, so a concurrent run (cron + user) can insert the same URL + # between our read and write. Upsert on conflict instead of letting + # IntegrityError abort the whole batch and lose every finding. conn.executemany( """INSERT INTO findings (run_id, topic_id, source, source_url, source_title, author, content, summary, engagement_score, relevance_score) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source_url) DO UPDATE SET + last_seen = datetime('now'), + sighting_count = sighting_count + 1, + engagement_score = max( + engagement_score, excluded.engagement_score), + run_id = excluded.run_id""", insert_rows, ) diff --git a/tests/test_store.py b/tests/test_store.py index 3de24fc..523d554 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -419,6 +419,65 @@ def test_store_findings_increments_sighting_count(temp_db, sample_report): conn.close() +def test_store_findings_upserts_on_concurrent_duplicate_url(temp_db, monkeypatch): + """A stale dedup read (a concurrent run that inserted the same URL between + our SELECT and INSERT) must upsert, not raise IntegrityError and lose the + whole batch. Regression for the SELECT-then-INSERT race in store_findings.""" + topic = store.add_topic("Test Topic") + finding = { + "source": "reddit", + "source_url": "https://reddit.com/race", + "source_title": "Race", + "engagement_score": 5.0, + } + + # First run inserts the URL. + run1 = store.record_run(topic["id"], source_mode="v3") + store.store_findings(run1, topic["id"], [finding]) + + # Force the dedup lookup to miss the now-existing URL, so store_findings + # takes the INSERT path for a row that is already present — exactly what a + # racing writer's stale read produces. + real_connect = store._connect + dedup_prefix = ( + "SELECT id, source_url, engagement_score FROM findings WHERE source_url IN" + ) + + class StaleReadConn: + def __init__(self, conn): + self._conn = conn + + def execute(self, sql, params=()): + if sql.strip().startswith(dedup_prefix): + return self._conn.execute( + "SELECT id, source_url, engagement_score FROM findings WHERE 0" + ) + return self._conn.execute(sql, params) + + def __getattr__(self, name): + return getattr(self._conn, name) + + monkeypatch.setattr( + store, "_connect", lambda *a, **k: StaleReadConn(real_connect(*a, **k)) + ) + + run2 = store.record_run(topic["id"], source_mode="v3") + # Without ON CONFLICT this raises sqlite3.IntegrityError on the UNIQUE + # source_url and rolls back the batch. + store.store_findings(run2, topic["id"], [{**finding, "engagement_score": 9.0}]) + + conn = sqlite3.connect(str(temp_db)) + rows = conn.execute( + "SELECT engagement_score, sighting_count FROM findings WHERE source_url = ?", + ("https://reddit.com/race",), + ).fetchall() + conn.close() + + assert len(rows) == 1 # not duplicated, not crashed + assert rows[0][0] == 9.0 # engagement upgraded via max() + assert rows[0][1] == 2 # sighting_count bumped by the conflict update + + def test_store_findings_skips_items_without_url(temp_db): """Test that findings without URLs are skipped.""" topic = store.add_topic("Test Topic") From ae65f9aad1e072898fff4eb152c76e71fbf1a24c Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 17 Jun 2026 21:08:30 -0400 Subject: [PATCH 02/50] fix(watchlist): validate webhook scheme and match Slack host exactly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _deliver_findings selected the Slack path with `"hooks.slack.com" in channel`, an unanchored substring test that ran before any scheme check — while the generic branch required https://. A delivery_channel like http://evil.example/hooks.slack.com was therefore treated as Slack and POSTed in cleartext to whatever host the URL actually named, leaking the notification payload while the operator believed Slack was configured. Parse the channel, require an https scheme, and match Slack on the exact hostname (parsed.hostname == "hooks.slack.com") rather than a substring. A non-https channel is now reported on stderr instead of being silently dropped. Adds regression tests for the cleartext-bypass URL and for a URL carrying the Slack host only in its path. --- skills/last30days/scripts/watchlist.py | 17 ++++++++++-- tests/test_watchlist_delivery.py | 36 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/skills/last30days/scripts/watchlist.py b/skills/last30days/scripts/watchlist.py index bf2903a..e855383 100644 --- a/skills/last30days/scripts/watchlist.py +++ b/skills/last30days/scripts/watchlist.py @@ -8,6 +8,7 @@ import json import subprocess import sys import time +import urllib.parse from pathlib import Path SCRIPT_DIR = Path(__file__).parent.resolve() @@ -28,10 +29,22 @@ def _deliver_findings(topic_name: str, counts: dict) -> None: mode = store.get_setting("delivery_mode", "announce") message = _format_delivery_message(topic_name, counts, mode) + # Require https before routing. The old "hooks.slack.com" in channel + # substring test ran before any scheme check, so a channel like + # http://evil.example/hooks.slack.com was treated as Slack and POSTed in + # cleartext to the wrong host. Match Slack on the exact hostname instead. + parsed = urllib.parse.urlparse(channel) + if parsed.scheme != "https": + print( + f"Delivery skipped: delivery_channel must be an https:// URL, got {channel!r}", + file=sys.stderr, + ) + return + try: - if "hooks.slack.com" in channel: + if parsed.hostname == "hooks.slack.com": _send_slack_webhook(channel, message) - elif channel.startswith("https://"): + else: _send_generic_webhook(channel, message) except Exception as e: # Don't fail the research run if delivery fails diff --git a/tests/test_watchlist_delivery.py b/tests/test_watchlist_delivery.py index 3d34408..7e6a224 100644 --- a/tests/test_watchlist_delivery.py +++ b/tests/test_watchlist_delivery.py @@ -225,6 +225,42 @@ def test_deliver_findings_handles_failure_gracefully(mock_post, mock_get_setting @patch('watchlist.http.post') +def test_deliver_findings_rejects_non_https_slack_substring(mock_post, mock_get_setting, capsys): + """A non-https channel that merely contains 'hooks.slack.com' must not be + sent. The old substring match POSTed it in cleartext to whatever host the + URL actually named.""" + mock_get_setting.side_effect = lambda key, default="": { + "delivery_channel": "http://evil.example/hooks.slack.com", + "delivery_mode": "announce", + }.get(key, default) + + watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2}) + + assert not mock_post.called + assert "https://" in capsys.readouterr().err + +@patch('watchlist.store.get_setting') +@patch('watchlist.http.post') + + +def test_deliver_findings_slack_match_is_exact_host(mock_post, mock_get_setting): + """An https URL with 'hooks.slack.com' only in the path routes as generic, + not Slack — the match is on the exact hostname, not a substring.""" + mock_get_setting.side_effect = lambda key, default="": { + "delivery_channel": "https://webhook.example.com/hooks.slack.com/x", + "delivery_mode": "announce", + }.get(key, default) + + watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2}) + + json_data = mock_post.call_args[1]["json_data"] + assert "message" in json_data # generic payload shape + assert "text" not in json_data # not the Slack {"text": ...} shape + +@patch('watchlist.store.get_setting') +@patch('watchlist.http.post') + + def test_deliver_findings_respects_delivery_mode(mock_post, mock_get_setting): """Test that different delivery modes produce different messages.""" mock_get_setting.side_effect = lambda key, default="": { From 7a5ded08ec3d67b0a79d65c8fbb134b500f0c0ce Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 17 Jun 2026 21:12:44 -0400 Subject: [PATCH 03/50] fix: fail fast and rank correctly on valid-looking input across CLI paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent spots silently did the wrong thing on input that looked fine: - last30days.py: malformed --plan JSON only warned to stderr, then proceeded with the internal planner and ran a full, API-consuming research the user did not request. Now raises SystemExit(2), matching the --plan file-read branch and parse_competitors_plan. - evaluate_search_quality.py: the judgment cache was keyed on the topic slug alone, so rerunning with a different --judge-model returned the prior model's grades and silently skewed precision@k / nDCG. The judge model is now stored in the cache and a mismatch forces a re-judge. - briefing.py: the weekly digest sliced this_week[:5] claiming "already sorted by engagement", but get_new_findings returns first_seen DESC — so it headlined the most recent items, not the highest-engagement ones. Now sorts by engagement before slicing, matching the daily path. Each fix has a regression test that fails on the prior behavior. --- skills/last30days/scripts/briefing.py | 10 +++- .../scripts/evaluate_search_quality.py | 9 ++- skills/last30days/scripts/last30days.py | 4 ++ tests/test_briefing_v3.py | 58 +++++++++++++++++++ tests/test_cli_v3.py | 22 +++++++ tests/test_evaluator_v3.py | 38 +++++++++++- 6 files changed, 138 insertions(+), 3 deletions(-) diff --git a/skills/last30days/scripts/briefing.py b/skills/last30days/scripts/briefing.py index ac8f611..1fe1abf 100644 --- a/skills/last30days/scripts/briefing.py +++ b/skills/last30days/scripts/briefing.py @@ -188,7 +188,15 @@ def generate_weekly() -> dict: "this_week_engagement": this_engagement, "last_week_engagement": last_engagement, "engagement_change_pct": round(engagement_change, 1), - "top_findings": this_week[:5], # Top 5 by engagement (already sorted) + # get_new_findings returns first_seen DESC, so sort by engagement + # before slicing — otherwise the digest headlines the most recent + # items, not the highest-engagement ones (the daily path keys on + # engagement too). + "top_findings": sorted( + this_week, + key=lambda f: f.get("engagement_score", 0), + reverse=True, + )[:5], }) result = { diff --git a/skills/last30days/scripts/evaluate_search_quality.py b/skills/last30days/scripts/evaluate_search_quality.py index c6834c6..5caa954 100644 --- a/skills/last30days/scripts/evaluate_search_quality.py +++ b/skills/last30days/scripts/evaluate_search_quality.py @@ -284,10 +284,17 @@ def get_judgments( cache_file.parent.mkdir(parents=True, exist_ok=True) if cache_file.exists(): payload = json.loads(cache_file.read_text()) - return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []} + # The cache key is the topic slug alone, but judgments are model- + # specific. Only reuse the cache when it was produced by the same judge + # model; otherwise re-judge, so a --judge-model change cannot return + # stale grades that silently skew precision@k / nDCG. Caches written + # before judge_model was recorded miss here and get refreshed once. + if payload.get("judge_model") == judge_model: + return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []} if not gemini_api_key or not items: return {} payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items)) + payload["judge_model"] = judge_model cache_file.write_text(json.dumps(payload, indent=2)) return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []} diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 1793642..b62c437 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -742,6 +742,10 @@ def main() -> int: external_plan = _json.loads(plan_str) except _json.JSONDecodeError as exc: sys.stderr.write(f"[Planner] Invalid --plan JSON: {exc}\n") + # Fail fast instead of silently dropping to the internal planner + # and burning a paid run the user did not ask for. Mirrors the + # --plan file-read branch above and parse_competitors_plan. + raise SystemExit(2) # Auto-resolve: use web search to discover subreddits/handles before planning. # This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms diff --git a/tests/test_briefing_v3.py b/tests/test_briefing_v3.py index 1b0a6fd..05c0c25 100644 --- a/tests/test_briefing_v3.py +++ b/tests/test_briefing_v3.py @@ -1,5 +1,6 @@ import tempfile import unittest +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest import mock @@ -28,6 +29,63 @@ class BriefingV3Tests(unittest.TestCase): store._db_override = old_db_override briefing.BRIEFS_DIR = old_briefs_dir + def test_generate_weekly_ranks_top_findings_by_engagement(self): + """Weekly digest top_findings must be the highest-engagement items, not + the most recent. get_new_findings returns first_seen DESC, so without an + explicit engagement sort the digest headlines recent low-engagement noise.""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "research.db" + briefs_dir = Path(tmpdir) / "briefs" + old_db_override = store._db_override + old_briefs_dir = briefing.BRIEFS_DIR + try: + store._db_override = db_path + briefing.BRIEFS_DIR = briefs_dir + topic = store.add_topic("test topic") + run_id = store.record_run( + topic["id"], source_mode="v3", status="completed" + ) + + now = datetime.now(timezone.utc) + # Finding i: i days ago, engagement i. The most recent (i=0) has + # the lowest engagement, so a recency sort and an engagement sort + # disagree on the top 5. + conn = store._connect() + try: + for i in range(6): + first_seen = (now - timedelta(days=i, hours=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + conn.execute( + """INSERT INTO findings + (run_id, topic_id, source, source_url, + source_title, engagement_score, first_seen) + VALUES (?, ?, 'reddit', ?, ?, ?, ?)""", + ( + run_id, + topic["id"], + f"https://example.com/{i}", + f"finding-{i}", + float(i), + first_seen, + ), + ) + conn.commit() + finally: + conn.close() + + result = briefing.generate_weekly() + top = result["topics"][0]["top_findings"] + scores = [f["engagement_score"] for f in top] + + self.assertEqual(len(top), 5) + self.assertEqual(scores, [5.0, 4.0, 3.0, 2.0, 1.0]) + # The most-recent, lowest-engagement finding is dropped, not kept. + self.assertNotIn(0.0, scores) + finally: + store._db_override = old_db_override + briefing.BRIEFS_DIR = old_briefs_dir + def test_save_briefing_uses_utf8_encoding(self): with tempfile.TemporaryDirectory() as tmpdir: old_briefs_dir = briefing.BRIEFS_DIR diff --git a/tests/test_cli_v3.py b/tests/test_cli_v3.py index b7c2f80..1e34617 100644 --- a/tests/test_cli_v3.py +++ b/tests/test_cli_v3.py @@ -64,6 +64,28 @@ class CliV3Tests(unittest.TestCase): self.assertIn("ranked_candidates", payload) self.assertIn("clusters", payload) + def test_invalid_plan_json_exits_nonzero(self): + """Malformed --plan JSON must fail fast, not silently fall back to the + internal planner and burn a paid run the user did not ask for.""" + result = subprocess.run( + [ + sys.executable, + "skills/last30days/scripts/last30days.py", + "test topic", + "--mock", + "--emit=json", + "--plan", + "{not valid json", + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + self.assertEqual(2, result.returncode, result.stderr) + self.assertIn("Invalid --plan JSON", result.stderr) + def test_parse_search_flag_normalizes_aliases_and_dedupes(self): self.assertEqual( ["grounding", "reddit", "hackernews"], diff --git a/tests/test_evaluator_v3.py b/tests/test_evaluator_v3.py index 2dd8612..a882f12 100644 --- a/tests/test_evaluator_v3.py +++ b/tests/test_evaluator_v3.py @@ -115,7 +115,14 @@ class EvaluatorV3Tests(unittest.TestCase): output_dir = Path(tmp) cache_dir = output_dir / "judgments" cache_dir.mkdir() - (cache_dir / "topic.json").write_text(json.dumps({"judgments": [{"id": "a", "grade": 3}]})) + (cache_dir / "topic.json").write_text( + json.dumps( + { + "judge_model": "gemini-3.1-flash-lite", + "judgments": [{"id": "a", "grade": 3}], + } + ) + ) cached = evaluator.get_judgments( output_dir=output_dir, slug="topic", @@ -138,6 +145,35 @@ class EvaluatorV3Tests(unittest.TestCase): ) self.assertEqual({}, skipped) + def test_get_judgments_remisses_on_judge_model_change(self): + """A cache written by a different judge model must not be reused; a + --judge-model change forces a re-judge instead of returning stale grades.""" + with tempfile.TemporaryDirectory() as tmp: + output_dir = Path(tmp) + cache_dir = output_dir / "judgments" + cache_dir.mkdir() + (cache_dir / "topic.json").write_text( + json.dumps( + { + "judge_model": "gemini-3.1-flash-lite", + "judgments": [{"id": "a", "grade": 3}], + } + ) + ) + # Same slug, different model, no API key to re-judge: the stale + # grades must NOT come back — an empty result signals "re-judge + # needed" rather than silently wrong numbers. + result = evaluator.get_judgments( + output_dir=output_dir, + slug="topic", + topic="test topic", + query_type="general", + items=[{"key": "a"}], + judge_model="gemini-2.5-pro", + gemini_api_key=None, + ) + self.assertEqual({}, result) + def test_create_eval_env_and_run_last30days(self): credential_env = { key: "" From 7e21004236a38f79fd9d70aa757a7a8c807ad403 Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Thu, 18 Jun 2026 01:41:40 +0000 Subject: [PATCH 04/50] fix(hackernews): avoid unsupported Algolia points filter --- skills/last30days/scripts/lib/hackernews.py | 23 +++++++++-- tests/test_hackernews.py | 42 +++++++++++++++++---- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.py index 3f68843..8c82652 100644 --- a/skills/last30days/scripts/lib/hackernews.py +++ b/skills/last30days/scripts/lib/hackernews.py @@ -31,6 +31,9 @@ DEPTH_CONFIG = { "deep": 60, } +MIN_STORY_POINTS = 2 +HN_OVERFETCH_MULTIPLIER = 2 + ENRICH_LIMITS = { "quick": 3, "default": 5, @@ -83,6 +86,7 @@ def search_hackernews( Dict with Algolia response (contains 'hits' list). """ count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + fetch_count = count * HN_OVERFETCH_MULTIPLIER from_ts = _date_to_unix(from_date) to_ts = _date_to_unix(to_date) + 86400 # Include the end date @@ -93,14 +97,15 @@ def search_hackernews( core_flat = _flatten_query_for_algolia(core) _log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})") - # Use relevance-sorted search with minimum engagement filter. + # Use relevance-sorted search with date filters. Algolia's HN index rejects + # engagement fields in numericFilters, so points are filtered client-side. # NOTE: restrictSearchableAttributes=title omitted intentionally — it would # miss Ask HN/Show HN threads where the topic appears in the body. params = { "query": core_flat, "tags": "story", - "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2", - "hitsPerPage": str(count), + "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts}", + "hitsPerPage": str(fetch_count), } # Algolia defaults to AND across query tokens, so a 4-5 word theme query # matches no stories. Mark all-but-the-first token as optional so Algolia @@ -121,7 +126,17 @@ def search_hackernews( _log(f"Search failed: {e}") return {"hits": [], "error": str(e)} - hits = response.get("hits", []) + raw_hits = response.get("hits", []) + qualifying_hits = [ + hit for hit in raw_hits + if (hit.get("points") or 0) > MIN_STORY_POINTS + ] + hits = qualifying_hits[:count] + dropped_low_engagement = len(raw_hits) - len(qualifying_hits) + if dropped_low_engagement: + _log(f"Filtered {dropped_low_engagement}/{len(raw_hits)} low-engagement stories") + if len(hits) != len(raw_hits): + response = {**response, "hits": hits} _log(f"Found {len(hits)} stories") return response diff --git a/tests/test_hackernews.py b/tests/test_hackernews.py index 3d4dd97..26f70cd 100644 --- a/tests/test_hackernews.py +++ b/tests/test_hackernews.py @@ -225,13 +225,17 @@ def test_search_hackernews_depth_config(mock_request): """Test that depth parameter controls hit count.""" mock_request.return_value = {"hits": [], "nbHits": 0} - # Quick mode should request 15 hits + # Quick mode returns up to 15 hits, but overfetches before client-side + # engagement filtering so low-point stories do not shrink result depth. hackernews.search_hackernews("test", "2026-01-01", "2026-01-31", depth="quick") call_args = mock_request.call_args[0] url = call_args[1] - assert "hitsPerPage=15" in url + expected_hits_per_page = ( + hackernews.DEPTH_CONFIG["quick"] * hackernews.HN_OVERFETCH_MULTIPLIER + ) + assert f"hitsPerPage={expected_hits_per_page}" in url @patch('lib.hackernews.http.request') @@ -267,16 +271,40 @@ def test_search_hackernews_http_error_handling(mock_request): def test_search_hackernews_engagement_filter(mock_request): - """Test that low-engagement stories are filtered.""" - mock_request.return_value = {"hits": [], "nbHits": 0} + """Test that low-engagement stories are filtered client-side.""" + mock_request.return_value = { + "hits": [ + create_mock_hit(object_id="low", points=2), + create_mock_hit(object_id="high", points=3), + ], + "nbHits": 2, + } - hackernews.search_hackernews("test", "2026-01-01", "2026-01-31") + result = hackernews.search_hackernews("test", "2026-01-01", "2026-01-31") call_args = mock_request.call_args[0] url = call_args[1] - # Should filter for points > 2 (URL-encoded) - assert "points" in url and "%3E2" in url + # Algolia rejects points in numericFilters; keep only supported date filters. + assert "points" not in url + assert [hit["objectID"] for hit in result["hits"]] == ["high"] + + +@patch('lib.hackernews.http.request') +def test_search_hackernews_truncates_after_overfetch(mock_request): + """Test that overfetching does not return more than the requested depth.""" + mock_request.return_value = { + "hits": [ + create_mock_hit(object_id=str(i), points=10) + for i in range(20) + ], + "nbHits": 20, + } + + result = hackernews.search_hackernews("test", "2026-01-01", "2026-01-31", depth="quick") + + assert len(result["hits"]) == 15 + assert [hit["objectID"] for hit in result["hits"]] == [str(i) for i in range(15)] # === Tests for parse_hackernews_response() === From 5b168c7382c6d6079ff0363bfafed0ec1af01567 Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Thu, 18 Jun 2026 01:30:02 +0000 Subject: [PATCH 05/50] ci: harden release supply chain --- .github/dependabot.yml | 7 +++++ .github/workflows/release.yml | 45 +++++++++++++++++++++------------ .github/workflows/scorecard.yml | 6 ++--- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 65091b3..58f5ba4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,3 +13,10 @@ updates: interval: weekly cooldown: default-days: 7 + + - package-ecosystem: gomod + directory: /mcp + schedule: + interval: weekly + cooldown: + default-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f17377a..5eafc3e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,10 +5,7 @@ on: tags: - "v*" -permissions: - contents: write - id-token: write - attestations: write +permissions: {} jobs: @@ -18,7 +15,7 @@ jobs: build-skill: runs-on: ubuntu-latest permissions: - contents: write + contents: read id-token: write attestations: write steps: @@ -34,12 +31,12 @@ jobs: test -f dist/last30days.skill - name: Attest .skill artifact provenance - uses: actions/attest-build-provenance@v4 + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 with: subject-path: dist/last30days.skill - name: Upload skill artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: last30days-skill path: dist/last30days.skill @@ -49,6 +46,13 @@ jobs: # zip layout; we only supply the pre-built binary via --skip-build. build-mcpb: runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + attestations: write + env: + MCPB_OUTPUT: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb + MCPB_PLATFORM: ${{ matrix.platform }} strategy: fail-fast: false matrix: @@ -69,9 +73,10 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: - go-version: stable + go-version-file: mcp/go.mod + cache: false - name: Install printing-press # Pin to a known-good PP release so the bundle command's behavior @@ -92,10 +97,11 @@ jobs: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} CGO_ENABLED: "0" + RELEASE_VERSION: ${{ github.ref_name }} run: | mkdir -p mcp/build go -C mcp build \ - -ldflags "-X main.Version=${{ github.ref_name }}" \ + -ldflags "-X main.Version=${RELEASE_VERSION}" \ -o build/last30days-pp-mcp \ ./cmd/last30days-pp-mcp @@ -108,24 +114,30 @@ jobs: printing-press bundle mcp \ --skip-build \ --binary mcp/build/last30days-pp-mcp \ - --platform ${{ matrix.platform }} \ - --output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb + --platform "${MCPB_PLATFORM}" \ + --output "${MCPB_OUTPUT}" + + - name: Attest .mcpb artifact provenance + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 + with: + subject-path: ${{ env.MCPB_OUTPUT }} - name: Upload .mcpb artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }} - path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb + path: ${{ env.MCPB_OUTPUT }} # Gather every platform artifact and attach to one GitHub release. release: needs: [build-skill, build-mcpb] runs-on: ubuntu-latest permissions: + actions: read contents: write steps: - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: dist merge-multiple: true @@ -133,8 +145,9 @@ jobs: - name: Create GitHub release env: GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} run: | - gh release create "${GITHUB_REF_NAME}" \ + gh release create "${RELEASE_TAG}" \ dist/last30days.skill \ dist/last30days-pp-mcp-*.mcpb \ --generate-notes \ diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 17e55c9..1be0bc1 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -46,7 +46,7 @@ jobs: persist-credentials: false - name: Run OpenSSF Scorecard - uses: ossf/scorecard-action@v2.4.3 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: scorecard.sarif results_format: sarif @@ -57,13 +57,13 @@ jobs: # Retain the raw SARIF as a build artifact for offline inspection. - name: Upload artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: scorecard-sarif path: scorecard.sarif retention-days: 5 - name: Upload SARIF to code-scanning - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 with: sarif_file: scorecard.sarif From d39d25f8d64bae1db3ff221e4baabd4277b1a5a5 Mon Sep 17 00:00:00 2001 From: 23241a6749 Date: Thu, 18 Jun 2026 06:11:41 +0000 Subject: [PATCH 06/50] fix: auto-tighten .env permissions to 0o600 instead of warning only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_perms() in check-config.sh warned about loose .env permissions but never fixed them. Now it runs chmod 600 before emitting the warning, so the next SessionStart doesn't re-warn. The .env creation path (write_setup_config) already used _open_secret_append with O_CREAT mode 0o600 + explicit chmod — that half was done. This closes the gap for pre-existing files and any path that bypasses the wizard. Closes #573. --- CHANGELOG.md | 4 ++++ CONTRIBUTORS.md | 2 +- hooks/scripts/check-config.sh | 3 ++- tests/test_last_run_state.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0604999..44575c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The X FROM lane (the subject's own timeline) now pulls up to 8 posts per handle (was 3); the about/related lanes stay modest. +### Fixed + +- Secrets `.env` and its parent config directory are now auto-tightened to `0o600`/`0o700` after creation, and `check-config.sh`'s `check_perms` now auto-fixes loose permissions with `chmod 600` instead of warning only ([#573](https://github.com/mvanhorn/last30days-skill/issues/573)) + ## [3.5.0] - 2026-06-18 ### Added diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 727608e..757c25e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -54,7 +54,7 @@ v3 has durable watchlist with multi-source storage and extended time windows. ## Past Contributors -- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)) +- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)); .env permission auto-fix ([#573](https://github.com/mvanhorn/last30days-skill/pull/599)) - [@JosephOIbrahim](https://github.com/JosephOIbrahim) - Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17)) - [@levineam](https://github.com/levineam) - Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16)) - [@jonthebeef](https://github.com/jonthebeef) - Early testing and feedback diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 8a07d7a..7452bc9 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -28,8 +28,9 @@ check_perms() { # every Linux session start and printed a false WARNING. perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "") if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then + chmod 600 "$file" 2>/dev/null || true echo "/last30days: WARNING — $file has permissions $perms (should be 600)." - echo " Fix: chmod 600 $file" + echo " Auto-fixed with: chmod 600 $file" fi } diff --git a/tests/test_last_run_state.py b/tests/test_last_run_state.py index 986bb8c..505dbd9 100644 --- a/tests/test_last_run_state.py +++ b/tests/test_last_run_state.py @@ -2,6 +2,8 @@ import io import json import os import re +import shutil +import stat import subprocess import sys import shutil @@ -267,5 +269,34 @@ class TestSkillMdFirstRunReference(unittest.TestCase): mock_write.assert_called_once() +class TestCheckPermsAutoFix(unittest.TestCase): + """check_perms should auto-fix loose .env permissions instead of warning only.""" + + def test_loose_env_is_tightened_by_check_perms(self): + with tempfile.TemporaryDirectory() as tmp: + config_dir = Path(tmp) / ".config" / "last30days" + config_dir.mkdir(parents=True) + env_file = config_dir / ".env" + env_file.write_text("SETUP_COMPLETE=true\n") + os.chmod(env_file, 0o644) + + env = os.environ.copy() + env["HOME"] = str(Path(tmp)) + env["LAST30DAYS_CONFIG_DIR"] = str(config_dir) + + result = subprocess.run( + ["bash", "hooks/scripts/check-config.sh"], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Auto-fixed", result.stdout) + self.assertEqual(stat.S_IMODE(os.stat(env_file).st_mode), 0o600) + + if __name__ == "__main__": unittest.main() From 0381df50255491232ceb11ec32df316e66b0a37b Mon Sep 17 00:00:00 2001 From: 23241a6749 Date: Thu, 18 Jun 2026 06:17:53 +0000 Subject: [PATCH 07/50] fix: only print auto-fixed message on successful chmod; remove duplicate import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile P1: silent chmod failure (e.g. read-only mount, wrong owner) would still print 'Auto-fixed'. Now the message is conditional on chmod exit code — success prints 'auto-fixed', failure prints the original 'Fix: chmod 600' manual instruction. Greptile P2: duplicate import shutil removed from test file. --- hooks/scripts/check-config.sh | 4 +--- tests/test_last_run_state.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 7452bc9..b48b104 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -28,9 +28,7 @@ check_perms() { # every Linux session start and printed a false WARNING. perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "") if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then - chmod 600 "$file" 2>/dev/null || true - echo "/last30days: WARNING — $file has permissions $perms (should be 600)." - echo " Auto-fixed with: chmod 600 $file" + chmod 600 "$file" && echo "/last30days: WARNING — $file had permissions $perms — auto-fixed with chmod 600" || echo "/last30days: WARNING — $file has permissions $perms (should be 600). Fix: chmod 600 $file" fi } diff --git a/tests/test_last_run_state.py b/tests/test_last_run_state.py index 505dbd9..c44fdf9 100644 --- a/tests/test_last_run_state.py +++ b/tests/test_last_run_state.py @@ -6,7 +6,6 @@ import shutil import stat import subprocess import sys -import shutil import tempfile import unittest from contextlib import redirect_stderr @@ -294,7 +293,7 @@ class TestCheckPermsAutoFix(unittest.TestCase): ) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("Auto-fixed", result.stdout) + self.assertIn("auto-fixed", result.stdout.lower()) self.assertEqual(stat.S_IMODE(os.stat(env_file).st_mode), 0o600) From f03d8a184c455c49d35af2825ab99546d2db71b3 Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:50:02 +0000 Subject: [PATCH 08/50] ci: enforce security workflow checks --- .github/workflows/security.yml | 23 ++++++----------------- tests/test_security_workflow.py | 32 +++++++++++++++----------------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index d25bbb2..024d979 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -24,12 +24,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - # Advisory-first: visibility before enforcement. This repo handles API keys, - # cookies, browser tokens, and local env files, so dependency CVEs should be - # visible in CI logs even before the project has a clean blocking baseline. - # Set continue-on-error: false once a clean baseline run is confirmed. + # Block known vulnerabilities in the locked Python dependency graph. - name: Run uv audit against locked dependencies - continue-on-error: true run: uv audit --locked secret-scan: @@ -44,18 +40,11 @@ jobs: fetch-depth: 0 persist-credentials: false - # Advisory-first: this reports verified secrets in pull requests and pushes to - # main, but does not block merges until maintainers confirm a clean baseline. - # The TruffleHog action automatically scans the PR range for pull_request - # events and the pushed commit range for push events. - # Set continue-on-error: false once a clean baseline run is confirmed. - # Contributor policy: never commit real secrets in fixtures, tests, docs, or - # examples; use obvious dummy values and env-based auth patterns instead. + # The action derives the commit range from the GitHub event and fails on + # verified secrets. Keep output limited to verified findings to avoid noisy + # unverified annotations. - name: Run TruffleHog OSS secret scan - if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2 - continue-on-error: true with: - path: ./ - version: v3.95.2 - extra_args: --only-verified + version: 3.95.2 + extra_args: --results=verified diff --git a/tests/test_security_workflow.py b/tests/test_security_workflow.py index 088548f..08399ea 100644 --- a/tests/test_security_workflow.py +++ b/tests/test_security_workflow.py @@ -16,32 +16,30 @@ def test_security_workflow_exists() -> None: assert WORKFLOW.is_file() -def test_security_workflow_runs_dependency_audit_advisory_first() -> None: +def test_security_workflow_runs_dependency_audit_as_blocking_check() -> None: text = _workflow_text() + dependency_audit_job = text.split("dependency-audit:", 1)[1].split("secret-scan:", 1)[0] assert "dependency-audit:" in text - assert "uv audit --locked" in text - assert "continue-on-error: true" in text - assert "Set continue-on-error: false once a clean baseline run is confirmed" in text + assert "uv audit --locked" in dependency_audit_job + assert "continue-on-error: true" not in dependency_audit_job def test_security_workflow_runs_secret_scan_for_pull_requests_and_main_pushes() -> None: text = _workflow_text() + secret_scan_job = text.split("secret-scan:", 1)[1] assert "secret-scan:" in text - assert "trufflesecurity/trufflehog" in text - assert "github.event_name == 'pull_request'" in text - assert "github.event_name == 'push'" in text - assert "--only-verified" in text - - -def test_security_workflow_documents_advisory_policy() -> None: - text = _workflow_text() - - assert "advisory-first" in text.lower() - assert "does not block merges" in text.lower() - assert "fixtures" in text.lower() - assert "env-based auth" in text.lower() + assert "pull_request:" in text + assert "push:" in text + assert "workflow_dispatch:" in text + assert "branches:\n - main" in text + assert "trufflesecurity/trufflehog" in secret_scan_job + assert "version: 3.95.2" in secret_scan_job + assert "extra_args: --results=verified" in secret_scan_job + assert "continue-on-error: true" not in secret_scan_job + assert "if: github.event_name" not in secret_scan_job + assert "path: ./" not in secret_scan_job def test_agent_guidance_mentions_secret_hygiene() -> None: From a6a7a2fb655fa5368754bf794c55d4078ad236a5 Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:59:57 +0000 Subject: [PATCH 09/50] ci: bump trufflehog action --- .github/workflows/security.yml | 4 ++-- tests/test_security_workflow.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 024d979..24f1248 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -44,7 +44,7 @@ jobs: # verified secrets. Keep output limited to verified findings to avoid noisy # unverified annotations. - name: Run TruffleHog OSS secret scan - uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2 + uses: trufflesecurity/trufflehog@d411fff7b8879a62509f3fa98c07f247ac089a51 # v3.95.5 with: - version: 3.95.2 + version: 3.95.5 extra_args: --results=verified diff --git a/tests/test_security_workflow.py b/tests/test_security_workflow.py index 08399ea..786f349 100644 --- a/tests/test_security_workflow.py +++ b/tests/test_security_workflow.py @@ -35,7 +35,7 @@ def test_security_workflow_runs_secret_scan_for_pull_requests_and_main_pushes() assert "workflow_dispatch:" in text assert "branches:\n - main" in text assert "trufflesecurity/trufflehog" in secret_scan_job - assert "version: 3.95.2" in secret_scan_job + assert "version: 3.95.5" in secret_scan_job assert "extra_args: --results=verified" in secret_scan_job assert "continue-on-error: true" not in secret_scan_job assert "if: github.event_name" not in secret_scan_job From e603c7b4f0caa88d6db5e73ce4947e840e53890e Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Thu, 18 Jun 2026 06:34:38 -0400 Subject: [PATCH 10/50] fix(watchlist): reject non-https delivery_channel at config time cmd_config stored any string as delivery_channel, so a misconfigured non-https URL surfaced only at delivery time via a stderr line the operator could miss hours later. Validate the scheme at write time and fail with a clear message, matching the guard in _deliver_findings. --- skills/last30days/scripts/watchlist.py | 11 +++++++++-- tests/test_watchlist_commands.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/skills/last30days/scripts/watchlist.py b/skills/last30days/scripts/watchlist.py index e855383..88c2de8 100644 --- a/skills/last30days/scripts/watchlist.py +++ b/skills/last30days/scripts/watchlist.py @@ -249,8 +249,15 @@ def cmd_config(args): print(json.dumps({"action": "config", "key": "daily_budget", "value": str(args.value)})) return if args.key == "delivery": - store.set_setting("delivery_channel", str(args.value)) - print(json.dumps({"action": "config", "key": "delivery_channel", "value": str(args.value)})) + value = str(args.value) + # Reject a non-https channel at write time so the operator gets + # immediate feedback, rather than discovering it via a stderr line + # buried in a research run hours later. Matches the delivery-time guard + # in _deliver_findings. + if value and urllib.parse.urlparse(value).scheme != "https": + raise SystemExit(f"delivery_channel must be an https:// URL, got {value!r}") + store.set_setting("delivery_channel", value) + print(json.dumps({"action": "config", "key": "delivery_channel", "value": value})) return raise SystemExit(f"Unknown config key: {args.key}") diff --git a/tests/test_watchlist_commands.py b/tests/test_watchlist_commands.py index 8f6b0b5..830a5fe 100644 --- a/tests/test_watchlist_commands.py +++ b/tests/test_watchlist_commands.py @@ -252,6 +252,18 @@ def test_cmd_config_delivery(temp_db, capsys): assert output["key"] == "delivery_channel" +def test_cmd_config_delivery_rejects_non_https(temp_db): + """A non-https delivery channel is rejected at write time, not stored.""" + args = Mock() + args.key = "delivery" + args.value = "http://evil.example/hooks.slack.com" + + with pytest.raises(SystemExit): + watchlist.cmd_config(args) + + assert not store.get_setting("delivery_channel") + + def test_cmd_config_budget(temp_db, capsys): """Test configuring daily budget.""" args = Mock() From faf8782dd4377b5e82a80e3abbe5b9ffe3d3b7e7 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Thu, 18 Jun 2026 06:35:46 -0400 Subject: [PATCH 11/50] fix(evaluator): warn when a stale-model judgment cache is discarded without a key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the cache existed under a different judge_model and no Gemini API key was set to re-judge, get_judgments returned {} silently, so a scheduled eval run scored every item as ungraded and reported zero precision@k / nDCG with no signal — the same silent-wrong-result class this PR removes elsewhere. Emit a stderr line in that path. The normal not-configured path (no cache) stays quiet. --- .../scripts/evaluate_search_quality.py | 11 ++++++++ tests/test_evaluator_v3.py | 26 ++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/skills/last30days/scripts/evaluate_search_quality.py b/skills/last30days/scripts/evaluate_search_quality.py index 5caa954..f40bc2d 100644 --- a/skills/last30days/scripts/evaluate_search_quality.py +++ b/skills/last30days/scripts/evaluate_search_quality.py @@ -282,6 +282,7 @@ def get_judgments( ) -> dict[str, int]: cache_file = output_dir / "judgments" / f"{slug}.json" cache_file.parent.mkdir(parents=True, exist_ok=True) + stale_cache = False if cache_file.exists(): payload = json.loads(cache_file.read_text()) # The cache key is the topic slug alone, but judgments are model- @@ -291,7 +292,17 @@ def get_judgments( # before judge_model was recorded miss here and get refreshed once. if payload.get("judge_model") == judge_model: return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []} + stale_cache = True if not gemini_api_key or not items: + if stale_cache: + # Discarded a different-model cache but can't re-judge. Returning {} + # scores every item as ungraded (zero precision@k / nDCG); say so + # rather than letting the run report silently wrong numbers. + sys.stderr.write( + f"[Eval] Cached judgments for {slug!r} were graded by a different " + f"judge model and no Gemini API key is set to re-judge; returning " + f"no grades (metrics for this topic will be zero).\n" + ) return {} payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items)) payload["judge_model"] = judge_model diff --git a/tests/test_evaluator_v3.py b/tests/test_evaluator_v3.py index a882f12..25fd62f 100644 --- a/tests/test_evaluator_v3.py +++ b/tests/test_evaluator_v3.py @@ -1,3 +1,5 @@ +import contextlib +import io import json import os import tempfile @@ -162,17 +164,21 @@ class EvaluatorV3Tests(unittest.TestCase): ) # Same slug, different model, no API key to re-judge: the stale # grades must NOT come back — an empty result signals "re-judge - # needed" rather than silently wrong numbers. - result = evaluator.get_judgments( - output_dir=output_dir, - slug="topic", - topic="test topic", - query_type="general", - items=[{"key": "a"}], - judge_model="gemini-2.5-pro", - gemini_api_key=None, - ) + # needed" rather than silently wrong numbers, and the discard is + # announced on stderr instead of failing silently. + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + result = evaluator.get_judgments( + output_dir=output_dir, + slug="topic", + topic="test topic", + query_type="general", + items=[{"key": "a"}], + judge_model="gemini-2.5-pro", + gemini_api_key=None, + ) self.assertEqual({}, result) + self.assertIn("different", stderr.getvalue()) def test_create_eval_env_and_run_last30days(self): credential_env = { From 87d2be30b1b851ba3e21fabff60b18a96e98a6c3 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Thu, 18 Jun 2026 06:37:26 -0400 Subject: [PATCH 12/50] fix(store): count conflict-resolved upserts as updates, not new The upsert closed the data-loss race, but new_count = len(insert_rows) still counted every ON CONFLICT row as a brand-new finding, inflating research_runs.findings_new and undercounting findings_updated on exactly the concurrent path the upsert handles. Re-derive the split after the write: an inserted URL whose sighting_count is now > 1 was a conflict (an update), so move it from new to updated. The regression test now asserts the run counters. --- skills/last30days/scripts/store.py | 15 +++++++++++++++ tests/test_store.py | 11 ++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.py index 22e1f7f..4e058c0 100644 --- a/skills/last30days/scripts/store.py +++ b/skills/last30days/scripts/store.py @@ -476,6 +476,21 @@ def store_findings( new_count = len(insert_rows) updated_count = len(update_rows) + if insert_rows: + # A row whose URL was inserted by a concurrent run between our SELECT + # and the upsert resolves via ON CONFLICT (an update, not a new row), + # bumping its sighting_count above 1. Re-derive the split so + # research_runs.findings_new isn't inflated by conflict-resolved rows + # (source_url is field index 3 in each insert tuple). + inserted_urls = [row[3] for row in insert_rows] + placeholders = ",".join("?" for _ in inserted_urls) + conflicted = conn.execute( + f"SELECT COUNT(*) FROM findings " + f"WHERE source_url IN ({placeholders}) AND sighting_count > 1", + inserted_urls, + ).fetchone()[0] + new_count -= conflicted + updated_count += conflicted _record_sightings(conn, run_id, topic_id, with_urls, existing_by_url) conn.execute( "UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?", diff --git a/tests/test_store.py b/tests/test_store.py index 523d554..dccf620 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -464,18 +464,27 @@ def test_store_findings_upserts_on_concurrent_duplicate_url(temp_db, monkeypatch run2 = store.record_run(topic["id"], source_mode="v3") # Without ON CONFLICT this raises sqlite3.IntegrityError on the UNIQUE # source_url and rolls back the batch. - store.store_findings(run2, topic["id"], [{**finding, "engagement_score": 9.0}]) + counts = store.store_findings(run2, topic["id"], [{**finding, "engagement_score": 9.0}]) + + # The conflict-resolved row is an update, not a new finding. The counters + # must reflect that, not inflate findings_new. + assert counts == {"new": 0, "updated": 1} conn = sqlite3.connect(str(temp_db)) rows = conn.execute( "SELECT engagement_score, sighting_count FROM findings WHERE source_url = ?", ("https://reddit.com/race",), ).fetchall() + run_counts = conn.execute( + "SELECT findings_new, findings_updated FROM research_runs WHERE id = ?", + (run2,), + ).fetchone() conn.close() assert len(rows) == 1 # not duplicated, not crashed assert rows[0][0] == 9.0 # engagement upgraded via max() assert rows[0][1] == 2 # sighting_count bumped by the conflict update + assert run_counts == (0, 1) # research_runs counters not inflated def test_store_findings_skips_items_without_url(temp_db): From c165fbfab50ede46383db6c92b7ced6df54634aa Mon Sep 17 00:00:00 2001 From: jesus alberto cornelio <365diascollaboration@gmail.com> Date: Thu, 18 Jun 2026 12:30:49 -0400 Subject: [PATCH 13/50] fix: make pre-research warning runtime-agnostic The warning in _render_pre_research_warning hardcoded 'Claude Code window' but this skill runs on Codex, Hermes, Gemini, Cursor, and 50+ other runtimes. A user on Gemini CLI would be told to open 'a fresh Claude Code window', which doesn't apply to them. Match the pattern already used by the sibling _render_degraded_run_warning, which correctly enumerates all supported runtimes. Signed-off-by: jesus alberto cornelio <365diascollaboration@gmail.com> --- skills/last30days/scripts/lib/render.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.py index a89fb8f..d96e276 100644 --- a/skills/last30days/scripts/lib/render.py +++ b/skills/last30days/scripts/lib/render.py @@ -442,8 +442,9 @@ def _render_pre_research_warning(report: schema.Report) -> list[str]: "- Subreddit-specific threads on dedicated communities", "- Topic-specific TikTok and Instagram creators", "", - "To fix: in a fresh Claude Code window, run `ToolSearch select:WebSearch` first,", - f"then rerun `/last30days {report.topic}`. The skill will resolve handles", + "To fix: in a fresh agent session (Claude Code, Codex, Hermes, Gemini, or any runtime),", + "ensure your runtime's web-search tool is active, then", + f"rerun `/last30days {report.topic}`. The skill will resolve handles", "and communities before calling the engine this time, producing richer results.", "", "If this topic really is abstract (e.g. \"AI regulation\") and doesn't need", From c4e5d7b50c21e469f76ea9ff8f7d7b8d9f203bdb Mon Sep 17 00:00:00 2001 From: jesus alberto cornelio <365diascollaboration@gmail.com> Date: Thu, 18 Jun 2026 12:51:24 -0400 Subject: [PATCH 14/50] fix: add exit 0 to check-config.sh to prevent hook error on session start With set -euo pipefail active, the final conditional expression '[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"' returns exit 1 when LAST_RUN_LINE is empty (no prior run yet). This causes every Claude Code session to show 'SessionStart:startup hook error' even when the plugin is correctly configured. Fixes #449 and #440 and #424. Signed-off-by: jesus alberto cornelio <365diascollaboration@gmail.com> --- hooks/scripts/check-config.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 4189027..b446946 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -171,3 +171,4 @@ else echo " 100 free credits, no credit card — scrapecreators.com" echo " last30days has no affiliation with any API provider." fi +exit 0 From b6e79032c22675bd4df30821c90fd322bcb4c1c8 Mon Sep 17 00:00:00 2001 From: jesus alberto cornelio <365diascollaboration@gmail.com> Date: Thu, 18 Jun 2026 13:04:41 -0400 Subject: [PATCH 15/50] fix: guard os.killpg/getpgid with hasattr for Windows compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spawn path already guards os.setsid with hasattr(os, 'setsid'), but the timeout cleanup path called os.killpg/os.getpgid unconditionally. On Windows these attributes don't exist, so any subprocess timeout raised AttributeError instead of a clean SubprocTimeout — making all YouTube transcript fetches silently fail on Windows. Mirror the existing spawn guard: check hasattr before the group-kill and fall back to proc.kill(), also add AttributeError to the caught exception set as a backstop. Fixes #588. Also related to #156. Signed-off-by: jesus alberto cornelio <365diascollaboration@gmail.com> --- skills/last30days/scripts/lib/subproc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.py index bc5772e..74e07d1 100644 --- a/skills/last30days/scripts/lib/subproc.py +++ b/skills/last30days/scripts/lib/subproc.py @@ -81,8 +81,11 @@ def run_with_timeout( stdout, stderr = proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: try: - os.killpg(os.getpgid(proc.pid), signal.SIGTERM) - except (ProcessLookupError, PermissionError, OSError): + if hasattr(os, "killpg") and hasattr(os, "getpgid"): + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + else: + proc.kill() + except (ProcessLookupError, PermissionError, OSError, AttributeError): proc.kill() proc.wait(timeout=5) raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s") From cffca3aeb806a3405b8568aa13a4c66975fb955c Mon Sep 17 00:00:00 2001 From: jesus alberto cornelio <365diascollaboration@gmail.com> Date: Thu, 18 Jun 2026 13:15:05 -0400 Subject: [PATCH 16/50] fix: wrap proc.wait in try/except to guarantee SubprocTimeout is raised MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile review on #624: if SIGTERM is ignored and proc.wait(timeout=5) expires, subprocess.TimeoutExpired would leak to callers who only catch SubprocTimeout. Wrap with try/except — force-kill on second timeout and do a final blocking wait before raising SubprocTimeout. Signed-off-by: jesus alberto cornelio <365diascollaboration@gmail.com> --- skills/last30days/scripts/lib/subproc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.py index 74e07d1..365f0e6 100644 --- a/skills/last30days/scripts/lib/subproc.py +++ b/skills/last30days/scripts/lib/subproc.py @@ -87,7 +87,11 @@ def run_with_timeout( proc.kill() except (ProcessLookupError, PermissionError, OSError, AttributeError): proc.kill() - proc.wait(timeout=5) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s") return SubprocResult( From 891797248d1d034949f64ac63e8248ea12275c37 Mon Sep 17 00:00:00 2001 From: jesus alberto cornelio <365diascollaboration@gmail.com> Date: Thu, 18 Jun 2026 13:21:52 -0400 Subject: [PATCH 17/50] fix: bound the SIGKILL escalation wait to prevent indefinite hang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile review on #624: the unbounded proc.wait() after proc.kill() can block forever on Linux if the process is in D-state (uninterruptible I/O wait) since SIGKILL cannot terminate it. Add timeout=5 and swallow the second TimeoutExpired — leave the process as a zombie rather than hanging the caller indefinitely. Signed-off-by: jesus alberto cornelio <365diascollaboration@gmail.com> --- skills/last30days/scripts/lib/subproc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/subproc.py b/skills/last30days/scripts/lib/subproc.py index 365f0e6..24fc421 100644 --- a/skills/last30days/scripts/lib/subproc.py +++ b/skills/last30days/scripts/lib/subproc.py @@ -91,7 +91,10 @@ def run_with_timeout( proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() - proc.wait() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass # process unkillable (e.g. D-state); leave as zombie raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s") return SubprocResult( From 9f77d3ac314d95052c6718d2605f934c24a9ee0a Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Mon, 22 Jun 2026 07:26:36 -0700 Subject: [PATCH 18/50] fix: exclude dev artifacts from Hermes skill scan (#656) --- .clawhubignore | 74 ++++++++++++++++++++++++++++++++++++++++++-------- .skillignore | 67 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 12 deletions(-) create mode 100644 .skillignore diff --git a/.clawhubignore b/.clawhubignore index 0158cb8..ac6906e 100644 --- a/.clawhubignore +++ b/.clawhubignore @@ -1,18 +1,68 @@ -# Exclude binary assets and dev/test artifacts from ClawHub bundle -assets/ -docs/ -fixtures/ -tests/ -plans/ -agents/ -variants/ -release-notes.md -SPEC.md -TASKS.md -SKILL-original.md +# ClawHub/Hermes packaging exclusions for repository-root scans. +# Mirrors .skillignore so non-runtime docs/dev artifacts stay out of the +# public bundle and install-time skill security scan. + +# VCS, local envs, caches, and generated outputs +.git/ +.venv/ +__pycache__/ +*.pyc +*.log *.jsonl *.mp3 *.jpeg *.jpg *.png *.gif +assets/ +skills/last30days/assets/ +.DS_Store +.coverage +htmlcov/ +dist/ +work/ +print/ + +# Repo/dev automation and host-specific package metadata +.github/ +.agents/ +.claude-plugin/ +hooks/ +mcp/ +gemini-extension.json +greptile.json +pyproject.toml + +# Non-runtime docs, plans, release notes, fixtures, and tests +docs/ +fixtures/ +tests/ +plans/ +agents/ +variants/ +media/ +README.md +CHANGELOG.md +AGENTS.md +CLAUDE.md +CONCEPTS.md +CONFIGURATION.md +CONTRIBUTORS.md +HERMES_SETUP.md +release-notes.md +SKILL-original.md +SPEC.md +TASKS.md + +# Dev/eval scripts shipped inside the skill tree but not needed at runtime +skills/last30days/scripts/build-skill.sh +skills/last30days/scripts/compare.sh +skills/last30days/scripts/evaluate_search_quality.py +skills/last30days/scripts/setup-keychain.sh +skills/last30days/scripts/setup-pass.sh +skills/last30days/scripts/test_device_auth.py +skills/last30days/scripts/test-v1-vs-v2.sh +skills/last30days/scripts/verify_v3.py + +# Keep visible: optional runtime watchlist/store/briefing feature scripts +# (`watchlist.py`, `store.py`, and `briefing.py`). diff --git a/.skillignore b/.skillignore new file mode 100644 index 0000000..cbcfb0c --- /dev/null +++ b/.skillignore @@ -0,0 +1,67 @@ +# Hermes install-time scanner/package exclusions for repository-root scans. +# Keep the public bundle focused on the runtime skill under skills/last30days/. + +# VCS, local envs, caches, and generated outputs +.git/ +.venv/ +__pycache__/ +*.pyc +*.log +*.jsonl +*.mp3 +*.jpeg +*.jpg +*.png +*.gif +assets/ +skills/last30days/assets/ +.DS_Store +.coverage +htmlcov/ +dist/ +work/ +print/ + +# Repo/dev automation and host-specific package metadata +.github/ +.agents/ +.claude-plugin/ +hooks/ +mcp/ +gemini-extension.json +greptile.json +pyproject.toml + +# Non-runtime docs, plans, release notes, fixtures, and tests +docs/ +fixtures/ +tests/ +plans/ +agents/ +variants/ +media/ +README.md +CHANGELOG.md +AGENTS.md +CLAUDE.md +CONCEPTS.md +CONFIGURATION.md +CONTRIBUTORS.md +HERMES_SETUP.md +release-notes.md +SKILL-original.md +SPEC.md +TASKS.md + +# Dev/eval scripts shipped inside the skill tree but not needed at runtime +skills/last30days/scripts/build-skill.sh +skills/last30days/scripts/compare.sh +skills/last30days/scripts/evaluate_search_quality.py +skills/last30days/scripts/setup-keychain.sh +skills/last30days/scripts/setup-pass.sh +skills/last30days/scripts/test_device_auth.py +skills/last30days/scripts/test-v1-vs-v2.sh +skills/last30days/scripts/verify_v3.py + +# Keep visible: optional runtime watchlist/store/briefing feature scripts +# (`watchlist.py`, `store.py`, and `briefing.py`). From 9124035d7eb538bd3321e989a2dd0af927677698 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:13:55 -0700 Subject: [PATCH 19/50] chore(deps): bump trufflesecurity/trufflehog from 3.95.2 to 3.95.5 (#647) Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.95.2 to 3.95.5. - [Release notes](https://github.com/trufflesecurity/trufflehog/releases) - [Commits](https://github.com/trufflesecurity/trufflehog/compare/17456f8c7d042d8c82c9a8ca9e937231f9f42e26...d411fff7b8879a62509f3fa98c07f247ac089a51) --- updated-dependencies: - dependency-name: trufflesecurity/trufflehog dependency-version: 3.95.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index d25bbb2..f0de441 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -53,7 +53,7 @@ jobs: # examples; use obvious dummy values and env-based auth patterns instead. - name: Run TruffleHog OSS secret scan if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' - uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2 + uses: trufflesecurity/trufflehog@d411fff7b8879a62509f3fa98c07f247ac089a51 # v3.95.5 continue-on-error: true with: path: ./ From 5a74d175593847e2b8527f8b59081720b36d36e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:14:08 -0700 Subject: [PATCH 20/50] chore(deps): bump actions/download-artifact from 4 to 8 (#648) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4 to 8. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4...v8) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '8' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f17377a..3cb36d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -125,7 +125,7 @@ jobs: contents: write steps: - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: path: dist merge-multiple: true From 8c2b1c39856e437136daec8e63b0975ebe49ae75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:14:21 -0700 Subject: [PATCH 21/50] chore(deps-dev): bump pytest from 9.0.3 to 9.1.0 (#649) Bumps [pytest](https://github.com/pytest-dev/pytest) from 9.0.3 to 9.1.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/9.0.3...9.1.0) --- updated-dependencies: - dependency-name: pytest dependency-version: 9.1.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 19a4324..782fb30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [] [dependency-groups] dev = [ - "pytest>=9.0.3,<10", + "pytest>=9.1.0,<10", "pytest-cov>=7,<8", ] diff --git a/uv.lock b/uv.lock index 8c28f2f..b72fd26 100644 --- a/uv.lock +++ b/uv.lock @@ -119,7 +119,7 @@ dev = [ [package.metadata.requires-dev] dev = [ - { name = "pytest", specifier = ">=9.0.3,<10" }, + { name = "pytest", specifier = ">=9.1.0,<10" }, { name = "pytest-cov", specifier = ">=7,<8" }, ] @@ -152,7 +152,7 @@ wheels = [ [[package]] name = "pytest" -version = "9.0.3" +version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -161,9 +161,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" }, ] [[package]] From a8506fc31555bb7b730ca8b29a71ce4c81a10ff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:14:52 -0700 Subject: [PATCH 22/50] chore(deps): bump actions/setup-go from 5 to 6 (#650) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5 to 6. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3cb36d4..66dc27a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: stable From f9a016e1309cbc1fe3329b220bd76997ae7b8076 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:15:04 -0700 Subject: [PATCH 23/50] chore(deps): bump actions/upload-artifact from 4 to 7 (#651) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 66dc27a..c13e7d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,7 @@ jobs: subject-path: dist/last30days.skill - name: Upload skill artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: last30days-skill path: dist/last30days.skill @@ -112,7 +112,7 @@ jobs: --output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb - name: Upload .mcpb artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }} path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb From 94006736088a8e6c8ae179cccbad7e63f47a8136 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Mon, 22 Jun 2026 20:16:04 -0400 Subject: [PATCH 24/50] fix(html_render): escape META marker text to close stored-XSS path (#598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _promote_meta_marker interpolated the captured META text straight into
{text}
with no escaping. The marker is deliberately exempted from the comment-strip pass, and the markdown reaching this stage can include LLM-synthesized content derived from untrusted web/social bodies — the same prompt-injection surface the link-scheme allowlist already guards. A crafted `` reaching the raw-form fallback would render as live markup in the saved, shareable HTML artifact. Normalize with html.unescape then html.escape so both the markdown-escaped and the raw fallback forms are escaped exactly once. Legitimate date/source-name markers (the only thing current callers emit) render unchanged. Not reachable via current internal callers, which feed only plain dates and source names; this hardens the boundary against future synthesized-content callers. --- skills/last30days/scripts/lib/html_render.py | 10 +++++- tests/test_html_render.py | 35 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/html_render.py b/skills/last30days/scripts/lib/html_render.py index 2a42354..f07efa5 100644 --- a/skills/last30days/scripts/lib/html_render.py +++ b/skills/last30days/scripts/lib/html_render.py @@ -574,7 +574,15 @@ def _promote_meta_marker(body: str) -> str: Both collapse to ``
TEXT
``. """ def replace(match: re.Match[str]) -> str: - text = match.group(1).strip() + # The marker survives the comment-strip pass, and the markdown reaching + # this point can include LLM-synthesized content derived from untrusted + # web/social bodies. The escaped-form branches below carry text the + # markdown pass already entity-escaped, while the raw-form fallbacks do + # not — so normalize with unescape, then escape exactly once. A crafted + # `` thus cannot render as live + # markup in the saved, shareable HTML artifact, and legitimate + # date/source-name markers render unchanged. + text = html.escape(html.unescape(match.group(1).strip())) return f'
{text}
' # Escaped form (most common after markdown conversion) diff --git a/tests/test_html_render.py b/tests/test_html_render.py index 7353e47..6e1344e 100644 --- a/tests/test_html_render.py +++ b/tests/test_html_render.py @@ -238,6 +238,41 @@ class HtmlRenderBehaviorTests(unittest.TestCase): self.assertNotIn(". Its text can come from LLM-synthesized content + derived from untrusted source bodies, so a crafted + `` must be escaped, not rendered. + """ + md = "intro\n\n\n\nmore" + body = html_render._markdown_to_html(md) + body = html_render._wrap_engine_footer(body) + body = html_render._promote_meta_marker(body) + self.assertNotIn(" -->" + ) + self.assertNotIn("<img src=x onerror=alert(1)>' + ) + + def test_meta_marker_preserves_plain_text(self): + """Legitimate date/source-name markers render unchanged (no double-escape).""" + body = html_render._promote_meta_marker( + "" + ) + self.assertEqual( + body, + '
2026-01-01 to 2026-01-31 · reddit, x
', + ) + def test_markdown_links_allow_relative_url(self): rendered = html_render._markdown_to_html("[home](/path?x=1#section)") self.assertIn( From 10e19f00fd8c4571e0245af96eca72c4f97684fe Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:16:15 -0700 Subject: [PATCH 25/50] chore: remove committed test-run artifact (#602) --- .gitattributes | 1 - .gitignore | 1 + test-run.log | 11 ----------- 3 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 test-run.log diff --git a/.gitattributes b/.gitattributes index 1139372..436a296 100644 --- a/.gitattributes +++ b/.gitattributes @@ -25,7 +25,6 @@ assets/ export-ignore # Historical + repo-only manifests SPEC.md export-ignore TASKS.md export-ignore -test-run.log export-ignore CONTRIBUTORS.md export-ignore HERMES_SETUP.md export-ignore CHANGELOG.md export-ignore diff --git a/.gitignore b/.gitignore index bd4656e..de09d3c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Private benchmark / evaluation artifacts — never push to upstream docs/comparison-results/ +test-run.log scripts/evaluate-synthesis.py scripts/generate-synthesis-inputs.py fixtures/polymarket_sample.json diff --git a/test-run.log b/test-run.log deleted file mode 100644 index c3b2b5a..0000000 --- a/test-run.log +++ /dev/null @@ -1,11 +0,0 @@ -📁 Output directory: /Users/mvanhorn/last30days-skill-private/docs/test-results/v1-vs-v2-20260206-231338 - -📦 Backing up current V2 SKILL.md... -📥 Installing V1 SKILL.md from upstream... - ✅ V1 installed (stripped: context:fork, agent:Explore, disable-model-invocation) - -========================================== - Running V1 — 17 queries -========================================== - -[V1] (1/17) prompting techniques for chatgpt for legal questions [PROMPTING+TOOL] From f092c9f5ba48af8e4acca292b067669bca35da27 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Mon, 22 Jun 2026 20:16:33 -0400 Subject: [PATCH 26/50] fix(cookie_extract): close world-readable window on temp cookie copy (#599) * fix(cookie_extract): close world-readable window on temp cookie copy _query_cookies_db copies the browser cookie DB (live X auth_token/ct0 session secrets) into a system temp file, then chmods it 0600. But shutil.copy2 copies the source file's mode onto the destination: Firefox cookies.sqlite is commonly 0644 (looser on WSL /mnt/c mounts), so between the copy and the chmod the decrypted secrets sat world-readable in shared /tmp, race-readable by another local user. Use shutil.copyfile, which writes content only and leaves the 0600 perms that mkstemp created intact, so the copy is never readable by others. The existing chmod is kept as defense-in-depth. Adds a regression test asserting the temp copy is 0600 the instant it exists, before the lock chmod runs. * test(cookie_extract): skip world-readable test on Windows mkstemp creates 0o666 (not 0o600) on Windows and the POSIX permission exposure does not apply there (_lock_temp_cookie_copy already no-ops on nt), so the 0o600 assertion would fail spuriously on a Windows CI runner. Matches the skipif guard on the sibling test_temp_cookie_db_copy_is_owner_only. --- .../last30days/scripts/lib/cookie_extract.py | 7 +++- tests/test_cookie_extract.py | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/cookie_extract.py b/skills/last30days/scripts/lib/cookie_extract.py index 5cc244e..23b24f3 100644 --- a/skills/last30days/scripts/lib/cookie_extract.py +++ b/skills/last30days/scripts/lib/cookie_extract.py @@ -157,7 +157,12 @@ def _query_cookies_db( tmp_path = None try: tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite") - shutil.copy2(str(db_path), tmp_path) + # mkstemp creates the file 0600. copy2 would copy the source's mode + # (Firefox cookies.sqlite is commonly 0644, looser on WSL /mnt/c) onto + # the temp file, leaving live session secrets world-readable in shared + # /tmp until the chmod below runs. copyfile writes content only and + # leaves the 0600 perms intact, closing that window. + shutil.copyfile(str(db_path), tmp_path) _lock_temp_cookie_copy(tmp_path) conn = sqlite3.connect(tmp_path) diff --git a/tests/test_cookie_extract.py b/tests/test_cookie_extract.py index 3059085..0346240 100644 --- a/tests/test_cookie_extract.py +++ b/tests/test_cookie_extract.py @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest +from lib import cookie_extract from lib.cookie_extract import ( extract_cookies, extract_firefox_cookies, @@ -123,6 +124,38 @@ class TestExtractFirefoxCookies: assert result == {"auth_token": "tok_abc123"} + @pytest.mark.skipif(os.name == "nt", reason="POSIX permission model does not apply on Windows; mkstemp is 0o666 there") + def test_temp_cookie_copy_never_world_readable(self, tmp_path): + """The temp copy must be private the instant it exists, not only after + the lock chmod. Regression for the TOCTOU window where copy2 widened the + 0600 mkstemp file to the source's 0644 before _lock_temp_cookie_copy ran. + """ + db_path = tmp_path / "cookies.sqlite" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE moz_cookies (name TEXT NOT NULL, value TEXT NOT NULL, host TEXT NOT NULL)" + ) + conn.execute( + "INSERT INTO moz_cookies (name, value, host) VALUES (?, ?, ?)", + ("auth_token", "tok_abc123", ".x.com"), + ) + conn.commit() + conn.close() + os.chmod(db_path, 0o644) # loose source perms, as Firefox ships them + + observed = {} + real_lock = cookie_extract._lock_temp_cookie_copy + + def spy(path): + # Mode of the copy as it exists right after copyfile, before chmod. + observed["mode_after_copy"] = os.stat(path).st_mode & 0o777 + return real_lock(path) + + with patch.object(cookie_extract, "_lock_temp_cookie_copy", side_effect=spy): + _query_cookies_db(db_path, ".x.com", ["auth_token"]) + + assert observed["mode_after_copy"] == 0o600 + def test_valid_cookies_extracted(self, mock_firefox_env): """Cookies for the target domain are returned correctly.""" profiles_dir = mock_firefox_env() From 220928f66471dca24e7512dfe834f3e67878ccce Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:16:50 -0700 Subject: [PATCH 27/50] chore: remove investigation dumps (#617) --- .gitignore | 1 + .../LEARNINGS.md | 107 ------------------ .../sessions/01-matt-vs-trevin.md | 19 ---- .../sessions/02-kanye-west.md | 15 --- .../sessions/03-kevin-rose.md | 11 -- .../sessions/04-peter-steinberger.md | 12 -- .../sessions/05-lan-xuezhao.md | 14 --- 7 files changed, 1 insertion(+), 178 deletions(-) delete mode 100644 docs/investigations/2026-06-17-x-search-and-funny/LEARNINGS.md delete mode 100644 docs/investigations/2026-06-17-x-search-and-funny/sessions/01-matt-vs-trevin.md delete mode 100644 docs/investigations/2026-06-17-x-search-and-funny/sessions/02-kanye-west.md delete mode 100644 docs/investigations/2026-06-17-x-search-and-funny/sessions/03-kevin-rose.md delete mode 100644 docs/investigations/2026-06-17-x-search-and-funny/sessions/04-peter-steinberger.md delete mode 100644 docs/investigations/2026-06-17-x-search-and-funny/sessions/05-lan-xuezhao.md diff --git a/.gitignore b/.gitignore index de09d3c..9f6998c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ docs/v2.1-tweets.md docs/30-day-anniversary-thread.md docs/30-day-anniversary-tweets.md variants/open/references/research.md +docs/investigations/ # OS / tool files .DS_Store diff --git a/docs/investigations/2026-06-17-x-search-and-funny/LEARNINGS.md b/docs/investigations/2026-06-17-x-search-and-funny/LEARNINGS.md deleted file mode 100644 index f96d050..0000000 --- a/docs/investigations/2026-06-17-x-search-and-funny/LEARNINGS.md +++ /dev/null @@ -1,107 +0,0 @@ -# last30days — X search + funny + laziness: consolidated bug inventory - -**Date:** 2026-06-17 -**Engine version under test:** 3.4.0 (current `main`, commit fce934b) -**Source evidence:** five live `/last30days` debug sessions, captured per-session in `sessions/`. -**Every line/behavior claim below was re-verified against current `main`, not taken on faith from the transcripts.** - -This is the grounding document for the fix plan. It examines each problem separately, -states the verified root cause with file/line, and rates fleet impact (the fixes ship to -100k+ users — most of whom do NOT have working browser-cookie X auth, so keyless paths and -honesty matter most at scale). - ---- - -## A. X search bugs - -### A1 — `from:{handle}` AND-bug (the headline X bug) · fleet impact: HIGH (every person/entity topic with X auth) -**Symptom:** Passing `--x-handle=mvanhorn` (or xuezhao, steipete, …) returns ~0 of the person's own tweets. The "X" column fills with unrelated keyword collisions instead. -**Verified root cause:** `bird_x.search_handles._search_one_handle` builds -`from:{handle} {core_topic} since:{from_date}` whenever `core_topic` is truthy -(`bird_x.py` ~373). X search is literal AND — so it only matches the person's tweets that *also contain the topic words* (usually their own name), which they never tweet. The unfiltered `from:{handle} since:` branch only fires when `topic is None`, and the caller (`pipeline.py:861`) always passes a truthy `topic`. So the real-timeline path is effectively unreachable. -**Proven fix exists in-code:** running `from:{handle} since:` (no AND) returned 40 real tweets in-session. -**User requirement (2026-06-17):** X must surface tweets **FROM** the person (engagement-weighted) — this is the `from:` lane, fixed to drop the topic-AND for person/handle topics. - -### A2 — No mention/“about” lane at all · fleet impact: HIGH -**Symptom:** Tweets *mentioning* the person (`@handle`) or *to* them never get collected as a category. -**Verified root cause:** the pipeline only runs `from:` (author) searches and keyword subqueries. There is no `@handle` / `to:handle` mention query anywhere (`bird_x.search_handles` does `from:` only; no other call site builds a mention query). -**User requirement (2026-06-17):** X must ALSO surface tweets **TO/ABOUT** the person (engagement-weighted) — a new mention lane, weighted by likes/reposts, deduped against the `from:` lane. - -### A3 — Handle search is silent on success · fleet impact: MED (observability; caused 3 wrong diagnoses in-session) -**Verified root cause:** `_search_one_handle` only `_log()`s on timeout / OSError / non-zero / invalid-JSON. A successful OR empty handle search emits zero log lines (`bird_x.py` ~382-405). The only `[Bird] Searching:` lines in task logs are the Phase-1 keyword subqueries, so the `from:` search looks like it never ran. -**Fix:** log the query + result count on success, like the keyword path. - -### A4 — `--diagnose` false-green for X · fleet impact: HIGH (fleet-wide trust bug) -**Symptom:** `--diagnose` reports `bird_authenticated: true` / `bird_username: "env AUTH_TOKEN"` even when X is effectively returning nothing, and labels the source `env AUTH_TOKEN` when the real lane is live browser-cookie extraction (`_AUTH_TOKEN_SOURCE: browser`). -**Verified root cause:** `pipeline.diagnose` reads `env.get_x_source_status` → `bird_status["authenticated"]` (`env.py:864`), a static credential-presence check, not a runtime probe. -**Fix:** diagnose should do a real 1-tweet probe and report the true auth lane (browser vs env vs keychain). At scale most users have NO working X auth; the footer/coverage must say "X: 0 — no working auth" honestly instead of showing green. - -### A5 — Off-topic X pollution · **CORRECTED at execution (2026-06-17)** -**Symptom:** "compound interest / compound nevus" junk filled the X column in the Matt-vs-Trevin run even though bird was failing the whole run. -**Original (transcript) hypothesis — DISPROVEN against current code:** the debug session blamed Digg's X-post enrichment side channel. False: Digg-enriched X posts live in `metadata["posts"]` and render ONLY as Digg-cluster quotes via `render._digg_posts_for` (returns `[]` for non-digg sources). Nothing adds them to `items_by_source["x"]`. Digg does NOT pollute the X column — a red herring. -**Actual verified root cause:** the X-column "compound" junk came from the A6 strongest-token fallback querying a bare generic token, whose results ARE parsed into X items. **A6 is the real fix; the planned "entity-filter Digg X posts" unit (U1) was dropped as a non-occurring path.** -**Residual minor note:** Digg-cluster X-post *quotes* could be tangentially off-topic, but they render as Digg quotes (not X items) and come from already-topic-matched clusters — low-severity, defer. - -### A6 — Strongest-token fallback collapses to a bare generic token · fleet impact: MED -**Verified root cause:** the last-chance retry picks `strongest = max(candidates, key=len)` then queries `f"{strongest} since:{from_date}"` (`bird_x.py:339-341`). For "trevin chow ai agents compound" it picks the longest token — `compound` — and dumps generic "compound" results into the shared X pool. -**Fix:** don't collapse to a single generic token; keep an entity anchor (handle/name) in the retry, or drop the subquery rather than over-broaden. - -### A7 — Name-collision / disambiguation gap · fleet impact: MED-HIGH (every mid-profile person) -**Symptom:** "Kevin Rose" pulled Kevin Warsh (Fed chair), Leon Rose (Knicks), Kevin Durant, Kevin Hart — 55 items, ~zero on-topic. "Lan Xuezhao" pulled Lanzhou noodle + cdrama edits. -**Root cause:** bare-name keyword subqueries are too collision-prone for mid-profile people; the engine has no disambiguation anchor (handle/domain/context) baked into the subqueries by default. -**Note:** larger relevance problem than the deterministic A1-A6 bugs. Candidate for a follow-up scope rather than the first PR. **Call out for the plan.** - -### NOT bugs — stop re-chasing (verified): -- **Bird is not broken.** Vendored `scripts/lib/vendor/bird-search/` v0.8.0, MIT, zero deps; works against live cookies. Peter deprecating the public `@steipete/bird` package is irrelevant — it's vendored. -- **`@steipete/sweet-cookie` is optional**, never a required dep (the "not installed" line is from the optional browser-cookie lane). -- **The parser is correct.** Metrics nest under `item["engagement"]` and `item["date"]`, not top-level — earlier "metrics = None" was a debug print-key mistake. - ---- - -## B. Funny things not showing up - -### B1 — Best Takes renders empty in normal use (the structural root cause) · fleet impact: HIGH -**Symptom:** Across the Kanye and Steinberger runs, no `## Best Takes` section appeared; the funniest lines ("Is anyone surprised? It's called TurkiYe", "I bet one of his kids will be a bully") never reached the synthesis. -**Verified root cause (two compounding):** -1. `rerank.score_fun` LLM-scores only when a reasoning `provider` exists in the **engine subprocess** (`pipeline.py:534`, `provider=reasoning_provider`). In normal `/last30days` usage the engine subprocess has NO paid reasoning provider (the hosting model is the planner but can't be called back into the subprocess), so fun scores come from the heuristic fallback (~38). `_render_best_takes` requires `fun_score >= _BEST_TAKE_FUNNY_FLOOR (40)` AND effective ≥ threshold (70 at medium) — so Best Takes returns `[]`. The vote-weighting shipped in #592 is moot without LLM fun scoring. -2. `_render_candidate` (the compact EVIDENCE block) renders top comments ONLY for the representative items of the top-`cluster_limit` (8) clusters (`render.py:166-171, 1208`). Funny comments on lower-ranked or non-representative items are never in the synthesis block at all. - -### B2 — The fun judgment is in the wrong place · design finding -The hosting model (Claude) is an excellent fun judge and *does* have an API. The engine subprocess is the one that can't LLM-score. So the fix direction is to **move fun SELECTION to the hosting model**: have the engine surface a comment-rich, vote-scored "Best Takes candidates" / "Top Community Comments" block (across more than the top-8 representative items) inside the EVIDENCE envelope, and have SKILL.md make weaving 2-3 of the funniest a hard gate. Vote scores (the #592 signal) become the ranking input the model selects from. **Soft vs hard fork — call out for the plan:** engine-surfaces-candidates + model-selects (structural) vs. just lowering the Best Takes heuristic floor (weak). - ---- - -## C. The AI is lazy / not reading what it finds - -### C1 — Compact stdout treated as the whole dataset · fleet impact: HIGH (every run) -**Symptom:** The model synthesized a news-shaped report off the compact EVIDENCE block and never opened the saved raw `.md` until prodded — missing comments, the subject's own quotes, a BULLY DELUXE release two days out, a Dutch court win, an allegation. -**Root cause (behavioral + structural):** the compact EVIDENCE block is a lossy index (top-8 clusters, representative items, truncated). The richer per-source comment/quote layer lives deeper in the saved raw file, which the model treats as optional. SKILL.md's "weave the funniest takes" / PRE-PRESENT SELF-CHECK exist but are skipped as formalities. -**Fix direction:** (a) structural — get the high-value layer (top comments with votes, the subject's own posts) INTO the synthesis-facing block so the model can't miss it; (b) behavioral — turn the self-check into an enforced gate (e.g. require ≥2 verbatim attributed quotes, lead with most-recent dated/upcoming event, test the thesis against highest-engagement items). - -### C2 — Fabricated / reconstructed citation URL · fleet impact: HIGH (correctness; a wrong link looks authoritative) -**Symptom:** In the Steinberger run the model linked @OtsileKole to a status ID that belonged to a different account — reconstructed from memory instead of copied from the raw file (a LAW 8 violation). -**Fix:** every URL in output copied verbatim from the raw data; plain-text fallback if not found; never reconstruct a status ID. - -### C3 — Meta-commentary about tooling leaks into the deliverable · fleet impact: MED (output quality) -**Symptom:** "the social-listening engine struck out … 'Kevin Rose' collided with Kevin Warsh …" — narrating the engine's own failure inside the user-facing report. -**Fix:** the synthesis presents what's true about the subject and quietly drops junk; engine-health notes belong in the footer/diagnostics, not the prose. - ---- - -## D. Formatting integrity (user constraint #4) - -### D1 — Cascading-repeat mangling of the "What I learned" block · fleet impact: unknown -**Symptom:** In the Kanye run the synthesis block repeated the same paragraphs many times with progressively growing left-indentation — a badly corrupted render. -**Status:** likely a model-output / terminal-streaming artifact rather than an engine-code bug (it's in the model's emitted prose, not the engine stdout). **Needs reproduction before claiming an engine cause — open question.** -**Hard constraint on ALL fixes:** changes to the engine's emitted output (especially adding a comments/Best-Takes-candidates block to the EVIDENCE envelope) must NOT break the envelope markers, the PASS-THROUGH FOOTER, the badge, or the `What I learned:` contract. Format is already fragile; every output-touching unit must preserve it and be diffed against a real run. - ---- - -## Priority for the fleet (100k+ users, most without working X auth) - -1. **A5 Digg X-pollution** + **A6 fallback drift** — hits everyone, no X auth needed. -2. **B1/B2 funny (Best Takes dark) + C1 laziness** — the product's headline value ("funniest comments on the whole internet"), every run, keyless. -3. **A4 diagnose false-green** — fleet-wide honesty. -4. **A1 from:-AND-bug + A2 mention lane** — the explicit FROM+ABOUT-with-weight requirement; deterministic, high correctness win for the X-auth subset. -5. **A3 silent logging** — cheap observability, do alongside. -6. **A7 disambiguation** + **D1 formatting cascade** — likely follow-up scope (bigger/uncertain). diff --git a/docs/investigations/2026-06-17-x-search-and-funny/sessions/01-matt-vs-trevin.md b/docs/investigations/2026-06-17-x-search-and-funny/sessions/01-matt-vs-trevin.md deleted file mode 100644 index 8e159f5..0000000 --- a/docs/investigations/2026-06-17-x-search-and-funny/sessions/01-matt-vs-trevin.md +++ /dev/null @@ -1,19 +0,0 @@ -# Session 01 — `/last30days Matt Van Horn vs Trevin Chow` - -**What this session proved:** the X handle search runs but is poisoned by the topic-AND, fails silently, and the X column is actually Digg-side-channel pollution. - -## Key evidence (verbatim engine/log lines) -- Engine fired keyword X searches only in the visible log: `[Bird] Searching: matt van horn printing press since:2026-05-19`. No `from:mvanhorn` line — because `search_handles` logs nothing on success (→ A3). -- GitHub person-mode worked (`[GitHub] Person-mode search for @mvanhorn`), so per-entity targeting was wired; X was the broken lane. -- The 13 "X posts" were `@imAbhishek9596`, `@lebojoycechauke`, etc. — none authored by @mvanhorn. -- Final verified mechanism: handle search built `from:mvanhorn matt van horn since:...` (topic AND'd onto the timeline → ~0). The clean `from:mvanhorn since:...` returned 40 real tweets. -- The off-topic "compound interest / compound nevus" X items came in through Digg's X-enrichment side channel (`[Digg] post-dedupe enriched ... clusters with X posts`), NOT bird — bird was failing the whole run under the 2-entity parallel fanout (`Bird search failed`). -- Strongest-token fallback collapsed `trevin chow ai agents compound` → bare token `compound`. - -## Red herrings burned (NOT bugs) -- "sweet-cookie missing = broken" — optional browser-cookie helper, never required. -- "auth dead" — standalone test failed only because `get_config()` + `set_credentials()` weren't called to inject cookies. -- "metrics = None parser bug" — print read wrong keys; parser nests under `engagement`/`date`. - -## Bugs surfaced → inventory IDs -A1 (from-AND), A3 (silent log), A4 (diagnose false-green), A5 (Digg pollution), A6 (fallback drift), A2 (no mention lane). diff --git a/docs/investigations/2026-06-17-x-search-and-funny/sessions/02-kanye-west.md b/docs/investigations/2026-06-17-x-search-and-funny/sessions/02-kanye-west.md deleted file mode 100644 index 1b95588..0000000 --- a/docs/investigations/2026-06-17-x-search-and-funny/sessions/02-kanye-west.md +++ /dev/null @@ -1,15 +0,0 @@ -# Session 02 — `/last30days Kanye West` - -**What this session proved:** the funniest, highest-engagement comments never reach the synthesis; the model synthesizes a news report off the lossy compact block; and the output formatting can corrupt. - -## Key evidence -- The best line of the month — `"Is anyone surprised? It's called TurkiYe"` (u/Drekkful, 764 upvotes), under a 3,895-upvote r/hiphopheads thread — was never mentioned. The model only read the compact "Ranked Evidence Clusters" (title + one snippet/item); the comment lived deeper in the saved raw file's per-source section. -- Also missed: `"I bet one of his kids will be a bully"` (BULLY album callback), `"anyone interested in my kidney? i need tickets"`, plus a 21,707-like TikTok comment. -- Missed real NEWS too (not just jokes): BULLY DELUXE dropping in 2 days, a Dutch court win clearing Arnhem shows (contradicted the "Europe is collapsing" thesis), an Italy ban, a serious allegation. -- Model's own root cause: *"I treated the compact stdout as the dataset. It is a lossy index … the actual comment text, full post bodies, and top-comment upvote counts only live in the raw file's All Items by Source section."* - -## Formatting corruption (constraint #4) -- The emitted `What I learned:` block repeated the same paragraphs many times with progressively growing left-indentation — a badly cascaded render (→ D1). Likely a model/terminal artifact; needs reproduction. - -## Bugs surfaced → inventory IDs -B1 (Best Takes empty / comments not in synthesis block), C1 (compact-as-dataset laziness), D1 (formatting cascade), plus contradiction-pass and recency-pass gaps in synthesis behavior. diff --git a/docs/investigations/2026-06-17-x-search-and-funny/sessions/03-kevin-rose.md b/docs/investigations/2026-06-17-x-search-and-funny/sessions/03-kevin-rose.md deleted file mode 100644 index b1c1f40..0000000 --- a/docs/investigations/2026-06-17-x-search-and-funny/sessions/03-kevin-rose.md +++ /dev/null @@ -1,11 +0,0 @@ -# Session 03 — `/last30days Kevin Rose` - -**What this session proved:** mid-profile names are swamped by collisions, and the model leaks tooling-failure meta-commentary into the deliverable. - -## Key evidence -- "Kevin Rose" pulled Kevin Warsh (new Fed chair), Leon Rose (Knicks), Kevin Durant, Kevin Hart — 55 engine items, ~zero genuinely about the Digg founder. The 123K-upvote r/technology haul was all other people. -- The synthesis opened with a paragraph narrating the engine's own miss: *"the social-listening engine struck out on Kevin Rose … 'Kevin Rose' collided with much louder newsmakers…"* — clutter the user must read past (→ C3). -- A disambiguated re-run (every subquery locked to "Digg founder" context) killed the Warsh/Durant noise and surfaced real signal — proving disambiguation is the lever, not a synthesis band-aid (→ A7). - -## Bugs surfaced → inventory IDs -A7 (name-collision / disambiguation), C3 (meta-commentary in output). diff --git a/docs/investigations/2026-06-17-x-search-and-funny/sessions/04-peter-steinberger.md b/docs/investigations/2026-06-17-x-search-and-funny/sessions/04-peter-steinberger.md deleted file mode 100644 index da9ced8..0000000 --- a/docs/investigations/2026-06-17-x-search-and-funny/sessions/04-peter-steinberger.md +++ /dev/null @@ -1,12 +0,0 @@ -# Session 04 — `/last30days Peter Steinberger` - -**What this session proved:** funny/community texture missed again (same root cause as Kanye), PLUS a fabricated citation URL and a pattern anchored on the weakest source. - -## Key evidence -- Genuinely funny skeptic comments existed and made ZERO of the report: `@theoldschooldk` (310 likes) *"Don't talk to me about the SPEND. Tell me what he BUILT"*, `u/alemorg` *"one of the most inefficient workflows on earth"*. Model: *"the compact emit didn't hand me a pre-built Best Takes section, so I didn't go dig for them."* -- **Fabricated URL (LAW 8):** linked `@OtsileKole` to a status ID that actually belonged to `@sabir_huss50540` — reconstructed from memory instead of copied from the raw file (→ C2). -- Dropped the subject's own first-person quote (`@OmarShahine` fireside: *"you cannot really automate taste or thoughtful design"*) — the highest-value citation tier. -- Anchored "build loops, not prompts" on a 210-view YouTube video while the raw file showed a rich multi-voice cluster (Boris Cherny coined it). Weak-source anchoring; ignored that all YouTube transcripts 429-failed (0/6). - -## Bugs surfaced → inventory IDs -B1 (Best Takes empty / funny missed), C1 (compact-as-dataset), C2 (fabricated URL), plus weak-source-anchoring + source-health-weighting gaps. diff --git a/docs/investigations/2026-06-17-x-search-and-funny/sessions/05-lan-xuezhao.md b/docs/investigations/2026-06-17-x-search-and-funny/sessions/05-lan-xuezhao.md deleted file mode 100644 index 6b7ec7c..0000000 --- a/docs/investigations/2026-06-17-x-search-and-funny/sessions/05-lan-xuezhao.md +++ /dev/null @@ -1,14 +0,0 @@ -# Session 05 — `/last30days Lan Xuezhao` - -**What this session proved (cleanest):** the from:-AND-bug AND the total absence of a mention lane — directly motivating the "FROM + TO/ABOUT, with weight" requirement. - -## Key evidence -- Engine X column = finance/VC keyword collisions (`@TheValueist`, `@arnaudmercier`, `@pierskicks`) + one `@hardmaru` "Thanks, Lan!" reply. None authored by @xuezhao. -- Verified mechanism: `--x-handle=xuezhao` → `from:xuezhao lan xuezhao since:...` (topic AND) → ~0. The unfiltered `from:xuezhao since:` branch is unreachable because topic is always truthy (→ A1). -- Structural gap: *"the engine only runs from: (authored-by) and keyword searches. There is no @xuezhao / to:xuezhao mention query anywhere in the pipeline"* (→ A2). -- Manual unfiltered `from:xuezhao` pulled 12 real posts (DeepSeek take, "new AI stack is global, sovereign and embedded", SpaceX grind) — genuinely interesting, all missed by the engine. -- Mentions were rich too (practitioners asking about her transcription/Hermes-dashboard rig) — exactly the "TO/ABOUT the person, with weight" lane the user now wants. -- Engagement counts came back 0 on the manual cookie-search field — note for weighting the mention/from lanes (resolve metrics per-tweet). - -## Bugs surfaced → inventory IDs -A1 (from-AND), A2 (no mention lane), A3 (silent log), plus the FROM+ABOUT-with-weight requirement. From 561e2242483e1828ef1ab5b03ce53d28a008cf41 Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:17:47 -0700 Subject: [PATCH 28/50] test: remove static security workflow check (#633) --- tests/test_security_workflow.py | 53 --------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 tests/test_security_workflow.py diff --git a/tests/test_security_workflow.py b/tests/test_security_workflow.py deleted file mode 100644 index 088548f..0000000 --- a/tests/test_security_workflow.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] -WORKFLOW = ROOT / ".github" / "workflows" / "security.yml" -# AGENTS.md is the canonical agent-guidance file; CLAUDE.md is a one-line -# pointer (`@AGENTS.md`) so anything Claude Code-shaped reads the same source. -AGENTS = ROOT / "AGENTS.md" - - -def _workflow_text() -> str: - return WORKFLOW.read_text(encoding="utf-8") - - -def test_security_workflow_exists() -> None: - assert WORKFLOW.is_file() - - -def test_security_workflow_runs_dependency_audit_advisory_first() -> None: - text = _workflow_text() - - assert "dependency-audit:" in text - assert "uv audit --locked" in text - assert "continue-on-error: true" in text - assert "Set continue-on-error: false once a clean baseline run is confirmed" in text - - -def test_security_workflow_runs_secret_scan_for_pull_requests_and_main_pushes() -> None: - text = _workflow_text() - - assert "secret-scan:" in text - assert "trufflesecurity/trufflehog" in text - assert "github.event_name == 'pull_request'" in text - assert "github.event_name == 'push'" in text - assert "--only-verified" in text - - -def test_security_workflow_documents_advisory_policy() -> None: - text = _workflow_text() - - assert "advisory-first" in text.lower() - assert "does not block merges" in text.lower() - assert "fixtures" in text.lower() - assert "env-based auth" in text.lower() - - -def test_agent_guidance_mentions_secret_hygiene() -> None: - text = AGENTS.read_text(encoding="utf-8") - - assert "Security hygiene" in text - assert "Never commit real API keys" in text - assert "skills/last30days/scripts/lib/env.py" in text - assert "fixtures" in text From 1b832a26bce52a17d0136f502f9021fa4651ced3 Mon Sep 17 00:00:00 2001 From: Shaan Majid <70789625+shaanmajid@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:18:11 -0700 Subject: [PATCH 29/50] chore: sync MCP manifest version to 3.6.0 (#606) --- mcp/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcp/manifest.json b/mcp/manifest.json index 71f2afa..46d7305 100644 --- a/mcp/manifest.json +++ b/mcp/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "0.3", "name": "last30days-pp-mcp", "display_name": "Last30Days", - "version": "3.0.0", + "version": "3.6.0", "description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.", "author": { "name": "Matt Van Horn", From 6291ac7ec2087b35a913a24c2bd83211672ac4f6 Mon Sep 17 00:00:00 2001 From: henkyermontero Date: Mon, 22 Jun 2026 18:18:43 -0700 Subject: [PATCH 30/50] fix: prevent first-run setup wizard from being skipped on new installs (#659) The branching rule in HOW TO INVOKE said "proceed to Step 0.5" which caused models to jump past the ## Step 0: First-Run Setup Wizard on every first invocation. Result: no browser cookie extraction, no yt-dlp, no Digg CLI, WebSearch-only synthesis with no X/YouTube/TikTok. Three changes: 1. SKILL.md: Add a FIRST-RUN GATE shell command (cat .env | grep -c SETUP_COMPLETE) between STEP 0 (WebSearch) and the branching rule. The observable output forces the model to check before any research. 2. SKILL.md: Update the branching rule to route through Step 0 wizard before Step 0.45/0.5/0.55. Add named failure mode entry (2026-06-22) documenting the root cause and the Fredy Montero reproduction. 3. SKILL.md + scripts/last30days.py: Add PYTHON VERSION GATE block that tells the model to display a brew/winget install command and STOP rather than silently falling back to WebSearch-only synthesis when Python 3.12+ is absent. Improve ensure_supported_python() error message with platform-specific install commands. Co-authored-by: Fredy Montero Co-authored-by: Claude Sonnet 4.6 --- skills/last30days/SKILL.md | 29 +++++++++++++++++++++++-- skills/last30days/scripts/last30days.py | 9 ++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index 0019513..a6d4d5f 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -230,7 +230,18 @@ ToolSearch select:WebSearch WebSearch is a **deferred tool** in Claude Code v2.1.114. The frontmatter of this file authorizes it (`allowed-tools: ... WebSearch`) but the runtime lists it as "schemas are NOT loaded." Calling WebSearch without `ToolSearch select:WebSearch` first will fail or do nothing. That friction is the documented cause of the second-most-common failure mode of this skill: the model sees "WebSearch is there but deferred," takes the low-friction path, skips Step 0.5 and 0.55, and runs the engine bare with only keyword search. The output looks fine but misses founder X timelines, GitHub repo activity, and subreddit-specific threads. -Load WebSearch first. No exceptions. Then proceed to the branching rule below. +Load WebSearch first. No exceptions. Then run the first-run gate below before anything else. + +**FIRST-RUN GATE — run this Bash command immediately after loading WebSearch, before reading the topic or doing any research:** + +```bash +cat ~/.config/last30days/.env 2>/dev/null | grep -c "SETUP_COMPLETE=true" || echo "FIRST_RUN_DETECTED" +``` + +- Output is `1` → setup is complete. Continue to the branching rule below. +- Output is `FIRST_RUN_DETECTED` (or `0`) → this is a first run. Jump immediately to `## Step 0: First-Run Setup Wizard` and complete it **before doing any topic research**. Do NOT proceed to Step 0.5, do NOT load WebSearch supplements, do NOT synthesize anything. The wizard installs yt-dlp (YouTube), the Digg CLI (via `npx`), and extracts browser cookies for X/Twitter and other sources. Skipping it produces a degraded WebSearch-only result that misrepresents the skill's capability to the user. + +**Named failure mode (2026-06-22, first-run setup skip - Fredy Montero run):** Model read "proceed to Step 0.5" in the branching rule and jumped there directly, bypassing `## Step 0: First-Run Setup Wizard` at line ~339. Result: no browser cookie extraction, no yt-dlp, no Digg CLI install, WebSearch-only synthesis with no X/YouTube/TikTok data. Root cause: the branching rule named Step 0.5 as the next step without mentioning the wizard. Fix: this gate and the updated branching rule below. **STEP 1 - RUN THE ENGINE. You MUST run `scripts/last30days.py` via Bash. Do not produce output from WebSearch alone.** @@ -238,7 +249,7 @@ The single most common failure mode of this skill is the model reading this file Branching rule: -- **If the user provided a topic** (e.g. `/last30days Kanye West`, `/last30days nvidia earnings`): proceed to Step 0.5 / Step 0.55 / Step 0.75 / Research Execution below. Do not skip straight to WebSearch. WebSearch is a **supplement after** the Python engine runs (see Step 2). It is **not a substitute**. +- **If the user provided a topic** (e.g. `/last30days Kanye West`, `/last30days nvidia earnings`): confirm the first-run gate above passed (output `1`), then proceed to `## Step 0: First-Run Setup Wizard` (or skip it if already confirmed complete), then continue to Step 0.45 / Step 0.5 / Step 0.55 / Step 0.75 / Research Execution below. Do not skip straight to WebSearch. WebSearch is a **supplement after** the Python engine runs (see Step 2). It is **not a substitute**. - **If the user provided no topic**: ask the user for a topic with a single short question. Do not run research. Do not run WebSearch. Wait. If you are about to write a response without having run `scripts/last30days.py` at least once, stop. Return to Research Execution and run the engine. Every valid output from this skill includes the emoji-tree footer (`✅ All agents reported back!`) that the engine produces data for. No footer means you did not run the skill. @@ -322,6 +333,20 @@ fi LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}" ``` +**PYTHON VERSION GATE — when the Runtime Preflight Bash block above exits with a Python version error:** + +If the preflight script emits `ERROR: last30days v3 requires Python 3.12+` (or `LAST30DAYS_PYTHON must point to Python 3.12+`) and exits, you MUST: + +1. Display this message to the user: + > "The last30days engine needs Python 3.12+. Your system has an older version. Install it with one command: + > - **Mac:** `brew install python@3.12` + > - **Windows:** `winget install Python.Python.3.12` + > + > Then re-run `/last30days ` and the setup wizard will configure everything automatically." +2. **Stop.** Do not attempt research. Do not fall back to WebSearch-only synthesis. + +WebSearch-only synthesis is not equivalent to running the engine — it misses Reddit community data, X/Twitter timelines, YouTube transcripts, TikTok, and Polymarket. Presenting it without disclosure misleads the user about what was actually searched. This is the same category of failure as a WebSearch-only run with no engine footer. + **Native-search signal (web coverage).** If you (the hosting model) have your own web-search tool available — e.g. Claude Code's `WebSearch`, which STEP 0 loads — export `LAST30DAYS_NATIVE_SEARCH=1` in the same shell before invoking the engine: ```bash diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 37e0a52..61aa3f0 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -24,10 +24,15 @@ def ensure_supported_python(version_info: tuple[int, int, int] | object | None = major, minor, micro = tuple(version_info[:3]) if (major, minor) >= MIN_PYTHON: return + req = f"{MIN_PYTHON[0]}.{MIN_PYTHON[1]}" sys.stderr.write( - "last30days v3 requires Python 3.12+.\n" + f"last30days v3 requires Python {req}+.\n" f"Detected Python {major}.{minor}.{micro}.\n" - "Install and use python3.12 or python3.13, then rerun this command.\n" + f"Install with:\n" + f" Mac: brew install python@{req}\n" + f" Windows: winget install Python.Python.{req}\n" + f" Linux: sudo apt install python{req} (or pyenv install {req})\n" + f"Then rerun: python{req} setup\n" ) raise SystemExit(1) From 9849de1396f19e03301d0fb579f8ec46f50cd057 Mon Sep 17 00:00:00 2001 From: henkyermontero Date: Mon, 22 Jun 2026 18:18:56 -0700 Subject: [PATCH 31/50] feat: consent-driven first-run onboarding (cookies + ScrapeCreators signup) (#660) * feat(setup): persist ScrapeCreators API key on signup success The GitHub device-auth signup (setup --github / --device-auth) returned the ScrapeCreators API key as JSON to stdout but nothing persisted it, so a successful signup never actually configured the paid sources. - Add setup_wizard.write_api_key(): secret-safe (0o600), idempotent, reuses _open_secret_append + _format_env_value (same path as write_setup_config), and never clobbers an existing key. - Add setup_wizard.mask_api_key(): prefix + last-4 display form. - Wire both into the CLI --github/--device-auth branch: on status==success, persist the key, set results['persisted'], and mask api_key in stdout so the secret never lands in the host model's captured Bash output. Covers plan U2. * feat(skill): consent-driven first-run onboarding in Step 0 The wizard fired but ran silently: the model invoked bare `setup`, which extracts cookies + installs tools + writes SETUP_COMPLETE with zero interaction. No consent before reading browser cookies, no macOS Full Disk Access remediation, and the ScrapeCreators GitHub signup was never offered. Rewrite Step 0 as an ordered, consent-first sequence the model drives in chat (the Python subprocess can't prompt): 1. Welcome 2. Ask cookie consent BEFORE reading; on decline run with FROM_BROWSER=off (skip reads, still install yt-dlp + Digg) 3. macOS Full Disk Access remediation on permission-denied + one retry 4. Offer the ScrapeCreators GitHub signup every first run, consent before launching the browser (setup --github) 5. Confirm active sources and proceed Remove the misleading 'follow the wizard's prompts end-to-end' line and add a named onboarding contract documenting why consent is conversational. Copy avoids a hard credit count (grant is server-side). Adds tests/test_onboarding_contract.py (7 contract assertions). Covers plan U1. * docs: document consent-driven first-run onboarding - CONFIGURATION.md: new 'First-run onboarding' section covering the three consent points (cookies, Full Disk Access, ScrapeCreators GitHub signup) and automatic key persistence. - AGENTS.md: extend the optional-sources rule to note onboarding is consent-driven and model-led, and that setup --github persists the key. - CHANGELOG.md: Unreleased entry (Added + Fixed) following #659. Covers plan U3. --------- Co-authored-by: Fredy Montero --- AGENTS.md | 1 + CHANGELOG.md | 6 ++ CONFIGURATION.md | 12 +++ skills/last30days/SKILL.md | 26 +++-- skills/last30days/scripts/last30days.py | 20 ++-- skills/last30days/scripts/lib/setup_wizard.py | 57 ++++++++++ tests/test_onboarding_contract.py | 73 +++++++++++++ tests/test_setup_openclaw.py | 50 +++++++++ tests/test_setup_wizard.py | 102 ++++++++++++++++++ 9 files changed, 335 insertions(+), 12 deletions(-) create mode 100644 tests/test_onboarding_contract.py diff --git a/AGENTS.md b/AGENTS.md index 47ef987..229f97c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,7 @@ Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`. - Git remote: origin = public (`mvanhorn/last30days-skill`) - Every `lib/*.py` call to `log.source_log(...)` must pass `tty_only=False`. The default is `True`, which silently drops every line when stderr isn't a TTY (Claude Code, Codex, CI, captured output) — turning source observability into invisible failure. Enforced by `tests/test_source_log_visibility.py`. - **CLI-gated optional sources** (Digg via `digg-pp-cli`, YouTube via `yt-dlp`) activate only when `shutil.which` resolves the binary on the **agent subprocess PATH** — not merely when the file exists on disk. First-run setup installs Digg through `@mvanhorn/printing-press-library` (default `$HOME/.local/bin`); Hermes/OpenClaw gateways often need that directory on PATH. Setup must distinguish PATH-visible installs from off-PATH binaries and must not claim "now active" unless the engine gate would pass. See `docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md`. +- **First-run onboarding is consent-driven and model-led.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth) — it cannot prompt, so consent lives in `SKILL.md` Step 0: the model asks before reading cookies, surfaces the macOS Full Disk Access fix on permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do not collapse Step 0 back into a bare silent `setup` call — the consent prompts are the feature. ## Security hygiene - Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents. diff --git a/CHANGELOG.md b/CHANGELOG.md index 45a6efb..2cb4f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659). + +### Fixed +- **First-run setup no longer runs silently.** The prior Step 0 told the model to run `setup` and "follow the wizard's prompts end-to-end", but the wizard has no prompts — so onboarding extracted cookies, installed tools, and wrote `SETUP_COMPLETE` with zero interaction and never offered the ScrapeCreators signup. Reproduced 2026-06-22 (Fredy Montero, fresh macOS). + ## [3.8.0] - 2026-06-21 ### Added diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 11aa4ad..562b1e7 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -49,6 +49,18 @@ The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Docume --- +## First-run onboarding + +On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It has three consent points: + +1. **Browser cookies** - the model asks before reading anything. On yes it extracts Firefox/Safari cookies (never Chrome, to avoid a macOS Keychain prompt) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup with `FROM_BROWSER=off` (skips all cookie reads, still installs the tools). +2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry. +3. **ScrapeCreators GitHub signup** - offered on every first run. On consent it runs `setup --github`, which opens a browser for GitHub device-auth and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, Threads, Pinterest, X, and YouTube comments/transcripts activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. + +Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation is specified in `skills/last30days/SKILL.md` Step 0. + +--- + ## API keys (`.env`) The skill reads keys from a `.env` file. Two locations are supported, in priority order: diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index a6d4d5f..687b5b6 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -363,18 +363,32 @@ The engine reads `LAST30DAYS_MEMORY_DIR` from either the process env or `~/.conf ## Step 0: First-Run Setup Wizard -Before proceeding to Step 1, handle first-run setup. +Before proceeding to Step 1, handle first-run setup. **You are the conversational driver.** The Python setup script does only mechanical work (cookie reads, tool installs, the GitHub device-auth flow) - it CANNOT prompt the user, because it runs as a non-interactive subprocess. So consent happens HERE, in chat: you ask, the user answers, and you gate each subprocess call on the answer. Do NOT just run `setup` and report the result - that is the silent-onboarding regression this section exists to prevent. **First-run detection (silent, no commands, no output to user):** - If `~/.config/last30days/.env` does NOT exist, this is a first run. - If the file exists and contains `SETUP_COMPLETE=true`, skip Step 0 entirely and go to Step 1 (CRITICAL: Parse User Intent below). Do NOT announce that setup is complete. The user does not need a status message on every run. -**If this IS a first run:** -- Run `python3 skills/last30days/scripts/last30days.py setup` (relative to the skill root) to launch the setup wizard. -- Follow the wizard's prompts end-to-end. The wizard handles platform detection (OpenClaw vs Claude Code), auto vs manual setup, browser cookie extraction, ScrapeCreators opt-in, a best-effort auto-install of the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only` — Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if the CLI is installed off-PATH; recommend-only if `npx` is unavailable), and the initial topic picker. -- After the wizard writes `SETUP_COMPLETE=true` to `~/.config/last30days/.env`, proceed to research. +**Named onboarding contract (2026-06-22, silent-wizard regression - Fredy Montero run):** the prior version of this step said "Run `setup` ... follow the wizard's prompts end-to-end." But `run_auto_setup()` has NO prompts - it extracts cookies, installs yt-dlp + Digg, and writes `SETUP_COMPLETE` with zero interaction. So the model ran the silent path, never asked consent before reading browser cookies, never surfaced the macOS Full Disk Access fix, and never offered the ScrapeCreators GitHub signup that unlocks TikTok/Instagram/X/Threads. The fix is the ordered, consent-first sequence below. Do not "simplify" it back to a bare `setup` call - the consent prompts are the feature. -The setup wizard lives as a Python module so it works across all hosts (Claude Code, Codex, Cursor, etc.) and the common-case (already set up) path through this file stays short. +**If this IS a first run, run this onboarding sequence in order. Each numbered step is a turn: present it, then wait for the user where it says to wait.** + +**1. Welcome.** One short branded line, e.g.: `Welcome to /last30days - let me get you set up (about 30 seconds).` + +**2. Cookie consent (ask BEFORE reading anything).** Tell the user you'd like to read their browser cookies and what it unlocks, then ask. Example: `I can read your browser cookies (Firefox/Safari) to unlock X/Twitter and other logged-in sources. Want me to? (yes / no)` **Wait for the answer.** + - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup` (relative to the skill root). This extracts cookies (Firefox/Safari by default - never Chrome, to avoid a Keychain prompt) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if installed off-PATH; recommend-only if `npx` is unavailable). + - On **no** → run the same command with cookie reads disabled for that invocation: `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. This skips all cookie extraction but STILL installs yt-dlp and Digg, and still writes `SETUP_COMPLETE`. Do not attempt any cookie read after a no. + +**3. Full Disk Access remediation (macOS only).** After the `setup` run, inspect its stderr. If it contains `Permission denied reading Cookies.binarycookies` and the platform is macOS, the OS blocked the read - surface the fix instead of swallowing it: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry of step 2's `setup` command. If the user skips, continue. + +**4. ScrapeCreators signup offer (every first run, consent BEFORE launching the browser).** Always offer this. Explain it grants free credits that unlock TikTok, Instagram, Threads, Pinterest, X, and YouTube comments/transcripts, and that it opens a GitHub authorization page in the browser. Do NOT hard-code a specific credit count - say "free credits" (the exact grant is set server-side). Ask, e.g.: `Want to unlock TikTok, Instagram, X and more? I can sign you up for ScrapeCreators with GitHub (free credits) - it opens a browser to authorize. (yes / no)` **Wait for the answer.** + - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --github`. Tell the user a browser window will open and to authorize with the code shown. On success the engine persists the key automatically and returns JSON with `"persisted": true` and a MASKED `api_key` (the raw key never appears - do not ask for or echo it). Confirm the paid sources are now active. + - On **timeout / denied** → tell the user it didn't complete and offer to retry or skip. + - On **no** → note they can run it anytime later by asking to set up ScrapeCreators, then continue. + +**5. Complete.** Once `SETUP_COMPLETE=true` is written, briefly confirm which sources are now active (read the `setup --github` JSON `persisted` field, or re-run `--diagnose`) and proceed to research. + +The setup wizard lives as a Python module so its mechanical work runs across all hosts (Claude Code, Codex, Cursor, etc.) while you drive the consent conversation above. The common-case (already set up) path through this file stays short. --- diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 61aa3f0..20350c7 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -673,12 +673,20 @@ def main() -> int: results = setup_wizard.run_openclaw_setup(config) print(json.dumps(results)) return 0 - if "--github" in extra_argv: - results = setup_wizard.run_github_auth() - print(json.dumps(results)) - return 0 - if "--device-auth" in extra_argv: - results = setup_wizard.run_full_device_auth() + if "--github" in extra_argv or "--device-auth" in extra_argv: + if "--github" in extra_argv: + results = setup_wizard.run_github_auth() + else: + results = setup_wizard.run_full_device_auth() + # Persist the returned key so the paid sources activate on the next + # run, and mask it in stdout so the secret never lands in the host + # model's captured Bash output. + api_key = results.get("api_key") + if results.get("status") == "success" and api_key: + results["persisted"] = setup_wizard.write_api_key(env.CONFIG_FILE, api_key) + results["api_key"] = setup_wizard.mask_api_key(api_key) + else: + results["persisted"] = False print(json.dumps(results)) return 0 sys.stderr.write("Running auto-setup...\n") diff --git a/skills/last30days/scripts/lib/setup_wizard.py b/skills/last30days/scripts/lib/setup_wizard.py index d1d6900..c545c82 100644 --- a/skills/last30days/scripts/lib/setup_wizard.py +++ b/skills/last30days/scripts/lib/setup_wizard.py @@ -330,6 +330,63 @@ def write_setup_config(env_path: Path, from_browser: str | None = None) -> bool: return False +def write_api_key(env_path: Path, api_key: str, key_name: str = "SCRAPECREATORS_API_KEY") -> bool: + """Append an API key to the .env file as a 0o600 secret. + + Reuses the same secret-safe write path as ``write_setup_config`` so the + value lands with restrictive permissions and round-trips through + ``env.load_env_file``. Idempotent: if ``key_name`` is already present in + the file, nothing is written and the existing value is preserved (we never + clobber a key the user may have set by hand). + + Args: + env_path: Path to the .env file (e.g. ~/.config/last30days/.env). + api_key: The raw key value to persist. + key_name: The env var name to write (default SCRAPECREATORS_API_KEY). + + Returns: + True if the key was written or already present, False on error or when + ``api_key`` is empty. + """ + if not api_key: + return False + try: + env_path = Path(env_path) + env_path.parent.mkdir(parents=True, exist_ok=True) + + existing_content = "" + if env_path.exists(): + existing_content = env_path.read_text(encoding="utf-8") + for line in existing_content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#") and "=" in stripped: + if stripped.split("=", 1)[0].strip() == key_name: + return True # Already configured; do not duplicate + + line = f"{key_name}={_format_env_value(api_key)}\n" + with _open_secret_append(env_path) as f: + if existing_content and not existing_content.endswith("\n"): + f.write("\n") + f.write(line) + + return True + + except OSError as exc: + logger.error("Failed to write API key to %s: %s", env_path, exc) + return False + + +def mask_api_key(api_key: str) -> str: + """Return a non-secret display form of an API key (prefix + last 4). + + Used so the key never appears verbatim in stdout the host model captures. + Short or empty keys collapse to a fixed placeholder. + """ + if not api_key or len(api_key) <= 8: + return "sc_…" + return f"{api_key[:3]}…{api_key[-4:]}" + + def get_setup_status_text(results: Dict[str, Any]) -> str: """Return a human-readable summary of auto-setup results. diff --git a/tests/test_onboarding_contract.py b/tests/test_onboarding_contract.py new file mode 100644 index 0000000..0dd302c --- /dev/null +++ b/tests/test_onboarding_contract.py @@ -0,0 +1,73 @@ +"""Contract tests for the consent-driven first-run onboarding in SKILL.md. + +These assert the structural guarantees of Step 0: consent is requested before +any cookie read, the decline and Full Disk Access branches are documented, the +ScrapeCreators signup is gated on a consent question, and the old silent-wizard +instruction is gone. They read SKILL.md as text (the model's runtime contract), +matching tests/test_runtime_preflight_contract.py. +""" + +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" + + +class TestOnboardingContract(unittest.TestCase): + def setUp(self): + self.text = SKILL_MD.read_text(encoding="utf-8") + # Scope assertions to the Step 0 section so generic substrings (e.g. + # "setup") elsewhere in the file do not satisfy ordering checks. + start = self.text.index("## Step 0: First-Run Setup Wizard") + end = self.text.index("## CRITICAL: Parse User Intent", start) + self.step0 = self.text[start:end] + + def test_cookie_consent_requested_before_setup_invocation(self): + """The cookie-consent question must appear before the first `setup` run.""" + consent_idx = self.step0.find("Cookie consent") + setup_idx = self.step0.find("last30days.py setup") + self.assertGreater(consent_idx, -1, "no Cookie consent step found") + self.assertGreater(setup_idx, -1, "no setup invocation found") + self.assertLess( + consent_idx, setup_idx, + "cookie consent must be requested before the setup command", + ) + + def test_decline_branch_uses_from_browser_off(self): + """Declining cookies must route to FROM_BROWSER=off (skip reads, keep installs).""" + self.assertIn("FROM_BROWSER=off", self.step0) + + def test_full_disk_access_remediation_present(self): + """The macOS permission-denied remediation must be documented.""" + self.assertIn("Permission denied reading Cookies.binarycookies", self.step0) + self.assertIn("Full Disk Access", self.step0) + + def test_scrapecreators_signup_gated_on_consent(self): + """The signup runs `setup --github` and is offered after a consent question.""" + self.assertIn("setup --github", self.step0) + offer_idx = self.step0.find("ScrapeCreators signup offer") + github_idx = self.step0.find("setup --github") + self.assertGreater(offer_idx, -1, "no ScrapeCreators signup offer step") + self.assertLess( + offer_idx, github_idx, + "the signup offer/consent must precede the --github invocation", + ) + + def test_signup_does_not_hardcode_credit_count(self): + """Onboarding copy must not assert an unverified credit number.""" + self.assertNotIn("1000 free credit", self.step0) + self.assertNotIn("1000 credits", self.step0) + + def test_old_silent_wizard_instruction_removed(self): + """The misleading 'follow the wizard's prompts' line must be gone.""" + self.assertNotIn("Follow the wizard's prompts end-to-end", self.text) + + def test_consent_is_conversational_contract_documented(self): + """The named onboarding contract explains why consent is in-chat.""" + self.assertIn("Named onboarding contract", self.step0) + self.assertIn("non-interactive subprocess", self.step0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_setup_openclaw.py b/tests/test_setup_openclaw.py index f062fdf..9a7625c 100644 --- a/tests/test_setup_openclaw.py +++ b/tests/test_setup_openclaw.py @@ -1,12 +1,16 @@ """Tests for OpenClaw setup and device auth functions.""" +import io import json +import sys import time +from contextlib import redirect_stdout from pathlib import Path from unittest.mock import patch, MagicMock, call import pytest +import last30days as cli from lib import setup_wizard @@ -504,3 +508,49 @@ class TestRunGithubAuth: result = setup_wizard.run_github_auth(timeout=1) assert result["status"] == "timeout" mock_subproc.assert_not_called() + + +class TestSetupGithubCliWiring: + """Tests for the `setup --github` CLI branch: persist + mask the key.""" + + def _run_setup_github(self, tmp_path, monkeypatch): + """Invoke `setup --github` in-process, return (parsed_json, env_path).""" + env_path = tmp_path / ".env" + monkeypatch.setattr(cli.env, "CONFIG_FILE", env_path) + monkeypatch.setattr(sys, "argv", ["last30days", "setup", "--github"]) + buf = io.StringIO() + with redirect_stdout(buf): + rc = cli.main() + assert rc == 0 + return json.loads(buf.getvalue()), env_path + + @patch("lib.setup_wizard.run_github_auth") + def test_success_persists_and_masks(self, mock_auth, tmp_path, monkeypatch): + """Success -> key written to .env, stdout JSON masked, persisted true.""" + mock_auth.return_value = { + "status": "success", "method": "device", + "api_key": "sc_live_supersecret9999", "user_code": "ABCD-1234", + } + + payload, env_path = self._run_setup_github(tmp_path, monkeypatch) + + # Key persisted to disk with the real value + assert "SCRAPECREATORS_API_KEY=sc_live_supersecret9999" in env_path.read_text() + # JSON reports persistence and the raw secret never appears in stdout + assert payload["persisted"] is True + assert payload["status"] == "success" + assert payload["api_key"] != "sc_live_supersecret9999" + assert "supersecret9999" not in json.dumps(payload) + # Useful non-secret fields survive + assert payload["user_code"] == "ABCD-1234" + + @patch("lib.setup_wizard.run_github_auth") + def test_timeout_persists_nothing(self, mock_auth, tmp_path, monkeypatch): + """Timeout -> no key on disk, persisted false.""" + mock_auth.return_value = {"status": "timeout", "user_code": "WXYZ-5678"} + + payload, env_path = self._run_setup_github(tmp_path, monkeypatch) + + assert payload["persisted"] is False + assert not env_path.exists() + assert payload["status"] == "timeout" diff --git a/tests/test_setup_wizard.py b/tests/test_setup_wizard.py index 576fbf2..7b8fa60 100644 --- a/tests/test_setup_wizard.py +++ b/tests/test_setup_wizard.py @@ -417,6 +417,108 @@ class TestWriteSetupConfig: assert "SETUP_COMPLETE=true" in lines[1] +class TestWriteApiKey: + """Tests for write_api_key() — persisting the ScrapeCreators signup key.""" + + def test_writes_key_with_secret_permissions(self): + """Key is written and the file is 0o600 (owner read/write only).""" + with tempfile.TemporaryDirectory() as tmpdir: + env_path = Path(tmpdir) / "subdir" / ".env" + + result = setup_wizard.write_api_key(env_path, "sc_live_abcdef123456") + + assert result is True + assert env_path.exists() + assert "SCRAPECREATORS_API_KEY=sc_live_abcdef123456" in env_path.read_text() + assert (env_path.stat().st_mode & 0o777) == 0o600 + + def test_value_round_trips_through_env_loader(self): + """Persisted key reloads to the exact original value.""" + from lib import env as env_mod + with tempfile.TemporaryDirectory() as tmpdir: + env_path = Path(tmpdir) / ".env" + + setup_wizard.write_api_key(env_path, "sc_live_abcdef123456") + + loaded = env_mod.load_env_file(env_path) + assert loaded["SCRAPECREATORS_API_KEY"] == "sc_live_abcdef123456" + + def test_idempotent_when_key_already_present(self): + """If the key already exists, do not duplicate or overwrite it.""" + with tempfile.TemporaryDirectory() as tmpdir: + env_path = Path(tmpdir) / ".env" + env_path.write_text("SCRAPECREATORS_API_KEY=existing_key\n") + + result = setup_wizard.write_api_key(env_path, "sc_new_value") + + assert result is True + content = env_path.read_text() + assert content.count("SCRAPECREATORS_API_KEY") == 1 + assert "existing_key" in content + assert "sc_new_value" not in content + + def test_appends_without_clobbering_other_keys(self): + """Existing unrelated keys are preserved.""" + with tempfile.TemporaryDirectory() as tmpdir: + env_path = Path(tmpdir) / ".env" + env_path.write_text("SETUP_COMPLETE=true\nFROM_BROWSER=firefox\n") + + setup_wizard.write_api_key(env_path, "sc_key_xyz") + + content = env_path.read_text() + assert "SETUP_COMPLETE=true" in content + assert "FROM_BROWSER=firefox" in content + assert "SCRAPECREATORS_API_KEY=sc_key_xyz" in content + + def test_value_with_whitespace_is_quoted(self): + """A pathological value with whitespace is quoted so it round-trips.""" + from lib import env as env_mod + with tempfile.TemporaryDirectory() as tmpdir: + env_path = Path(tmpdir) / ".env" + + setup_wizard.write_api_key(env_path, "key with space") + + content = env_path.read_text() + assert 'SCRAPECREATORS_API_KEY="key with space"' in content + assert env_mod.load_env_file(env_path)["SCRAPECREATORS_API_KEY"] == "key with space" + + def test_empty_key_returns_false_and_writes_nothing(self): + """An empty api_key persists nothing and reports failure.""" + with tempfile.TemporaryDirectory() as tmpdir: + env_path = Path(tmpdir) / ".env" + + assert setup_wizard.write_api_key(env_path, "") is False + assert not env_path.exists() + + def test_unwritable_target_returns_false(self): + """Unwritable target dir -> False, no exception escapes.""" + with tempfile.TemporaryDirectory() as tmpdir: + ro_dir = Path(tmpdir) / "ro" + ro_dir.mkdir() + ro_dir.chmod(0o500) # no write + try: + result = setup_wizard.write_api_key(ro_dir / "sub" / ".env", "sc_key") + assert result is False + finally: + ro_dir.chmod(0o700) # restore so tempdir cleanup succeeds + + +class TestMaskApiKey: + """Tests for mask_api_key() — non-secret display form.""" + + def test_masks_long_key(self): + masked = setup_wizard.mask_api_key("sc_live_abcdef123456") + assert "abcdef" not in masked + assert masked.endswith("3456") + assert masked.startswith("sc_") + + def test_short_key_collapses_to_placeholder(self): + assert setup_wizard.mask_api_key("short") == "sc_…" + + def test_empty_key_collapses_to_placeholder(self): + assert setup_wizard.mask_api_key("") == "sc_…" + + class TestCookieExtractionBrowsers: """Tests for env.cookie_extraction_browsers() — the shared browser policy.""" From 4d69ae6a1faf66ada8e2e8999d46c5c792b3a973 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Mon, 22 Jun 2026 18:18:01 -0700 Subject: [PATCH 32/50] fix(setup): resolve Greptile findings on first-run gate + onboarding (#659, #660) - First-run gate: replace `grep -c || echo` (emits `0\nFIRST_RUN_DETECTED` on a fresh install) with `grep -q ... && echo 1 || echo FIRST_RUN_DETECTED` so the gate emits exactly one token. (P1 on #659) - Python version gate: add the Linux `apt`/`pyenv` install line that the engine error message already prints. (P2 on #659) - ScrapeCreators signup: document the success-but-persisted:false branch so a failed key write is surfaced instead of silently claiming sources active. (P2 on #660) Follow-up to @henkyermontero's first-run setup fixes. --- skills/last30days/SKILL.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index 687b5b6..858281b 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -235,11 +235,13 @@ Load WebSearch first. No exceptions. Then run the first-run gate below before an **FIRST-RUN GATE — run this Bash command immediately after loading WebSearch, before reading the topic or doing any research:** ```bash -cat ~/.config/last30days/.env 2>/dev/null | grep -c "SETUP_COMPLETE=true" || echo "FIRST_RUN_DETECTED" +grep -q "SETUP_COMPLETE=true" ~/.config/last30days/.env 2>/dev/null && echo "1" || echo "FIRST_RUN_DETECTED" ``` +This emits exactly one token: `1` or `FIRST_RUN_DETECTED`, never both. + - Output is `1` → setup is complete. Continue to the branching rule below. -- Output is `FIRST_RUN_DETECTED` (or `0`) → this is a first run. Jump immediately to `## Step 0: First-Run Setup Wizard` and complete it **before doing any topic research**. Do NOT proceed to Step 0.5, do NOT load WebSearch supplements, do NOT synthesize anything. The wizard installs yt-dlp (YouTube), the Digg CLI (via `npx`), and extracts browser cookies for X/Twitter and other sources. Skipping it produces a degraded WebSearch-only result that misrepresents the skill's capability to the user. +- Output is `FIRST_RUN_DETECTED` → this is a first run. Jump immediately to `## Step 0: First-Run Setup Wizard` and complete it **before doing any topic research**. Do NOT proceed to Step 0.5, do NOT load WebSearch supplements, do NOT synthesize anything. The wizard installs yt-dlp (YouTube), the Digg CLI (via `npx`), and extracts browser cookies for X/Twitter and other sources. Skipping it produces a degraded WebSearch-only result that misrepresents the skill's capability to the user. **Named failure mode (2026-06-22, first-run setup skip - Fredy Montero run):** Model read "proceed to Step 0.5" in the branching rule and jumped there directly, bypassing `## Step 0: First-Run Setup Wizard` at line ~339. Result: no browser cookie extraction, no yt-dlp, no Digg CLI install, WebSearch-only synthesis with no X/YouTube/TikTok data. Root cause: the branching rule named Step 0.5 as the next step without mentioning the wizard. Fix: this gate and the updated branching rule below. @@ -341,6 +343,7 @@ If the preflight script emits `ERROR: last30days v3 requires Python 3.12+` (or ` > "The last30days engine needs Python 3.12+. Your system has an older version. Install it with one command: > - **Mac:** `brew install python@3.12` > - **Windows:** `winget install Python.Python.3.12` + > - **Linux:** `sudo apt install python3.12` (or `pyenv install 3.12`) > > Then re-run `/last30days ` and the setup wizard will configure everything automatically." 2. **Stop.** Do not attempt research. Do not fall back to WebSearch-only synthesis. @@ -383,6 +386,7 @@ Before proceeding to Step 1, handle first-run setup. **You are the conversationa **4. ScrapeCreators signup offer (every first run, consent BEFORE launching the browser).** Always offer this. Explain it grants free credits that unlock TikTok, Instagram, Threads, Pinterest, X, and YouTube comments/transcripts, and that it opens a GitHub authorization page in the browser. Do NOT hard-code a specific credit count - say "free credits" (the exact grant is set server-side). Ask, e.g.: `Want to unlock TikTok, Instagram, X and more? I can sign you up for ScrapeCreators with GitHub (free credits) - it opens a browser to authorize. (yes / no)` **Wait for the answer.** - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --github`. Tell the user a browser window will open and to authorize with the code shown. On success the engine persists the key automatically and returns JSON with `"persisted": true` and a MASKED `api_key` (the raw key never appears - do not ask for or echo it). Confirm the paid sources are now active. + - On **success but `"persisted": false`** (auth completed yet the key write failed - e.g. a permissions error on `~/.config/last30days/.env`) → do NOT claim the paid sources are active. Tell the user the signup worked but saving the key failed, and have them add `SCRAPECREATORS_API_KEY=` to `~/.config/last30days/.env` manually (the raw key is masked in output, so re-run `setup --github` or retrieve it from scrapecreators.com). - On **timeout / denied** → tell the user it didn't complete and offer to retry or skip. - On **no** → note they can run it anytime later by asking to set up ScrapeCreators, then continue. From aecab67c81c38895bdf6a6e97a32dac6da1ea916 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Mon, 22 Jun 2026 19:16:14 -0700 Subject: [PATCH 33/50] feat(setup): restore the v3.0.0 first-run NUX wizard (#661) * feat(setup): restore the v3.0.0 first-run NUX wizard on the consent-driven foundation Step 0 now has two branches. Claude Code (and any host with AskUserQuestion) gets the restored original guided NUX: welcome message, Auto/Manual/Skip setup modal, cookie-consent modal, ScrapeCreators signup offer, TikTok/Instagram INCLUDE_SOURCES opt-in, and a first-topic picker. Hosts without modals (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent Non-Modal Prose Flow. Builds on #659 (first-run gate) and #660 (consent-driven prose, key-persist, Full Disk Access remediation) - all of that is preserved, not reverted. Additive: the source inventory is current, not the v3.0.0 set. - Digg threaded into the install messaging everywhere yt-dlp/YouTube appears (welcome list, Auto-setup option, manual guide). Install already existed; this is the copy. - ScrapeCreators credit count restored to "10,000 free calls". - Hard "ALWAYS execute Step 0 BEFORE Step 1" gate restored to resist re-erosion. - Manual-setup guide refreshed to the current source matrix (Digg, youtube comments, SC Reddit/YouTube backups, Perplexity, Bluesky) with append-only .env safety rules. - Threads/Pinterest intentionally not surfaced in onboarding (power-user INCLUDE_SOURCES only). - tests/test_onboarding_contract.py rewritten for the two-branch structure and to lock the flow against silent re-erosion. - Docs synced: CONFIGURATION.md, AGENTS.md, CHANGELOG.md; original wizard captured at docs/reference/old-nux-wizard-v3.0.0.md. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Q37ombFQdv9uLbKm2y8vBD * fix(setup): resolve Greptile findings on the restored NUX wizard - Skip+Skip path now writes SETUP_COMPLETE: picking "Skip for now" at the setup choice wrote no .env flag, so the first-run gate re-fired on every invocation. The skip branch now persists SETUP_COMPLETE=true and goes to the topic picker. - Modal step labels (Step 1/2/3) now match the sequence descriptor; the body jumped from Step 1 to Step 4. - Non-Modal Prose Flow now honors an existing BROWSER_CONSENT=true (skip re-asking when consent was granted in a prior session). - Contract test: symmetric Full Disk Access assertion on the prose branch + a guard that the Skip path writes SETUP_COMPLETE. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Q37ombFQdv9uLbKm2y8vBD * fix(setup): restore masked-key retrieval hint on persisted:false path When setup --github succeeds but the key write fails, both flows told the user to add SCRAPECREATORS_API_KEY= manually but dropped the hint on how to obtain the value (the raw key is masked in output). Restored the parenthetical: re-run setup --github or retrieve it from scrapecreators.com. (Greptile) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Q37ombFQdv9uLbKm2y8vBD --------- Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- AGENTS.md | 2 +- CHANGELOG.md | 1 + CONFIGURATION.md | 11 +- docs/reference/old-nux-wizard-v3.0.0.md | 205 ++++++++++++++++++++++++ skills/last30days/SKILL.md | 166 +++++++++++++++++-- tests/test_onboarding_contract.py | 164 ++++++++++++++----- 6 files changed, 495 insertions(+), 54 deletions(-) create mode 100644 docs/reference/old-nux-wizard-v3.0.0.md diff --git a/AGENTS.md b/AGENTS.md index 229f97c..af27dbb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,7 +42,7 @@ Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`. - Git remote: origin = public (`mvanhorn/last30days-skill`) - Every `lib/*.py` call to `log.source_log(...)` must pass `tty_only=False`. The default is `True`, which silently drops every line when stderr isn't a TTY (Claude Code, Codex, CI, captured output) — turning source observability into invisible failure. Enforced by `tests/test_source_log_visibility.py`. - **CLI-gated optional sources** (Digg via `digg-pp-cli`, YouTube via `yt-dlp`) activate only when `shutil.which` resolves the binary on the **agent subprocess PATH** — not merely when the file exists on disk. First-run setup installs Digg through `@mvanhorn/printing-press-library` (default `$HOME/.local/bin`); Hermes/OpenClaw gateways often need that directory on PATH. Setup must distinguish PATH-visible installs from off-PATH binaries and must not claim "now active" unless the engine gate would pass. See `docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md`. -- **First-run onboarding is consent-driven and model-led.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth) — it cannot prompt, so consent lives in `SKILL.md` Step 0: the model asks before reading cookies, surfaces the macOS Full Disk Access fix on permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do not collapse Step 0 back into a bare silent `setup` call — the consent prompts are the feature. +- **First-run onboarding is consent-driven, model-led, and host-split.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth) — it cannot prompt, so consent lives in `SKILL.md` Step 0. Step 0 has TWO branches: a **Claude Code Modal Flow** (the restored v3.0.0 `AskUserQuestion`-driven NUX — welcome, Auto/Manual/Skip, cookie consent, ScrapeCreators offer, `INCLUDE_SOURCES` opt-in, first-topic picker) for hosts with modals, and a **Non-Modal Prose Flow** for hosts without (OpenClaw, Codex, Cursor, Gemini CLI). Both ask before reading cookies, surface the macOS Full Disk Access fix on permission-denied, and offer the ScrapeCreators GitHub signup (10,000 free calls) on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do NOT collapse the modal flow back into a bare silent `setup` call or flatten it to prose-only — the guided modals are the feature (they eroded once and were restored). The onboarding contract is locked by `tests/test_onboarding_contract.py`. Threads/Pinterest are intentionally not surfaced in onboarding (power-user `INCLUDE_SOURCES` only). ## Security hygiene - Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb4f75..6b6f28c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Restored the v3.0.0 first-run NUX wizard (Claude Code Modal Flow).** Step 0 now restores the original guided, `AskUserQuestion`-driven onboarding that eroded over time: a welcome message, an Auto/Manual/Skip setup modal, a cookie-consent modal, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. It is gated to hosts with modals; hosts without (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent **Non-Modal Prose Flow**. Digg is threaded into the install messaging alongside yt-dlp everywhere it appears, the ScrapeCreators credit count is `10,000 free calls`, and the flow is locked against re-erosion by `tests/test_onboarding_contract.py`. Builds on the consent-driven foundation from #659/#660. Original wizard captured at `docs/reference/old-nux-wizard-v3.0.0.md`. - **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659). ### Fixed diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 562b1e7..bb5887b 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -51,13 +51,18 @@ The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Docume ## First-run onboarding -On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It has three consent points: +On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It takes one of two forms depending on the host: + +- **Claude Code Modal Flow** - the restored v3.0.0 guided NUX, used on hosts with `AskUserQuestion` (Claude Code). A welcome message, then modals for Auto/Manual/Skip setup, cookie consent, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. +- **Non-Modal Prose Flow** - the same work done conversationally on hosts without modals (OpenClaw, Codex, Cursor, Gemini CLI, raw CLI). + +Both share the same consent points: 1. **Browser cookies** - the model asks before reading anything. On yes it extracts Firefox/Safari cookies (never Chrome, to avoid a macOS Keychain prompt) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup with `FROM_BROWSER=off` (skips all cookie reads, still installs the tools). 2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry. -3. **ScrapeCreators GitHub signup** - offered on every first run. On consent it runs `setup --github`, which opens a browser for GitHub device-auth and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, Threads, Pinterest, X, and YouTube comments/transcripts activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. +3. **ScrapeCreators GitHub signup** - offered on every first run (10,000 free calls). On consent it runs `setup --github`, which opens a browser for GitHub device-auth (or registers instantly via the `gh` CLI when installed) and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, X, YouTube comments, and the SC Reddit/YouTube backups activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. (Threads and Pinterest are not surfaced in onboarding but remain available via `INCLUDE_SOURCES`.) -Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation is specified in `skills/last30days/SKILL.md` Step 0. +Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation and both host flows are specified in `skills/last30days/SKILL.md` Step 0. The original v3.0.0 wizard is captured at `docs/reference/old-nux-wizard-v3.0.0.md`. --- diff --git a/docs/reference/old-nux-wizard-v3.0.0.md b/docs/reference/old-nux-wizard-v3.0.0.md new file mode 100644 index 0000000..c33ff8c --- /dev/null +++ b/docs/reference/old-nux-wizard-v3.0.0.md @@ -0,0 +1,205 @@ +# Original v3.0.0 First-Run NUX Wizard (reference capture) + +Captured verbatim from `SKILL.md` at git commit `0a9ff16` (v3.0.0, 2026-04-08), +the first-run setup wizard Matt built. Preserved here for provenance and as the +source for the restored modal NUX (see docs/plans/2026-06-22-001-feat-restore-nux-wizard-plan.md). +This is a historical snapshot - the live wizard in SKILL.md Step 0 uses the CURRENT +source inventory (Digg, youtube_comments, SC backups) and omits Threads/Pinterest. + +```markdown +## Step 0: First-Run Setup Wizard + +**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever. + +To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section **silently** and proceed to Step 1. **Do NOT say "Setup is complete" or any other status message — just move on.** The user doesn't need to be told setup is done every time they run the skill. + +**When first run is detected, detect your platform first:** + +**If you do NOT have WebSearch capability (OpenClaw, Codex, raw CLI):** Run the OpenClaw setup flow below. +**If you DO have WebSearch (Claude Code):** Run the standard setup flow below. + +--- + +### OpenClaw / Non-WebSearch Setup Flow + +Run environment detection first: +```bash +python3 "${SKILL_ROOT}/scripts/last30days.py" setup --openclaw +``` + +Read the JSON output. It tells you what's already configured. Display a status summary: + +``` +👋 Welcome to /last30days! + +Detected: +{✅ or ❌} yt-dlp (YouTube search) +{✅ or ❌} X/Twitter ({method} configured) +{✅ or ❌} ScrapeCreators (TikTok, Instagram, Reddit backup) +{✅ or ❌} Web search ({backend} configured) +``` + +Then for each missing item, offer setup in priority order: + +1. **ScrapeCreators** (if not configured): "ScrapeCreators adds TikTok and Instagram search (plus a Reddit backup if public Reddit gets rate-limited). 10,000 free calls, no credit card. (No referrals, no kickbacks - we don't get a cut.)" + - Option A: "ScrapeCreators via GitHub (recommended)" -- Check if `gh` CLI was detected in the environment detection output above. If gh IS detected: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". Before running the command, display: "Registering via GitHub CLI..." If gh is NOT detected: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". Before running the command, display: "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 "${SKILL_ROOT}/scripts/last30days.py" setup --github`, parse JSON output. Tries PAT first (if `gh` is installed), falls back to device flow which copies a one-time code to your clipboard and opens your browser. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to .env. + - Option B: "I have a key" -- accept paste, write to .env + - Option C: "Skip for now" + +2. **X/Twitter** (if not configured): "X search finds tweets and conversations. To unlock X: add FROM_BROWSER=auto (reads browser cookies, free), XAI_API_KEY (no browser access, api.x.ai), or AUTH_TOKEN+CT0 (manual cookies)." + - Option A: "I have an xAI API key" (recommended for servers -- persistent, no expiry). Write XAI_API_KEY to .env. + - Option B: "I have AUTH_TOKEN + CT0 from my browser" -- accept both, write to .env + - Option C: "Skip for now" + +3. **YouTube** (if yt-dlp not found): "YouTube search needs yt-dlp. Run: `pip install yt-dlp`" + +4. **Web search** (if no Brave/Exa/Serper key): "A web search key enables smarter results. Brave Search is free for 2,000 queries/month at brave.com/search/api" + +After setup, write `SETUP_COMPLETE=true` to .env and proceed to research. + +**Skip to "END OF FIRST-RUN WIZARD" below after completing the OpenClaw flow.** + +--- + +### Claude Code Setup Flow (Standard) + +**You MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text -> (2) setup modal -> (3) run setup if chosen -> (4) optional ScrapeCreators modal -> (5) topic picker. You MUST start at step 1.** + +**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion - do NOT repeat any of the welcome text inside the AskUserQuestion call.** + +Welcome to /last30days! + +I research any topic across Reddit, X, YouTube, and other sources - synthesizing what people are actually saying right now. + +Auto setup gives you 5 core sources for free in 30 seconds: +- X/Twitter - reads your x.com browser cookies to authenticate (not saved to disk). Chrome on macOS will prompt for Keychain access. +- Reddit with comments - public JSON, no API key needed +- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars) +- Hacker News + Polymarket + GitHub (if `gh` CLI installed) - always on, zero config + +Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation. + +**Then call AskUserQuestion with ONLY this question and these options - no additional text:** + +Question: "How would you like to set up?" +Options: +- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp for YouTube" +- "Manual setup - show me what to configure" +- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if gh installed), Web" + +**If the user picks 1 (Auto setup):** + +**Before running the setup command, get cookie consent:** + +Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`. If it does, skip the consent prompt and run setup directly. + +If `BROWSER_CONSENT=true` is NOT present, **call AskUserQuestion:** +Question: "Auto setup will scan your browser for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. Chrome on macOS will prompt for Keychain access. OK to proceed?" +Options: +- "Yes, scan my cookies for X" - Run setup as normal. Append `BROWSER_CONSENT=true` to .env after setup completes. +- "Skip X, just set up YouTube" - Run setup with YouTube only (install yt-dlp). Do not scan cookies. +- "I have an xAI API key instead" - Ask them to paste it, write XAI_API_KEY to .env. Then install yt-dlp. + +Run the setup subcommand: +```bash +cd {SKILL_DIR} && python3 scripts/last30days.py setup +``` +Show the user the results (what cookies were found, whether yt-dlp was installed). + +**Then show the optional ScrapeCreators offer (plain text, then modal):** + +Want TikTok and Instagram too? ScrapeCreators adds those platforms - 10,000 free calls, no credit card. It also serves as a Reddit backup if public Reddit ever gets rate-limited. + +**Before showing the ScrapeCreators modal, check for `gh` CLI:** Run `which gh` via Bash silently. Store the result as gh_available (true if found, false if not). + +**Call AskUserQuestion:** +Question: "Want to add TikTok, Instagram, and Reddit backup via ScrapeCreators? (We don't get a cut.)" +Options: +- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". If NOT gh_available: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". After the user selects this option: If gh_available, display "Registering via GitHub CLI..." before running the command. If NOT gh_available, display "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V on Mac, Ctrl+V on Windows/Linux)." Then run `cd {SKILL_DIR} && python3 scripts/last30days.py setup --github` via Bash with a 5-minute timeout. This tries PAT auth first (if `gh` CLI is installed, zero browser needed), then falls back to GitHub device flow which copies a one-time code to your clipboard and opens GitHub in your browser. Parse the JSON stdout. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to `~/.config/last30days/.env`. If `method` is `pat`, show: "You're in! Registered via GitHub CLI - zero browser needed. 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is true, show: "You're in! (The authorization code was copied to your clipboard automatically.) 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is false, show: "You're in! 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `status` is `timeout` or `error`, show: "GitHub auth didn't complete. No worries - you can sign up at scrapecreators.com instead or try again later." Then offer the web signup option. +- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env +- "I have a key" - accept the key, write to .env +- "Skip for now" - proceed without ScrapeCreators + +**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:** + +Your ScrapeCreators key powers TikTok, Instagram, Threads, Pinterest, and YouTube comments. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.) + +**Call AskUserQuestion:** +Question: "Which ScrapeCreators sources do you want on?" +Options: +- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "TikTok and Instagram are on, plus Reddit backup if public Reddit has issues. You can add threads, pinterest, youtube_comments to INCLUDE_SOURCES anytime." +- "Everything - TikTok, Instagram, Threads, Pinterest, YouTube comments" - append `INCLUDE_SOURCES=tiktok,instagram,threads,pinterest,youtube_comments` to ~/.config/last30days/.env. Confirm: "All ScrapeCreators sources are on." +- "Just the basics - let's run our first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will serve as Reddit backup. You can add sources to INCLUDE_SOURCES in your .env anytime." + +**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:** + +**Call AskUserQuestion:** +Question: "What do you want to research first?" +Options: +- "Claude Code vs Codex" - tech comparison +- "Sam Altman" - person in the news +- "Warriors Basketball" - sports +- "AI Legal Prompting Techniques" - niche/professional +- "Type my own topic" + +If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly. + +**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.** + +**If the user picks 2 (Manual setup):** +Show them this guide (present as plain text, not blockquoted): + +The magic of /last30days is Reddit comments + X posts together - and both are free. Here's how to unlock each source. + +Add these to `~/.config/last30days/.env`: + +X/Twitter (pick one - this is the most important): +- `FROM_BROWSER=auto` - free. Reads your x.com login cookies at search time to authenticate. Cookies are read live each run, not saved to disk. Chrome on macOS will prompt for Keychain access the first time. Firefox and Safari don't. +- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers or if you don't want cookie scanning. +- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com -> F12 -> Application -> Cookies) + +Reddit (free, works out of the box): +- Public JSON gives you threads + top comments with upvote counts. No setup required. +- `SCRAPECREATORS_API_KEY=xxx` - optional backup source if public Reddit gets rate-limited. +- `OPENAI_API_KEY=xxx` - optional fallback if public Reddit search has trouble finding threads. + +YouTube (free, open source): +- Run `brew install yt-dlp` - free, open source, 190K+ GitHub stars. Enables YouTube search and transcripts. + +Bonus: TikTok, Instagram, Threads, Pinterest, YouTube comments (ScrapeCreators): +- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com. +- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the most popular ones. Add threads, pinterest, youtube_comments for more. + +GitHub Issues/PRs (free, no key needed): +- If you have the `gh` CLI installed (`brew install gh`), GitHub search is automatic. No API key required. + +Perplexity Sonar Pro (AI-synthesized research via OpenRouter): +- `OPENROUTER_API_KEY=xxx` - adds AI-synthesized research with citations as an additive source alongside Reddit/X/YouTube. Returns structured narratives with specific dates, names, and numbers that social sources miss. ~$0.02/run. +- After adding your key, set `INCLUDE_SOURCES=perplexity` (or append to existing, e.g. `INCLUDE_SOURCES=tiktok,instagram,perplexity`). +- Use `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query) on topics that need serious investigation. +- Bonus: also powers the planning and reranking engine if you don't have a Gemini/OpenAI/xAI key. + +Other bonus sources (add anytime): +- `EXA_API_KEY=xxx` - semantic web search, 1K free/month (exa.ai) +- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password) +- `BRAVE_API_KEY=xxx` - Brave web search + +Always add this last line: `SETUP_COMPLETE=true` + +**CRITICAL: NEVER overwrite an existing .env file.** Before writing ANY key to `~/.config/last30days/.env`: +1. Check if the file exists: `test -f ~/.config/last30days/.env` +2. If it exists, READ it first, then APPEND only missing keys using `>>` (double redirect) +3. NEVER use `>` (single redirect) which destroys existing content +4. If it doesn't exist, create it: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env` + +**Then call AskUserQuestion:** +Question: "How do you want to add your keys?" +Options: +- "Open .env in my editor" - Creates the file with a commented template and opens it. You edit, save, and come back. +- "Paste keys here" - Paste your API keys and I'll write the file for you. +- "I'll do it myself" - I'll tell you the file path and you handle it. + +**If the user picks "Open .env in editor":** +Create `~/.config/last30days/.env` if it doesn't exist (check first!), pre-populated with this template: +``` +``` diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index 858281b..a5b7c56 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -366,33 +366,175 @@ The engine reads `LAST30DAYS_MEMORY_DIR` from either the process env or `~/.conf ## Step 0: First-Run Setup Wizard -Before proceeding to Step 1, handle first-run setup. **You are the conversational driver.** The Python setup script does only mechanical work (cookie reads, tool installs, the GitHub device-auth flow) - it CANNOT prompt the user, because it runs as a non-interactive subprocess. So consent happens HERE, in chat: you ask, the user answers, and you gate each subprocess call on the answer. Do NOT just run `setup` and report the result - that is the silent-onboarding regression this section exists to prevent. +**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even when the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST run the wizard BEFORE any research. The topic is preserved - research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. It takes about 30 seconds and only runs once, ever. + +**You are the conversational driver.** The Python setup script does only mechanical work (cookie reads, tool installs, the GitHub device-auth flow) - it CANNOT prompt the user, because it runs as a non-interactive subprocess. So consent happens HERE, in chat: you ask, the user answers, and you gate each subprocess call on the answer. Do NOT just run `setup` and report the result - that is the silent-onboarding regression this section exists to prevent. **First-run detection (silent, no commands, no output to user):** - If `~/.config/last30days/.env` does NOT exist, this is a first run. - If the file exists and contains `SETUP_COMPLETE=true`, skip Step 0 entirely and go to Step 1 (CRITICAL: Parse User Intent below). Do NOT announce that setup is complete. The user does not need a status message on every run. -**Named onboarding contract (2026-06-22, silent-wizard regression - Fredy Montero run):** the prior version of this step said "Run `setup` ... follow the wizard's prompts end-to-end." But `run_auto_setup()` has NO prompts - it extracts cookies, installs yt-dlp + Digg, and writes `SETUP_COMPLETE` with zero interaction. So the model ran the silent path, never asked consent before reading browser cookies, never surfaced the macOS Full Disk Access fix, and never offered the ScrapeCreators GitHub signup that unlocks TikTok/Instagram/X/Threads. The fix is the ordered, consent-first sequence below. Do not "simplify" it back to a bare `setup` call - the consent prompts are the feature. +**Named onboarding contracts:** +- *(2026-06-22, silent-wizard regression - Fredy Montero run):* a prior version said "Run `setup` ... follow the wizard's prompts end-to-end." But `run_auto_setup()` has NO prompts - it extracts cookies, installs yt-dlp + Digg, and writes `SETUP_COMPLETE` with zero interaction. The model ran the silent path, never asked cookie consent, never surfaced the macOS Full Disk Access fix, and never offered the ScrapeCreators signup. Consent must be conversational. +- *(2026-06-22, NUX restoration):* the original v3.0.0 Claude Code wizard was a guided, modal-driven flow (welcome → Auto/Manual/Skip → cookie consent → ScrapeCreators offer → source opt-in → first-topic picker) that eroded over time. It is restored below as the **Claude Code Modal Flow**. Do NOT collapse it back into a bare prose call - the guided modals are the feature. Reference capture: `docs/reference/old-nux-wizard-v3.0.0.md`. -**If this IS a first run, run this onboarding sequence in order. Each numbered step is a turn: present it, then wait for the user where it says to wait.** +**Platform split - run exactly ONE branch:** +- **If you HAVE WebSearch and AskUserQuestion (Claude Code):** run the **Claude Code Modal Flow** immediately below. +- **If you do NOT (OpenClaw, Codex, Cursor, Gemini CLI, raw CLI):** run the **Non-Modal Prose Flow** further down. It does the same work conversationally, without modals. + +--- + +### Claude Code Modal Flow + +**Follow these steps IN ORDER. Do NOT skip ahead to research. The sequence is: (1) welcome text → (2) setup modal → (3) run setup if chosen → (4) ScrapeCreators offer modal → (5) source opt-in modal → (6) first-topic picker. Start at step 1.** + +**Step 1 - Welcome.** Display this welcome text ONCE as a normal message (not blockquoted). + +Welcome to /last30days! + +I research any topic across Reddit, X, YouTube, and more - synthesizing what people are actually saying right now. + +Auto setup gives you the core sources free in about 30 seconds: +- X/Twitter - reads your browser cookies to authenticate (read live each run, never saved to disk). +- Reddit with comments - public JSON, no API key needed. +- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars). +- Digg - trending news, GitHub stars, and pipeline feeds - installs the free, keyless Digg CLI. +- Hacker News + Polymarket + GitHub (auto-on if the `gh` CLI is installed) - always on, zero config. + +Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation. + +**Step 2 - Setup choice.** Then IMMEDIATELY call AskUserQuestion with ONLY this question and these options (do not repeat the welcome text inside the modal): + +Question: "How would you like to set up?" +Options: +- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp (YouTube) and the Digg CLI" +- "Manual setup - show me what to configure" +- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if `gh` installed), Web" + +**Step 3 - Run setup based on the choice.** + +**If the user picks Skip for now:** write `SETUP_COMPLETE=true` to `~/.config/last30days/.env` (append-only; run `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env` first if the file does not exist) so the wizard does NOT re-fire on every subsequent run, then skip straight to Step 6 (the topic picker). Do not run any `setup` command - the always-on sources (Reddit, HN, Polymarket, GitHub, Web) need no setup. + +**If the user picks Auto setup:** + +Get cookie consent first. Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`; if so, skip the consent prompt and run setup directly. Otherwise **call AskUserQuestion:** +Question: "Auto setup will scan your browser (Firefox/Safari) for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. OK to proceed?" +Options: +- "Yes, scan my cookies for X" - run `python3 skills/last30days/scripts/last30days.py setup` (relative to the skill root). Append `BROWSER_CONSENT=true` to `.env` after setup completes. +- "Skip X, just set up YouTube + Digg" - run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. Skips all cookie reads; still installs yt-dlp and Digg. +- "I have an xAI API key instead" - ask them to paste it, write `XAI_API_KEY` to `.env`, then run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup` (installs yt-dlp + Digg, no cookie read). + +The `setup` run extracts cookies (Firefox/Safari by default - never Chrome, to avoid a macOS Keychain prompt) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if installed off-PATH; recommend-only if `npx` is unavailable). Show the user what was found and installed - including whether Digg landed on PATH (active) or off-PATH (installed but not yet active). + +**macOS Full Disk Access remediation.** After the `setup` run, inspect its stderr. If it contains `Permission denied reading Cookies.binarycookies` and the platform is macOS, the OS blocked the read - surface the fix instead of swallowing it: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry of the `setup` command. If the user skips, continue. + +**Step 4: ScrapeCreators offer (every first run).** Show this as plain text, then a modal: + +ScrapeCreators adds TikTok and Instagram - 10,000 free calls, no credit card. Your key also powers YouTube comments, a YouTube transcript fallback (used only when yt-dlp gets rate-limited), and a Reddit backup (if public Reddit gets rate-limited). (We don't get a cut.) + +Before the modal, run `which gh` via Bash silently; store as gh_available. + +**Call AskUserQuestion:** +Question: "Want to add TikTok, Instagram, and the ScrapeCreators backups? (We don't get a cut.)" +Options: +- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available, description: "Registers via GitHub CLI in ~2 seconds - no browser." If NOT gh_available, description: "Copies a one-time code to your clipboard and opens GitHub to authorize." After selection: if gh_available, display "Registering via GitHub CLI..."; if not, display "I'll copy a one-time code to your clipboard and open GitHub. When prompted for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 skills/last30days/scripts/last30days.py setup --github` with a 5-minute timeout. Parse the JSON. On `status == "success"` the engine persists the key automatically and returns `"persisted": true` with a MASKED `api_key` (the raw key never appears - do not ask for or echo it); confirm "You're in! 10,000 free calls. TikTok, Instagram, YouTube comments, and the Reddit/YouTube backups are now active." On `status == "success"` but `"persisted": false` (key write failed, e.g. a permissions error), do NOT claim sources are active - tell the user the signup worked but saving the key failed, and have them add `SCRAPECREATORS_API_KEY=` to `~/.config/last30days/.env` manually (the raw key is masked in output, so re-run `setup --github` or retrieve it from scrapecreators.com to get the value). On `status` `timeout`/`error`, show "GitHub auth didn't complete - no worries, sign up at scrapecreators.com or try again later," then offer the web option. +- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash, then ask them to paste the API key. Write `SCRAPECREATORS_API_KEY={key}` to `~/.config/last30days/.env`. +- "I have a key" - accept the key, write to `.env`. +- "Skip for now" - proceed without ScrapeCreators. + +**Step 5: Source opt-in (only if a ScrapeCreators key was saved, not if skipped).** Plain text then modal: + +Your ScrapeCreators key powers TikTok, Instagram, and YouTube comments. Want TikTok and Instagram on for every run? (Each adds one ScrapeCreators call per search.) + +**Call AskUserQuestion:** +Question: "Which ScrapeCreators sources do you want on?" +Options: +- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to `~/.config/last30days/.env`. Confirm: "TikTok and Instagram are on, plus the Reddit/YouTube backups if the free sources get rate-limited." +- "Just the basics - let's run my first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will still serve as the Reddit and YouTube backups. You can add sources to `INCLUDE_SOURCES` in your `.env` anytime." + +**Step 6: First-topic picker.** Once `SETUP_COMPLETE=true` is written, **call AskUserQuestion:** +Question: "What do you want to research first?" +Options: +- "Claude Code vs Codex" - tech comparison +- "Sam Altman" - person in the news +- "Warriors Basketball" - sports +- "AI Legal Prompting Techniques" - niche/professional +- "Type my own topic" + +If the user picks an example, run research with it. If "Type my own", ask what they want. **If the user already supplied a topic with the command (e.g. `/last30days Mercer Island`), SKIP this picker and use their topic directly.** + +**END OF FIRST-RUN WIZARD.** Everything in the Modal Flow ONLY runs on first run. If `SETUP_COMPLETE=true` exists, skip ALL of it - no welcome, no modals, no topic picker - and go straight to research (Parse User Intent). + +**If the user picked Manual setup** at Step 2, follow the **Manual Setup Guide** below instead of the Auto branch (the guide writes `SETUP_COMPLETE=true` itself), then continue to Step 6. + +--- + +### Non-Modal Prose Flow + +For hosts without interactive modal prompts (OpenClaw, Codex, Cursor, Gemini CLI, raw CLI). Same work, done conversationally. Run in order; wait where it says to wait. **1. Welcome.** One short branded line, e.g.: `Welcome to /last30days - let me get you set up (about 30 seconds).` -**2. Cookie consent (ask BEFORE reading anything).** Tell the user you'd like to read their browser cookies and what it unlocks, then ask. Example: `I can read your browser cookies (Firefox/Safari) to unlock X/Twitter and other logged-in sources. Want me to? (yes / no)` **Wait for the answer.** - - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup` (relative to the skill root). This extracts cookies (Firefox/Safari by default - never Chrome, to avoid a Keychain prompt) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if installed off-PATH; recommend-only if `npx` is unavailable). - - On **no** → run the same command with cookie reads disabled for that invocation: `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. This skips all cookie extraction but STILL installs yt-dlp and Digg, and still writes `SETUP_COMPLETE`. Do not attempt any cookie read after a no. +**2. Cookie consent (ask BEFORE reading anything).** First check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env` (e.g. granted in a prior Claude Code session); if so, skip this prompt and run `setup` directly. Otherwise ask. Example: `I can read your browser cookies (Firefox/Safari) to unlock X/Twitter and other logged-in sources. Want me to? (yes / no)` **Wait for the answer.** + - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup` (and append `BROWSER_CONSENT=true` to `.env` after it completes). Extracts cookies (Firefox/Safari, never Chrome) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; activates only when on the agent subprocess PATH, typically `$HOME/.local/bin`; reports honestly if off-PATH; recommend-only if `npx` is unavailable). + - On **no** → run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. Skips all cookie reads; still installs yt-dlp and Digg, still writes `SETUP_COMPLETE`. -**3. Full Disk Access remediation (macOS only).** After the `setup` run, inspect its stderr. If it contains `Permission denied reading Cookies.binarycookies` and the platform is macOS, the OS blocked the read - surface the fix instead of swallowing it: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry of step 2's `setup` command. If the user skips, continue. +**3. Full Disk Access remediation (macOS only).** After `setup`, inspect stderr. If it contains `Permission denied reading Cookies.binarycookies` on macOS, surface: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry. If skipped, continue. -**4. ScrapeCreators signup offer (every first run, consent BEFORE launching the browser).** Always offer this. Explain it grants free credits that unlock TikTok, Instagram, Threads, Pinterest, X, and YouTube comments/transcripts, and that it opens a GitHub authorization page in the browser. Do NOT hard-code a specific credit count - say "free credits" (the exact grant is set server-side). Ask, e.g.: `Want to unlock TikTok, Instagram, X and more? I can sign you up for ScrapeCreators with GitHub (free credits) - it opens a browser to authorize. (yes / no)` **Wait for the answer.** - - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --github`. Tell the user a browser window will open and to authorize with the code shown. On success the engine persists the key automatically and returns JSON with `"persisted": true` and a MASKED `api_key` (the raw key never appears - do not ask for or echo it). Confirm the paid sources are now active. - - On **success but `"persisted": false`** (auth completed yet the key write failed - e.g. a permissions error on `~/.config/last30days/.env`) → do NOT claim the paid sources are active. Tell the user the signup worked but saving the key failed, and have them add `SCRAPECREATORS_API_KEY=` to `~/.config/last30days/.env` manually (the raw key is masked in output, so re-run `setup --github` or retrieve it from scrapecreators.com). +**4. ScrapeCreators signup offer (every first run, consent BEFORE launching the browser).** Explain it grants 10,000 free calls that unlock TikTok, Instagram, YouTube comments, plus a Reddit backup and a YouTube transcript fallback, and that it opens a GitHub authorization page. Ask, e.g.: `Want to unlock TikTok, Instagram, and more? I can sign you up for ScrapeCreators with GitHub (10,000 free calls) - it opens a browser to authorize. (yes / no)` **Wait for the answer.** + - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --github`. A browser window opens; the user authorizes with the code shown. On success the engine persists the key automatically and returns `"persisted": true` with a MASKED `api_key` (never ask for or echo the raw key). Confirm the paid sources are active. + - On **success but `"persisted": false`** (auth completed yet the key write failed) → do NOT claim sources are active. Tell the user signup worked but saving failed, and have them add `SCRAPECREATORS_API_KEY=` to `~/.config/last30days/.env` manually (the raw key is masked in output, so re-run `setup --github` or retrieve it from scrapecreators.com to get the value). - On **timeout / denied** → tell the user it didn't complete and offer to retry or skip. - - On **no** → note they can run it anytime later by asking to set up ScrapeCreators, then continue. + - On **no** → note they can run it later by asking to set up ScrapeCreators, then continue. **5. Complete.** Once `SETUP_COMPLETE=true` is written, briefly confirm which sources are now active (read the `setup --github` JSON `persisted` field, or re-run `--diagnose`) and proceed to research. -The setup wizard lives as a Python module so its mechanical work runs across all hosts (Claude Code, Codex, Cursor, etc.) while you drive the consent conversation above. The common-case (already set up) path through this file stays short. +--- + +### Manual Setup Guide + +Shown when a Claude Code user picks "Manual setup", or for anyone who wants to configure by hand. Present as plain text (not blockquoted). + +The magic of /last30days is Reddit comments + X posts together - and both are free. Add these to `~/.config/last30days/.env`: + +**X/Twitter (pick one - the most important source):** +- `FROM_BROWSER=auto` - free. Reads your x.com login cookies live at search time (Firefox/Safari, never saved to disk). +- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers. +- `XQUIK_API_KEY=xxx` - keyless-style X via Xquik. +- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com → F12 → Application → Cookies). + +**Reddit (free, works out of the box):** +- Public JSON gives threads + top comments with upvote counts. No setup required. +- `SCRAPECREATORS_API_KEY=xxx` - optional backup if public Reddit gets rate-limited. + +**YouTube (free, open source):** +- Run `brew install yt-dlp` (or `pip install yt-dlp`) - enables YouTube search + transcripts. +- `SCRAPECREATORS_API_KEY=xxx` - optional server-side transcript fallback, used only when yt-dlp is rate-limited/bot-gated. + +**Digg (free, keyless):** +- Run `npx @mvanhorn/printing-press-library install digg --cli-only` - installs the Digg CLI for trending news, GitHub stars, and pipeline feeds. Activates when `digg-pp-cli` is on your PATH (typically `$HOME/.local/bin`). + +**GitHub Issues/PRs (free, no key needed):** +- If the `gh` CLI is installed and authed (`brew install gh && gh auth login`), GitHub search is automatic. No API key required. + +**Bonus: TikTok, Instagram, YouTube comments (ScrapeCreators):** +- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com. +- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the popular ones. (Threads and Pinterest are also available via `INCLUDE_SOURCES=threads,pinterest` for power users.) + +**Other optional sources (add anytime):** +- `PERPLEXITY_API_KEY=xxx` (or `OPENROUTER_API_KEY=xxx`) - AI-synthesized research with citations; set `INCLUDE_SOURCES=perplexity`. +- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password). +- `BRAVE_API_KEY=xxx` or `EXA_API_KEY=xxx` - web search backends. + +**CRITICAL: NEVER overwrite an existing `.env`.** Before writing ANY key: +1. Check if the file exists: `test -f ~/.config/last30days/.env` +2. If it exists, READ it, then APPEND only missing keys with `>>` (double redirect). +3. NEVER use `>` (single redirect) - it destroys existing content. +4. If it doesn't exist: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env` + +Always add this last line: `SETUP_COMPLETE=true`. Then proceed to research. + +The setup wizard's mechanical work lives in a Python module so it runs across all hosts (Claude Code, Codex, Cursor, etc.) while you drive the consent conversation above. The common-case (already set up) path through this file stays short. --- diff --git a/tests/test_onboarding_contract.py b/tests/test_onboarding_contract.py index 0dd302c..81f042b 100644 --- a/tests/test_onboarding_contract.py +++ b/tests/test_onboarding_contract.py @@ -1,10 +1,16 @@ -"""Contract tests for the consent-driven first-run onboarding in SKILL.md. +"""Contract tests for the restored first-run NUX wizard in SKILL.md. -These assert the structural guarantees of Step 0: consent is requested before -any cookie read, the decline and Full Disk Access branches are documented, the -ScrapeCreators signup is gated on a consent question, and the old silent-wizard -instruction is gone. They read SKILL.md as text (the model's runtime contract), -matching tests/test_runtime_preflight_contract.py. +Step 0 has two branches: a **Claude Code Modal Flow** (AskUserQuestion-driven, +the restored v3.0.0 NUX) and a **Non-Modal Prose Flow** for hosts without modals +(OpenClaw, Codex, Cursor, Gemini CLI). These tests assert the structural +guarantees of both branches, plus the cross-cutting copy rules: the hard +"Step 0 before Step 1" gate, Digg threaded alongside yt-dlp, the 10,000-free-calls +credit count, and Threads/Pinterest kept out of the onboarding offers. They read +SKILL.md as text - the model's runtime contract - matching +tests/test_runtime_preflight_contract.py. + +These lock the flow against silent re-erosion (the failure mode that orphaned the +wizard in PR #659 and flattened it before this restoration). """ import unittest @@ -17,54 +23,136 @@ SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" class TestOnboardingContract(unittest.TestCase): def setUp(self): self.text = SKILL_MD.read_text(encoding="utf-8") - # Scope assertions to the Step 0 section so generic substrings (e.g. - # "setup") elsewhere in the file do not satisfy ordering checks. + # Scope assertions to Step 0 so generic substrings elsewhere in the file + # do not satisfy ordering/presence checks. start = self.text.index("## Step 0: First-Run Setup Wizard") end = self.text.index("## CRITICAL: Parse User Intent", start) self.step0 = self.text[start:end] + # Branch slices. + modal_start = self.step0.index("### Claude Code Modal Flow") + prose_start = self.step0.index("### Non-Modal Prose Flow") + manual_start = self.step0.index("### Manual Setup Guide") + self.modal = self.step0[modal_start:prose_start] + self.prose = self.step0[prose_start:manual_start] + self.manual = self.step0[manual_start:] - def test_cookie_consent_requested_before_setup_invocation(self): - """The cookie-consent question must appear before the first `setup` run.""" - consent_idx = self.step0.find("Cookie consent") - setup_idx = self.step0.find("last30days.py setup") - self.assertGreater(consent_idx, -1, "no Cookie consent step found") - self.assertGreater(setup_idx, -1, "no setup invocation found") - self.assertLess( - consent_idx, setup_idx, - "cookie consent must be requested before the setup command", - ) + # --- Platform split + hard gate --- - def test_decline_branch_uses_from_browser_off(self): - """Declining cookies must route to FROM_BROWSER=off (skip reads, keep installs).""" - self.assertIn("FROM_BROWSER=off", self.step0) + def test_platform_split_present(self): + """Step 0 routes modal-capable hosts and prose hosts to distinct flows.""" + self.assertIn("Platform split", self.step0) + self.assertIn("### Claude Code Modal Flow", self.step0) + self.assertIn("### Non-Modal Prose Flow", self.step0) + + def test_hard_gate_step0_before_step1(self): + """The erosion-resistant gate that orphaned the wizard in #659 is restored.""" + self.assertIn("ALWAYS execute Step 0 BEFORE Step 1", self.step0) + + # --- Modal flow: the restored NUX, stages in order --- + + def test_modal_flow_stage_order(self): + """Welcome -> setup modal -> cookie consent -> SC offer -> opt-in -> picker.""" + anchors = [ + "Welcome to /last30days!", + "How would you like to set up?", + "scan your browser", # cookie-consent modal + "Want to add TikTok, Instagram, and the ScrapeCreators backups?", # SC offer + "Which ScrapeCreators sources do you want on?", # source opt-in + "What do you want to research first?", # topic picker + ] + idxs = [self.modal.find(a) for a in anchors] + for a, i in zip(anchors, idxs): + self.assertGreater(i, -1, f"modal flow missing stage anchor: {a!r}") + self.assertEqual(idxs, sorted(idxs), "modal flow stages are out of order") + + def test_modal_uses_askuserquestion(self): + self.assertIn("AskUserQuestion", self.modal) + + def test_modal_cookie_consent_before_setup(self): + consent = self.modal.find("scan your browser") + setup = self.modal.find("last30days.py setup") + self.assertGreater(consent, -1, "no cookie-consent modal in modal flow") + self.assertGreater(setup, -1, "no setup invocation in modal flow") + self.assertLess(consent, setup, "cookie consent must precede setup in modal flow") + + def test_topic_picker_skips_when_topic_supplied(self): + """The picker documents skipping when the user already gave a topic.""" + self.assertIn("What do you want to research first?", self.modal) + self.assertIn("SKIP this picker", self.modal) + + # --- Prose flow: same work, modal-free --- + + def test_prose_flow_has_no_modals(self): + self.assertNotIn("AskUserQuestion", self.prose) + + def test_prose_cookie_consent_before_setup(self): + consent = self.prose.find("Cookie consent") + setup = self.prose.find("last30days.py setup") + self.assertGreater(consent, -1, "no cookie-consent step in prose flow") + self.assertGreater(setup, -1, "no setup invocation in prose flow") + self.assertLess(consent, setup, "cookie consent must precede setup in prose flow") + + def test_prose_decline_uses_from_browser_off(self): + self.assertIn("FROM_BROWSER=off", self.prose) + + # --- Full Disk Access remediation (both branches) --- def test_full_disk_access_remediation_present(self): - """The macOS permission-denied remediation must be documented.""" - self.assertIn("Permission denied reading Cookies.binarycookies", self.step0) - self.assertIn("Full Disk Access", self.step0) + self.assertIn("Permission denied reading Cookies.binarycookies", self.modal) + self.assertIn("Full Disk Access", self.modal) + self.assertIn("Permission denied reading Cookies.binarycookies", self.prose) + self.assertIn("Full Disk Access", self.prose) - def test_scrapecreators_signup_gated_on_consent(self): - """The signup runs `setup --github` and is offered after a consent question.""" - self.assertIn("setup --github", self.step0) - offer_idx = self.step0.find("ScrapeCreators signup offer") - github_idx = self.step0.find("setup --github") - self.assertGreater(offer_idx, -1, "no ScrapeCreators signup offer step") - self.assertLess( - offer_idx, github_idx, - "the signup offer/consent must precede the --github invocation", - ) + def test_skip_path_writes_setup_complete(self): + """The 'Skip for now' setup choice must write SETUP_COMPLETE or the wizard loops.""" + skip_idx = self.modal.find("If the user picks Skip for now") + self.assertGreater(skip_idx, -1, "no Skip-for-now handling in modal flow") + # The skip branch must persist the completion flag in its own paragraph. + skip_para = self.modal[skip_idx:skip_idx + 400] + self.assertIn("SETUP_COMPLETE=true", skip_para) - def test_signup_does_not_hardcode_credit_count(self): - """Onboarding copy must not assert an unverified credit number.""" + # --- ScrapeCreators signup + persisted edge case --- + + def test_scrapecreators_signup_present_both_branches(self): + self.assertIn("setup --github", self.modal) + self.assertIn("setup --github", self.prose) + + def test_persisted_false_edge_case_documented(self): + self.assertIn('"persisted": false', self.step0) + + # --- Digg threaded alongside yt-dlp everywhere it appears --- + + def test_digg_threaded_with_ytdlp(self): + self.assertIn("Digg", self.modal) + self.assertIn("Digg", self.prose) + self.assertIn("Digg", self.manual) + # The Auto-setup modal option names both tools together. + self.assertIn("yt-dlp (YouTube) and the Digg CLI", self.modal) + + # --- Credit count = 10,000, no conflicting numbers in onboarding --- + + def test_credit_count_is_10000(self): + self.assertIn("10,000 free calls", self.step0) + self.assertNotIn("1,000 free", self.step0) self.assertNotIn("1000 free credit", self.step0) self.assertNotIn("1000 credits", self.step0) + self.assertNotIn("100 free call", self.step0) + + # --- Threads/Pinterest kept out of the onboarding offers --- + + def test_threads_pinterest_absent_from_modal_and_prose(self): + """They stay a power-user INCLUDE_SOURCES note in the manual guide only.""" + self.assertNotIn("Threads", self.modal) + self.assertNotIn("Pinterest", self.modal) + self.assertNotIn("Threads", self.prose) + self.assertNotIn("Pinterest", self.prose) + + # --- Legacy guarantees retained --- def test_old_silent_wizard_instruction_removed(self): - """The misleading 'follow the wizard's prompts' line must be gone.""" self.assertNotIn("Follow the wizard's prompts end-to-end", self.text) def test_consent_is_conversational_contract_documented(self): - """The named onboarding contract explains why consent is in-chat.""" self.assertIn("Named onboarding contract", self.step0) self.assertIn("non-interactive subprocess", self.step0) From 977f0beed5ad4dbc604f6762a4b27ab8efbd3504 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:18:29 -0700 Subject: [PATCH 34/50] chore: bump version to 3.8.1 Restored the v3.0.0 first-run NUX wizard (#661) on the consent-driven onboarding foundation (#659/#660): the guided Claude Code Modal Flow (welcome, Auto/Manual/Skip, cookie consent, ScrapeCreators offer, source opt-in, topic picker) with a Non-Modal Prose Flow fallback. Digg threaded into the install copy; 10,000-free-calls restored; hard Step-0-before-Step-1 gate restored; flow locked by contract tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Q37ombFQdv9uLbKm2y8vBD --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 2 ++ gemini-extension.json | 2 +- pyproject.toml | 2 +- skills/last30days/SKILL.md | 12 ++++++------ uv.lock | 2 +- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c7befc8..27d6bc9 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ { "name": "last30days", "description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.", - "version": "3.8.0", + "version": "3.8.1", "author": { "name": "Matt Van Horn", "url": "https://github.com/mvanhorn" diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e1f4049..b73d037 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "last30days", - "version": "3.8.0", + "version": "3.8.1", "description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.", "author": { "name": "Matt Van Horn", diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b6f28c..9132a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.8.1] - 2026-06-22 + ### Added - **Restored the v3.0.0 first-run NUX wizard (Claude Code Modal Flow).** Step 0 now restores the original guided, `AskUserQuestion`-driven onboarding that eroded over time: a welcome message, an Auto/Manual/Skip setup modal, a cookie-consent modal, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. It is gated to hosts with modals; hosts without (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent **Non-Modal Prose Flow**. Digg is threaded into the install messaging alongside yt-dlp everywhere it appears, the ScrapeCreators credit count is `10,000 free calls`, and the flow is locked against re-erosion by `tests/test_onboarding_contract.py`. Builds on the consent-driven foundation from #659/#660. Original wizard captured at `docs/reference/old-nux-wizard-v3.0.0.md`. - **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659). diff --git a/gemini-extension.json b/gemini-extension.json index 5b065ef..b284038 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "last30days-skill", - "version": "3.8.0", + "version": "3.8.1", "description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.", "settings": [ { diff --git a/pyproject.toml b/pyproject.toml index 782fb30..853aed2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "last30days-skill" -version = "3.8.0" +version = "3.8.1" description = "Multi-source last-30-days research skill" readme = "README.md" requires-python = ">=3.12" diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index a5b7c56..0354865 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -1,6 +1,6 @@ --- name: last30days -version: "3.8.0" +version: "3.8.1" description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web." argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react' allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch @@ -264,7 +264,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl --- -# last30days v3.8.0: Research Any Topic from the Last 30 Days +# last30days v3.8.1: Research Any Topic from the Last 30 Days > **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section. @@ -856,8 +856,8 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe # the Read tool result. Examples: # Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days # Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days -# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days/SKILL.md -# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days +# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days/SKILL.md +# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days # scripts/last30days.py is always a direct child of SKILL_DIR (every install layout # packages SKILL.md and scripts/ as siblings). SKILL_DIR="" @@ -1196,8 +1196,8 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next # the Read tool result. Examples: # Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days # Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days -# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days/SKILL.md -# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.0/skills/last30days +# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days/SKILL.md +# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.8.1/skills/last30days # scripts/last30days.py is always a direct child of SKILL_DIR (every install layout # packages SKILL.md and scripts/ as siblings). SKILL_DIR="" diff --git a/uv.lock b/uv.lock index b72fd26..9f310f3 100644 --- a/uv.lock +++ b/uv.lock @@ -106,7 +106,7 @@ wheels = [ [[package]] name = "last30days-skill" -version = "3.8.0" +version = "3.8.1" source = { virtual = "." } [package.dev-dependencies] From dab238ea7fb5e5cc640a894c42dbd319fa262e88 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Tue, 23 Jun 2026 09:46:59 -0700 Subject: [PATCH 35/50] fix: avoid enterprise boilerplate in company size inference --- .gitignore | 1 + skills/last30days/scripts/lib/hiring_signals.py | 12 ++++++++++-- tests/test_hiring_signals.py | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7b7b27e..2a7b02e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ htmlcov/ # Internal planning docs (ce:plan output) — keep local, don't publish docs/plans/ +docs/brainstorms/ .context/ /work diff --git a/skills/last30days/scripts/lib/hiring_signals.py b/skills/last30days/scripts/lib/hiring_signals.py index 7a16977..e4e7de9 100644 --- a/skills/last30days/scripts/lib/hiring_signals.py +++ b/skills/last30days/scripts/lib/hiring_signals.py @@ -96,6 +96,14 @@ def analyze( def infer_company_size(items: list[schema.SourceItem], *, topic: str = "") -> str: """Infer a coarse company-size tier from jobs evidence.""" + topic_lower = topic.lower() + firmographic_text = " ".join( + " ".join([ + str(item.metadata.get("company_size") or ""), + topic, + ]) + for item in items + ).lower() text = " ".join( " ".join([ item.title, @@ -111,9 +119,9 @@ def infer_company_size(items: list[schema.SourceItem], *, topic: str = "") -> st # never the job-description body - JDs list enterprise customers (e.g. # "trusted by Microsoft, Google"), which would misclassify a startup as # mega-cap and suppress its real signals. - if re.search(r"\b(apple|uber|google|microsoft|amazon|meta|netflix)\b", topic.lower()): + if re.search(r"\b(apple|uber|google|microsoft|amazon|meta|netflix)\b", topic_lower): return "mega-cap" - if count >= 200 or re.search(r"\b(fortune 500|thousands of employees)\b", text): + if count >= 200 or re.search(r"\b(fortune 500|thousands of employees)\b", firmographic_text): return "large-enterprise" if count >= 35 or re.search(r"\b(series [cd]|public company)\b", text): return "growth" diff --git a/tests/test_hiring_signals.py b/tests/test_hiring_signals.py index b6199d6..ae7ef0d 100644 --- a/tests/test_hiring_signals.py +++ b/tests/test_hiring_signals.py @@ -39,6 +39,23 @@ class HiringSignalsTests(unittest.TestCase): self.assertEqual("mega-cap", summary["company_size_tier"]) self.assertIn("too diffuse", summary["omitted_reason"]) + def test_fortune_500_customer_boilerplate_does_not_make_startup_large_enterprise(self): + items = [ + job( + "Founding Enterprise Solutions Engineer", + "Help Fortune 500 customers adopt SSO, SOC 2, and procurement workflows.", + "Sales", + ), + job( + "Security Platform Engineer", + "Build enterprise security, audit, and admin workflows for Fortune 500 customers.", + "Engineering", + ), + ] + summary = hiring_signals.analyze(items, explicit=False, topic="Listen Labs") + self.assertEqual("startup", summary["company_size_tier"]) + self.assertTrue(summary["include"]) + def test_explicit_mode_keeps_low_confidence_signal(self): items = [job("Customer Success Manager", "support enterprise customers", "Success")] summary = hiring_signals.analyze(items, explicit=True, topic="Acme") From c8ab309f45326a95192ffecf9001edd22f003def Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 17:01:32 -0700 Subject: [PATCH 36/50] fix: gate browser-cookie reads by command policy (#670) * fix: gate browser-cookie reads by command policy * fix: honor browser cookie hard-disable in setup * fix: keep setup diagnose cookie-safe --- CONFIGURATION.md | 14 +- mcp/internal/tools/research.go | 13 +- mcp/internal/tools/research_test.go | 8 ++ skills/last30days/scripts/last30days.py | 47 +++++-- skills/last30days/scripts/lib/env.py | 57 +++++--- skills/last30days/scripts/lib/pipeline.py | 31 ++++- skills/last30days/scripts/lib/setup_wizard.py | 46 +++---- tests/test_chromium_browsers.py | 7 +- tests/test_env_cookies.py | 41 ++++-- tests/test_security_boundaries.py | 124 ++++++++++++++++++ tests/test_setup_wizard.py | 19 ++- 11 files changed, 318 insertions(+), 89 deletions(-) create mode 100644 tests/test_security_boundaries.py diff --git a/CONFIGURATION.md b/CONFIGURATION.md index bb5887b..b6ec8a4 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -44,6 +44,7 @@ The engine's `.env` reader doesn't expand `$HOME` — only the tilde, via `Path( - `--save-dir ` - one-off output location. **Flag wins over env var.** If neither flag nor env var is set, the engine does not write a file (DB persistence is independent — see `LAST30DAYS_STORE` below). - `--output ` - write the rendered output to an exact file path, using the format selected by `--emit`. - `--save-suffix ` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`). +- `--no-browser-cookies` - hard-disable browser-cookie extraction for this run, even when `FROM_BROWSER` is configured. MCP and folder-mode hosts use this for safe defaults. The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1. @@ -58,7 +59,7 @@ On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_C Both share the same consent points: -1. **Browser cookies** - the model asks before reading anything. On yes it extracts Firefox/Safari cookies (never Chrome, to avoid a macOS Keychain prompt) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup with `FROM_BROWSER=off` (skips all cookie reads, still installs the tools). +1. **Browser cookies** - the model asks before reading anything. On yes it runs `setup --allow-browser-cookies`, which extracts Firefox/Safari cookies (never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser is explicitly configured) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup without `--allow-browser-cookies` (or with `FROM_BROWSER=off`), which skips all cookie reads and still installs the tools. 2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry. 3. **ScrapeCreators GitHub signup** - offered on every first run (10,000 free calls). On consent it runs `setup --github`, which opens a browser for GitHub device-auth (or registers instantly via the `gh` CLI when installed) and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, X, YouTube comments, and the SC Reddit/YouTube backups activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. (Threads and Pinterest are not surfaced in onboarding but remain available via `INCLUDE_SOURCES`.) @@ -129,10 +130,11 @@ CT0= # OR Xquik key-based X search # XQUIK_API_KEY= # OR cookie-jar (free; logs in via your browser session). -# Unset = Firefox + Safari (silent). FROM_BROWSER=auto also tries the Chromium -# family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it only prompts -# for macOS Keychain access on the browser that actually holds your X cookies. -# Or name a single browser, e.g. brave/edge. On Windows only Firefox is supported. +# Unset = no browser-cookie reads. FROM_BROWSER=auto tries Firefox/Safari and +# the Chromium family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it +# only prompts for macOS Keychain access on the browser that actually holds your +# X cookies. Or name a single browser, e.g. brave/edge. On Windows only Firefox +# is supported. # FROM_BROWSER=firefox # Bluesky @@ -142,7 +144,7 @@ BSKY_APP_PASSWORD= After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant). -**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search. +**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a safe preflight report for source availability, config source, browser-cookie plan, external command availability, and write destinations without reading browser cookies or running live provider probes. ### Perplexity source modes diff --git a/mcp/internal/tools/research.go b/mcp/internal/tools/research.go index 150ca80..42ecdb5 100644 --- a/mcp/internal/tools/research.go +++ b/mcp/internal/tools/research.go @@ -74,10 +74,7 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc { )), nil } - runArgs := []string{topic, "--emit=" + emit} - if save { - runArgs = append(runArgs, "--save") - } + runArgs := researchRunArgs(topic, emit, save) res, runErr := engine.Run(ctx, engine.RunOptions{ CacheDir: cacheDir, @@ -90,6 +87,14 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc { } } +func researchRunArgs(topic, emit string, save bool) []string { + runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"} + if save { + runArgs = append(runArgs, "--save") + } + return runArgs +} + func requireString(args map[string]any, name string) (string, error) { raw, ok := args[name] if !ok { diff --git a/mcp/internal/tools/research_test.go b/mcp/internal/tools/research_test.go index 0edb684..ed06e49 100644 --- a/mcp/internal/tools/research_test.go +++ b/mcp/internal/tools/research_test.go @@ -97,6 +97,14 @@ func TestBoolArgument(t *testing.T) { } } +func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) { + args := researchRunArgs("OpenAI", "compact", false) + want := []string{"OpenAI", "--emit=compact", "--no-browser-cookies"} + if strings.Join(args, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("args = %#v, want %#v", args, want) + } +} + func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) { // Validation failures are returned as MCP tool errors (not Go errors) // so Claude sees a structured failure with a readable message rather diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 20350c7..bc14ce4 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -293,6 +293,8 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--debug", action="store_true", help="Enable HTTP debug logging") parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures") parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability") + parser.add_argument("--no-browser-cookies", action="store_true", + help="Disable browser-cookie extraction even when FROM_BROWSER is configured") parser.add_argument("--save-dir", help="Optional directory for saving the rendered output") parser.add_argument("--output", help="Optional exact file path for saving the rendered output") parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output") @@ -618,7 +620,7 @@ def _write_last_run(topic: str, report: "schema.Report") -> None: pass -def _propagate_config_to_environ() -> None: +def _propagate_config_to_environ(config: dict[str, object]) -> None: """Push relevant env keys to os.environ so provider modules can read them. The env.get_config() function reads from a .env file, but providers.py @@ -626,17 +628,30 @@ def _propagate_config_to_environ() -> None: XAI_BASE_URL overrides are silently ignored. This is a no-op for keys that are already set in process env. """ - try: - config = env.get_config() - except Exception: - return for key in ("OPENAI_BASE_URL", "XAI_BASE_URL"): val = config.get(key) if val and not os.environ.get(key): os.environ[key] = val -_propagate_config_to_environ() +def _setup_allows_browser_cookies(args: argparse.Namespace, extra_argv: list[str]) -> bool: + return ( + not args.no_browser_cookies + and not args.diagnose + and "--allow-browser-cookies" in extra_argv + ) + + +def _config_policy_for_args(args: argparse.Namespace, topic: str, extra_argv: list[str]) -> env.ConfigLoadPolicy: + if args.no_browser_cookies: + browser_mode = "off" + elif args.diagnose: + browser_mode = "plan_only" + elif topic.lower() == "setup": + browser_mode = "read" if _setup_allows_browser_cookies(args, extra_argv) else "off" + else: + browser_mode = "read" + return env.ConfigLoadPolicy(browser_cookies=browser_mode) def main() -> int: @@ -647,7 +662,9 @@ def main() -> int: if args.debug: os.environ["LAST30DAYS_DEBUG"] = "1" - config = env.get_config() + topic = " ".join(args.topic).strip() + config = env.get_config(policy=_config_policy_for_args(args, topic, extra_argv)) + _propagate_config_to_environ(config) # Env-var fallback for --save-dir, mirroring the LAST30DAYS_STORE pattern below. # Uses `is None` / `is not None` checks (not truthy `or`) at every layer so that @@ -666,7 +683,6 @@ def main() -> int: os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"] # Handle setup subcommand - topic = " ".join(args.topic).strip() if topic.lower() == "setup": from lib import setup_wizard if "--openclaw" in extra_argv: @@ -690,14 +706,17 @@ def main() -> int: print(json.dumps(results)) return 0 sys.stderr.write("Running auto-setup...\n") - results = setup_wizard.run_auto_setup(config) + results = setup_wizard.run_auto_setup( + config, + allow_browser_cookies=_setup_allows_browser_cookies(args, extra_argv), + ) # Persist FROM_BROWSER only when every service's cookies came from the # SAME single browser — then we can fast-path future runs to it. If # different services matched different browsers, or none matched, leave - # FROM_BROWSER unset: the safe default (Firefox/Safari) then covers all - # of them with no Keychain prompt. We deliberately do NOT pin "auto" - # here (it would re-probe Chrome and re-trigger the prompt) nor a single - # browser (it would silently skip the service that used the other one). + # FROM_BROWSER unset so the safe default remains no browser-cookie + # reads. We deliberately do NOT pin "auto" here (it would re-probe + # Chrome and re-trigger the prompt) nor a single browser (it would + # silently skip the service that used the other one). found_browsers = set(results.get("cookies_found", {}).values()) from_browser = found_browsers.pop() if len(found_browsers) == 1 else None setup_wizard.write_setup_config(env.CONFIG_FILE, from_browser=from_browser) @@ -706,7 +725,7 @@ def main() -> int: return 0 requested_sources = resolve_requested_sources(args.search, config) - diag = pipeline.diagnose(config, requested_sources) + diag = pipeline.diagnose(config, requested_sources, safe=args.diagnose) if args.diagnose: print(json.dumps(diag, indent=2, sort_keys=True)) diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index ab59f35..c052510 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -77,6 +77,20 @@ class OpenAIAuth: codex_auth_file: str +BrowserCookieMode = Literal["off", "read", "plan_only"] + + +@dataclass(frozen=True) +class ConfigLoadPolicy: + """Local-read gates for configuration loading. + + Bare library calls use the safe default: no browser-cookie extraction. CLI + entry points can opt into narrower behavior after parsing command intent. + """ + + browser_cookies: BrowserCookieMode = "off" + + def _check_file_permissions(path: Path) -> None: """Warn to stderr if a secrets file has overly permissive permissions.""" if os.name == "nt": @@ -328,7 +342,7 @@ def _find_project_env() -> Path | None: return None -def get_config() -> dict[str, Any]: +def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]: """Load configuration from multiple sources. Priority (highest wins): @@ -337,6 +351,7 @@ def get_config() -> dict[str, Any]: 3. ~/.config/last30days/.env (global config) 4. macOS Keychain items prefixed ``last30days-`` (Darwin only) """ + policy = policy or ConfigLoadPolicy() # Load from global config file file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {} @@ -477,12 +492,15 @@ def get_config() -> dict[str, Any]: else: config['_CONFIG_SOURCE'] = 'env_only' - # Extract browser credentials if configured - browser_creds = extract_browser_credentials(config) - for key, value in browser_creds.items(): - if not config.get(key): - config[key] = value - config[f"_{key}_SOURCE"] = "browser" + config['_BROWSER_COOKIE_MODE'] = policy.browser_cookies + config['_BROWSER_COOKIE_BROWSERS'] = cookie_extraction_browsers(config) + + if policy.browser_cookies == "read": + browser_creds = extract_browser_credentials(config) + for key, value in browser_creds.items(): + if not config.get(key): + config[key] = value + config[f"_{key}_SOURCE"] = "browser" return config @@ -508,16 +526,17 @@ COOKIE_DOMAINS: dict[str, dict[str, Any]] = { def cookie_extraction_browsers(config: dict[str, Any]) -> list[str]: """Browsers to try for cookie extraction, honoring FROM_BROWSER. - Default (FROM_BROWSER unset): Firefox and Safari only. These read local - files silently with no system dialogs. The Chromium family (Chrome, Brave, - Edge, Vivaldi, Opera, Arc, Chromium) is skipped because reading their - cookies on macOS requires the browser's Safe Storage Keychain key, which - triggers a system password prompt that cannot be reliably suppressed. On - Windows only Firefox cookie extraction is supported; Chrome and Edge use - DPAPI-encrypted cookie stores that are not yet supported. + Default (FROM_BROWSER unset): no browser-cookie reads. The Chromium family + (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium) is available only when + explicitly selected because reading their cookies on macOS requires the + browser's Safe Storage Keychain key, which triggers a system password prompt + that cannot be reliably suppressed. On Windows only Firefox cookie + extraction is supported; Chrome and Edge use DPAPI-encrypted cookie stores + that are not yet supported. - ``FROM_BROWSER=`` - a single browser (e.g. ``firefox``, ``brave``, ``edge``, ``arc``). + - ``FROM_BROWSER=firefox,safari`` - a comma-separated explicit browser list. - ``FROM_BROWSER=auto`` - also try every Chromium browser (user accepts the Keychain dialog when needed). - ``FROM_BROWSER=off`` - returns [] (extraction disabled). @@ -528,14 +547,20 @@ def cookie_extraction_browsers(config: dict[str, Any]) -> list[str]: """ silent_browsers = ["firefox", "safari"] chromium_browsers = ["chrome", "brave", "edge", "vivaldi", "opera", "arc", "chromium"] + known_browsers = silent_browsers + chromium_browsers from_browser = (config.get("FROM_BROWSER") or "").strip().lower() + if not from_browser: + return [] if from_browser == "off": return [] - if from_browser in silent_browsers or from_browser in chromium_browsers: + if "," in from_browser: + browsers = [b.strip() for b in from_browser.split(",") if b.strip()] + return [b for b in browsers if b in known_browsers] + if from_browser in known_browsers: return [from_browser] if from_browser == "auto": return silent_browsers + chromium_browsers - return list(silent_browsers) + return [] diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index 46a1851..7a9c810 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -162,10 +162,15 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non return available -def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None) -> dict[str, Any]: +def diagnose( + config: dict[str, Any], + requested_sources: list[str] | None = None, + *, + safe: bool = False, +) -> dict[str, Any]: requested_sources = normalize_requested_sources(requested_sources) google_key = _google_key(config) - x_status = env.get_x_source_status(config, probe=True) + x_status = env.get_x_source_status(config, probe=not safe) native_web_backend = None if config.get("BRAVE_API_KEY"): native_web_backend = "brave" @@ -185,6 +190,22 @@ def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None) reasoning_provider_available = any( providers_status[name] for name in ("google", "openai", "xai", "openrouter") ) + external_commands = { + "yt-dlp": bool(which("yt-dlp")), + "digg-pp-cli": bool(which("digg-pp-cli")), + "gh": bool(which("gh")), + } + credential_destinations = { + "global_env": str(env.CONFIG_FILE) if env.CONFIG_FILE else None, + } + browser_cookies = { + "mode": config.get("_BROWSER_COOKIE_MODE", "off"), + "browsers": list(config.get("_BROWSER_COOKIE_BROWSERS") or []), + "reads_values": False if safe else config.get("_BROWSER_COOKIE_MODE") == "read", + } + local_writes: list[dict[str, str]] = [] + if config.get("LAST30DAYS_MEMORY_DIR"): + local_writes.append({"kind": "report", "path": str(config.get("LAST30DAYS_MEMORY_DIR"))}) return { "providers": providers_status, "local_mode": not reasoning_provider_available, @@ -201,6 +222,12 @@ def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None) "has_scrapecreators": bool(config.get("SCRAPECREATORS_API_KEY")), "has_github": bool(config.get("GITHUB_TOKEN") or which("gh")), "available_sources": available_sources(config, requested_sources), + "safe": safe, + "config_source": config.get("_CONFIG_SOURCE"), + "browser_cookies": browser_cookies, + "external_commands": external_commands, + "credential_destinations": credential_destinations, + "local_writes": local_writes, } diff --git a/skills/last30days/scripts/lib/setup_wizard.py b/skills/last30days/scripts/lib/setup_wizard.py index c545c82..3fe19b9 100644 --- a/skills/last30days/scripts/lib/setup_wizard.py +++ b/skills/last30days/scripts/lib/setup_wizard.py @@ -28,12 +28,12 @@ def is_first_run(config: Dict[str, Any]) -> bool: return not config.get("SETUP_COMPLETE") -def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]: +def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = False) -> Dict[str, Any]: """Perform the auto-setup actions. - - Runs cookie extraction for all registered domains, trying the browsers - from ``env.cookie_extraction_browsers()`` (honors ``FROM_BROWSER``; - defaults to Firefox/Safari, so no Chrome Keychain prompt) + - Optionally runs cookie extraction for all registered domains, trying the + browsers from ``env.cookie_extraction_browsers()``. Browser reads are off + unless ``allow_browser_cookies`` is true. - Checks if yt-dlp is installed - Best-effort install of digg-pp-cli (Printing Press library) @@ -49,31 +49,31 @@ def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]: digg_stderr: present when digg_action is install_failed digg_path: present when digg_action is installed_off_path (binary on disk, not on PATH) """ - from . import cookie_extract from .env import COOKIE_DOMAINS, cookie_extraction_browsers cookies_found: Dict[str, str] = {} - # Honor FROM_BROWSER and default to the silent browsers (Firefox/Safari). - # Using "auto" here used to probe Chrome unconditionally, triggering a - # "Chrome Safe Storage" Keychain prompt on first run that the steady-state - # path deliberately avoids. Chrome is now opt-in via FROM_BROWSER=chrome|auto. - # An empty list (FROM_BROWSER=off) makes the inner loop a no-op. - browsers = cookie_extraction_browsers(config) + if allow_browser_cookies: + from . import cookie_extract - for source_name, spec in COOKIE_DOMAINS.items(): - domain = spec["domain"] - cookie_names = spec["cookies"] + cookie_config = dict(config) + if not (cookie_config.get("FROM_BROWSER") or "").strip(): + cookie_config["FROM_BROWSER"] = "firefox,safari" + browsers = cookie_extraction_browsers(cookie_config) - for browser in browsers: - try: - result = cookie_extract.extract_cookies_with_source(browser, domain, cookie_names) - except Exception as exc: - logger.debug("Cookie extraction failed for %s via %s: %s", source_name, browser, exc) - continue - if result is not None and result[0]: - cookies_found[source_name] = result[1] - break # Found cookies for this service, stop trying browsers + for source_name, spec in COOKIE_DOMAINS.items(): + domain = spec["domain"] + cookie_names = spec["cookies"] + + for browser in browsers: + try: + result = cookie_extract.extract_cookies_with_source(browser, domain, cookie_names) + except Exception as exc: + logger.debug("Cookie extraction failed for %s via %s: %s", source_name, browser, exc) + continue + if result is not None and result[0]: + cookies_found[source_name] = result[1] + break # Found cookies for this service, stop trying browsers # Check yt-dlp availability and install via Homebrew if missing ytdlp_action: str diff --git a/tests/test_chromium_browsers.py b/tests/test_chromium_browsers.py index 8ebf1cd..18d212a 100644 --- a/tests/test_chromium_browsers.py +++ b/tests/test_chromium_browsers.py @@ -117,15 +117,14 @@ class TestEnvBrowserSelection: assert browser in tried, f"auto should try {browser}" @patch("lib.cookie_extract.extract_cookies") - def test_default_still_silent_only(self, mock_extract): - """Default (no FROM_BROWSER) stays Firefox+Safari - no Keychain prompt.""" + def test_default_skips_browser_cookie_reads(self, mock_extract): + """Default (no FROM_BROWSER) reads no local browser cookies.""" mock_extract.return_value = None config = _base_config() extract_browser_credentials(config) - tried = {call[0][0] for call in mock_extract.call_args_list} - assert tried == {"firefox", "safari"} + mock_extract.assert_not_called() # --------------------------------------------------------------------------- diff --git a/tests/test_env_cookies.py b/tests/test_env_cookies.py index 72dc2df..d667a09 100644 --- a/tests/test_env_cookies.py +++ b/tests/test_env_cookies.py @@ -5,7 +5,7 @@ from unittest.mock import patch import pytest -from lib.env import extract_browser_credentials, COOKIE_DOMAINS +from lib.env import ConfigLoadPolicy, extract_browser_credentials, COOKIE_DOMAINS def _base_config(**overrides): @@ -55,17 +55,13 @@ class TestExtractBrowserCredentials: mock_extract.assert_not_called() @patch("lib.cookie_extract.extract_cookies") - def test_no_from_browser_defaults_to_silent(self, mock_extract): - """Default (no FROM_BROWSER): tries Firefox and Safari only, skips Chrome.""" + def test_no_from_browser_skips_all(self, mock_extract): + """Default (no FROM_BROWSER): reads no browser cookies.""" mock_extract.return_value = None config = _base_config() result = extract_browser_credentials(config) assert result == {} - # Should try firefox and safari but NOT chrome - browser_args = [call[0][0] for call in mock_extract.call_args_list] - assert "firefox" in browser_args - assert "safari" in browser_args - assert "chrome" not in browser_args + mock_extract.assert_not_called() @patch("lib.cookie_extract.extract_cookies") def test_from_browser_firefox_only(self, mock_extract): @@ -105,14 +101,14 @@ class TestExtractBrowserCredentials: class TestGetConfigCookieIntegration: - """Integration test: get_config() calls extract_browser_credentials.""" + """Integration tests for policy-gated cookie extraction in get_config().""" @patch("lib.cookie_extract.extract_cookies") @patch("lib.env._find_project_env", return_value=None) @patch("lib.env.load_env_file", return_value={}) @patch("lib.env._load_keychain", return_value={}) @patch("lib.env.get_openai_auth") - def test_get_config_injects_cookies( + def test_get_config_default_does_not_extract_cookies( self, mock_openai, mock_keychain, mock_load, mock_proj, mock_extract ): from lib.env import get_config, OpenAIAuth @@ -128,5 +124,30 @@ class TestGetConfigCookieIntegration: } with patch.dict(os.environ, env_patch, clear=False): config = get_config() + assert config["AUTH_TOKEN"] is None + assert config["CT0"] is None + mock_extract.assert_not_called() + + @patch("lib.cookie_extract.extract_cookies") + @patch("lib.env._find_project_env", return_value=None) + @patch("lib.env.load_env_file", return_value={}) + @patch("lib.env._load_keychain", return_value={}) + @patch("lib.env.get_openai_auth") + def test_get_config_with_cookie_policy_injects_cookies( + self, mock_openai, mock_keychain, mock_load, mock_proj, mock_extract + ): + from lib.env import get_config, OpenAIAuth + mock_openai.return_value = OpenAIAuth( + token=None, source="none", status="missing", + account_id=None, codex_auth_file="/fake", + ) + mock_extract.return_value = {"auth_token": "browser_tok", "ct0": "browser_ct0"} + env_patch = { + "SETUP_COMPLETE": "true", + "FROM_BROWSER": "auto", + "LAST30DAYS_CONFIG_DIR": "", + } + with patch.dict(os.environ, env_patch, clear=False): + config = get_config(policy=ConfigLoadPolicy(browser_cookies="read")) assert config["AUTH_TOKEN"] == "browser_tok" assert config["CT0"] == "browser_ct0" diff --git a/tests/test_security_boundaries.py b/tests/test_security_boundaries.py new file mode 100644 index 0000000..d8c58fc --- /dev/null +++ b/tests/test_security_boundaries.py @@ -0,0 +1,124 @@ +"""Regression tests for agent-host local-read boundaries.""" + +from __future__ import annotations + +import importlib +import io +import json +import os +import sys +from contextlib import redirect_stderr, redirect_stdout +from unittest import mock + +import last30days as cli + + +def test_importing_cli_does_not_load_config_or_propagate_endpoints(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("XAI_BASE_URL", raising=False) + with mock.patch("lib.env.get_config", side_effect=AssertionError("import loaded config")): + importlib.reload(cli) + assert os.environ.get("OPENAI_BASE_URL") is None + assert os.environ.get("XAI_BASE_URL") is None + + +def test_diagnose_uses_plan_only_cookie_policy_and_safe_pipeline(monkeypatch): + seen: dict[str, object] = {} + + def fake_get_config(*, policy): + seen["policy"] = policy + return {"_BROWSER_COOKIE_MODE": policy.browser_cookies, "_BROWSER_COOKIE_BROWSERS": ["firefox"]} + + with mock.patch.object(cli.env, "get_config", side_effect=fake_get_config), \ + mock.patch.object(cli.pipeline, "diagnose", return_value={"ok": True}) as diagnose, \ + mock.patch.object(sys, "argv", ["last30days.py", "--diagnose"]): + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + assert cli.main() == 0 + + assert seen["policy"].browser_cookies == "plan_only" + diagnose.assert_called_once_with( + {"_BROWSER_COOKIE_MODE": "plan_only", "_BROWSER_COOKIE_BROWSERS": ["firefox"]}, + None, + safe=True, + ) + assert json.loads(stdout.getvalue()) == {"ok": True} + + +def test_setup_without_cookie_flag_disables_browser_cookie_setup(monkeypatch): + with mock.patch.object(cli.env, "get_config", return_value={}), \ + mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \ + mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \ + mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \ + mock.patch.object(sys, "argv", ["last30days.py", "setup"]): + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + assert cli.main() == 0 + + assert setup.call_args.kwargs["allow_browser_cookies"] is False + + +def test_setup_cookie_flag_allows_browser_cookie_setup(monkeypatch): + with mock.patch.object(cli.env, "get_config", return_value={}), \ + mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \ + mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \ + mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \ + mock.patch.object(sys, "argv", ["last30days.py", "setup", "--allow-browser-cookies"]): + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + assert cli.main() == 0 + + assert setup.call_args.kwargs["allow_browser_cookies"] is True + + +def test_no_browser_cookies_overrides_setup_cookie_flag(monkeypatch): + seen: dict[str, object] = {} + + def fake_get_config(*, policy): + seen["policy"] = policy + return {} + + with mock.patch.object(cli.env, "get_config", side_effect=fake_get_config), \ + mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \ + mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \ + mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \ + mock.patch.object( + sys, + "argv", + ["last30days.py", "--no-browser-cookies", "setup", "--allow-browser-cookies"], + ): + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + assert cli.main() == 0 + + assert seen["policy"].browser_cookies == "off" + assert setup.call_args.kwargs["allow_browser_cookies"] is False + + +def test_diagnose_overrides_setup_cookie_flag(monkeypatch): + seen: dict[str, object] = {} + + def fake_get_config(*, policy): + seen["policy"] = policy + return {} + + with mock.patch.object(cli.env, "get_config", side_effect=fake_get_config), \ + mock.patch("lib.setup_wizard.run_auto_setup", return_value={"cookies_found": {}}) as setup, \ + mock.patch("lib.setup_wizard.write_setup_config", return_value=True), \ + mock.patch("lib.setup_wizard.get_setup_status_text", return_value="ok"), \ + mock.patch.object( + sys, + "argv", + ["last30days.py", "--diagnose", "setup", "--allow-browser-cookies"], + ): + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + assert cli.main() == 0 + + assert seen["policy"].browser_cookies == "plan_only" + assert setup.call_args.kwargs["allow_browser_cookies"] is False diff --git a/tests/test_setup_wizard.py b/tests/test_setup_wizard.py index 7b8fa60..fc3f962 100644 --- a/tests/test_setup_wizard.py +++ b/tests/test_setup_wizard.py @@ -50,7 +50,7 @@ class TestRunAutoSetup: mock_which.return_value = "/usr/local/bin/yt-dlp" config = {} - results = setup_wizard.run_auto_setup(config) + results = setup_wizard.run_auto_setup(config, allow_browser_cookies=True) assert "x" in results["cookies_found"] assert results["cookies_found"]["x"] == "chrome" @@ -69,6 +69,7 @@ class TestRunAutoSetup: results = setup_wizard.run_auto_setup(config) assert results["cookies_found"] == {} + mock_extract.assert_not_called() assert results["ytdlp_installed"] is False assert results["ytdlp_action"] == "no_homebrew" @@ -80,7 +81,7 @@ class TestRunAutoSetup: mock_which.return_value = None config = {} - results = setup_wizard.run_auto_setup(config) + results = setup_wizard.run_auto_setup(config, allow_browser_cookies=True) assert results["cookies_found"] == {} @@ -99,7 +100,7 @@ class TestRunAutoSetup: mock_which.return_value = None config = {} - results = setup_wizard.run_auto_setup(config) + results = setup_wizard.run_auto_setup(config, allow_browser_cookies=True) assert results["cookies_found"]["x"] == "firefox" assert results["cookies_found"]["truthsocial"] == "firefox" @@ -522,12 +523,11 @@ class TestMaskApiKey: class TestCookieExtractionBrowsers: """Tests for env.cookie_extraction_browsers() — the shared browser policy.""" - def test_default_excludes_chrome(self): - """FROM_BROWSER unset -> Firefox/Safari only, never Chrome (no prompt).""" + def test_default_disables_extraction(self): + """FROM_BROWSER unset -> no browser-cookie reads.""" from lib import env browsers = env.cookie_extraction_browsers({}) - assert "chrome" not in browsers - assert browsers == ["firefox", "safari"] + assert browsers == [] def test_off_disables_extraction(self): from lib import env @@ -550,13 +550,12 @@ class TestWizardDoesNotProbeChromeByDefault: def test_default_run_never_requests_chrome(self, _mock_which, mock_extract): setup_wizard.run_auto_setup({}) requested_browsers = {call.args[0] for call in mock_extract.call_args_list} - assert "chrome" not in requested_browsers - assert requested_browsers <= {"firefox", "safari"} + assert requested_browsers == set() @patch("lib.cookie_extract.extract_cookies_with_source", return_value=None) @patch("shutil.which", return_value=None) def test_from_browser_auto_does_request_chrome(self, _mock_which, mock_extract): - setup_wizard.run_auto_setup({"FROM_BROWSER": "auto"}) + setup_wizard.run_auto_setup({"FROM_BROWSER": "auto"}, allow_browser_cookies=True) requested_browsers = {call.args[0] for call in mock_extract.call_args_list} assert "chrome" in requested_browsers From 565bb03b63d78123d26e8b9df161e1842d6e2dc3 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 16:55:49 -0700 Subject: [PATCH 37/50] fix: extend cookie-read gate to watchlist cron and warn on bad FROM_BROWSER - watchlist._run_topic now passes --no-browser-cookies, matching the MCP host: the unattended cron never probes browser cookies (no silent Chromium read / macOS Keychain prompt when FROM_BROWSER=auto is set interactively). - cookie_extraction_browsers warns to stderr on an unrecognized FROM_BROWSER value (typo or unknown name in a comma list) instead of silently returning no cookies, matching the repo's no-silent-failure rule. - Tests: research run defaults to read; --no-browser-cookies flips it to off; watchlist subprocess carries --no-browser-cookies. --- skills/last30days/scripts/lib/env.py | 27 ++++++++++++++++++----- skills/last30days/scripts/watchlist.py | 5 +++++ tests/test_security_boundaries.py | 30 ++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index c052510..29d4949 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -553,13 +553,30 @@ def cookie_extraction_browsers(config: dict[str, Any]) -> list[str]: return [] if from_browser == "off": return [] - if "," in from_browser: - browsers = [b.strip() for b in from_browser.split(",") if b.strip()] - return [b for b in browsers if b in known_browsers] - if from_browser in known_browsers: - return [from_browser] if from_browser == "auto": return silent_browsers + chromium_browsers + if "," in from_browser: + requested = [b.strip() for b in from_browser.split(",") if b.strip()] + resolved = [b for b in requested if b in known_browsers] + unknown = [b for b in requested if b not in known_browsers] + if unknown: + sys.stderr.write( + "[last30days] WARNING: FROM_BROWSER ignored unrecognized browser(s): " + f"{', '.join(unknown)} (known: {', '.join(known_browsers)})\n" + ) + sys.stderr.flush() + return resolved + if from_browser in known_browsers: + return [from_browser] + # Non-empty, not off/auto, not a known browser, not a list: unrecognized. + # Warn rather than fail silently so a typo (FROM_BROWSER=chrme) is visible + # instead of looking like "no cookies found". + sys.stderr.write( + f"[last30days] WARNING: FROM_BROWSER='{from_browser}' is not a recognized " + f"browser; no cookies will be read (known: {', '.join(known_browsers)}, " + "or 'auto'/'off')\n" + ) + sys.stderr.flush() return [] diff --git a/skills/last30days/scripts/watchlist.py b/skills/last30days/scripts/watchlist.py index bf2903a..37958b2 100644 --- a/skills/last30days/scripts/watchlist.py +++ b/skills/last30days/scripts/watchlist.py @@ -171,6 +171,11 @@ def _run_topic(topic: dict) -> dict: "--quick", "--lookback-days", "90", + # Watchlist is an unattended cron host: never probe browser + # cookies (matches the MCP server). Avoids a silent Chromium + # read / unattended macOS Keychain prompt when a user has set + # FROM_BROWSER=auto for interactive use. + "--no-browser-cookies", ], capture_output=True, text=True, diff --git a/tests/test_security_boundaries.py b/tests/test_security_boundaries.py index d8c58fc..d9e056a 100644 --- a/tests/test_security_boundaries.py +++ b/tests/test_security_boundaries.py @@ -122,3 +122,33 @@ def test_diagnose_overrides_setup_cookie_flag(monkeypatch): assert seen["policy"].browser_cookies == "plan_only" assert setup.call_args.kwargs["allow_browser_cookies"] is False + + +def test_research_run_defaults_to_browser_cookie_read(): + """A plain research run reads cookies (the path that powers X auth).""" + parser = cli.build_parser() + args, extra = parser.parse_known_args(["some topic"]) + policy = cli._config_policy_for_args(args, "some topic", extra) + assert policy.browser_cookies == "read" + + +def test_no_browser_cookies_flag_disables_research_run_cookie_read(): + """--no-browser-cookies flips a research run to the no-read policy.""" + parser = cli.build_parser() + args, extra = parser.parse_known_args(["--no-browser-cookies", "some topic"]) + policy = cli._config_policy_for_args(args, "some topic", extra) + assert policy.browser_cookies == "off" + + +def test_watchlist_subprocess_disables_browser_cookies(): + """The unattended watchlist cron must never probe browser cookies.""" + import watchlist + + fake_result = mock.Mock(returncode=1, stdout="", stderr="boom") + with mock.patch.object(watchlist, "store") as store, \ + mock.patch.object(watchlist.subprocess, "run", return_value=fake_result) as run: + store.record_run.return_value = 1 + watchlist._run_topic({"id": 1, "name": "test topic", "search_queries": None}) + + argv = run.call_args.args[0] + assert "--no-browser-cookies" in argv From 25cfced305964ba3cab0d347ade1d683eb0c7f98 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 15:07:38 -0700 Subject: [PATCH 38/50] fix: trust project config explicitly --- CONFIGURATION.md | 16 +-- skills/last30days/scripts/last30days.py | 5 +- skills/last30days/scripts/lib/env.py | 46 +++++++-- skills/last30days/scripts/lib/pipeline.py | 7 ++ tests/test_project_config_trust.py | 114 ++++++++++++++++++++++ tests/test_security_boundaries.py | 38 ++++++++ 6 files changed, 210 insertions(+), 16 deletions(-) create mode 100644 tests/test_project_config_trust.py diff --git a/CONFIGURATION.md b/CONFIGURATION.md index b6ec8a4..5f32b7f 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -69,14 +69,14 @@ Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work l ## API keys (`.env`) -The skill reads keys from a `.env` file. Two locations are supported, in priority order: +The skill reads keys from a `.env` file. Two locations are supported: -1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present. -2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback. +1. **`~/.config/last30days/.env`** at the user level (global default) - loaded by default. +2. **`.claude/last30days.env`** in the current project directory (project-scoped) - loaded only when trusted by setting `LAST30DAYS_TRUST_PROJECT_CONFIG=1` in the process environment or global config. Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't. -The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case. +The project-scoped file is useful for **intentional per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), then opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or `~/.config/last30days/.env`. Folder-mode hosts such as Codex desktop do not trust hidden project config by default, and discovery stops at the git root so unrelated parent folders cannot silently influence runs. **Source-by-source** - what each key unlocks: @@ -144,7 +144,7 @@ BSKY_APP_PASSWORD= After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant). -**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a safe preflight report for source availability, config source, browser-cookie plan, external command availability, and write destinations without reading browser cookies or running live provider probes. +**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a safe preflight report for source availability, config source, browser-cookie plan, external command availability, write destinations, and ignored untrusted project config without reading browser cookies or running live provider probes. ### Perplexity source modes @@ -334,9 +334,9 @@ The schedule field stored on each topic is metadata - the actual cron / Task Sch The skill is built to flex around different client environments. Four patterns that compose well: -### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders) +### 1. Trusted per-client `.claude/last30days.env` -The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents: +When each client has its own working directory, drop a `.claude/last30days.env` into the client folder and opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or global `~/.config/last30days/.env`. The skill loads the project file only after that trust signal. Typical contents: ```bash LAST30DAYS_MEMORY_DIR=C:\Users\\Clients\acme\Research\Last30Days @@ -345,7 +345,7 @@ INCLUDE_SOURCES=tiktok,instagram BSKY_HANDLE=.bsky.social ``` -`cd` into the client folder, run `/last30days ` as normal, no flags or wrappers. Combine with `--save-suffix=` per run if you also need to differentiate filenames within that folder. +`cd` into the client folder, run `/last30days ` as normal, no wrappers. Combine with `--save-suffix=` per run if you also need to differentiate filenames within that folder. ### 2. Per-client save dir + suffix wrapper diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index bc14ce4..e45e036 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -651,7 +651,10 @@ def _config_policy_for_args(args: argparse.Namespace, topic: str, extra_argv: li browser_mode = "read" if _setup_allows_browser_cookies(args, extra_argv) else "off" else: browser_mode = "read" - return env.ConfigLoadPolicy(browser_cookies=browser_mode) + return env.ConfigLoadPolicy( + browser_cookies=browser_mode, + inspect_ignored_project_config=args.diagnose, + ) def main() -> int: diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index 29d4949..7da9f97 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -84,11 +84,29 @@ BrowserCookieMode = Literal["off", "read", "plan_only"] class ConfigLoadPolicy: """Local-read gates for configuration loading. - Bare library calls use the safe default: no browser-cookie extraction. CLI - entry points can opt into narrower behavior after parsing command intent. + Bare library calls use the safe default: no browser-cookie extraction and no + project-scoped config. CLI entry points can opt into narrower behavior after + parsing command intent. """ browser_cookies: BrowserCookieMode = "off" + allow_project_config: bool = False + inspect_ignored_project_config: bool = False + + +def _truthy(value: Any) -> bool: + if value is None: + return False + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _project_config_trusted(policy: ConfigLoadPolicy, file_env: dict[str, Any]) -> bool: + if policy.allow_project_config: + return True + return _truthy( + os.environ.get("LAST30DAYS_TRUST_PROJECT_CONFIG") + or file_env.get("LAST30DAYS_TRUST_PROJECT_CONFIG") + ) def _check_file_permissions(path: Path) -> None: @@ -329,13 +347,15 @@ def _find_project_env() -> Path | None: """Find per-project .env by walking up from cwd. Searches for .claude/last30days.env in each parent directory, - stopping at the user's home directory or filesystem root. + stopping at the git root, user's home directory, or filesystem root. """ cwd = Path.cwd() for parent in [cwd, *cwd.parents]: candidate = parent / '.claude' / 'last30days.env' if candidate.exists(): return candidate + if (parent / ".git").exists(): + break # Stop at filesystem root or home if parent == Path.home() or parent == parent.parent: break @@ -347,7 +367,7 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]: Priority (highest wins): 1. Environment variables (os.environ) - 2. .claude/last30days.env (per-project config) + 2. Trusted .claude/last30days.env (per-project config) 3. ~/.config/last30days/.env (global config) 4. macOS Keychain items prefixed ``last30days-`` (Darwin only) """ @@ -355,9 +375,18 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]: # Load from global config file file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {} - # Load from per-project config (overrides global) - project_env_path = _find_project_env() + # Load per-project config only when trust comes from process env, global + # user config, or an explicit policy. A project file cannot grant trust to + # itself because it is not parsed until after this decision. + project_config_trusted = _project_config_trusted(policy, file_env) + project_env_path = _find_project_env() if project_config_trusted else None project_env = load_env_file(project_env_path) if project_env_path else {} + ignored_project_env_path = None + ignored_project_keys: list[str] = [] + if not project_config_trusted and policy.inspect_ignored_project_config: + ignored_project_env_path = _find_project_env() + if ignored_project_env_path: + ignored_project_keys = sorted(load_env_file(ignored_project_env_path).keys()) # Merge file sources: project > global merged_env = {**file_env, **project_env} @@ -444,6 +473,7 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]: # Optional SearXNG instance for the keyless-search fallback rung. ('LAST30DAYS_SEARXNG_URL', None), ('FROM_BROWSER', None), + ('LAST30DAYS_TRUST_PROJECT_CONFIG', None), ('SETUP_COMPLETE', None), ('INCLUDE_SOURCES', ''), ('EXCLUDE_SOURCES', ''), @@ -491,7 +521,9 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]: config['_CONFIG_SOURCE'] = 'pass' else: config['_CONFIG_SOURCE'] = 'env_only' - + if ignored_project_env_path: + config['_IGNORED_PROJECT_CONFIG'] = str(ignored_project_env_path) + config['_IGNORED_PROJECT_CONFIG_KEYS'] = ignored_project_keys config['_BROWSER_COOKIE_MODE'] = policy.browser_cookies config['_BROWSER_COOKIE_BROWSERS'] = cookie_extraction_browsers(config) diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index 7a9c810..504eb04 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -203,6 +203,10 @@ def diagnose( "browsers": list(config.get("_BROWSER_COOKIE_BROWSERS") or []), "reads_values": False if safe else config.get("_BROWSER_COOKIE_MODE") == "read", } + ignored_project_keys = list(config.get("_IGNORED_PROJECT_CONFIG_KEYS") or []) + ignored_endpoint_overrides = [ + key for key in ignored_project_keys if key in {"OPENAI_BASE_URL", "XAI_BASE_URL"} + ] local_writes: list[dict[str, str]] = [] if config.get("LAST30DAYS_MEMORY_DIR"): local_writes.append({"kind": "report", "path": str(config.get("LAST30DAYS_MEMORY_DIR"))}) @@ -224,6 +228,9 @@ def diagnose( "available_sources": available_sources(config, requested_sources), "safe": safe, "config_source": config.get("_CONFIG_SOURCE"), + "ignored_project_config": config.get("_IGNORED_PROJECT_CONFIG"), + "ignored_project_config_keys": ignored_project_keys, + "ignored_endpoint_overrides": ignored_endpoint_overrides, "browser_cookies": browser_cookies, "external_commands": external_commands, "credential_destinations": credential_destinations, diff --git a/tests/test_project_config_trust.py b/tests/test_project_config_trust.py new file mode 100644 index 0000000..3e6157a --- /dev/null +++ b/tests/test_project_config_trust.py @@ -0,0 +1,114 @@ +"""Tests for trusted project-scoped configuration.""" + +from __future__ import annotations + +from unittest import mock + +from lib import env, pipeline + + +def _neutral_secret_sources(): + return ( + mock.patch.object(env, "_load_keychain", return_value={}), + mock.patch.object(env, "_load_pass", return_value={}), + ) + + +def test_untrusted_project_config_is_ignored_by_default(tmp_path, monkeypatch): + project_env = tmp_path / ".claude" / "last30days.env" + project_env.parent.mkdir() + project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(env, "CONFIG_FILE", None) + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config() + + assert cfg["XAI_API_KEY"] is None + assert cfg["_CONFIG_SOURCE"] == "env_only" + + +def test_project_config_loads_with_global_trust_signal(tmp_path, monkeypatch): + global_env = tmp_path / "global.env" + global_env.write_text("LAST30DAYS_TRUST_PROJECT_CONFIG=1\n", encoding="utf-8") + project_dir = tmp_path / "project" + project_env = project_dir / ".claude" / "last30days.env" + project_env.parent.mkdir(parents=True) + project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8") + monkeypatch.chdir(project_dir) + monkeypatch.setattr(env, "CONFIG_FILE", global_env) + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config() + + assert cfg["XAI_API_KEY"] == "xai-project" + assert cfg["_CONFIG_SOURCE"].startswith(f"project:{project_env}") + + +def test_project_config_discovery_stops_at_git_root(tmp_path, monkeypatch): + outside_env = tmp_path / ".claude" / "last30days.env" + outside_env.parent.mkdir() + outside_env.write_text("XAI_API_KEY=outside\n", encoding="utf-8") + repo = tmp_path / "repo" + workdir = repo / "nested" + workdir.mkdir(parents=True) + (repo / ".git").mkdir() + monkeypatch.chdir(workdir) + monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "1") + monkeypatch.setattr(env, "CONFIG_FILE", None) + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config() + + assert cfg["XAI_API_KEY"] is None + assert cfg["_CONFIG_SOURCE"] == "env_only" + + +def test_global_config_loads_when_project_config_is_untrusted(tmp_path, monkeypatch): + global_env = tmp_path / "global.env" + global_env.write_text("XAI_API_KEY=global\n", encoding="utf-8") + project_dir = tmp_path / "project" + project_env = project_dir / ".claude" / "last30days.env" + project_env.parent.mkdir(parents=True) + project_env.write_text("XAI_API_KEY=project\n", encoding="utf-8") + monkeypatch.chdir(project_dir) + monkeypatch.setattr(env, "CONFIG_FILE", global_env) + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config() + + assert cfg["XAI_API_KEY"] == "global" + assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}") + + +def test_diagnose_reports_ignored_untrusted_endpoint_override(tmp_path, monkeypatch): + project_env = tmp_path / ".claude" / "last30days.env" + project_env.parent.mkdir() + project_env.write_text( + "OPENAI_BASE_URL=https://attacker.example\nOPENAI_API_KEY=sk-not-reported\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(env, "CONFIG_FILE", None) + monkeypatch.setenv("OPENAI_API_KEY", "sk-global") + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config( + policy=env.ConfigLoadPolicy(inspect_ignored_project_config=True) + ) + diag = pipeline.diagnose(cfg, safe=True) + + assert cfg["OPENAI_API_KEY"] == "sk-global" + assert cfg["OPENAI_BASE_URL"] is None + assert diag["ignored_project_config"] == str(project_env) + assert diag["ignored_endpoint_overrides"] == ["OPENAI_BASE_URL"] + assert "sk-not-reported" not in str(diag) diff --git a/tests/test_security_boundaries.py b/tests/test_security_boundaries.py index d9e056a..05325ea 100644 --- a/tests/test_security_boundaries.py +++ b/tests/test_security_boundaries.py @@ -11,6 +11,7 @@ from contextlib import redirect_stderr, redirect_stdout from unittest import mock import last30days as cli +from lib import env def test_importing_cli_does_not_load_config_or_propagate_endpoints(monkeypatch): @@ -152,3 +153,40 @@ def test_watchlist_subprocess_disables_browser_cookies(): argv = run.call_args.args[0] assert "--no-browser-cookies" in argv + + +def test_project_config_ignored_by_default_and_cannot_self_trust(tmp_path, monkeypatch): + project_env = tmp_path / ".claude" / "last30days.env" + project_env.parent.mkdir() + project_env.write_text( + "LAST30DAYS_TRUST_PROJECT_CONFIG=1\nOPENAI_BASE_URL=https://example.invalid\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(env, "CONFIG_FILE", None) + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + + with mock.patch.object(env, "_load_keychain", return_value={}), \ + mock.patch.object(env, "_load_pass", return_value={}): + cfg = env.get_config() + + assert cfg["OPENAI_BASE_URL"] is None + assert cfg["_CONFIG_SOURCE"] == "env_only" + + +def test_project_config_loads_with_process_trust_signal(tmp_path, monkeypatch): + project_env = tmp_path / ".claude" / "last30days.env" + project_env.parent.mkdir() + project_env.write_text("OPENAI_BASE_URL=https://trusted.example\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(env, "CONFIG_FILE", None) + monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "1") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + + with mock.patch.object(env, "_load_keychain", return_value={}), \ + mock.patch.object(env, "_load_pass", return_value={}): + cfg = env.get_config() + + assert cfg["OPENAI_BASE_URL"] == "https://trusted.example" + assert cfg["_CONFIG_SOURCE"].startswith(f"project:{project_env}") From c5ff5505b30d60ba5f2222d0c185d5bad43d4256 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 15:41:18 -0700 Subject: [PATCH 39/50] fix: tighten project config trust reporting --- skills/last30days/scripts/lib/env.py | 14 ++--- skills/last30days/scripts/lib/pipeline.py | 9 +++- tests/test_project_config_trust.py | 63 ++++++++++++++++++++++- 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index 7da9f97..76e83e1 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -103,10 +103,10 @@ def _truthy(value: Any) -> bool: def _project_config_trusted(policy: ConfigLoadPolicy, file_env: dict[str, Any]) -> bool: if policy.allow_project_config: return True - return _truthy( - os.environ.get("LAST30DAYS_TRUST_PROJECT_CONFIG") - or file_env.get("LAST30DAYS_TRUST_PROJECT_CONFIG") - ) + process_value = os.environ.get("LAST30DAYS_TRUST_PROJECT_CONFIG") + if process_value is not None: + return _truthy(process_value) + return _truthy(file_env.get("LAST30DAYS_TRUST_PROJECT_CONFIG")) def _check_file_permissions(path: Path) -> None: @@ -658,9 +658,11 @@ def get_x_source_with_method(config: dict[str, Any]) -> tuple[str | None, str]: return None, "none" -def config_exists() -> bool: +def config_exists(policy: ConfigLoadPolicy | None = None) -> bool: """Check if any configuration source exists.""" - if _find_project_env(): + policy = policy or ConfigLoadPolicy() + file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE and CONFIG_FILE.exists() else {} + if _project_config_trusted(policy, file_env) and _find_project_env(): return True if CONFIG_FILE: return CONFIG_FILE.exists() diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index 504eb04..4d08f3d 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -204,8 +204,15 @@ def diagnose( "reads_values": False if safe else config.get("_BROWSER_COOKIE_MODE") == "read", } ignored_project_keys = list(config.get("_IGNORED_PROJECT_CONFIG_KEYS") or []) + endpoint_override_keys = { + "BSKY_SEARCH_HOST", + "LAST30DAYS_SEARXNG_URL", + "OPENAI_BASE_URL", + "XAI_BASE_URL", + "XIAOHONGSHU_API_BASE", + } ignored_endpoint_overrides = [ - key for key in ignored_project_keys if key in {"OPENAI_BASE_URL", "XAI_BASE_URL"} + key for key in ignored_project_keys if key in endpoint_override_keys ] local_writes: list[dict[str, str]] = [] if config.get("LAST30DAYS_MEMORY_DIR"): diff --git a/tests/test_project_config_trust.py b/tests/test_project_config_trust.py index 3e6157a..8f5f8bc 100644 --- a/tests/test_project_config_trust.py +++ b/tests/test_project_config_trust.py @@ -49,6 +49,25 @@ def test_project_config_loads_with_global_trust_signal(tmp_path, monkeypatch): assert cfg["_CONFIG_SOURCE"].startswith(f"project:{project_env}") +def test_empty_process_trust_signal_overrides_global_trust_signal(tmp_path, monkeypatch): + global_env = tmp_path / "global.env" + global_env.write_text("LAST30DAYS_TRUST_PROJECT_CONFIG=1\n", encoding="utf-8") + project_dir = tmp_path / "project" + project_env = project_dir / ".claude" / "last30days.env" + project_env.parent.mkdir(parents=True) + project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8") + monkeypatch.chdir(project_dir) + monkeypatch.setattr(env, "CONFIG_FILE", global_env) + monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "") + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config() + + assert cfg["XAI_API_KEY"] is None + assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}") + + def test_project_config_discovery_stops_at_git_root(tmp_path, monkeypatch): outside_env = tmp_path / ".claude" / "last30days.env" outside_env.parent.mkdir() @@ -88,11 +107,46 @@ def test_global_config_loads_when_project_config_is_untrusted(tmp_path, monkeypa assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}") +def test_config_exists_ignores_untrusted_project_config(tmp_path, monkeypatch): + project_env = tmp_path / ".claude" / "last30days.env" + project_env.parent.mkdir() + project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(env, "CONFIG_FILE", None) + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + + assert env.config_exists() is False + + +def test_config_exists_reports_trusted_project_config(tmp_path, monkeypatch): + project_env = tmp_path / ".claude" / "last30days.env" + project_env.parent.mkdir() + project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(env, "CONFIG_FILE", None) + monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "1") + + assert env.config_exists() is True + + +def test_config_exists_reports_global_config(tmp_path, monkeypatch): + global_env = tmp_path / "global.env" + global_env.write_text("XAI_API_KEY=xai-global\n", encoding="utf-8") + monkeypatch.setattr(env, "CONFIG_FILE", global_env) + monkeypatch.delenv("LAST30DAYS_TRUST_PROJECT_CONFIG", raising=False) + + assert env.config_exists() is True + + def test_diagnose_reports_ignored_untrusted_endpoint_override(tmp_path, monkeypatch): project_env = tmp_path / ".claude" / "last30days.env" project_env.parent.mkdir() project_env.write_text( - "OPENAI_BASE_URL=https://attacker.example\nOPENAI_API_KEY=sk-not-reported\n", + "BSKY_SEARCH_HOST=https://bsky-attacker.example\n" + "LAST30DAYS_SEARXNG_URL=https://searxng-attacker.example\n" + "OPENAI_BASE_URL=https://attacker.example\n" + "OPENAI_API_KEY=sk-not-reported\n" + "XIAOHONGSHU_API_BASE=https://xhs-attacker.example\n", encoding="utf-8", ) monkeypatch.chdir(tmp_path) @@ -110,5 +164,10 @@ def test_diagnose_reports_ignored_untrusted_endpoint_override(tmp_path, monkeypa assert cfg["OPENAI_API_KEY"] == "sk-global" assert cfg["OPENAI_BASE_URL"] is None assert diag["ignored_project_config"] == str(project_env) - assert diag["ignored_endpoint_overrides"] == ["OPENAI_BASE_URL"] + assert diag["ignored_endpoint_overrides"] == [ + "BSKY_SEARCH_HOST", + "LAST30DAYS_SEARXNG_URL", + "OPENAI_BASE_URL", + "XIAOHONGSHU_API_BASE", + ] assert "sk-not-reported" not in str(diag) From 5f86484c0a00fa78208f24d44479f11b64b47d8c Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 16:58:51 -0700 Subject: [PATCH 40/50] fix: report YOUTUBE_SSH_HOST as ignored endpoint override + test deny precedence - pipeline diagnose now lists LAST30DAYS_YOUTUBE_SSH_HOST among ignored endpoint overrides; a malicious project config setting it would redirect yt-dlp through an attacker SSH host, so it belongs in the highlight set. - Test: explicit process LAST30DAYS_TRUST_PROJECT_CONFIG=0 denies trust even when global config sets =1 (process-wins precedence was previously only covered for the empty-string case). --- skills/last30days/scripts/lib/pipeline.py | 1 + tests/test_project_config_trust.py | 24 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index 4d08f3d..52f18eb 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -207,6 +207,7 @@ def diagnose( endpoint_override_keys = { "BSKY_SEARCH_HOST", "LAST30DAYS_SEARXNG_URL", + "LAST30DAYS_YOUTUBE_SSH_HOST", "OPENAI_BASE_URL", "XAI_BASE_URL", "XIAOHONGSHU_API_BASE", diff --git a/tests/test_project_config_trust.py b/tests/test_project_config_trust.py index 8f5f8bc..3784643 100644 --- a/tests/test_project_config_trust.py +++ b/tests/test_project_config_trust.py @@ -68,6 +68,26 @@ def test_empty_process_trust_signal_overrides_global_trust_signal(tmp_path, monk assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}") +def test_explicit_zero_process_trust_signal_overrides_global_trust_signal(tmp_path, monkeypatch): + """An explicit process `=0` is a deny and wins over a global `=1`.""" + global_env = tmp_path / "global.env" + global_env.write_text("LAST30DAYS_TRUST_PROJECT_CONFIG=1\n", encoding="utf-8") + project_dir = tmp_path / "project" + project_env = project_dir / ".claude" / "last30days.env" + project_env.parent.mkdir(parents=True) + project_env.write_text("XAI_API_KEY=xai-project\n", encoding="utf-8") + monkeypatch.chdir(project_dir) + monkeypatch.setattr(env, "CONFIG_FILE", global_env) + monkeypatch.setenv("LAST30DAYS_TRUST_PROJECT_CONFIG", "0") + + keychain, pass_store = _neutral_secret_sources() + with keychain, pass_store: + cfg = env.get_config() + + assert cfg["XAI_API_KEY"] is None + assert cfg["_CONFIG_SOURCE"].startswith(f"global:{global_env}") + + def test_project_config_discovery_stops_at_git_root(tmp_path, monkeypatch): outside_env = tmp_path / ".claude" / "last30days.env" outside_env.parent.mkdir() @@ -144,6 +164,7 @@ def test_diagnose_reports_ignored_untrusted_endpoint_override(tmp_path, monkeypa project_env.write_text( "BSKY_SEARCH_HOST=https://bsky-attacker.example\n" "LAST30DAYS_SEARXNG_URL=https://searxng-attacker.example\n" + "LAST30DAYS_YOUTUBE_SSH_HOST=attacker-host\n" "OPENAI_BASE_URL=https://attacker.example\n" "OPENAI_API_KEY=sk-not-reported\n" "XIAOHONGSHU_API_BASE=https://xhs-attacker.example\n", @@ -164,9 +185,10 @@ def test_diagnose_reports_ignored_untrusted_endpoint_override(tmp_path, monkeypa assert cfg["OPENAI_API_KEY"] == "sk-global" assert cfg["OPENAI_BASE_URL"] is None assert diag["ignored_project_config"] == str(project_env) - assert diag["ignored_endpoint_overrides"] == [ + assert sorted(diag["ignored_endpoint_overrides"]) == [ "BSKY_SEARCH_HOST", "LAST30DAYS_SEARXNG_URL", + "LAST30DAYS_YOUTUBE_SSH_HOST", "OPENAI_BASE_URL", "XIAOHONGSHU_API_BASE", ] From 91d588719952aaa45c30821b41b8965d0be35670 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 15:08:21 -0700 Subject: [PATCH 41/50] fix: reject stale CLI and MCP flags --- mcp/internal/tools/research.go | 4 +-- mcp/internal/tools/research_test.go | 12 +++++++ skills/last30days/scripts/last30days.py | 35 ++++++++++++++++++++ tests/test_cli_v3.py | 40 +++++++++++++++++++++++ tests/test_doc_flag_contract.py | 43 +++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tests/test_doc_flag_contract.py diff --git a/mcp/internal/tools/research.go b/mcp/internal/tools/research.go index 42ecdb5..7f052ad 100644 --- a/mcp/internal/tools/research.go +++ b/mcp/internal/tools/research.go @@ -36,7 +36,7 @@ func Register(s *server.MCPServer, cfg Config) { mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")), mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")), mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")), - mcplib.WithReadOnlyHintAnnotation(true), + mcplib.WithReadOnlyHintAnnotation(false), mcplib.WithDestructiveHintAnnotation(false), mcplib.WithOpenWorldHintAnnotation(true), ), @@ -90,7 +90,7 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc { func researchRunArgs(topic, emit string, save bool) []string { runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"} if save { - runArgs = append(runArgs, "--save") + runArgs = append(runArgs, "--save-dir", "~/Documents/Last30Days") } return runArgs } diff --git a/mcp/internal/tools/research_test.go b/mcp/internal/tools/research_test.go index ed06e49..5f2c1a0 100644 --- a/mcp/internal/tools/research_test.go +++ b/mcp/internal/tools/research_test.go @@ -105,6 +105,18 @@ func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) { } } +func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) { + args := researchRunArgs("OpenAI", "html", true) + got := strings.Join(args, "\x00") + if strings.Contains(got, "--save\x00") || strings.HasSuffix(got, "--save") { + t.Fatalf("args still include unsupported --save: %#v", args) + } + want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "~/Documents/Last30Days"} + if got != strings.Join(want, "\x00") { + t.Fatalf("args = %#v, want %#v", args, want) + } +} + func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) { // Validation failures are returned as MCP tool errors (not Go errors) // so Claude sees a structured failure with a readable message rather diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index e45e036..88c3767 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -642,6 +642,40 @@ def _setup_allows_browser_cookies(args: argparse.Namespace, extra_argv: list[str ) +SETUP_PASSTHROUGH_FLAGS = { + "--allow-browser-cookies", + "--device-auth", + "--github", + "--openclaw", +} + +SKILL_ONLY_FLAGS = { + "--agent", +} + + +def _validate_extra_argv(parser: argparse.ArgumentParser, topic: str, extra_argv: list[str]) -> None: + if not extra_argv: + return + if topic.lower() == "setup": + unsupported = [arg for arg in extra_argv if arg not in SETUP_PASSTHROUGH_FLAGS] + if unsupported: + parser.error( + "unsupported setup argument(s): " + + ", ".join(unsupported) + + f"; supported setup passthrough flags are {', '.join(sorted(SETUP_PASSTHROUGH_FLAGS))}" + ) + return + skill_only = [arg for arg in extra_argv if arg in SKILL_ONLY_FLAGS] + if skill_only: + parser.error( + "unsupported Python CLI argument(s): " + + ", ".join(skill_only) + + "; these are skill arguments and must not be forwarded to scripts/last30days.py" + ) + parser.error("unsupported Python CLI argument(s): " + ", ".join(extra_argv)) + + def _config_policy_for_args(args: argparse.Namespace, topic: str, extra_argv: list[str]) -> env.ConfigLoadPolicy: if args.no_browser_cookies: browser_mode = "off" @@ -666,6 +700,7 @@ def main() -> int: os.environ["LAST30DAYS_DEBUG"] = "1" topic = " ".join(args.topic).strip() + _validate_extra_argv(parser, topic, extra_argv) config = env.get_config(policy=_config_policy_for_args(args, topic, extra_argv)) _propagate_config_to_environ(config) diff --git a/tests/test_cli_v3.py b/tests/test_cli_v3.py index 52814e1..d85c3dc 100644 --- a/tests/test_cli_v3.py +++ b/tests/test_cli_v3.py @@ -147,6 +147,46 @@ class CliV3Tests(unittest.TestCase): self.assertEqual(["biosecurity"], args.topic) self.assertEqual([], extra) + def test_research_unknown_flag_fails_before_config_load(self): + with mock.patch.object( + cli.env, "get_config", side_effect=AssertionError("config should not load") + ), mock.patch.object(sys, "argv", ["last30days.py", "topic", "--save"]): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc: + cli.main() + self.assertEqual(2, exc.exception.code) + self.assertIn("--save", stderr.getvalue()) + + def test_agent_is_skill_argument_not_python_cli_flag(self): + with mock.patch.object( + cli.env, "get_config", side_effect=AssertionError("config should not load") + ), mock.patch.object(sys, "argv", ["last30days.py", "topic", "--agent"]): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc: + cli.main() + self.assertEqual(2, exc.exception.code) + self.assertIn("skill arguments", stderr.getvalue()) + + def test_setup_passthrough_flags_remain_scoped_to_setup(self): + with mock.patch.object(cli.env, "get_config", return_value={}), \ + mock.patch("lib.setup_wizard.run_github_auth", return_value={"status": "cancelled"}), \ + mock.patch.object(sys, "argv", ["last30days.py", "setup", "--github"]): + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + rc = cli.main() + self.assertEqual(0, rc) + + def test_setup_rejects_unknown_passthrough_flag_before_config_load(self): + with mock.patch.object( + cli.env, "get_config", side_effect=AssertionError("config should not load") + ), mock.patch.object(sys, "argv", ["last30days.py", "setup", "--bad"]): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc: + cli.main() + self.assertEqual(2, exc.exception.code) + self.assertIn("--bad", stderr.getvalue()) + def test_ensure_supported_python_rejects_old_interpreter_with_actionable_error(self): stderr = io.StringIO() with redirect_stderr(stderr): diff --git a/tests/test_doc_flag_contract.py b/tests/test_doc_flag_contract.py new file mode 100644 index 0000000..5858475 --- /dev/null +++ b/tests/test_doc_flag_contract.py @@ -0,0 +1,43 @@ +"""Documentation contract for Python CLI and wrapper-only flags.""" + +from __future__ import annotations + +from pathlib import Path + +import last30days as cli + +ROOT = Path(__file__).resolve().parents[1] +CONFIGURATION = ROOT / "CONFIGURATION.md" +SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" + + +def _parser_flags() -> set[str]: + parser = cli.build_parser() + flags: set[str] = set() + for action in parser._actions: + flags.update(action.option_strings) + return flags + + +def test_configuration_documents_new_safety_flags(): + text = CONFIGURATION.read_text(encoding="utf-8") + flags = _parser_flags() + assert "--no-browser-cookies" in flags + assert "--no-browser-cookies" in text + assert "--save-dir" in text + assert "--output" in text + + +def test_save_is_not_documented_as_python_cli_flag(): + text = CONFIGURATION.read_text(encoding="utf-8") + assert "--save-dir " in text + assert "--save " not in text + assert "`--save`" not in text + + +def test_agent_is_documented_as_skill_argument_not_python_flag(): + text = SKILL_MD.read_text(encoding="utf-8") + start = text.index("## Agent Mode (--agent flag)") + agent_section = text[start:start + 2000] + assert "If `--agent` appears in ARGUMENTS" in agent_section + assert "Skill tool" in text From 472eb88bba550a54b18693844fb7d3ab92ef3dca Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 15:42:34 -0700 Subject: [PATCH 42/50] fix: align stale CLI and MCP save behavior --- mcp/internal/tools/research.go | 7 ++++++- mcp/internal/tools/research_test.go | 10 ++++++++++ skills/last30days/scripts/last30days.py | 6 +++++- tests/test_cli_v3.py | 12 ++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/mcp/internal/tools/research.go b/mcp/internal/tools/research.go index 7f052ad..1232d13 100644 --- a/mcp/internal/tools/research.go +++ b/mcp/internal/tools/research.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "os" "strings" mcplib "github.com/mark3labs/mcp-go/mcp" @@ -90,7 +91,11 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc { func researchRunArgs(topic, emit string, save bool) []string { runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"} if save { - runArgs = append(runArgs, "--save-dir", "~/Documents/Last30Days") + saveDir := os.Getenv("LAST30DAYS_MEMORY_DIR") + if saveDir == "" { + saveDir = "~/Documents/Last30Days" + } + runArgs = append(runArgs, "--save-dir", saveDir) } return runArgs } diff --git a/mcp/internal/tools/research_test.go b/mcp/internal/tools/research_test.go index 5f2c1a0..2a257b8 100644 --- a/mcp/internal/tools/research_test.go +++ b/mcp/internal/tools/research_test.go @@ -106,6 +106,7 @@ func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) { } func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) { + t.Setenv("LAST30DAYS_MEMORY_DIR", "") args := researchRunArgs("OpenAI", "html", true) got := strings.Join(args, "\x00") if strings.Contains(got, "--save\x00") || strings.HasSuffix(got, "--save") { @@ -117,6 +118,15 @@ func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) { } } +func TestResearchRunArgsSaveUsesMemoryDirEnvOverride(t *testing.T) { + t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports") + args := researchRunArgs("OpenAI", "html", true) + want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "/tmp/last30days-reports"} + if strings.Join(args, "\x00") != strings.Join(want, "\x00") { + t.Fatalf("args = %#v, want %#v", args, want) + } +} + func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) { // Validation failures are returned as MCP tool errors (not Go errors) // so Claude sees a structured failure with a readable message rather diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 88c3767..5746a54 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -667,12 +667,16 @@ def _validate_extra_argv(parser: argparse.ArgumentParser, topic: str, extra_argv ) return skill_only = [arg for arg in extra_argv if arg in SKILL_ONLY_FLAGS] + other_unknown = [arg for arg in extra_argv if arg not in SKILL_ONLY_FLAGS] if skill_only: - parser.error( + message = ( "unsupported Python CLI argument(s): " + ", ".join(skill_only) + "; these are skill arguments and must not be forwarded to scripts/last30days.py" ) + if other_unknown: + message += "; also unsupported: " + ", ".join(other_unknown) + parser.error(message) parser.error("unsupported Python CLI argument(s): " + ", ".join(extra_argv)) diff --git a/tests/test_cli_v3.py b/tests/test_cli_v3.py index d85c3dc..45d7fc1 100644 --- a/tests/test_cli_v3.py +++ b/tests/test_cli_v3.py @@ -167,6 +167,18 @@ class CliV3Tests(unittest.TestCase): self.assertEqual(2, exc.exception.code) self.assertIn("skill arguments", stderr.getvalue()) + def test_agent_error_includes_other_unknown_flags(self): + with mock.patch.object( + cli.env, "get_config", side_effect=AssertionError("config should not load") + ), mock.patch.object(sys, "argv", ["last30days.py", "topic", "--agent", "--save"]): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as exc: + cli.main() + self.assertEqual(2, exc.exception.code) + message = stderr.getvalue() + self.assertIn("--agent", message) + self.assertIn("--save", message) + def test_setup_passthrough_flags_remain_scoped_to_setup(self): with mock.patch.object(cli.env, "get_config", return_value={}), \ mock.patch("lib.setup_wizard.run_github_auth", return_value={"status": "cancelled"}), \ From f6edf563b4814fde8908a98995155edc6de5f47f Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 15:08:58 -0700 Subject: [PATCH 43/50] docs: align host security contracts --- CONFIGURATION.md | 2 +- README.md | 2 +- skills/last30days/SKILL.md | 16 +++++----- skills/last30days/scripts/lib/ui.py | 6 ++-- tests/test_codex_host_contract.py | 37 +++++++++++++++++++++++ tests/test_doc_security_contract.py | 46 +++++++++++++++++++++++++++++ 6 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 tests/test_codex_host_contract.py create mode 100644 tests/test_doc_security_contract.py diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 5f32b7f..fc69674 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -238,7 +238,7 @@ Accepts the same comma-separated names and aliases as `--search` (`web` → grou `/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=` to pin one): 1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY` -2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`) +2. **OpenAI** - `OPENAI_API_KEY` only. Codex ChatGPT auth at `~/.codex/auth.json` is intentionally not used as an OpenAI provider credential. 3. **xAI** - `XAI_API_KEY` 4. **OpenRouter** - `OPENROUTER_API_KEY` (Sonar fallback for the Perplexity source / `--deep-research`; also usable as a reasoning provider) 5. **Local / deterministic** - always available, lowest quality diff --git a/README.md b/README.md index 4c2653a..18e3b3d 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ These platforms don't have relationships with each other. X doesn't know what Re | X / Twitter | Log into x.com in any browser, or set `XQUIK_API_KEY` / `XAI_API_KEY` | Browser cookies are free; keys are provider-specific | | YouTube | `brew install yt-dlp` | Free | | Bluesky | App password from bsky.app | Free | -| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG | +| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls, then PAYG | | Perplexity Sonar / Search API / Deep Research | Perplexity key, or OpenRouter key as Sonar fallback | Pay as you go | | Web search | Brave Search key | 2,000 free queries/month | diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index 0354865..cb15cd7 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -417,14 +417,14 @@ Options: **If the user picks Auto setup:** -Get cookie consent first. Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`; if so, skip the consent prompt and run setup directly. Otherwise **call AskUserQuestion:** +Get cookie consent first. Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`; if so, skip the consent prompt and run `setup --allow-browser-cookies` directly. Otherwise **call AskUserQuestion:** Question: "Auto setup will scan your browser (Firefox/Safari) for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. OK to proceed?" Options: -- "Yes, scan my cookies for X" - run `python3 skills/last30days/scripts/last30days.py setup` (relative to the skill root). Append `BROWSER_CONSENT=true` to `.env` after setup completes. +- "Yes, scan my cookies for X" - run `python3 skills/last30days/scripts/last30days.py setup --allow-browser-cookies` (relative to the skill root). Append `BROWSER_CONSENT=true` to `.env` after setup completes. - "Skip X, just set up YouTube + Digg" - run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. Skips all cookie reads; still installs yt-dlp and Digg. - "I have an xAI API key instead" - ask them to paste it, write `XAI_API_KEY` to `.env`, then run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup` (installs yt-dlp + Digg, no cookie read). -The `setup` run extracts cookies (Firefox/Safari by default - never Chrome, to avoid a macOS Keychain prompt) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if installed off-PATH; recommend-only if `npx` is unavailable). Show the user what was found and installed - including whether Digg landed on PATH (active) or off-PATH (installed but not yet active). +The consented `setup --allow-browser-cookies` run extracts cookies (Firefox/Safari by default - never Chrome, to avoid a macOS Keychain prompt, unless `FROM_BROWSER=auto` or a named Chromium browser was explicitly configured) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; Digg activates only when the binary is on the **agent subprocess PATH**, typically `$HOME/.local/bin`; setup reports honestly if installed off-PATH; recommend-only if `npx` is unavailable). Show the user what was found and installed - including whether Digg landed on PATH (active) or off-PATH (installed but not yet active). **macOS Full Disk Access remediation.** After the `setup` run, inspect its stderr. If it contains `Permission denied reading Cookies.binarycookies` and the platform is macOS, the OS blocked the read - surface the fix instead of swallowing it: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry of the `setup` command. If the user skips, continue. @@ -475,8 +475,8 @@ For hosts without interactive modal prompts (OpenClaw, Codex, Cursor, Gemini CLI **1. Welcome.** One short branded line, e.g.: `Welcome to /last30days - let me get you set up (about 30 seconds).` -**2. Cookie consent (ask BEFORE reading anything).** First check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env` (e.g. granted in a prior Claude Code session); if so, skip this prompt and run `setup` directly. Otherwise ask. Example: `I can read your browser cookies (Firefox/Safari) to unlock X/Twitter and other logged-in sources. Want me to? (yes / no)` **Wait for the answer.** - - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup` (and append `BROWSER_CONSENT=true` to `.env` after it completes). Extracts cookies (Firefox/Safari, never Chrome) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; activates only when on the agent subprocess PATH, typically `$HOME/.local/bin`; reports honestly if off-PATH; recommend-only if `npx` is unavailable). +**2. Cookie consent (ask BEFORE reading anything).** First check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env` (e.g. granted in a prior Claude Code session); if so, skip this prompt and run `setup --allow-browser-cookies` directly. Otherwise ask. Example: `I can read your browser cookies (Firefox/Safari) to unlock X/Twitter and other logged-in sources. Want me to? (yes / no)` **Wait for the answer.** + - On **yes** → run `python3 skills/last30days/scripts/last30days.py setup --allow-browser-cookies` (and append `BROWSER_CONSENT=true` to `.env` after it completes). Extracts cookies (Firefox/Safari, never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser was explicitly configured) and best-effort installs yt-dlp (YouTube) and the free, keyless Digg CLI (`digg-pp-cli` via `@mvanhorn/printing-press-library install digg --cli-only`; activates only when on the agent subprocess PATH, typically `$HOME/.local/bin`; reports honestly if off-PATH; recommend-only if `npx` is unavailable). - On **no** → run `FROM_BROWSER=off python3 skills/last30days/scripts/last30days.py setup`. Skips all cookie reads; still installs yt-dlp and Digg, still writes `SETUP_COMPLETE`. **3. Full Disk Access remediation (macOS only).** After `setup`, inspect stderr. If it contains `Permission denied reading Cookies.binarycookies` on macOS, surface: `macOS blocked the cookie read. To enable X/Twitter: System Settings > Privacy & Security > Full Disk Access > enable your terminal (or the Claude app), then I can retry.` Offer ONE retry. If skipped, continue. @@ -487,7 +487,7 @@ For hosts without interactive modal prompts (OpenClaw, Codex, Cursor, Gemini CLI - On **timeout / denied** → tell the user it didn't complete and offer to retry or skip. - On **no** → note they can run it later by asking to set up ScrapeCreators, then continue. -**5. Complete.** Once `SETUP_COMPLETE=true` is written, briefly confirm which sources are now active (read the `setup --github` JSON `persisted` field, or re-run `--diagnose`) and proceed to research. +**5. Complete.** Once `SETUP_COMPLETE=true` is written, briefly confirm which sources are now active (read the `setup --github` JSON `persisted` field, or re-run safe `--diagnose`) and proceed to research. For Codex desktop, Cursor, Gemini CLI, and raw folder-mode hosts, hidden `.claude/last30days.env` project config is ignored unless `LAST30DAYS_TRUST_PROJECT_CONFIG=1` is set from the process environment or global config; do not tell the user a project file is active unless diagnose reports it as the config source. --- @@ -1978,7 +1978,7 @@ Want another prompt? Just tell me what you're creating next. - Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth) - Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth) - Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data) -- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits) +- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (10,000 free calls, then PAYG) - Optionally sends search queries to Brave Search API, Parallel AI API, Perplexity API (`api.perplexity.ai`), or OpenRouter API for web search / synthesis - Fetches public Reddit thread data from `reddit.com` for engagement metrics - Stores research findings in local SQLite database (watchlist mode only) @@ -1991,7 +1991,7 @@ Want another prompt? Just tell me what you're creating next. - Does not log, cache, or write API keys to output files - Does not send data to any endpoint not listed above - Hacker News and Polymarket sources are always available (no API key, no binary dependency) -- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable. +- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable. - Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output **Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed) diff --git a/skills/last30days/scripts/lib/ui.py b/skills/last30days/scripts/lib/ui.py index f68ff14..8a37afe 100644 --- a/skills/last30days/scripts/lib/ui.py +++ b/skills/last30days/scripts/lib/ui.py @@ -200,7 +200,7 @@ Just start with "last30" and talk to me like normal. # Shorter promo for single missing key PROMO_SINGLE_KEY = { - "reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 100 free credits, no CC - scrapecreators.com\n", + "reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 10,000 free calls, no CC - scrapecreators.com\n", "x": "\n💡 Unlock X: log into x.com in your browser, then re-run. " "Firefox works on all platforms. Safari works on macOS (detected automatically). " "Chrome, Brave, Edge, Arc, Vivaldi, Opera, or Chromium on macOS require " @@ -214,7 +214,7 @@ BIRD_AUTH_HELP = f""" {Colors.YELLOW}Bird authentication failed.{Colors.RESET} To fix this: -1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env or .claude/last30days.env +1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env, or to trusted .claude/last30days.env with LAST30DAYS_TRUST_PROJECT_CONFIG=1 2. Or set XAI_API_KEY for the xAI fallback backend """ @@ -222,7 +222,7 @@ BIRD_AUTH_HELP_PLAIN = """ Bird authentication failed. To fix this: -1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env or .claude/last30days.env +1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env, or to trusted .claude/last30days.env with LAST30DAYS_TRUST_PROJECT_CONFIG=1 2. Or set XAI_API_KEY for the xAI fallback backend """ diff --git a/tests/test_codex_host_contract.py b/tests/test_codex_host_contract.py new file mode 100644 index 0000000..a610721 --- /dev/null +++ b/tests/test_codex_host_contract.py @@ -0,0 +1,37 @@ +"""Host-contract tests for non-modal agent runtimes.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" + + +def _prose_flow() -> str: + text = SKILL_MD.read_text(encoding="utf-8") + start = text.index("### Non-Modal Prose Flow") + end = text.index("### Manual Setup Guide", start) + return text[start:end] + + +def test_non_modal_hosts_are_named(): + prose = _prose_flow() + for host in ("Codex", "Cursor", "Gemini CLI", "raw CLI"): + assert host in prose + + +def test_non_modal_cookie_consent_uses_engine_allow_flag(): + prose = _prose_flow() + consent = prose.index("Cookie consent") + allow = prose.index("setup --allow-browser-cookies") + decline = prose.index("FROM_BROWSER=off") + assert consent < allow + assert consent < decline + + +def test_non_modal_completion_mentions_safe_diagnose_and_project_trust(): + prose = _prose_flow() + assert "safe `--diagnose`" in prose + assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in prose + assert "Codex desktop" in prose diff --git a/tests/test_doc_security_contract.py b/tests/test_doc_security_contract.py new file mode 100644 index 0000000..3bb7ff2 --- /dev/null +++ b/tests/test_doc_security_contract.py @@ -0,0 +1,46 @@ +"""Security-copy contract tests for local reads and credential destinations.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CONFIGURATION = ROOT / "CONFIGURATION.md" +SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" +UI_PY = ROOT / "skills" / "last30days" / "scripts" / "lib" / "ui.py" + + +def test_cookie_setup_requires_explicit_allow_flag_in_docs(): + config = CONFIGURATION.read_text(encoding="utf-8") + skill = SKILL_MD.read_text(encoding="utf-8") + assert "setup --allow-browser-cookies" in config + assert "setup --allow-browser-cookies" in skill + assert "Unset = no browser-cookie reads" in config + + +def test_project_config_trust_is_documented(): + config = CONFIGURATION.read_text(encoding="utf-8") + skill = SKILL_MD.read_text(encoding="utf-8") + assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in config + assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in skill + assert "Folder-mode hosts such as Codex desktop do not trust hidden project config by default" in config + + +def test_codex_auth_not_advertised_as_openai_fallback(): + config = CONFIGURATION.read_text(encoding="utf-8") + assert "Codex ChatGPT auth" in config + assert "intentionally not used" in config + assert "or Codex auth" not in config + + +def test_scrapecreators_copy_uses_canonical_free_call_count(): + text = "\n".join( + [ + CONFIGURATION.read_text(encoding="utf-8"), + SKILL_MD.read_text(encoding="utf-8"), + UI_PY.read_text(encoding="utf-8"), + ] + ) + assert "10,000 free calls" in text + assert "100 free credits" not in text + assert "1,000 free" not in text From 207cfa1e7456cb2aaecd07089b281adb61cf848c Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 15:43:23 -0700 Subject: [PATCH 44/50] test: strengthen host doc contracts --- tests/test_codex_host_contract.py | 8 ++++++-- tests/test_doc_security_contract.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_codex_host_contract.py b/tests/test_codex_host_contract.py index a610721..4f02116 100644 --- a/tests/test_codex_host_contract.py +++ b/tests/test_codex_host_contract.py @@ -10,8 +10,12 @@ SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" def _prose_flow() -> str: text = SKILL_MD.read_text(encoding="utf-8") - start = text.index("### Non-Modal Prose Flow") - end = text.index("### Manual Setup Guide", start) + start_marker = "### Non-Modal Prose Flow" + end_marker = "### Manual Setup Guide" + start = text.find(start_marker) + assert start != -1, f"missing section marker: {start_marker}" + end = text.find(end_marker, start) + assert end != -1, f"missing section marker: {end_marker}" return text[start:end] diff --git a/tests/test_doc_security_contract.py b/tests/test_doc_security_contract.py index 3bb7ff2..02a1f71 100644 --- a/tests/test_doc_security_contract.py +++ b/tests/test_doc_security_contract.py @@ -6,6 +6,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] CONFIGURATION = ROOT / "CONFIGURATION.md" +README = ROOT / "README.md" SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" UI_PY = ROOT / "skills" / "last30days" / "scripts" / "lib" / "ui.py" @@ -37,6 +38,7 @@ def test_scrapecreators_copy_uses_canonical_free_call_count(): text = "\n".join( [ CONFIGURATION.read_text(encoding="utf-8"), + README.read_text(encoding="utf-8"), SKILL_MD.read_text(encoding="utf-8"), UI_PY.read_text(encoding="utf-8"), ] From e3a481df370135257aa6f54c5af5d5cfbb007a78 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 24 Jun 2026 18:29:23 -0700 Subject: [PATCH 45/50] fix: align release artifact upload action pin --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5eafc3e..95b09be 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: subject-path: dist/last30days.skill - name: Upload skill artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: last30days-skill path: dist/last30days.skill @@ -123,7 +123,7 @@ jobs: subject-path: ${{ env.MCPB_OUTPUT }} - name: Upload .mcpb artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }} path: ${{ env.MCPB_OUTPUT }} From ae229e1f3fe648f36b433efe5c45ce8f89adb57c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 01:31:24 +0000 Subject: [PATCH 46/50] chore(deps): bump github.com/mark3labs/mcp-go in /mcp Bumps [github.com/mark3labs/mcp-go](https://github.com/mark3labs/mcp-go) from 0.54.0 to 0.55.0. - [Release notes](https://github.com/mark3labs/mcp-go/releases) - [Commits](https://github.com/mark3labs/mcp-go/compare/v0.54.0...v0.55.0) --- updated-dependencies: - dependency-name: github.com/mark3labs/mcp-go dependency-version: 0.55.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mcp/go.mod | 2 +- mcp/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mcp/go.mod b/mcp/go.mod index 597b5de..9a1857c 100644 --- a/mcp/go.mod +++ b/mcp/go.mod @@ -2,7 +2,7 @@ module github.com/mvanhorn/last30days-skill/mcp go 1.25.5 -require github.com/mark3labs/mcp-go v0.54.0 +require github.com/mark3labs/mcp-go v0.55.0 require ( github.com/google/jsonschema-go v0.4.2 // indirect diff --git a/mcp/go.sum b/mcp/go.sum index bbbc4dd..bbc555a 100644 --- a/mcp/go.sum +++ b/mcp/go.sum @@ -14,8 +14,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mark3labs/mcp-go v0.54.0 h1:PZhQvd+5xrT43cUoiaKn/hDcvLUhcLc1twSEKYPTcTA= -github.com/mark3labs/mcp-go v0.54.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= +github.com/mark3labs/mcp-go v0.55.0 h1:lJfz2aoctiwK+sI991+uIYwmKNIBciI+O7zsyDsa4U8= +github.com/mark3labs/mcp-go v0.55.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= From ba5c8b6aebd6031fc62c1e1f2161606f920a6e0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 01:31:27 +0000 Subject: [PATCH 47/50] chore(deps): bump actions/download-artifact from 4.3.0 to 8.0.1 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.3.0 to 8.0.1. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/d3f86a106a0bac45b974a628896c90dbdf5c8093...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95b09be..3150c0f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -137,7 +137,7 @@ jobs: contents: write steps: - name: Download all artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist merge-multiple: true From bdb4827bc723de2916bd0cde89b87715e96e99ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 01:31:43 +0000 Subject: [PATCH 48/50] chore(deps): bump trufflesecurity/trufflehog from 3.95.5 to 3.95.6 Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.95.5 to 3.95.6. - [Release notes](https://github.com/trufflesecurity/trufflehog/releases) - [Commits](https://github.com/trufflesecurity/trufflehog/compare/d411fff7b8879a62509f3fa98c07f247ac089a51...30d5bb91af1a771378349dbbb0c82129392acf70) --- updated-dependencies: - dependency-name: trufflesecurity/trufflehog dependency-version: 3.95.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 24f1248..fd5c1ad 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -44,7 +44,7 @@ jobs: # verified secrets. Keep output limited to verified findings to avoid noisy # unverified annotations. - name: Run TruffleHog OSS secret scan - uses: trufflesecurity/trufflehog@d411fff7b8879a62509f3fa98c07f247ac089a51 # v3.95.5 + uses: trufflesecurity/trufflehog@30d5bb91af1a771378349dbbb0c82129392acf70 # v3.95.6 with: version: 3.95.5 extra_args: --results=verified From edbb3daa3bd73942aa81ef40455c6a251ea270de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 01:31:49 +0000 Subject: [PATCH 49/50] chore(deps): bump actions/setup-go from 5.6.0 to 6.4.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.6.0 to 6.4.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/40f1582b2485089dde7abd97c1529aa768e1baff...4a3601121dd01d1626a1e23e37211e3254c1c06c) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 6.4.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95b09be..8a86807 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version-file: mcp/go.mod cache: false From db8a2ad5ab79f2714e949991577b6d0fb69efe59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 01:33:02 +0000 Subject: [PATCH 50/50] chore(deps): bump actions/checkout from 6.0.3 to 7.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.3 to 7.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/df4cb1c069e1874edd31b4311f1884172cec0e10...9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- .github/workflows/security.yml | 4 ++-- .github/workflows/validate.yml | 2 +- .github/workflows/zizmor.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95b09be..3d1ff80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: attestations: write steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false @@ -68,7 +68,7 @@ jobs: platform: linux/amd64 steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 1be0bc1..7e1abda 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 24f1248..1c3fe64 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -17,7 +17,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -35,7 +35,7 @@ jobs: contents: read steps: - name: Checkout full history for diff-aware scanning - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9c7eaef..fb2e963 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -15,7 +15,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 3ad091a..462fb61 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -15,7 +15,7 @@ jobs: security-events: write steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false