Compare commits
522 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cae3843828 | |||
| c7460f6114 | |||
| 40b42b7bed | |||
| 0b35e0c2c8 | |||
| ab124fc8f9 | |||
| c7cc880c36 | |||
| acdc397fec | |||
| 4fa89d772d | |||
| 698bfd0c3d | |||
| ddc761d2c2 | |||
| 6835eb8b1e | |||
| 55885bcacc | |||
| e93c8249d8 | |||
| 1004324ad3 | |||
| 9b4684d5f3 | |||
| 475a51e5dd | |||
| c3591ee056 | |||
| ee003df621 | |||
| 47f532fe78 | |||
| 5d21d9e924 | |||
| 9fcbd7b472 | |||
| d82c6a0948 | |||
| 52f53312ff | |||
| a22c8e7576 | |||
| 57f8d2c87a | |||
| b0fdfb0d51 | |||
| a940c98965 | |||
| cb4e93cff1 | |||
| 53fb6104c9 | |||
| 21c54d646c | |||
| f310b82c14 | |||
| ef6c279ceb | |||
| 0188da7ce7 | |||
| 9067158574 | |||
| 5cc8e7cceb | |||
| e82dbd41e1 | |||
| 4b071d2605 | |||
| b2ab9bf45b | |||
| afb27826e9 | |||
| 3d17afb39e | |||
| 0f205e64e1 | |||
| 2e8dcaf746 | |||
| e0011c07e3 | |||
| b92fbd4201 | |||
| 4c186a182c | |||
| 69a2510637 | |||
| 1434895279 | |||
| 0c0c51f1fe | |||
| 0b6109dd77 | |||
| eda1ffe668 | |||
| 0612ff7605 | |||
| a4e7eca516 | |||
| 906d9614f5 | |||
| 48fecf0fa4 | |||
| b6efa6445f | |||
| fcff56b57c | |||
| bb08c50bdd | |||
| 3d33d97a35 | |||
| f60d477d74 | |||
| 94138da0ae | |||
| 1809f08f9d | |||
| cd10487c6c | |||
| 1bc916fc04 | |||
| 6bfc957e55 | |||
| 9ef66ff61c | |||
| a7560a2d2e | |||
| 529b3f3c1b | |||
| b8db4341ed | |||
| 90a49a37bc | |||
| 8c86ffda9e | |||
| 6172f0395e | |||
| 7999421b57 | |||
| 9e833f3ab0 | |||
| 29d1dd6965 | |||
| 748b6eae08 | |||
| 3e354ff7f2 | |||
| cce0f687f0 | |||
| 7103df779b | |||
| a21c463b6f | |||
| 1b559129fc | |||
| 779c486d67 | |||
| b201aa52c2 | |||
| 2ae2718ec9 | |||
| 91d1f9a17c | |||
| 4e06976079 | |||
| 0b4d8b1bbd | |||
| b9e3c21cbf | |||
| 987b9b995a | |||
| 85143d3c4e | |||
| d57b147749 | |||
| 472207164a | |||
| ce489808fa | |||
| 074dc2569e | |||
| d7709a56a0 | |||
| d9d0019af8 | |||
| 1ba7a2bb6a | |||
| 8dbf7219fb | |||
| 279a406e95 | |||
| 666c8eaf45 | |||
| da7bc30a02 | |||
| c0e9fbf80b | |||
| 4c7b87b86d | |||
| bfe1453c66 | |||
| 249dbb6e23 | |||
| 5cd86083c1 | |||
| a50761a1f1 | |||
| 2471ea23ce | |||
| 1758502190 | |||
| e3082d7d48 | |||
| f7767cf3de | |||
| c9cb532cba | |||
| f323ec6e7a | |||
| ec9c8e32a6 | |||
| 5c6e671596 | |||
| 274a92df46 | |||
| cda646a5a0 | |||
| 6f1ae7ce63 | |||
| 612c58f873 | |||
| c9fb132fa0 | |||
| 95aaaf261b | |||
| ce70edb7fa | |||
| e72103ccb0 | |||
| 650d5be709 | |||
| a1ebe21684 | |||
| 6895bde883 | |||
| a7ac3f6d32 | |||
| 9c40c8f5c4 | |||
| 3af5a9fc15 | |||
| 14e6d62cd1 | |||
| 89a3a8d3d6 | |||
| d5ab765b64 | |||
| 01aac1ab20 | |||
| 8a0fcf40fb | |||
| ec6c2e4d2f | |||
| f0b1403a40 | |||
| dbc7621fc8 | |||
| a6712f3cbf | |||
| 573555c76c | |||
| a2ed2f3167 | |||
| f780817a50 | |||
| 5901a0b51a | |||
| 01aef34ca4 | |||
| 51781fe02d | |||
| 5ccda4036b | |||
| 4142665a4f | |||
| 92b664e742 | |||
| 27e17245b6 | |||
| f9a3c919bc | |||
| ca8574da75 | |||
| 7a574f9189 | |||
| 729c2a5a47 | |||
| a7ba39820d | |||
| cfe827bb7b | |||
| 7faae4e19b | |||
| 7ee64fcee3 | |||
| 89e699a35a | |||
| 249c7a4c04 | |||
| 2626d0ff2f | |||
| 50d44bf208 | |||
| 691c21234c | |||
| e71ac74cf0 | |||
| b58c2e7947 | |||
| 4488271f79 | |||
| d5a8f2836a | |||
| 224e84a043 | |||
| 665f9a893b | |||
| 9585ec4b84 | |||
| cb44afeb19 | |||
| 39cca461aa | |||
| 3176b07842 | |||
| 1d927db51d | |||
| b8cfb9d03c | |||
| 5c4fe1ed1d | |||
| 86c5303c1a | |||
| aab5395778 | |||
| ed49713e4d | |||
| de3b692b1c | |||
| 586b7e29f9 | |||
| e4f7e9b175 | |||
| 7650a68638 | |||
| 4e14c9a259 | |||
| d1efed83d4 | |||
| 1d55d8bef1 | |||
| 2f4374ba9f | |||
| 943e18d1c3 | |||
| 4b027919c7 | |||
| af5f0ec4e3 | |||
| d31d5f3b3a | |||
| d1ecc2f062 | |||
| c245847046 | |||
| d3cafd2980 | |||
| 22e095fa0a | |||
| dd69abe669 | |||
| 0a1a5799ae | |||
| c410a2d141 | |||
| f462577eaa | |||
| 9bf0a79a70 | |||
| 4ce7a0ebc9 | |||
| 3b0b021e2c | |||
| 9c020b1393 | |||
| a06e221552 | |||
| 6b62b867b5 | |||
| cb27c689b5 | |||
| 298310ca34 | |||
| ffa2e31e9b | |||
| 0270c12352 | |||
| 7abcb9d108 | |||
| 021bdbf60e | |||
| 06b68a883a | |||
| 91e606a369 | |||
| 24c6567731 | |||
| f1beee7e81 | |||
| fc45949239 | |||
| c74fbd1b0d | |||
| 856da6dc8e | |||
| f53bde5420 | |||
| 5f4e7e73c3 | |||
| 602dbde2db | |||
| 98dd5dce29 | |||
| f5a0a18ec0 | |||
| 6287cf0aeb | |||
| 6340fadf31 | |||
| 961572de22 | |||
| 541717af30 | |||
| ae8c32327f | |||
| c4e70ee40a | |||
| 8f31ed3179 | |||
| ee04fba116 | |||
| 030a2c8fe6 | |||
| 4bbfee4055 | |||
| 750176314a | |||
| 1e82659bb4 | |||
| 16bfbd8ce9 | |||
| a5b3ca1f3c | |||
| 9e00fe5b6d | |||
| f5bbc03aab | |||
| 51424be41d | |||
| 6a1d93bdb5 | |||
| 6318f7449e | |||
| 76215629b9 | |||
| 47b924c87f | |||
| cc620449ce | |||
| 443b05f330 | |||
| 9b9bd42e0f | |||
| f857a5101f | |||
| 784624c002 | |||
| 3e78f45438 | |||
| a0bbac22e6 | |||
| a1ab6fc4b6 | |||
| a28668ad30 | |||
| cea36329af | |||
| 344731b95a | |||
| 6e87b83d78 | |||
| a9d2674147 | |||
| 7d2868b9d1 | |||
| 3f7c8c5022 | |||
| 433ac8d2ed | |||
| 545a8bdc33 | |||
| 732d4ae76b | |||
| ad52872ecf | |||
| 283dbae20c | |||
| 9a550c0ccb | |||
| 62072dacfe | |||
| e0775b9d18 | |||
| 34365e68fe | |||
| d4d603d540 | |||
| d5f3083b82 | |||
| 4bd20f3f4b | |||
| c09e67ce76 | |||
| 5b8b021318 | |||
| a61852b34e | |||
| f168eecd67 | |||
| 65951b3bd7 | |||
| 80cc0aae13 | |||
| d979c3640a | |||
| 97c7636126 | |||
| 287327df60 | |||
| 6db296ca68 | |||
| 1e365e573b | |||
| 8a0dea2884 | |||
| e219c65ea7 | |||
| 0afd78dabc | |||
| 91dba43d1f | |||
| eefeba9537 | |||
| 979e6dece3 | |||
| fcf047c3f3 | |||
| a824fa3c3a | |||
| c966c93653 | |||
| 2c149b73e4 | |||
| 0abd39abaa | |||
| f067f90151 | |||
| 811f044367 | |||
| 7bd11506bd | |||
| d3b0a45a94 | |||
| 7a4a0ee223 | |||
| 3494926ef2 | |||
| e389a0ecc4 | |||
| 39cfd722f8 | |||
| 0e427ab98a | |||
| 9a9d735ef9 | |||
| 25b86de81c | |||
| 945f11d048 | |||
| 0cd5772834 | |||
| e64ed92b10 | |||
| 030dda7399 | |||
| 3e36224927 | |||
| 8bec337c99 | |||
| 42eb5f5273 | |||
| 6e91e8106c | |||
| 51b6762f4b | |||
| b582bffcb3 | |||
| bd7e46fbfd | |||
| 770b91e023 | |||
| 46269c46a5 | |||
| 963a6c8a15 | |||
| fcf428266d | |||
| 824a4a9595 | |||
| eee3359248 | |||
| 50d062b88b | |||
| 0aed01555a | |||
| d9c9edffb8 | |||
| f74169fdaa | |||
| 22a2e1d254 | |||
| 76fb9de86b | |||
| ead5ce5466 | |||
| 3f0d1b410b | |||
| db8a2ad5ab | |||
| f48f6d337e | |||
| 66428a8a8e | |||
| edbb3daa3b | |||
| bdb4827bc7 | |||
| ba5c8b6aeb | |||
| ae229e1f3f | |||
| d1398ef3a1 | |||
| 82bc473e29 | |||
| e3a481df37 | |||
| c65bd764dd | |||
| 50d8101013 | |||
| 5f49ad2b0c | |||
| e12330df3f | |||
| cc5598de84 | |||
| a431340fd2 | |||
| 136b4f90c1 | |||
| 6c9f1ada5c | |||
| 5eea498e7d | |||
| 9b9439d25b | |||
| 55f7aa5850 | |||
| 89a5685953 | |||
| dd5bdbb707 | |||
| 36d6f29684 | |||
| 9961d4396f | |||
| 21c0a9291a | |||
| 207cfa1e74 | |||
| f6edf563b4 | |||
| 834535ec15 | |||
| 472eb88bba | |||
| 91d5887199 | |||
| 0f98304f6e | |||
| 5f86484c0a | |||
| c5ff5505b3 | |||
| 25cfced305 | |||
| 565bb03b63 | |||
| c8ab309f45 | |||
| c3649f30f5 | |||
| ed42d8aa26 | |||
| dab238ea7f | |||
| 56fd24bf14 | |||
| f00162ca89 | |||
| 977f0beed5 | |||
| aecab67c81 | |||
| 4d69ae6a1f | |||
| 9849de1396 | |||
| 6291ac7ec2 | |||
| e65914a9c1 | |||
| fda62bd215 | |||
| 0fa5542dd4 | |||
| 1b832a26bc | |||
| 561e224248 | |||
| 220928f664 | |||
| f092c9f5ba | |||
| 10e19f00fd | |||
| 9400673608 | |||
| f9a016e130 | |||
| a8506fc315 | |||
| 8c2b1c3985 | |||
| 5a74d17559 | |||
| 9124035d7e | |||
| 9f77d3ac31 | |||
| 7f6fd4efd4 | |||
| a8472599b1 | |||
| 8979bc74e4 | |||
| 8fdab71bd6 | |||
| 9450f1f29b | |||
| 64ed8aaf29 | |||
| 22a21e8f2f | |||
| b8122a28d8 | |||
| 03848365dd | |||
| e0102f817d | |||
| e5354f702a | |||
| 7b4ae7afb8 | |||
| e14af82f12 | |||
| ef193c093c | |||
| 8d1c08568d | |||
| b371c0d31f | |||
| 859a63146f | |||
| bed9679699 | |||
| afba3ba528 | |||
| 0a5bdee16d | |||
| 75badd9117 | |||
| 634b95b787 | |||
| 96ab2e276b | |||
| 891797248d | |||
| cffca3aeb8 | |||
| b6e79032c2 | |||
| c4e5d7b50c | |||
| c165fbfab5 | |||
| 87d2be30b1 | |||
| faf8782dd4 | |||
| e603c7b4f0 | |||
| a6a7a2fb65 | |||
| f03d8a184c | |||
| 0381df5025 | |||
| d39d25f8d6 | |||
| 3fa91fce90 | |||
| 214a8fa059 | |||
| 5634bb3126 | |||
| e1f5e4686d | |||
| a1bda56650 | |||
| 73c9a84c88 | |||
| fbc828ad4a | |||
| ad69960683 | |||
| d9380c629b | |||
| 5b168c7382 | |||
| fce934bd31 | |||
| 7e21004236 | |||
| 7a5ded08ec | |||
| ae65f9aad1 | |||
| 6ef466d43f | |||
| 073ce4278d | |||
| c320908ee2 | |||
| 2d2803f62e | |||
| c2d7412411 | |||
| 885fea5ba5 | |||
| 5b5ed6ddd1 | |||
| ff22e1cd18 | |||
| 9eff7be21c | |||
| 91987294cd | |||
| 334c813cef | |||
| 30c84d7878 | |||
| 9fff90b62b | |||
| ccca10bbaf | |||
| 6a07685c3e | |||
| fa0f840a89 | |||
| faecb03b7f | |||
| 87ef4db772 | |||
| be8438459a | |||
| d434947e03 | |||
| d31f26eb17 | |||
| 09c163cec8 | |||
| ef8cc69fdd | |||
| 9d0fb3180b | |||
| 0283cf2d25 | |||
| b7e92d49c0 | |||
| 3360ef3b23 | |||
| 8333a9a1aa | |||
| 6bed7a3dc5 | |||
| 9a59d88b11 | |||
| 402b1eedf4 | |||
| 1cc1b2efa1 | |||
| 8d10d34010 | |||
| d6a4dd0822 | |||
| ffe4d3477d | |||
| 3bad700f0b | |||
| 3e0f5ebe1e | |||
| 29bb548772 | |||
| 3ecfc9126c | |||
| b70f52f426 | |||
| 7eaa4e0b11 | |||
| 07e6b59e62 | |||
| 76a7ceb553 | |||
| 84b21434dd | |||
| 7f95ca7c21 | |||
| a86c8a162b | |||
| 92c104b911 | |||
| b8e9ba92c7 | |||
| b6fa5b13e6 | |||
| 7d90275d57 | |||
| b034c7373b | |||
| a4d837d74e | |||
| 5f499d4eec | |||
| cd15598e62 | |||
| 884e82bd4f | |||
| 3a300a91b6 | |||
| 019be0f853 | |||
| 557eaec9ee | |||
| 46fe7570ad | |||
| 76b9e4b98f | |||
| 1c6cece934 | |||
| 36af50f1b6 | |||
| c5b4a82917 | |||
| 09cd085b47 | |||
| 2655d8f4c8 | |||
| 41f5e480ab | |||
| 82f9340c91 | |||
| 367904f156 | |||
| 7f211acbdb | |||
| 509fd3a553 | |||
| 61d46b54ee | |||
| e7b7e61237 | |||
| a547a0a948 | |||
| 8ea048b988 | |||
| 1b23a3e900 | |||
| 35f12cb9ea | |||
| a1afbce84c | |||
| 15781bfcc0 | |||
| 236c17a3b8 | |||
| b0b9e6ac5b | |||
| 5ede01745c | |||
| 19f8019323 | |||
| b2a762aa24 | |||
| 6b0261ff0e | |||
| 378be7cfd5 |
@@ -1,14 +1,14 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
"displayName": "last30days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
"source": "url",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill.git"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
"description": "Marketplace hosting the last30days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.3.2",
|
||||
"version": "3.21.0",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.3.2",
|
||||
"version": "3.21.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
@@ -10,5 +10,21 @@
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
"keywords": [
|
||||
"competitor research",
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"youtube",
|
||||
"tiktok",
|
||||
"instagram",
|
||||
"trends",
|
||||
"prompts",
|
||||
"polymarket",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"hacker-news"
|
||||
]
|
||||
}
|
||||
|
||||
+65
-12
@@ -1,18 +1,71 @@
|
||||
# Exclude binary assets and dev/test artifacts from ClawHub bundle
|
||||
assets/
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
release-notes.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
SKILL-original.md
|
||||
# ClawHub/Hermes packaging exclusions for repository-root scans.
|
||||
# Mirrors .skillignore so non-runtime docs/dev artifacts stay out of the
|
||||
# public bundle and install-time skill security scan.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
skills/last30days/assets/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
work/
|
||||
print/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
mcp/
|
||||
gemini-extension.json
|
||||
greptile.json
|
||||
pyproject.toml
|
||||
|
||||
# Non-runtime docs, plans, release notes, fixtures, and tests
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
media/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
CONCEPTS.md
|
||||
CONFIGURATION.md
|
||||
CONTRIBUTORS.md
|
||||
HERMES_SETUP.md
|
||||
release-notes.md
|
||||
SKILL-original.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
|
||||
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
|
||||
skills/last30days/scripts/build-skill.sh
|
||||
skills/last30days/scripts/compare.sh
|
||||
skills/last30days/scripts/evaluate_search_quality.py
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
skills/last30days/scripts/setup-pass.sh
|
||||
skills/last30days/scripts/test_device_auth.py
|
||||
skills/last30days/scripts/test-v1-vs-v2.sh
|
||||
skills/last30days/scripts/verify_v3.py
|
||||
|
||||
# Keep visible: optional runtime watchlist/store/briefing feature scripts
|
||||
# (`watchlist.py`, `store.py`, and `briefing.py`).
|
||||
|
||||
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
|
||||
skills/last30days/scripts/lib/vendor/
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.21.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and the web.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"email": "mvanhorn@gmail.com",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"competitor research",
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"youtube",
|
||||
"tiktok",
|
||||
"instagram",
|
||||
"trends",
|
||||
"prompts",
|
||||
"polymarket",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"hacker-news"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "last30days",
|
||||
"shortDescription": "Research what people are saying about a topic now.",
|
||||
"longDescription": "last30days adds a Codex skill for researching any topic based on recent discussion and engagement signals across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and the web.",
|
||||
"developerName": "Matt Van Horn",
|
||||
"category": "Research",
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Read",
|
||||
"Write"
|
||||
],
|
||||
"websiteURL": "https://github.com/mvanhorn/last30days-skill",
|
||||
"defaultPrompt": [
|
||||
"TikTok shop trends",
|
||||
"Codex vs Cursor",
|
||||
"best travel credit cards"
|
||||
],
|
||||
"brandColor": "#6F42C1"
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,6 @@ assets/ export-ignore
|
||||
# Historical + repo-only manifests
|
||||
SPEC.md export-ignore
|
||||
TASKS.md export-ignore
|
||||
test-run.log export-ignore
|
||||
CONTRIBUTORS.md export-ignore
|
||||
HERMES_SETUP.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
|
||||
@@ -2,18 +2,39 @@
|
||||
|
||||
<!-- What does this PR do? 1-3 sentences. -->
|
||||
|
||||
## Changes
|
||||
|
||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
||||
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- How did you verify this works? -->
|
||||
- [ ] `uv run pytest`
|
||||
- [ ] Added or updated tests that would catch a regression, or explained why not below
|
||||
|
||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
||||
## Changelog
|
||||
|
||||
## Related Issues
|
||||
If this change should appear in the next release notes, add a fragment under `changelog.d/` (see `changelog.d/README.md` and [CONTRIBUTING.md](../CONTRIBUTING.md)). Do **not** edit `CHANGELOG.md` or bump version/manifest files in this PR.
|
||||
|
||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
||||
- [ ] Added `changelog.d/<pr-or-issue>.<type>.md` (types: `added`, `changed`, `fixed`, `removed`, `deprecated`, `security`)
|
||||
- [ ] Skip changelog — chore/internal only (also add the `skip-changelog` label)
|
||||
|
||||
## Agent disclosure
|
||||
|
||||
### AI review
|
||||
|
||||
Summarize the review your coding agent ran: main risks checked, what it flagged, and what you changed or verified as a result.
|
||||
|
||||
### Security
|
||||
|
||||
Note any input handling, command execution, path handling, auth, secrets, or dependency risks reviewed, plus follow-up needed. Write `N/A` if none apply.
|
||||
|
||||
## Notes
|
||||
|
||||
Call out follow-up work, host-specific behavior, or risks.
|
||||
|
||||
### Relationship to this change
|
||||
|
||||
Disclose employment, contracting, equity, or other paid ties to a company/product/service this PR adds or meaningfully promotes (example: you work at the API vendor being integrated).
|
||||
|
||||
- [ ] None
|
||||
- [ ] Yes — disclosure: <!-- who / what relationship -->
|
||||
|
||||
## Related issues
|
||||
|
||||
<!-- Fixes #123 / Relates to #456 — or N/A -->
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
This file contains Copilot-specific additions. See AGENTS.md for the shared cross-tool governance layer.
|
||||
|
||||
# Copilot-specific guidance
|
||||
|
||||
## Test generation
|
||||
|
||||
- Prefer unittest.TestCase for generated tests to match the existing test suite.
|
||||
- Mock external calls with unittest.mock.patch.
|
||||
|
||||
## Pull request reminders
|
||||
|
||||
Before suggesting a pull request:
|
||||
|
||||
- Confirm that pytest passes.
|
||||
- For changes that belong in the next release notes, add a `changelog.d/<n>.<type>.md` fragment (do not edit `CHANGELOG.md` or bump version manifests). See `CONTRIBUTING.md` / `AGENTS.md` § Changelog and releases and fill the PR template’s Agent disclosure + Relationship sections.
|
||||
- If changes were made anywhere under skills/last30days/, confirm the install copy has been refreshed with:
|
||||
|
||||
npx skills add . -g -y
|
||||
|
||||
## Vendor exclusion zone
|
||||
|
||||
- Never suggest changes to skills/last30days/scripts/lib/vendor/.
|
||||
- Treat skills/last30days/scripts/lib/vendor/ as a no-touch zone.
|
||||
|
||||
## CI expectations
|
||||
|
||||
GitHub CI runs:
|
||||
|
||||
- pytest
|
||||
- ruff
|
||||
|
||||
Generated changes should pass both before review is requested.
|
||||
|
||||
## CLI examples
|
||||
|
||||
When suggesting CLI usage examples for safe local testing, default to:
|
||||
|
||||
--emit=compact --mock
|
||||
@@ -0,0 +1,22 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: uv
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: gomod
|
||||
directory: /mcp
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
Executable
+221
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a lockstep release: towncrier changelog + bump every version surface.
|
||||
|
||||
Usage (from repo root):
|
||||
python3 .github/scripts/prepare_release.py --bump patch
|
||||
python3 .github/scripts/prepare_release.py --version 3.19.0
|
||||
python3 .github/scripts/prepare_release.py --bump minor --dry-run
|
||||
|
||||
Do not edit CHANGELOG.md or version manifests in feature PRs — add a
|
||||
changelog.d/ fragment instead. This script is for release PRs only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
|
||||
PYPROJECT = ROOT / "pyproject.toml"
|
||||
UV_LOCK = ROOT / "uv.lock"
|
||||
|
||||
JSON_VERSION_FILES = (
|
||||
ROOT / ".claude-plugin" / "plugin.json",
|
||||
ROOT / ".codex-plugin" / "plugin.json",
|
||||
ROOT / ".grok-plugin" / "plugin.json",
|
||||
ROOT / "gemini-extension.json",
|
||||
)
|
||||
|
||||
MARKETPLACE_FILES = (
|
||||
ROOT / ".claude-plugin" / "marketplace.json",
|
||||
ROOT / ".grok-plugin" / "marketplace.json",
|
||||
)
|
||||
|
||||
_VERSION_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$")
|
||||
_PYPROJECT_VERSION_RE = re.compile(
|
||||
r'^(version\s*=\s*")([^"]+)(")\s*$', re.MULTILINE
|
||||
)
|
||||
_SKILL_FRONTMATTER_VERSION_RE = re.compile(
|
||||
r'^(version:\s*")([^"]+)(")\s*$', re.MULTILINE
|
||||
)
|
||||
_SKILL_HEADER_RE = re.compile(
|
||||
r"^(# last30days v)(\d+\.\d+\.\d+)(:)", re.MULTILINE
|
||||
)
|
||||
_UV_LOCK_PACKAGE_RE = re.compile(
|
||||
r'(?ms)^(\[\[package\]\]\nname = "last30days-skill"\nversion = ")([^"]+)(")'
|
||||
)
|
||||
|
||||
|
||||
def _parse_version(text: str) -> tuple[int, int, int]:
|
||||
match = _VERSION_RE.fullmatch(text.strip())
|
||||
if not match:
|
||||
raise SystemExit(f"Invalid semver (expected X.Y.Z): {text!r}")
|
||||
return int(match.group(1)), int(match.group(2)), int(match.group(3))
|
||||
|
||||
|
||||
def _format_version(parts: tuple[int, int, int]) -> str:
|
||||
return f"{parts[0]}.{parts[1]}.{parts[2]}"
|
||||
|
||||
|
||||
def read_current_version() -> str:
|
||||
text = PYPROJECT.read_text(encoding="utf-8")
|
||||
match = _PYPROJECT_VERSION_RE.search(text)
|
||||
if not match:
|
||||
raise SystemExit("Could not find [project].version in pyproject.toml")
|
||||
return match.group(2)
|
||||
|
||||
|
||||
def next_version(current: str, bump: str) -> str:
|
||||
major, minor, patch = _parse_version(current)
|
||||
if bump == "major":
|
||||
return _format_version((major + 1, 0, 0))
|
||||
if bump == "minor":
|
||||
return _format_version((major, minor + 1, 0))
|
||||
if bump == "patch":
|
||||
return _format_version((major, minor, patch + 1))
|
||||
raise SystemExit(f"Unknown bump kind: {bump!r}")
|
||||
|
||||
|
||||
def _replace_once(path: Path, pattern: re.Pattern[str], new: str, label: str) -> None:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
updated, count = pattern.subn(rf"\g<1>{new}\g<3>", text, count=1)
|
||||
if count != 1:
|
||||
raise SystemExit(f"{path.relative_to(ROOT)}: expected one {label} match, found {count}")
|
||||
path.write_text(updated, encoding="utf-8")
|
||||
|
||||
|
||||
def bump_pyproject(version: str) -> None:
|
||||
_replace_once(PYPROJECT, _PYPROJECT_VERSION_RE, version, "version")
|
||||
|
||||
|
||||
def bump_skill_md(version: str) -> None:
|
||||
text = SKILL_MD.read_text(encoding="utf-8")
|
||||
text2, n1 = _SKILL_FRONTMATTER_VERSION_RE.subn(
|
||||
rf"\g<1>{version}\g<3>", text, count=1
|
||||
)
|
||||
text3, n2 = _SKILL_HEADER_RE.subn(rf"\g<1>{version}\g<3>", text2, count=1)
|
||||
if n1 != 1 or n2 != 1:
|
||||
raise SystemExit(
|
||||
f"SKILL.md: expected one frontmatter version and one H1 version, "
|
||||
f"found frontmatter={n1} header={n2}"
|
||||
)
|
||||
SKILL_MD.write_text(text3, encoding="utf-8")
|
||||
|
||||
|
||||
def bump_json_version(path: Path, version: str) -> None:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if "version" not in data:
|
||||
raise SystemExit(f"{path.relative_to(ROOT)}: missing top-level version")
|
||||
data["version"] = version
|
||||
path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def bump_marketplace(path: Path, version: str) -> None:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
plugins = data.get("plugins") or []
|
||||
if not plugins:
|
||||
raise SystemExit(f"{path.relative_to(ROOT)}: plugins[] is empty")
|
||||
plugins[0]["version"] = version
|
||||
path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def bump_uv_lock(version: str) -> None:
|
||||
text = UV_LOCK.read_text(encoding="utf-8")
|
||||
updated, count = _UV_LOCK_PACKAGE_RE.subn(rf"\g<1>{version}\g<3>", text, count=1)
|
||||
if count != 1:
|
||||
raise SystemExit(f"uv.lock: expected one last30days-skill package stanza, found {count}")
|
||||
UV_LOCK.write_text(updated, encoding="utf-8")
|
||||
|
||||
|
||||
def run_towncrier(version: str, *, dry_run: bool) -> None:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"towncrier",
|
||||
"build",
|
||||
"--version",
|
||||
version,
|
||||
"--yes",
|
||||
]
|
||||
if dry_run:
|
||||
cmd.append("--draft")
|
||||
subprocess.run(cmd, cwd=ROOT, check=True)
|
||||
|
||||
|
||||
def bump_all(version: str) -> list[str]:
|
||||
touched: list[str] = []
|
||||
bump_pyproject(version)
|
||||
touched.append(str(PYPROJECT.relative_to(ROOT)))
|
||||
bump_skill_md(version)
|
||||
touched.append(str(SKILL_MD.relative_to(ROOT)))
|
||||
for path in JSON_VERSION_FILES:
|
||||
bump_json_version(path, version)
|
||||
touched.append(str(path.relative_to(ROOT)))
|
||||
for path in MARKETPLACE_FILES:
|
||||
bump_marketplace(path, version)
|
||||
touched.append(str(path.relative_to(ROOT)))
|
||||
bump_uv_lock(version)
|
||||
touched.append(str(UV_LOCK.relative_to(ROOT)))
|
||||
return touched
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--bump", choices=("major", "minor", "patch"))
|
||||
group.add_argument("--version", help="Explicit X.Y.Z to set")
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print the planned version and towncrier draft; do not write files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-towncrier",
|
||||
action="store_true",
|
||||
help="Only bump version surfaces (changelog already prepared)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
current = read_current_version()
|
||||
version = args.version or next_version(current, args.bump)
|
||||
_parse_version(version)
|
||||
if args.version:
|
||||
parsed_new = _parse_version(version)
|
||||
parsed_cur = _parse_version(current)
|
||||
if parsed_new < parsed_cur:
|
||||
raise SystemExit(f"Refusing to downgrade {current} → {version}")
|
||||
if parsed_new == parsed_cur and not args.dry_run:
|
||||
raise SystemExit(
|
||||
f"Refusing to re-release {current}; pass --bump or a newer --version "
|
||||
"(use --dry-run to preview towncrier output for the current version)"
|
||||
)
|
||||
|
||||
print(f"Current version: {current}")
|
||||
print(f"Next version: {version}")
|
||||
|
||||
if args.dry_run:
|
||||
if not args.skip_towncrier:
|
||||
run_towncrier(version, dry_run=True)
|
||||
print("Dry run only — no files written.")
|
||||
return 0
|
||||
|
||||
if not args.skip_towncrier:
|
||||
run_towncrier(version, dry_run=False)
|
||||
print("Updated CHANGELOG.md via towncrier")
|
||||
|
||||
touched = bump_all(version)
|
||||
print("Bumped lockstep files:")
|
||||
for path in touched:
|
||||
print(f" - {path}")
|
||||
print(f"\nNext: open a release PR, merge, then tag v{version} (tag-release workflow).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Print the lockstep version string from a manifest file on stdin.
|
||||
|
||||
Used by .github/workflows/changelog-guard.yml so version parsing stays out of
|
||||
the YAML ``run: |`` block (column-0 Python inside that block breaks Actions).
|
||||
|
||||
Usage:
|
||||
git show REF:path | python3 .github/scripts/read_manifest_version.py path
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def version_from(path: str, text: str) -> str:
|
||||
if path.endswith("pyproject.toml"):
|
||||
match = re.search(r'(?m)^version\s*=\s*"([^"]+)"\s*$', text)
|
||||
return match.group(1) if match else ""
|
||||
if path.endswith("SKILL.md"):
|
||||
match = re.search(r'(?m)^version:\s*"([^"]+)"\s*$', text)
|
||||
return match.group(1) if match else ""
|
||||
if path.endswith("uv.lock"):
|
||||
match = re.search(
|
||||
r'(?ms)^\[\[package\]\]\nname = "last30days-skill"\nversion = "([^"]+)"',
|
||||
text,
|
||||
)
|
||||
return match.group(1) if match else ""
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise SystemExit(f"invalid JSON in {path}: {exc}") from exc
|
||||
if path.endswith("marketplace.json"):
|
||||
plugins = data.get("plugins") or []
|
||||
return plugins[0].get("version", "") if plugins else ""
|
||||
return data.get("version", "") or ""
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 2:
|
||||
print(
|
||||
"usage: read_manifest_version.py PATH < manifest",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
path = argv[1]
|
||||
text = sys.stdin.read()
|
||||
sys.stdout.write(version_from(path, text))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -0,0 +1,153 @@
|
||||
name: Changelog guard
|
||||
|
||||
# Non-release PRs must not edit CHANGELOG.md or bump lockstep version strings.
|
||||
# Content edits to SKILL.md / pyproject.toml / uv.lock are fine.
|
||||
# Release PRs (label: release) are exempt. Engine changes need a changelog
|
||||
# fragment unless labeled skip-changelog.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, labeled, unlabeled]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
guard:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Enforce changelog / version lockstep rules
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
LABELS="$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/labels" --jq '.[].name')"
|
||||
IS_RELEASE=0
|
||||
SKIP_CHANGELOG=0
|
||||
if printf '%s\n' "${LABELS}" | grep -qx 'release'; then
|
||||
IS_RELEASE=1
|
||||
fi
|
||||
if printf '%s\n' "${LABELS}" | grep -qx 'skip-changelog'; then
|
||||
SKIP_CHANGELOG=1
|
||||
fi
|
||||
|
||||
mapfile -t CHANGED < <(git diff --name-only "${BASE_SHA}...${HEAD_SHA}")
|
||||
|
||||
changed_changelog=0
|
||||
for path in "${CHANGED[@]}"; do
|
||||
if [ "${path}" = "CHANGELOG.md" ]; then
|
||||
changed_changelog=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${IS_RELEASE}" -eq 1 ]; then
|
||||
echo "PR has label 'release' — version/CHANGELOG edits allowed."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${changed_changelog}" -eq 1 ]; then
|
||||
# One-time towncrier adoption: replacing ## [Unreleased] with the
|
||||
# start marker / notice is allowed. Adding release-note bullets
|
||||
# (+### sections) is not.
|
||||
cl_diff="$(git diff "${BASE_SHA}...${HEAD_SHA}" -- CHANGELOG.md || true)"
|
||||
if printf '%s\n' "${cl_diff}" | grep -q 'towncrier release notes start' \
|
||||
&& ! printf '%s\n' "${cl_diff}" | grep -qE '^\+### '; then
|
||||
echo "Allowing towncrier bootstrap CHANGELOG.md header change."
|
||||
changed_changelog=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${changed_changelog}" -eq 1 ]; then
|
||||
echo "::error::Do not edit CHANGELOG.md in feature PRs."
|
||||
echo "Add changelog.d/<n>.<type>.md instead (see changelog.d/README.md)."
|
||||
echo "Release PRs created via Actions → Prepare release use the 'release' label."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Keep version parsing in .github/scripts/ — a prior inline
|
||||
# python3 -c block used column-0 source and made Actions refuse
|
||||
# to parse this workflow (every run failed with empty jobs).
|
||||
version_at() {
|
||||
local ref="$1"
|
||||
local path="$2"
|
||||
# Fail closed: do not swallow helper/parse errors with || true.
|
||||
# Callers only invoke this after git cat-file confirms the blob.
|
||||
git show "${ref}:${path}" \
|
||||
| python3 .github/scripts/read_manifest_version.py "${path}"
|
||||
}
|
||||
|
||||
VERSION_PATHS=(
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
skills/last30days/SKILL.md
|
||||
.claude-plugin/plugin.json
|
||||
.claude-plugin/marketplace.json
|
||||
.codex-plugin/plugin.json
|
||||
.grok-plugin/plugin.json
|
||||
.grok-plugin/marketplace.json
|
||||
gemini-extension.json
|
||||
)
|
||||
|
||||
bumps=()
|
||||
for path in "${VERSION_PATHS[@]}"; do
|
||||
# Only compare when the file exists on both sides.
|
||||
if ! git cat-file -e "${BASE_SHA}:${path}" 2>/dev/null; then
|
||||
continue
|
||||
fi
|
||||
if ! git cat-file -e "${HEAD_SHA}:${path}" 2>/dev/null; then
|
||||
continue
|
||||
fi
|
||||
base_v="$(version_at "${BASE_SHA}" "${path}")"
|
||||
head_v="$(version_at "${HEAD_SHA}" "${path}")"
|
||||
if [ -n "${base_v}" ] && [ -n "${head_v}" ] && [ "${base_v}" != "${head_v}" ]; then
|
||||
bumps+=("${path}: ${base_v} → ${head_v}")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${#bumps[@]}" -gt 0 ]; then
|
||||
echo "::error::Non-release PRs must not bump lockstep version strings."
|
||||
printf ' - %s\n' "${bumps[@]}"
|
||||
echo "Run Actions → Prepare release to cut a version bump PR."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
has_fragment=0
|
||||
for path in "${CHANGED[@]}"; do
|
||||
case "${path}" in
|
||||
changelog.d/*.md)
|
||||
base="$(basename "${path}")"
|
||||
if [ "${base}" != "README.md" ]; then
|
||||
has_fragment=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
touches_engine=0
|
||||
for path in "${CHANGED[@]}"; do
|
||||
case "${path}" in
|
||||
skills/last30days/scripts/*|skills/last30days/SKILL.md|mcp/*)
|
||||
touches_engine=1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${touches_engine}" -eq 1 ] && [ "${has_fragment}" -eq 0 ] && [ "${SKIP_CHANGELOG}" -eq 0 ]; then
|
||||
echo "::error::Engine/skill changes need a changelog.d fragment (or the skip-changelog label)."
|
||||
echo "See changelog.d/README.md"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Changelog guard passed."
|
||||
@@ -0,0 +1,31 @@
|
||||
name: OSV-Scanner
|
||||
|
||||
# Scheduled OSV-Scanner workflow for vulnerability drift detection.
|
||||
# Scans the repository lockfiles (uv.lock, mcp/go.sum) on a weekly schedule
|
||||
# and uploads results to GitHub code scanning, so newly disclosed CVEs in
|
||||
# the dependency tree are visible even between PRs.
|
||||
#
|
||||
# Advisory-first: fail-on-vuln is false until maintainers confirm a clean
|
||||
# baseline, matching the pattern in security.yml.
|
||||
#
|
||||
# Separate from the pip-audit job in security.yml (which runs on every PR
|
||||
# and push) and from the dependency-review gate (which blocks on new
|
||||
# vulnerable deps at PR time). This workflow fills the scheduled-drift gap.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Weekly, Mondays at 12:30 UTC.
|
||||
- cron: "30 12 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
scan-scheduled:
|
||||
uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
with:
|
||||
# Advisory-first: surface results in code scanning without blocking.
|
||||
fail-on-vuln: false
|
||||
@@ -0,0 +1,130 @@
|
||||
name: Prepare release
|
||||
|
||||
# Opens a chore(release) PR that runs towncrier + lockstep version bumps.
|
||||
# Merge of that PR is tagged by tag-release.yml; tag push / dispatch runs release.yml.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
bump:
|
||||
description: Semver bump kind (ignored when version is set)
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- patch
|
||||
- minor
|
||||
- major
|
||||
default: patch
|
||||
version:
|
||||
description: Optional explicit X.Y.Z (overrides bump)
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Install project (towncrier)
|
||||
run: uv sync --group dev
|
||||
|
||||
- name: Prepare release files
|
||||
id: prep
|
||||
env:
|
||||
BUMP: ${{ inputs.bump }}
|
||||
EXPLICIT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${EXPLICIT_VERSION}" ]; then
|
||||
uv run python .github/scripts/prepare_release.py --version "${EXPLICIT_VERSION}"
|
||||
VERSION="${EXPLICIT_VERSION}"
|
||||
else
|
||||
uv run python .github/scripts/prepare_release.py --bump "${BUMP}"
|
||||
VERSION="$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")"
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "branch=release/v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create release branch and PR
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ steps.prep.outputs.version }}
|
||||
BRANCH: ${{ steps.prep.outputs.branch }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git"
|
||||
|
||||
if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then
|
||||
echo "Branch ${BRANCH} already exists on origin — aborting to avoid clobbering."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git switch -c "${BRANCH}"
|
||||
git add \
|
||||
CHANGELOG.md \
|
||||
changelog.d \
|
||||
pyproject.toml \
|
||||
uv.lock \
|
||||
skills/last30days/SKILL.md \
|
||||
.claude-plugin/plugin.json \
|
||||
.claude-plugin/marketplace.json \
|
||||
.codex-plugin/plugin.json \
|
||||
.grok-plugin/plugin.json \
|
||||
.grok-plugin/marketplace.json \
|
||||
gemini-extension.json
|
||||
git status --short
|
||||
if git diff --cached --quiet; then
|
||||
echo "No release changes staged (empty changelog.d?)."
|
||||
exit 1
|
||||
fi
|
||||
git commit -m "chore(release): bump version to ${VERSION}"
|
||||
git push -u origin HEAD
|
||||
|
||||
gh label create release --description "Automated version lockstep release PR" --color 0E8A16 2>/dev/null || true
|
||||
gh label create skip-changelog --description "PR has nothing for release notes" --color BFDADC 2>/dev/null || true
|
||||
|
||||
BODY="$(cat <<EOF
|
||||
## Summary
|
||||
|
||||
Automated release preparation for **v${VERSION}**.
|
||||
|
||||
- Built \`CHANGELOG.md\` from \`changelog.d/\` via towncrier
|
||||
- Bumped every lockstep version surface (skill, pyproject, plugin/marketplace manifests, uv.lock)
|
||||
|
||||
## Test plan
|
||||
|
||||
- [ ] \`uv run pytest\` (CI)
|
||||
- [ ] Confirm \`tests/test_plugin_contract.py::test_versions_match_across_manifests\` passes
|
||||
- [ ] After merge, confirm tag \`v${VERSION}\` is created and [Release](../actions/workflows/release.yml) attaches artifacts
|
||||
|
||||
EOF
|
||||
)"
|
||||
# Strip leading spaces from heredoc indentation for readable PR body
|
||||
BODY="$(printf '%s\n' "${BODY}" | sed 's/^ //')"
|
||||
|
||||
gh pr create \
|
||||
--title "chore(release): bump version to ${VERSION}" \
|
||||
--body "${BODY}" \
|
||||
--label "release" \
|
||||
--base "${DEFAULT_BRANCH}" \
|
||||
--head "${BRANCH}"
|
||||
+155
-10
@@ -4,28 +4,173 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
# tag-release.yml dispatches this because GITHUB_TOKEN tag pushes do not
|
||||
# start other workflows.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: Tag to release (e.g. v3.18.2)
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
permissions: {}
|
||||
|
||||
env:
|
||||
# Tag push uses ref_name (vX.Y.Z); dispatch from tag-release passes inputs.tag.
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
build-and-release:
|
||||
# Build the existing .skill artifact (Claude Code / Codex / Cursor install
|
||||
# surface). Unchanged from prior versions; just isolated into its own job
|
||||
# so the .mcpb matrix can run in parallel.
|
||||
build-skill:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ env.RELEASE_TAG }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
- name: Attest .skill artifact provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
files: dist/last30days.skill
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
subject-path: dist/last30days.skill
|
||||
|
||||
- name: Upload skill artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: last30days-skill
|
||||
path: dist/last30days.skill
|
||||
|
||||
# Cross-compile the Go MCP server for each Claude Desktop platform and
|
||||
# package each as a .mcpb. MCPB v0.3 is a ZIP containing the checked-in
|
||||
# manifest and the pre-built binary at the manifest's entry point.
|
||||
build-mcpb:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
env:
|
||||
MCPB_OUTPUT: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- goos: darwin
|
||||
goarch: arm64
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ env.RELEASE_TAG }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
# Build the MCP binary with a toolchain newer than the floor in
|
||||
# mcp/go.mod. Keeping this explicit also avoids a runtime toolchain
|
||||
# download during the cross-compile.
|
||||
go-version: "1.26"
|
||||
cache: false
|
||||
|
||||
- name: Sync engine into vendored/
|
||||
run: bash mcp/scripts/sync-engine.sh
|
||||
|
||||
- name: Build MCP binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: "0"
|
||||
RELEASE_VERSION: ${{ env.RELEASE_TAG }}
|
||||
run: |
|
||||
mkdir -p mcp/build
|
||||
go -C mcp build \
|
||||
-ldflags "-X main.Version=${RELEASE_VERSION}" \
|
||||
-o build/last30days-pp-mcp \
|
||||
./cmd/last30days-pp-mcp
|
||||
|
||||
- name: Bundle .mcpb
|
||||
# Keep this equivalent to printing-press's bundle layout without
|
||||
# downloading a separate packager: manifest.json at the ZIP root and
|
||||
# the executable at its declared server.entry_point.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
entry_point="$(jq -er '.server.entry_point' mcp/manifest.json)"
|
||||
test "${entry_point}" = "bin/last30days-pp-mcp"
|
||||
|
||||
staging="${RUNNER_TEMP}/last30days-mcpb"
|
||||
output="${GITHUB_WORKSPACE}/${MCPB_OUTPUT}"
|
||||
mkdir -p "${staging}/bin" "$(dirname "${output}")"
|
||||
cp mcp/manifest.json "${staging}/manifest.json"
|
||||
cp mcp/build/last30days-pp-mcp "${staging}/${entry_point}"
|
||||
chmod 0755 "${staging}/${entry_point}"
|
||||
|
||||
(
|
||||
cd "${staging}"
|
||||
zip -q -X "${output}" manifest.json "${entry_point}"
|
||||
)
|
||||
unzip -Z1 "${output}" | grep -Fxq "manifest.json"
|
||||
unzip -Z1 "${output}" | grep -Fxq "${entry_point}"
|
||||
|
||||
- name: Attest .mcpb artifact provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: ${{ env.MCPB_OUTPUT }}
|
||||
|
||||
- name: Upload .mcpb artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: ${{ env.MCPB_OUTPUT }}
|
||||
|
||||
# Gather every platform artifact and attach to one GitHub release.
|
||||
release:
|
||||
needs: [build-skill, build-mcpb]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
# gh release create --verify-tag shells out to git, so the job needs a
|
||||
# checkout with the tag present; without it the step fails with
|
||||
# "fatal: not a git repository".
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ env.RELEASE_TAG }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: Create GitHub release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ env.RELEASE_TAG }}
|
||||
run: |
|
||||
gh release create "${RELEASE_TAG}" \
|
||||
dist/last30days.skill \
|
||||
dist/last30days-pp-mcp-*.mcpb \
|
||||
--generate-notes \
|
||||
--verify-tag
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
name: Scorecard
|
||||
|
||||
# OpenSSF Scorecard tracks broader repo security-health drift (branch
|
||||
# protection, token permissions, pinned actions, dangerous workflows, CI
|
||||
# tests, maintenance signals) on a schedule, complementing the per-diff
|
||||
# dependency-audit and secret-scan jobs in security.yml.
|
||||
#
|
||||
# Advisory-first: this workflow only measures and publishes a score, and it
|
||||
# never blocks merges. It runs on the default branch (Scorecard needs repo-level
|
||||
# data and a token, so it is not meaningful on PR forks) plus a weekly schedule
|
||||
# so regressions in security health surface even when no code changes.
|
||||
|
||||
on:
|
||||
branch_protection_rule:
|
||||
schedule:
|
||||
# Weekly, Mondays at 07:00 UTC.
|
||||
- cron: '0 7 * * 1'
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
# Top-level token is read-only; the analysis job widens only what it needs.
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
analysis:
|
||||
name: Scorecard analysis
|
||||
runs-on: ubuntu-latest
|
||||
# Job-level permissions fully replace the top-level block (unlisted scopes
|
||||
# default to none), so the reads checkout and Scorecard need are explicit.
|
||||
permissions:
|
||||
# Needed by actions/checkout to clone the repo, and by Scorecard to read
|
||||
# workflow files for its Dangerous-Workflow / Token-Permissions checks.
|
||||
contents: read
|
||||
actions: read
|
||||
# Needed to upload the SARIF results to the code-scanning dashboard.
|
||||
security-events: write
|
||||
# Needed to publish results and obtain a badge (uses OIDC, no secrets).
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run OpenSSF Scorecard
|
||||
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
||||
with:
|
||||
results_file: scorecard.sarif
|
||||
results_format: sarif
|
||||
# Publishes results to the OpenSSF REST API for the public badge and
|
||||
# trend tracking. Set to false if maintainers prefer to keep the
|
||||
# score private (the SARIF upload below still works either way).
|
||||
publish_results: true
|
||||
|
||||
# Retain the raw SARIF as a build artifact for offline inspection.
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: scorecard-sarif
|
||||
path: scorecard.sarif
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload SARIF to code-scanning
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
with:
|
||||
sarif_file: scorecard.sarif
|
||||
@@ -7,61 +7,83 @@ on:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
# Block known vulnerabilities in the locked Python dependency graph.
|
||||
- name: Run uv audit against locked dependencies
|
||||
run: uv audit --locked
|
||||
|
||||
- name: Export locked dependency set
|
||||
run: |
|
||||
uv export \
|
||||
--locked \
|
||||
--all-groups \
|
||||
--no-hashes \
|
||||
--format requirements.txt \
|
||||
--output-file /tmp/last30days-requirements.txt
|
||||
dependency-review:
|
||||
name: Dependency review
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
||||
# visible in CI logs even before the project has a clean blocking baseline.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
- name: Run pip-audit against locked dependencies
|
||||
continue-on-error: true
|
||||
run: uvx --python 3.12 pip-audit -r /tmp/last30days-requirements.txt --progress-spinner=off
|
||||
- name: Review dependency changes
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
|
||||
|
||||
secret-scan:
|
||||
name: Secret scan
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
||||
# The TruffleHog action automatically scans the PR range for pull_request
|
||||
# events and the pushed commit range for push events.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
||||
# The action derives the commit range from the GitHub event and fails on
|
||||
# verified secrets. Keep output limited to verified findings to avoid noisy
|
||||
# unverified annotations.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
uses: trufflesecurity/trufflehog@v3.95.2
|
||||
continue-on-error: true
|
||||
uses: trufflesecurity/trufflehog@6f3c981e7b77f235fd2702dd74af25fc4b72bf11 # v3.96.0
|
||||
with:
|
||||
path: ./
|
||||
version: v3.95.2
|
||||
extra_args: --only-verified
|
||||
version: 3.95.5
|
||||
extra_args: --results=verified
|
||||
|
||||
sast-scan:
|
||||
name: SAST scan
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
container:
|
||||
image: semgrep/semgrep@sha256:06938c1f365d3f67b8cedd8bc117607ae64253f88a0e768e9da9408548927dd6 # v1.167.0
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Advisory-first: visibility before enforcement. Semgrep CE scans
|
||||
# the repository with the community rule set (Python, shell, YAML,
|
||||
# JavaScript, Go) to catch source-level security bugs before they
|
||||
# reach production. Set continue-on-error: false once a clean baseline
|
||||
# is confirmed.
|
||||
- name: Run Semgrep SAST scan
|
||||
continue-on-error: true
|
||||
env:
|
||||
SEMGREP_SEND_METRICS: off
|
||||
run: semgrep scan --config=auto
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
name: Tag release
|
||||
|
||||
# After a prepare-release PR merges to main, create the vX.Y.Z tag and
|
||||
# dispatch release.yml (GITHUB_TOKEN tag pushes do not start other workflows).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
tag:
|
||||
runs-on: ubuntu-latest
|
||||
# Only act on the release-prep commit shape produced by prepare-release.yml
|
||||
# (or an equivalent manual chore(release) commit). Quote the expression —
|
||||
# a bare `chore(release):` colon is invalid YAML and fails the whole
|
||||
# workflow before any job runs. Use contains (not startsWith) so merge
|
||||
# commits whose subject is "Merge pull request #N …" still match when the
|
||||
# PR title is in the body.
|
||||
if: "contains(github.event.head_commit.message, 'chore(release): bump version to ')"
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Create annotated tag and dispatch Release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
HEAD_MSG: ${{ github.event.head_commit.message }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scan every line — merge commits put the chore(release) title in the body.
|
||||
VERSION="$(printf '%s\n' "${HEAD_MSG}" | sed -n 's/^chore(release): bump version to \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -n1)"
|
||||
if [ -z "${VERSION}" ]; then
|
||||
echo "Could not parse version from commit message: ${HEAD_MSG}"
|
||||
exit 1
|
||||
fi
|
||||
TAG="v${VERSION}"
|
||||
|
||||
PY_VERSION="$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")"
|
||||
if [ "${PY_VERSION}" != "${VERSION}" ]; then
|
||||
echo "Commit message version (${VERSION}) does not match pyproject.toml (${PY_VERSION})"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Require the merged PR to carry the repository-controlled `release`
|
||||
# label so a matching title alone cannot mint a tag.
|
||||
PR_NUMBER="$(gh api "repos/${REPOSITORY}/commits/${HEAD_SHA}/pulls" \
|
||||
--jq 'map(select(.base.ref == env.DEFAULT_BRANCH)) | .[0].number // empty')"
|
||||
if [ -z "${PR_NUMBER}" ]; then
|
||||
echo "No PR found for ${HEAD_SHA} into ${DEFAULT_BRANCH} — refusing to tag."
|
||||
exit 1
|
||||
fi
|
||||
if ! gh api "repos/${REPOSITORY}/issues/${PR_NUMBER}/labels" \
|
||||
--jq '.[].name' | grep -qx 'release'; then
|
||||
echo "PR #${PR_NUMBER} lacks the 'release' label — refusing to tag."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then
|
||||
echo "Tag ${TAG} already exists locally — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then
|
||||
echo "Tag ${TAG} already exists on origin — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${REPOSITORY}.git"
|
||||
git tag -a "${TAG}" -m "Release ${TAG}"
|
||||
git push origin "refs/tags/${TAG}"
|
||||
echo "Created and pushed ${TAG}"
|
||||
|
||||
# GITHUB_TOKEN tag pushes do not trigger release.yml; dispatch it.
|
||||
gh workflow run release.yml \
|
||||
--ref "${DEFAULT_BRANCH}" \
|
||||
-f "tag=${TAG}"
|
||||
echo "Dispatched release.yml for ${TAG}"
|
||||
@@ -6,21 +6,63 @@ on:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run test suite
|
||||
run: uv run pytest
|
||||
run: uv run pytest --cov --cov-report=term-missing
|
||||
|
||||
eval:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Score research quality
|
||||
run: uv run pytest tests/eval -x -s
|
||||
|
||||
mcp-tests:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version: "1.25.5"
|
||||
cache-dependency-path: mcp/go.sum
|
||||
|
||||
- name: Run MCP Go tests
|
||||
run: go test -race ./...
|
||||
working-directory: mcp
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: GitHub Actions Security Analysis with zizmor 🌈
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
zizmor:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor 🌈
|
||||
uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1
|
||||
+17
@@ -1,5 +1,6 @@
|
||||
# Private benchmark / evaluation artifacts — never push to upstream
|
||||
docs/comparison-results/
|
||||
test-run.log
|
||||
scripts/evaluate-synthesis.py
|
||||
scripts/generate-synthesis-inputs.py
|
||||
fixtures/polymarket_sample.json
|
||||
@@ -7,6 +8,7 @@ docs/v2.1-tweets.md
|
||||
docs/30-day-anniversary-thread.md
|
||||
docs/30-day-anniversary-tweets.md
|
||||
variants/open/references/research.md
|
||||
docs/investigations/
|
||||
|
||||
# OS / tool files
|
||||
.DS_Store
|
||||
@@ -20,14 +22,29 @@ mise.toml
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Local secrets/config. Keep tracked examples if added later.
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Root vendor/ is accidental - real vendored client lives at scripts/lib/vendor/bird-search/
|
||||
/vendor/
|
||||
|
||||
# build artifact from scripts/build-skill.sh
|
||||
/dist/
|
||||
|
||||
# Go MCP bundle build outputs - source of truth for vendored/ stays under
|
||||
# skills/last30days/scripts/; build/ holds cross-compiled binaries + .mcpb files.
|
||||
# vendored/ lives inside the engine package because //go:embed cannot reach
|
||||
# outside its own package directory; the .gitkeep anchor stays tracked so
|
||||
# the embed pattern always finds a match even before sync-engine runs.
|
||||
/mcp/internal/engine/vendored/*
|
||||
!/mcp/internal/engine/vendored/.gitkeep
|
||||
/mcp/build/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
docs/brainstorms/
|
||||
.context/
|
||||
|
||||
/work
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"owner": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"description": "Marketplace for the last30days research plugin",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.21.0",
|
||||
"category": "productivity",
|
||||
"source": {
|
||||
"source": "url",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill.git"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"keywords": [
|
||||
"last30days",
|
||||
"last 30 days"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.21.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"email": "mvanhorn@gmail.com",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"last30days",
|
||||
"last 30 days"
|
||||
],
|
||||
"skills": "./skills/"
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
# Hermes install-time scanner/package exclusions for repository-root scans.
|
||||
# Keep the public bundle focused on the runtime skill under skills/last30days/.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
skills/last30days/assets/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
work/
|
||||
print/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
mcp/
|
||||
gemini-extension.json
|
||||
greptile.json
|
||||
pyproject.toml
|
||||
|
||||
# Non-runtime docs, plans, release notes, fixtures, and tests
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
media/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
CONCEPTS.md
|
||||
CONFIGURATION.md
|
||||
CONTRIBUTORS.md
|
||||
HERMES_SETUP.md
|
||||
release-notes.md
|
||||
SKILL-original.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
|
||||
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
|
||||
skills/last30days/scripts/build-skill.sh
|
||||
skills/last30days/scripts/compare.sh
|
||||
skills/last30days/scripts/evaluate_search_quality.py
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
skills/last30days/scripts/setup-pass.sh
|
||||
skills/last30days/scripts/test_device_auth.py
|
||||
skills/last30days/scripts/test-v1-vs-v2.sh
|
||||
skills/last30days/scripts/verify_v3.py
|
||||
|
||||
# Keep visible: optional runtime watchlist/store/briefing feature scripts
|
||||
# (`watchlist.py`, `store.py`, and `briefing.py`).
|
||||
|
||||
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
|
||||
skills/last30days/scripts/lib/vendor/
|
||||
@@ -1,6 +1,6 @@
|
||||
# last30days Skill
|
||||
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, Grok (xAI), and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition / runtime spec the model reads when the slash command fires
|
||||
@@ -10,7 +10,10 @@ Agent Skills package for researching any topic across Reddit, X, YouTube, and we
|
||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||
- `CONFIGURATION.md` — user-facing knobs (env vars, flags, per-host install patterns); keep in sync per the rules below
|
||||
- `CHANGELOG.md` — structured release history (launch copy lives in GitHub Releases)
|
||||
- `CHANGELOG.md` — structured release history built by towncrier at release time (launch copy lives in GitHub Releases)
|
||||
- `changelog.d/` — per-PR news fragments; feature PRs write here, never edit `CHANGELOG.md` directly
|
||||
- `CONTRIBUTING.md` — setup, fragments, and release notes for humans and agents (towncrier is release-only)
|
||||
- `.github/scripts/prepare_release.py` — lockstep version bump + towncrier build (release PRs only)
|
||||
- `HERMES_SETUP.md` — install instructions for the Hermes harness specifically
|
||||
|
||||
## Orientation
|
||||
@@ -21,7 +24,9 @@ Agent Skills package for researching any topic across Reddit, X, YouTube, and we
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only).
|
||||
# Saves to $LAST30DAYS_MEMORY_DIR when set in shell or ~/.config/last30days/.env;
|
||||
# add --save-dir <path> for a one-off override. Mirrors LAST30DAYS_STORE convention.
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # copies skill into ~/.agents/skills/<name>/ (frozen at install time); re-run to sync working-tree edits — see Rules below
|
||||
|
||||
@@ -30,14 +35,32 @@ uv run pytest # full suite
|
||||
uv run pytest tests/test_dedupe_v3.py # single file
|
||||
uv run pytest tests/test_dedupe_v3.py -k some_case # single case
|
||||
uv run pytest --cov # with coverage (skips lib/vendor/)
|
||||
|
||||
# Release prep (maintainers / release automation — not feature PRs):
|
||||
# Prefer GitHub Actions → "Prepare release". Local equivalent:
|
||||
uv run python .github/scripts/prepare_release.py --bump patch # or --version X.Y.Z
|
||||
```
|
||||
|
||||
Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
|
||||
|
||||
## Changelog and releases (agents)
|
||||
|
||||
Agents open most PRs. Follow this so `CHANGELOG.md` stops conflicting and versions stay lockstep:
|
||||
|
||||
1. **Feature/fix PRs:** add `changelog.d/<pr-or-issue>.<type>.md` (`added` / `changed` / `fixed` / `removed` / `deprecated` / `security`) when the change belongs in the next release notes. See `changelog.d/README.md` and `CONTRIBUTING.md`. Fill the PR template’s Summary, Agent disclosure, and Relationship sections.
|
||||
2. **Never** edit `CHANGELOG.md` in a feature PR. **Never** bump version strings in `pyproject.toml`, `SKILL.md`, plugin/marketplace JSON, or `uv.lock` outside a release PR. CI (`changelog-guard.yml`) enforces this.
|
||||
3. **Nothing for release notes:** omit the fragment, check Skip changelog in the template, and add the `skip-changelog` label.
|
||||
4. **Cutting a release:** run Actions → **Prepare release** (patch/minor/major). That opens a `chore(release): bump version to X.Y.Z` PR which runs towncrier and bumps every lockstep surface. Merging to `main` triggers **Tag release**, which pushes `vX.Y.Z` and existing `release.yml` publishes `.skill` / `.mcpb` artifacts. Do not hand-edit ten version files. Contributors do not need a global towncrier install — `uv sync --group dev` (or the Action) provides it for release prep only.
|
||||
5. Lockstep gate remains `tests/test_plugin_contract.py::test_versions_match_across_manifests`. Workflow contract: `tests/test_changelog_workflow.py`.
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` copies the skill into `~/.agents/skills/<name>/` (real directory) and, for harnesses that support symlinked skill dirs, drops a per-host symlink pointing at that copy. **Working-tree edits do NOT propagate automatically** — the `~/.agents/skills/<name>/` copy is frozen at install time. To sync after edits, re-run `npx skills add . -g -y`. For live-edit on a dev machine, replace the install copy with a symlink to the working tree: `ln -sfn "$PWD/skills/last30days" ~/.agents/skills/last30days` (run from the repo root).
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
- Do not reduce `fail_under` in `pyproject.toml` (`[tool.coverage.report]`) without documenting why in the PR. The coverage gate is a floor meant to rise over time, not to be relaxed when new code is under-tested.
|
||||
- Every `lib/*.py` call to `log.source_log(...)` must pass `tty_only=False`. The default is `True`, which silently drops every line when stderr isn't a TTY (Claude Code, Codex, CI, captured output) — turning source observability into invisible failure. Enforced by `tests/test_source_log_visibility.py`.
|
||||
- **CLI-gated optional sources** (Digg via `digg-pp-cli`, YouTube via `yt-dlp`) activate only when `shutil.which` resolves the binary on the **agent subprocess PATH** — not merely when the file exists on disk. First-run setup installs Digg through `@mvanhorn/printing-press-library` (default `$HOME/.local/bin`); Hermes/OpenClaw gateways often need that directory on PATH. Setup must distinguish PATH-visible installs from off-PATH binaries and must not claim "now active" unless the engine gate would pass. See `docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md`.
|
||||
- **First-run onboarding is consent-driven, model-led, and host-split.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth, and emitting the engine-owned welcome via `--welcome`) — it cannot prompt, so consent lives in `SKILL.md` Step 0. Two flows avoid model-authored prose that Claude Code folds or the model skips: in the **Modal Flow** the welcome pitch is embedded in the setup modal's question (the AskUserQuestion modal is the only always-fully-visible surface — a separate welcome message or `--welcome` Bash run gets buried behind "ctrl+o to expand"); the **Non-Modal Prose Flow** still uses `last30days.py --welcome` (relayed verbatim) since it has no modal. The GitHub device code is surfaced by a two-command split — `setup --github-start` returns the code fast (foreground, copies to clipboard) and `setup --github-poll` waits for authorization (`setup --github` still chains both for back-compat). Step 0 has TWO branches: a **Claude Code Modal Flow** (the restored v3.0.0 `AskUserQuestion`-driven NUX — welcome, Auto/Manual/Skip, cookie consent, ScrapeCreators offer, `INCLUDE_SOURCES` opt-in, first-topic picker) for hosts with modals, and a **Non-Modal Prose Flow** for hosts without (OpenClaw, Codex, Cursor, Gemini CLI, Grok). Both ask before reading cookies, surface the macOS Full Disk Access fix on permission-denied, and offer the ScrapeCreators GitHub signup (10,000 free calls) on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do NOT collapse the modal flow back into a bare silent `setup` call or flatten it to prose-only — the guided modals are the feature (they eroded once and were restored). The onboarding contract is locked by `tests/test_onboarding_contract.py`. The Step 5 source opt-in is two tiers, both comment-enabled: **Recommended** (TikTok + Instagram posts AND top comments, plus YouTube comments — `INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`) and **Everything** (also Threads + Pinterest). Comments are on by default (posts on → comments on for all three platforms); **Threads and Pinterest are the only opt-in extras**, appearing only in the Step 5 Everything option, never in the welcome or the Step 4 offer. Instagram comments are fetched via ScrapeCreators (`/v2/instagram/post/comments`, ranked by `comment_like_count`) with full vote-weighting parity to YouTube/TikTok (a dedicated `_instagram_engagement` carve-out, the `_VOTE_LOG_REFERENCE`/label/threshold entries). The cross-platform "Top Community Comments" list (`render._render_top_comments`) selects **round-robin by within-platform rank** (every platform's #1, then #2, then #3) so a viral platform can't crowd out a smaller one, and drops the per-platform absolute floor so a less-watched video's killer low-vote comment still surfaces.
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
@@ -45,6 +68,10 @@ Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
|
||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||
|
||||
## Maintaining README translations
|
||||
|
||||
`README.md` is the canonical English README. When changing it, reflect the same substantive updates in `README.fr.md`, `README.de.md`, `README.es.md`, `README.pt-BR.md`, `README.ja.md`, and `README.zh-CN.md`, preserving commands, links, tables, and reciprocal language navigation.
|
||||
|
||||
## Maintaining CONFIGURATION.md
|
||||
|
||||
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
||||
@@ -53,7 +80,7 @@ Update `CONFIGURATION.md` when:
|
||||
|
||||
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
||||
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Hermes…)
|
||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Grok, Hermes…)
|
||||
- adding a new optional source that requires its own credential
|
||||
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
||||
|
||||
@@ -61,6 +88,46 @@ Keep the existing structure organized by how often each layer is touched: per-ru
|
||||
|
||||
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
||||
|
||||
## Plugin manifests (Grok)
|
||||
|
||||
The repo doubles as a native Grok Build plugin via `.grok-plugin/plugin.json` + `.grok-plugin/marketplace.json`. Grok also reads `.claude-plugin/*` for compatibility; the native pair is the first-class lane and what an official xAI marketplace listing points at. The self-hosted catalog uses a bare Git URL source (`{"source":"url","url":"https://github.com/mvanhorn/last30days-skill.git"}`) so `grok plugin marketplace add mvanhorn/last30days-skill` tracks HEAD — not a self-referential local `path: "."` (Grok does not enumerate those). Version lockstep with Claude/Codex/Gemini manifests is enforced by `tests/test_plugin_contract.py`. Validate with `grok plugin validate .`.
|
||||
|
||||
## Submitting to the xAI plugin marketplace
|
||||
|
||||
Getting last30days into xAI's official catalog (`xai-org/plugin-marketplace`) is an outbound PR to *their* repo — an index that only points at our source, so nothing of last30days is vendored there. Do this **after** the change you want to ship has merged to `main`: the entry pins a commit that must already exist.
|
||||
|
||||
1. Fork `xai-org/plugin-marketplace` and branch from `main`.
|
||||
2. Get the commit to pin — a full 40-char lowercase SHA; a branch, tag, or short SHA is rejected by their validator:
|
||||
```bash
|
||||
git ls-remote https://github.com/mvanhorn/last30days-skill.git HEAD
|
||||
```
|
||||
3. Add one entry to their `.grok-plugin/marketplace.json` under `plugins[]`, a remote source pinned to that SHA:
|
||||
```json
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"category": "productivity",
|
||||
"source": {
|
||||
"source": "url",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill.git",
|
||||
"sha": "<full-40-char-sha-from-step-2>"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"keywords": ["last30days", "last 30 days"]
|
||||
}
|
||||
```
|
||||
4. Regenerate their component index (never hand-edit it) and validate exactly as their CI does:
|
||||
```bash
|
||||
python3 scripts/generate-plugin-index.py
|
||||
python3 scripts/validate-catalog.py
|
||||
python3 scripts/generate-plugin-index.py --check
|
||||
```
|
||||
5. Open the PR, fill in their template, and wait for code-owner review.
|
||||
|
||||
To roll out a later update in their catalog, bump the pinned `sha` in the existing entry — never open a second, parallel entry.
|
||||
|
||||
Do not confuse this with our own `.grok-plugin/marketplace.json`: that file makes this repo directly addable as a Grok marketplace (`grok plugin marketplace add mvanhorn/last30days-skill`) and uses a **bare URL** source (no SHA) so it tracks HEAD; the xAI entry above lives in *their* repo and uses a **remote** source pinned to a SHA.
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
|
||||
+430
-1
@@ -5,10 +5,429 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
This project uses [towncrier](https://towncrier.readthedocs.io/). Upcoming notes live in [`changelog.d/`](changelog.d/); do not edit this file in feature PRs.
|
||||
|
||||
<!-- towncrier release notes start -->
|
||||
|
||||
## [3.21.0] - 2026-08-14
|
||||
|
||||
### Changed
|
||||
|
||||
- X backend priority changed: bird (browser cookies) is now first in the auto chain, ahead of xai/xurl/xquik. Cookies beat XAI_API_KEY when both are present. Grok CLI is demoted to opt-in only: a leftover `~/.grok/auth.json` no longer steals the X lane. Pin `LAST30DAYS_X_BACKEND=grok` to enable it explicitly.
|
||||
|
||||
|
||||
## [3.20.0] - 2026-08-14
|
||||
|
||||
### Added
|
||||
|
||||
- X search now judges corpus on-topic ratio and retries once with a wider AND query when the initial results are mostly off-topic (e.g., phrase-quoted "Rome Italy" returning AS Roma sports accounts). Multi-word search queries use unquoted AND as the primary variant instead of phrase-quoting. Handles extracted from entity_extract are now filtered for the from: lane based on whether their already-retrieved posts are on-topic (≥2 on-topic hits and ≥50% ratio), not just frequency. Extracted handles that qualify for the from: lane AND the topic into the query (`from:handle Rome`) to prevent off-topic timelines from filling the X budget. Explicit --x-handle and --x-related handles always get the from: lane without ANDing the topic. Source status reflects off-topic floods as a warning artifact, not a failure. First-party floor immunity remains conservative (explicit handles only, not promoted commentators).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Amazon review enrichment now starts at search time instead of after all other sources finish, ensuring multi-source runs have a useful budget (up to 180s) rather than leftover crumbs. Previously, a run that spent 269s on retrieval would leave only 11s for reviews, causing all Bright Data pulls to time out. Budgets below 90s now skip the lane entirely instead of firing doomed short pulls that spend credits without returning reviews.
|
||||
- Grok session expiry is now detected locally by parsing `expires_at` from `~/.grok/auth.json`. Doctor reports expired sessions as **degraded** (not ok) with the expiry timestamp and a hint to run `grok login --device-auth` if refresh fails. Research-time availability still attempts grok when credentials exist (expired access_token does not prove the refresh_token is dead). When the Grok CLI returns "Not signed in" or `invalid_grant` mid-run, the pipeline now reports `auth-failed` with a proper fix hint instead of a generic PARTIAL outcome, and falls back to the next X backend.
|
||||
|
||||
|
||||
## [3.19.0] - 2026-08-14
|
||||
|
||||
### Security
|
||||
|
||||
- Source URLs containing unsafe schemes or Markdown delimiters are now rendered as inert escaped text instead of raw Markdown. ([#886](https://github.com/mvanhorn/last30days-skill/issues/886))
|
||||
- SessionStart `check-config.sh` now rejects non-identifier `.env` keys before `printf -v` (blocking array-subscript command substitution) and loads `.claude/last30days.env` only when `LAST30DAYS_TRUST_PROJECT_CONFIG` is set in the process environment or global config, matching `lib/env.py`.
|
||||
|
||||
### Added
|
||||
|
||||
- **Amazon buyer signals** — a new opt-in `amazon` source, backed by the Bright Data CLI. On shopping-intent topics it pulls discovered products with live ratings and prices, plus a capped sample of recent written reviews woven in as buyer voice.
|
||||
|
||||
The signal it exists for is *drift*: an all-time rating from thousands of ratings set against the average of only the reviews inside the last 30 days. When those disagree, something changed this month, and the review text says what. The emoji footer names each product and the direction it moved — `📦 Amazon: 3 products │ Chill Max XL 4.4★→3.8★ ↓, Deluxe Bag 4.7★→5.0★, BLUEY Set 4.8★ new` — rather than reporting inventory counts.
|
||||
|
||||
Off by default and dual-gated: the `brightdata` CLI must be on PATH and logged in, *and* the run must ask for the source (`--search ...,amazon` or `INCLUDE_SOURCES=amazon`). It never auto-fires from inferred intent. Use `--amazon-query` when the product keyword differs from the topic — a person topic searches their company's product line, not their name. `LAST30DAYS_AMAZON_DOMAIN` selects a non-US marketplace.
|
||||
|
||||
Billing is one credit per request against a 5,000/month free tier, so a typical run costs 4 credits regardless of how many reviews come back.
|
||||
- Reddit keyless discovery now falls back to the arctic-shift archive when the shreddit listing partials return nothing — hosts on datacenter egress (where Reddit 403s `/svc/shreddit`) keep scored Reddit discovery, score backfill, and discover-mode listings instead of reporting `auth-failed`.
|
||||
- X search now works with no X credential at all. Install the Grok CLI (`curl -fsSL https://x.ai/cli/install.sh | bash`, then `grok login`) and last30days reaches X with no X account, no browser cookies, and no `XAI_API_KEY` — on any host, including Claude Code, Codex, Cursor and GrokBot. It sits ahead of the browser-cookie path by default; pin `LAST30DAYS_X_BACKEND=bird` to keep cookies. Covers three lanes for a person or company topic: posts by the subject, posts @-mentioning them, and posts naming them in plain text (which is most of the discussion, and which a mention-only search misses).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Source URLs in the saved raw report and internal evidence output now render as clickable markdown links instead of plain text. ([#886](https://github.com/mvanhorn/last30days-skill/issues/886))
|
||||
- Hacker News comments no longer vanish from every per-source path. HN comments arrive as `{author, text, points}` while downstream readers key on `score`/`excerpt`, and `_normalize_hackernews` stored them raw, so `render._top_comments_list` filtered `(c.get("score") or 0) >= 5` against a key that was never present and rejected the entire source. HN comments are now remapped like the YouTube and TikTok ones, and the HN floor is 0 because the Algolia API returns `points: null` for every comment child, which makes any positive threshold unmeetable. A comment with no vote signal renders without a fabricated "(0 points)". ([#889](https://github.com/mvanhorn/last30days-skill/issues/889))
|
||||
- Polymarket topics spelled out in full ("artificial general intelligence") now match markets titled in shorthand ("AGI by 2030?"). Previously the topic filter and the relevance floor both compared full words against an acronym, so every on-topic market was dropped and the run reported zero results — indistinguishable from the source genuinely having none. ([#891](https://github.com/mvanhorn/last30days-skill/issues/891))
|
||||
- Reddit HTTP 429/403 on the keyless lanes is no longer reported as a clean `no-results`: the failure now survives the worker-thread hop into the run outcome, so `source_status` carries `rate-limited`/`auth-failed` with the status in `detail` and `doctor --postmortem` lists Reddit under Failed instead of "No failures on the last run." `doctor --probe` now checks the RSS endpoint the engine actually uses (the old `/r/all/hot.json` probe is permanently 403 keyless) and counts a 403/429 there as blocked rather than reachable. Under `LAST30DAYS_STRICT_EXIT` a blocked Reddit run now exits 3 instead of 0. ([#899](https://github.com/mvanhorn/last30days-skill/issues/899))
|
||||
- On Windows, the setup wizard's npx-based installs (Digg, arXiv, Techmeme) always failed silently because `shutil.which("npx")` resolves `PATHEXT` but `subprocess.run` given the bare string `"npx"` does not. Windows users also got macOS-only Homebrew guidance when yt-dlp was missing. Both are fixed: the resolved npx path is now passed through, and Windows gets `pip install yt-dlp` guidance instead. ([#904](https://github.com/mvanhorn/last30days-skill/issues/904))
|
||||
- `--web-backend=keyless` is now accepted by the CLI, matching what `CONFIGURATION.md` already documented. The keyless web-search floor was already fully supported internally; only the argument parser rejected the value. ([#905](https://github.com/mvanhorn/last30days-skill/issues/905))
|
||||
- arXiv no longer returns zero results for natural-language multi-word topics. The exact-phrase quoted query now retries unquoted once when it matches nothing, instead of silently dropping arXiv from the report. ([#908](https://github.com/mvanhorn/last30days-skill/issues/908))
|
||||
- Truth Social search no longer fails with a Cloudflare-triggered HTTP 403 on every request. Requests now send browser-like headers, the same fix already applied to Reddit. ([#909](https://github.com/mvanhorn/last30days-skill/issues/909))
|
||||
- `--emit=compact --save-dir` runs now save the complete debug artifact (all clusters plus every per-source item, with the emoji footer citing the actual written path) instead of the compact stdout render, which had made most collected evidence unrecoverable from the raw file. ([#923](https://github.com/mvanhorn/last30days-skill/issues/923))
|
||||
- The GitHub source no longer reports zero results when the planner writes search qualifiers into the topic (e.g. `open source AI stars:>1000 created:>2025-03-20`). `search_github` appends its own `created:>{from_date}` window, and two `created:` qualifiers collide: GitHub honors the first and ignores the appended window, so out-of-window items are fetched and then dropped wholesale by the local date filter, surfacing as a silent `no-results`. Qualifiers are now stripped from the topic before the query is built (including comma/semicolon-glued forms and quoted values such as `label:"bug fix"`), topic terms glued after a qualifier value are preserved, and a qualifier-only topic reports an explicit error instead of searching the whole site. ([#949](https://github.com/mvanhorn/last30days-skill/issues/949))
|
||||
- The GitHub source no longer returns zero results once a credential is available. GitHub rejects authenticated `/search/issues` requests that carry neither `is:issue` nor `is:pull-request` with HTTP 422, while anonymous requests are still accepted without one — so the source worked until a user ran `gh auth login` or set `GITHUB_TOKEN`, then failed silently while `doctor` still reported it healthy. Authenticated searches now run both qualifier-scoped queries and merge them, deduped by item id and re-sorted by reaction count, which keeps issues and pull requests in the same result set; appending a single qualifier would have dropped roughly 87% of matches on a typical topic. The unauthenticated path is unchanged. When one partition fails but the other returns items, the surviving items are now kept and the source is reported as partial rather than silently claiming success — full-failure (both partitions return nothing) is still a clear failure. ([#967](https://github.com/mvanhorn/last30days-skill/issues/967))
|
||||
- Out-of-window evidence no longer leads the ranked output. Items whose dates fall outside the run's window were flagged `[date:low]` but ranked normally, so a 2025-10 video took the #1 cluster in a 2026-07 brief and a 2025-12 one took #5. Candidates whose every dated item is out of window are now demoted in both the fusion sort and `_final_score`, so they still appear as evidence but never above in-window material; items with no date at all are untouched, since an unknown date is a coverage gap rather than a stale item. The freshness verdict ("only N of M dated items are from the last 7 days") also reaches the pass-through footer instead of only the report body.
|
||||
- X runs on a person or company no longer discard the subject's own posts. A post almost never contains its own author's name, so lexical relevance scored it at zero and the retrieval floor pruned it — a run for "Peter Steinberger steipete" fetched 8 posts by him and reported none of them. Fixed across the chain: planner scaffolding words no longer count as topic signal, posts by a handle the run is searching are exempt from the floor, auto-discovered handles now reach the first-party protections (previously only `--x-handle` did), quoted proper-noun phrases survive into the provider query instead of degrading into a token conjunction, and the subject of the topic gets a higher per-author cap than incidental accounts. When no real handle can be identified at all, the X floor is skipped rather than pruning against lexical name tokens. The thin-source retry path defers the X floor the same way Phase 1 does, so a subject-authored post recovered on retry is not discarded before handle resolution.
|
||||
- `--github-user` no longer returns unrelated repos for people whose PR search comes back empty or is unavailable. Person mode now falls back to the selected user's public GitHub events and returns only in-window `PushEvent` activity attributed to that actor, instead of treating repository-level `pushed_at` as proof that the selected user pushed. A pinned `--github-user` that still yields nothing is recorded as `no-results` instead of passing silently.
|
||||
|
||||
|
||||
## [3.18.4] - 2026-07-28
|
||||
|
||||
### Fixed
|
||||
|
||||
- Tag release workflow YAML now parses on every main push; merge commits can mint `vX.Y.Z` tags again. ([#880](https://github.com/mvanhorn/last30days-skill/issues/880))
|
||||
- YouTube yt-dlp search under comparison-mode fan-out no longer self-throttles into 120s timeouts: concurrent yt-dlp invocations are process-wide capped, identical searches are deduped within a run, and a search timeout is recorded as `timeout` rather than `no-results`. `LAST30DAYS_YT_SEARCH_TIMEOUT` configures the search deadline.
|
||||
|
||||
|
||||
## [3.18.3] - 2026-07-25
|
||||
|
||||
### Fixed
|
||||
|
||||
- Top Community Comments ranking blends thread relevance with vote strength and applies the relevance floor only when enough on-topic candidates exist. ([#701](https://github.com/mvanhorn/last30days-skill/issues/701))
|
||||
- Chromium cookie extraction now searches every browser profile for a matching cookie set, and reuses the Keychain/AES key across profiles in one scan. ([#725](https://github.com/mvanhorn/last30days-skill/issues/725))
|
||||
- Synthesis contract is echoed at the top of the evidence envelope so hosts that truncate stdout still see the "synthesize, don't dump" directive. ([#727](https://github.com/mvanhorn/last30days-skill/issues/727))
|
||||
- `store_findings` no longer raises `TypeError` when a re-sighted finding carries `engagement_score: None`. ([#796](https://github.com/mvanhorn/last30days-skill/issues/796))
|
||||
- Explicit --plan payloads with an invalid schema now exit with a field-specific error instead of silently running a deterministic plan. ([#841](https://github.com/mvanhorn/last30days-skill/issues/841))
|
||||
- Company-topic runs no longer auto-add the `jobs` source when an explicit `--search` / `requested_sources` filter is set; `--hiring-signals` still forces jobs. ([#842](https://github.com/mvanhorn/last30days-skill/issues/842))
|
||||
- Report footer path now matches the collision-safe path actually reserved by save_output. ([#850](https://github.com/mvanhorn/last30days-skill/issues/850))
|
||||
- YouTube ScrapeCreators transcript rescue is logged instead of being masked as a hard failure. ([#851](https://github.com/mvanhorn/last30days-skill/issues/851))
|
||||
- Polymarket domain-sweep topics no longer drop every market after noise-word stripping removes terms like "ai". ([#859](https://github.com/mvanhorn/last30days-skill/issues/859))
|
||||
- Reddit fetch windows now track the requested date range so short `--days` runs no longer pull a depth-default month and discard everything outside the window. ([#860](https://github.com/mvanhorn/last30days-skill/issues/860))
|
||||
|
||||
|
||||
## [3.18.2] - 2026-07-25
|
||||
|
||||
### Added
|
||||
|
||||
- Release preparation now builds CHANGELOG.md from changelog.d fragments via towncrier and bumps every plugin/marketplace lockstep version surface through an automated Prepare release workflow (no more shared Unreleased edits).
|
||||
|
||||
### Fixed
|
||||
|
||||
- `--trustpilot-domain` (and plan-level `trustpilot_domain`) now auto-activates the opt-in Trustpilot source for the run instead of silently no-oping when `INCLUDE_SOURCES` / `--search` omit it. ([#873](https://github.com/mvanhorn/last30days-skill/issues/873))
|
||||
- Scraped evidence text can no longer inject structural `##` markdown headings into the EVIDENCE FOR SYNTHESIS block — continuation lines stay indented and leading ATX markers are escaped. ([#874](https://github.com/mvanhorn/last30days-skill/issues/874))
|
||||
|
||||
|
||||
## [3.18.1] - 2026-07-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- General reports no longer promote unanchored fallback entity misses, zero-score clusters, or comments attached only to rejected evidence into synthesis. ([#863](https://github.com/mvanhorn/last30days-skill/pull/863))
|
||||
- YouTube transcript fetches now reuse a completed VTT left on disk when yt-dlp times out, honor `.env` values for caption languages, and allow keyed runs to tune the 12-second fast-fail timeout. ([#864](https://github.com/mvanhorn/last30days-skill/pull/864))
|
||||
- Comparison / vs-mode no longer silently drops entities beyond 4. Entity ceiling is now `COMPETITORS_MAX + 1` (7), truncation warns on stderr naming dropped entities, and `--competitors-plan` implies competitor mode so a vs-topic + plan keeps all named peers (plan remains targeting-only; discover-N via bare `--competitors` is unchanged) ([#868](https://github.com/mvanhorn/last30days-skill/issues/868), [#870](https://github.com/mvanhorn/last30days-skill/pull/870)).
|
||||
- Docs now match Reddit ScrapeCreators search backup semantics: empty-only by default (not "when public Reddit is unavailable" / rate-limited). `CONFIGURATION.md` documents `LAST30DAYS_REDDIT_SC_MIN_ITEMS`; `SKILL.md` Security, Manual setup, NUX, and the Reddit backend pin describe the real empty-path / thinness-floor / SC-primary knobs. NUX Step 4/5 no longer claim SC Reddit comment enrichment or `public + ScrapeCreators` merge on the default free path (comments stay keyless via shreddit) ([#867](https://github.com/mvanhorn/last30days-skill/issues/867), [#869](https://github.com/mvanhorn/last30days-skill/pull/869)).
|
||||
- X search via xurl pins app-only bearer auth so OAuth1-signed multi-word queries no longer 401. ([#855](https://github.com/mvanhorn/last30days-skill/pull/855))
|
||||
- Bird X retries normalize cleanly and empty result sets stay empty instead of erroring. ([#840](https://github.com/mvanhorn/last30days-skill/pull/840))
|
||||
|
||||
## [3.18.0] - 2026-07-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Discovery is now a three-command host-judged protocol (SKILL.md LAW 11: "YOU ARE THE JUDGE"): `--discover --nominate-only` writes a nominations bundle and a fenced judging digest, the hosting model writes a judgments file (short names, junk flags, worthiness) and later an angles file, and `--discover --judgments <file>` / `--discover --finalize [--angles <file>]` complete the run. No API key is ever needed for host-judged trending. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
- Discovery protocol runs enrich at the normal-research tier (default depth, 4 workers, `LAST30DAYS_ENRICH_BUDGET_SECONDS` default 450s) instead of the 240s quick sweep; one-shot `--discover` keeps the quick tier unchanged. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
- Displayed discovery ranks now descend by the card's velocity score, and survivors sharing evidence (same top comment or 2+ shared URLs) fold into the higher-velocity story. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
|
||||
### Removed
|
||||
|
||||
- The engine-side discovery LLM judge (`lib/discovery_judge.py` and all reasoning-provider resolution in the discovery path). One-shot cron runs use deterministic heuristic names, velocity-only order, and no angles, with one loud stderr note pointing at the host-judged protocol. Keyed one-shot users lose provider naming/angles by design - the protocol replaces them. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
|
||||
## [3.17.0] - 2026-07-21
|
||||
|
||||
### Added
|
||||
|
||||
- Discovery trend cards now lead with short judged topic names: a stage-1 LLM judge gives each nominated cluster a 2-6 word searchable name (with a deterministic fallback namer), replacing raw post titles as card headings, and blends a 0-100 content-worthiness score into the ranking. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Junk-shape gate in discovery: help-me / beginner / personal-musing post shapes flagged by the judge (or the deterministic classifier at the seed-source floor) lose the single-source ranking bypass and need cross-source corroboration to rank. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Stage-2 angle pass: every discovery trend card carries engine-owned `**Podcast angle:**` and `**X article angle:**` lines, so the brief doubles as a content-pipeline worksheet. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Persistent discovery topic queue: `--discover` runs record surfaced topics in research.db (on by default; `LAST30DAYS_DISCOVERY_QUEUE=off` opts out, `--mock` never writes, `--save-dir` scopes the store), re-surfaced or covered topics get a `**Pipeline:**` annotation on their card, and `queue list` / `queue cover "<name>"` manage the queue from the CLI. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Discovery JSON export schema 1.1: per-topic `podcast_angle` / `x_article_angle` plus the queue fields `previously_surfaced_count`, `last_surfaced`, and `covered` join the discovery export contract; every existing key is preserved. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
|
||||
## [3.16.0] - 2026-07-15
|
||||
|
||||
### Added
|
||||
|
||||
- YouTube comments now fetch free via yt-dlp first; ScrapeCreators is a failure-only fallback, dropping the paid-key requirement for comment enrichment. ([#827](https://github.com/mvanhorn/last30days-skill/pull/827))
|
||||
- `GITHUB_TOKEN` is registered end-to-end (.env, keychain, setup scripts, doctor) so the GitHub source stops rate-limiting keyed users. ([#793](https://github.com/mvanhorn/last30days-skill/pull/793))
|
||||
- Opt-in overridable per-source result caps for high-volume topics; defaults unchanged when unset. ([#717](https://github.com/mvanhorn/last30days-skill/pull/717))
|
||||
- `OPENROUTER_BASE_URL` override, mirroring the existing OPENAI/XAI base-URL knobs. ([#703](https://github.com/mvanhorn/last30days-skill/pull/703))
|
||||
- `LAST30DAYS_MCP_TIMEOUT` accepts bare integer seconds as documented, not just Go duration strings. ([#765](https://github.com/mvanhorn/last30days-skill/pull/765))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Keyless web search now works on hosts where DuckDuckGo anomaly-blocks the egress IP (a 202 challenge page with no results — common on datacenter/VPS IPs). Added Startpage as a second keyless rung (DuckDuckGo → Startpage → configured SearXNG), so the web floor still returns results there. Also hardened `_strip_html` to drop `<style>`/`<script>` contents so inline CSS can't leak into a title or snippet.
|
||||
- Web/grounding results are no longer discarded when one of them is a reddit.com URL whose enrichment fetch fails. Reddit enrichment is a best-effort secondary fetch; its HTTP failures (e.g. a 403 on a datacenter IP) were being attributed to the whole web source, which then reported "0 items — HTTP 403" despite having retrieved good results. Its failures are now isolated from the source's outcome.
|
||||
- Very long topic names no longer crash `save_output` (ENAMETOOLONG): slugify truncates at 180 chars with a stable hash suffix so distinct topics stay distinct. ([#786](https://github.com/mvanhorn/last30days-skill/pull/786))
|
||||
- Quick depth honors the plan's explicit sources instead of trimming them away. ([#664](https://github.com/mvanhorn/last30days-skill/pull/664))
|
||||
- X search on Windows/Node 24: valid Bird CLI JSON on stdout is trusted even when the process exits non-zero. ([#813](https://github.com/mvanhorn/last30days-skill/pull/813))
|
||||
- 17 `.get(key, 0)` sites are now None-safe, fixing sort/math crashes on stored data with null fields. ([#822](https://github.com/mvanhorn/last30days-skill/pull/822))
|
||||
- Non-ASCII characters in URLs are percent-encoded component-wise before urllib, fixing the latin-1 encode crash. ([#822](https://github.com/mvanhorn/last30days-skill/pull/822), supersedes [#821](https://github.com/mvanhorn/last30days-skill/pull/821))
|
||||
- `LAST30DAYS_DEBUG` is registered and resolved lazily; fixes the `http.DEBUG` AttributeError in xai_x. ([#770](https://github.com/mvanhorn/last30days-skill/pull/770))
|
||||
- `DEGRADED_TRANSCRIPT_THRESHOLD` set in .env is picked up. ([#807](https://github.com/mvanhorn/last30days-skill/pull/807))
|
||||
- One bad video no longer marks the whole ScrapeCreators transcript source failed. ([#830](https://github.com/mvanhorn/last30days-skill/pull/830))
|
||||
- Chromium cookie temp copies keep 0600 permissions for their whole lifetime. ([#764](https://github.com/mvanhorn/last30days-skill/pull/764))
|
||||
- Thin-source retries forward pinned subreddits/hashtags/creators instead of retrying generically. ([#795](https://github.com/mvanhorn/last30days-skill/pull/795))
|
||||
|
||||
## [3.15.0] - 2026-07-14
|
||||
|
||||
### Added
|
||||
|
||||
- `doctor` is now a four-state audit instead of a flat config prediction: every source is grouped into **WORKING** (verified this run, last run, or keyless-always-on), **TURNED ON - UNVERIFIED** (configured/opted-in but no run evidence), **NOT WORKING** (configured but failing, or the last run errored), or **COULD BE ON** (an available capability not yet configured). Each source renders on its own labeled line, so GitHub (and every other source) is no longer buried in a cluster.
|
||||
- `doctor --postmortem`: reads the last run's `last-report.json` (any age, labeled) and reports, per source, what actually happened - Failed / Partial / Succeeded / Skipped with details and fix hints - so "what broke on that run?" is answerable after the fact.
|
||||
- `doctor --probe`: a bounded live test that verifies WORKING instead of guessing. It also auto-fires when there is no fresh run. Each source is probed concurrently under a per-source deadline (`LAST30DAYS_DOCTOR_PROBE_TIMEOUT`, default 10s) so a slow source can never hang the command. Scope is free HTTP endpoints + keyless CLIs only; credit-gated sources (X, TikTok, Instagram, Threads, …) are never live-probed and stay UNVERIFIED.
|
||||
- `doctor` now surfaces **CLI health**: sources needing a downloaded binary (`yt-dlp`, `digg-pp-cli`, `techmeme-pp-cli`, `arxiv-pp-cli`, `trustpilot-pp-cli`, optional `gh`) carry an inline `[CLI: name ✓]` marker and a dedicated CLI-health block, visibly distinct from keyless sources.
|
||||
- `doctor` now audits **techmeme, arXiv, and trustpilot** (they run in research but were previously absent from the health surface), and surfaces **backup lanes** (Reddit ScrapeCreators backfill, YouTube SC transcript/search backstop used when yt-dlp is rate-limited, X cookie-vs-`XAI_API_KEY` dual path) and **comment lanes** (youtube/tiktok/instagram) as indented sub-lines.
|
||||
- `doctor --json` gains `audit_state`, `cli`, `backups`, `comments`, and `run_outcome` per source plus a top-level `mode`, all additive - every existing key is preserved.
|
||||
|
||||
### Fixed
|
||||
|
||||
- `doctor` no longer reports Threads as Ready when it will not run: SC-gated opt-in sources now honor `INCLUDE_SOURCES` (mirrors the correct LinkedIn gating), so Threads shows COULD BE ON until opted in. TikTok/Instagram stay on-by-default.
|
||||
|
||||
## [3.14.0] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Global trending: bare `--discover` (no domain) sweeps every river feed's own hot list (r/all, Hacker News front page, Digg) with no keyword gate - `/last30days trending` now works. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
- Discovery is now two-stage: a listing sweep nominates candidate topics, then each nomination gets a full research pass (Reddit with comments, X, YouTube, Techmeme, arXiv, HN, Polymarket, web) before ranking - Techmeme and arXiv reach discovery for the first time, and every trend card can carry a verbatim community-voice quote with attribution plus a cross-source corroboration badge. `--discover-shallow` skips the research passes for a faster, thinner sweep. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
- Discovery confidence floor: every topic must clear cross-source confirmation or a genuinely strong single-source spike; when nothing clears, the run reports an honest "Nothing solid this window" (JSON `outcome: nothing-solid` with the closest `weak_signal` named) instead of ranking noise. The discovery JSON contract gains `outcome`, `weak_signal`, and per-topic `top_comment` / `corroboration_count`. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Discovery no longer emits ranked junk on quiet or over-broad domains (the "sports" sweep that returned five 1-like tweets): sub-floor evidence never ranks. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
- An explicit `--search` source boundary now holds through discovery's research passes, not just the listing sweep; `--discover-shallow` without `--discover` errors instead of silently running a full research pass; enrichment stragglers can no longer keep the process alive past the wall-clock budget. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
|
||||
## [3.13.1] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Doctor `library` line: reports how many saved research briefs the local library holds (cheap glob, never a full parse), so the report's "From your library" block is explained on the health surface. The block itself now carries a one-line explainer with the `LAST30DAYS_LIBRARY_CONTEXT=off` opt-out. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Doctor no longer reports X as `Off` when the bird CLI plus browser-cookie consent serve X fine at runtime: the cookie-backed path now reads **Ready**, with an honest note that the session is verified only at run time and `XAI_API_KEY` is the key-backed alternative. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
- Doctor's YouTube note no longer reads as broken when yt-dlp is healthy: it affirms search + transcripts work, scopes the transcription key to caption-free videos, and correctly attributes comment text to ScrapeCreators (key + `youtube_comments` opt-in) with an actionable fix line - never to yt-dlp. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
- Doctor's Web line on Claude Code now says host-native web search is active instead of `degraded ... keyless`, and names the host rather than an env var the user never set. Messaging only; engine web behavior unchanged. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
- The report footer no longer prints `no results` lines for zero-item sources; failure signal stays in the Source Coverage / Partial Coverage evidence blocks, and the `Raw results saved to` line still renders when every source is empty. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
|
||||
## [3.13.0] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Xiaohongshu (RED) documented as a first-class requested-only source, with auto-detection of a logged-in local browser-session service: last30days probes `http://localhost:18060` then `http://host.docker.internal:18060` when the source is opted in; `XIAOHONGSHU_API_BASE` remains the explicit override. Zero probing and zero behavior change for users who have not opted in. ([#766](https://github.com/mvanhorn/last30days-skill/pull/766), thanks @yuzhiyang1)
|
||||
- DripStack as an opt-in source: premium financial newsletter and analyst-writeup search (free public API, no key), complementing StockTwits retail sentiment and Polymarket odds with professional analyst signal. Ships default-off; requests route through the shared HTTP layer and honor the 30-day window. ([#791](https://github.com/mvanhorn/last30days-skill/pull/791), thanks @zimoo354)
|
||||
- Persistent opt-in for both new sources via `INCLUDE_SOURCES=xiaohongshu` / `INCLUDE_SOURCES=dripstack` in `.env`, matching the LinkedIn/Perplexity pattern; per-run `--search` still works. ([#812](https://github.com/mvanhorn/last30days-skill/pull/812))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Whitespace in comma-separated `INCLUDE_SOURCES` values no longer silently breaks any source's persisted opt-in. ([#812](https://github.com/mvanhorn/last30days-skill/pull/812))
|
||||
- DripStack article bodies (subtitle/lede) now reach ranking and synthesis instead of only the capped snippet; the Xiaohongshu doctor prescription no longer recommends an env pin that disables auto-probing. ([#811](https://github.com/mvanhorn/last30days-skill/pull/811))
|
||||
- Release hygiene: SKILL.md body header and uv.lock are regenerated with the version bump (both were missed in the 3.12.0 cut and hotfixed on main).
|
||||
|
||||
## [3.12.0] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Typed per-run source outcomes: every run records what actually happened per source (`ok`, `no-results`, `partial`, `rate-limited`, `auth-failed`, `unreachable`, `timeout`, `schema-drift`, `skipped-unconfigured`, `error`) in `source_status`, with doctor-aligned states and fix hints - silence is never mistaken for coverage. ([#797](https://github.com/mvanhorn/last30days-skill/pull/797))
|
||||
- Versioned agent JSON export profile: `--emit=json --json-profile=agent` returns a stable machine contract (`schema_version` 1.2) with `source_status`, clusters, ranked results with joinable `candidate_id`, and freshness verdicts; `--json-profile=raw` keeps the legacy dump byte-identical. ([#798](https://github.com/mvanhorn/last30days-skill/pull/798), [#810](https://github.com/mvanhorn/last30days-skill/pull/810))
|
||||
- Research-quality eval harness: recorded-fixture regression suite scoring runs on citation grounding, recency compliance, cluster coherence, coverage, and determinism against per-fixture floors, in CI. ([#799](https://github.com/mvanhorn/last30days-skill/pull/799))
|
||||
- `--drill`: re-research one cluster of the cached report in depth without a full re-run. ([#800](https://github.com/mvanhorn/last30days-skill/pull/800))
|
||||
- `--discover`: topic-less trending sweeps over listing feeds with velocity-ranked story clusters and ready-to-run research commands. ([#801](https://github.com/mvanhorn/last30days-skill/pull/801))
|
||||
- `library feed`: renders every saved brief into a browsable HTML library with a topic-grouped index and a subscribable Atom feed; hand-written pages are preserved with backups. ([#802](https://github.com/mvanhorn/last30days-skill/pull/802))
|
||||
- `library search`: SQLite FTS5 full-text search across saved briefs and store sightings, plus a passive "From your library" section when new runs overlap past research; scoped `--save-dir` libraries stay fully isolated from the shared store. ([#803](https://github.com/mvanhorn/last30days-skill/pull/803))
|
||||
- `--register` audience templates: `exec`, `dev`, and `creator` presets reshape section order and budgets for the reader; `eli5` is unified into the same mechanism. ([#804](https://github.com/mvanhorn/last30days-skill/pull/804))
|
||||
- `--verify-freshness`: typed per-claim act-time verdicts (`current` / `stale` / `contradicted` / `unsupported`) with point re-fetch of Polymarket lines, GitHub stars, and StockTwits sentiment, inline or post-hoc over the cached report; closes the recency-promise audit gap. ([#805](https://github.com/mvanhorn/last30days-skill/pull/805), closes [#769](https://github.com/mvanhorn/last30days-skill/issues/769))
|
||||
- `--corpus`: register local directories as a private, offline, deterministic source; matching notes rank alongside social evidence under a LOCAL ONLY badge and are excluded from hosted publishing and agent JSON by default. ([#808](https://github.com/mvanhorn/last30days-skill/pull/808))
|
||||
- Native Grok Build (xAI) plugin and marketplace lane: `.grok-plugin/plugin.json` + `.grok-plugin/marketplace.json` so `grok plugin install mvanhorn/last30days-skill` and `grok plugin marketplace add mvanhorn/last30days-skill` work as first-class install paths. The self-hosted catalog uses a bare Git URL source (tracks HEAD); submitting to the official `xai-org/plugin-marketplace` remains a post-merge SHA-pinned outbound PR documented in `AGENTS.md`.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Session-start hook no longer deadlocks under Homebrew bash 5.3: removed every heredoc from `check-config.sh` (bash 5.3 can block forever in `heredoc_write` inside command substitution). ([#809](https://github.com/mvanhorn/last30days-skill/pull/809))
|
||||
- Trustpilot transient-error retries keep their domain parameters. ([#794](https://github.com/mvanhorn/last30days-skill/pull/794))
|
||||
- Hosted same-day saves no longer overwrite earlier reports, and `save_output` never silently overwrites date-stamped files. ([#784](https://github.com/mvanhorn/last30days-skill/pull/784), [#785](https://github.com/mvanhorn/last30days-skill/pull/785))
|
||||
- `.env` reads as UTF-8 (with BOM tolerance and locale fallback) on Windows. ([#780](https://github.com/mvanhorn/last30days-skill/pull/780), [#715](https://github.com/mvanhorn/last30days-skill/pull/715))
|
||||
- `FUN_LEVEL` and `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` are registered in `env.py` so `.env` values are no longer silently ignored; doctor detects `GITHUB_TOKEN` from the process environment. ([#708](https://github.com/mvanhorn/last30days-skill/pull/708), [#732](https://github.com/mvanhorn/last30days-skill/pull/732), [#782](https://github.com/mvanhorn/last30days-skill/pull/782))
|
||||
- File descriptors close promptly across the engine (`open()` wrapped in `with`). ([#775](https://github.com/mvanhorn/last30days-skill/pull/775))
|
||||
|
||||
## [3.11.0] - 2026-07-05
|
||||
|
||||
### Added
|
||||
|
||||
- `last30days doctor`: a unified health command that aggregates every source's probe state into a single grouped report with copy-pasteable fix prescriptions. Layered design: dependency probes (missing/broken/timeout detection), backend-chain descriptors (predict-then-report, never a network call), a centralized prescription registry shared by doctor and quality nudges, and an aggregator with grouped rendering. Replaces the fragmented health knowledge previously spread across `--diagnose`, `--preflight`, `lib/health.py`, and post-run nudges. ([#753](https://github.com/mvanhorn/last30days-skill/pull/753))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Techmeme: `search` results are now windowed to each record's own ISO date instead of stamping every record with today's date, so years-old archive headlines can no longer surface as current news. Dated in-window records take result-cap slots first; undated records (old `techmeme-pp-cli` binary or upstream markup change) degrade gracefully with a logged upgrade hint. The sync machinery is removed because `search` never read the local cache. ([#752](https://github.com/mvanhorn/last30days-skill/pull/752))
|
||||
- LinkedIn now renders in the emoji-tree footer (👔 with likes/comments), the `## Stats` engagement summary, and with the correct "LinkedIn" label. Previously LinkedIn items were counted in `## Stats` but silently dropped from the footer because `_FOOTER_SOURCES`, `ENGAGEMENT_DISPLAY`, and `SOURCE_LABELS` all omitted the source - an 8-item LinkedIn run looked like the source never ran. ([#758](https://github.com/mvanhorn/last30days-skill/pull/758))
|
||||
|
||||
## [3.10.0] - 2026-07-04
|
||||
|
||||
### Added
|
||||
|
||||
- Instagram comments as a first-class ScrapeCreators source: `instagram.enrich_with_comments` fetches top comments via `GET /v2/instagram/post/comments` (ranked by `comment_like_count`), gated by `SCRAPECREATORS_API_KEY` + `instagram_comments` in `INCLUDE_SOURCES`. Full vote-weighting parity with YouTube/TikTok - a dedicated `_instagram_engagement` gives IG posts the same top-comment ranking carve-out, and IG comments render with a "likes" label. ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
|
||||
- Comments are now on by default: the first-run Step 5 Recommended tier enables top comments for TikTok, Instagram, and YouTube (`INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`); the Everything tier adds Threads + Pinterest. Comments were previously an opt-in "Everything" feature. ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
|
||||
|
||||
### Changed
|
||||
|
||||
- The cross-platform "Top Community Comments" list now selects **round-robin by within-platform rank** (every platform's #1, then #2, then #3) instead of a global vote-magnitude sort, so the top-3-of-each-platform outranks the 4th-of-any and each platform's #1 is guaranteed a slot - a viral platform can no longer sweep the list. The list also drops the per-platform absolute vote floor so a less-watched video's high-signal low-vote comment still surfaces (the per-candidate card keeps its floor). ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run wizard: the welcome pitch is embedded directly in the setup modal (the only always-visible surface) instead of a separate `--welcome` message that Claude Code folds behind "ctrl+o to expand"; the cookie-consent and ScrapeCreators-offer copy now name every installed CLI (yt-dlp, Digg, arXiv, Techmeme) and describe the key's real reach (auto Reddit enrichment + YouTube search backstop), with the GitHub device code auto-copied to the clipboard. ([#750](https://github.com/mvanhorn/last30days-skill/pull/750))
|
||||
|
||||
## [3.9.4] - 2026-07-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run wizard: the welcome message and the ScrapeCreators GitHub device code are now engine-driven instead of model-authored, because a real cold run showed the model skipping the welcome and never surfacing the device code no matter how forceful the SKILL.md prose. The welcome is printed by a new `last30days.py --welcome` command that Step 1 relays verbatim (single source of truth; it can't be skipped or drift), and the GitHub device flow is split into `setup --github-start` (submits, copies the code to the clipboard, prints it to stdout, opens the browser, returns immediately) and `setup --github-poll` (waits for authorization and persists the key). The one-shot `setup --github` still chains both. The code now always appears in the command output, and the "on your clipboard" claim is only made when the copy actually succeeded. ([#748](https://github.com/mvanhorn/last30days-skill/pull/748))
|
||||
|
||||
## [3.9.3] - 2026-07-04
|
||||
|
||||
### Added
|
||||
|
||||
- Optional remote research API backend (env-driven). When both `LAST30DAYS_API_KEY` and `LAST30DAYS_API_BASE` are set in the process environment (never read from `.env`), a search runs through the configured remote endpoint (submit -> poll with stderr progress -> render) instead of local sources; with either unset, behavior is byte-identical to local-only. Opt-in and inert by default (no built-in endpoint); the key is confined to the `Authorization` header and never logged or persisted. Handles the clarify gate and 401/402/429 paths. ([#747](https://github.com/mvanhorn/last30days-skill/pull/747))
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run wizard: the welcome message is now mandated before the setup modal (it was being skipped), the Auto-setup option lists every installed CLI (yt-dlp, Digg, arXiv, Techmeme, not just two), and the ScrapeCreators GitHub signup reliably surfaces the device code with an "it's on your clipboard, just paste" hint as a required step instead of leaving the user staring at a spinner. ([#746](https://github.com/mvanhorn/last30days-skill/pull/746))
|
||||
- ScrapeCreators GitHub signup: an already-linked account whose `.env` is cold no longer fails with the misleading "GitHub auth didn't complete." The `Authorized but failed to fetch API key` case now gets an honest branch (auth worked; the account is likely already linked -- get your key from scrapecreators.com and paste it), and `fetch_api_key` logs the `/profile` response field names (never values) so a full auto-fetch can follow. ([#746](https://github.com/mvanhorn/last30days-skill/pull/746))
|
||||
|
||||
## [3.9.2] - 2026-07-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Trustpilot source returned 0 items on company topics: the engine passed raw topic names to a domain-keyed CLI (`info ThriftBooks` -> HTTP 404) and parallel subqueries raced concurrent Chrome WAF-cookie harvests. Company names now resolve to their Trustpilot review-page domain via the CLI's search (per-topic cache; name-match mandatory, ambiguous cases fall back rather than misattributing another company's reviews), a new `--trustpilot-domain` flag pins the domain explicitly (verbatim, bypasses the brand-shape gate, per-entity `trustpilot_domain` in `--competitors-plan`), the WAF session warms once per 240s window behind a lock at first fetch, Trustpilot is capped to one fetch per run and excluded from the thin-source retry, and headless `--auto-resolve` fills a verified domain hint. SKILL.md Step 0.5d documents the resolution flow. ([#745](https://github.com/mvanhorn/last30days-skill/pull/745))
|
||||
|
||||
## [3.9.1] - 2026-07-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run setup wizard: the browser-cookie scan now tries Chrome/Chromium first (Keychain, no Full Disk Access) before Safari, so macOS users logged into X in Chrome authenticate in ~2s instead of hitting the Safari Full Disk Access dead-end. The winning browser is pinned for later runs only when it is Firefox/Safari, so Chrome never re-triggers the Keychain prompt. Consent copy leads with Chrome and the one-time "Always Allow" cue. ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
|
||||
- ScrapeCreators GitHub signup now surfaces the device code immediately (emitted to stdout so a backgrounded caller shows it at once, instead of a spinner until the process exits), validates the `XXXX-XXXX` code shape before copying/labeling it, short-circuits an already-registered account without a fresh device dance, and masks the API key on every status (not just success). Removed the false "GitHub CLI ~2 seconds, no browser" promise. ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
|
||||
- ScrapeCreators source opt-in is now two real tiers. The Step 5 choices were previously identical — a key auto-ran TikTok, Instagram, Threads, and YouTube comments regardless of `INCLUDE_SOURCES`, and Pinterest's opt-in silently ignored a persisted `INCLUDE_SOURCES`. Threads, YouTube comments, and Pinterest are now genuine `INCLUDE_SOURCES` opt-ins: **Recommended** = TikTok + Instagram + the rate-limit backups; **Everything** = also Threads, Pinterest, and YouTube/TikTok/Instagram comments. "ScrapeCreators backups" is now defined inline (keeps Reddit/YouTube working at rate limits). ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
|
||||
|
||||
## [3.9.0] - 2026-07-03
|
||||
|
||||
### Added
|
||||
|
||||
- StockTwits as a source, gated to ticker/crypto topics only. Surfaces a retail sentiment ratio (self-reported Bullish/Bearish tags) and message volume on a resolved symbol. Inert on non-financial topics: an unambiguous finance-vocabulary gate (cashtags, "stock", "earnings", "dividend", "crypto", named coins) keeps it from injecting stock chatter into general runs, and it degrades to an empty lane if the public API fails without touching other sources. ([#658](https://github.com/mvanhorn/last30days-skill/pull/658), thanks @wtiwana)
|
||||
- LinkedIn as a source via ScrapeCreators, surfacing articles as high-signal results with date-range filtering, gated behind `INCLUDE_SOURCES`. ([#702](https://github.com/mvanhorn/last30days-skill/pull/702))
|
||||
- arXiv and Techmeme sources (default-on) plus Trustpilot (opt-in). ([#709](https://github.com/mvanhorn/last30days-skill/pull/709))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Runtime preflight now auto-provisions a uv-managed CPython 3.12 on hosts that have `uv` but no system Python 3.12+ (most agent sandboxes), instead of hard-failing the version gate. The install is bounded by a 30s HTTP timeout, matches an existing managed `>=3.12` interpreter before downloading, and announces the one-time ~28MB download on stderr rather than installing silently; hosts without `uv` still get the original clear error. Setup invocations now honor `LAST30DAYS_PYTHON` so first-run setup works on the same hosts. ([#738](https://github.com/mvanhorn/last30days-skill/pull/738), thanks @buntysomroy; setup-interpreter fix adapted from #699 by @SeanGearin)
|
||||
- Setup wizard summary now displays the install status of the arXiv/Techmeme pp_sources CLIs, so users can see whether they landed on PATH. ([#741](https://github.com/mvanhorn/last30days-skill/pull/741), thanks @23241a6749)
|
||||
- `--diagnose` / `--preflight` no longer falsely reports X as unreachable when X auth comes from `FROM_BROWSER` browser cookies. These modes run in `plan_only` and skip cookie extraction for privacy (no Keychain access), so X was dropped from `available_sources` even though a real run authenticates fine. A new side-effect-free `env.x_pending_browser_auth` predicate now reports X as available-pending-browser-auth (and surfaces an `x_pending_browser_auth` flag in `--diagnose`) by keying only on the already-resolved browser list — no cookie is read. Covers every configured browser, including Chrome. ([#692](https://github.com/mvanhorn/last30days-skill/issues/692); first reported and fixed by @23241a6749 in #700)
|
||||
|
||||
### Internal
|
||||
|
||||
- Tightened Hermes `.skillignore` regression coverage: the test now fails if an ignored path is deleted without updating the ignore list, or if a runtime-contract file is accidentally ignored. ([#739](https://github.com/mvanhorn/last30days-skill/pull/739), thanks @SyntaxSawdust)
|
||||
|
||||
## [3.8.3] - 2026-06-25
|
||||
|
||||
### Added
|
||||
|
||||
- Free Reddit gets dedicated-subreddit lanes: entity-home subs (e.g. r/Kanye for "Kanye West", via the new `--dedicated-subreddits` flag) are pulled in full from top+hot+new listings and exempt from the relevance floor, since the whole sub is the topic. Fixes the over-aggressive floor that dropped on-topic posts whose titles lacked the entity name.
|
||||
- `reddit_arctic` resolves upvote counts for threads found only via RSS search (which carries no score) using the free, keyless arctic-shift archive — batched, paced, cached, and graceful-degrading. Reddit now gets headlines-with-points and best-comments-with-points entirely for free, at parity with ScrapeCreators.
|
||||
- `LAST30DAYS_REDDIT_SC_MIN_ITEMS` (default 0 = unchanged empty-only behavior): set above 0 to let the ScrapeCreators backup backfill a thin free Reddit run instead of sitting idle. Backfilled items merge deduped by post id.
|
||||
|
||||
### Removed
|
||||
|
||||
- The permanently-403 `search.json` Tier 0 is gone from the keyless Reddit path; discovery is RSS breadth + shreddit listing partials (real scores) + the dedicated-sub lanes, with no wasted 403 calls.
|
||||
|
||||
## [3.8.2] - 2026-06-25
|
||||
|
||||
### Added
|
||||
|
||||
- Advisory Semgrep SAST scan runs on every push/PR as part of the Security workflow, catching source-level security bugs using Semgrep CE community rules ([#563](https://github.com/mvanhorn/last30days-skill/issues/563))
|
||||
- Scheduled OSV-Scanner vulnerability-drift workflow scans repository lockfiles weekly and uploads SARIF results to GitHub code scanning, catching newly disclosed CVEs in the dependency tree even between PRs ([#571](https://github.com/mvanhorn/last30days-skill/issues/571))
|
||||
- `LAST30DAYS_REDDIT_BACKEND=scrapecreators` makes ScrapeCreators the primary Reddit backend with the public path as fallback. Users with a ScrapeCreators key who were getting shallow public data will now get full nested comment trees by setting this flag ([#589](https://github.com/mvanhorn/last30days-skill/issues/589))
|
||||
- MCP Go tests (`mcp/`) now run in CI on every push/PR alongside the Python test suite, so MCP server regressions are caught before merge ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
- PR dependency review gate blocks merges that introduce new vulnerable dependencies ([#551](https://github.com/mvanhorn/last30days-skill/issues/551))
|
||||
|
||||
### Changed
|
||||
|
||||
- Citations are now renderer-aware (LAW 8). On hidden-link hosts (Claude Code) every citation stays an inline `[name](url)` link as before; on visible-URL hosts (Codex, Cursor, Gemini CLI, raw CLI) citations render as plain source labels so the narrative no longer turns into `label (https://...)` URL soup. The host is detected deterministically from the `CLAUDECODE` environment variable, and full URLs remain reachable through the engine footer and the saved raw file.
|
||||
|
||||
### Fixed
|
||||
|
||||
- The query-plan invocation guidance now warns against wrapping the heredoc in `bash -lc '...'` / `zsh -lc '...'`, whose single quotes terminate at the first apostrophe in a ranking string and abort the engine run with `unmatched "` on Codex. The quoted `<<'PLAN_EOF'` heredoc is already apostrophe-safe; the `-lc` wrapper was the hazard.
|
||||
- Firefox profile detection on Linux now checks `$XDG_CONFIG_HOME/mozilla/firefox` (or its default `~/.config/mozilla/firefox`) in addition to `~/.mozilla/firefox`, fixing cookie extraction on distros that honour the XDG Base Directory Specification ([#667](https://github.com/mvanhorn/last30days-skill/issues/667))
|
||||
|
||||
## [3.8.1] - 2026-06-22
|
||||
|
||||
### Added
|
||||
- **Restored the v3.0.0 first-run NUX wizard (Claude Code Modal Flow).** Step 0 now restores the original guided, `AskUserQuestion`-driven onboarding that eroded over time: a welcome message, an Auto/Manual/Skip setup modal, a cookie-consent modal, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. It is gated to hosts with modals; hosts without (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent **Non-Modal Prose Flow**. Digg is threaded into the install messaging alongside yt-dlp everywhere it appears, the ScrapeCreators credit count is `10,000 free calls`, and the flow is locked against re-erosion by `tests/test_onboarding_contract.py`. Builds on the consent-driven foundation from #659/#660. Original wizard captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
- **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659).
|
||||
|
||||
### Fixed
|
||||
- **First-run setup no longer runs silently.** The prior Step 0 told the model to run `setup` and "follow the wizard's prompts end-to-end", but the wizard has no prompts — so onboarding extracted cookies, installed tools, and wrote `SETUP_COMPLETE` with zero interaction and never offered the ScrapeCreators signup. Reproduced 2026-06-22 (Fredy Montero, fresh macOS).
|
||||
|
||||
## [3.8.0] - 2026-06-21
|
||||
|
||||
### Added
|
||||
|
||||
- **Single X source with backend failover.** X is now one source backed by an ordered chain of interchangeable backends (xai, bird, xurl, xquik) with runtime failover, rather than separate sources. The key-based xquik backend reaches parity with bird, gaining the X-quality ranking and FROM/ABOUT handle lanes, so hosts that cannot supply browser cookies (OpenClaw, CI/cron, headless harnesses) get real X coverage from an xquik key alone. Handle lanes run via the first handle-capable backend in the chain even when a non-capable backend (xai/xurl) is primary. (#622)
|
||||
|
||||
## [3.7.1] - 2026-06-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- GitHub repo stars are no longer mislabeled as "reactions" in the report footer. Repo cards use a distinct `stars` engagement key, velocity cards use `merged_prs`, and genuine issue/PR reaction counts keep their own `reactions` key. (#645, closes #642)
|
||||
- Hacker News returned zero stories on every run: the Algolia query sent `points>2`, which the HN index no longer accepts as a filterable attribute, so every request 400'd. Dropped the server-side `points` filter; low-engagement demotion still happens at parse time. (#639)
|
||||
- Polymarket surfaced off-topic markets and rendered a mangled footer. The relevance filter was fed the per-subquery string instead of the stable topic, and market labels were truncated mid-article into fragments like "an Anthropic Claude model score at: an 19%". Now filters on the stable topic and cleans the labels. (#640)
|
||||
|
||||
## [3.7.0] - 2026-06-20
|
||||
|
||||
### Added
|
||||
|
||||
- **Direct Perplexity API support.** When `PERPLEXITY_API_KEY` is set it is preferred over OpenRouter for the Perplexity source, unlocking first-party Search API results and async Deep Research. Adds `LAST30DAYS_PERPLEXITY_MODE=sonar|search|both` plus model, search-context, domain/language/country, recency, and reasoning-effort knobs. OpenRouter stays the Sonar compatibility fallback when no direct key is set. Async Deep Research preserves request id, status, idempotency key, poll count, lifecycle timestamps, and failure metadata in raw artifacts. (#629, by @sk-holmes)
|
||||
|
||||
### Changed
|
||||
|
||||
- `check-config.sh` now parses env files in pure bash (no `sed` / `tr`), which also fixes the YouTube-availability hint breaking in minimal environments that lack those tools. (#629)
|
||||
|
||||
## [3.6.1] - 2026-06-20
|
||||
|
||||
### Added
|
||||
|
||||
- **ScrapeCreators transcript fallback.** When `SCRAPECREATORS_API_KEY` is set, YouTube transcripts fall back to the ScrapeCreators transcript endpoint after the keyless yt-dlp cascade fails (fetched server-side, so no 429 / cookies / PO tokens). yt-dlp stays primary and a credit is only spent on a genuine failure, never on success and never on a video proven to have no captions. With a key, yt-dlp also fails over fast (one short-timeout attempt) so a 429 hands off to ScrapeCreators in roughly 17s instead of roughly 90s. (#637, idea from #595)
|
||||
- **YouTube comments default-on.** Comment enrichment now activates whenever a ScrapeCreators key is present (bounded to the top ~3 videos by engagement, ~3 credits per run) instead of requiring `INCLUDE_SOURCES=youtube_comments`. Suppress with `EXCLUDE_SOURCES=youtube_comments`. TikTok/Instagram comments remain `INCLUDE_SOURCES` opt-ins. (#637)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Salvage partial YouTube transcripts on non-zero yt-dlp exit.** With the default `en,es,pt` languages an English video wrote `en.vtt` then 429'd on `es`/`pt`, and the already-written transcript was discarded and retried back into the rate limit. Any VTT on disk is now read before the failure is classified, which fixes the dominant `0/N transcripts` case. (#636)
|
||||
- **Windows transcript crash on subprocess timeout.** Guarded the SIGKILL escalation path in `run_with_timeout` against `os.killpg` / `os.getpgid` raising `AttributeError` on Windows (they are POSIX-only), mirroring the primary path's guard. (#638, reported in #588)
|
||||
|
||||
## [3.6.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- **First-party X posts are no longer buried.** A post authored by one of the run's resolved handles (`--x-handle`, `--x-related`, the GitHub user) is now treated as first-class evidence: it is exempt from the entity-miss demotion (a post never repeats its own author's name, so the body-text grounding check used to zero out the subject's own highest-signal posts) and gets a small authorship credit. Third-party collision-noise suppression is unchanged.
|
||||
- **Engagement rescue for on-topic X posts.** A high-engagement X post that is first-party or entity-grounded gets a `final_score` floor scaled by its engagement percentile within the run's X pool, so a viral on-topic post can't sit at ~0. Off-topic name-collision posts are explicitly excluded.
|
||||
- **First-party interaction signal.** A first-party post directed at another account (a reply / leading @mention) is floated into the visible band regardless of like-count and tagged `interaction:→@handle` in the EVIDENCE block, so the synthesis reads it as a relationship signal rather than low-engagement noise. New **LAW 10** in SKILL.md teaches the model to surface first-party posts and read the interaction tag.
|
||||
|
||||
### Changed
|
||||
|
||||
- The X FROM lane (the subject's own timeline) now pulls up to 8 posts per handle (was 3); the about/related lanes stay modest.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Secrets `.env` and its parent config directory are now auto-tightened to `0o600`/`0o700` after creation, and `check-config.sh`'s `check_perms` now auto-fixes loose permissions with `chmod 600` instead of warning only ([#573](https://github.com/mvanhorn/last30days-skill/issues/573))
|
||||
|
||||
## [3.5.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- **X surfaces tweets FROM and ABOUT a person, both engagement-weighted.** The handle search now pulls the person's real timeline (`from:handle since:`, topic used for ranking only — never AND'd into the query, which previously matched only tweets where they wrote their own name and returned ~0), and a new mention lane (`@handle since:`) surfaces what others say to/about them, excluding their own tweets and deduping against the FROM lane ([#610](https://github.com/mvanhorn/last30days-skill/pull/610)).
|
||||
- **`## Top Community Comments` block.** The engine now surfaces vote-ranked community comments across all candidates (not just the top-cluster representatives), per-platform-normalized, into the EVIDENCE-for-synthesis block, so the funniest/sharpest crowd reactions reach the synthesizing model even when no LLM fun-scorer is available. Paired with a new SKILL.md **LAW 9** that requires weaving ≥2 verbatim attributed comments, copying URLs verbatim, and never narrating the tooling in the deliverable ([#608](https://github.com/mvanhorn/last30days-skill/pull/608)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`--diagnose` honesty.** X status now reflects a real 1-tweet probe (downgrades from green when X is effectively dead; fail-open on a transient timeout) and reports the true auth lane (browser / env / keychain) instead of a hardcoded `env AUTH_TOKEN`. Handle/mention searches log query + result count on success, not only on failure ([#609](https://github.com/mvanhorn/last30days-skill/pull/609)).
|
||||
- **X column de-pollution.** The last-chance keyword retry no longer collapses a multi-word subquery to a bare generic token (e.g. `compound`); it keeps an entity anchor ([#607](https://github.com/mvanhorn/last30days-skill/pull/607)).
|
||||
- **Mandatory person-aware subquery disambiguation.** Collision-prone person names (Kevin Rose vs Kevin Warsh, Lan Xuezhao vs Lanzhou) must anchor every subquery with the resolved company/role/domain context ([#611](https://github.com/mvanhorn/last30days-skill/pull/611)).
|
||||
|
||||
## [3.4.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- **Crowd-vote weighting in the fun judge (Best Takes).** The fun judge now factors how many upvotes/likes each top comment earned. Comment vote counts are fed into the LLM prompt (as traction, not funniness), and Best-Takes selection ranks by an effective score — `fun_score` plus a bounded, per-platform-normalized, relevance-confidence-scaled crowd nudge — so genuinely funny, crowd-loved, on-topic comments surface while off-topic virality and high-voted-but-unfunny rants are excluded. `FUN_LEVEL=medium` stays the default and applies the signal as a meaningful factor ([#592](https://github.com/mvanhorn/last30days-skill/pull/592)).
|
||||
- **Digg added to first-run setup.** The free, keyless `digg-pp-cli` is now auto-installed during the first-run wizard (best-effort via the Printing Press installer, with a recommend-only fallback), so the already-built Digg AI-news source activates automatically for new users instead of silently never appearing ([#590](https://github.com/mvanhorn/last30days-skill/pull/590)).
|
||||
|
||||
- **`LAST30DAYS_YOUTUBE_SSH_HOST` transcript routing** — yt-dlp transcript fetch runs on the remote SSH host via a mktemp + cat pipeline ([#422](https://github.com/mvanhorn/last30days-skill/pull/422)).
|
||||
- Browser-cookie auth for X/Twitter now covers the full Chromium family on macOS - Brave, Microsoft Edge, Vivaldi, Opera, Arc, and Chromium - alongside the existing Chrome, Firefox, and Safari. They all share Chrome's v10 AES-128-CBC decryption, differing only in profile path and Keychain service name, so they run through one shared decryption core. The profile finder probes both the modern `Default/Network/Cookies` layout (Chromium >= 96) and the legacy flat `Default/Cookies`, and Chrome now resolves through that same finder so it picks up the modern layout too. Set `FROM_BROWSER=auto` to try every browser, or `FROM_BROWSER=<name>` (e.g. `brave`, `edge`, `arc`) to target one. Verified end-to-end on real Brave and Edge installs ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
|
||||
- **First-party positioning research + pitch-vs-pulse synthesis (company / product / service topics).** A new mandatory research step captures each entity's current stated positioning from first-party sources (homepage, docs, pricing) rather than from memory. The fetched pitch grounds `What it is` descriptions (entities described as they pitch themselves today), helps reject unrelated brand-name noise, and feeds an evidence-triggered prose beat: when the month's conversation directly supports a specific claim, cuts against one, or is squarely about the pitched ground, the synthesis says so anchored to the top thread — and stays silent when the pulse is orthogonal to the pitch, because a manufactured connection is worse than omission. Claims are tested at matched altitude (specific claims against specific threads; broad taglines are never graded against individual items), and statements stay windowed to the 30 days — no trend verdicts. Scoped to entities with an identifiable first party: people are always excluded (even founders whose companies qualify), as are events, abstract concepts, and ownerless topics like Bitcoin; the beat requires positioning fetched during the run, never from memory.
|
||||
|
||||
### Changed
|
||||
@@ -17,17 +436,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **SSH routing failures no longer present as "0 results"** — `search_youtube` surfaces non-zero SSH exit codes as an explicit `error` field ([#422](https://github.com/mvanhorn/last30days-skill/pull/422)).
|
||||
- `extract_browser_credentials()` silently ignored Brave even though the lower-level `cookie_extract` layer already supported it: `FROM_BROWSER=brave` fell back to Firefox/Safari and `FROM_BROWSER=auto` never tried Brave. The env wiring now passes Brave - and the rest of the Chromium family - through to the extractor ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
|
||||
- Chromium cookie extraction now fetches the macOS Keychain key lazily - only when an encrypted cookie actually needs decrypting. Previously the key was fetched as soon as the cookie DB existed, so `FROM_BROWSER=auto` could trigger a Keychain prompt for every installed Chromium browser. Now only the browser that actually holds the requested cookie prompts ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
|
||||
- YouTube transcript budget prioritises recent videos (by a combination of views and recency) instead of views alone, preventing transcript slots from being consumed by old high-view-count videos that would be discarded by strict_recent freshness pruning ([#531](https://github.com/mvanhorn/last30days-skill/issues/531))
|
||||
- YouTube items with successfully extracted transcripts are no longer pruned by title-only relevance scoring; the transcript content proves substantive topical coverage even when the video title has low lexical overlap with the query ([#468](https://github.com/mvanhorn/last30days-skill/issues/468))
|
||||
- First-run setup wizard in SKILL.md now references the existing Python setup wizard (`last30days.py setup`) instead of the missing `nux-wizard.md` file, so first-run setup actually runs on new installs. ([#574](https://github.com/mvanhorn/last30days-skill/issues/574))
|
||||
- `check-config.sh` no longer exits 1 on the ScrapeCreators-configured path when no prior run exists (empty `LAST_RUN_LINE`) — swapped `&&` guard for an `if` block that always exits cleanly ([#463](https://github.com/mvanhorn/last30days-skill/issues/463))
|
||||
- `check-config.sh` no longer exits 1 when a `.env` value contains an unbalanced quote — replaced `xargs` (which interprets quotes) with `sed` for whitespace trimming in `load_env_vars` ([#506](https://github.com/mvanhorn/last30days-skill/issues/506))
|
||||
- X/Twitter `.env` template now includes `CT0` alongside `AUTH_TOKEN` in the example skeleton ([CONFIGURATION.md](CONFIGURATION.md)), and the just-in-time unlock wizard offers AUTH_TOKEN/CT0 cookie entry ([#396](https://github.com/mvanhorn/last30days-skill/issues/396))
|
||||
- `check-config.sh` no longer counts X as an active source when only `AUTH_TOKEN` is set without `CT0` — both cookies are now required to credit X in the source count ([#396](https://github.com/mvanhorn/last30days-skill/issues/396))
|
||||
- Firefox cookie extraction now falls back to scanning non-default profiles when the default profile has no matching X cookies, fixing multi-profile setups where login lives on a non-default profile ([#498](https://github.com/mvanhorn/last30days-skill/issues/498))
|
||||
- `subproc.py` `run_with_timeout()` now guards `os.killpg` / `os.getpgid` with `hasattr`, preventing an uncaught `AttributeError` crash when a subprocess times out on Windows where these functions don't exist ([#527](https://github.com/mvanhorn/last30days-skill/issues/527))
|
||||
- Entity-grounding rerank demotion now keys on the head token of the primary entity instead of requiring the full multi-word phrase as a contiguous substring. A high-engagement on-entity item (e.g. a 323-pt HN thread titled "Stripe is friendly to 'friendly fraud'") is no longer demoted to score 0 on a `Stripe payments` query just because it lacks the trailing search-hint word. The intended demotion still fires for items that never name the brand at all. The keyless Reddit comment-enrichment slot selection (`_slot_priority`), which mirrors this signal, was updated to the same head-token grounding so the two paths stay consistent.
|
||||
- `--plan` / `--competitors-plan` file reads now specify `encoding="utf-8"` and catch `UnicodeDecodeError`, preventing crashes on non-ASCII content like accented entity names on Windows (cp1252). `check_perms()` in `check-config.sh` now skips the POSIX 600-permission check on MSYS/MinGW/Cygwin where `stat` runs in noacl mode. `skill_meta.py` `read_skill_version()` now passes `encoding="utf-8"` so SKILL.md emoji doesn't break version detection on Windows. ([#549](https://github.com/mvanhorn/last30days-skill/issues/549))
|
||||
|
||||
|
||||
## [3.3.2] - 2026-06-06
|
||||
|
||||
### Fixed
|
||||
|
||||
- YouTube transcript extraction now falls back through `en,es,pt` (configurable via `LAST30DAYS_YT_SUB_LANGS`) instead of English-only, so non-English videos with auto-captions in any of those three languages now contribute transcripts to the brief ([#469](https://github.com/mvanhorn/last30days-skill/issues/469))
|
||||
- Keyless Reddit comment enrichment now spends its limited slots on entity-matching posts first (mirroring rerank's entity-miss demotion signal) instead of raw upvote order, so off-topic high-upvote threads from broad subreddits no longer consume the comment budget only to be demoted afterward ([#484](https://github.com/mvanhorn/last30days-skill/pull/484))
|
||||
|
||||
## [3.3.1] - 2026-05-30
|
||||
|
||||
+44
@@ -39,3 +39,47 @@ The research flow available with no API keys: source data is gathered by scrapin
|
||||
### Comment-enrichment slots
|
||||
|
||||
The small, depth-dependent budget of Reddit posts whose comments get fetched in the Keyless path. Slot selection is relevance-aware: posts that pass Entity grounding claim slots first, so the budget is not spent on high-engagement posts that final ranking will demote anyway.
|
||||
|
||||
## Discovery
|
||||
|
||||
### Discovery
|
||||
|
||||
The topic-less research mode: instead of researching a named topic, it finds what is worth researching. On a reasoning-model host it runs as a three-leg host-judged protocol: leg 1 sweeps the river listings and writes a nominations bundle, the host judges every Nomination (name, junk, worthiness) into a judgments file, leg 2 resumes from the bundle and runs the Enrichment passes, and leg 3 applies host-written content angles and renders the brief. Headless/cron runs keep the one-shot form - same sweep and enrichment, deterministic heuristics in place of the judge, no angles. Either way every surviving topic must clear the Confidence floor before it is shown. Global Discovery (no domain given) sweeps every river feed's own hot list with no keyword gate; domain Discovery scopes and keyword-gates the sweep.
|
||||
|
||||
### Nomination
|
||||
|
||||
A named candidate topic produced by Discovery's listing sweep: clustered items from the river feeds, given a short searchable name plus a Junk shape flag and a content-worthiness score that blends into its seed rank. On protocol runs the hosting model judges all three via the judgments file - the engine's deterministic heuristics only fill rows the host left absent; on headless one-shot runs deterministic distillation supplies the name and junk flag and no worthiness signal exists. A Nomination is only a candidate - its blended seed rank decides which topics deserve an Enrichment pass and the display order of survivors; the Confidence floor judgment and the displayed velocity score are computed from the enriched evidence, never the seed score. The Nomination's name doubles as its Enrichment pass search query and its research handoff, so naming happens before enrichment, never at render time.
|
||||
|
||||
### Handoff checkpoint
|
||||
|
||||
The persisted state that lets Discovery's protocol pause for host judgment and resume in a later invocation: the nominations bundle (the full judge pool with its seed evidence, written by leg 1, awaiting the host's judgments) and the pending report (the enriched, floored, ranked round written by leg 2, awaiting the host's angles). A checkpoint is identity-bound - every host-written file must echo the checkpoint's bundle id, and a mismatch is a fix-the-id-and-retry error, never a redo of the expensive leg - and time-bound by its own TTL so a stale round is rejected rather than resumed. Checkpoints also carry provenance the resume legs enforce: mock and real state never cross, degraded sweep coverage survives into later legs instead of reading as clean, and an explicitly scoped store is the only place its checkpoints are looked for. Structurally empty checkpoint state is treated as corruption and fails closed - it never becomes an authoritative-looking empty result.
|
||||
|
||||
### Enrichment pass
|
||||
|
||||
A full research-pipeline run executed on one Nomination's topic name during Discovery. This is what gives a trend card the whole multi-source corpus (community comments, prediction markets, keyword-driven sources that have no hot-list of their own) instead of thin listing evidence. Enrichment passes run in parallel against a wall-clock budget; a pass that fails or outruns the budget downgrades its topic to nomination-only evidence, never fails the run.
|
||||
|
||||
### Confidence floor
|
||||
|
||||
The absolute evidence bar every Discovery topic must clear before it may rank: an engagement junk-gate first, then either independent cross-source corroboration or a genuinely strong single-source spike. Topics with a Junk shape get a stricter read: the single-source spike bypass is off, and their corroboration is counted against the seed listing sources the sweep actually found - never the enriched corpus, because an Enrichment pass makes almost any topic look multi-source. The floor is absolute, not relative to the current pool - a relative bar would degrade with the pool, which is the failure it exists to prevent. Its thresholds are deliberately tunable; the behavior contract is only that sub-floor evidence never ranks.
|
||||
|
||||
### Nothing-solid
|
||||
|
||||
The honest empty outcome of a Discovery run in which zero topics cleared the Confidence floor. A first-class result, not an error: the run reports that nothing in the window was strong enough to call a trend, and names the closest sub-floor candidate (the weak signal, preferring a non-junk-shaped one) so the user knows where the signal petered out. Rendering junk instead of Nothing-solid is the named failure this outcome replaced.
|
||||
|
||||
### Junk shape
|
||||
|
||||
A classification applied to a Nomination whose leading item reads as a help-me question, beginner ask, or personal musing rather than a story - the post shapes that engagement alone cannot distinguish from news. Its force depends on who flagged it: a host junk verdict is authoritative and excludes the Nomination from Enrichment passes outright (it can still appear as the weak signal in a Nothing-solid brief), while a heuristic junk shape on a row the host never judged only removes the Confidence floor's single-source bypass, so that topic surfaces solely with independent seed-source corroboration.
|
||||
|
||||
### Topic queue
|
||||
|
||||
The persistent memory of what Discovery has surfaced: each surfaced topic is recorded per research store, so later runs can annotate repeats ("surfaced Nth time") and the user can mark stories Covered. On by default for every real Discovery run, with an engine toggle to disable; mock runs never write it.
|
||||
|
||||
Identity in the queue is annotate-only: a new topic name that closely matches an earlier row (exact normalized match, else entity overlap) annotates the rendered card but never merges or rewrites rows - a false match costs one noisy line, never a hidden story. Queue annotations always describe the state before the current run, and a failed queue write degrades to a warning; it must never destroy a finished run's output.
|
||||
|
||||
### Covered
|
||||
|
||||
The user-set status on a Topic queue row meaning "I already produced content for this story." Set by marking a topic covered by its exact name; surfaced is the only other status. A resurfacing never un-covers a row, and a new name that fuzzily matches a Covered row is born Covered - so the mark survives the judge (now the hosting model) renaming the same story across runs instead of silently re-pitching it.
|
||||
|
||||
## Flagged ambiguities
|
||||
|
||||
- "Enrichment" is used for two distinct things: Comment-enrichment slots (fetching comments for already-ranked Reddit posts in the Keyless path) and Discovery's Enrichment pass (a full research run per Nomination). Context disambiguates; prefer the full term when writing.
|
||||
|
||||
+315
-21
@@ -27,48 +27,152 @@ This is a focused **configuration reference** maintained alongside the engine. T
|
||||
Each run produces one file per topic, slug-named:
|
||||
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
||||
|
||||
### Recommended `.env` entry
|
||||
|
||||
`.env` files don't travel between machines or harnesses, so set `LAST30DAYS_MEMORY_DIR` explicitly in `~/.config/last30days/.env` once per host. The `/last30days` slash command works without it (the SKILL.md wrapper has its own default), but **bare engine invocations** — `python3 scripts/last30days.py ...` from cron jobs, scripts, or agents that bypass the wrapper — silently no-op the file save unless the engine sees the env var. Mirrors the `LAST30DAYS_STORE` env-or-flag convention.
|
||||
|
||||
```bash
|
||||
# ~/.config/last30days/.env (pick ONE — uncomment the line that matches your OS)
|
||||
LAST30DAYS_MEMORY_DIR=~/Documents/Last30Days # POSIX — defaults to this path when unset
|
||||
# LAST30DAYS_MEMORY_DIR=C:\Users\<user>\Documents\Last30Days # Windows
|
||||
# LAST30DAYS_LIBRARY_OWNER=Your Name # Optional Atom feed author
|
||||
# LAST30DAYS_LIBRARY_CONTEXT=off # Disable prior-run context (default: on)
|
||||
```
|
||||
|
||||
The engine's `.env` reader doesn't expand `$HOME` — only the tilde, via `Path().expanduser()` downstream. Use `~/...` or an absolute path; **don't** write the literal string `$HOME/...` into your `.env` (it gets stored verbatim and breaks path resolution).
|
||||
|
||||
**Per-run overrides:**
|
||||
- `--save-dir <path>` - one-off output location.
|
||||
|
||||
- `--save-dir <path>` - one-off output location. **Flag wins over env var.** If neither flag nor env var is set, the engine does not write a file (DB persistence is independent — see `LAST30DAYS_STORE` below).
|
||||
- `--output <file>` - write the rendered output to an exact file path, using the format selected by `--emit`.
|
||||
- `--json-profile {agent,raw}` - select the research JSON shape used with `--emit=json`. `agent` is the default, versioned workflow contract; `raw` preserves the full internal `Report` dump for debugging and power users. See the [JSON export reference](docs/reference/json-export.md).
|
||||
- `--corpus <dir>` - add a local `.md`/`.txt` directory as a private ranked source; repeat the flag for multiple directories. PDFs are extracted only when `pdftotext` is on PATH and otherwise skip with a note. File modification time supplies recency, so the normal research window applies.
|
||||
- `--corpus-all-time` - include relevant registered files whose modification time is older than the current research window. Without this flag, a 30-day run includes only files modified in those 30 days.
|
||||
- `--register {default,exec,dev,creator,eli5}` - shape a standard single-topic Markdown or HTML research brief for its audience. `exec` is decisions-first with five core findings and numbers up top; `dev` gives GitHub, code, and technical signals more room; `creator` leads with hooks, Best Takes, community reactions, and virality metrics; `eli5` keeps the established evidence layout and asks the synthesizing agent for accessible language. Registers do not change retrieval, JSON exports, discovery, drill, library feed/search, or comparison output.
|
||||
- `--discover [domain]` - trending discovery, two-stage: a river-listing sweep NOMINATES candidate topics, then each nomination gets a full research pass (Reddit with comments, X, YouTube, Techmeme, arXiv, HN, Polymarket, web) before ranking. Bare `--discover` (no domain) is **global trending**: every feed's own hot list (r/all rising/top-week, Hacker News front/best, Digg clusters when `digg-pp-cli` is on PATH) with no keyword gate; with a domain, the sweep is category-scoped and keyword-gated, and broad X activity joins when an X backend is authenticated. Every topic must clear a confidence floor (cross-source confirmation or a genuinely strong single-source spike); when nothing clears it the run reports "Nothing solid this window" instead of ranked noise. Run without a positional topic; it is mutually exclusive with `--drill`. `--emit=json` uses the separate versioned discovery contract (now with `outcome`, `weak_signal`, per-topic `top_comment` and `corroboration_count`) documented in the [JSON export reference](docs/reference/json-export.md).
|
||||
- `--discover-shallow` - skip discovery's per-topic research passes and rank on listing evidence only. Faster and thinner; the confidence floor still applies. An explicit `--search` source list bounds both the sweep and the research passes. On a protocol run (below), adding it to the `--nominate-only` leg marks the bundle quick-tier so the resume leg uses the faster shallow research pass.
|
||||
- `--nominate-only` - leg 1 of the three-command host-judged discovery protocol (agent hosts; SKILL.md drives it - one-shot `--discover` stays the scripting/cron form with deterministic topic names and no angles). With `--discover [domain]`: sweep the listings, write the nominations bundle (`discover-nominations.json` in the save dir, TTL one hour) for host judgment, print a judging digest, and stop - no enrichment, no queue writes. A zero-nomination sweep prints the nothing-solid brief directly.
|
||||
- `--judgments <path>` - leg 2: resume from the nominations bundle, applying the host judgments file (`{"bundle_id": "...", "judgments": [{"id", "name", "junk", "worthiness"}, ...]}`, bound to the bundle by `bundle_id`). Runs the per-topic research passes (deep tier by default; budget tunable via `LAST30DAYS_ENRICH_BUDGET_SECONDS` below), writes the pending report (`discover-pending.json`), and prints per-topic angle inputs. Requires `--discover`.
|
||||
- `--finalize` - leg 3: apply optional host angles to the pending report, render the final discovery brief, save artifacts, and record the topic queue (retries are idempotent - the pending file stays in place within its TTL). Offline; requires `--discover`.
|
||||
- `--angles <path>` - optional host angles file for `--discover --finalize` (`{"bundle_id": "...", "angles": [{"id", "podcast", "x_article"}, ...]}`, sentences capped at 200 chars); omitting it ships the brief without angle lines. All three protocol legs must share one `--save-dir` (handoff files live there, else in `~/.config/last30days/`); contract failures (missing/stale/unbound handoff files) exit 2 with the remedy on stderr, and `--mock` protocol legs require `--save-dir` to stay side-effect-free.
|
||||
- `--drill <target>` - deep follow-up over the fresh `~/.config/last30days/last-report.json` cache. Accepts a 1-based index (`--drill "cluster 3"` or `--drill "3"`) or a fuzzy cluster title/entity description. It re-fetches only sources that contributed to the matched cluster, enables their deep comment/transcript enrichment paths, merges/dedupes the evidence, and replaces the cache so drills can chain. Run it without a positional topic; if the cache is absent or expired, run a normal research pass first.
|
||||
- `--verify-freshness` - opt into an act-time verification pass for conservatively extracted, source-grounded claims (Polymarket odds/end dates, GitHub stars, StockTwits sentiment ratios, and explicit status assertions). With a topic, verification runs after research; without a topic, it re-verifies the fresh `last-report.json` cache without repeating research. Verdicts are `current`, `stale`, `contradicted`, or `unsupported` and include evidence timestamps. Set `LAST30DAYS_VERIFY_FRESHNESS=on` in `.env` to make the pass default for normal research runs.
|
||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||
- `--no-browser-cookies` - hard-disable browser-cookie extraction for this run, even when `FROM_BROWSER` is configured. MCP and folder-mode hosts use this for safe defaults.
|
||||
- `--publish-html` - with `--emit=html`, publish the rendered HTML to `ht-ml.app` after local output/save-dir writes. This is explicit opt-in only; pages are public by default.
|
||||
- `library feed` - scan `LAST30DAYS_MEMORY_DIR` plus `~/.local/share/last30days/briefs/`, then write a self-contained `index.html`, valid Atom `feed.xml`, and browser-ready pages under `briefs/`. The index is reverse-chronological and grouped by topic. For direct engine use: `python3 skills/last30days/scripts/last30days.py library feed`; use `--save-dir <path>` to scan and write another library directory.
|
||||
- `library feed --publish` - publish each rendered brief and the HTML index through `ht-ml.app`. The generated `feed.xml` remains a first-class local artifact because this HTML host does not serve Atom with an XML content type. Host the output directory on any static host (for example, GitHub Pages) to make `feed.xml` subscribable. Publishing is explicit opt-in and pages are public by default; public pages may be crawled or indexed.
|
||||
- `library search "<query>"` - incrementally sync `LAST30DAYS_MEMORY_DIR` and `~/.local/share/last30days/briefs/` through the shared library scanner, then run offline SQLite FTS5 across those briefs plus dated per-run sightings in `~/.local/share/last30days/research.db`. Results are grouped by topic run. The sibling search index lives at `~/.local/share/last30days/library.db`; hand edits, renames, and deletes are picked up on sync, and a corrupt index is rebuilt automatically.
|
||||
- `LAST30DAYS_LIBRARY_OWNER=<name>` - optional feed-level Atom author. Defaults to `last30days research library`.
|
||||
- `LAST30DAYS_LIBRARY_CONTEXT=on|off` - controls passive prior-run context on fresh research reports. It defaults to `on`; matching saved research appears in a short `From your library` section. Set `off` to skip the local index read and leave reports unchanged. Mock runs, eval replays, and internal fan-out subruns do not load library context, keeping fixtures deterministic.
|
||||
- `--publish-password <password>` - optional shared password for `--publish-html` or `library feed --publish`. Prefer `LAST30DAYS_PUBLISH_PASSWORD=<password>` instead so the password is not visible in the process list or shell history. Use a unique non-personal password; never reuse the user's own password. The provider's update key is treated as secret and is not written to stdout, HTML, raw output, or `.publish.json` metadata.
|
||||
- `--preflight` - print a human-readable permission preflight. It reports config source, project config trust/ignore state, browser-cookie plan, planned writes, optional commands, source availability, and endpoint overrides without reading browser cookies, writing setup/config/report files, or running research. Add `--emit=json` for the separate machine-readable preflight contract (`--json-profile` does not change it); use `--diagnose` when you need the full source diagnostic JSON.
|
||||
- `--welcome` - print the first-run welcome text (engine-owned; the skill relays it verbatim on first run). Safe: prints and exits, no reads or writes.
|
||||
- `--record-fixtures <dir>` - developer-only, hidden flag that records scrubbed source responses for the offline research-quality eval harness. It writes `<dir>/http.json`; see the [eval reference](docs/reference/eval.md) before recording or committing fixtures.
|
||||
- `setup --github-start` / `setup --github-poll` - the two-command ScrapeCreators GitHub device-auth split. `--github-start` submits the device flow, copies the code to the clipboard, opens the browser, and returns the code immediately (foreground); `--github-poll` waits for you to authorize and persists the key. `setup --github` still runs both in one shot for back-compat.
|
||||
|
||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||
|
||||
Every completed research pass writes a structured `last-report.json` cache beside `last-run.json`. HTML follow-up renders use it so `--emit=html --synthesis-file` can reuse report metadata/footer without fetching sources again; `--drill <target>` uses it as the grounded starting point for targeted re-research; bare `--verify-freshness` updates only the cached report's claim verdicts. Reuse is intentionally short-lived: `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` defaults to `3600` (one hour). Set it to another integer number of seconds to tune the window, or `0` to disable report-cache reuse and post-run follow-ups.
|
||||
|
||||
---
|
||||
|
||||
## First-run onboarding
|
||||
|
||||
On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It takes one of two forms depending on the host:
|
||||
|
||||
- **Claude Code Modal Flow** - the restored v3.0.0 guided NUX, used on hosts with `AskUserQuestion` (Claude Code). A welcome message, then modals for Auto/Manual/Skip setup, cookie consent, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker.
|
||||
- **Non-Modal Prose Flow** - the same work done conversationally on hosts without modals (OpenClaw, Codex, Cursor, Gemini CLI, Grok, raw CLI).
|
||||
|
||||
Both share the same consent points:
|
||||
|
||||
1. **Browser cookies** - the model asks before reading anything. On yes it runs `setup --allow-browser-cookies`, which extracts Firefox/Safari cookies (never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser is explicitly configured) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup without `--allow-browser-cookies` (or with `FROM_BROWSER=off`), which skips all cookie reads and still installs the tools.
|
||||
2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry.
|
||||
3. **ScrapeCreators GitHub signup** - offered on every first run (10,000 free calls). On consent it runs `setup --github`, which opens a browser for GitHub device-auth (or registers instantly via the `gh` CLI when installed) and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, empty-path Reddit search backup, and the YouTube transcript fallback activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. The Step 5 opt-in has two tiers, both comment-enabled: **Recommended** (TikTok + Instagram posts AND top comments, plus YouTube comments — `INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`) and **Everything**, which also adds Threads + Pinterest. Comments are on by default; Threads and Pinterest are the only opt-in extras.
|
||||
|
||||
Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation and both host flows are specified in `skills/last30days/SKILL.md` Step 0. The original v3.0.0 wizard is captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
|
||||
---
|
||||
|
||||
## API keys (`.env`)
|
||||
|
||||
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
||||
The skill reads keys from a `.env` file. Two locations are supported:
|
||||
|
||||
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
||||
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
||||
1. **`~/.config/last30days/.env`** at the user level (global default) - loaded by default.
|
||||
2. **`.claude/last30days.env`** in the current project directory (project-scoped) - loaded only when trusted by setting `LAST30DAYS_TRUST_PROJECT_CONFIG=1` in the process environment or global config.
|
||||
|
||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||
|
||||
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
||||
The project-scoped file is useful for **intentional per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), then opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or `~/.config/last30days/.env`. Folder-mode hosts such as Codex desktop do not trust hidden project config by default, and discovery stops at the git root so unrelated parent folders cannot silently influence runs. The SessionStart status hook (`hooks/scripts/check-config.sh`) uses the same trust gate — an untrusted repo's `.claude/last30days.env` is not read at session start.
|
||||
|
||||
**`LAST30DAYS_API_KEY`** + **`LAST30DAYS_API_BASE`** - optional remote-API backend. Set BOTH to route research through a remote API endpoint instead of running the local sources: `LAST30DAYS_API_BASE` is the endpoint (there is no built-in default), and `LAST30DAYS_API_KEY` is the bearer key for it. When both are set (and `--mock` is not passed), the engine submits the topic to that endpoint, polls with progress on stderr, and prints the server's report; none of the per-source keys below are used for that run. A configured local corpus is the privacy exception: the engine bypasses the hosted backend and runs locally rather than forwarding file-derived input. Non-default `--register` selections are forwarded with the request so server-side synthesis uses the same audience preset. Leave either unset to run local sources exactly as normal. Unlike the other keys here, these two are read only from the **process environment** (export them in your shell or host config) - they are deliberately not loaded from the `.env` files above, so a project-scoped `.env` can never silently redirect research to a remote endpoint. The remote endpoint does not return the local `Report` needed for the versioned agent JSON profile; use `--emit=json --json-profile=raw` for its existing server-response JSON contract.
|
||||
|
||||
**`BRIGHTDATA_API_KEY`** - optional, for the `amazon` source. The Bright Data CLI normally owns its own auth via `brightdata login`, so this is only needed if you would rather keep an explicit key in `.env` or the keychain. It is resolved through the standard config layering and passed to the CLI through the child process environment, never on the command line (where it would be readable from `/proc/<pid>/cmdline` by other local users on a shared host).
|
||||
|
||||
**`LAST30DAYS_AMAZON_DOMAIN`** - optional, default `https://www.amazon.com`. The marketplace the `amazon` source searches; set it to `https://www.amazon.co.uk`, `https://www.amazon.de`, and so on. Product URLs are validated against this host, so records from other marketplaces are rejected.
|
||||
|
||||
### Local corpus (your files)
|
||||
|
||||
Register persistent directories with `LAST30DAYS_CORPUS_DIRS`. Separate paths with `:` on macOS/Linux (the platform path separator is `;` on Windows):
|
||||
|
||||
```bash
|
||||
# ~/.config/last30days/.env
|
||||
LAST30DAYS_CORPUS_DIRS=~/notes:~/meeting-transcripts
|
||||
# LAST30DAYS_CORPUS_IN_EXPORT=1 # explicit agent-JSON opt-in; off by default
|
||||
```
|
||||
|
||||
The slash-command experience remains primary: ask `/last30days` to include your registered notes. For direct engine scripting or development, the equivalent one-off invocation is:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "MCP servers" \
|
||||
--corpus ~/notes --corpus ~/meeting-transcripts
|
||||
```
|
||||
|
||||
**Privacy:** corpus files are read locally, never sent through a source HTTP client, never forwarded to `LAST30DAYS_API_BASE`, never included in remote reranker/fun-scoring prompts, and do not consume network-source concurrency or retry budget. Matches appear in a badged **From your files** section. Corpus candidates are removed from `--publish-html`, `library feed --publish`, and the versioned agent JSON export by default, including corpus-derived cluster titles and source outcomes. Set `LAST30DAYS_CORPUS_IN_EXPORT=1` only when you intentionally want corpus results in the agent JSON written to local stdout/files. The unversioned `--json-profile=raw` debug dump remains a full local report and can contain corpus text; do not redirect it to an external system unless that is intentional. Extracted text is cached by file mtime in `~/.config/last30days/corpus-cache.json` with mode `0600`; a corpus-bearing `last-report.json` cache is also tightened to `0600`. Delete either cache at any time to clear it.
|
||||
|
||||
**Source-by-source** - what each key unlocks:
|
||||
|
||||
| Source | Key(s) | Required for | Free tier |
|
||||
|---|---|---|---|
|
||||
| Reddit (public) | none | always on | yes |
|
||||
| Local corpus | `--corpus <dir>` or `LAST30DAYS_CORPUS_DIRS` | private `.md`/`.txt`; `.pdf` when `pdftotext` is on PATH | yes (offline) |
|
||||
| Reddit (public) | none (default free keyless path). With `SCRAPECREATORS_API_KEY`: empty-only search backup by default; `LAST30DAYS_REDDIT_SC_MIN_ITEMS=<N>` backfills thin free runs; `LAST30DAYS_REDDIT_BACKEND=scrapecreators` pins SC primary with free fallback | always on; SC knobs require `SCRAPECREATORS_API_KEY` | yes |
|
||||
| Hacker News | none | always on | yes |
|
||||
| Polymarket | none | always on | yes |
|
||||
| StockTwits | none | auto-on for ticker/crypto topics only (gated by symbol detection); never registered for non-financial topics | yes (public API, ~200 req/hr per IP) |
|
||||
| DripStack | none | opt-in only: per run with `--search dripstack`, or persistently with `INCLUDE_SOURCES=dripstack` in `.env`. Searches premium financial newsletters and analyst writeups via a free, public search API — no key needed. Never active without the opt-in. | yes when opted in (public API, no auth) |
|
||||
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
||||
| YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes |
|
||||
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid |
|
||||
| YouTube | `yt-dlp` CLI installed; `SCRAPECREATORS_API_KEY` adds a server-side transcript fallback used only when yt-dlp fails (429 / bot-gate) | always on if `yt-dlp` present; SC transcript fallback default-on when key set (no credit spent unless yt-dlp fails) | yes |
|
||||
| YouTube comments | `yt-dlp` CLI installed — **free and keyless, no API key and no opt-in needed**. Falls back to `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` containing `youtube_comments` only when yt-dlp is absent. Suppress with `EXCLUDE_SOURCES=youtube_comments`. | top comments (by likes) on the top ~3 videos by engagement | yes — free via yt-dlp (no credits spent) |
|
||||
| TikTok comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok_comments` (**on by default** — Step 5 Recommended tier) | top comments (by `digg_count`) on the top ~3 TikTok posts | ~3 calls/run; 10K free calls |
|
||||
| Instagram comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram_comments` (**on by default** — Step 5 Recommended tier) | top comments (by `comment_like_count`) on the top ~3 Instagram posts, via `/v2/instagram/post/comments` | ~3 calls/run; 10K free calls |
|
||||
| Digg | `digg-pp-cli` on PATH (auto-installed during first-run setup via `npx -y @mvanhorn/printing-press-library@0.1.16 install digg --cli-only`; binary defaults to `$HOME/.local/bin` — Hermes/OpenClaw agent subprocesses must inherit that dir on PATH for Digg to activate; prior pp-digg installs use the same path) | always on if `digg-pp-cli` on PATH | yes (free, keyless, read-only) |
|
||||
| arXiv | `arxiv-pp-cli` on PATH (auto-installed during first-run setup via `npx -y @mvanhorn/printing-press-library@0.1.16 install arxiv --cli-only`) | always on if `arxiv-pp-cli` on PATH; fires on research/technical topics and stays quiet otherwise (relevance + 365-day recency gating) | yes (free, keyless) |
|
||||
| Techmeme | `techmeme-pp-cli` on PATH (auto-installed via `... install techmeme --cli-only`) | always on if `techmeme-pp-cli` on PATH; searches Techmeme's live archive and keeps only headlines dated within the research window (undated headlines flow through as low-confidence) | yes (free, keyless) |
|
||||
| Trustpilot | `trustpilot-pp-cli` on PATH (NOT auto-installed; install on demand via `npx -y @mvanhorn/printing-press-library@0.1.16 install trustpilot --cli-only`) + (`INCLUDE_SOURCES` contains `trustpilot` **or** an explicit `--trustpilot-domain` / plan-level `trustpilot_domain`) | **opt-in, off by default**; `--trustpilot-domain=<domain>` (and per-entity `trustpilot_domain` in `--competitors-plan`) auto-activates the source for that run and bypasses the brand-shape gate. Persist with `INCLUDE_SOURCES=trustpilot` to skip per-run auto-enable. `EXCLUDE_SOURCES=trustpilot` still wins. Bare company names auto-resolve to the review-page domain via the CLI's search only when the source is already active. The session warms once before the search fan-out; a stale session does a ~10s headless-Chrome WAF-cookie harvest (set `LAST30DAYS_TRUSTPILOT_NO_BROWSER=1` to disable in cron/CI) | yes (no API key; cookie-replay after the one-time harvest) |
|
||||
| Amazon | `brightdata` CLI on PATH **and logged in** (NOT auto-installed: `npm i -g @brightdata/cli` then `brightdata login`) + (`INCLUDE_SOURCES` contains `amazon` **or** `--search` includes `amazon`) | product records with live rating, rating count, and price, plus a capped sample of recent written reviews woven as buyer voice; the emoji footer shows each product's all-time-vs-last-30-days drift | **opt-in, off by default**. Free tier is 5,000 requests/month (~$7.50 equivalent); a typical run spends 4 (1 product search + up to 3 review pulls) regardless of how many reviews come back, since billing is per request. Past the free tier it bills the account balance at $1.50 per 1,000 records (~$0.32 for a default run). `--amazon-query=<keyword>` sets the product keyword when it differs from the topic; `LAST30DAYS_AMAZON_DOMAIN` selects a non-US marketplace. `EXCLUDE_SOURCES=amazon` wins. Never auto-fires: the model requests it per run or the user enables it durably |
|
||||
| X / Twitter | one of: a signed-in `grok` CLI (no X credential), `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `XQUIK_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | grok = no X credential, draws on your Grok plan; cookie-jar / Bird = free; Xquik / xAI / ScrapeCreators = key-based |
|
||||
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
||||
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
||||
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
||||
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
||||
| LinkedIn | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `linkedin` | LinkedIn posts + articles (articles rank as high signal on person topics) | 10K free calls; power-user opt-in, not offered during first-run onboarding |
|
||||
| Xiaohongshu (RED) | logged-in x-mcp browser plugin or `xiaohongshu-mcp` service; optional `XIAOHONGSHU_API_BASE` for custom URLs | requested-only via `--search xhs` or `--search xiaohongshu`; auto-probes `http://localhost:18060` then `http://host.docker.internal:18060` | no last30days API key; depends on your local browser-session service |
|
||||
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
||||
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
||||
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
||||
| Perplexity Deep Research | `OPENROUTER_API_KEY` | `--deep-research` flag (~$0.90/query) | no |
|
||||
| Perplexity Sonar / Search API / Deep Research | `PERPLEXITY_API_KEY` (preferred) or `OPENROUTER_API_KEY` (Sonar fallback) | `INCLUDE_SOURCES=perplexity`; `--deep-research` flag (~$0.90/query) | no |
|
||||
| Caption-free transcription | `GROQ_API_KEY` (free tier, preferred) or `OPENAI_API_KEY` (paid backstop); requires `ffmpeg` | Whisper transcription for audio/video without captions (groundwork: module shipped, not yet auto-invoked by the engine) | Groq free tier is generous; needs ffmpeg installed |
|
||||
| Jobs / careers pages | none for public ATS pages; web backend improves fallback discovery | `--hiring-signals` and strong Hiring Signals in standard company reports | yes |
|
||||
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
||||
|
||||
**YouTube transcript tuning.** `LAST30DAYS_YT_SUB_LANGS` controls the comma-separated caption-language priority passed to yt-dlp and defaults to `en,es,pt`. When `SCRAPECREATORS_API_KEY` is available, yt-dlp uses one fast attempt before the paid fallback; set `LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT` to the number of seconds allowed for that attempt when a throttled host needs longer than the 12-second default. A VTT completed before the timeout is reused rather than discarded. `LAST30DAYS_YT_SEARCH_TIMEOUT` sets the per-search yt-dlp deadline (default 120s). Comparison-mode fan-out also caps concurrent yt-dlp processes process-wide and caches identical searches within a run so redundant `ytsearch` calls do not self-throttle the same IP.
|
||||
|
||||
**X backend priority (bird first).** The default X backend chain is bird (browser cookies) → xai (API key) → xurl (OAuth2 CLI) → xquik (API key). Cookies beat `XAI_API_KEY` when both are present. A leftover grok login never steals the X lane; see below.
|
||||
|
||||
**Grok CLI (opt-in backup).** Install the Grok CLI (`curl -fsSL https://x.ai/cli/install.sh | bash`) and run `grok login`, and X can work with no X account, no browser cookies, and no `XAI_API_KEY`. However, grok is **opt-in only**: a leftover `~/.grok/auth.json` must never steal the X lane. Pin `LAST30DAYS_X_BACKEND=grok` to enable it. It is not "free" in the way the cookie path is: calls draw on your Grok plan, and depth costs several calls per run because the underlying tool caps each search at 10 posts. Results are validated before use — every returned post's ID is decoded to confirm it falls inside the requested date range, because the retrieval is performed by a language model and can otherwise return confident, well-formed posts that were never searched for.
|
||||
|
||||
**X on cookie-less hosts.** Bird (the free X source) scrapes X using your logged-in browser cookies (`AUTH_TOKEN`/`CT0`), which agent hosts like OpenClaw, CI, or headless runs often can't supply — and scraping carries some account risk. On those, set `XQUIK_API_KEY` (or `XAI_API_KEY`) for full, ranked X coverage from a single API key: the same engagement-based ranking, first-party authorship, and handle (from/mentions) lanes the native X source gets. `--diagnose` reports whether the key is working (and flags an unpaid key).
|
||||
|
||||
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
||||
|
||||
```bash
|
||||
@@ -81,10 +185,28 @@ BRAVE_API_KEY=<your-brave-key>
|
||||
# Optional sources
|
||||
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
# Xiaohongshu is requested-only: run with --search xhs after starting a local
|
||||
# browser-session service. Defaults probe localhost, then host.docker.internal.
|
||||
# XIAOHONGSHU_API_BASE=http://localhost:18060
|
||||
# Add perplexity to INCLUDE_SOURCES when you want the paid Perplexity source.
|
||||
# PERPLEXITY_API_KEY=<your-perplexity-key>
|
||||
# INCLUDE_SOURCES=tiktok,instagram,perplexity
|
||||
# LAST30DAYS_PERPLEXITY_MODE=sonar # sonar | search | both
|
||||
# LAST30DAYS_PERPLEXITY_MODEL=sonar-pro # sonar | sonar-pro | sonar-reasoning-pro
|
||||
|
||||
# X authentication (one option only)
|
||||
XAI_API_KEY=<your-xai-key>
|
||||
# OR cookie-jar (no key needed; logs in via your browser session)
|
||||
AUTH_TOKEN=<your-auth-token>
|
||||
CT0=<your-ct0-token>
|
||||
# OR xAI API key (paid)
|
||||
# XAI_API_KEY=<your-xai-key>
|
||||
# OR Xquik key-based X search
|
||||
# XQUIK_API_KEY=<your-xquik-key>
|
||||
# OR cookie-jar (free; logs in via your browser session).
|
||||
# Unset = no browser-cookie reads. FROM_BROWSER=auto tries Firefox/Safari and
|
||||
# the Chromium family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it
|
||||
# only prompts for macOS Keychain access on the browser that actually holds your
|
||||
# X cookies. Or name a single browser, e.g. brave/edge. On Windows only Firefox
|
||||
# is supported.
|
||||
# FROM_BROWSER=firefox
|
||||
|
||||
# Bluesky
|
||||
@@ -94,7 +216,37 @@ BSKY_APP_PASSWORD=<your-app-password>
|
||||
|
||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --preflight` for a human permission summary or `python3 scripts/last30days.py --diagnose` for full JSON diagnostics. Both are safe: they report source availability, config source, browser-cookie plan, external command availability, write destinations, and ignored untrusted project config without reading browser cookies or running live provider probes.
|
||||
|
||||
### Perplexity source modes
|
||||
|
||||
Perplexity is a paid opt-in source. A direct `PERPLEXITY_API_KEY` unlocks first-party Perplexity features. `OPENROUTER_API_KEY` remains a Sonar compatibility fallback only; Perplexity Search API and async Deep Research call Perplexity directly.
|
||||
|
||||
`LAST30DAYS_PERPLEXITY_MODE` controls normal `perplexity` source runs:
|
||||
|
||||
| Value | Behavior | Calls |
|
||||
|---|---|---|
|
||||
| `sonar` (default) | Sonar synthesis plus citations. | one Sonar call |
|
||||
| `search` | Raw ranked Search API rows; best when you want source aggregation over prose. | one Search API call |
|
||||
| `both` | Sonar synthesis plus raw ranked Search API rows, deduped by URL. | one Search API call and one Sonar call |
|
||||
|
||||
`--deep-research` ignores `LAST30DAYS_PERPLEXITY_MODE` and uses `sonar-deep-research`. With `PERPLEXITY_API_KEY`, it submits to Perplexity's async Sonar endpoint and polls with a hard wall-clock timeout. The async request uses a deterministic idempotency key derived from the request body. If the request is still running at timeout, fails remotely, or polling hits a transport/rate-limit error after the async id exists, the raw artifact records the async request id, idempotency key, last status, lifecycle timestamps returned by Perplexity, poll count, and timeout/error fields so you can inspect or resume by id outside the run. With only `OPENROUTER_API_KEY`, it keeps the OpenRouter synchronous fallback.
|
||||
|
||||
Perplexity-specific env vars:
|
||||
|
||||
| Env var | Default | Applies to | Notes |
|
||||
|---|---|---|---|
|
||||
| `LAST30DAYS_PERPLEXITY_MODE` | `sonar` | normal Perplexity source runs | `sonar`, `search`, or `both`; `search` and `both` require `PERPLEXITY_API_KEY`. |
|
||||
| `LAST30DAYS_PERPLEXITY_MODEL` | `sonar-pro` | direct Sonar only | Supported: `sonar`, `sonar-pro`, `sonar-reasoning-pro`. `--deep-research` forces `sonar-deep-research`. |
|
||||
| `LAST30DAYS_PERPLEXITY_MAX_RESULTS` | `10` | Search API | Clamped to Perplexity's 1..20 range. |
|
||||
| `LAST30DAYS_PERPLEXITY_SEARCH_CONTEXT_SIZE` | provider default | Search API | `low`, `medium`, or `high`; omitted unless set. |
|
||||
| `LAST30DAYS_PERPLEXITY_SEARCH_MODE` | provider default | direct Sonar | `web`, `academic`, or `sec`. |
|
||||
| `LAST30DAYS_PERPLEXITY_DOMAIN_FILTER` | unset | Search API and direct Sonar | Comma-separated domains, max 20. |
|
||||
| `LAST30DAYS_PERPLEXITY_LANGUAGE_FILTER` | unset | Search API and direct Sonar | Comma-separated ISO 639-1 language codes, max 20. |
|
||||
| `LAST30DAYS_PERPLEXITY_COUNTRY` | unset | Search API | Two-letter country code such as `US`. |
|
||||
| `LAST30DAYS_PERPLEXITY_RECENCY_FILTER` | unset | Search API and direct Sonar | `hour`, `day`, `week`, `month`, or `year`. |
|
||||
| `LAST30DAYS_PERPLEXITY_REASONING_EFFORT` | unset | direct Sonar | `minimal`, `low`, `medium`, or `high`. |
|
||||
| `LAST30DAYS_PERPLEXITY_DEEP_TIMEOUT_SECONDS` | `600` | direct async Deep Research | Wall-clock polling deadline. |
|
||||
|
||||
### Encrypted credential sources (Keychain / pass)
|
||||
|
||||
@@ -106,6 +258,13 @@ to disk, never logged). Both are **lowest-priority and additive** — an explici
|
||||
sources, so a box that merely has `pass` installed pays no decrypt cost when
|
||||
everything is already in `.env`.
|
||||
|
||||
Effective credential priority is: process env > trusted project config
|
||||
(`.claude/last30days.env`) > global config (`~/.config/last30days/.env`) >
|
||||
macOS Keychain > `pass`(1). The SessionStart status hook also checks for
|
||||
Keychain item **presence** under `last30days-<KEY>` without reading secret
|
||||
values, so a Keychain-only setup is treated as configured instead of showing the
|
||||
first-run welcome again.
|
||||
|
||||
| Platform | Source | Store keys with | Lookup convention |
|
||||
|---|---|---|---|
|
||||
| macOS | Keychain | `scripts/setup-keychain.sh` | service name `last30days-<KEY>` |
|
||||
@@ -131,6 +290,35 @@ export LAST30DAYS_PASS_PREFIX="secrets/last30days/" # default: last30days/
|
||||
|
||||
Both sources cover the same key set as the `.env` skeleton above.
|
||||
|
||||
#### Reusing existing macOS Keychain items
|
||||
|
||||
If you already have keys stored under another Keychain naming convention, you
|
||||
can reference them without copying the secret by setting non-secret alias
|
||||
metadata in `LAST30DAYS_KEYCHAIN_ALIASES`. The loader still checks
|
||||
`last30days-<KEY>` first; aliases are fallback lookups only.
|
||||
|
||||
```bash
|
||||
# ~/.config/last30days/.env
|
||||
LAST30DAYS_KEYCHAIN_ALIASES={"XAI_API_KEY":{"account":"keychain-user","service":"existing-xai-api-key"},"BRAVE_API_KEY":"existing-brave-api-key"}
|
||||
```
|
||||
|
||||
Each JSON key must be one of the supported env-var names (`XAI_API_KEY`,
|
||||
`SCRAPECREATORS_API_KEY`, `BRAVE_API_KEY`, etc). A string value means "use this
|
||||
service name with the current user account"; an object can specify both
|
||||
`account` and `service`. Lists are allowed for fallback order:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_KEYCHAIN_ALIASES={"XAI_API_KEY":[{"account":"keychain-user","service":"existing-xai-api-key"},{"service":"last-resort-xai"}]}
|
||||
```
|
||||
|
||||
The alias value contains no secret material; it is safe to keep in `.env` as
|
||||
configuration. The secret itself remains in its original Keychain item and is
|
||||
read directly by the engine process.
|
||||
|
||||
Write `LAST30DAYS_KEYCHAIN_ALIASES` as a single-line JSON value in `.env`.
|
||||
Multiline JSON formatting is not supported because `.env` files are parsed
|
||||
line-by-line.
|
||||
|
||||
### Bluesky app-password format and search host
|
||||
|
||||
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
||||
@@ -141,6 +329,26 @@ The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical a
|
||||
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||
```
|
||||
|
||||
### Default source set (`LAST30DAYS_DEFAULT_SEARCH`)
|
||||
|
||||
By default the engine decides the source set per query (everything available, minus `EXCLUDE_SOURCES`). To pin a **fixed** source set for every run without passing `--search` each time — and without patching `SKILL.md`, which a release would overwrite — set:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_DEFAULT_SEARCH=reddit,x,youtube,hn
|
||||
```
|
||||
|
||||
Accepts the same comma-separated names and aliases as `--search` (`web` → grounding, `hn` → hackernews, `bsky` → bluesky, `xhs` → xiaohongshu). Precedence: an explicit `--search` on the command line always wins; `LAST30DAYS_DEFAULT_SEARCH` applies only when the flag is omitted; when neither is set, per-query behavior is unchanged. `INCLUDE_SOURCES` / `EXCLUDE_SOURCES` keep their existing additive/subtractive roles on whichever set is selected.
|
||||
|
||||
### Audience register (`LAST30DAYS_REGISTER`)
|
||||
|
||||
The default standard brief stays balanced and byte-compatible with prior releases. To keep a named audience preset across runs, set one of the supported values:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_REGISTER=exec # default | exec | dev | creator | eli5
|
||||
```
|
||||
|
||||
An explicit `--register` wins over `LAST30DAYS_REGISTER`; the environment/config value defaults to `default`. Presets are intentionally named and bounded - arbitrary prompt or template files are not accepted. Existing `ELI5_MODE=true` configurations continue to resolve to the `eli5` register when no explicit register is selected, but new configuration should use `LAST30DAYS_REGISTER=eli5`.
|
||||
|
||||
---
|
||||
|
||||
## Reasoning provider priority
|
||||
@@ -148,9 +356,9 @@ BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||
|
||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
||||
2. **OpenAI** - `OPENAI_API_KEY` only. Codex ChatGPT auth at `~/.codex/auth.json` is intentionally not used as an OpenAI provider credential.
|
||||
3. **xAI** - `XAI_API_KEY`
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (also unlocks `--deep-research`)
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (Sonar fallback for the Perplexity source / `--deep-research`; also usable as a reasoning provider)
|
||||
5. **Local / deterministic** - always available, lowest quality
|
||||
|
||||
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
||||
@@ -161,16 +369,17 @@ When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model
|
||||
|
||||
The search-source preference ladder, strict best-to-floor:
|
||||
|
||||
1. **Host-native search** - Claude Code's `WebSearch`, and the equivalents on Codex / Gemini. Best results; used automatically on hosts that have it. Signalled to the engine via `LAST30DAYS_NATIVE_SEARCH=1` (the skill sets this for you when your host has a native search tool) so the engine does not run a worse search underneath it.
|
||||
1. **Host web search** - whatever web-search capability the agent session already has: built-in search, a deferred web-search tool that must be loaded first, or an installed connector such as Brave, Firecrawl, Exa, Serper, or another provider. Best results; used automatically on hosts that have it. A failed lookup for one specific tool name is not fatal when another web-search capability is available. Signalled to the engine via `LAST30DAYS_NATIVE_SEARCH=1` (the skill sets this for you when your agent session has web search) so the engine does not run a worse search underneath it.
|
||||
2. **Paid engine backend** - one of `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY`, auto-detected in that order. Override per-run with `--web-backend=<name>`.
|
||||
3. **Keyless engine floor** - zero-key web search (DuckDuckGo, plus an optional SearXNG instance) and zero-key page fetch (Jina Reader). Runs only when the host has **no** native search **and** no paid key is set, so headless/cron and hosts without a built-in search tool still get general-web coverage. Force it explicitly with `--web-backend=keyless`.
|
||||
3. **Keyless engine floor** - zero-key web search (DuckDuckGo, plus an optional SearXNG instance) and zero-key page fetch (Jina Reader). Runs only when the agent session has **no** host web search **and** no paid key is set, so headless/cron and hosts without a search tool still get general-web coverage. Force it explicitly with `--web-backend=keyless`.
|
||||
|
||||
Relevant env vars:
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_NATIVE_SEARCH=1` | Tells the engine your host has native search; suppresses the keyless floor. Set automatically by the skill on capable hosts. Leave unset on hosts without a native search tool so the floor runs. |
|
||||
| `LAST30DAYS_NATIVE_SEARCH=1` | Tells the engine your agent session has host-side web search; suppresses the keyless floor. Set automatically by the skill when web search is available. Leave unset when the agent has no web-search tool so the floor runs. |
|
||||
| `LAST30DAYS_SEARXNG_URL=<base-url>` | Optional. A SearXNG instance used as the keyless-search fallback rung when DuckDuckGo returns nothing. |
|
||||
| `LAST30DAYS_TRUSTPILOT_NO_BROWSER=1` | Optional. Truthy value disables the Trustpilot source's one-time headless-Chrome WAF-cookie harvest, so an automated/headless run (cron, CI, the eval harness) never spawns a browser. Trustpilot still degrades to empty gracefully. |
|
||||
|
||||
Privacy note: the keyless floor sends the query (to DuckDuckGo / your SearXNG instance) and any fetched URL (to Jina Reader) to those third parties. It is intended for public-research use; results may be cached snapshots. It never runs when native search or a paid backend is in play.
|
||||
|
||||
@@ -190,6 +399,56 @@ The engine treats public jobs/careers postings as evidence of focus or priority
|
||||
|
||||
---
|
||||
|
||||
## Health check (`doctor`)
|
||||
|
||||
One command answers "what could be on, what's turned on, what's working, and what isn't" — a four-state audit (WORKING / TURNED ON - UNVERIFIED / NOT WORKING / COULD BE ON), one line per source, with a CLI-health block for sources that need a downloaded binary, indented backup/comment sub-lanes, the backend the next run will use (for chained sources), and an exact fix on anything that isn't working:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py doctor # four-state audit (text)
|
||||
python3 skills/last30days/scripts/last30days.py doctor --json # machine contract
|
||||
python3 skills/last30days/scripts/last30days.py doctor --cached # serve the cached report while fresh
|
||||
python3 skills/last30days/scripts/last30days.py doctor --postmortem # what actually broke on the last run
|
||||
python3 skills/last30days/scripts/last30days.py doctor --probe # bounded live test (free/CLI sources)
|
||||
```
|
||||
|
||||
Slash-command form: `/last30days doctor`. Reporting problems is a successful run — the exit code is always 0, no browser cookies are read, and no secret values appear anywhere (key presence is booleans only). Backends within a chained source are probed sequentially with a 5-second budget per binary probe, so a chained source's worst-case check time is additive across its backends (only reached when several binaries hang at once).
|
||||
|
||||
`doctor --postmortem` reads the last run's `last-report.json` (any age, labeled) and reports what actually happened per source — Failed / Partial / Succeeded / Skipped, with details and fix hints — so a run that returned less than expected can be diagnosed after the fact. It makes no network calls.
|
||||
|
||||
**Network note:** plain `doctor` with a fresh run, `--cached`, and `--json` make **no** network calls. `doctor --probe` — and a plain `doctor` when there is **no** fresh run to learn from — run a **bounded** live test to verify WORKING instead of guessing. The probe is scoped to free HTTP endpoints (Reddit, Hacker News, Polymarket, GitHub) plus keyless CLIs; credit-gated sources (X, TikTok, Instagram, Threads, …) are never probed, so no ScrapeCreators credits are spent and no auth rate limits are tripped. Each source is probed concurrently under a per-source deadline so a slow source can never hang the command.
|
||||
|
||||
Every live run writes its JSON result to `~/.config/last30days/doctor-cache.json` (beside `last-run.json`; honors `LAST30DAYS_CONFIG_DIR`). `doctor --cached` returns that stored report when it is younger than the TTL, and falls through to a live run — rewriting the cache — when it is stale, absent, or corrupt. The cache also self-invalidates on configuration change: the payload carries a schema stamp plus a fingerprint of non-secret config signals (which credentials are present as booleans, the `LAST30DAYS_X_BACKEND` / `LAST30DAYS_REDDIT_BACKEND` pin values, and `INCLUDE_SOURCES`), so adding or removing a key, changing a pin, or toggling an opt-in source makes the next `--cached` call run live — no raw secret ever enters the fingerprint or the file. Every report also carries `from_cache` (true/false) and `generated_at` (when the report was built), in the `--json` top level and as a final `generated: … (cached|live)` text line, so you can always tell how old a cached answer is. A failed cache write is never fatal — doctor prints a one-line stderr warning and continues. An explicit `doctor` without `--cached` always runs live and refreshes the cache.
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_DOCTOR_TTL` | Freshness window for `doctor --cached`, in **seconds**. Defaults to `900` (15 minutes). `0` makes every `--cached` call run live. |
|
||||
| `LAST30DAYS_DOCTOR_PROBE_TIMEOUT` | Per-source deadline (**seconds**) for `doctor --probe` live checks. Defaults to `10`. Caps each concurrent probe so a slow source cannot hang the command. |
|
||||
| `LAST30DAYS_X_BACKEND` | Pins the X backend (`bird` / `xai` / `xurl` / `xquik` / `grok`); doctor renders the pin and predicts "will use" accordingly. The unpinned auto chain is bird → xai → xurl → xquik (grok is opt-in only). Pin `grok` to enable it; a leftover `~/.grok/auth.json` is never auto-selected. |
|
||||
| `LAST30DAYS_REDDIT_BACKEND` | `scrapecreators` makes ScrapeCreators the primary Reddit backend; doctor renders Reddit's conditional routing with the pin applied. |
|
||||
| `LAST30DAYS_REDDIT_SC_MIN_ITEMS` | Integer thinness floor for ScrapeCreators Reddit **search** backfill. Default `0` = empty-only (free path keeps any non-empty result; no credit spend). Set above `0` to backfill when free yield is below that count; merged results dedupe by post id. Requires `SCRAPECREATORS_API_KEY`. Ignored when `LAST30DAYS_REDDIT_BACKEND=scrapecreators` (SC is already primary). |
|
||||
|
||||
Web search has **no** env pin — pin it per-run with `--web-backend=<name>` only (see [Web search backend priority](#web-search-backend-priority)).
|
||||
|
||||
### Strict exit for degraded runs
|
||||
|
||||
By default a research run exits `0` even when a source failed mid-run (rate-limited, auth-failed, unreachable, timeout, schema-drift) — the report still renders, with the failure annotated in the per-source footer and a partial-coverage warning. Wrappers that need to distinguish degraded coverage from success (cron briefs, CI, downstream agents) can opt in:
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_STRICT_EXIT` | Truthy (`1`/`true`/`yes`/`on`): the engine exits `3` when any source outcome is neither `ok`, `no-results`, nor `skipped-unconfigured`. A one-line `strict-exit: degraded sources: ...` note goes to stderr. Default (unset): exit `0`, unchanged behavior. |
|
||||
|
||||
Exit codes with the flag on: `0` clean run, `3` completed-but-degraded (report was produced), non-zero others unchanged (hard failures). Same hybrid pattern as `LAST30DAYS_DEBUG` — works shell-exported or in `.env`.
|
||||
|
||||
---
|
||||
|
||||
## Debug mode (`--debug`)
|
||||
|
||||
Add `--debug` to any run to emit verbose `[DEBUG]` log lines to stderr from the source modules (X API, HTTP, etc.). Helpful for diagnosing API errors or unexpected behavior.
|
||||
|
||||
**Always-on alternative:** set `LAST30DAYS_DEBUG=true` in your `.env` or export it from your shell. The flag still works as before; the env var is purely additive — works whether shell-exported or set in `.env`.
|
||||
|
||||
---
|
||||
|
||||
## Trend monitoring (`--store` + watchlist + briefings)
|
||||
|
||||
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
||||
@@ -202,6 +461,28 @@ Adding `--store` to any run persists every finding to a SQLite database (default
|
||||
|
||||
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
||||
|
||||
### Discovery topic queue (`LAST30DAYS_DISCOVERY_QUEUE`)
|
||||
|
||||
`--discover` runs remember what they surfaced (table `discovery_topics` in the same research.db). Re-surfaced topics get a `**Pipeline:**` line on their card ("surfaced 2nd time", "marked covered") so the discovery brief doubles as a podcast / X-article content pipeline. On by default for real runs; `--mock` runs never write. With `--save-dir`, queue rows land in that directory's scoped `research.db`, never the global one.
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_DISCOVERY_QUEUE` | Set to `off` to disable queue writes and card annotations. Any other value (or unset) keeps the queue on. Works shell-exported or in `.env`. |
|
||||
| `LAST30DAYS_ENRICH_BUDGET_SECONDS` | Wall-clock budget (seconds) for the deep-tier per-topic research batch on the discovery resume leg (`--discover --judgments <file>`). Default `450`; unset/invalid/non-positive values fall back to it. The one-shot `--discover` path keeps its fixed quick-tier 240s budget regardless. Works shell-exported or in `.env`. |
|
||||
|
||||
Manage the queue from the engine CLI:
|
||||
|
||||
```bash
|
||||
# Uncovered surfaced topics (name, domain, surface_count, last_surfaced, status)
|
||||
python3 skills/last30days/scripts/last30days.py queue list
|
||||
|
||||
# Mark a topic done after you record the episode / publish the article.
|
||||
# Requires the exact topic name; unknown names exit 2 instead of no-opping.
|
||||
python3 skills/last30days/scripts/last30days.py queue cover "Gemma 4 chat templates"
|
||||
```
|
||||
|
||||
Both respect `--save-dir` scoping.
|
||||
|
||||
### `watchlist.py` - recurring topics
|
||||
|
||||
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
||||
@@ -244,9 +525,22 @@ The schedule field stored on each topic is metadata - the actual cron / Task Sch
|
||||
|
||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||
|
||||
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
||||
**Codex note:** the repository includes `.codex-plugin/plugin.json` so Codex can treat the existing
|
||||
`skills/last30days/SKILL.md` tree as plugin metadata without maintaining a separate Codex copy.
|
||||
The Codex marketplace catalog points at the repository root URL: Codex clones the repo, reads the
|
||||
root `.codex-plugin/plugin.json`, and loads skills from `./skills/`. The Agent Skills install
|
||||
command documented in the README remains the broadest cross-host path.
|
||||
|
||||
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
||||
**Grok note:** the repository includes `.grok-plugin/plugin.json` and `.grok-plugin/marketplace.json`
|
||||
so xAI's Grok Build CLI (`grok`) can install last30days as a native plugin. Grok also reads the
|
||||
Claude Code manifests for compatibility; the native pair is the first-class lane. The Grok
|
||||
marketplace catalog uses a bare Git URL source (no commit pin) so `grok plugin marketplace add
|
||||
mvanhorn/last30days-skill` tracks HEAD — the same pattern as the Codex catalog. `npx skills add`
|
||||
remains a valid cross-host fallback.
|
||||
|
||||
### 1. Trusted per-client `.claude/last30days.env`
|
||||
|
||||
When each client has its own working directory, drop a `.claude/last30days.env` into the client folder and opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or global `~/.config/last30days/.env`. The skill loads the project file only after that trust signal. Typical contents:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||
@@ -255,7 +549,7 @@ INCLUDE_SOURCES=tiktok,instagram
|
||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||
```
|
||||
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
|
||||
### 2. Per-client save dir + suffix wrapper
|
||||
|
||||
@@ -321,5 +615,5 @@ This is the right home for client-specific changes you don't intend to upstream
|
||||
|
||||
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
||||
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
||||
- Engine spec (some sections stale; SKILL.md wins on conflicts): [`SPEC.md`](SPEC.md)
|
||||
- Shared package vocabulary and engine/harness terminology: [`CONCEPTS.md`](CONCEPTS.md)
|
||||
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# Contributing
|
||||
|
||||
Thanks for helping with last30days. Most PRs here are opened by coding agents following [`AGENTS.md`](AGENTS.md); this file is the short path for humans and agents alike.
|
||||
|
||||
## Setup
|
||||
|
||||
Python **3.12+**. From the repo root:
|
||||
|
||||
```bash
|
||||
uv sync --group dev
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
That installs pytest/coverage and **towncrier** into the project env. You do **not** need a global towncrier install for normal contributions.
|
||||
|
||||
## Day-to-day PRs (no towncrier CLI)
|
||||
|
||||
1. Make your change and add/update tests.
|
||||
2. If it should show up in the next release notes, add a fragment:
|
||||
```bash
|
||||
# Prefer the PR or issue number when you know it:
|
||||
# changelog.d/<number>.<type>.md
|
||||
# Orphan (no number yet):
|
||||
# changelog.d/+short-slug.<type>.md
|
||||
```
|
||||
Types: `added`, `changed`, `fixed`, `removed`, `deprecated`, `security`.
|
||||
Details: [`changelog.d/README.md`](changelog.d/README.md).
|
||||
3. Fill out [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md) — Summary (“what does this PR do”), Testing, Changelog, Agent disclosure, Relationship.
|
||||
4. Do **not** edit `CHANGELOG.md` and do **not** bump version strings in `pyproject.toml`, `SKILL.md`, plugin/marketplace JSON, or `uv.lock`. CI enforces that.
|
||||
|
||||
Chores with nothing for the release notes: check Skip changelog in the template and add the `skip-changelog` label.
|
||||
|
||||
Fragments are plain Markdown files. **towncrier is only used when cutting a release** (locally via `uv run` or in GitHub Actions) — contributors never run it for a feature PR.
|
||||
|
||||
## Releases (maintainers)
|
||||
|
||||
Prefer **Actions → Prepare release** (patch / minor / major). That opens a lockstep version PR (towncrier builds `CHANGELOG.md`, bumps every plugin/marketplace surface). Merging to `main` tags `vX.Y.Z` and the existing Release workflow publishes artifacts.
|
||||
|
||||
Local equivalent (after `uv sync --group dev`):
|
||||
|
||||
```bash
|
||||
uv run python .github/scripts/prepare_release.py --bump patch # or --version X.Y.Z
|
||||
```
|
||||
|
||||
More detail: `AGENTS.md` § Changelog and releases, and `docs/solutions/workflow-issues/towncrier-lockstep-release.md`.
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
uv run pytest
|
||||
uv run pytest tests/test_dedupe_v3.py -k some_case
|
||||
uv run pytest --cov
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
Never commit real API keys, cookies, tokens, or `.env` contents. Use dummy values in tests and fixtures.
|
||||
+1
-3
@@ -54,9 +54,7 @@ v3 has durable watchlist with multi-source storage and extended time windows.
|
||||
|
||||
## Past Contributors
|
||||
|
||||
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 encoding fix ([#550](https://github.com/mvanhorn/last30days-skill/pull/550)), Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)), browser cookie message clarity ([#387](https://github.com/mvanhorn/last30days-skill/issues/387))
|
||||
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)); .env permission auto-fix ([#573](https://github.com/mvanhorn/last30days-skill/pull/599)); MCP Go tests in CI ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
- [@JosephOIbrahim](https://github.com/JosephOIbrahim) - Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
|
||||
- [@levineam](https://github.com/levineam) - Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
|
||||
- [@jonthebeef](https://github.com/jonthebeef) - Early testing and feedback
|
||||
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes: utf-8 plan-file reads, skill_meta version-detection encoding, MSYS noacl permission check skip ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); first-run setup wizard fix: SKILL.md references existing Python setup wizard instead of missing nux-wizard.md ([#574](https://github.com/mvanhorn/last30days-skill/pull/574))
|
||||
- [@23241a6749](https://github.com/23241a6749) - check-config.sh xargs unterminated quote fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506))
|
||||
|
||||
+7
-3
@@ -4,17 +4,19 @@ This guide covers installing last30days on Hermes AI Agent.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Hermes installed** - See https://github.com/mercurial-tf/hermes
|
||||
1. **Hermes installed** - See https://github.com/NousResearch/hermes-agent
|
||||
2. **Python 3.12+** - `brew install python@3.12` or similar
|
||||
3. **yt-dlp** (optional, for YouTube) - `brew install yt-dlp`
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
hermes skills install mvanhorn/last30days-skill/skills/last30days --force
|
||||
```
|
||||
|
||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
||||
The explicit `skills/last30days` path fetches the skill straight from this repo's current default branch and deploys it under `~/.hermes/skills/`. `--force` is required because Hermes's install-time security scanner returns a `caution` verdict for this skill — it flags benign patterns such as reading your own API keys from the environment and calling `subprocess` to run `yt-dlp`/`bird`. `--force` accepts the caution verdict and installs (it also reinstalls over any existing copy).
|
||||
|
||||
**Why the explicit path?** The shorter `hermes skills install mvanhorn/last30days-skill` currently resolves through the skills.sh index, which is serving an older cached snapshot of this repo (from before the skill moved under `skills/last30days/`). Use the explicit `.../skills/last30days` path above until the index re-crawls — tracked in [vercel-labs/skills#1602](https://github.com/vercel-labs/skills/issues/1602).
|
||||
|
||||
### Developer / live-edit alternative
|
||||
|
||||
@@ -47,6 +49,7 @@ On first run, the skill will guide you through setup:
|
||||
1. **Auto setup** (~30 seconds)
|
||||
- Scans browser cookies for X/Twitter
|
||||
- Checks/installs yt-dlp for YouTube
|
||||
- Best-effort install of `digg-pp-cli` for Digg AI-news clusters (via `@mvanhorn/printing-press-library`; binary lands in `$HOME/.local/bin` — ensure your Hermes gateway PATH includes it, or Digg stays off even after install)
|
||||
- Configures free sources (Reddit, HN, Polymarket)
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
@@ -65,6 +68,7 @@ On first run, the skill will guide you through setup:
|
||||
- **Hacker News** - Tech discussions via Algolia
|
||||
- **Polymarket** - Prediction markets
|
||||
- **YouTube** - Search and transcripts (requires yt-dlp)
|
||||
- **Digg** - AI-news story clusters (requires `digg-pp-cli` on the agent PATH; auto-installed to `$HOME/.local/bin` during setup when `npx` is available)
|
||||
|
||||
### Requires API Key
|
||||
- **X/Twitter** - xAI API key or browser cookies
|
||||
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
# /last30days
|
||||
|
||||
[English](README.md) | [Français](README.fr.md) | Deutsch | [Español](README.es.md) | [Português (Brasil)](README.pt-BR.md) | [日本語](README.ja.md) | [简体中文](README.zh-CN.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://trendshift.io/repositories/21997" target="_blank">
|
||||
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**Eine von einem KI-Agenten gesteuerte Suchmaschine, die nach Upvotes, Likes und echtem Geld gewichtet – nicht nach Redaktionen.**
|
||||
|
||||
Dieses README beschreibt die aktuelle v3-Pipeline. Die Laufzeitspezifikation der Skill liegt in [skills/last30days/SKILL.md](skills/last30days/SKILL.md) und ist maßgeblich für das aktuelle Verhalten von Befehlen und Setup.
|
||||
|
||||
**Claude Code (empfohlen – automatische Updates über den Marketplace):**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI oder einer von 50+ [Agent Skills](https://agentskills.io)-Hosts:**
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
(`-g` installiert global für deinen Benutzer, also in allen Projekten verfügbar. Lass das Flag weg, wenn du die Installation auf ein Projekt beschränken willst.)
|
||||
|
||||
Weitere Installationswege (claude.ai im Browser, OpenClaw, manuell) findest du unten im Abschnitt [Installation](#installation).
|
||||
|
||||
Null Konfiguration. Reddit, HN, Polymarket und GitHub funktionieren sofort. Führe die Skill einmal aus, und der Setup-Assistent schaltet X, YouTube, TikTok, arXiv, Techmeme und mehr in 30 Sekunden frei.
|
||||
|
||||
---
|
||||
|
||||
Upvotes von Reddit. Likes von X. YouTube-Transkripte. TikTok-Engagement. Polymarket-Quoten, gedeckt durch echtes Geld und Insiderwissen. Das sind Millionen Menschen, die jeden Tag mit ihrer Aufmerksamkeit und ihrem Geldbeutel abstimmen. /last30days durchsucht all das parallel, gewichtet nach dem, womit echte Menschen tatsächlich interagieren, und ein KI-Agent fasst es als Juror zu einem einzigen Briefing zusammen.
|
||||
|
||||
Google aggregiert Redaktionen. /last30days durchsucht Menschen.
|
||||
|
||||
Diese Suche bekommst du nirgendwo sonst, weil keine einzelne KI Zugriff auf alles hat. Google erfasst weder Reddit-Kommentare noch X-Beiträge. ChatGPT hat einen Deal mit Reddit, kann aber weder X noch TikTok durchsuchen. Gemini hat YouTube, aber kein Reddit. Claude hat nichts davon nativ. Jede Plattform ist ein abgeschotteter Garten mit eigener API, eigenen Tokens, eigener Authentifizierung. Aber du kannst deine eigenen Schlüssel und Browser-Sessions mitbringen – und plötzlich durchsucht ein KI-Agent alle gleichzeitig, wägt sie gegeneinander ab und sagt dir, was wirklich zählt.
|
||||
|
||||
Das ist der eigentliche Durchbruch. Keine bessere Suchmaschine, sondern ein Dutzend getrennter Plattformen, die ein Agent miteinander verbindet.
|
||||
|
||||
```
|
||||
/last30days Peter Steinberger
|
||||
```
|
||||
|
||||
Du hast morgen ein Meeting. Du googelst die Person. Du bekommst ihr LinkedIn-Profil von 2023. /last30days zeigt dir, was sie diesen Monat wirklich macht: bei OpenAI eingestiegen, um an Codex zu arbeiten, kämpft gegen Anthropics Verbot von Drittanbieter-Agenten, hat 23 PRs mit 85 % Merge-Rate geliefert, baut „LobsterOS“ für geräteübergreifende Agentensteuerung – und ein Thread in r/ClaudeCode kam auf 569 Upvotes bei der Frage, ob sie ein Held oder „unerträglich“ ist. Verteilt über X-Beiträge, Reddit-Threads, YouTube-Transkripte und GitHub-Commits. Nichts davon stand bei Google.
|
||||
|
||||
## Warum es das gibt
|
||||
|
||||
Ich habe es gebaut, um bei KI Schritt zu halten. Alles ändert sich täglich, und die Nerds auf Reddit und X wissen es immer zuerst. Ich brauchte bessere Prompts, und die Trainingsdaten lagen immer Monate hinter dem, was die Community längst herausgefunden hatte.
|
||||
|
||||
Daraus wurde etwas Größeres. Heute lasse ich es vor einem Sales-Call laufen, um die Wahrheit der letzten 30 Tage über ein Unternehmen zu kennen. Vor einem Meeting, um die aktuellen Tweets und Podcast-Transkripte meines Gegenübers zu lesen. Vor einer Reise nach Disney World, um zu wissen, welche Attraktionen geschlossen sind und was die Community über Genie+ sagt. Bevor ich irgendetwas baue, um zu wissen, an welchen Problemen die Leute wirklich hängen.
|
||||
|
||||
Wenn du dich mit einem CEO triffst: Hast du alle Tweets und YouTube-Transkripte der letzten 30 Tage gelesen? Ich schon.
|
||||
|
||||
## Quellen, gewichtet von den Menschen
|
||||
|
||||
| Quelle | Was dir die Menschen sagen |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | Die ungefilterte Meinung. Top-Kommentare mit echten Upvote-Zahlen, kostenlos, ohne API-Schlüssel. Die echten Meinungen, die Google vergräbt. |
|
||||
| **X / Twitter** | Die spontane Einschätzung, der Experten-Thread, die erste Reaktion auf eine Eilmeldung. Zuerst informiert, zuerst am Streiten. |
|
||||
| **YouTube** | Die 45-minütige Tiefenanalyse. Vollständige Transkripte, durchsucht nach den 5 zitierfähigen Sätzen, auf die es ankommt. |
|
||||
| **TikTok** | Der Creator, der 3,6 Millionen Menschen mit einer Sichtweise erreicht, die du bei Google nie findest. |
|
||||
| **Instagram Reels** | Die Perspektive der Influencer, inklusive Transkript des Gesprochenen. Das Signal der visuellen Kultur. |
|
||||
| **Hacker News** | Der Konsens der Entwickler. 825 Punkte, 899 Kommentare. Wo technische Leute wirklich streiten. |
|
||||
| **Polymarket** | Keine Meinungen. Quoten. Gedeckt durch echtes Geld. 96 % Wahrscheinlichkeit bei Albumverkäufen. 4 % bei einer Übernahme. |
|
||||
| **GitHub** | Für Personen: PR-Tempo, Top-Repos nach Sternen, Release Notes. Für Themen: Issues und Discussions. |
|
||||
| **Digg** | Kuratierte Story-Cluster aus Diggs AI-1000-Leaderboard (rund 1000 KI-Accounts mit hohem Signal auf X), mit zuordenbaren Inline-Zitaten und ganz ohne X-Authentifizierung. Wird automatisch aktiv, sobald `digg-pp-cli` im PATH liegt. |
|
||||
| **arXiv** | Die Fachartikel hinter dem Hype. Neue Forschung im Zeitfenster, kostenlos, ohne API-Schlüssel. Wird automatisch aktiv, sobald `arxiv-pp-cli` im PATH liegt (das Erst-Setup installiert es). |
|
||||
| **Techmeme** | Die redaktionelle Ebene der Tech-News, begrenzt auf dein 30-Tage-Fenster. Kostenlos, ohne API-Schlüssel. Wird automatisch aktiv, sobald `techmeme-pp-cli` im PATH liegt (das Erst-Setup installiert es). |
|
||||
| **LinkedIn** | Das berufliche Signal. Beiträge und Artikel, wobei Artikel als starkes Signal gewichtet werden. |
|
||||
| **StockTwits** | Die Stimmung der Trader. Aktiviert sich automatisch, wenn dein Thema ein Ticker oder eine Kryptowährung ist. |
|
||||
| **Threads** | Die Textebene nach Twitter. Gespräche von Creators und Marken. |
|
||||
| **Pinterest** | Visuelle Entdeckung. Pins, gespeicherte Beiträge und Kommentare zu Produkten und Ideen. |
|
||||
| **Xiaohongshu (RED)** | Chinesische Signale zu Lifestyle, Produkten und Creators. Wird ausdrücklich mit `--search xhs` angefordert, wenn lokal ein eingeloggtes x-mcp-Browser-Plugin oder ein `xiaohongshu-mcp`-Dienst läuft. |
|
||||
| **Bluesky** | Die dezentrale soziale Ebene. AT-Protocol-Beiträge aus der Abwanderung nach Twitter. |
|
||||
| **Perplexity** | Belegte Sonar-Synthese, Rohtreffer der Search API und Deep Research. |
|
||||
| **Web** | Die redaktionelle Berichterstattung, die Blog-Vergleiche. Ein Signal von vielen, nicht das einzige. |
|
||||
|
||||
Die Community steuert laufend weitere bei. Truth Social und andere Nischenquellen stecken bereits in der Engine, weitere folgen.
|
||||
|
||||
Ein Reddit-Thread mit 1.500 Upvotes ist ein stärkeres Signal als ein Blogbeitrag, den niemand gelesen hat. Ein TikTok mit 3,6 Millionen Aufrufen sagt mehr darüber aus, was kulturell relevant ist, als jede Pressemitteilung. Polymarket-Quoten mit 66.000 $ Handelsvolumen dahinter lassen sich schwerer wegdiskutieren als die Vermutung eines Kommentators.
|
||||
|
||||
Die Synthese sortiert nach dem, womit echte Menschen tatsächlich interagiert haben. Soziale Relevanz, nicht SEO-Relevanz.
|
||||
|
||||
## Wofür die Leute es wirklich nutzen
|
||||
|
||||
**Vor einem Meeting.** `/last30days Peter Steinberger` – beim Codex-Team von OpenAI eingestiegen, kämpft gegen Anthropics Verbot von Drittanbieter-Agenten, 23 PRs mit 85 % Merge-Rate auf GitHub gemergt, baut LobsterOS für geräteübergreifende Agentensteuerung. r/ClaudeCode: „Seit OpenClaw erschienen ist, war allgemein bekannt: Wer es über etwas anderes als die API laufen lässt, fliegt irgendwann raus“ (227 Upvotes). Das steht so nicht auf LinkedIn.
|
||||
|
||||
**Um Hiring-Signale zu lesen.** `/last30days Listen Labs --hiring-signals` – aktuelle Stellenanzeigen und Karriereseiten werden zu zitierten Belegen für Schwerpunktverschiebungen: Einstellungen in Enterprise Security, Customer Success, Infrastruktur oder Produktausbau. Der Bericht sagt, was das Hiring zu signalisieren scheint, nicht was die Roadmap liefern wird.
|
||||
|
||||
**Um ein Thema vor seinem Höhepunkt zu finden.** Frag `/last30days what's exploding in AI agents?`, und die Skill wechselt in den Discovery-Modus: Die Engine durchkämmt Reddit-Kategorielisten, die Front- und Best-Stories von Hacker News, Diggs AI-1000-Feed und X, sofern du authentifiziert bist. Dein Agent bewertet die Vorschläge (Namen, Müllfilterung, inhaltliche Relevanz) und schreibt Podcast- und X-Artikel-Ansätze. Am Ende bekommst du 5 bis 10 nach Velocity sortierte Themen. Jedes Ergebnis enthält quellenübergreifende Zahlen, ein Momentum-Label und einen startklaren Folgebefehl `/last30days "<topic>"`.
|
||||
|
||||
**Wenn etwas erscheint.** `/last30days Kanye West` – Großbritannien hat sein Visum blockiert, das Wireless Festival wurde abgesagt, die Sponsoren sind abgesprungen. Aber BULLY stieg auf Platz 2 der Billboard-Charts ein. Fantano kam aus seinem „Yay sabbatical“ zurück, um es zu rezensieren (653.000 Aufrufe). Beim SoFi Homecoming holte er Lauryn Hill und Travis Scott für 44 Songs auf die Bühne. Polymarket: „Wird Kanye wieder twittern?“ 86 % Ja. 23 Reddit-Threads, 17 YouTube-Videos, 86.000 Upvotes.
|
||||
|
||||
**Um Tools zu vergleichen.** `/last30days OpenClaw vs Hermes vs Paperclip` – „Das sind keine Konkurrenten, das sind Schichten.“ OpenClaw ist die ausführende Ebene (351.000 GitHub-Sterne, produktiv), Hermes ist das sich selbst verbessernde Gehirn (31.000 Sterne), Paperclip ist das Organigramm (49.000 Sterne). Die Sternzahlen kommen live aus der GitHub-API, nicht aus veralteten Blogbeiträgen. Vergleichstabelle mit Architektur, Speicher, Sicherheit und idealem Einsatzzweck. Laut @IMJustinBrooke: „OpenClaw = Glumanda, Hermes = Glurak.“
|
||||
|
||||
**Um die Welt zu verstehen.** `/last30days Iran vs USA` – Tag 38 des Krieges. Trumps Ultimatum bis Dienstag, damit der Iran die Straße von Hormus wieder öffnet. Zwei US-Kampfjets abgeschossen. Öl bei 126 $ pro Barrel. Die IEA nannte es „die größte Versorgungsstörung in der Geschichte des globalen Ölmarkts“. Polymarket: Waffenstillstand bis zum 31. Dezember bei 74 %. 27 X-Beiträge, 10 YouTube-Videos, 20 Prognosemärkte.
|
||||
|
||||
**Vor einer Reise.** `/last30days Universal Epic Universe` – die Erweiterung ist bereits im Bau. Baugenehmigung „Project 680“ eingereicht. Eine Feuerwerksshow ist über die Infrastruktur belegt, aber noch nicht angekündigt. Wartezeiten: Mine-Cart Madness im Schnitt 148 Minuten. Noch keine Jahreskarte, und die Einheimischen sind genervt. Stardust Racers steht bis zum 5. April wegen Renovierung still.
|
||||
|
||||
**Um schnell etwas zu lernen.** `/last30days Nano Banana Pro prompting` – JSON-strukturierte Prompts lösen den Tag-Wildwuchs ab. Das verschachtelte Format von @pictsbyai verhindert „Concept Bleeding“. Bearbeiten schlägt neu generieren. Und danach schreibt dir die Skill einen produktionsreifen Prompt, der genau das umsetzt, was die Community als funktionierend beschrieben hat.
|
||||
|
||||
## Was neu ist
|
||||
|
||||
Seit der Ankündigung von v3.3 im Mai und mit Stand v3.11.1 (Juli 2026): 175 gemergte PRs – 122 davon von 52 Beitragenden aus der Community – verteilt auf 15 Releases. Das ist gelandet.
|
||||
|
||||
### Erstklassig auf OpenAI Codex
|
||||
|
||||
/last30days ist jetzt ein natives Codex-Plugin mit geführtem Setup – keine Portierung, sondern ein vollwertiger Bürger. Renderer-bewusste Zitate sorgen dafür, dass die Codex-Ausgabe sich wie ein Briefing liest und nicht wie eine URL-Suppe (#694), und dieselbe Engine läuft auf Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw und 50+ Agent-Skills-Hosts. Codex-Plugin-Manifest von [@rfoust](https://github.com/rfoust) (#686), Codex-Auth-Fix von [@tmchow](https://github.com/tmchow) (#698).
|
||||
|
||||
### arXiv, Techmeme und Digg – kostenlos, ohne API-Schlüssel
|
||||
|
||||
arXiv liefert die Fachartikel hinter dem Hype, Techmeme die redaktionelle Tech-News-Ebene – kostenlos, ohne einen einzigen Schlüssel, und das Erst-Setup installiert ihre CLIs, sodass sie sich von selbst aktivieren (#709). Diggs AI-1000-Story-Cluster kommen genauso ohne X-Authentifizierung an: Das Setup installiert dir die kostenlose Digg-CLI (#590). Trustpilot ist optional zuschaltbar für Recherchen zu Consumer-Marken.
|
||||
|
||||
### Reddit gratis, mit echten Scores und Top-Kommentaren
|
||||
|
||||
Reddits öffentliche .json-API ist gestorben; der kostenlose Weg kam stärker zurück. Schlüsselloses RSS plus Shreddit-Scraping (#457), gezielte Subreddit-Suche mit echten Upvote-Zahlen über arctic-shift (#696) und eine Relevanzschwelle, damit ein viraler Off-Topic-Beitrag dein Briefing nicht kapert (#488, danke [@rzachsmith](https://github.com/rzachsmith)). Kein API-Schlüssel. Echte Scores. Top-Kommentare inklusive.
|
||||
|
||||
### Die besten Kommentare in jedem Briefing
|
||||
|
||||
Kommentare sind jetzt eine quellenübergreifend standardmäßig aktive Ebene: Instagram-Kommentare mit rangbasierter Vielfalt, damit fünf zugespitzte Meinungen nicht alle aus einem einzigen Beitrag stammen (#751), YouTube-Kommentare plus ein Transkript-Backup über ScrapeCreators, falls yt-dlp scheitert (#637), und von der Community hochgevotete Kommentare, die in die Best-Takes-Wertung einfließen, damit die witzigsten Zeilen die Bewertung überleben (#592, #608).
|
||||
|
||||
### Ein einziger doctor-Befehl
|
||||
|
||||
Bitte um einen Health-Check: doctor prüft jede Quelle und verschreibt dann die genauen Korrekturen – welcher Schlüssel fehlt, welche CLI nicht im PATH liegt, welches Cookie abgelaufen ist (#753). Kein Rätselraten mehr, warum X so wenig geliefert hat.
|
||||
|
||||
### Die X-Suche, neu gebaut
|
||||
|
||||
Die X-Pipeline wurde von Grund auf überarbeitet: FROM- und ABOUT-Lanes, damit sowohl die eigenen Beiträge einer Person als auch das Gespräch über sie einsortiert werden (#610), personenbezogene Auflösung mehrdeutiger Unterabfragen (#611), Verifizierung der Urheberschaft aus erster Hand samt Ranking nach Interaktionssignalen (#613) und eine einzige X-Quelle mit automatischem Backend-Failover (#622). Dazu ein ehrliches `--diagnose`, das die Authentifizierung wirklich prüft (#609).
|
||||
|
||||
### Weitere Quellen sind dazugekommen
|
||||
|
||||
LinkedIn über ScrapeCreators, mit Artikeln als starkem Signal ([@ravstr](https://github.com/ravstr), #702). StockTwits aktiviert sich automatisch bei Ticker- und Krypto-Themen ([@wtiwana](https://github.com/wtiwana), #658). Perplexity hat direkte API-Modi und asynchrone Deep Research dazubekommen ([@sk-holmes](https://github.com/sk-holmes), #629).
|
||||
|
||||
### Von der Community gehärtet
|
||||
|
||||
Die Sicherheitswelle war fast vollständig Community-Arbeit: Fixes für Stored XSS im HTML-Renderer ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), abgesicherte temporäre Cookie-Dateien, eine gegen Supply-Chain-Angriffe gehärtete CI mit OpenSSF Scorecard und Build-Provenance-Attestierung ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), Semgrep- und OSV-Scanner-Scans plus ein Dependency-Review-Gate für jeden PR ([@23241a6749](https://github.com/23241a6749)), eine Mindestgrenze für die Testabdeckung, eingeführt bei 60 % und inzwischen auf 84 % angehoben ([@gourab5139014](https://github.com/gourab5139014)), und ein Hermes-Sicherheitsscan, der inzwischen keinen einzigen CRITICAL-Befund mehr enthält (#768).
|
||||
|
||||
### Reicht weiter
|
||||
|
||||
Hebräisch und andere nichtlateinische Sprachen ([@dudyme](https://github.com/dudyme)). CJK-taugliche Tokenisierung für chinesische Quellen ([@An-idd](https://github.com/An-idd)). Eine Welle an Windows-Kompatibilität. Cookie-Extraktion für die gesamte Chromium-Familie – Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov)) – plus macOS Keychain und pass(1) unter Linux als Quellen für Zugangsdaten. Historischer Rückblick mit `--as-of` ([@chiyi-creator](https://github.com/chiyi-creator)). Automatisch bereitgestelltes Python 3.12 über uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` zum Auslesen der Stellenseiten eines Unternehmens. Watchlist-Deltas zwischen zwei Durchläufen.
|
||||
|
||||
### Weiterhin ab Werk dabei seit v3
|
||||
|
||||
Die Grundlagen aus v3 sind alle noch da: das Pre-Research-Hirn, das die richtigen Handles, Subreddits und Hashtags ermittelt, bevor ein einziger API-Aufruf rausgeht (gebaut von [@j-sperling](https://github.com/j-sperling)); die Best-Takes-Wertung, die Humor und Viralität neben Relevanz berücksichtigt; quellenübergreifendes Cluster-Merging; Vergleiche in einem Durchgang („CLI vs MCP“ in 3 Minuten statt 12); automatisch gefundene `--competitors`-Vergleiche; der GitHub-Personenmodus (`--github-user=steipete`); der ELI5-Modus („eli5 on“ nach jedem Durchlauf); und teilbare, in sich geschlossene HTML-Briefings (`--emit=html`). Die Konfigurationsschalter stehen in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Installation
|
||||
|
||||
| Umgebung | Installation | Updates |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (empfohlen) | `/plugin marketplace add mvanhorn/last30days-skill` | Automatisch über den Marketplace, oder `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill`, dann `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI oder einer von 50+ [Agent Skills](https://agentskills.io)-Hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (Browser) | [`last30days.skill` herunterladen](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) und über claude.ai > Customize > Skills > + > Create skill > Upload a skill hochladen | Neu herunterladen und erneut hochladen |
|
||||
| **Claude Desktop** | [Die `.mcpb` für deine Plattform herunterladen](https://github.com/mvanhorn/last30days-skill/releases/latest) und in Settings > Extensions ziehen | Neu herunterladen und das neue Bundle hineinziehen |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (empfohlen)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Empfohlen, weil der Claude-Code-Marketplace die Updates für dich übernimmt: Der Plugin-Cache ist versioniert und aktualisiert sich automatisch, sobald ein neues Release erscheint. Mit `claude plugin update last30days@last30days-skill` erzwingst du eine Prüfung.
|
||||
|
||||
Wenn du lieber den Agent-Skills-Installationsweg unter Claude Code nutzt, wird auch der unterstützt:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
Das native Plugin und die `npx skills`-Installation können nebeneinander existieren. Beachte aber: Claude Code dedupliziert nicht über Installationsmethoden hinweg. Wenn sowohl das Marketplace-Plugin als auch die `npx skills`-Kopie aktiv sind, taucht `/last30days` doppelt auf. Nutze pro Rechner eine Installationsmethode.
|
||||
|
||||
### Grok (xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) installiert last30days als natives Plugin. Die direkte Installation folgt dem Repository:
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Oder füge dieses Repository als Marketplace-Quelle hinzu und installiere anschließend über den Plugin-Namen:
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
Mit `--trust` überspringst du die Installationsbestätigung. Aktualisieren kannst du mit `grok plugin update last30days`. Grok liest aus Kompatibilitätsgründen auch die Claude-Code-Manifeste; das native `.grok-plugin/`-Paar ist der bevorzugte Weg – und genau darauf verweist ein offizieller Eintrag im [xAI-Marketplace](https://github.com/xai-org/plugin-marketplace). `npx skills add` bleibt ein gültiger Fallback über alle Hosts hinweg.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI und weitere Agent-Skills-Hosts
|
||||
|
||||
Installiere über die offene [Agent Skills](https://agentskills.io)-CLI – sie unterstützt 50+ Hosts, darunter `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose` und weitere (vollständige Liste im [Repository vercel-labs/skills](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
Das Flag `-g` (global) installiert in dein Benutzerverzeichnis, sodass die Skill in allen Projekten verfügbar ist. Ohne `-g` installiert `npx skills` projektlokal nach `./.skills/` (und wird mit dem Repository eingecheckt). Für ein Werkzeug, mit dem du die ganze Welt recherchierst, willst du die globale Installation.
|
||||
|
||||
Codex Desktop und andere Hosts, die auf Ordnerebene arbeiten, funktionieren sowohl in gewöhnlichen Ordnern als auch in Git-Repositories. Bitte den Host-Agenten vor der ersten Recherche, das mitgelieferte `scripts/last30days.py --preflight` aus dem geladenen Skill-Verzeichnis auszuführen; in einem Checkout des Quellcodes lautet der entsprechende Befehl `python3 skills/last30days/scripts/last30days.py --preflight`. Er zeigt dir, woher die Konfiguration stammt, welche Browser-Cookies gelesen würden, welche Dateien geschrieben würden, welche optionalen Befehle es gibt und welche Projektkonfiguration ignoriert wird – ohne Cookies zu lesen, Dateien zu schreiben oder eine Recherche zu starten.
|
||||
|
||||
Standardmäßig wird für den Host installiert, den `npx skills` erkennt. Um gezielt einen (oder mehrere) anzusprechen:
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Später aktualisieren mit:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
Oder aktualisiere alles, was du global über `npx skills` installiert hast:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
Auflisten und entfernen kannst du mit `npx skills list -g` und `npx skills remove last30days -g`.
|
||||
|
||||
### claude.ai (Browser)
|
||||
|
||||
1. [`last30days.skill` herunterladen](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) – aus dem neuesten Release
|
||||
2. Geh zu [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. Klicke im Skills-Panel auf `+`, dann auf `Create skill` > `Upload a skill`, und wähle die Datei aus oder zieh sie hinein
|
||||
|
||||
Aktiviere vorher unter Capabilities die Option „Code execution and file creation“ – ohne sie laufen Skills nicht.
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop installiert `/last30days` als MCP-Server über ein `.mcpb`-Bundle (ein Model-Context-Protocol-Paket zum Ein-Klick-Installieren).
|
||||
|
||||
1. Öffne das [neueste Release](https://github.com/mvanhorn/last30days-skill/releases/latest) und lade die `.mcpb` für deine Plattform herunter:
|
||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Öffne Claude Desktop, geh zu Settings > Extensions und zieh die Datei hinein.
|
||||
3. Füge auf Nachfrage die API-Schlüssel für die Quellen ein, die du aktivieren willst. Jedes Feld ist optional – überspringst du alle, fällt die Engine auf den reinen Web-Modus zurück. Die Schlüssel landen im Schlüsselbund deines Betriebssystems.
|
||||
4. Starte Claude Desktop neu. Bitte Claude, „zu Peter Steinberger zu recherchieren“ oder zu einem beliebigen anderen Thema, und es ruft das Tool `research` auf.
|
||||
|
||||
**Voraussetzung auf dem Host:** Python 3.12+ im PATH. Das Bundle bringt den Quellcode der Engine mit, nutzt aber deinen lokalen Python-Interpreter. Unter Windows installierst du ihn von [python.org](https://www.python.org/downloads/); macOS und die meisten Linux-Distributionen bringen bereits eine kompatible Version mit.
|
||||
|
||||
**Die Schlüssel werden nicht mit der Claude-Code-Skill geteilt.** Claude Desktop und Claude Code halten bewusst getrennte Speicher für Zugangsdaten. Wenn du `~/.config/last30days/.env` bereits für die Claude-Code-Skill eingerichtet hast, gibst du dieselben Schlüssel hier einmalig erneut ein.
|
||||
|
||||
Windows-Unterstützung ist zurückgestellt, bis die plattformspezifischen Einstiegspunkte im Manifest geklärt sind; verfolgt wird das in einem eigenen Issue.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
Für X/Twitter-Aktionen außerhalb der `/last30days`-Recherche – Tweets oder
|
||||
Antworten posten, Follower exportieren, Medien verwalten, Accounts beobachten
|
||||
und Verlosungen auswerten – nutzt du [TweetClaw](https://github.com/Xquik-dev/tweetclaw)
|
||||
als ergänzendes OpenClaw-Plugin. TweetClaw wird von Xquik-dev gepflegt und ist
|
||||
hier nur als optionale Ergänzung aufgeführt, nicht als Abhängigkeit oder
|
||||
Empfehlung von last30days.
|
||||
|
||||
### Manuell (für Entwickler)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
Der Symlink hält die Installation beim Bearbeiten mit deinem Arbeitsverzeichnis synchron – erneutes Kopieren entfällt. Für `claude.ai` baust du die `.skill`-Datei aus dem Quellcode: `bash skills/last30days/scripts/build-skill.sh` erzeugt `dist/last30days.skill`.
|
||||
|
||||
Reddit (mit Kommentaren), Hacker News, Polymarket und GitHub funktionieren sofort. Null Konfiguration. Führe `/last30days` einmal aus, und der Setup-Assistent schaltet in 30 Sekunden weitere Quellen frei, darunter die kostenlosen CLIs für arXiv und Techmeme.
|
||||
|
||||
## Bring deine eigenen Schlüssel mit
|
||||
|
||||
Diese Plattformen haben nichts miteinander zu tun. X weiß nicht, was Reddit denkt. YouTube sieht TikTok nicht. Aber du kannst deine eigenen API-Schlüssel und Browser-Tokens mitbringen – und hast auf einen Schlag Zugriff auf alle gleichzeitig.
|
||||
|
||||
| Quellen | Was du brauchst | Kosten |
|
||||
|---------|---------------|------|
|
||||
| Reddit (mit Kommentaren) + HN + Polymarket + GitHub + StockTwits | Nichts | Kostenlos |
|
||||
| arXiv + Techmeme | Kostenlose CLIs, die das Erst-Setup automatisch installiert | Kostenlos |
|
||||
| X / Twitter | In einem beliebigen Browser bei x.com anmelden, oder `XQUIK_API_KEY` / `XAI_API_KEY` setzen | Browser-Cookies sind kostenlos; Schlüssel hängen vom Anbieter ab |
|
||||
| YouTube | `brew install yt-dlp` | Kostenlos |
|
||||
| Bluesky | App-Passwort von bsky.app | Kostenlos |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + YouTube-Kommentare | Ein ScrapeCreators-Schlüssel | 10.000 kostenlose Aufrufe, danach nutzungsabhängig |
|
||||
| Xiaohongshu (RED) | Ein eingeloggtes x-mcp-Browser-Plugin oder einen `xiaohongshu-mcp`-Dienst laufen lassen und die Quelle mit `--search xhs` pro Durchlauf oder `INCLUDE_SOURCES=xiaohongshu` in `.env` zuschalten; last30days probiert automatisch `http://localhost:18060` und danach `http://host.docker.internal:18060`, oder du setzt `XIAOHONGSHU_API_BASE` für eine eigene URL | Kein last30days-API-Schlüssel nötig; hängt von deinem lokalen Browser-Session-Dienst ab |
|
||||
| DripStack (Premium-Finanznewsletter) | Zuschaltbar: `--search dripstack` pro Durchlauf, oder `INCLUDE_SOURCES=dripstack` in `.env` | Kein Schlüssel; kostenlose öffentliche Such-API |
|
||||
| Perplexity Sonar / Search API / Deep Research | Ein Perplexity-Schlüssel, oder ein OpenRouter-Schlüssel als Sonar-Fallback | Nutzungsabhängig |
|
||||
| Websuche | Ein Brave-Search-Schlüssel | 2.000 kostenlose Anfragen pro Monat |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
|
||||
Unter macOS kannst du Schlüssel im System-Schlüsselbund statt in einer `.env`-Datei ablegen. Die Skill liest sie automatisch aus, allerdings mit der niedrigsten Priorität – bei einer Kollision gewinnen weiterhin `.env`-Dateien und die Prozessumgebung.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Die Einträge werden für den aktuellen Benutzer unter dem Dienstnamen `last30days-<KEY>` gespeichert. Auf Nicht-Darwin-Plattformen tut der Loader nichts, für Linux- und Windows-Nutzer ändert sich also am Verhalten nichts.
|
||||
|
||||
Du hast bereits Schlüssel unter anderen Keychain-Dienstnamen? Dann setz das nicht geheime Mapping `LAST30DAYS_KEYCHAIN_ALIASES`, das in [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items) beschrieben ist, statt Geheimnisse zu kopieren.
|
||||
|
||||
Die vollständige Schlüsselmatrix pro Quelle, die Priorität der Reasoning-Anbieter und die Priorität der Websuche-Backends stehen in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Konfiguration
|
||||
|
||||
Zwei Dinge, die du vermutlich schon am ersten Tag wissen willst:
|
||||
|
||||
**Wo die Rechercheergebnisse landen.** `LAST30DAYS_MEMORY_DIR` zeigt standardmäßig auf `~/Documents/Last30Days/` (unter Windows: `C:\Users\<you>\Documents\Last30Days\`). Überschreib das, indem du die Umgebungsvariable in deiner Shell auf einen beliebigen Pfad setzt, oder mit `--save-dir <path>` pro Durchlauf. Nutze `--output <file>`, wenn du das gerenderte Ergebnis an einem exakten Pfad brauchst – im Format, das `--emit` vorgibt. Mit `--save-suffix=<name>` hältst du mehrere Varianten desselben Themas auseinander (etwa pro Kunde). Jeder Durchlauf mit `--save-dir` erzeugt `<slug>-raw[-suffix].md`. Mit `python3 skills/last30days/scripts/last30days.py --preflight` siehst du vor einer Recherche, welche Dateien geschrieben würden.
|
||||
|
||||
**Strukturierte Ausgabe für Agenten und Workflows.** Bitte `/last30days` um maschinenlesbares JSON, dann bekommst du das stabile, versionierte Agentenprofil. Für den direkten Einsatz der Engine in Skripten oder in der Entwicklung führst du `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json` aus; `--json-profile=raw` brauchst du nur, wenn du den unversionierten internen `Report`-Dump willst. Siehe die [Feldreferenz des JSON-Exports samt Versionierungsrichtlinie](docs/reference/json-export.md).
|
||||
|
||||
**Discovery ohne festes Thema.** Frag `/last30days what's trending in AI agents?`, um ein sortiertes Discovery-Briefing zu bekommen, statt ein Thema zu recherchieren, das du ohnehin kennst. Auf einem Agenten-Host läuft dafür das dreistufige, vom Host bewertete Protokoll (das Modell benennt Themen, filtert Müll heraus, bewertet ihre Relevanz und schreibt die inhaltlichen Ansätze). Für den direkten Einsatz der Engine in Skripten oder per Cron führst du `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` aus (einmaliger Lauf: deterministische Themennamen, keine Ansätze); mit `--emit=json` bekommst du den versionierten Discovery-Vertrag. Discovery schließt ein positionsbasiertes Thema und `--drill` gegenseitig aus.
|
||||
|
||||
**Trendbeobachtung über mehrere Durchläufe.** Der Standardmodus erzeugt pro Durchlauf einen frischen Markdown-Snapshot. Um Erkenntnisse über die Zeit zu sammeln, hängst du `--store` an, damit sie in einer SQLite-Datenbank landen, und nutzt dann [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) für geplante Durchläufe (auf Wunsch mit Zustellung per Slack oder Webhook bei neuen Funden) sowie [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) für tägliche oder wöchentliche Zusammenfassungen. Das vollständige Taktmuster steht in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
**Eine abonnierbare Recherche-Bibliothek.** Bitte `/last30days`, deinen Bibliotheks-Feed zu bauen, oder nutze für Skripting und Entwicklung direkt `python3 skills/last30days/scripts/last30days.py library feed`. Das verwandelt gespeicherte Briefings in eine `index.html`, ein lokales Atom-`feed.xml` und lesbare Briefing-Seiten. Hänge `--publish` nur an, wenn der HTML-Index und die Briefing-Seiten gehostet werden sollen; das Veröffentlichen ist eine bewusste Entscheidung und standardmäßig öffentlich. Damit der Atom-Feed wirklich abonnierbar wird, hoste das erzeugte Ausgabeverzeichnis bei einem statischen Anbieter wie GitHub Pages.
|
||||
|
||||
**Durchsuche alles, was du schon recherchiert hast.** Frag `/last30days search my library for MCP servers` oder `/last30days have I researched MCP servers before?`. Für den direkten Einsatz der Engine führst du `python3 skills/last30days/scripts/last30days.py library search "MCP servers"` aus. Die Suche läuft offline und deterministisch: Sie indexiert nach und nach dieselben gespeicherten Briefings, die auch der Bibliotheks-Feed nutzt, führt passende Treffer aus dem Store je Durchlauf zusammen und gruppiert die Ergebnisse nach Thema und Datum. Neue Durchläufe blenden außerdem einen kompakten Abschnitt **From your library** („aus deiner Bibliothek“) ein, wenn frühere Recherchen das aktuelle Thema überschneiden; mit `LAST30DAYS_LIBRARY_CONTEXT=off` schaltest du diesen passiven Kontext ab.
|
||||
|
||||
Wrapper-Skripte pro Kunde, eigene Kategorie-Subreddits und der experimentelle Beta-Kanal für Anpassungen in Arbeit sind ebenfalls in [CONFIGURATION.md](CONFIGURATION.md) dokumentiert.
|
||||
|
||||
## Showcase: Recherche-Feeds aus der Community
|
||||
|
||||
Du hast mit last30days ein wiederkehrendes KI-Update, eine Marktbeobachtung oder eine herrlich spezielle Obsession veröffentlicht? Teil die URL deiner öffentlichen Bibliothek – oder die Atom-URL, sobald `feed.xml` bei einem statischen Anbieter liegt – im [Showcase-Thread der Community](https://github.com/mvanhorn/last30days-skill/issues/532). Community-Feeds werden hier verlinkt, sobald ihre Besitzer sie einreichen; bis dahin ist der Thread die Sammelstelle.
|
||||
|
||||
## So funktioniert es
|
||||
|
||||
1. **Du tippst ein Thema ein.** Person, Unternehmen, Produkt, Technologie, „X vs Y“. Alles ist möglich.
|
||||
2. **Der Agent klärt, wer zählt.** Er findet X-Handles (auch die von Gründerinnen und Gründern), GitHub-Repos, Subreddits, TikTok-Hashtags und YouTube-Kanäle. Bei „Kanye West“ weiß er, dass r/hiphopheads, @kanyewest und „bully review“ auf YouTube dazugehören. Bei „OpenClaw“ löst er openclaw/openclaw auf GitHub auf und holt die aktuellen Sternzahlen.
|
||||
3. **Alle Quellen werden parallel durchsucht.** Erweiterung über mehrere Suchanfragen. Ergebnisse gewichtet nach Engagement, Relevanz und Aktualität.
|
||||
4. **Die Tiefe, die sonst niemand hat.** Vollständige YouTube-Transkripte aus Reaktionsvideos. Die besten Reddit-Kommentare samt Upvote-Zahlen. TikTok-Captions. Polymarket-Quoten. Nicht nur Titel und Links.
|
||||
5. **Dieselbe Geschichte, zusammengeführt.** Das Wireless Festival auf Reddit angekündigt, auf X diskutiert, Ticketpreise auf TikTok – das ergibt einen Cluster, nicht drei getrennte Einträge.
|
||||
6. **Zu einem Briefing verdichtet.** Auf konkreten Daten fußend. Nach Quelle belegt. Sortiert nach dem, womit Menschen wirklich interagieren. Nicht „hier ist, was ich gefunden habe“, sondern „hier ist, was zählt“.
|
||||
7. **Danach wird es dein Experte.** Nach einem einzigen Durchlauf weiß deine Claude-Sitzung alles, was die Community weiß. Stell Rückfragen. Lass sie Prompts schreiben, E-Mails entwerfen, Reisen planen, Systeme entwerfen – immer verankert in dem, was gerade wirklich stimmt.
|
||||
|
||||
## Was die Leute sagen
|
||||
|
||||
> „Ich habe eine Claude-Code-Skill gefunden, die zu jedem Thema die letzten 30 Tage auf Reddit, X, YouTube und HN recherchiert. Und dann schreibt sie dir die Prompts. Vor jedem Text, den ich schreibe, habe ich das bisher von Hand auf Reddit und X gemacht. Tab für Tab. Thread für Thread. Genau das ist der Teil, der 90 Minuten frisst. Der fällt jetzt weg.“ – @itsjasonai
|
||||
|
||||
> „Diese eine Skill hat meinen kompletten Recherche-Workflow ersetzt. Du gibst ihr ein Thema, sie holt sich von Reddit, X und dem Web, worüber die Leute wirklich reden. Keine alten Blogbeiträge. Echte Gespräche aus den letzten 30 Tagen.“ – @itswilsoncharles
|
||||
|
||||
> „5 der 10 Trending-Repos heute auf GitHub sind Claude-Tools. Nummer 1: mvanhorn/last30days-skill“ – @yieldhunter95
|
||||
|
||||
## Open Source
|
||||
|
||||
MIT-Lizenz. Kein Tracking. Keine Analytics. Deine Recherche bleibt auf deinem Rechner. Über 2.700 Tests.
|
||||
|
||||
Gebaut mit Python 3.12+, yt-dlp, Node.js (mitgelieferter Bird-Client für die X-Suche) und der ScrapeCreators-API. Architektur der v3-Engine von [@j-sperling](https://github.com/j-sperling).
|
||||
|
||||
Wie du einen PR aufmachst, steht in [CONTRIBUTING.md](CONTRIBUTING.md), die vollständige Liste der Community-Beitragenden in [CONTRIBUTORS.md](CONTRIBUTORS.md) und die Versionshistorie in [CHANGELOG.md](CHANGELOG.md).
|
||||
|
||||
## Sternverlauf
|
||||
|
||||
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
# /last30days
|
||||
|
||||
[English](README.md) | [Français](README.fr.md) | [Deutsch](README.de.md) | Español | [Português (Brasil)](README.pt-BR.md) | [日本語](README.ja.md) | [简体中文](README.zh-CN.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://trendshift.io/repositories/21997" target="_blank">
|
||||
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**Un buscador dirigido por un agente de IA que puntúa por votos positivos, likes y dinero real, no por redacciones.**
|
||||
|
||||
Este README documenta el pipeline v3 actual. La especificación de ejecución de la skill vive en [skills/last30days/SKILL.md](skills/last30days/SKILL.md), que es la referencia definitiva sobre el comportamiento de los comandos y la configuración.
|
||||
|
||||
**Claude Code (recomendado — actualizaciones automáticas vía marketplace):**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, o cualquiera de los 50+ hosts de [Agent Skills](https://agentskills.io):**
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
(`-g` instala de forma global para tu usuario, así que la tienes disponible en todos tus proyectos. Omite ese flag si prefieres limitar la instalación a un proyecto.)
|
||||
|
||||
Más formas de instalarlo (claude.ai web, OpenClaw, manual) en la sección [Instalación](#instalación) de más abajo.
|
||||
|
||||
Cero configuración. Reddit, HN, Polymarket y GitHub funcionan de inmediato. Ejecútalo una vez y el asistente de configuración desbloquea X, YouTube, TikTok, arXiv, Techmeme y más en 30 segundos.
|
||||
|
||||
---
|
||||
|
||||
Los votos positivos de Reddit. Los likes de X. Las transcripciones de YouTube. La interacción en TikTok. Las cuotas de Polymarket, respaldadas por dinero real y por información privilegiada. Eso son millones de personas votando cada día con su atención y su cartera. /last30days lo busca todo en paralelo, lo puntúa según aquello con lo que la gente interactúa de verdad, y un agente de IA hace de juez para sintetizarlo en un único informe.
|
||||
|
||||
Google agrega redacciones. /last30days busca personas.
|
||||
|
||||
Esta búsqueda no la consigues en ningún otro sitio, porque ninguna IA tiene acceso a todo. Google no toca los comentarios de Reddit ni las publicaciones de X. ChatGPT tiene un acuerdo con Reddit, pero no puede buscar en X ni en TikTok. Gemini tiene YouTube, pero no Reddit. Claude no tiene ninguno de forma nativa. Cada plataforma es un jardín amurallado con su propia API, sus propios tokens y su propia autenticación. Pero tú puedes aportar tus claves y tus sesiones de navegador y, de golpe, un agente de IA las consulta todas a la vez, las compara entre sí y te dice qué importa de verdad.
|
||||
|
||||
Ese es el desbloqueo. No se trata de un buscador mejor, sino de una docena de plataformas incomunicadas que un agente conecta entre sí.
|
||||
|
||||
```
|
||||
/last30days Peter Steinberger
|
||||
```
|
||||
|
||||
Mañana tienes una reunión. Buscas a la persona en Google. Te sale su LinkedIn de 2023. /last30days te da lo que está haciendo de verdad este mes: se ha incorporado a OpenAI para trabajar en Codex, pelea contra el veto de Anthropic a los agentes de terceros, ha entregado 23 PR con un 85 % de tasa de merge, construye «LobsterOS» para controlar agentes entre dispositivos, y un hilo de r/ClaudeCode llegó a 569 votos positivos debatiendo si es un héroe o un «insoportable». Todo repartido entre publicaciones de X, hilos de Reddit, transcripciones de YouTube y commits de GitHub. Nada de eso estaba en Google.
|
||||
|
||||
## Por qué existe esto
|
||||
|
||||
Lo construí para no quedarme atrás en IA. Todo cambia cada día y los frikis de Reddit y de X siempre se enteran primero. Necesitaba mejores prompts, y los datos de entrenamiento siempre iban meses por detrás de lo que la comunidad ya había averiguado.
|
||||
|
||||
Pero acabó siendo algo más grande. Ahora lo lanzo antes de una llamada comercial, para conocer la verdad de los últimos 30 días sobre una empresa. Antes de una reunión, para leer los tuits recientes y las transcripciones de podcasts de la otra persona. Antes de un viaje a Disney World, para saber qué atracciones están cerradas y qué opina la comunidad sobre Genie+. Antes de construir nada, para saber con qué problemas se está encontrando la gente de verdad.
|
||||
|
||||
Si te vas a reunir con un CEO, ¿te has leído todos sus tuits y todas sus transcripciones de YouTube de los últimos 30 días? Yo sí.
|
||||
|
||||
## Fuentes, puntuadas por la gente
|
||||
|
||||
| Fuente | Lo que te dice la gente |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | La opinión sin filtros. Los mejores comentarios con su recuento real de votos positivos, gratis y sin clave de API. Las opiniones reales que Google entierra. |
|
||||
| **X / Twitter** | La reacción en caliente, el hilo del experto, la primera respuesta a una noticia de última hora. Los primeros en enterarse, los primeros en discutir. |
|
||||
| **YouTube** | El análisis a fondo de 45 minutos. Transcripciones completas, rastreadas para sacar las 5 frases citables que importan. |
|
||||
| **TikTok** | El creador que llega a 3,6 millones de personas con una lectura que nunca encontrarás en Google. |
|
||||
| **Instagram Reels** | La mirada de los influencers, con transcripción de lo que dicen. La señal de la cultura visual. |
|
||||
| **Hacker News** | El consenso de los desarrolladores. 825 puntos, 899 comentarios. Donde la gente técnica discute de verdad. |
|
||||
| **Polymarket** | No son opiniones. Son cuotas. Respaldadas por dinero real. 96 % de probabilidad en ventas de un álbum. 4 % en una adquisición. |
|
||||
| **GitHub** | Para personas: ritmo de PR, mejores repositorios por estrellas, notas de versión. Para temas: issues y discusiones. |
|
||||
| **Digg** | Grupos de noticias seleccionados del ranking AI 1000 de Digg (unas 1000 cuentas de IA con mucha señal en X), con citas atribuibles integradas y sin necesidad de autenticarte en X. Se activa solo cuando `digg-pp-cli` está en el PATH. |
|
||||
| **arXiv** | Los artículos científicos que hay detrás del ruido. Investigación nueva dentro de la ventana, gratis y sin clave de API. Se activa solo cuando `arxiv-pp-cli` está en el PATH (la configuración inicial lo instala). |
|
||||
| **Techmeme** | La capa editorial de la actualidad tecnológica, acotada a tu ventana de 30 días. Gratis y sin clave de API. Se activa solo cuando `techmeme-pp-cli` está en el PATH (la configuración inicial lo instala). |
|
||||
| **LinkedIn** | La señal profesional. Publicaciones y artículos, con los artículos ponderados como señal fuerte. |
|
||||
| **StockTwits** | El sentimiento de los traders. Se activa automáticamente cuando tu tema es un ticker o una criptomoneda. |
|
||||
| **Threads** | La capa de texto posterior a Twitter. Conversaciones de creadores y marcas. |
|
||||
| **Pinterest** | Descubrimiento visual. Pines, guardados y comentarios sobre productos e ideas. |
|
||||
| **Xiaohongshu (RED)** | Señales chinas sobre estilo de vida, productos y creadores. Se pide de forma explícita con `--search xhs` cuando tienes corriendo en local un plugin de navegador x-mcp con sesión iniciada o un servicio `xiaohongshu-mcp`. |
|
||||
| **Bluesky** | La capa social descentralizada. Publicaciones de AT Protocol surgidas de la migración posterior a Twitter. |
|
||||
| **Perplexity** | La síntesis fundamentada de Sonar, los resultados en bruto de la Search API y Deep Research. |
|
||||
| **Web** | La cobertura editorial, las comparativas de los blogs. Una señal entre muchas, no la única. |
|
||||
|
||||
La comunidad no para de sumar fuentes. Truth Social y otras fuentes de nicho ya están en el motor, y vienen más.
|
||||
|
||||
Un hilo de Reddit con 1.500 votos positivos es una señal más fuerte que una entrada de blog que no leyó nadie. Un TikTok con 3,6 millones de visualizaciones dice más sobre lo que es culturalmente relevante que cualquier nota de prensa. Unas cuotas de Polymarket respaldadas por 66.000 dólares de volumen son más difíciles de rebatir que la corazonada de un tertuliano.
|
||||
|
||||
La síntesis ordena según aquello con lo que la gente real ha interactuado de verdad. Relevancia social, no relevancia SEO.
|
||||
|
||||
## Para qué lo usa la gente en realidad
|
||||
|
||||
**Antes de una reunión.** `/last30days Peter Steinberger` — se ha incorporado al equipo de Codex de OpenAI, pelea contra el veto de Anthropic a los agentes de terceros, 23 PR mergeadas con un 85 % de tasa de merge en GitHub, construye LobsterOS para controlar agentes entre dispositivos. r/ClaudeCode: «Desde que salió OpenClaw, todo el mundo sabía que, si lo pasabas por algo que no fuera la API, acabarías baneado» (227 votos positivos). Eso no está en LinkedIn.
|
||||
|
||||
**Para leer señales de contratación.** `/last30days Listen Labs --hiring-signals` — las ofertas de empleo y las páginas de carreras actuales se convierten en pruebas citadas de un cambio de prioridades: contratación en seguridad para empresa, customer success, infraestructura o expansión de producto. El informe dice lo que la contratación parece señalar, no lo que la hoja de ruta va a entregar.
|
||||
|
||||
**Para encontrar el tema antes de su pico.** Pregunta `/last30days what's exploding in AI agents?` y la skill cambia a modo descubrimiento: el motor barre los listados por categoría de Reddit, la portada y las mejores historias de Hacker News, el feed AI 1000 de Digg y X si estás autenticado; tu agente evalúa las candidaturas (nombres, filtrado de ruido, interés real) y escribe enfoques para pódcast o para un artículo en X; después obtienes entre 5 y 10 temas ordenados por velocidad. Cada resultado incluye cifras de varias fuentes, una etiqueta de impulso y un comando `/last30days "<topic>"` listo para lanzar.
|
||||
|
||||
**Cuando sale algo nuevo.** `/last30days Kanye West` — el Reino Unido le bloqueó el visado, el Wireless Festival se canceló, los patrocinadores huyeron. Pero BULLY debutó en el número 2 del Billboard. Fantano volvió de su «Yay sabbatical» para reseñarlo (653.000 visualizaciones). En el SoFi Homecoming sacó al escenario a Lauryn Hill y a Travis Scott para 44 canciones. Polymarket: «¿Volverá Kanye a tuitear?» 86 % sí. 23 hilos de Reddit, 17 vídeos de YouTube, 86.000 votos positivos.
|
||||
|
||||
**Para comparar herramientas.** `/last30days OpenClaw vs Hermes vs Paperclip` — «No son competidores, son capas.» OpenClaw es la capa de ejecución (351.000 estrellas en GitHub, en producción), Hermes es el cerebro que se mejora a sí mismo (31.000 estrellas), Paperclip es el organigrama (49.000 estrellas). El número de estrellas se saca en directo de la API de GitHub, no de entradas de blog caducadas. Tabla comparativa con arquitectura, memoria, seguridad y caso de uso ideal. Según @IMJustinBrooke: «OpenClaw = Charmander, Hermes = Charizard.»
|
||||
|
||||
**Para entender el mundo.** `/last30days Iran vs USA` — día 38 de la guerra. El ultimátum de Trump, con plazo hasta el martes, para que Irán reabra el estrecho de Ormuz. Dos aviones de combate estadounidenses derribados. El petróleo a 126 dólares el barril. La AIE lo calificó como «la mayor interrupción de suministro de la historia del mercado mundial del petróleo». Polymarket: alto el fuego antes del 31 de diciembre al 74 %. 27 publicaciones de X, 10 vídeos de YouTube, 20 mercados de predicción.
|
||||
|
||||
**Antes de un viaje.** `/last30days Universal Epic Universe` — la ampliación ya está en obras. Licencia «Project 680» presentada. El espectáculo de fuegos artificiales está confirmado por la infraestructura, pero sin anunciar. Tiempos de espera: Mine-Cart Madness promedia 148 minutos. Todavía no hay pase anual, y los vecinos están hartos. Stardust Racers cerrada por reforma hasta el 5 de abril.
|
||||
|
||||
**Para aprender algo rápido.** `/last30days Nano Banana Pro prompting` — los prompts estructurados en JSON están sustituyendo al amontonamiento de etiquetas. El formato anidado de @pictsbyai evita el «concept bleeding». Editar gana a regenerar. Y después te escribe un prompt de producción aplicando exactamente lo que la comunidad ha dicho que funciona.
|
||||
|
||||
## Novedades
|
||||
|
||||
Desde el anuncio de la v3.3 en mayo y hasta la v3.11.1 (julio de 2026): 175 PR mergeadas —122 de ellas de 52 colaboradores de la comunidad— repartidas en 15 versiones. Esto es lo que ha entrado.
|
||||
|
||||
### Ciudadano de primera en OpenAI Codex
|
||||
|
||||
/last30days ya es un plugin nativo de Codex con configuración guiada: no es un port, es un ciudadano de primera. Las citas tienen en cuenta el renderizador, así que la salida en Codex se lee como un informe y no como una sopa de URL (#694), y el mismo motor funciona en Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw y 50+ hosts de Agent Skills. Manifiesto del plugin de Codex por [@rfoust](https://github.com/rfoust) (#686), corrección de autenticación en Codex por [@tmchow](https://github.com/tmchow) (#698).
|
||||
|
||||
### arXiv, Techmeme y Digg: gratis y sin claves de API
|
||||
|
||||
arXiv aporta los artículos científicos que hay detrás del ruido y Techmeme la capa editorial de la actualidad tecnológica: gratis, sin una sola clave, y la configuración inicial instala sus CLI para que se activen solas (#709). Los grupos de noticias AI 1000 de Digg llegan igual, sin autenticarte en X: la configuración instala por ti la CLI gratuita de Digg (#590). Trustpilot está disponible como opción para investigar marcas de consumo.
|
||||
|
||||
### Reddit gratis, con puntuaciones reales y mejores comentarios
|
||||
|
||||
La API pública .json de Reddit desapareció; la vía gratuita volvió más fuerte. RSS sin clave y scraping de shreddit (#457), descubrimiento de subreddits específicos con recuentos reales de votos positivos vía arctic-shift (#696), y un umbral de relevancia para que una publicación viral fuera de tema no secuestre tu informe (#488, gracias [@rzachsmith](https://github.com/rzachsmith)). Sin clave de API. Puntuaciones reales. Con los mejores comentarios incluidos.
|
||||
|
||||
### Los mejores comentarios en cada informe
|
||||
|
||||
Los comentarios son ya una capa activada por defecto en todas las fuentes: comentarios de Instagram con diversidad basada en el ranking, para que cinco opiniones rotundas no salgan todas de la misma publicación (#751), comentarios de YouTube más un respaldo de transcripción vía ScrapeCreators para cuando yt-dlp falla (#637), y comentarios votados por la comunidad ponderados dentro de Best Takes, para que las mejores frases sobrevivan a la puntuación (#592, #608).
|
||||
|
||||
### Un único comando doctor
|
||||
|
||||
Pide una revisión y doctor comprueba todas las fuentes y receta los arreglos exactos: qué clave falta, qué CLI no está en el PATH, qué cookie ha caducado (#753). Se acabó adivinar por qué X ha devuelto tan poco.
|
||||
|
||||
### La búsqueda en X, reconstruida
|
||||
|
||||
El pipeline de X se rehízo de arriba abajo: carriles FROM y ABOUT para que se posicionen tanto las publicaciones de una persona como la conversación sobre ella (#610), desambiguación de subconsultas según la persona buscada (#611), verificación de la autoría de primera mano con ranking por señales de interacción (#613), y una única fuente X con conmutación automática entre backends (#622). Además, un `--diagnose` honesto que comprueba de verdad la autenticación (#609).
|
||||
|
||||
### Se han sumado más fuentes
|
||||
|
||||
LinkedIn vía ScrapeCreators, con los artículos como señal fuerte ([@ravstr](https://github.com/ravstr), #702). StockTwits se activa automáticamente en temas de tickers y cripto ([@wtiwana](https://github.com/wtiwana), #658). Perplexity ha ganado modos de API directos y Deep Research asíncrono ([@sk-holmes](https://github.com/sk-holmes), #629).
|
||||
|
||||
### Endurecido por la comunidad
|
||||
|
||||
La oleada de seguridad fue casi por completo trabajo de la comunidad: correcciones de XSS almacenado en el renderizador HTML ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), archivos temporales de cookies blindados, CI endurecida frente a ataques a la cadena de suministro con OpenSSF Scorecard y atestación de procedencia de las builds ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), análisis con Semgrep y OSV-Scanner más un control de revisión de dependencias en cada PR ([@23241a6749](https://github.com/23241a6749)), un mínimo de cobertura de pruebas fijado al 60 % y elevado desde entonces al 84 % ([@gourab5139014](https://github.com/gourab5139014)), y un análisis de seguridad de Hermes que ya no arroja ningún hallazgo CRITICAL (#768).
|
||||
|
||||
### Llega más lejos
|
||||
|
||||
Hebreo y otros idiomas no latinos ([@dudyme](https://github.com/dudyme)). Tokenización adaptada a CJK para las fuentes chinas ([@An-idd](https://github.com/An-idd)). Una oleada de compatibilidad con Windows. Extracción de cookies en toda la familia Chromium —Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov))— además del llavero de macOS y pass(1) en Linux como orígenes de credenciales. Consulta histórica hacia atrás con `--as-of` ([@chiyi-creator](https://github.com/chiyi-creator)). Instalación automática de Python 3.12 mediante uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` para leer las páginas de empleo de una empresa. Diferencias de la lista de seguimiento entre ejecuciones.
|
||||
|
||||
### Lo que ya venía de serie desde la v3
|
||||
|
||||
Los cimientos de la v3 siguen todos aquí: el cerebro previo a la investigación, que identifica las cuentas, subreddits y hashtags correctos antes de que salga una sola llamada a la API (obra de [@j-sperling](https://github.com/j-sperling)); la puntuación Best Takes, que valora el humor y la viralidad además de la relevancia; la fusión de clústeres entre fuentes; las comparativas en una sola pasada («CLI vs MCP» en 3 minutos, no en 12); las comparativas `--competitors` descubiertas de forma automática; el modo persona de GitHub (`--github-user=steipete`); el modo ELI5 («eli5 on» después de cualquier ejecución); y los informes HTML autocontenidos y compartibles (`--emit=html`). Los ajustes de configuración están en [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Instalación
|
||||
|
||||
| Entorno | Instalación | Actualizaciones |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recomendado) | `/plugin marketplace add mvanhorn/last30days-skill` | Automáticas vía marketplace, o `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` y después `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, o cualquiera de los 50+ hosts de [Agent Skills](https://agentskills.io)** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Descarga `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) y súbelo desde claude.ai > Customize > Skills > + > Create skill > Upload a skill | Volver a descargar y volver a subir |
|
||||
| **Claude Desktop** | [Descarga el `.mcpb` de tu plataforma](https://github.com/mvanhorn/last30days-skill/releases/latest) y arrástralo a Settings > Extensions | Volver a descargar y arrastrar el nuevo paquete |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recomendado)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Es la opción recomendada porque el marketplace de Claude Code se encarga de las actualizaciones por ti: la caché del plugin está versionada y se refresca sola cuando se publica una versión nueva. Ejecuta `claude plugin update last30days@last30days-skill` para forzar una comprobación.
|
||||
|
||||
Si prefieres usar la vía de instalación de Agent Skills en Claude Code, también está soportada:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
El plugin nativo y la instalación con `npx skills` pueden convivir. Ojo: Claude Code no deduplica entre métodos de instalación. Si tienes activos a la vez el plugin del marketplace y la copia de `npx skills`, `/last30days` aparecerá dos veces. Usa un solo método de instalación por máquina.
|
||||
|
||||
### Grok (xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) instala last30days como plugin nativo. La instalación directa sigue el repositorio:
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
O añade este repositorio como fuente de marketplace y luego instálalo por nombre de plugin:
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
Añade `--trust` para saltarte la confirmación de instalación. Actualiza con `grok plugin update last30days`. Grok también lee los manifiestos de Claude Code por compatibilidad; el par nativo `.grok-plugin/` es la vía principal, y es a lo que apunta una entrada oficial en el [marketplace de xAI](https://github.com/xai-org/plugin-marketplace). `npx skills add` sigue siendo una alternativa válida para cualquier host.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI y otros hosts de Agent Skills
|
||||
|
||||
Instálalo con la CLI abierta de [Agent Skills](https://agentskills.io): soporta 50+ hosts, entre ellos `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose` y más (lista completa en el [repositorio vercel-labs/skills](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
El flag `-g` (global) instala en tu directorio de usuario, de modo que la skill queda disponible en todos los proyectos. Sin `-g`, `npx skills` instala solo en el proyecto, dentro de `./.skills/` (y se versiona con el repositorio). Para una herramienta que sirve para investigar el mundo entero, lo que quieres es la instalación global.
|
||||
|
||||
Codex de escritorio y otros hosts que trabajan a nivel de carpeta funcionan tanto en carpetas normales como en repositorios Git. Antes de la primera investigación, pídele al agente anfitrión que ejecute el `scripts/last30days.py --preflight` incluido desde el directorio de la skill cargada; en un clon del código fuente, el comando equivalente es `python3 skills/last30days/scripts/last30days.py --preflight`. Te muestra de dónde sale la configuración, qué cookies del navegador se leerían, qué archivos se escribirían, qué comandos opcionales hay y qué configuración de proyecto se ignora, todo ello sin leer cookies, sin escribir archivos y sin lanzar ninguna investigación.
|
||||
|
||||
Por defecto se instala para el host que detecte `npx skills`. Para apuntar a uno concreto (o a varios):
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Para actualizar más adelante:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
O actualiza todo lo que hayas instalado globalmente con `npx skills`:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
Puedes listarlo y desinstalarlo con `npx skills list -g` y `npx skills remove last30days -g`.
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Descarga `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) de la última versión publicada
|
||||
2. Entra en [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. Pulsa el botón `+` del panel de Skills, luego `Create skill` > `Upload a skill`, y busca o arrastra el archivo
|
||||
|
||||
Activa antes «Code execution and file creation» en Capabilities: sin eso, las skills no se ejecutan.
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop instala `/last30days` como servidor MCP mediante un paquete `.mcpb` (un paquete de Model Context Protocol de un solo clic).
|
||||
|
||||
1. Entra en la [última versión publicada](https://github.com/mvanhorn/last30days-skill/releases/latest) y descarga el `.mcpb` de tu plataforma:
|
||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Abre Claude Desktop, ve a Settings > Extensions y arrastra el archivo ahí.
|
||||
3. Cuando te las pida, pega las claves de API de las fuentes que quieras activar. Todos los campos son opcionales: si los saltas todos, el motor se queda en modo solo web. Las claves se guardan en el llavero de tu sistema operativo.
|
||||
4. Reinicia Claude Desktop. Pídele a Claude que «investigue a Peter Steinberger», o cualquier otro tema, y llamará a la herramienta `research`.
|
||||
|
||||
**Requisito del anfitrión:** Python 3.12+ en el PATH. El paquete incluye el código del motor, pero usa tu intérprete de Python local. En Windows, instálalo desde [python.org](https://www.python.org/downloads/); macOS y la mayoría de distribuciones de Linux ya traen una versión compatible.
|
||||
|
||||
**Las claves no se comparten con la skill de Claude Code.** Claude Desktop y Claude Code mantienen almacenes de credenciales separados a propósito. Si ya configuraste `~/.config/last30days/.env` para la skill de Claude Code, aquí tendrás que introducir esas mismas claves una vez.
|
||||
|
||||
La compatibilidad con Windows queda aplazada hasta resolver los puntos de entrada por plataforma del manifiesto; el seguimiento se hace en una incidencia aparte.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
Para flujos de acción en X/Twitter fuera de la investigación de `/last30days` —publicar
|
||||
tuits o respuestas, exportar seguidores, gestionar medios, monitorizar cuentas y
|
||||
resolver sorteos— usa [TweetClaw](https://github.com/Xquik-dev/tweetclaw) como
|
||||
plugin complementario de OpenClaw. TweetClaw lo mantiene Xquik-dev y aparece aquí
|
||||
únicamente como opción complementaria: no es una dependencia ni una recomendación
|
||||
de last30days.
|
||||
|
||||
### Manual (para desarrolladores)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
El enlace simbólico mantiene la instalación sincronizada con tu copia de trabajo a medida que editas, sin necesidad de volver a copiar nada. Para `claude.ai`, compila el archivo `.skill` desde el código fuente: `bash skills/last30days/scripts/build-skill.sh` genera `dist/last30days.skill`.
|
||||
|
||||
Reddit (con comentarios), Hacker News, Polymarket y GitHub funcionan de inmediato. Cero configuración. Ejecuta `/last30days` una vez y el asistente de configuración desbloquea más fuentes en 30 segundos, incluidas las CLI gratuitas de arXiv y Techmeme.
|
||||
|
||||
## Aporta tus propias claves
|
||||
|
||||
Estas plataformas no tienen ninguna relación entre sí. X no sabe lo que piensa Reddit. YouTube no ve TikTok. Pero tú puedes aportar tus claves de API y tus tokens de navegador y, de golpe, tienes acceso a todas a la vez.
|
||||
|
||||
| Fuentes | Lo que necesitas | Coste |
|
||||
|---------|---------------|------|
|
||||
| Reddit (con comentarios) + HN + Polymarket + GitHub + StockTwits | Nada | Gratis |
|
||||
| arXiv + Techmeme | CLI gratuitas, instaladas automáticamente por la configuración inicial | Gratis |
|
||||
| X / Twitter | Inicia sesión en x.com en cualquier navegador, o define `XQUIK_API_KEY` / `XAI_API_KEY` | Las cookies del navegador son gratis; las claves dependen del proveedor |
|
||||
| YouTube | `brew install yt-dlp` | Gratis |
|
||||
| Bluesky | Una contraseña de aplicación de bsky.app | Gratis |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + comentarios de YouTube | Una clave de ScrapeCreators | 10.000 llamadas gratis y luego pago por uso |
|
||||
| Xiaohongshu (RED) | Ten corriendo un plugin de navegador x-mcp con sesión iniciada o un servicio `xiaohongshu-mcp`, y activa la fuente con `--search xhs` por ejecución o con `INCLUDE_SOURCES=xiaohongshu` en `.env`; last30days prueba automáticamente `http://localhost:18060` y después `http://host.docker.internal:18060`, o usa `XIAOHONGSHU_API_BASE` para una URL propia | No hace falta clave de API de last30days; depende de tu servicio local de sesión de navegador |
|
||||
| DripStack (boletines financieros premium) | Opcional: `--search dripstack` por ejecución, o `INCLUDE_SOURCES=dripstack` en `.env` | Sin clave; API de búsqueda pública y gratuita |
|
||||
| Perplexity Sonar / Search API / Deep Research | Una clave de Perplexity, o una clave de OpenRouter como alternativa para Sonar | Pago por uso |
|
||||
| Búsqueda web | Una clave de Brave Search | 2.000 consultas gratis al mes |
|
||||
|
||||
### Llavero de macOS (opcional)
|
||||
|
||||
En macOS puedes guardar las claves en el llavero del sistema en lugar de en un archivo `.env`. La skill las recoge automáticamente como la fuente de menor prioridad: si hay conflicto, siguen ganando los archivos `.env` y el entorno del proceso.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Las entradas se guardan con el nombre de servicio `last30days-<KEY>` para el usuario actual. En plataformas que no son Darwin el cargador no hace nada, así que para quienes usan Linux o Windows no cambia el comportamiento.
|
||||
|
||||
¿Ya tienes claves guardadas con otros nombres de servicio en el llavero? Define el mapeo no secreto `LAST30DAYS_KEYCHAIN_ALIASES` que se describe en [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items), en lugar de copiar secretos.
|
||||
|
||||
Consulta [CONFIGURATION.md](CONFIGURATION.md) para ver la matriz completa de claves por fuente, el orden de prioridad de los proveedores de razonamiento y el de los backends de búsqueda web.
|
||||
|
||||
## Configuración
|
||||
|
||||
Dos cosas que seguramente querrás saber desde el primer día:
|
||||
|
||||
**Dónde se guardan los archivos de investigación.** `LAST30DAYS_MEMORY_DIR` apunta por defecto a `~/Documents/Last30Days/` (en Windows: `C:\Users\<you>\Documents\Last30Days\`). Puedes cambiarlo definiendo esa variable de entorno en tu shell con la ruta que quieras, o con `--save-dir <path>` en una ejecución concreta. Usa `--output <file>` cuando necesites el resultado renderizado en una ruta exacta, con el formato que elijas en `--emit`. Usa `--save-suffix=<name>` para mantener separadas varias variantes del mismo tema (por cliente, por ejemplo). Cada ejecución con `--save-dir` genera `<slug>-raw[-suffix].md`. Ejecuta `python3 skills/last30days/scripts/last30days.py --preflight` para revisar qué se va a escribir antes de lanzar una investigación.
|
||||
|
||||
**Salida estructurada para agentes y flujos de trabajo.** Pídele a `/last30days` JSON legible por máquina y obtendrás el perfil de agente estable y versionado. Para usar el motor directamente en scripts o en desarrollo, ejecuta `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json`; añade `--json-profile=raw` solo si necesitas el volcado interno sin versionar de `Report`. Consulta la [referencia de campos de la exportación JSON y la política de versionado](docs/reference/json-export.md).
|
||||
|
||||
**Descubrimiento sin tema.** Pregunta `/last30days what's trending in AI agents?` para obtener un informe de descubrimiento ordenado, en lugar de investigar un tema que ya conoces. En un host con agente esto ejecuta el protocolo de tres comandos arbitrado por el host (el modelo propone los temas, filtra el ruido, puntúa lo que merece la pena y escribe los enfoques de contenido). Para usar el motor directamente en scripts o en cron, ejecuta `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` (una sola pasada: nombres de tema deterministas, sin enfoques); añade `--emit=json` para el contrato de descubrimiento versionado. El descubrimiento es incompatible con un tema posicional y con `--drill`.
|
||||
|
||||
**Seguimiento de tendencias entre ejecuciones.** El modo por defecto genera una instantánea Markdown nueva en cada ejecución. Para ir acumulando hallazgos con el tiempo, añade `--store` y se guardarán en una base de datos SQLite; después usa [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) para ejecuciones programadas (con envío opcional por Slack o webhook cuando aparezcan hallazgos nuevos) y [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) para resúmenes diarios o semanales. El patrón de cadencia completo está en [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
**Una biblioteca de investigación a la que suscribirse.** Pídele a `/last30days` que genere el feed de tu biblioteca, o usa directamente `python3 skills/last30days/scripts/last30days.py library feed` para scripting y desarrollo. Convierte los informes guardados en un `index.html`, un `feed.xml` Atom local y páginas de informe legibles. Añade `--publish` solo cuando quieras alojar el índice HTML y las páginas de informe; publicar es una decisión explícita y por defecto es público. Para que el feed Atom se pueda seguir de verdad, aloja el directorio de salida generado en un alojamiento estático como GitHub Pages.
|
||||
|
||||
**Busca en todo lo que ya has investigado.** Pregunta `/last30days search my library for MCP servers` o `/last30days have I researched MCP servers before?`. Para usar el motor directamente, ejecuta `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`. La búsqueda es offline y determinista: indexa de forma incremental los mismos informes guardados que usa el feed de la biblioteca, fusiona las coincidencias registradas en el almacén de cada ejecución y agrupa los resultados por tema y fecha. Las ejecuciones nuevas muestran además una sección compacta **From your library** («desde tu biblioteca») cuando una investigación anterior se solapa con el tema actual; define `LAST30DAYS_LIBRARY_CONTEXT=off` para desactivar ese contexto pasivo.
|
||||
|
||||
Los scripts envoltorio por cliente, los subreddits de categoría personalizados y el canal beta experimental para personalizaciones en curso también están documentados en [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Escaparate: feeds de investigación de la comunidad
|
||||
|
||||
¿Has publicado con last30days una actualización periódica sobre IA, un seguimiento de mercado o una obsesión maravillosamente específica? Comparte la URL de tu biblioteca pública —o la URL de Atom, una vez alojado `feed.xml` en un alojamiento estático— en [el hilo de escaparate de la comunidad](https://github.com/mvanhorn/last30days-skill/issues/532). Los feeds de la comunidad se irán enlazando aquí a medida que sus autores los envíen; mientras tanto, el hilo es el punto de recogida.
|
||||
|
||||
## Cómo funciona
|
||||
|
||||
1. **Escribes un tema.** Una persona, una empresa, un producto, una tecnología, «X vs Y». Lo que sea.
|
||||
2. **El agente averigua quién importa.** Encuentra las cuentas de X (incluidas las de fundadores), los repositorios de GitHub, los subreddits, los hashtags de TikTok y los canales de YouTube. Para «Kanye West» sabe que hay que mirar r/hiphopheads, @kanyewest y «bully review» en YouTube. Para «OpenClaw» resuelve openclaw/openclaw en GitHub y trae el número de estrellas en directo.
|
||||
3. **Todas las fuentes se consultan en paralelo.** Expansión con varias consultas. Resultados puntuados por interacción, relevancia y frescura.
|
||||
4. **La profundidad que no tiene nadie más.** Transcripciones completas de YouTube de vídeos de reacción. Los mejores comentarios de Reddit con su recuento de votos positivos. Los textos de los TikTok. Las cuotas de Polymarket. No solo títulos y enlaces.
|
||||
5. **La misma historia, fusionada.** El Wireless Festival anunciado en Reddit, comentado en X y con los precios de las entradas en TikTok: un solo clúster, no tres entradas distintas.
|
||||
6. **Sintetizado en un único informe.** Anclado en datos concretos. Citado por fuente. Ordenado según aquello con lo que la gente interactúa de verdad. No es «esto es lo que he encontrado», es «esto es lo que importa».
|
||||
7. **Y después se convierte en tu experto.** Tras una sola ejecución, tu sesión de Claude sabe todo lo que sabe la comunidad. Haz preguntas de seguimiento. Pídele que escriba prompts, redacte correos, planifique viajes o diseñe arquitecturas, todo anclado en lo que es real ahora mismo.
|
||||
|
||||
## Lo que dice la gente
|
||||
|
||||
> «He encontrado una skill de Claude Code que investiga cualquier tema en Reddit, X, YouTube y HN de los últimos 30 días. Y luego te escribe los prompts. Antes de cada contenido que escribo, hacía esa búsqueda a mano en Reddit y X. Pestaña a pestaña. Hilo a hilo. Esa es la parte que se lleva 90 minutos. Esto la elimina.» —@itsjasonai
|
||||
|
||||
> «Esta única skill ha sustituido todo mi flujo de investigación. Le das un tema y rastrea Reddit, X y la web para sacar de qué está hablando la gente de verdad. Nada de entradas de blog viejas. Conversaciones reales de los últimos 30 días.» —@itswilsoncharles
|
||||
|
||||
> «5 de los 10 repos en tendencia hoy en GitHub son herramientas de Claude. El número 1: mvanhorn/last30days-skill» —@yieldhunter95
|
||||
|
||||
## Código abierto
|
||||
|
||||
Licencia MIT. Sin rastreo. Sin analíticas. Tu investigación se queda en tu máquina. Más de 2.700 pruebas.
|
||||
|
||||
Construido con Python 3.12+, yt-dlp, Node.js (cliente Bird incorporado para la búsqueda en X) y la API de ScrapeCreators. Arquitectura del motor v3 de [@j-sperling](https://github.com/j-sperling).
|
||||
|
||||
Consulta [CONTRIBUTING.md](CONTRIBUTING.md) para abrir un PR, [CONTRIBUTORS.md](CONTRIBUTORS.md) para la lista completa de colaboradores de la comunidad y [CHANGELOG.md](CHANGELOG.md) para el historial de versiones.
|
||||
|
||||
## Evolución de las estrellas
|
||||
|
||||
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
# /last30days
|
||||
|
||||
[English](README.md) | Français | [Deutsch](README.de.md) | [Español](README.es.md) | [Português (Brasil)](README.pt-BR.md) | [日本語](README.ja.md) | [简体中文](README.zh-CN.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://trendshift.io/repositories/21997" target="_blank">
|
||||
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**Un moteur de recherche piloté par un agent IA, qui classe les résultats selon les upvotes, les likes et l'argent réel — pas selon des rédacteurs.**
|
||||
|
||||
Ce README décrit le pipeline v3 actuel. La spécification d'exécution de la skill se trouve dans [skills/last30days/SKILL.md](skills/last30days/SKILL.md), qui fait référence pour le comportement des commandes et de la configuration.
|
||||
|
||||
**Claude Code (recommandé — mises à jour automatiques via la marketplace) :**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, ou l'un des 50+ hôtes [Agent Skills](https://agentskills.io) :**
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
(`-g` installe la skill globalement pour votre utilisateur, donc disponible dans tous vos projets. Omettez ce flag pour une installation limitée au projet.)
|
||||
|
||||
D'autres options d'installation (claude.ai web, OpenClaw, manuelle) dans la section [Installation](#installation) ci-dessous.
|
||||
|
||||
Zéro configuration. Reddit, HN, Polymarket et GitHub fonctionnent immédiatement. Lancez la skill une fois : l'assistant de configuration débloque X, YouTube, TikTok, arXiv, Techmeme et d'autres sources en 30 secondes.
|
||||
|
||||
---
|
||||
|
||||
Les upvotes de Reddit. Les likes de X. Les transcriptions YouTube. L'engagement TikTok. Les cotes Polymarket, adossées à de l'argent réel et à des informations d'initiés. Ce sont des millions de personnes qui votent chaque jour avec leur attention et leur portefeuille. /last30days interroge tout cela en parallèle, classe les résultats selon ce avec quoi les gens interagissent vraiment, et un agent IA joue le rôle de juge pour en tirer un seul brief.
|
||||
|
||||
Google agrège des rédactions. /last30days interroge les gens.
|
||||
|
||||
Cette recherche est introuvable ailleurs, parce qu'aucune IA n'a accès à l'ensemble. Google ne touche ni aux commentaires Reddit ni aux posts X. ChatGPT a un accord avec Reddit mais ne sait chercher ni sur X ni sur TikTok. Gemini a YouTube mais pas Reddit. Claude n'a nativement accès à aucun des trois. Chaque plateforme est un jardin clos, avec son API, ses tokens et son authentification. Mais vous pouvez apporter vos propres clés et vos sessions de navigateur : d'un coup, un agent IA peut toutes les interroger en même temps, les comparer entre elles et vous dire ce qui compte vraiment.
|
||||
|
||||
C'est ça, le déclic. Pas un meilleur moteur de recherche. Une douzaine de plateformes cloisonnées, reliées par un agent.
|
||||
|
||||
```
|
||||
/last30days Peter Steinberger
|
||||
```
|
||||
|
||||
Vous avez une réunion demain. Vous cherchez la personne sur Google. Vous tombez sur son LinkedIn de 2023. /last30days vous donne ce qu'elle fait vraiment ce mois-ci : elle a rejoint OpenAI pour travailler sur Codex, elle conteste l'interdiction des agents tiers décrétée par Anthropic, elle a livré 23 PR avec un taux de merge de 85 %, elle construit « LobsterOS » pour piloter des agents entre appareils, et un fil r/ClaudeCode a atteint 569 upvotes en débattant de savoir si elle est un héros ou « insupportable ». Le tout dispersé entre des posts X, des fils Reddit, des transcriptions YouTube et des commits GitHub. Rien de tout ça n'était sur Google.
|
||||
|
||||
## Pourquoi ce projet existe
|
||||
|
||||
Je l'ai construit pour suivre le rythme de l'IA. Tout change chaque jour, et les passionnés de Reddit et de X sont toujours au courant les premiers. J'avais besoin de meilleurs prompts, et les données d'entraînement avaient toujours plusieurs mois de retard sur ce que la communauté avait déjà compris.
|
||||
|
||||
Mais c'est devenu quelque chose de plus large. Aujourd'hui je le lance avant un rendez-vous commercial, pour connaître la vérité des 30 derniers jours sur une entreprise. Avant une réunion, pour lire les tweets récents et les transcriptions de podcasts de mon interlocuteur. Avant un séjour à Disney World, pour savoir quelles attractions sont fermées et ce que la communauté pense de Genie+. Avant de construire quoi que ce soit, pour savoir sur quels problèmes les gens butent réellement.
|
||||
|
||||
Si vous rencontrez un PDG, avez-vous lu tous ses tweets et toutes ses transcriptions YouTube des 30 derniers jours ? Moi, oui.
|
||||
|
||||
## Les sources, classées par les gens
|
||||
|
||||
| Source | Ce que les gens vous disent |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | L'avis brut. Les meilleurs commentaires avec leur vrai nombre d'upvotes, gratuit, sans clé API. Les vraies opinions que Google enterre. |
|
||||
| **X / Twitter** | La réaction à chaud, le fil d'expert, la première réaction à l'actualité. Premiers informés, premiers à débattre. |
|
||||
| **YouTube** | L'analyse approfondie de 45 minutes. Des transcriptions complètes, fouillées pour en extraire les 5 phrases citables qui comptent. |
|
||||
| **TikTok** | Le créateur qui touche 3,6 millions de personnes avec un angle que vous ne trouverez jamais sur Google. |
|
||||
| **Instagram Reels** | Le regard des influenceurs, avec la transcription de ce qui est dit. Le signal de la culture visuelle. |
|
||||
| **Hacker News** | Le consensus des développeurs. 825 points, 899 commentaires. Là où les gens techniques débattent vraiment. |
|
||||
| **Polymarket** | Pas des opinions. Des cotes. Adossées à de l'argent réel. 96 % de probabilité sur des ventes d'album. 4 % sur une acquisition. |
|
||||
| **GitHub** | Pour les personnes : rythme des PR, meilleurs dépôts par étoiles, notes de version. Pour les sujets : issues et discussions. |
|
||||
| **Digg** | Des groupes d'articles sélectionnés depuis le classement AI 1000 de Digg (environ 1000 comptes IA à fort signal sur X), avec des citations attribuables intégrées (sans authentification X). Activé automatiquement quand `digg-pp-cli` est présent dans le PATH. |
|
||||
| **arXiv** | Les articles scientifiques derrière le battage médiatique. La recherche publiée dans la fenêtre, gratuit, sans clé API. Activé automatiquement quand `arxiv-pp-cli` est présent dans le PATH (la configuration initiale l'installe). |
|
||||
| **Techmeme** | La couche éditoriale de l'actu tech, restreinte à votre fenêtre de 30 jours. Gratuit, sans clé API. Activé automatiquement quand `techmeme-pp-cli` est présent dans le PATH (la configuration initiale l'installe). |
|
||||
| **LinkedIn** | Le signal professionnel. Posts et articles, les articles étant pondérés comme signal fort. |
|
||||
| **StockTwits** | Le sentiment des traders. S'active automatiquement quand votre sujet est un ticker ou une crypto. |
|
||||
| **Threads** | La couche texte de l'après-Twitter. Les conversations des créateurs et des marques. |
|
||||
| **Pinterest** | La découverte visuelle. Épingles, enregistrements et commentaires sur des produits et des idées. |
|
||||
| **Xiaohongshu (RED)** | Les signaux chinois sur le lifestyle, les produits et les créateurs. À demander explicitement avec `--search xhs` quand un plugin de navigateur x-mcp connecté ou un service `xiaohongshu-mcp` tourne en local. |
|
||||
| **Bluesky** | La couche sociale décentralisée. Les posts AT Protocol issus de la migration post-Twitter. |
|
||||
| **Perplexity** | La synthèse Sonar sourcée, les résultats bruts de la Search API et Deep Research. |
|
||||
| **Web** | La couverture éditoriale, les comparatifs de blogs. Un signal parmi d'autres, pas le seul. |
|
||||
|
||||
La communauté en ajoute sans cesse. Truth Social et d'autres sources de niche sont déjà dans le moteur, et d'autres arrivent.
|
||||
|
||||
Un fil Reddit à 1 500 upvotes est un signal plus fort qu'un billet de blog que personne n'a lu. Un TikTok à 3,6 millions de vues en dit plus sur ce qui compte culturellement qu'un communiqué de presse. Des cotes Polymarket adossées à 66 000 $ de volume sont plus difficiles à contester que l'intuition d'un éditorialiste.
|
||||
|
||||
La synthèse classe selon ce avec quoi de vraies personnes ont vraiment interagi. La pertinence sociale, pas la pertinence SEO.
|
||||
|
||||
## Ce que les gens en font vraiment
|
||||
|
||||
**Avant une réunion.** `/last30days Peter Steinberger` — a rejoint l'équipe Codex d'OpenAI, conteste l'interdiction des agents tiers décrétée par Anthropic, 23 PR mergées avec un taux de merge de 85 % sur GitHub, construit LobsterOS pour piloter des agents entre appareils. r/ClaudeCode : « Depuis la sortie d'OpenClaw, tout le monde savait que si vous passiez par autre chose que l'API, vous finiriez par être banni » (227 upvotes). Ça, ce n'est pas sur LinkedIn.
|
||||
|
||||
**Pour lire les signaux de recrutement.** `/last30days Listen Labs --hiring-signals` — les offres d'emploi et les pages carrières actuelles deviennent des preuves citées de changements de priorités : recrutements en sécurité entreprise, customer success, infrastructure ou expansion produit. Le rapport dit ce que le recrutement semble signaler, pas ce que la roadmap va livrer.
|
||||
|
||||
**Pour repérer un sujet avant son pic.** Demandez `/last30days what's exploding in AI agents?` et la skill bascule en mode découverte : le moteur balaie les listings de catégories Reddit, la une et les meilleures histoires de Hacker News, le flux AI 1000 de Digg, et X si vous êtes authentifié ; votre agent évalue les candidats (noms, filtrage du bruit, intérêt réel) et rédige des angles pour un podcast ou un article X ; vous obtenez ensuite 5 à 10 sujets classés par vélocité. Chaque résultat comprend des chiffres multi-sources, une étiquette de momentum et une commande `/last30days "<topic>"` prête à lancer.
|
||||
|
||||
**Quand quelque chose sort.** `/last30days Kanye West` — le Royaume-Uni a bloqué son visa, le Wireless Festival est annulé, les sponsors ont fui. Mais BULLY est entré n° 2 au Billboard. Fantano est revenu de son « Yay sabbatical » pour le chroniquer (653 000 vues). SoFi Homecoming a fait monter Lauryn Hill et Travis Scott sur scène pour 44 titres. Polymarket : « Kanye tweetera-t-il de nouveau ? » 86 % de oui. 23 fils Reddit, 17 vidéos YouTube, 86 000 upvotes.
|
||||
|
||||
**Pour comparer des outils.** `/last30days OpenClaw vs Hermes vs Paperclip` — « Ce ne sont pas des concurrents, ce sont des couches. » OpenClaw est la couche d'exécution (351 000 étoiles GitHub, en production), Hermes est le cerveau qui s'améliore tout seul (31 000 étoiles), Paperclip est l'organigramme (49 000 étoiles). Nombres d'étoiles récupérés en direct via l'API GitHub, pas repris de billets de blog périmés. Tableau comparatif avec architecture, mémoire, sécurité et cas d'usage idéal. Selon @IMJustinBrooke : « OpenClaw = Salamèche, Hermes = Dracaufeu. »
|
||||
|
||||
**Pour comprendre le monde.** `/last30days Iran vs USA` — 38e jour de guerre. Ultimatum de Trump, fixé à mardi, pour que l'Iran rouvre le détroit d'Ormuz. Deux avions de combat américains abattus. Le pétrole à 126 $ le baril. L'AIE parle de « la plus grande perturbation d'approvisionnement de l'histoire du marché pétrolier mondial ». Polymarket : cessez-le-feu avant le 31 décembre à 74 %. 27 posts X, 10 vidéos YouTube, 20 marchés de prédiction.
|
||||
|
||||
**Avant un voyage.** `/last30days Universal Epic Universe` — l'extension est déjà en construction. Permis « Project 680 » déposé. Spectacle de feux d'artifice confirmé par les travaux mais toujours pas annoncé. Temps d'attente : Mine-Cart Madness à 148 minutes en moyenne. Toujours pas de pass annuel, et les habitants s'agacent. Stardust Racers fermé pour rénovation jusqu'au 5 avril.
|
||||
|
||||
**Pour apprendre vite.** `/last30days Nano Banana Pro prompting` — les prompts structurés en JSON remplacent l'empilement de tags. Le format imbriqué de @pictsbyai évite le « concept bleeding ». Mieux vaut éditer que régénérer. Et ensuite, la skill vous écrit un prompt de production en appliquant exactement ce que la communauté a validé.
|
||||
|
||||
## Nouveautés
|
||||
|
||||
Depuis l'annonce de la v3.3 en mai, et jusqu'à la v3.11.1 (juillet 2026) : 175 PR mergées — dont 122 venant de 52 contributeurs de la communauté — réparties sur 15 versions. Voici ce qui a atterri.
|
||||
|
||||
### Citoyen de première classe sur OpenAI Codex
|
||||
|
||||
/last30days est désormais un plugin Codex natif avec configuration guidée : pas un portage, un vrai citoyen de première classe. Les citations tiennent compte du rendu, ce qui fait que la sortie Codex se lit comme un brief et non comme une soupe d'URL (#694), et le même moteur tourne sur Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw et 50+ hôtes Agent Skills. Manifeste du plugin Codex par [@rfoust](https://github.com/rfoust) (#686), correctif d'authentification Codex par [@tmchow](https://github.com/tmchow) (#698).
|
||||
|
||||
### arXiv, Techmeme et Digg — gratuits, sans clé API
|
||||
|
||||
arXiv apporte les articles scientifiques derrière le battage médiatique et Techmeme la couche éditoriale de l'actu tech — gratuits, sans aucune clé, et la configuration initiale installe leurs CLI pour qu'ils s'activent tout seuls (#709). Les groupes d'articles AI 1000 de Digg arrivent de la même façon, sans authentification X : la configuration installe pour vous la CLI Digg gratuite (#590). Trustpilot est disponible en option pour la recherche sur les marques grand public.
|
||||
|
||||
### Reddit gratuit, avec de vrais scores et les meilleurs commentaires
|
||||
|
||||
L'API .json publique de Reddit a disparu ; la voie gratuite est revenue plus forte. Flux RSS sans clé et scraping de shreddit (#457), découverte de subreddits dédiés avec de vrais décomptes d'upvotes via arctic-shift (#696), et un seuil de pertinence pour qu'un post viral hors sujet ne détourne pas votre brief (#488, merci [@rzachsmith](https://github.com/rzachsmith)). Pas de clé API. De vrais scores. Les meilleurs commentaires inclus.
|
||||
|
||||
### Les meilleurs commentaires dans chaque brief
|
||||
|
||||
Les commentaires sont maintenant une couche activée par défaut sur toutes les sources : commentaires Instagram avec une diversité fondée sur le rang, pour que cinq avis tranchés ne viennent pas tous du même post (#751), commentaires YouTube plus une récupération de transcription via ScrapeCreators quand yt-dlp échoue (#637), et commentaires plébiscités par la communauté intégrés au scoring Best Takes, pour que les meilleures punchlines survivent au classement (#592, #608).
|
||||
|
||||
### Une seule commande doctor
|
||||
|
||||
Demandez un diagnostic : doctor teste chaque source, puis prescrit les correctifs exacts — quelle clé manque, quelle CLI est absente du PATH, quel cookie a expiré (#753). Fini de deviner pourquoi X est revenu à vide.
|
||||
|
||||
### La recherche X, reconstruite
|
||||
|
||||
Le pipeline X a été repensé de fond en comble : des voies FROM et ABOUT pour que les posts d'une personne et la conversation à son sujet soient classés tous les deux (#610), désambiguïsation des sous-requêtes selon la personne visée (#611), vérification de la paternité des posts avec classement par signaux d'interaction (#613), et une source X unique avec bascule automatique entre backends (#622). Plus un `--diagnose` honnête qui teste vraiment l'authentification (#609).
|
||||
|
||||
### De nouvelles sources
|
||||
|
||||
LinkedIn via ScrapeCreators, avec les articles comme signal fort ([@ravstr](https://github.com/ravstr), #702). StockTwits s'active automatiquement sur les sujets liés aux tickers et aux cryptos ([@wtiwana](https://github.com/wtiwana), #658). Perplexity a gagné des modes API directs et Deep Research en asynchrone ([@sk-holmes](https://github.com/sk-holmes), #629).
|
||||
|
||||
### Durci par la communauté
|
||||
|
||||
La vague sécurité est presque entièrement le fait de la communauté : correctifs XSS stocké dans le rendu HTML ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), fichiers temporaires de cookies verrouillés, CI durcie contre les attaques de chaîne d'approvisionnement avec OpenSSF Scorecard et attestation de provenance des builds ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), analyses Semgrep et OSV-Scanner plus un contrôle de revue des dépendances sur chaque PR ([@23241a6749](https://github.com/23241a6749)), un seuil plancher de couverture de tests instauré à 60 % puis relevé à 84 % ([@gourab5139014](https://github.com/gourab5139014)), et un audit de sécurité Hermes désormais sans aucune finding CRITICAL (#768).
|
||||
|
||||
### Une portée plus large
|
||||
|
||||
L'hébreu et les langues non latines ([@dudyme](https://github.com/dudyme)). Une tokenisation adaptée au CJK pour les sources chinoises ([@An-idd](https://github.com/An-idd)). Une vague d'améliorations sur Windows. L'extraction des cookies sur toute la famille Chromium — Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov)) — plus le trousseau macOS et pass(1) sous Linux comme sources d'identifiants. Le retour en arrière historique avec `--as-of` ([@chiyi-creator](https://github.com/chiyi-creator)). L'installation automatique de Python 3.12 via uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` pour lire les pages emploi d'une entreprise. Les écarts de watchlist d'une exécution à l'autre.
|
||||
|
||||
### Toujours livré depuis la v3
|
||||
|
||||
Les fondations de la v3 sont toujours là : le cerveau de pré-recherche qui identifie les bons comptes, subreddits et hashtags avant le moindre appel API (construit par [@j-sperling](https://github.com/j-sperling)) ; le scoring Best Takes, qui prend en compte l'humour et la viralité en plus de la pertinence ; la fusion de clusters entre sources ; les comparaisons en une seule passe (« CLI vs MCP » en 3 minutes, pas 12) ; les comparaisons `--competitors` découvertes automatiquement ; le mode personne de GitHub (`--github-user=steipete`) ; le mode ELI5 (« eli5 on » après n'importe quelle exécution) ; et des briefs HTML autonomes et partageables (`--emit=html`). Les options de configuration sont détaillées dans [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Installation
|
||||
|
||||
| Environnement | Installation | Mises à jour |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recommandé) | `/plugin marketplace add mvanhorn/last30days-skill` | Automatiques via la marketplace, ou `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` puis `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, ou l'un des 50+ hôtes [Agent Skills](https://agentskills.io)** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Téléchargez `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) et envoyez-le via claude.ai > Customize > Skills > + > Create skill > Upload a skill | Retélécharger et renvoyer |
|
||||
| **Claude Desktop** | [Téléchargez le `.mcpb` de votre plateforme](https://github.com/mvanhorn/last30days-skill/releases/latest) et glissez-le dans Settings > Extensions | Retélécharger et glisser le nouveau bundle |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recommandé)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Recommandé parce que la marketplace Claude Code gère les mises à jour pour vous : le cache du plugin est versionné et se rafraîchit automatiquement à chaque nouvelle version publiée. Lancez `claude plugin update last30days@last30days-skill` pour forcer une vérification.
|
||||
|
||||
Si vous préférez passer par le chemin d'installation Agent Skills sur Claude Code, c'est également pris en charge :
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
Le plugin natif et l'installation `npx skills` peuvent coexister. Attention : Claude Code ne déduplique pas entre méthodes d'installation. Si le plugin de la marketplace et la copie `npx skills` sont actifs tous les deux, `/last30days` apparaîtra en double. Utilisez une seule méthode d'installation par machine.
|
||||
|
||||
### Grok (xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) installe last30days comme plugin natif. L'installation directe suit le dépôt :
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Ou ajoutez ce dépôt comme source de marketplace, puis installez par nom de plugin :
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
Ajoutez `--trust` pour sauter la confirmation d'installation. Mettez à jour avec `grok plugin update last30days`. Grok lit aussi les manifestes Claude Code par compatibilité ; la paire native `.grok-plugin/` reste la voie principale, et c'est elle que pointe une entrée officielle dans la [marketplace xAI](https://github.com/xai-org/plugin-marketplace). `npx skills add` reste une solution de repli valable, tous hôtes confondus.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI et autres hôtes Agent Skills
|
||||
|
||||
Installez via la CLI ouverte [Agent Skills](https://agentskills.io) — elle prend en charge 50+ hôtes, dont `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose` et d'autres (liste complète sur le [dépôt vercel-labs/skills](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
Le flag `-g` (global) installe dans votre répertoire utilisateur, ce qui rend la skill disponible dans tous vos projets. Sans `-g`, `npx skills` installe localement dans `./.skills/` (versionné avec le dépôt). Pour un outil qui sert à explorer le monde entier, c'est bien l'installation globale que vous voulez.
|
||||
|
||||
Codex desktop et les autres hôtes qui travaillent au niveau du dossier fonctionnent aussi bien dans un dossier ordinaire que dans un dépôt Git. Avant la première recherche, demandez à l'agent hôte de lancer le `scripts/last30days.py --preflight` fourni depuis le répertoire de la skill chargée ; dans un clone du dépôt source, la commande équivalente est `python3 skills/last30days/scripts/last30days.py --preflight`. Elle affiche l'origine de la configuration, le plan de lecture des cookies de navigateur, les fichiers qui seront écrits, les commandes optionnelles et la configuration projet ignorée — sans lire de cookies, sans écrire de fichier et sans lancer de recherche.
|
||||
|
||||
Par défaut, l'installation cible l'hôte que `npx skills` détecte. Pour en viser un en particulier (ou plusieurs) :
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Pour mettre à jour plus tard :
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
Ou mettez à jour tout ce que vous avez installé globalement via `npx skills` :
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
Listez et désinstallez avec `npx skills list -g` et `npx skills remove last30days -g`.
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Téléchargez `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) depuis la dernière version publiée
|
||||
2. Allez sur [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. Cliquez sur le bouton `+` du panneau Skills, puis sur `Create skill` > `Upload a skill`, et déposez le fichier
|
||||
|
||||
Activez d'abord « Code execution and file creation » dans Capabilities — sans cela, les skills ne s'exécutent pas.
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop installe `/last30days` comme serveur MCP via un bundle `.mcpb` (un paquet Model Context Protocol en un clic).
|
||||
|
||||
1. Ouvrez la [dernière version publiée](https://github.com/mvanhorn/last30days-skill/releases/latest) et téléchargez le `.mcpb` correspondant à votre plateforme :
|
||||
- macOS Apple Silicon : `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel : `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64 : `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Ouvrez Claude Desktop, allez dans Settings > Extensions et glissez-y le fichier.
|
||||
3. Quand l'application vous les demande, collez les clés API des sources que vous voulez activer. Tous les champs sont facultatifs : si vous les ignorez tous, le moteur se rabat sur le mode web uniquement. Les clés sont stockées dans le trousseau de votre système.
|
||||
4. Redémarrez Claude Desktop. Demandez à Claude de « faire des recherches sur Peter Steinberger », ou sur n'importe quel sujet, et il appellera l'outil `research`.
|
||||
|
||||
**Prérequis côté hôte :** Python 3.12+ dans le PATH. Le bundle embarque le code du moteur mais utilise votre interpréteur Python local. Installez-le depuis [python.org](https://www.python.org/downloads/) sous Windows ; macOS et la plupart des distributions Linux fournissent déjà une version compatible.
|
||||
|
||||
**Les clés ne sont pas partagées avec la skill Claude Code.** Claude Desktop et Claude Code maintiennent délibérément des stockages d'identifiants distincts. Si vous avez déjà configuré `~/.config/last30days/.env` pour la skill Claude Code, il faudra ressaisir les mêmes clés ici, une fois.
|
||||
|
||||
La prise en charge de Windows est reportée le temps de régler les points d'entrée par plateforme dans le manifeste ; le suivi se fait dans une issue dédiée.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
Pour les workflows d'action sur X/Twitter en dehors des recherches `/last30days` —
|
||||
publier des tweets ou des réponses, exporter des abonnés, gérer les médias,
|
||||
surveiller des comptes, organiser des tirages au sort — utilisez
|
||||
[TweetClaw](https://github.com/Xquik-dev/tweetclaw), le plugin OpenClaw
|
||||
complémentaire. TweetClaw est maintenu par Xquik-dev et n'est mentionné que comme
|
||||
option complémentaire : ce n'est ni une dépendance ni une recommandation de last30days.
|
||||
|
||||
### Installation manuelle (développeurs)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
Le lien symbolique garde l'installation synchronisée avec votre copie de travail au fil de vos modifications — inutile de recopier quoi que ce soit. Pour `claude.ai`, construisez le fichier `.skill` depuis les sources : `bash skills/last30days/scripts/build-skill.sh` produit `dist/last30days.skill`.
|
||||
|
||||
Reddit (avec les commentaires), Hacker News, Polymarket et GitHub fonctionnent immédiatement. Zéro configuration. Lancez `/last30days` une fois : l'assistant de configuration débloque d'autres sources en 30 secondes, dont les CLI gratuites arXiv et Techmeme.
|
||||
|
||||
## Apportez vos propres clés
|
||||
|
||||
Ces plateformes n'ont aucune relation entre elles. X ignore ce que pense Reddit. YouTube ne voit pas TikTok. Mais vous pouvez apporter vos propres clés API et vos tokens de navigateur, et vous avez soudain accès à toutes en même temps.
|
||||
|
||||
| Sources | Ce qu'il vous faut | Coût |
|
||||
|---------|---------------|------|
|
||||
| Reddit (avec les commentaires) + HN + Polymarket + GitHub + StockTwits | Rien | Gratuit |
|
||||
| arXiv + Techmeme | Des CLI gratuites, installées automatiquement à la configuration initiale | Gratuit |
|
||||
| X / Twitter | Connectez-vous à x.com dans n'importe quel navigateur, ou définissez `XQUIK_API_KEY` / `XAI_API_KEY` | Les cookies de navigateur sont gratuits ; les clés dépendent du fournisseur |
|
||||
| YouTube | `brew install yt-dlp` | Gratuit |
|
||||
| Bluesky | Un mot de passe d'application depuis bsky.app | Gratuit |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + commentaires YouTube | Une clé ScrapeCreators | 10 000 appels gratuits, puis paiement à l'usage |
|
||||
| Xiaohongshu (RED) | Faites tourner un plugin de navigateur x-mcp connecté ou un service `xiaohongshu-mcp`, puis activez la source avec `--search xhs` pour une exécution ou `INCLUDE_SOURCES=xiaohongshu` dans `.env` ; last30days teste automatiquement `http://localhost:18060` puis `http://host.docker.internal:18060`, ou utilisez `XIAOHONGSHU_API_BASE` pour une URL personnalisée | Aucune clé API last30days ; dépend de votre service local de session de navigateur |
|
||||
| DripStack (newsletters financières premium) | Sur activation : `--search dripstack` pour une exécution, ou `INCLUDE_SOURCES=dripstack` dans `.env` | Aucune clé ; API de recherche publique et gratuite |
|
||||
| Perplexity Sonar / Search API / Deep Research | Une clé Perplexity, ou une clé OpenRouter en repli pour Sonar | Paiement à l'usage |
|
||||
| Recherche web | Une clé Brave Search | 2 000 requêtes gratuites par mois |
|
||||
|
||||
### Trousseau macOS (facultatif)
|
||||
|
||||
Sous macOS, vous pouvez stocker vos clés dans le trousseau système plutôt que dans un fichier `.env`. La skill les récupère automatiquement, comme source de plus faible priorité : en cas de conflit, les fichiers `.env` et les variables d'environnement du processus l'emportent toujours.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Les entrées sont enregistrées sous le nom de service `last30days-<KEY>` pour l'utilisateur courant. Sur les plateformes non Darwin, le chargeur ne fait rien : aucun changement de comportement pour les utilisateurs Linux et Windows.
|
||||
|
||||
Vous avez déjà des clés sous d'autres noms de service dans le trousseau ? Définissez la correspondance non secrète `LAST30DAYS_KEYCHAIN_ALIASES` décrite dans [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items), plutôt que de recopier vos secrets.
|
||||
|
||||
Voir [CONFIGURATION.md](CONFIGURATION.md) pour la matrice complète des clés par source, l'ordre de priorité des fournisseurs de raisonnement et celui des backends de recherche web.
|
||||
|
||||
## Configuration
|
||||
|
||||
Deux choses que vous voudrez sans doute savoir dès le premier jour :
|
||||
|
||||
**Où sont enregistrés les fichiers de recherche.** `LAST30DAYS_MEMORY_DIR` vaut par défaut `~/Documents/Last30Days/` (sous Windows : `C:\Users\<you>\Documents\Last30Days\`). Redéfinissez cette variable d'environnement dans votre shell pour pointer ailleurs, ou passez `--save-dir <path>` sur une exécution. Utilisez `--output <file>` quand vous voulez le résultat rendu à un chemin précis, dans le format choisi par `--emit`. Utilisez `--save-suffix=<name>` pour garder séparées plusieurs variantes d'un même sujet (par client, par exemple). Chaque exécution avec `--save-dir` produit `<slug>-raw[-suffix].md`. Lancez `python3 skills/last30days/scripts/last30days.py --preflight` pour vérifier les écritures prévues avant une recherche.
|
||||
|
||||
**Sortie structurée pour les agents et les workflows.** Demandez à `/last30days` du JSON exploitable par une machine pour obtenir le profil d'agent stable et versionné. Pour un usage direct du moteur en script ou en développement, lancez `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json` ; n'ajoutez `--json-profile=raw` que si vous avez besoin du dump interne non versionné de `Report`. Voir la [référence des champs de l'export JSON et la politique de versionnement](docs/reference/json-export.md).
|
||||
|
||||
**Découverte sans sujet imposé.** Demandez `/last30days what's trending in AI agents?` pour obtenir un brief de découverte classé, au lieu de rechercher un sujet que vous connaissez déjà. Sur un hôte agentique, cela déclenche le protocole en trois commandes arbitré par l'hôte (le modèle propose les sujets, écarte le bruit, note leur intérêt et rédige les angles éditoriaux). Pour un usage direct du moteur en script ou en cron, lancez `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` (en une passe : noms de sujets déterministes, sans angles) ; ajoutez `--emit=json` pour le contrat de découverte versionné. La découverte est incompatible avec un sujet positionnel et avec `--drill`.
|
||||
|
||||
**Suivi des tendances d'une exécution à l'autre.** Le mode par défaut produit un instantané Markdown à chaque exécution. Pour accumuler les résultats dans le temps, ajoutez `--store` afin de les conserver dans une base SQLite, puis utilisez [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) pour les exécutions planifiées (avec envoi facultatif sur Slack ou via webhook à chaque nouveau résultat) et [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) pour des synthèses quotidiennes ou hebdomadaires. Le schéma de cadence complet est dans [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
**Une bibliothèque de recherche à laquelle s'abonner.** Demandez à `/last30days` de générer le flux de votre bibliothèque, ou utilisez directement `python3 skills/last30days/scripts/last30days.py library feed` pour vos scripts et vos développements. La commande transforme les briefs enregistrés en un `index.html`, un `feed.xml` Atom local et des pages de brief lisibles. N'ajoutez `--publish` que si vous voulez héberger l'index HTML et les pages de brief ; la publication est un choix explicite, et publique par défaut. Pour rendre le flux Atom réellement abonnable, hébergez le répertoire de sortie généré sur un hébergeur statique comme GitHub Pages.
|
||||
|
||||
**Cherchez dans tout ce que vous avez déjà recherché.** Demandez `/last30days search my library for MCP servers` ou `/last30days have I researched MCP servers before?`. Pour un usage direct du moteur, lancez `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`. La recherche est hors ligne et déterministe : elle indexe au fil de l'eau les mêmes briefs enregistrés que le flux de bibliothèque, y fusionne les occurrences correspondantes conservées dans le store, et regroupe les résultats par sujet et par date. Les nouvelles exécutions affichent aussi une section compacte **From your library** (« depuis votre bibliothèque ») quand des recherches antérieures recoupent le sujet en cours ; définissez `LAST30DAYS_LIBRARY_CONTEXT=off` pour désactiver ce contexte passif.
|
||||
|
||||
Les scripts d'encapsulation par client, les subreddits de catégorie personnalisés et le canal bêta expérimental pour les personnalisations en cours sont également documentés dans [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Vitrine : les flux de recherche de la communauté
|
||||
|
||||
Vous avez publié une veille IA récurrente, un suivi de marché ou une obsession merveilleusement pointue avec last30days ? Partagez l'URL de votre bibliothèque publique — ou l'URL Atom une fois `feed.xml` hébergé sur un hébergeur statique — dans [le fil vitrine de la communauté](https://github.com/mvanhorn/last30days-skill/issues/532). Les flux communautaires seront listés ici au fur et à mesure que leurs auteurs les proposeront ; en attendant, le fil sert de point de collecte.
|
||||
|
||||
## Comment ça marche
|
||||
|
||||
1. **Vous saisissez un sujet.** Une personne, une entreprise, un produit, une technologie, « X vs Y ». N'importe quoi.
|
||||
2. **L'agent identifie qui compte.** Il trouve les comptes X (y compris ceux des fondateurs), les dépôts GitHub, les subreddits, les hashtags TikTok, les chaînes YouTube. Pour « Kanye West », il sait qu'il faut r/hiphopheads, @kanyewest et « bully review » sur YouTube. Pour « OpenClaw », il identifie openclaw/openclaw sur GitHub et récupère le nombre d'étoiles en direct.
|
||||
3. **Toutes les sources interrogées en parallèle.** Expansion multi-requêtes. Résultats classés selon l'engagement, la pertinence et la fraîcheur.
|
||||
4. **Une profondeur que personne d'autre n'a.** Les transcriptions YouTube complètes des vidéos de réaction. Les meilleurs commentaires Reddit avec leur nombre d'upvotes. Les légendes TikTok. Les cotes Polymarket. Pas seulement des titres et des liens.
|
||||
5. **Une même histoire, fusionnée.** Le Wireless Festival annoncé sur Reddit, commenté sur X, avec le prix des billets sur TikTok : un seul cluster, pas trois entrées distinctes.
|
||||
6. **Synthétisé en un seul brief.** Ancré dans des données précises. Sourcé. Classé selon ce avec quoi les gens interagissent vraiment. Pas « voilà ce que j'ai trouvé », mais « voilà ce qui compte ».
|
||||
7. **Ensuite, la skill devient votre experte.** Après une seule exécution, votre session Claude sait tout ce que sait la communauté. Posez vos questions de suivi. Faites-lui écrire des prompts, rédiger des e-mails, planifier des voyages, concevoir des architectures — le tout ancré dans la réalité du moment.
|
||||
|
||||
## Ce que les gens en disent
|
||||
|
||||
> « J'ai trouvé une skill Claude Code qui fait des recherches sur n'importe quel sujet à travers Reddit, X, YouTube et HN sur les 30 derniers jours. Et elle écrit les prompts à votre place. Avant chaque contenu que j'écris, je faisais ces recherches à la main sur Reddit et X. Onglet par onglet. Fil par fil. C'est la partie qui prend 90 minutes. Elle disparaît. » — @itsjasonai
|
||||
|
||||
> « Cette seule skill a remplacé tout mon workflow de recherche. Vous lui donnez un sujet, elle récupère sur Reddit, X et le web ce dont les gens parlent vraiment. Pas de vieux billets de blog. De vraies conversations des 30 derniers jours. » — @itswilsoncharles
|
||||
|
||||
> « 5 des 10 dépôts tendance du jour sur GitHub sont des outils Claude. N° 1 : mvanhorn/last30days-skill » — @yieldhunter95
|
||||
|
||||
## Open source
|
||||
|
||||
Licence MIT. Aucun tracking. Aucune analytics. Vos recherches restent sur votre machine. Plus de 2 700 tests.
|
||||
|
||||
Construit avec Python 3.12+, yt-dlp, Node.js (client Bird intégré pour la recherche X) et l'API ScrapeCreators. Architecture du moteur v3 par [@j-sperling](https://github.com/j-sperling).
|
||||
|
||||
Voir [CONTRIBUTING.md](CONTRIBUTING.md) pour ouvrir une PR, [CONTRIBUTORS.md](CONTRIBUTORS.md) pour la liste complète des contributeurs de la communauté, et [CHANGELOG.md](CHANGELOG.md) pour l'historique des versions.
|
||||
|
||||
## Évolution des étoiles
|
||||
|
||||
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
|
||||
+382
@@ -0,0 +1,382 @@
|
||||
# /last30days
|
||||
|
||||
[English](README.md) | [Français](README.fr.md) | [Deutsch](README.de.md) | [Español](README.es.md) | [Português (Brasil)](README.pt-BR.md) | 日本語 | [简体中文](README.zh-CN.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://trendshift.io/repositories/21997" target="_blank">
|
||||
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**編集者ではなく、アップボート・いいね・実際に動いたお金でランク付けする、AIエージェント主導の検索エンジンです。**
|
||||
|
||||
このREADMEは現行のv3パイプラインについて説明しています。実行時のスキル仕様は [skills/last30days/SKILL.md](skills/last30days/SKILL.md) にあり、コマンドとセットアップの挙動についてはそちらが最新かつ正式なものです。
|
||||
|
||||
**Claude Code(推奨 — マーケットプレイス経由で自動更新):**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex、Cursor、Copilot、Gemini CLI、その他50以上の [Agent Skills](https://agentskills.io) ホスト:**
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
(`-g` を付けるとユーザー単位でグローバルにインストールされ、すべてのプロジェクトで使えます。プロジェクト単位に限定したい場合はこのフラグを外してください。)
|
||||
|
||||
その他のインストール方法(claude.aiのウェブ版、OpenClaw、手動)は下の [インストール](#インストール) セクションにあります。
|
||||
|
||||
設定は不要です。Reddit、HN、Polymarket、GitHub はすぐに使えます。一度実行すれば、セットアップウィザードが30秒で X、YouTube、TikTok、arXiv、Techmeme などを有効にします。
|
||||
|
||||
---
|
||||
|
||||
Reddit のアップボート。X のいいね。YouTube の文字起こし。TikTok のエンゲージメント。実際のお金とインサイダー情報に裏打ちされた Polymarket のオッズ。つまり、毎日何百万人もの人が自分の注意と財布で投票しているということです。/last30days はそのすべてを並行して検索し、実際に人々が反応したかどうかでスコアを付け、AIエージェントが判定役となって1本のブリーフにまとめます。
|
||||
|
||||
Google は編集者を束ねます。/last30days は人を検索します。
|
||||
|
||||
この検索は他のどこでも手に入りません。単独のAIがすべてにアクセスできないからです。Google の検索は Reddit のコメントにも X の投稿にも届きません。ChatGPT は Reddit と提携していますが、X も TikTok も検索できません。Gemini には YouTube がありますが Reddit がありません。Claude はそのどれもネイティブには持っていません。どのプラットフォームも、独自のAPI・独自のトークン・独自の認証を備えた閉じた庭です。しかし自分のキーとブラウザセッションを持ち込めば、AIエージェントが一度にすべてを検索し、互いに突き合わせてスコアを付け、本当に重要なことを教えてくれるようになります。
|
||||
|
||||
そこが突破口です。優れた検索エンジンが1つ増えるという話ではありません。断絶していた十数のプラットフォームを、エージェントが橋渡しするのです。
|
||||
|
||||
```
|
||||
/last30days Peter Steinberger
|
||||
```
|
||||
|
||||
明日、打ち合わせがあるとします。その人を Google で調べると、出てくるのは2023年の LinkedIn です。/last30days なら、その人が今月実際にやっていることが分かります。Codex に取り組むため OpenAI に参加し、サードパーティ製エージェントを禁じた Anthropic の方針と争い、23本のPRをマージ率85%で送り、デバイスをまたいでエージェントを操作する「LobsterOS」を作っていて、さらに r/ClaudeCode では彼が英雄なのか「鼻につく」のかという議論が569アップボートを集めている。それらは X の投稿、Reddit のスレッド、YouTube の文字起こし、GitHub のコミットに散らばっていて、どれも Google には出てきませんでした。
|
||||
|
||||
## なぜ作ったのか
|
||||
|
||||
AIの動きに追いつくために作りました。何もかもが日々変わり、Reddit と X の濃い人たちがいつも真っ先に把握しています。もっと良いプロンプトが必要でしたが、学習データはコミュニティがすでに突き止めたことより常に数か月遅れていました。
|
||||
|
||||
ただ、そこからもっと大きなものになりました。今では商談の前に走らせて、その会社について直近30日間の実情を押さえます。打ち合わせの前には、相手の最近のツイートやポッドキャストの文字起こしを読むために。ディズニー・ワールドに行く前には、どのアトラクションが休止中で、Genie+ についてコミュニティが何と言っているかを知るために。何かを作り始める前には、人々が実際にどんな問題にぶつかっているかを知るために。
|
||||
|
||||
CEOと会うとして、直近30日間のツイートと YouTube の文字起こしを全部読んできましたか。私は読んでいます。
|
||||
|
||||
## 人々がスコアを付けた情報源
|
||||
|
||||
| 情報源 | 人々が教えてくれること |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | フィルターのかかっていない本音。実際のアップボート数付きのトップコメントが、無料・APIキーなしで手に入ります。Google が埋もれさせてしまう本当の意見です。 |
|
||||
| **X / Twitter** | 勢いのある一言、専門家のスレッド、速報への最初の反応。誰よりも早く知り、誰よりも早く議論が始まります。 |
|
||||
| **YouTube** | 45分の掘り下げ。文字起こし全文を検索し、引用に値する5つの文だけを取り出します。 |
|
||||
| **TikTok** | Google では絶対に見つからない切り口で360万人に届いているクリエイター。 |
|
||||
| **Instagram Reels** | 話した内容の文字起こし付きで届く、インフルエンサーの視点。ビジュアル文化のシグナルです。 |
|
||||
| **Hacker News** | 開発者の総意。825ポイント、899コメント。技術寄りの人たちが本気で議論している場所です。 |
|
||||
| **Polymarket** | 意見ではなく、オッズ。実際のお金が裏付けています。アルバムの売上に96%、買収に4%といった具合です。 |
|
||||
| **GitHub** | 人物について: PRの勢い、スター数の多いリポジトリ、リリースノート。トピックについて: Issue と Discussion。 |
|
||||
| **Digg** | Digg の AI 1000 リーダーボード(X 上でシグナルの強いAI関連アカウント約1000件)から集めたストーリークラスター。出典をたどれるインライン引用付きで、X の認証は不要です。`digg-pp-cli` が PATH にあると自動的に有効になります。 |
|
||||
| **arXiv** | 話題の裏側にある論文。対象期間に出た新しい研究が、無料・APIキーなしで手に入ります。`arxiv-pp-cli` が PATH にあると自動的に有効になります(初回セットアップでインストールされます)。 |
|
||||
| **Techmeme** | テックニュースの編集レイヤーを、対象の30日間に絞って取得します。無料・APIキーなし。`techmeme-pp-cli` が PATH にあると自動的に有効になります(初回セットアップでインストールされます)。 |
|
||||
| **LinkedIn** | ビジネス面のシグナル。投稿と記事を拾い、記事は強いシグナルとして重み付けします。 |
|
||||
| **StockTwits** | トレーダーの温度感。調べる対象が銘柄コードや暗号資産のときに自動で有効になります。 |
|
||||
| **Threads** | Twitter 以後のテキストの層。クリエイターやブランドの会話です。 |
|
||||
| **Pinterest** | ビジュアル起点の発見。プロダクトやアイデアに対するピン・保存・コメント。 |
|
||||
| **Xiaohongshu(RED)** | 中国のライフスタイル・プロダクト・クリエイターのシグナル。ログイン済みの x-mcp ブラウザプラグイン、または `xiaohongshu-mcp` サービスがローカルで動いているときに、`--search xhs` で明示的に指定して使います。 |
|
||||
| **Bluesky** | 分散型のソーシャル層。Twitter 以後の移住で生まれた AT Protocol の投稿です。 |
|
||||
| **Perplexity** | 根拠付きの Sonar による統合、Search API の生の結果、そして Deep Research。 |
|
||||
| **Web** | 編集記事や、ブログの比較記事。数あるシグナルの1つであって、唯一のものではありません。 |
|
||||
|
||||
コミュニティが今も情報源を増やし続けています。Truth Social をはじめとするニッチな情報源もすでにエンジンに入っていて、さらに追加予定です。
|
||||
|
||||
1,500アップボートの Reddit スレッドは、誰にも読まれなかったブログ記事よりも強いシグナルです。360万回再生の TikTok は、プレスリリースよりも「今、文化的に何が効いているか」を語ります。6.6万ドルの出来高に裏打ちされた Polymarket のオッズは、評論家の当て推量よりも反論しにくいものです。
|
||||
|
||||
この統合処理は、実在の人々が実際に反応したかどうかで順位を付けます。SEO上の関連性ではなく、社会的な関連性です。
|
||||
|
||||
## みんなが実際に使っている場面
|
||||
|
||||
**打ち合わせの前に。** `/last30days Peter Steinberger` — OpenAI の Codex チームに参加、サードパーティ製エージェントを禁じた Anthropic の方針と対立、GitHub で23本のPRをマージ率85%でマージ、デバイスをまたいでエージェントを操作する LobsterOS を開発中。r/ClaudeCode では「OpenClaw が出てからずっと、API 以外の経路で動かせばいずれBANされると広く知られていた」(227アップボート)。これは LinkedIn には載っていません。
|
||||
|
||||
**採用シグナルを読むために。** `/last30days Listen Labs --hiring-signals` — 現在の求人ページやキャリアページが、注力領域の変化を示す引用可能な根拠になります。エンタープライズ向けセキュリティ、カスタマーサクセス、インフラ、プロダクト拡張といった採用の動きです。レポートが述べるのは「採用が何を示唆しているように見えるか」であって、「ロードマップが何を出すか」ではありません。
|
||||
|
||||
**ピークを迎える前の話題を見つけるために。** `/last30days what's exploding in AI agents?` と尋ねると、スキルはディスカバリーモードに切り替わります。エンジンが Reddit のカテゴリー一覧、Hacker News のフロントページとベストストーリー、Digg の AI 1000 フィード、そして認証済みであれば X を横断してさらいます。次にエージェントが候補を審査し(名前の妥当性、ノイズの除去、記事になるか)、ポッドキャストや X 記事の切り口を書きます。最後に、勢いの強さで並べた5〜10件のトピックが返ってきます。各結果には情報源をまたいだ数値、モメンタムのラベル、そしてそのまま実行できる `/last30days "<topic>"` が付いてきます。
|
||||
|
||||
**何かが出たとき。** `/last30days Kanye West` — イギリスがビザを却下、Wireless Festival は中止、スポンサーは離脱。それでも BULLY は Billboard 初登場2位。Fantano は「Yay sabbatical」から復帰してレビューを公開(65.3万回再生)。SoFi Homecoming では Lauryn Hill と Travis Scott を迎えて44曲を披露。Polymarket では「Kanye はまたツイートするか?」が「はい」86%。Reddit のスレッド23件、YouTube の動画17本、アップボート8.6万件。
|
||||
|
||||
**ツールを比べるために。** `/last30days OpenClaw vs Hermes vs Paperclip` — 「これらは競合ではなくレイヤーだ」。OpenClaw は実行を担うレイヤー(GitHub スター35.1万、稼働中)、Hermes は自己改善する頭脳(スター3.1万)、Paperclip は組織図(スター4.9万)。スター数は古いブログ記事からではなく GitHub API からその場で取得しています。アーキテクチャ、メモリ、セキュリティ、向いている用途を並べた比較表付き。@IMJustinBrooke いわく「OpenClaw = ヒトカゲ、Hermes = リザードン」。
|
||||
|
||||
**世界の動きを理解するために。** `/last30days Iran vs USA` — 開戦から38日目。トランプ大統領はイランに対し、ホルムズ海峡の再開について火曜日を期限とする最後通告。米軍機2機が撃墜。原油は1バレル126ドル。IEA はこれを「世界の石油市場の歴史上最大の供給途絶」と呼びました。Polymarket では12月31日までの停戦が74%。X の投稿27件、YouTube の動画10本、予測市場20件。
|
||||
|
||||
**旅行の前に。** `/last30days Universal Epic Universe` — 拡張エリアはすでに着工済み。「Project 680」の建設許可が申請されています。花火ショーはインフラの痕跡から確認できるものの、まだ発表はありません。待ち時間は Mine-Cart Madness が平均148分。年間パスはまだ出ておらず、地元の人たちは不満を漏らしています。Stardust Racers は4月5日まで改修で運休。
|
||||
|
||||
**手早く学ぶために。** `/last30days Nano Banana Pro prompting` — JSON で構造化したプロンプトが、タグの寄せ集めに取って代わりつつあります。@pictsbyai の入れ子形式は「コンセプトの混線」を防ぎます。作り直すより、編集を前提にしたワークフローのほうが結果が出ます。そのうえで、コミュニティが「これは効く」と言った内容をそのまま使って、実運用向けのプロンプトを書いてくれます。
|
||||
|
||||
## 最近の変更
|
||||
|
||||
5月の v3.3 発表以降、v3.11.1(2026年7月)時点までで、15回のリリースにわたり175本のPRがマージされました。うち122本はコミュニティの52人によるものです。以下がその内容です。
|
||||
|
||||
### OpenAI Codex での一級対応
|
||||
|
||||
/last30days は、ガイド付きセットアップを備えた Codex のネイティブプラグインになりました。移植版ではなく、一級の対応です。レンダラーを踏まえた引用処理によって、Codex での出力はURLの羅列ではなくブリーフとして読めるようになり(#694)、同じエンジンが Claude Code、Cursor、Copilot、Gemini CLI、Claude Desktop、OpenClaw、そして50以上の Agent Skills ホストで動きます。Codex のプラグインマニフェストは [@rfoust](https://github.com/rfoust)(#686)、Codex の認証まわりの修正は [@tmchow](https://github.com/tmchow)(#698)によるものです。
|
||||
|
||||
### arXiv、Techmeme、Digg — 無料、APIキー不要
|
||||
|
||||
arXiv は話題の裏側にある論文を、Techmeme はテックニュースの編集レイヤーを持ち込みます。いずれも無料でキーは一切不要、しかも初回セットアップが各CLIをインストールするので自動的に有効になります(#709)。Digg の AI 1000 ストーリークラスターも同じように、X の認証なしで届きます。セットアップが無料の Digg CLI を入れてくれます(#590)。Trustpilot は消費者向けブランドの調査用に、任意で有効にできます。
|
||||
|
||||
### 無料の Reddit が、実数のスコアとトップコメント付きで復活
|
||||
|
||||
Reddit の公開 .json API は終了しましたが、無料の経路はより強くなって戻ってきました。キー不要の RSS と shreddit のスクレイピング(#457)、arctic-shift 経由で実際のアップボート数まで取れるサブレディット特定(#696)、そして話題から外れたバズ投稿にブリーフを乗っ取られないようにする関連性の下限(#488、[@rzachsmith](https://github.com/rzachsmith) に感謝)。APIキーは不要。スコアは実数。トップコメントも込みです。
|
||||
|
||||
### どのブリーフにも最高のコメントを
|
||||
|
||||
コメントは今や、どの情報源でも既定で有効なレイヤーです。Instagram のコメントは順位に基づいて分散させ、尖った意見5件が同じ投稿ばかりから出ないようにしています(#751)。YouTube のコメントに加えて、yt-dlp が失敗したときのために ScrapeCreators による文字起こしのバックアップも用意しました(#637)。さらに、コミュニティの投票で支持されたコメントを Best Takes のスコアに反映し、いちばん面白い一言が選別を生き延びるようにしています(#592、#608)。
|
||||
|
||||
### doctor コマンド1つで
|
||||
|
||||
ヘルスチェックを頼めば、doctor がすべての情報源を試したうえで、必要な対処をそのまま提示します。どのキーが足りないのか、どのCLIが PATH に入っていないのか、どのクッキーが期限切れなのか(#753)。X の結果が薄かった理由を当てずっぽうで探す必要はもうありません。
|
||||
|
||||
### X 検索の作り直し
|
||||
|
||||
X のパイプラインを一から作り直しました。FROM レーンと ABOUT レーンを設けて、本人の投稿と本人についての会話の両方が順位付けされるようにし(#610)、対象人物に応じてサブクエリの曖昧さを解消し(#611)、本人による投稿かどうかを裏付けたうえでインタラクションのシグナルで順位を付け(#613)、バックエンドを自動で切り替える単一の X ソースにまとめました(#622)。さらに、認証を実際に確かめる正直な `--diagnose` も入っています(#609)。
|
||||
|
||||
### 情報源が増えました
|
||||
|
||||
ScrapeCreators 経由の LinkedIn。記事は強いシグナルとして扱います([@ravstr](https://github.com/ravstr)、#702)。StockTwits は銘柄コードや暗号資産の話題で自動的に有効になります([@wtiwana](https://github.com/wtiwana)、#658)。Perplexity は直接APIモードと非同期の Deep Research に対応しました([@sk-holmes](https://github.com/sk-holmes)、#629)。
|
||||
|
||||
### コミュニティによる堅牢化
|
||||
|
||||
セキュリティ面の改善は、ほぼすべてコミュニティの手によるものです。HTML レンダラーの格納型XSSの修正([@iliaal](https://github.com/iliaal)、[@aaronjmars](https://github.com/aaronjmars))、クッキーの一時ファイルの権限強化、OpenSSF Scorecard とビルド来歴の証明を組み込んだサプライチェーン耐性のあるCI([@shaanmajid](https://github.com/shaanmajid)、[@hammadxcm](https://github.com/hammadxcm)、[@aniruddh909](https://github.com/aniruddh909))、Semgrep と OSV-Scanner によるスキャンおよびPRごとの依存関係レビューゲート([@23241a6749](https://github.com/23241a6749))、60%で導入し現在は84%まで引き上げたテストカバレッジの下限([@gourab5139014](https://github.com/gourab5139014))、そして CRITICAL の指摘がゼロになった Hermes のセキュリティスキャン(#768)。
|
||||
|
||||
### 届く範囲が広がりました
|
||||
|
||||
ヘブライ語をはじめとする非ラテン文字の言語に対応([@dudyme](https://github.com/dudyme))。中国語の情報源向けに CJK を考慮したトークナイズ([@An-idd](https://github.com/An-idd))。Windows 対応の改善もまとめて入りました。Chromium 系ブラウザ全体(Brave、Edge、Vivaldi、Opera、Arc)からのクッキー抽出([@andrey-esipov](https://github.com/andrey-esipov))に加え、macOS のキーチェーンと Linux の pass(1) も認証情報の取得元として使えます。`--as-of` による過去時点の振り返り([@chiyi-creator](https://github.com/chiyi-creator))。uv 経由での Python 3.12 の自動セットアップ([@buntysomroy](https://github.com/buntysomroy))。企業の求人ページを読む `--hiring-signals`。実行と実行のあいだのウォッチリスト差分。
|
||||
|
||||
### v3 から引き続き入っているもの
|
||||
|
||||
v3 の土台はすべて健在です。APIコールを1件も投げる前に、適切なアカウント・サブレディット・ハッシュタグを特定する事前リサーチの頭脳([@j-sperling](https://github.com/j-sperling) が構築)。関連性だけでなくユーモアやバイラル性も見る Best Takes のスコアリング。情報源をまたいだクラスターの統合。1回のパスで済む比較(「CLI vs MCP」が12分ではなく3分)。自動で候補を見つける `--competitors` 比較。GitHub の人物モード(`--github-user=steipete`)。ELI5 モード(実行後に「eli5 on」)。そして共有できる自己完結型の HTML ブリーフ(`--emit=html`)。設定項目は [CONFIGURATION.md](CONFIGURATION.md) にまとまっています。
|
||||
|
||||
## インストール
|
||||
|
||||
| 環境 | インストール | 更新 |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code**(推奨) | `/plugin marketplace add mvanhorn/last30days-skill` | マーケットプレイス経由で自動、または `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok**(xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` のあとに `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex、Cursor、Copilot、Gemini CLI、その他50以上の [Agent Skills](https://agentskills.io) ホスト** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai**(ウェブ) | [`last30days.skill` をダウンロード](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill)し、claude.ai > Customize > Skills > + > Create skill > Upload a skill からアップロード | ダウンロードし直してアップロードし直す |
|
||||
| **Claude Desktop** | [お使いのプラットフォーム向けの `.mcpb` をダウンロード](https://github.com/mvanhorn/last30days-skill/releases/latest)し、Settings > Extensions にドラッグ | ダウンロードし直して新しいバンドルをドラッグ |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code(推奨)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Claude Code のマーケットプレイスが更新を代わりにやってくれるため、これが推奨です。プラグインのキャッシュはバージョン管理されていて、新しいリリースが公開されると自動で更新されます。`claude plugin update last30days@last30days-skill` を実行すれば、その場で確認を強制できます。
|
||||
|
||||
Claude Code で Agent Skills 経由のインストールを使いたい場合も、それはそれで対応しています。
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
ネイティブプラグインと `npx skills` でのインストールは共存できます。ただし Claude Code はインストール方法をまたいだ重複排除を行いません。マーケットプレイス版のプラグインと `npx skills` のコピーを両方とも有効にしていると、`/last30days` が2件表示されます。1台につきインストール方法は1つにしてください。
|
||||
|
||||
### Grok(xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces)(`grok`)は last30days をネイティブプラグインとしてインストールします。直接インストールする場合はリポジトリを追跡します。
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
あるいは、このリポジトリをマーケットプレイスのソースとして追加してから、プラグイン名でインストールすることもできます。
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
インストール時の確認を省きたい場合は `--trust` を付けてください。更新は `grok plugin update last30days` です。Grok は互換性のために Claude Code のマニフェストも読みますが、第一の経路はネイティブの `.grok-plugin/` のペアで、[xAI のマーケットプレイス](https://github.com/xai-org/plugin-marketplace)への公式掲載もこちらを指しています。`npx skills add` は、どのホストでも使える代替手段として引き続き有効です。
|
||||
|
||||
### Codex、Cursor、Copilot、Gemini CLI、その他の Agent Skills ホスト
|
||||
|
||||
オープンな [Agent Skills](https://agentskills.io) の CLI からインストールします。`codex`、`cursor`、`github-copilot`、`gemini-cli`、`claude-code`、`windsurf`、`cline`、`continue`、`roo`、`aider-desk`、`opencode`、`goose` など50以上のホストに対応しています(全一覧は [vercel-labs/skills リポジトリ](https://github.com/vercel-labs/skills)にあります)。
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
`-g`(グローバル)フラグを付けるとユーザーディレクトリにインストールされ、スキルをすべてのプロジェクトで使えます。`-g` を付けない場合、`npx skills` はプロジェクト内の `./.skills/` にインストールし、リポジトリと一緒にコミットされます。世界中を調べるためのツールなので、通常はグローバルが向いています。
|
||||
|
||||
Codex のデスクトップ版など、フォルダ単位で動くホストは、Git リポジトリでも普通のフォルダでも動作します。最初の調査を始める前に、読み込み済みのスキルディレクトリから同梱の `scripts/last30days.py --preflight` を実行するようホストのエージェントに頼んでください。ソースをチェックアウトしている場合、同等のコマンドは `python3 skills/last30days/scripts/last30days.py --preflight` です。設定の取得元、ブラウザのクッキーをどう扱う予定か、どのファイルを書き込む予定か、任意で使えるコマンド、無視されるプロジェクト設定を表示します。クッキーの読み取りもファイルの書き込みも調査の実行もしません。
|
||||
|
||||
既定では、`npx skills` が検出したホスト向けにインストールされます。特定のホスト(または複数)を指定するには次のようにします。
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
あとから更新するには次のようにします。
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
`npx skills` でグローバルに入れたものをまとめて更新することもできます。
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
一覧表示と削除は `npx skills list -g` と `npx skills remove last30days -g` で行えます。
|
||||
|
||||
### claude.ai(ウェブ)
|
||||
|
||||
1. 最新リリースから [`last30days.skill` をダウンロード](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill)します
|
||||
2. [claude.ai > Customize > Skills](https://claude.ai/customize/skills) を開きます
|
||||
3. Skills パネルの `+` ボタンをクリックし、`Create skill` > `Upload a skill` と進んで、ファイルを選択するかドロップします
|
||||
|
||||
先に Capabilities で「Code execution and file creation」を有効にしてください。これがないとスキルは動きません。
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop では、`.mcpb` バンドル(ワンクリック版の Model Context Protocol パッケージ)を使って `/last30days` を MCP サーバーとしてインストールします。
|
||||
|
||||
1. [最新リリース](https://github.com/mvanhorn/last30days-skill/releases/latest)を開き、お使いのプラットフォーム向けの `.mcpb` をダウンロードします:
|
||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Claude Desktop を開き、Settings > Extensions に移動して、ファイルをドラッグします。
|
||||
3. 求められたら、有効にしたい情報源のAPIキーを貼り付けます。どの項目も任意です。すべて省略した場合、エンジンはウェブのみのモードに切り替わります。キーはOSのキーチェーンに保存されます。
|
||||
4. Claude Desktop を再起動します。Claude に「Peter Steinberger について調べて」などと頼めば、`research` ツールが呼び出されます。
|
||||
|
||||
**ホスト側の要件:** PATH の通った Python 3.12以上。バンドルにはエンジンのソースが含まれますが、実行にはローカルの Python インタプリタを使います。Windows では [python.org](https://www.python.org/downloads/) からインストールしてください。macOS とたいていの Linux ディストリビューションには、対応するバージョンが最初から入っています。
|
||||
|
||||
**キーは Claude Code のスキルとは共有されません。** Claude Desktop と Claude Code は、設計上それぞれ別に認証情報を保管しています。Claude Code のスキル用にすでに `~/.config/last30days/.env` を設定していても、ここで同じキーをもう一度だけ入力する必要があります。
|
||||
|
||||
Windows のサポートは、マニフェストのプラットフォーム別エントリーポイントが整理されるまで見送りとなっており、専用のIssueで追跡しています。
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
`/last30days` の調査以外で X/Twitter を操作したい場合 — ツイートや返信の投稿、
|
||||
フォロワーのエクスポート、メディアの扱い、モニタリング、プレゼント企画の抽選など —
|
||||
には、OpenClaw の補助プラグインとして [TweetClaw](https://github.com/Xquik-dev/tweetclaw)
|
||||
を使ってください。TweetClaw は Xquik-dev が管理しており、ここでは任意の補助的な
|
||||
選択肢として挙げているだけです。last30days の依存でも推奨でもありません。
|
||||
|
||||
### 手動インストール(開発者向け)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
シンボリックリンクにしておけば、編集するたびに作業ツリーとインストール先が同期するので、コピーし直す必要はありません。`claude.ai` 用には、ソースから `.skill` ファイルをビルドしてください。`bash skills/last30days/scripts/build-skill.sh` で `dist/last30days.skill` が生成されます。
|
||||
|
||||
Reddit(コメント込み)、Hacker News、Polymarket、GitHub はすぐに使えます。設定は不要です。`/last30days` を一度実行すれば、セットアップウィザードが30秒でさらに多くの情報源を有効にします。無料の arXiv と Techmeme の CLI も含まれます。
|
||||
|
||||
## 自分のキーを持ち込む
|
||||
|
||||
これらのプラットフォーム同士には何のつながりもありません。X は Reddit が何を考えているかを知りませんし、YouTube に TikTok は見えていません。しかし自分のAPIキーとブラウザのトークンを持ち込めば、それらすべてに一度にアクセスできるようになります。
|
||||
|
||||
| 情報源 | 必要なもの | 費用 |
|
||||
|---------|---------------|------|
|
||||
| Reddit(コメント込み)+ HN + Polymarket + GitHub + StockTwits | 不要 | 無料 |
|
||||
| arXiv + Techmeme | 無料のCLI。初回セットアップが自動でインストールします | 無料 |
|
||||
| X / Twitter | 任意のブラウザで x.com にログインするか、`XQUIK_API_KEY` / `XAI_API_KEY` を設定 | ブラウザのクッキーは無料。キーの料金は提供元によります |
|
||||
| YouTube | `brew install yt-dlp` | 無料 |
|
||||
| Bluesky | bsky.app のアプリパスワード | 無料 |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + YouTube のコメント | ScrapeCreators のキー | 1万リクエストまで無料、以降は従量課金 |
|
||||
| Xiaohongshu(RED) | ログイン済みの x-mcp ブラウザプラグインか `xiaohongshu-mcp` サービスを動かしたうえで、実行ごとに `--search xhs` を付けるか `.env` に `INCLUDE_SOURCES=xiaohongshu` を設定して有効化します。last30days は `http://localhost:18060`、次に `http://host.docker.internal:18060` の順に自動で接続を試し、独自のURLを使う場合は `XIAOHONGSHU_API_BASE` を指定します | last30days 側のAPIキーは不要。ローカルのブラウザセッションのサービス次第です |
|
||||
| DripStack(有料の金融ニュースレター) | 任意で有効化: 実行ごとに `--search dripstack`、または `.env` に `INCLUDE_SOURCES=dripstack` | キー不要。無料の公開検索APIを使います |
|
||||
| Perplexity Sonar / Search API / Deep Research | Perplexity のキー、または Sonar の代替として OpenRouter のキー | 従量課金 |
|
||||
| ウェブ検索 | Brave Search のキー | 月2,000クエリまで無料 |
|
||||
|
||||
### macOS のキーチェーン(任意)
|
||||
|
||||
macOS では、キーを `.env` ファイルではなくシステムのキーチェーンに保存できます。スキルは最も優先度の低い取得元として自動的に読み込むため、衝突した場合は `.env` ファイルとプロセスの環境変数が優先されます。
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
項目は現在のユーザー向けに、サービス名 `last30days-<KEY>` で保存されます。Darwin 以外のプラットフォームではローダーは何もしないため、Linux や Windows のユーザーにとって挙動は変わりません。
|
||||
|
||||
すでに別のサービス名でキーチェーンにキーを保存している場合は、秘密情報をコピーする代わりに、[CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items) で説明している秘密情報ではないマッピング `LAST30DAYS_KEYCHAIN_ALIASES` を設定してください。
|
||||
|
||||
情報源ごとのキーの一覧、推論プロバイダーの優先順位、ウェブ検索バックエンドの優先順位については [CONFIGURATION.md](CONFIGURATION.md) を参照してください。
|
||||
|
||||
## 設定
|
||||
|
||||
初日に知っておくとよいことが2つあります。
|
||||
|
||||
**調査ファイルの保存先。** `LAST30DAYS_MEMORY_DIR` の既定値は `~/Documents/Last30Days/` です(Windows では `C:\Users\<you>\Documents\Last30Days\`)。変更したい場合は、シェルでこの環境変数に任意のパスを設定するか、実行ごとに `--save-dir <path>` を指定します。レンダリング結果を特定のパスに出力したいときは `--output <file>` を使い、形式は `--emit` で選びます。同じトピックの複数のバリエーションを分けて残したいときは `--save-suffix=<name>` を使ってください(クライアントごとに分ける場合など)。`--save-dir` を付けた実行では `<slug>-raw[-suffix].md` が生成されます。調査を走らせる前に書き込み予定を確認するには `python3 skills/last30days/scripts/last30days.py --preflight` を実行してください。
|
||||
|
||||
**エージェントやワークフロー向けの構造化出力。** `/last30days` に機械可読なJSONを求めると、安定したバージョン付きのエージェント向けプロファイルが返ります。スクリプトや開発でエンジンを直接使う場合は `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json` を実行してください。バージョン管理されていない内部の `Report` のダンプが必要なときだけ `--json-profile=raw` を追加します。[JSONエクスポートのフィールド一覧とバージョニング方針](docs/reference/json-export.md)も参照してください。
|
||||
|
||||
**トピックを決めないディスカバリー。** すでに知っているトピックを調べる代わりに、順位付きのディスカバリーブリーフがほしいときは `/last30days what's trending in AI agents?` と尋ねてください。エージェントを備えたホストでは、ホストが判定する3コマンドのプロトコルが走ります(モデルがトピックを挙げ、ノイズを除き、取り上げる価値を採点し、コンテンツの切り口を書きます)。スクリプトや cron でエンジンを直接使う場合は `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` を実行します(単発実行。トピック名は決定論的で、切り口は付きません)。バージョン付きのディスカバリー契約がほしい場合は `--emit=json` を追加してください。ディスカバリーは、位置引数のトピックや `--drill` とは併用できません。
|
||||
|
||||
**実行をまたいだトレンド監視。** 既定のモードでは、実行のたびに新しい Markdown のスナップショットが作られます。時間をかけて結果を蓄積したい場合は `--store` を付けて SQLite データベースに保存し、定期実行には [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py)(新しい結果が出たときの Slack や Webhook への通知も任意で設定できます)、日次・週次のまとめには [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) を使ってください。運用サイクルの全体像は [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings) にあります。
|
||||
|
||||
**購読できる調査ライブラリ。** `/last30days` にライブラリのフィードを作らせるか、スクリプトや開発用には `python3 skills/last30days/scripts/last30days.py library feed` を直接使ってください。保存済みのブリーフが `index.html`、ローカルの Atom 形式の `feed.xml`、読みやすいブリーフのページに変換されます。HTML のインデックスとブリーフのページをホスティングしたいときだけ `--publish` を付けてください。公開は明示的に選ぶ形で、既定では誰でも見られる状態になります。Atom フィードを実際に購読できるようにするには、生成された出力ディレクトリを GitHub Pages のような静的ホスティングに置いてください。
|
||||
|
||||
**これまで調べたものをすべて検索する。** `/last30days search my library for MCP servers` や `/last30days have I researched MCP servers before?` と尋ねてください。エンジンを直接使う場合は `python3 skills/last30days/scripts/last30days.py library search "MCP servers"` を実行します。この検索はオフラインかつ決定論的です。ライブラリのフィードが使うのと同じ保存済みブリーフを少しずつインデックス化し、実行ごとにストアへ記録された該当分をまとめ、トピックと日付で結果をグループ化します。新しく実行したときも、過去の調査が今回のトピックと重なっていれば、**From your library**(あなたのライブラリから)というコンパクトなセクションが表示されます。この受動的な文脈表示をやめたい場合は `LAST30DAYS_LIBRARY_CONTEXT=off` を設定してください。
|
||||
|
||||
クライアントごとのラッパースクリプト、カテゴリー用のサブレディットのカスタマイズ、作業中のカスタマイズを試す実験的なベータチャンネルについても [CONFIGURATION.md](CONFIGURATION.md) に記載しています。
|
||||
|
||||
## ショーケース: コミュニティの調査フィード
|
||||
|
||||
last30days で、定期的なAIのまとめ、市場ウォッチ、あるいは見事にニッチな偏愛を公開しましたか。公開ライブラリのURL(または `feed.xml` を静的ホスティングに置いたあとの Atom のURL)を[コミュニティのショーケーススレッド](https://github.com/mvanhorn/last30days-skill/issues/532)で共有してください。コミュニティのフィードは、作者から届き次第ここにリンクしていきます。それまでのあいだは、このスレッドが集約先です。
|
||||
|
||||
## 仕組み
|
||||
|
||||
1. **トピックを入力します。** 人物、企業、プロダクト、技術、「X vs Y」。何でもかまいません。
|
||||
2. **エージェントが「誰が重要か」を特定します。** X のアカウント(創業者を含む)、GitHub のリポジトリ、サブレディット、TikTok のハッシュタグ、YouTube のチャンネルを見つけます。「Kanye West」なら r/hiphopheads、@kanyewest、YouTube の「bully review」だと分かります。「OpenClaw」なら GitHub 上の openclaw/openclaw を特定し、スター数をその場で取得します。
|
||||
3. **すべての情報源を並行して検索します。** 複数クエリへの展開。結果はエンゲージメント、関連性、新しさでスコア付けされます。
|
||||
4. **他にはない深さ。** リアクション動画の YouTube 全文文字起こし。アップボート数付きの Reddit のトップコメント。TikTok のキャプション。Polymarket のオッズ。タイトルとリンクだけではありません。
|
||||
5. **同じ話題はまとめます。** Reddit で告知され、X で語られ、TikTok にチケット価格が出た Wireless Festival は、3件の別々の項目ではなく1つのクラスターになります。
|
||||
6. **1本のブリーフに統合します。** 具体的なデータに基づき、情報源を明示し、実際に人々が反応したかどうかで順位を付けます。「見つけたものはこれです」ではなく「重要なのはこれです」を返します。
|
||||
7. **そのあとは、あなたの専門家になります。** 一度実行すれば、あなたの Claude のセッションはコミュニティが知っていることをすべて把握しています。続けて質問してください。プロンプトを書かせる、メールを下書きさせる、旅程を立てさせる、システム構成を設計させる。どれも「今、実際に起きていること」に基づきます。
|
||||
|
||||
## 使っている人の声
|
||||
|
||||
> 「Reddit、X、YouTube、HN を横断して直近30日のあらゆるトピックを調べてくれる Claude Code のスキルを見つけた。しかもプロンプトまで書いてくれる。書く記事ごとに、これまでは Reddit と X を手作業で調べていた。タブごと、スレッドごとに。そこが90分かかっていた部分だ。それがなくなる。」 — @itsjasonai
|
||||
|
||||
> 「このスキル1つで、私の調査ワークフローがまるごと置き換わった。トピックを渡すと、Reddit、X、ウェブから人々が本当に話していることを拾ってくる。古いブログ記事ではなく、直近30日の生の会話だ。」 — @itswilsoncharles
|
||||
|
||||
> 「今日 GitHub でトレンド入りしているリポジトリ10件のうち5件が Claude 関連のツール。1位は mvanhorn/last30days-skill」 — @yieldhunter95
|
||||
|
||||
## オープンソース
|
||||
|
||||
MIT ライセンス。トラッキングなし。アナリティクスなし。調査結果はあなたのマシンに残ります。テストは2,700件以上。
|
||||
|
||||
Python 3.12以上、yt-dlp、Node.js(X 検索用に同梱した Bird クライアント)、ScrapeCreators API で構築しています。v3 のエンジンアーキテクチャは [@j-sperling](https://github.com/j-sperling) によるものです。
|
||||
|
||||
PRの出し方は [CONTRIBUTING.md](CONTRIBUTING.md)、コミュニティの貢献者の一覧は [CONTRIBUTORS.md](CONTRIBUTORS.md)、バージョン履歴は [CHANGELOG.md](CHANGELOG.md) を参照してください。
|
||||
|
||||
## スター数の推移
|
||||
|
||||
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
|
||||
@@ -1,5 +1,11 @@
|
||||
# /last30days
|
||||
|
||||
English | [Français](README.fr.md) | [Deutsch](README.de.md) | [Español](README.es.md) | [Português (Brasil)](README.pt-BR.md) | [日本語](README.ja.md) | [简体中文](README.zh-CN.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
@@ -28,7 +34,7 @@ npx skills add mvanhorn/last30days-skill -g
|
||||
|
||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
||||
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, arXiv, Techmeme, and more in 30 seconds.
|
||||
|
||||
---
|
||||
|
||||
@@ -58,7 +64,7 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
|
||||
| Source | What the people tell you |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | The unfiltered take. Top comments with upvote counts, free via public JSON. The real opinions that Google buries. |
|
||||
| **Reddit** | The unfiltered take. Top comments with real upvote counts, free, no API key. The real opinions that Google buries. |
|
||||
| **X / Twitter** | The hot take, the expert thread, the breaking reaction. First to know, first to argue. |
|
||||
| **YouTube** | The 45-minute deep dive. Full transcripts searched for the 5 quotable sentences that matter. |
|
||||
| **TikTok** | The creator reaching 3.6M people with a take you'll never find on Google. |
|
||||
@@ -67,13 +73,18 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
||||
| **arXiv** | The papers behind the hype. New research in the window, free, no API key. Auto-enabled when `arxiv-pp-cli` is on PATH (first-run setup installs it). |
|
||||
| **Techmeme** | The tech-news editorial layer, date-windowed to your 30 days. Free, no API key. Auto-enabled when `techmeme-pp-cli` is on PATH (first-run setup installs it). |
|
||||
| **LinkedIn** | The professional signal. Posts and articles, with articles weighted as high signal. |
|
||||
| **StockTwits** | Trader sentiment. Auto-activates when your topic is a ticker or crypto. |
|
||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||
| **Xiaohongshu (RED)** | Chinese lifestyle, product, and creator signals. Requested explicitly with `--search xhs` when a logged-in x-mcp browser plugin or `xiaohongshu-mcp` service is running locally. |
|
||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||
| **Perplexity** | Grounded web search with citations via Sonar Pro. |
|
||||
| **Perplexity** | Grounded Sonar synthesis, raw Search API rows, and Deep Research. |
|
||||
| **Web** | The editorial coverage, the blog comparisons. One signal of many, not the only one. |
|
||||
|
||||
Community contributors keep adding more. Truth Social, Xiaohongshu (RED), and others are in the engine with more on the way.
|
||||
Community contributors keep adding more. Truth Social and other niche sources are in the engine with more on the way.
|
||||
|
||||
A Reddit thread with 1,500 upvotes is a stronger signal than a blog post nobody read. A TikTok with 3.6M views tells you more about what's culturally relevant than a press release. Polymarket odds backed by $66K in volume are harder to argue with than a pundit's guess.
|
||||
|
||||
@@ -85,6 +96,8 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
**To read hiring signals.** `/last30days Listen Labs --hiring-signals` - current jobs and careers pages become cited evidence for focus shifts: hiring into enterprise security, customer success, infrastructure, or product expansion. The report says what the hiring appears to signal, not what the roadmap will ship.
|
||||
|
||||
**To find the topic before it peaks.** Ask `/last30days what's exploding in AI agents?` and the skill switches to discovery mode: the engine sweeps Reddit category listings, Hacker News front/best stories, Digg's AI 1000 feed, and X when authenticated; your agent judges the nominations (names, junk filtering, content-worthiness) and writes podcast / X-article angles; then you get 5-10 velocity-ranked topics. Every result includes cross-source numbers, a momentum label, and a ready-to-run `/last30days "<topic>"` follow-up.
|
||||
|
||||
**When something drops.** `/last30days Kanye West` - UK blocked his visa, Wireless Festival canceled, sponsors fled. But BULLY debuted #2 on Billboard. Fantano came back from his "Yay sabbatical" to review it (653K views). SoFi Homecoming brought out Lauryn Hill and Travis Scott for 44 songs. Polymarket: "Will Kanye tweet again?" 86% Yes. 23 Reddit threads, 17 YouTube videos, 86K upvotes.
|
||||
|
||||
**To compare tools.** `/last30days OpenClaw vs Hermes vs Paperclip` - "These aren't competitors, they're layers." OpenClaw is the executor (351K GitHub stars, live), Hermes is the self-improving brain (31K stars), Paperclip is the org chart (49K stars). Star counts pulled live from the GitHub API, not stale blog posts. Side-by-side table with architecture, memory, security, best-for. Per @IMJustinBrooke: "OpenClaw = Charmander, Hermes = Charizard."
|
||||
@@ -95,86 +108,59 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
**To learn something fast.** `/last30days Nano Banana Pro prompting` - JSON-structured prompts are replacing tag soup. @pictsbyai's nested format prevents "concept bleeding." Edit-first workflow beats regeneration. Then it writes you a production prompt using exactly what the community said works.
|
||||
|
||||
## What v3 Changed
|
||||
## What's new
|
||||
|
||||
### Shareable HTML briefs
|
||||
Since the v3.3 announcement in May, as of v3.11.1 (July 2026): 175 merged PRs - 122 of them from 52 community contributors - across 15 releases. This is what landed.
|
||||
|
||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
||||
### First-class on OpenAI Codex
|
||||
|
||||
```
|
||||
/last30days OpenClaw --emit=html
|
||||
```
|
||||
/last30days is now a native Codex plugin with guided setup - not a port, a first-class citizen. Renderer-aware citations mean Codex output reads like a brief instead of URL soup (#694), and the same engine runs on Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw, and 50+ Agent Skills hosts. Codex plugin manifest by [@rfoust](https://github.com/rfoust) (#686), Codex auth fix by [@tmchow](https://github.com/tmchow) (#698).
|
||||
|
||||
or just ask in plain language:
|
||||
### arXiv, Techmeme, and Digg - free, no API keys
|
||||
|
||||
```
|
||||
/last30days OpenClaw, give me a shareable HTML brief
|
||||
/last30days Cursor IDE for slack
|
||||
/last30days Anthropic earnings export as html
|
||||
```
|
||||
arXiv brings the papers behind the hype and Techmeme brings the editorial tech-news layer - free, zero keys, and first-run setup installs their CLIs so they activate automatically (#709). Digg's AI 1000 story clusters arrive without X auth the same way - setup installs the free Digg CLI for you (#590). Trustpilot ships opt-in for consumer-brand research.
|
||||
|
||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
||||
### Free Reddit grew real scores and top comments
|
||||
|
||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
||||
Reddit's public .json API died; the free path came back stronger. Keyless RSS + shreddit scraping (#457), dedicated-subreddit discovery with real upvote counts via arctic-shift (#696), and a relevance floor so a viral off-topic post can't hijack your brief (#488, thanks [@rzachsmith](https://github.com/rzachsmith)). No API key. Real scores. Top comments included.
|
||||
|
||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
||||
### The best comments in every brief
|
||||
|
||||
### Intelligent search: the killer feature
|
||||
Comments are now a default-on layer across sources: Instagram comments with rank-based diversity so five hot takes don't all come from one post (#751), YouTube comments plus a ScrapeCreators transcript backup for when yt-dlp strikes out (#637), and crowd-voted comments weighted into Best Takes so the community's funniest lines survive scoring (#592, #608).
|
||||
|
||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
### One doctor command
|
||||
|
||||
This is why v3 finds content v2 never could. "Paperclip" resolves @dotta. "Dave Morin" resolves @davemorin plus @OpenClaw plus the TWiST podcast. "Peter Steinberger" resolves @steipete on X and steipete on GitHub. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags - resolved before a single API call fires.
|
||||
Ask for a health check and the doctor runs every source, then prescribes exact fixes - which key is missing, which CLI is off PATH, which cookie expired (#753). No more guessing why X came back thin.
|
||||
|
||||
### Best Takes
|
||||
### X search, rebuilt
|
||||
|
||||
Reddit and X people are funny. The old engine buried their best stuff because it scored for relevance, not cleverness. v3 has a second judge that scores every result for humor, wit, and virality alongside the relevance score. Tommy Lloyd's "My Michael Jordan is Steve Kerr" scores low on relevance to "Arizona Basketball" but off the charts on fun. Now every brief ends with a "Best Takes" section - the cleverest one-liners, the most viral quotes, the reactions that make you want to share the research. Built in, not a toggle.
|
||||
The X pipeline got a ground-up overhaul: FROM and ABOUT lanes so a person's own posts and the conversation about them both rank (#610), person-aware subquery disambiguation (#611), first-party authorship grounding with interaction-signal ranking (#613), and a single X source with automatic backend failover (#622). Plus an honest `--diagnose` that actually probes auth (#609).
|
||||
|
||||
### Cross-source cluster merging
|
||||
### More sources joined
|
||||
|
||||
When the same story appears on Reddit, X, and YouTube, v3 merges them into one cluster instead of showing three separate items. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
LinkedIn via ScrapeCreators, with articles as high signal ([@ravstr](https://github.com/ravstr), #702). StockTwits auto-activates for ticker and crypto topics ([@wtiwana](https://github.com/wtiwana), #658). Perplexity grew direct API modes and async Deep Research ([@sk-holmes](https://github.com/sk-holmes), #629).
|
||||
|
||||
### Single-pass comparisons
|
||||
### Hardened by the community
|
||||
|
||||
"CLI vs MCP" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides simultaneously. Same depth, 3 minutes.
|
||||
The security wave was almost entirely community work: stored-XSS fixes in the HTML renderer ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), locked-down cookie temp files, supply-chain-hardened CI with OpenSSF Scorecard and build provenance attestation ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), Semgrep and OSV-Scanner scans plus a PR dependency-review gate ([@23241a6749](https://github.com/23241a6749)), a test-coverage floor introduced at 60% and since raised to 84% ([@gourab5139014](https://github.com/gourab5139014)), and a Hermes security scan cleared of every CRITICAL finding (#768).
|
||||
|
||||
### Auto-discovered competitor comparisons
|
||||
### Reaches further
|
||||
|
||||
`/last30days OpenAI --competitors` tells the hosting reasoning model to discover the top 2 peers via WebSearch (Anthropic, xAI), run Step 0.55 per entity, and invoke the engine with `"OpenAI vs Anthropic vs xAI"` and a per-entity `--competitors-plan` JSON. The engine fans out 3 full pipelines in parallel, saves a `*-raw.md` file per entity, and merges them into a 3-way comparison. Same mechanics power `/last30days "OpenAI vs Anthropic vs xAI"` directly.
|
||||
Hebrew and non-Latin languages ([@dudyme](https://github.com/dudyme)). CJK-aware tokenization for Chinese sources ([@An-idd](https://github.com/An-idd)). A Windows compatibility wave. Cookie extraction across the full Chromium family - Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov)) - plus macOS Keychain and Linux pass(1) credential sources. `--as-of` historical lookback ([@chiyi-creator](https://github.com/chiyi-creator)). Auto-provisioned Python 3.12 via uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` for reading a company's job pages. Watchlist deltas between runs.
|
||||
|
||||
### GitHub person-mode
|
||||
### Still in the box from v3
|
||||
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. Instead of "who mentioned this name in an issue body," it answers: what are they shipping and where is it landing?
|
||||
|
||||
`/last30days Peter Steinberger --github-user=steipete` shows 22 PRs merged across 3 repos at 85% merge rate. Own projects with README summaries, star counts, and top feature requests. Release notes for what shipped this month. The synthesizer weaves it into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
### ELI5 mode
|
||||
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations - just clearer. "Arizona wins by being physical" instead of "Arizona's identity is paint scoring (50%+ shooting, 9th nationally)." Say "eli5 off" to go back.
|
||||
|
||||
### Everything else in v3
|
||||
|
||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||
- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset).
|
||||
- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`.
|
||||
- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed).
|
||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||
- **Polymarket odds, not dollars.** The % odds are the magic. Dollar volumes removed from display.
|
||||
- **Per-author cap.** Max 3 items per author prevents any single voice from dominating your brief.
|
||||
- **Entity disambiguation.** When the engine resolves handles, the synthesis trusts them. No more Mallorca resorts winning over Washington athletic clubs.
|
||||
- **OpenClaw first-class citizen.** Auto-resolve for engine-side pre-research. Device auth for frictionless ScrapeCreators signup.
|
||||
- **1,012 tests passing.**
|
||||
The v3 foundations are all still here: the pre-research brain that resolves the right handles, subreddits, and hashtags before a single API call fires (built by [@j-sperling](https://github.com/j-sperling)); Best Takes scoring for humor and virality alongside relevance; cross-source cluster merging; single-pass comparisons ("CLI vs MCP" in 3 minutes, not 12); auto-discovered `--competitors` comparisons; GitHub person-mode (`--github-user=steipete`); ELI5 mode ("eli5 on" after any run); and shareable, self-contained HTML briefs (`--emit=html`). Configuration knobs live in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install | Updates |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` then `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via claude.ai > Customize > Skills > + > Create skill > Upload a skill | Re-download and re-upload |
|
||||
| **Claude Desktop** | [Download the `.mcpb` for your platform](https://github.com/mvanhorn/last30days-skill/releases/latest) and drag into Settings > Extensions | Re-download and drag the new bundle in |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recommended)
|
||||
@@ -193,6 +179,23 @@ npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
|
||||
The native plugin and the `npx skills` install can coexist. Note that Claude Code does not dedupe across install methods: if you have both the marketplace plugin and the `npx skills` copy active, `/last30days` will show two entries. Use one install method per machine.
|
||||
|
||||
### Grok (xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) installs last30days as a native plugin. Direct install tracks the repository:
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Or add this repo as a marketplace source, then install by plugin name:
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
Add `--trust` to skip the install confirmation. Update with `grok plugin update last30days`. Grok also reads the Claude Code manifests for compatibility; the native `.grok-plugin/` pair is the first-class lane (and what an official [xAI marketplace](https://github.com/xai-org/plugin-marketplace) listing points at). `npx skills add` remains a valid cross-host fallback.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
||||
|
||||
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
|
||||
@@ -203,6 +206,8 @@ npx skills add mvanhorn/last30days-skill -g
|
||||
|
||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
||||
|
||||
Codex desktop and other folder-mode hosts can work in ordinary folders as well as Git repos. Before first research, ask the host agent to run the bundled `scripts/last30days.py --preflight` from the loaded skill directory; in a source checkout, the equivalent command is `python3 skills/last30days/scripts/last30days.py --preflight`. It shows the config source, browser-cookie plan, planned writes, optional commands, and ignored project config without reading cookies, writing files, or running research.
|
||||
|
||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
||||
|
||||
```bash
|
||||
@@ -229,17 +234,41 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) from the latest release
|
||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||
3. Click the `+` button in the Skills panel and drop the file in
|
||||
2. Go to [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. Click the `+` button in the Skills panel > click on `Create skill` > `Upload a skill` and browse/drop the file in
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop installs `/last30days` as an MCP server via a `.mcpb` bundle (a one-click Model Context Protocol package).
|
||||
|
||||
1. Go to the [latest release](https://github.com/mvanhorn/last30days-skill/releases/latest) and download the `.mcpb` for your platform:
|
||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Open Claude Desktop, go to Settings > Extensions, and drag the file in.
|
||||
3. When prompted, paste API keys for the sources you want to enable. Every field is optional — the engine degrades to web-only mode if you skip them all. Keys are stored in your OS keychain.
|
||||
4. Restart Claude Desktop. Ask Claude to "research Peter Steinberger" or any topic and it will call the `research` tool.
|
||||
|
||||
**Host requirement:** Python 3.12+ on PATH. The bundle ships the engine source but uses your local Python interpreter. Install from [python.org](https://www.python.org/downloads/) on Windows; macOS and most Linux distros ship a compatible version.
|
||||
|
||||
**Keys don't sync with the Code skill.** Claude Desktop and Claude Code maintain separate credential stores by design. If you already configured `~/.config/last30days/.env` for the Code skill, you'll re-enter the same keys here once.
|
||||
|
||||
Windows support is deferred until per-platform manifest entry points are sorted out; track in a follow-up issue.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
For X/Twitter action workflows outside `/last30days` research, such as posting
|
||||
tweets or replies, follower export, media handling, monitors, and giveaway
|
||||
draws, use [TweetClaw](https://github.com/Xquik-dev/tweetclaw) as the companion
|
||||
OpenClaw plugin. TweetClaw is maintained by Xquik-dev and is listed only as an
|
||||
optional companion path, not a last30days dependency or endorsement.
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
```bash
|
||||
@@ -249,7 +278,7 @@ ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
|
||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds, including the free arXiv and Techmeme CLIs.
|
||||
|
||||
## Bring your own keys
|
||||
|
||||
@@ -257,12 +286,15 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
|
||||
| Sources | What you need | Cost |
|
||||
|---------|---------------|------|
|
||||
| Reddit (with comments) + HN + Polymarket + GitHub | Nothing | Free |
|
||||
| X / Twitter | Log into x.com in any browser | Free |
|
||||
| Reddit (with comments) + HN + Polymarket + GitHub + StockTwits | Nothing | Free |
|
||||
| arXiv + Techmeme | Free CLIs, auto-installed by first-run setup | Free |
|
||||
| X / Twitter | Log into x.com in any browser, or set `XQUIK_API_KEY` / `XAI_API_KEY` | Browser cookies are free; keys are provider-specific |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + YouTube comments | ScrapeCreators key | 10,000 free calls, then PAYG |
|
||||
| Xiaohongshu (RED) | Run a logged-in x-mcp browser plugin or `xiaohongshu-mcp` service and opt in with `--search xhs` per run or `INCLUDE_SOURCES=xiaohongshu` in `.env`; last30days auto-probes `http://localhost:18060` then `http://host.docker.internal:18060`, or use `XIAOHONGSHU_API_BASE` for a custom URL | No last30days API key; depends on your local browser-session service |
|
||||
| DripStack (premium financial newsletters) | Opt-in: `--search dripstack` per run, or `INCLUDE_SOURCES=dripstack` in `.env` | No key; free public search API |
|
||||
| Perplexity Sonar / Search API / Deep Research | Perplexity key, or OpenRouter key as Sonar fallback | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
@@ -283,18 +315,32 @@ skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
|
||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||
|
||||
Already have keys under different Keychain service names? Set the non-secret `LAST30DAYS_KEYCHAIN_ALIASES` mapping described in [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items) instead of copying secrets.
|
||||
|
||||
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
||||
|
||||
## Configuration
|
||||
|
||||
Two things you'll likely want to know on day one:
|
||||
|
||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--output <file>` when you need the rendered result at an exact path, using the format selected by `--emit`. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each `--save-dir` run produces `<slug>-raw[-suffix].md`.
|
||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--output <file>` when you need the rendered result at an exact path, using the format selected by `--emit`. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each `--save-dir` run produces `<slug>-raw[-suffix].md`. Run `python3 skills/last30days/scripts/last30days.py --preflight` to review planned writes before a research run.
|
||||
|
||||
**Structured output for agents and workflows.** Ask `/last30days` for machine-readable JSON to receive the stable, versioned agent profile. For direct engine use in scripts or development, run `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json`; add `--json-profile=raw` only when you need the unversioned internal `Report` dump. See the [JSON export field reference and versioning policy](docs/reference/json-export.md).
|
||||
|
||||
**Topic-less discovery.** Ask `/last30days what's trending in AI agents?` to get a ranked discovery brief instead of researching a topic you already know - on an agent host this runs the three-command host-judged protocol (the model names topics, filters junk, scores worthiness, and writes the content angles). For direct engine use in scripts or cron, run `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` (one-shot: deterministic topic names, no angles); add `--emit=json` for the versioned discovery contract. Discovery is mutually exclusive with a positional topic and `--drill`.
|
||||
|
||||
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
**A subscribable research library.** Ask `/last30days` to build your library feed, or use `python3 skills/last30days/scripts/last30days.py library feed` directly for scripting and development. It turns saved briefs into `index.html`, a local Atom `feed.xml`, and readable brief pages. Add `--publish` only when you want the HTML index and brief pages hosted; publishing is explicit opt-in and public by default. To make the Atom feed subscribable, host the generated output directory on a static host such as GitHub Pages.
|
||||
|
||||
**Search everything you've researched.** Ask `/last30days search my library for MCP servers` or `/last30days have I researched MCP servers before?`. For direct engine use, run `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`. Search is offline and deterministic: it incrementally indexes the same saved briefs used by the library feed, merges matching per-run store sightings, and groups results by topic and date. Fresh runs also surface a compact **From your library** section when prior research overlaps the current topic; set `LAST30DAYS_LIBRARY_CONTEXT=off` to disable that passive context.
|
||||
|
||||
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Showcase: community research feeds
|
||||
|
||||
Published a recurring AI update, market watch, or wonderfully narrow obsession with last30days? Share the public library URL—or the Atom URL after hosting `feed.xml` on a static host—in [the community showcase thread](https://github.com/mvanhorn/last30days-skill/issues/532). Community feeds will be linked here as their owners submit them; the thread is the collection point in the meantime.
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||
@@ -315,11 +361,11 @@ Per-client wrapper scripts, custom category-peer subreddits, and the experimenta
|
||||
|
||||
## Open source
|
||||
|
||||
MIT license. No tracking. No analytics. Your research stays on your machine. 1,012 tests.
|
||||
MIT license. No tracking. No analytics. Your research stays on your machine. 2,700+ tests.
|
||||
|
||||
Built with Python 3.12+, yt-dlp, Node.js (vendored Bird client for X search), and ScrapeCreators API. v3 engine architecture by [@j-sperling](https://github.com/j-sperling).
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) to open a PR, [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list of community contributors, and [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
|
||||
## Star History
|
||||
|
||||
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
# /last30days
|
||||
|
||||
[English](README.md) | [Français](README.fr.md) | [Deutsch](README.de.md) | [Español](README.es.md) | Português (Brasil) | [日本語](README.ja.md) | [简体中文](README.zh-CN.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://trendshift.io/repositories/21997" target="_blank">
|
||||
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**Um buscador conduzido por um agente de IA, que pontua por votos positivos, curtidas e dinheiro de verdade — não por redações.**
|
||||
|
||||
Este README descreve o pipeline v3 atual. A especificação de execução da skill fica em [skills/last30days/SKILL.md](skills/last30days/SKILL.md), que é a referência definitiva sobre o comportamento dos comandos e da configuração.
|
||||
|
||||
**Claude Code (recomendado — atualizações automáticas via marketplace):**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, ou qualquer um dos 50+ hosts do [Agent Skills](https://agentskills.io):**
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
(`-g` instala globalmente para o seu usuário, então fica disponível em todos os projetos. Omita essa flag se quiser limitar a instalação a um projeto.)
|
||||
|
||||
Outras formas de instalar (claude.ai web, OpenClaw, manual) estão na seção [Instalação](#instalação), mais abaixo.
|
||||
|
||||
Configuração zero. Reddit, HN, Polymarket e GitHub funcionam de imediato. Rode uma vez e o assistente de configuração libera X, YouTube, TikTok, arXiv, Techmeme e mais em 30 segundos.
|
||||
|
||||
---
|
||||
|
||||
Os votos positivos do Reddit. As curtidas do X. As transcrições do YouTube. O engajamento no TikTok. As probabilidades do Polymarket, lastreadas em dinheiro de verdade e em informação privilegiada. São milhões de pessoas votando todo dia com a atenção e com o bolso. O /last30days busca tudo isso em paralelo, pontua pelo que as pessoas realmente engajam e um agente de IA atua como juiz para sintetizar tudo em um único briefing.
|
||||
|
||||
O Google agrega redações. O /last30days busca pessoas.
|
||||
|
||||
Essa busca você não encontra em nenhum outro lugar, porque nenhuma IA sozinha tem acesso a tudo. O Google não alcança nem os comentários do Reddit nem as publicações do X. O ChatGPT tem acordo com o Reddit, mas não consegue buscar no X nem no TikTok. O Gemini tem o YouTube, mas não o Reddit. O Claude não tem nenhum deles de forma nativa. Cada plataforma é um jardim murado, com API, tokens e autenticação próprios. Mas você pode trazer suas próprias chaves e sessões de navegador e, de repente, um agente de IA busca em todas ao mesmo tempo, compara umas com as outras e diz o que realmente importa.
|
||||
|
||||
É esse o destravamento. Não é um buscador melhor: é uma dúzia de plataformas isoladas, conectadas por um agente.
|
||||
|
||||
```
|
||||
/last30days Peter Steinberger
|
||||
```
|
||||
|
||||
Você tem uma reunião amanhã. Procura a pessoa no Google. Aparece o LinkedIn dela de 2023. O /last30days entrega o que ela está fazendo de fato neste mês: entrou na OpenAI para trabalhar no Codex, enfrenta o veto da Anthropic a agentes de terceiros, entregou 23 PRs com 85 % de taxa de merge, constrói o "LobsterOS" para controlar agentes entre dispositivos, e uma thread no r/ClaudeCode chegou a 569 votos positivos discutindo se ele é um herói ou "insuportável". Tudo espalhado entre publicações no X, threads no Reddit, transcrições do YouTube e commits no GitHub. Nada disso estava no Google.
|
||||
|
||||
## Por que isso existe
|
||||
|
||||
Construí para acompanhar o ritmo da IA. Tudo muda todo dia, e o pessoal do Reddit e do X sempre sabe primeiro. Eu precisava de prompts melhores, e os dados de treinamento estavam sempre meses atrás do que a comunidade já tinha descoberto.
|
||||
|
||||
Mas virou algo maior. Hoje eu rodo antes de uma call de vendas, para saber a verdade dos últimos 30 dias sobre uma empresa. Antes de uma reunião, para ler os tweets recentes e as transcrições de podcast de quem vou encontrar. Antes de uma viagem à Disney World, para saber quais brinquedos estão fechados e o que a comunidade acha do Genie+. Antes de construir qualquer coisa, para saber em quais problemas as pessoas realmente estão esbarrando.
|
||||
|
||||
Se você vai se reunir com um CEO, já leu todos os tweets e todas as transcrições do YouTube dos últimos 30 dias? Eu li.
|
||||
|
||||
## Fontes, pontuadas pelas pessoas
|
||||
|
||||
| Fonte | O que as pessoas te contam |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | A opinião sem filtro. Os melhores comentários com a contagem real de votos positivos, de graça e sem chave de API. As opiniões de verdade que o Google enterra. |
|
||||
| **X / Twitter** | A opinião quente, a thread do especialista, a primeira reação ao factual. Os primeiros a saber, os primeiros a discutir. |
|
||||
| **YouTube** | A análise de 45 minutos. Transcrições completas, garimpadas atrás das 5 frases citáveis que importam. |
|
||||
| **TikTok** | O criador que alcança 3,6 milhões de pessoas com uma leitura que você nunca vai achar no Google. |
|
||||
| **Instagram Reels** | O olhar dos influenciadores, com a transcrição do que é falado. O sinal da cultura visual. |
|
||||
| **Hacker News** | O consenso da turma de desenvolvimento. 825 pontos, 899 comentários. Onde o pessoal técnico discute de verdade. |
|
||||
| **Polymarket** | Não são opiniões. São probabilidades. Lastreadas em dinheiro de verdade. 96 % de confiança em vendas de um álbum. 4 % em uma aquisição. |
|
||||
| **GitHub** | Para pessoas: ritmo de PRs, principais repositórios por estrelas, notas de versão. Para assuntos: issues e discussions. |
|
||||
| **Digg** | Agrupamentos de histórias curados a partir do ranking AI 1000 do Digg (cerca de 1000 contas de IA com alto sinal no X), com citações atribuíveis embutidas e sem exigir autenticação no X. Ativa sozinho quando `digg-pp-cli` está no PATH. |
|
||||
| **arXiv** | Os artigos científicos por trás do hype. Pesquisa nova dentro da janela, de graça e sem chave de API. Ativa sozinho quando `arxiv-pp-cli` está no PATH (a configuração inicial instala). |
|
||||
| **Techmeme** | A camada editorial do noticiário de tecnologia, limitada à sua janela de 30 dias. De graça e sem chave de API. Ativa sozinho quando `techmeme-pp-cli` está no PATH (a configuração inicial instala). |
|
||||
| **LinkedIn** | O sinal profissional. Publicações e artigos, com os artigos ponderados como sinal forte. |
|
||||
| **StockTwits** | O humor dos traders. Ativa automaticamente quando seu assunto é um ticker ou uma cripto. |
|
||||
| **Threads** | A camada de texto do pós-Twitter. Conversas de criadores e marcas. |
|
||||
| **Pinterest** | Descoberta visual. Pins, itens salvos e comentários sobre produtos e ideias. |
|
||||
| **Xiaohongshu (RED)** | Sinais chineses sobre estilo de vida, produtos e criadores. É pedido explicitamente com `--search xhs` quando há um plugin de navegador x-mcp logado ou um serviço `xiaohongshu-mcp` rodando localmente. |
|
||||
| **Bluesky** | A camada social descentralizada. Publicações do AT Protocol vindas da migração pós-Twitter. |
|
||||
| **Perplexity** | A síntese fundamentada do Sonar, os resultados brutos da Search API e o Deep Research. |
|
||||
| **Web** | A cobertura editorial, as comparações de blog. Um sinal entre muitos, não o único. |
|
||||
|
||||
A comunidade não para de acrescentar fontes. Truth Social e outras fontes de nicho já estão no motor, e vêm mais por aí.
|
||||
|
||||
Uma thread do Reddit com 1.500 votos positivos é um sinal mais forte do que um post de blog que ninguém leu. Um TikTok com 3,6 milhões de visualizações diz mais sobre o que é culturalmente relevante do que qualquer release de imprensa. Probabilidades do Polymarket lastreadas em US$ 66 mil de volume são bem mais difíceis de contestar do que o palpite de um comentarista.
|
||||
|
||||
A síntese ordena pelo que as pessoas de verdade realmente engajaram. Relevância social, não relevância de SEO.
|
||||
|
||||
## Para que as pessoas realmente usam
|
||||
|
||||
**Antes de uma reunião.** `/last30days Peter Steinberger` — entrou no time do Codex na OpenAI, enfrenta o veto da Anthropic a agentes de terceiros, 23 PRs mergeados com 85 % de taxa de merge no GitHub, constrói o LobsterOS para controlar agentes entre dispositivos. r/ClaudeCode: "Desde que o OpenClaw saiu, todo mundo já sabia que, se você rodasse por qualquer coisa que não fosse a API, uma hora ia ser banido" (227 votos positivos). Isso não está no LinkedIn.
|
||||
|
||||
**Para ler sinais de contratação.** `/last30days Listen Labs --hiring-signals` — as vagas e páginas de carreira atuais viram evidência citada de mudança de prioridade: contratação em segurança para empresas, customer success, infraestrutura ou expansão de produto. O relatório diz o que a contratação parece sinalizar, não o que o roadmap vai entregar.
|
||||
|
||||
**Para achar o assunto antes do pico.** Pergunte `/last30days what's exploding in AI agents?` e a skill muda para o modo descoberta: o motor varre as listagens por categoria do Reddit, a capa e as melhores histórias do Hacker News, o feed AI 1000 do Digg e o X quando você está autenticado; seu agente avalia as indicações (nomes, filtragem de ruído, se rende conteúdo) e escreve ângulos para podcast ou para um artigo no X; depois você recebe de 5 a 10 assuntos ordenados por velocidade. Cada resultado traz números de várias fontes, um rótulo de momentum e um comando `/last30days "<topic>"` pronto para rodar.
|
||||
|
||||
**Quando alguma coisa é lançada.** `/last30days Kanye West` — o Reino Unido bloqueou o visto dele, o Wireless Festival foi cancelado, os patrocinadores fugiram. Mas BULLY estreou em 2º na Billboard. Fantano voltou do "Yay sabbatical" para resenhar o disco (653 mil visualizações). No SoFi Homecoming, ele levou Lauryn Hill e Travis Scott ao palco para 44 músicas. Polymarket: "Kanye vai tuitar de novo?" 86 % sim. 23 threads no Reddit, 17 vídeos no YouTube, 86 mil votos positivos.
|
||||
|
||||
**Para comparar ferramentas.** `/last30days OpenClaw vs Hermes vs Paperclip` — "Não são concorrentes, são camadas." O OpenClaw é a camada de execução (351 mil estrelas no GitHub, em produção), o Hermes é o cérebro que se aprimora sozinho (31 mil estrelas), o Paperclip é o organograma (49 mil estrelas). As contagens de estrelas vêm ao vivo da API do GitHub, não de posts de blog desatualizados. Tabela lado a lado com arquitetura, memória, segurança e melhor caso de uso. Segundo @IMJustinBrooke: "OpenClaw = Charmander, Hermes = Charizard."
|
||||
|
||||
**Para entender o mundo.** `/last30days Iran vs USA` — dia 38 da guerra. O ultimato de Trump, com prazo até terça, para o Irã reabrir o Estreito de Ormuz. Dois caças americanos abatidos. Petróleo a US$ 126 o barril. A AIE chamou o episódio de "a maior interrupção de fornecimento da história do mercado global de petróleo". Polymarket: cessar-fogo até 31 de dezembro a 74 %. 27 publicações no X, 10 vídeos no YouTube, 20 mercados de previsão.
|
||||
|
||||
**Antes de uma viagem.** `/last30days Universal Epic Universe` — a expansão já está em obras. Alvará do "Project 680" protocolado. Show de fogos confirmado pela infraestrutura, mas ainda não anunciado. Tempo de espera: Mine-Cart Madness com média de 148 minutos. Ainda sem passe anual, e os moradores estão irritados. Stardust Racers fechada para reforma até 5 de abril.
|
||||
|
||||
**Para aprender algo rápido.** `/last30days Nano Banana Pro prompting` — prompts estruturados em JSON estão substituindo a sopa de tags. O formato aninhado do @pictsbyai evita o "concept bleeding". Editar ganha de regerar. E depois a skill escreve um prompt de produção usando exatamente o que a comunidade disse que funciona.
|
||||
|
||||
## Novidades
|
||||
|
||||
Desde o anúncio da v3.3 em maio e até a v3.11.1 (julho de 2026): 175 PRs mergeados — 122 deles de 52 pessoas da comunidade — distribuídos em 15 versões. Foi isso que entrou.
|
||||
|
||||
### Cidadão de primeira classe no OpenAI Codex
|
||||
|
||||
O /last30days agora é um plugin nativo do Codex com configuração guiada: não é um port, é cidadão de primeira classe. As citações levam o renderizador em conta, então a saída no Codex se lê como um briefing e não como uma sopa de URLs (#694), e o mesmo motor roda no Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw e em 50+ hosts do Agent Skills. Manifesto do plugin do Codex por [@rfoust](https://github.com/rfoust) (#686), correção de autenticação no Codex por [@tmchow](https://github.com/tmchow) (#698).
|
||||
|
||||
### arXiv, Techmeme e Digg — de graça, sem chaves de API
|
||||
|
||||
O arXiv traz os artigos científicos por trás do hype e o Techmeme traz a camada editorial do noticiário de tecnologia — de graça, sem nenhuma chave, e a configuração inicial instala as CLIs deles para que ativem sozinhos (#709). Os agrupamentos de histórias AI 1000 do Digg chegam do mesmo jeito, sem autenticação no X: a configuração instala a CLI gratuita do Digg para você (#590). O Trustpilot está disponível como opção para pesquisa de marcas de consumo.
|
||||
|
||||
### Reddit gratuito, com pontuações reais e melhores comentários
|
||||
|
||||
A API pública .json do Reddit morreu; o caminho gratuito voltou mais forte. RSS sem chave e scraping do shreddit (#457), descoberta de subreddits específicos com contagem real de votos positivos via arctic-shift (#696) e um piso de relevância para que um post viral fora do tema não sequestre seu briefing (#488, valeu [@rzachsmith](https://github.com/rzachsmith)). Sem chave de API. Pontuações reais. Melhores comentários incluídos.
|
||||
|
||||
### Os melhores comentários em cada briefing
|
||||
|
||||
Os comentários agora são uma camada ligada por padrão em todas as fontes: comentários do Instagram com diversidade baseada em ranking, para que cinco opiniões fortes não venham todas do mesmo post (#751), comentários do YouTube mais um backup de transcrição via ScrapeCreators para quando o yt-dlp falha (#637), e comentários votados pela comunidade entrando com peso no Best Takes, para que as melhores tiradas sobrevivam à pontuação (#592, #608).
|
||||
|
||||
### Um único comando doctor
|
||||
|
||||
Peça um diagnóstico: o doctor testa cada fonte e receita as correções exatas — qual chave está faltando, qual CLI não está no PATH, qual cookie expirou (#753). Chega de adivinhar por que o X voltou fraco.
|
||||
|
||||
### A busca no X, reconstruída
|
||||
|
||||
O pipeline do X foi refeito do zero: faixas FROM e ABOUT para que tanto as publicações da própria pessoa quanto a conversa sobre ela sejam ranqueadas (#610), desambiguação de subconsultas conforme a pessoa buscada (#611), verificação de autoria de primeira mão com ranqueamento por sinais de interação (#613) e uma única fonte X com failover automático entre backends (#622). Além de um `--diagnose` honesto, que testa a autenticação de verdade (#609).
|
||||
|
||||
### Mais fontes entraram
|
||||
|
||||
LinkedIn via ScrapeCreators, com artigos como sinal forte ([@ravstr](https://github.com/ravstr), #702). O StockTwits ativa sozinho em assuntos de ticker e cripto ([@wtiwana](https://github.com/wtiwana), #658). O Perplexity ganhou modos de API diretos e Deep Research assíncrono ([@sk-holmes](https://github.com/sk-holmes), #629).
|
||||
|
||||
### Endurecido pela comunidade
|
||||
|
||||
A onda de segurança foi quase toda trabalho da comunidade: correções de XSS armazenado no renderizador HTML ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), arquivos temporários de cookie protegidos, CI endurecida contra ataques à cadeia de suprimentos com OpenSSF Scorecard e atestação de proveniência de build ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), varreduras com Semgrep e OSV-Scanner mais um portão de revisão de dependências em cada PR ([@23241a6749](https://github.com/23241a6749)), um piso de cobertura de testes criado em 60 % e desde então elevado para 84 % ([@gourab5139014](https://github.com/gourab5139014)), e uma varredura de segurança do Hermes que hoje não tem nenhum achado CRITICAL (#768).
|
||||
|
||||
### Alcança mais longe
|
||||
|
||||
Hebraico e outras línguas não latinas ([@dudyme](https://github.com/dudyme)). Tokenização adaptada a CJK para fontes chinesas ([@An-idd](https://github.com/An-idd)). Uma onda de compatibilidade com Windows. Extração de cookies em toda a família Chromium — Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov)) — além do Keychain do macOS e do pass(1) no Linux como origens de credenciais. Consulta retroativa com `--as-of` ([@chiyi-creator](https://github.com/chiyi-creator)). Provisionamento automático do Python 3.12 via uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` para ler as páginas de vagas de uma empresa. Deltas de watchlist entre execuções.
|
||||
|
||||
### O que já vinha de fábrica desde a v3
|
||||
|
||||
As bases da v3 continuam todas aqui: o cérebro de pré-pesquisa, que identifica os handles, subreddits e hashtags certos antes de disparar uma única chamada de API (construído por [@j-sperling](https://github.com/j-sperling)); a pontuação Best Takes, que considera humor e viralidade além de relevância; a fusão de clusters entre fontes; as comparações em uma única passada ("CLI vs MCP" em 3 minutos, não em 12); as comparações `--competitors` descobertas automaticamente; o modo pessoa do GitHub (`--github-user=steipete`); o modo ELI5 ("eli5 on" depois de qualquer execução); e os briefings HTML autocontidos e compartilháveis (`--emit=html`). Os ajustes de configuração estão em [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Instalação
|
||||
|
||||
| Ambiente | Instalação | Atualizações |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recomendado) | `/plugin marketplace add mvanhorn/last30days-skill` | Automáticas via marketplace, ou `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` e depois `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, ou qualquer um dos 50+ hosts do [Agent Skills](https://agentskills.io)** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Baixe `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) e envie por claude.ai > Customize > Skills > + > Create skill > Upload a skill | Baixar de novo e enviar de novo |
|
||||
| **Claude Desktop** | [Baixe o `.mcpb` da sua plataforma](https://github.com/mvanhorn/last30days-skill/releases/latest) e arraste para Settings > Extensions | Baixar de novo e arrastar o novo pacote |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recomendado)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Recomendado porque o marketplace do Claude Code cuida das atualizações por você: o cache do plugin é versionado e se atualiza sozinho quando sai uma versão nova. Rode `claude plugin update last30days@last30days-skill` para forçar uma verificação.
|
||||
|
||||
Se preferir usar o caminho de instalação do Agent Skills no Claude Code, ele também é suportado:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
O plugin nativo e a instalação com `npx skills` podem conviver. Só atenção: o Claude Code não deduplica entre métodos de instalação. Se você tiver ativos ao mesmo tempo o plugin do marketplace e a cópia do `npx skills`, o `/last30days` vai aparecer duas vezes. Use um método de instalação por máquina.
|
||||
|
||||
### Grok (xAI Build CLI)
|
||||
|
||||
O [Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) instala o last30days como plugin nativo. A instalação direta acompanha o repositório:
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Ou adicione este repositório como fonte de marketplace e depois instale pelo nome do plugin:
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
Acrescente `--trust` para pular a confirmação de instalação. Atualize com `grok plugin update last30days`. O Grok também lê os manifestos do Claude Code por compatibilidade; o par nativo `.grok-plugin/` é o caminho principal — e é para ele que aponta um registro oficial no [marketplace da xAI](https://github.com/xai-org/plugin-marketplace). O `npx skills add` continua sendo uma alternativa válida em qualquer host.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI e outros hosts do Agent Skills
|
||||
|
||||
Instale pela CLI aberta do [Agent Skills](https://agentskills.io) — ela suporta 50+ hosts, entre eles `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose` e outros (lista completa no [repositório vercel-labs/skills](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
A flag `-g` (global) instala no seu diretório de usuário, então a skill fica disponível em todos os projetos. Sem `-g`, o `npx skills` instala só no projeto, dentro de `./.skills/` (e vai versionado junto com o repositório). Para uma ferramenta feita para pesquisar o mundo inteiro, o que você quer é a instalação global.
|
||||
|
||||
O Codex desktop e outros hosts que trabalham no nível de pasta funcionam tanto em pastas comuns quanto em repositórios Git. Antes da primeira pesquisa, peça ao agente host que rode o `scripts/last30days.py --preflight` que acompanha a skill, a partir do diretório da skill carregada; em um clone do código-fonte, o comando equivalente é `python3 skills/last30days/scripts/last30days.py --preflight`. Ele mostra de onde vem a configuração, quais cookies do navegador seriam lidos, quais arquivos seriam escritos, quais comandos opcionais existem e qual configuração de projeto está sendo ignorada — sem ler cookies, sem escrever arquivos e sem rodar pesquisa nenhuma.
|
||||
|
||||
Por padrão, a instalação vale para o host que o `npx skills` detectar. Para mirar em um específico (ou em vários):
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Para atualizar depois:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
Ou atualize tudo que você instalou globalmente pelo `npx skills`:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
Dá para listar e remover com `npx skills list -g` e `npx skills remove last30days -g`.
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Baixe `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) da versão mais recente
|
||||
2. Vá em [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. Clique no botão `+` no painel de Skills, depois em `Create skill` > `Upload a skill`, e escolha ou arraste o arquivo
|
||||
|
||||
Ative antes "Code execution and file creation" em Capabilities — sem isso, as skills não rodam.
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
O Claude Desktop instala o `/last30days` como servidor MCP por meio de um pacote `.mcpb` (um pacote Model Context Protocol de um clique).
|
||||
|
||||
1. Vá até a [versão mais recente](https://github.com/mvanhorn/last30days-skill/releases/latest) e baixe o `.mcpb` da sua plataforma:
|
||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Abra o Claude Desktop, vá em Settings > Extensions e arraste o arquivo para lá.
|
||||
3. Quando for solicitado, cole as chaves de API das fontes que quiser ativar. Todo campo é opcional — se pular todos, o motor cai para o modo só web. As chaves ficam guardadas no chaveiro do seu sistema operacional.
|
||||
4. Reinicie o Claude Desktop. Peça ao Claude para "pesquisar sobre Peter Steinberger", ou sobre qualquer outro assunto, e ele vai chamar a ferramenta `research`.
|
||||
|
||||
**Requisito do host:** Python 3.12+ no PATH. O pacote traz o código do motor, mas usa o seu interpretador Python local. No Windows, instale a partir do [python.org](https://www.python.org/downloads/); o macOS e a maioria das distribuições Linux já vêm com uma versão compatível.
|
||||
|
||||
**As chaves não são compartilhadas com a skill do Claude Code.** O Claude Desktop e o Claude Code mantêm armazenamentos de credenciais separados de propósito. Se você já configurou o `~/.config/last30days/.env` para a skill do Claude Code, vai precisar digitar essas mesmas chaves aqui uma vez.
|
||||
|
||||
O suporte a Windows está adiado até que os pontos de entrada por plataforma no manifesto sejam resolvidos; o acompanhamento fica em uma issue à parte.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
Para fluxos de ação no X/Twitter fora da pesquisa do `/last30days` — publicar
|
||||
tweets ou respostas, exportar seguidores, cuidar de mídia, monitorar contas e
|
||||
apurar sorteios — use o [TweetClaw](https://github.com/Xquik-dev/tweetclaw) como
|
||||
plugin complementar do OpenClaw. O TweetClaw é mantido pelo Xquik-dev e aparece
|
||||
aqui apenas como opção complementar: não é dependência nem recomendação do
|
||||
last30days.
|
||||
|
||||
### Manual (para quem desenvolve)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
O symlink mantém a instalação em sincronia com sua árvore de trabalho conforme você edita — não precisa copiar de novo. Para o `claude.ai`, compile o arquivo `.skill` a partir do código-fonte: `bash skills/last30days/scripts/build-skill.sh` gera `dist/last30days.skill`.
|
||||
|
||||
Reddit (com comentários), Hacker News, Polymarket e GitHub funcionam de imediato. Configuração zero. Rode `/last30days` uma vez e o assistente de configuração libera mais fontes em 30 segundos, incluindo as CLIs gratuitas do arXiv e do Techmeme.
|
||||
|
||||
## Traga suas próprias chaves
|
||||
|
||||
Essas plataformas não têm relação nenhuma entre si. O X não sabe o que o Reddit pensa. O YouTube não enxerga o TikTok. Mas você pode trazer suas próprias chaves de API e tokens de navegador e, de repente, tem acesso a todas ao mesmo tempo.
|
||||
|
||||
| Fontes | O que você precisa | Custo |
|
||||
|---------|---------------|------|
|
||||
| Reddit (com comentários) + HN + Polymarket + GitHub + StockTwits | Nada | De graça |
|
||||
| arXiv + Techmeme | CLIs gratuitas, instaladas automaticamente pela configuração inicial | De graça |
|
||||
| X / Twitter | Faça login em x.com em qualquer navegador, ou defina `XQUIK_API_KEY` / `XAI_API_KEY` | Os cookies do navegador são gratuitos; as chaves dependem do provedor |
|
||||
| YouTube | `brew install yt-dlp` | De graça |
|
||||
| Bluesky | Uma senha de aplicativo do bsky.app | De graça |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + comentários do YouTube | Uma chave do ScrapeCreators | 10.000 chamadas gratuitas e depois pagamento por uso |
|
||||
| Xiaohongshu (RED) | Deixe rodando um plugin de navegador x-mcp logado ou um serviço `xiaohongshu-mcp` e habilite a fonte com `--search xhs` por execução ou com `INCLUDE_SOURCES=xiaohongshu` no `.env`; o last30days testa automaticamente `http://localhost:18060` e depois `http://host.docker.internal:18060`, ou use `XIAOHONGSHU_API_BASE` para uma URL própria | Não precisa de chave de API do last30days; depende do seu serviço local de sessão de navegador |
|
||||
| DripStack (newsletters financeiras premium) | Opcional: `--search dripstack` por execução, ou `INCLUDE_SOURCES=dripstack` no `.env` | Sem chave; API de busca pública e gratuita |
|
||||
| Perplexity Sonar / Search API / Deep Research | Uma chave do Perplexity, ou uma chave do OpenRouter como alternativa para o Sonar | Pagamento por uso |
|
||||
| Busca na web | Uma chave do Brave Search | 2.000 consultas gratuitas por mês |
|
||||
|
||||
### Keychain do macOS (opcional)
|
||||
|
||||
No macOS você pode guardar as chaves no Keychain do sistema em vez de em um arquivo `.env`. A skill as encontra automaticamente como a origem de menor prioridade — em caso de conflito, os arquivos `.env` e o ambiente do processo continuam ganhando.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Os itens ficam guardados sob o nome de serviço `last30days-<KEY>` para o usuário atual. Em plataformas que não são Darwin o carregador não faz nada, então não há mudança de comportamento para quem usa Linux ou Windows.
|
||||
|
||||
Já tem chaves guardadas com outros nomes de serviço no Keychain? Defina o mapeamento não secreto `LAST30DAYS_KEYCHAIN_ALIASES` descrito em [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items), em vez de copiar segredos.
|
||||
|
||||
Veja [CONFIGURATION.md](CONFIGURATION.md) para a matriz completa de chaves por fonte, a ordem de prioridade dos provedores de raciocínio e a dos backends de busca web.
|
||||
|
||||
## Configuração
|
||||
|
||||
Duas coisas que você provavelmente vai querer saber no primeiro dia:
|
||||
|
||||
**Onde os arquivos de pesquisa são salvos.** O `LAST30DAYS_MEMORY_DIR` aponta por padrão para `~/Documents/Last30Days/` (no Windows: `C:\Users\<you>\Documents\Last30Days\`). Para mudar, defina essa variável de ambiente no seu shell com o caminho que quiser, ou use `--save-dir <path>` em uma execução específica. Use `--output <file>` quando precisar do resultado renderizado em um caminho exato, no formato escolhido por `--emit`. Use `--save-suffix=<name>` para manter separadas várias variações do mesmo assunto (por cliente, por exemplo). Cada execução com `--save-dir` gera `<slug>-raw[-suffix].md`. Rode `python3 skills/last30days/scripts/last30days.py --preflight` para conferir o que será escrito antes de disparar uma pesquisa.
|
||||
|
||||
**Saída estruturada para agentes e fluxos de trabalho.** Peça ao `/last30days` um JSON legível por máquina e você recebe o perfil de agente estável e versionado. Para usar o motor direto em scripts ou no desenvolvimento, rode `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json`; use `--json-profile=raw` só quando precisar do dump interno não versionado do `Report`. Veja a [referência de campos da exportação JSON e a política de versionamento](docs/reference/json-export.md).
|
||||
|
||||
**Descoberta sem assunto definido.** Pergunte `/last30days what's trending in AI agents?` para receber um briefing de descoberta ordenado, em vez de pesquisar um assunto que você já conhece. Em um host com agente, isso executa o protocolo de três comandos arbitrado pelo host (o modelo nomeia os assuntos, filtra ruído, avalia o que vale a pena e escreve os ângulos de conteúdo). Para usar o motor direto em scripts ou no cron, rode `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` (passada única: nomes de assunto determinísticos, sem ângulos); acrescente `--emit=json` para o contrato de descoberta versionado. A descoberta é mutuamente exclusiva com um assunto posicional e com `--drill`.
|
||||
|
||||
**Monitoramento de tendências entre execuções.** O modo padrão gera um snapshot Markdown novo a cada execução. Para acumular achados ao longo do tempo, acrescente `--store` e eles ficam guardados em um banco SQLite; depois use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) para execuções agendadas (com envio opcional por Slack ou webhook quando surgirem achados novos) e [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) para resumos diários ou semanais. O padrão de cadência completo está em [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
**Uma biblioteca de pesquisa que dá para assinar.** Peça ao `/last30days` que monte o feed da sua biblioteca, ou use direto `python3 skills/last30days/scripts/last30days.py library feed` para scripts e desenvolvimento. Ele transforma os briefings salvos em um `index.html`, um `feed.xml` Atom local e páginas de briefing legíveis. Acrescente `--publish` só quando quiser hospedar o índice HTML e as páginas de briefing; publicar é uma decisão explícita e, por padrão, é público. Para o feed Atom ficar realmente assinável, hospede o diretório de saída gerado em um serviço estático como o GitHub Pages.
|
||||
|
||||
**Busque em tudo que você já pesquisou.** Pergunte `/last30days search my library for MCP servers` ou `/last30days have I researched MCP servers before?`. Para usar o motor direto, rode `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`. A busca é offline e determinística: ela indexa aos poucos os mesmos briefings salvos que o feed da biblioteca usa, junta as ocorrências correspondentes registradas no store a cada execução e agrupa os resultados por assunto e data. Execuções novas também exibem uma seção compacta **From your library** ("da sua biblioteca") quando pesquisas anteriores se sobrepõem ao assunto atual; defina `LAST30DAYS_LIBRARY_CONTEXT=off` para desativar esse contexto passivo.
|
||||
|
||||
Scripts wrapper por cliente, subreddits de categoria personalizados e o canal beta experimental para personalizações em andamento também estão documentados em [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Vitrine: feeds de pesquisa da comunidade
|
||||
|
||||
Publicou com o last30days um panorama recorrente de IA, um acompanhamento de mercado ou uma obsessão maravilhosamente específica? Compartilhe a URL da sua biblioteca pública — ou a URL do Atom, depois de hospedar o `feed.xml` em um serviço estático — na [thread de vitrine da comunidade](https://github.com/mvanhorn/last30days-skill/issues/532). Os feeds da comunidade serão linkados aqui conforme as pessoas os enviarem; enquanto isso, a thread é o ponto de coleta.
|
||||
|
||||
## Como funciona
|
||||
|
||||
1. **Você digita um assunto.** Pessoa, empresa, produto, tecnologia, "X vs Y". Qualquer coisa.
|
||||
2. **O agente descobre quem importa.** Ele encontra os perfis do X (inclusive de fundadores), os repositórios do GitHub, os subreddits, as hashtags do TikTok e os canais do YouTube. Para "Kanye West" ele sabe que o caminho é r/hiphopheads, @kanyewest e "bully review" no YouTube. Para "OpenClaw" ele resolve openclaw/openclaw no GitHub e busca a contagem de estrelas ao vivo.
|
||||
3. **Todas as fontes buscadas em paralelo.** Expansão com várias consultas. Resultados pontuados por engajamento, relevância e frescor.
|
||||
4. **A profundidade que ninguém mais tem.** Transcrições completas do YouTube de vídeos de reação. Os melhores comentários do Reddit com a contagem de votos positivos. As legendas dos TikToks. As probabilidades do Polymarket. Não só títulos e links.
|
||||
5. **Mesma história, unificada.** O Wireless Festival anunciado no Reddit, discutido no X e com preço de ingresso no TikTok vira um cluster só, não três itens separados.
|
||||
6. **Sintetizado em um único briefing.** Ancorado em dados específicos. Citado por fonte. Ordenado pelo que as pessoas realmente engajam. Não é "olha o que eu encontrei", é "olha o que importa".
|
||||
7. **E então ele vira o seu especialista.** Depois de uma única execução, sua sessão do Claude sabe tudo o que a comunidade sabe. Faça perguntas de acompanhamento. Peça para escrever prompts, redigir e-mails, planejar viagens, desenhar arquiteturas — tudo ancorado no que é real agora.
|
||||
|
||||
## O que as pessoas estão dizendo
|
||||
|
||||
> "Achei uma skill do Claude Code que pesquisa qualquer assunto no Reddit, X, YouTube e HN dos últimos 30 dias. E ainda escreve os prompts pra você. Antes de cada conteúdo que eu escrevo, eu fazia essa busca na mão no Reddit e no X. Aba por aba. Thread por thread. É justamente essa a parte que leva 90 minutos. Isso elimina ela." — @itsjasonai
|
||||
|
||||
> "Essa skill sozinha substituiu todo o meu fluxo de pesquisa. Você dá um assunto e ela raspa Reddit, X e a web atrás do que as pessoas estão falando de verdade. Nada de post de blog velho. Conversas reais dos últimos 30 dias." — @itswilsoncharles
|
||||
|
||||
> "5 dos 10 repositórios em alta no GitHub hoje são ferramentas do Claude. O nº 1: mvanhorn/last30days-skill" — @yieldhunter95
|
||||
|
||||
## Código aberto
|
||||
|
||||
Licença MIT. Sem rastreamento. Sem analytics. Sua pesquisa fica na sua máquina. Mais de 2.700 testes.
|
||||
|
||||
Construído com Python 3.12+, yt-dlp, Node.js (cliente Bird embarcado para a busca no X) e a API do ScrapeCreators. Arquitetura do motor v3 por [@j-sperling](https://github.com/j-sperling).
|
||||
|
||||
Veja [CONTRIBUTING.md](CONTRIBUTING.md) para abrir um PR, [CONTRIBUTORS.md](CONTRIBUTORS.md) para a lista completa de quem contribuiu e [CHANGELOG.md](CHANGELOG.md) para o histórico de versões.
|
||||
|
||||
## Histórico de estrelas
|
||||
|
||||
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
|
||||
+381
@@ -0,0 +1,381 @@
|
||||
# /last30days
|
||||
|
||||
[English](README.md) | [Français](README.fr.md) | [Deutsch](README.de.md) | [Español](README.es.md) | [Português (Brasil)](README.pt-BR.md) | [日本語](README.ja.md) | 简体中文
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days——由 AI 智能体驱动、搜索真实用户而非编辑内容的搜索引擎" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending 单日排名第一的仓库" />
|
||||
</a>
|
||||
<br/>
|
||||
<a href="https://trendshift.io/repositories/21997" target="_blank">
|
||||
<img src="https://trendshift.io/api/badge/repositories/21997" alt="mvanhorn/last30days-skill | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**一个由 AI 智能体驱动的搜索引擎:按赞同票、点赞和真金白银评分,而不是由编辑决定。**
|
||||
|
||||
本文档对应当前的 v3 流水线。运行时 Skill 规范位于 [skills/last30days/SKILL.md](skills/last30days/SKILL.md),最新命令与配置行为以该文件为准。
|
||||
|
||||
**Claude Code(推荐——通过 marketplace 自动更新):**
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex、Cursor、Copilot、Gemini CLI,或其他 50 多个支持 [Agent Skills](https://agentskills.io) 的宿主:**
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
(`-g` 会安装到当前用户的全局环境,所有项目均可使用;去掉该参数则仅安装到当前项目。)
|
||||
|
||||
更多安装方式(claude.ai 网页版、OpenClaw、手动安装)见下方[安装](#安装)章节。
|
||||
|
||||
开箱即用。Reddit、Hacker News、Polymarket 和 GitHub 无需配置即可搜索。首次运行时,配置向导会在 30 秒内帮你解锁 X、YouTube、TikTok、arXiv、Techmeme 等更多来源。
|
||||
|
||||
---
|
||||
|
||||
Reddit 的赞同票、X 的点赞、YouTube 的完整字幕、TikTok 的互动数据,以及由真金白银和内幕信息支撑的 Polymarket 概率——每天都有数百万人用注意力和钱包投票。`/last30days` 会并行搜索这些平台,按照真实用户的参与度评分,再由 AI 智能体裁判综合成一份简报。
|
||||
|
||||
Google 聚合编辑选出的内容,`/last30days` 搜索真实的人。
|
||||
|
||||
你无法从别的单一搜索产品获得这些结果,因为没有哪个 AI 天生能访问所有平台。Google 搜不到 Reddit 评论和 X 帖子;ChatGPT 虽然与 Reddit 合作,却无法搜索 X 或 TikTok;Gemini 能访问 YouTube,却没有 Reddit;Claude 原生不具备这些能力。每个平台都是一座围墙花园,有自己的 API、令牌和认证机制。但只要接入你自己的密钥和浏览器会话,AI 智能体就能同时搜索所有平台、横向比较信号,并告诉你真正值得关注的内容。
|
||||
|
||||
这才是关键:不是再造一个更好的搜索引擎,而是让一个智能体把十几个彼此割裂的平台连接起来。
|
||||
|
||||
```
|
||||
/last30days Peter Steinberger
|
||||
```
|
||||
|
||||
假设你明天要和一个人开会。用 Google 搜他,你看到的可能还是 2023 年的 LinkedIn 页面;`/last30days` 告诉你的则是他这个月真正做了什么:加入 OpenAI 参与 Codex、反对 Anthropic 禁止第三方智能体、提交 23 个 PR 且合并率达到 85%、打造用于跨设备智能体控制的 “LobsterOS”,以及 r/ClaudeCode 上一场获得 569 个赞同票的争论——他究竟是英雄,还是“令人难以忍受”。这些信息散落在 X 帖子、Reddit 讨论、YouTube 字幕和 GitHub 提交中,Google 上根本没有。
|
||||
|
||||
## 为什么要做这个项目
|
||||
|
||||
最初,我做它是为了跟上 AI 的变化。这个领域每天都在变,而 Reddit 和 X 上的极客通常最先发现新东西。我需要更好的提示词,但模型训练数据总比社区已经摸索出的经验慢几个月。
|
||||
|
||||
后来,它变成了更大的东西。现在,销售通话前,我用它了解一家公司过去 30 天的真实动态;开会前,我用它读完对方最近的推文和播客字幕;去迪士尼世界前,我用它确认哪些项目停运、社区怎么看 Genie+;开始做任何产品前,我用它找出人们真正遇到的问题。
|
||||
|
||||
如果你要见一位 CEO,你读过他过去 30 天的所有推文和 YouTube 字幕吗?我读过。
|
||||
|
||||
## 由真实用户评分的信息源
|
||||
|
||||
| 来源 | 人们会告诉你什么 |
|
||||
|------|------------------|
|
||||
| **Reddit** | 未经过滤的真实看法。免费获取带实际赞同数的热门评论,无需 API 密钥;那些常被 Google 埋没的真实意见。 |
|
||||
| **X / Twitter** | 犀利观点、专家长帖和突发事件的第一反应。最早知道,也最早争论。 |
|
||||
| **YouTube** | 45 分钟的深度内容。搜索完整字幕,只提取真正值得引用的 5 句话。 |
|
||||
| **TikTok** | 一个触达 360 万人的创作者观点——你永远不会在 Google 上搜到。 |
|
||||
| **Instagram Reels** | 带口播字幕的影响者视角,反映视觉文化的信号。 |
|
||||
| **Hacker News** | 开发者共识:825 分、899 条评论,技术从业者真正交锋的地方。 |
|
||||
| **Polymarket** | 不是观点,而是由真金白银支撑的概率:专辑销量 96%,收购概率 4%。 |
|
||||
| **GitHub** | 搜人时查看 PR 速度、按 Star 排名的热门仓库和发行说明;搜主题时查看 Issue 与 Discussion。 |
|
||||
| **Digg** | 来自 Digg AI 1000 排行榜(约 1,000 个高信号 X 账号)的精选话题聚类,包含可追溯的行内引用,无需 X 认证。当 PATH 中存在 `digg-pp-cli` 时自动启用。 |
|
||||
| **arXiv** | 热点背后的论文。免费查找时间窗口内的新研究,无需 API 密钥。当 PATH 中存在 `arxiv-pp-cli` 时自动启用(首次配置会安装)。 |
|
||||
| **Techmeme** | 科技新闻的编辑视角,并按你设定的 30 天窗口筛选。免费且无需 API 密钥。当 PATH 中存在 `techmeme-pp-cli` 时自动启用(首次配置会安装)。 |
|
||||
| **LinkedIn** | 职业领域的信号。搜索帖子和文章,其中文章被视为高价值信号。 |
|
||||
| **StockTwits** | 交易者情绪。当主题是股票代码或加密货币时自动启用。 |
|
||||
| **Threads** | 后 Twitter 时代的文字内容层,汇集创作者和品牌的讨论。 |
|
||||
| **Pinterest** | 视觉发现:围绕产品和创意的 Pin、收藏与评论。 |
|
||||
| **小红书(RED)** | 来自中国生活方式、产品和创作者的信号。当本机运行已登录的 x-mcp 浏览器插件或 `xiaohongshu-mcp` 服务时,通过 `--search xhs` 显式启用。 |
|
||||
| **Bluesky** | 去中心化的社交内容层,搜索 Twitter 用户迁移后产生的 AT Protocol 帖子。 |
|
||||
| **Perplexity** | 基于来源的 Sonar 综合结果、原始 Search API 数据和 Deep Research。 |
|
||||
| **Web** | 编辑报道和博客对比。它只是众多信号之一,而不是唯一来源。 |
|
||||
|
||||
社区贡献者仍在不断加入更多平台。Truth Social 等垂直来源已经进入引擎,更多来源也在路上。
|
||||
|
||||
一条获得 1,500 个赞同票的 Reddit 帖子,信号强度高于一篇无人阅读的博客;一个拥有 360 万次观看的 TikTok,比新闻稿更能说明当下的文化热点;一个有 6.6 万美元成交量支撑的 Polymarket 概率,也比评论员的猜测更难反驳。
|
||||
|
||||
综合排序依据的是人们真正参与过的内容——看社会相关性,而不是 SEO 相关性。
|
||||
|
||||
## 大家实际上怎么用它
|
||||
|
||||
**开会之前。** `/last30days Peter Steinberger`——加入 OpenAI Codex 团队、反对 Anthropic 禁止第三方智能体、GitHub 上合并了 23 个 PR 且合并率达 85%、正在开发跨设备智能体控制系统 LobsterOS。r/ClaudeCode 上的一条评论说:“自从 OpenClaw 发布之后,大家就知道,只要你不是通过 API 运行它,迟早会被封。”(227 个赞同票)。这些不会出现在 LinkedIn 上。
|
||||
|
||||
**判断招聘信号。** `/last30days Listen Labs --hiring-signals`——把最新职位和招聘页面变成有引用依据的证据,从中判断公司是否正转向企业安全、客户成功、基础设施或产品扩张。报告只解释招聘看起来释放了什么信号,不会武断预测路线图一定会交付什么。
|
||||
|
||||
**在话题爆发前发现它。** 输入 `/last30days what's exploding in AI agents?`,Skill 会切换到发现模式:引擎扫描 Reddit 分类列表、Hacker News 的 front/best 故事、Digg AI 1000 信息流,以及认证后的 X;随后由你的智能体评审候选主题(命名、过滤垃圾、判断内容价值),并写出播客或 X 长文的切入角度;最终给出 5–10 个按增长速度排序的话题。每条结果都包含跨平台数据、势头标签,以及可直接运行的 `/last30days "<topic>"` 后续命令。
|
||||
|
||||
**突发事件发生时。** `/last30days Kanye West`——英国拒绝其签证,Wireless Festival 取消演出,赞助商纷纷离场;但《BULLY》首周登上 Billboard 第二名。Fantano 结束自己的 “Yay sabbatical” 回归评测(65.3 万次观看);SoFi Homecoming 请来 Lauryn Hill 和 Travis Scott,共演出 44 首歌。Polymarket:“Kanye 还会再发推吗?”86% 认为会。共找到 23 个 Reddit 主题、17 个 YouTube 视频和 8.6 万次赞同。
|
||||
|
||||
**比较工具。** `/last30days OpenClaw vs Hermes vs Paperclip`——“它们并非竞品,而是处于不同层次。”OpenClaw 是执行层(GitHub 35.1 万 Star,已上线),Hermes 是会自我改进的大脑(3.1 万 Star),Paperclip 是组织结构图(4.9 万 Star)。Star 数来自 GitHub API 的实时数据,不是过期博客。报告会提供架构、记忆、安全性和适用场景的横向表格。正如 @IMJustinBrooke 所说:“OpenClaw = 小火龙,Hermes = 喷火龙。”
|
||||
|
||||
**理解世界。** `/last30days Iran vs USA`——战争进入第 38 天。特朗普要求伊朗在周二的最后期限前重新开放霍尔木兹海峡;两架美国战机被击落;油价涨至每桶 126 美元。IEA 称之为“全球石油市场史上最大规模的供应中断”。Polymarket 认为 12 月 31 日前停火的概率为 74%。共找到 27 条 X 帖子、10 个 YouTube 视频和 20 个预测市场。
|
||||
|
||||
**旅行之前。** `/last30days Universal Epic Universe`——扩建工程已经开工,“Project 680” 许可已提交;基础设施证实将有烟花表演,但官方尚未公布。Mine-Cart Madness 平均排队 148 分钟;年票仍未推出,当地居民对此不满;Stardust Racers 将停运翻修至 4 月 5 日。
|
||||
|
||||
**快速学习。** `/last30days Nano Banana Pro prompting`——JSON 结构化提示词正在取代标签堆砌;@pictsbyai 的嵌套格式能避免“概念串色”;以编辑为先的工作流优于反复重新生成。随后,它会严格依据社区验证有效的方法,为你写出一条可用于生产的提示词。
|
||||
|
||||
## 最近更新
|
||||
|
||||
自 5 月发布 v3.3 公告以来,截至 v3.11.1(2026 年 7 月),项目已在 15 个版本中合并 175 个 PR,其中 122 个来自 52 位社区贡献者。下面是主要变化。
|
||||
|
||||
### 正式支持 OpenAI Codex
|
||||
|
||||
`/last30days` 现在是带引导式配置的原生 Codex 插件——不是简单移植,而是一等公民。针对不同渲染器优化的引用格式,让 Codex 输出读起来像简报,而不是一团 URL(#694)。同一套引擎也运行在 Claude Code、Cursor、Copilot、Gemini CLI、Claude Desktop、OpenClaw 以及 50 多个 Agent Skills 宿主上。Codex 插件清单由 [@rfoust](https://github.com/rfoust) 贡献(#686),Codex 认证修复由 [@tmchow](https://github.com/tmchow) 贡献(#698)。
|
||||
|
||||
### arXiv、Techmeme 与 Digg——免费,无需 API 密钥
|
||||
|
||||
arXiv 提供热点背后的论文,Techmeme 提供科技新闻的编辑视角;二者均免费、无需密钥,首次配置会安装相应 CLI 并自动启用(#709)。Digg 的 AI 1000 话题聚类同样无需 X 认证——配置过程会自动安装免费的 Digg CLI(#590)。此外还加入了可选的 Trustpilot 来源,适合消费品牌研究。
|
||||
|
||||
### 免费 Reddit 搜索也有真实评分和热门评论
|
||||
|
||||
Reddit 的公开 `.json` API 停止工作后,免费的数据通路以更强的方式回归:无密钥 RSS + shreddit 抓取(#457)、通过 arctic-shift 发现垂直 subreddit 并获取真实赞同数(#696),以及相关性下限,防止病毒式传播但偏题的帖子劫持整份简报(#488,感谢 [@rzachsmith](https://github.com/rzachsmith))。无需 API 密钥,提供真实评分和热门评论。
|
||||
|
||||
### 每份简报都收录最好的评论
|
||||
|
||||
评论现在是各来源默认启用的一层:Instagram 评论采用基于排名的多样性机制,避免五条热门观点全来自同一篇帖子(#751);YouTube 评论配合 ScrapeCreators 字幕回退,以应对 yt-dlp 失效(#637);经过社区投票的评论还会计入 Best Takes 的权重,让最有趣的金句不会在评分中消失(#592、#608)。
|
||||
|
||||
### 一个 doctor 命令解决健康检查
|
||||
|
||||
要求执行健康检查时,doctor 会逐一测试所有来源,并给出精确修复建议:缺少哪个密钥、哪个 CLI 不在 PATH、哪个 Cookie 已过期(#753)。不必再猜为什么 X 的结果这么少。
|
||||
|
||||
### 重构 X 搜索
|
||||
|
||||
X 流水线经过彻底重构:新增 FROM 和 ABOUT 两条通路,让某人的原创帖子和外界对他的讨论都能进入排名(#610);按人物感知的子查询消歧(#611);基于第一方作者身份的信息归属,并结合互动信号排序(#613);统一的 X 来源与自动后端故障转移(#622)。另外,`--diagnose` 现在会真正探测认证状态,如实报告问题(#609)。
|
||||
|
||||
### 更多信息源加入
|
||||
|
||||
通过 ScrapeCreators 接入 LinkedIn,并将文章视为高价值信号([@ravstr](https://github.com/ravstr),#702)。StockTwits 会在股票代码和加密货币主题下自动启用([@wtiwana](https://github.com/wtiwana),#658)。Perplexity 新增直接 API 模式和异步 Deep Research([@sk-holmes](https://github.com/sk-holmes),#629)。
|
||||
|
||||
### 在社区协作下进一步加固
|
||||
|
||||
这一轮安全改进几乎全部来自社区:修复 HTML 渲染器中的存储型 XSS([@iliaal](https://github.com/iliaal)、[@aaronjmars](https://github.com/aaronjmars));收紧 Cookie 临时文件权限;通过 OpenSSF Scorecard 和构建来源证明加固 CI 供应链([@shaanmajid](https://github.com/shaanmajid)、[@hammadxcm](https://github.com/hammadxcm)、[@aniruddh909](https://github.com/aniruddh909));增加 Semgrep、OSV-Scanner 扫描以及 PR 依赖审查门禁([@23241a6749](https://github.com/23241a6749));测试覆盖率门槛从 60% 起步,现已提高到 84%([@gourab5139014](https://github.com/gourab5139014));Hermes 安全扫描中的所有 CRITICAL 问题也已清零(#768)。
|
||||
|
||||
### 覆盖范围更广
|
||||
|
||||
支持希伯来语和其他非拉丁文字语言([@dudyme](https://github.com/dudyme));为中文来源加入 CJK 感知的分词([@An-idd](https://github.com/An-idd));推进一系列 Windows 兼容性改进;支持从完整 Chromium 浏览器家族提取 Cookie——Brave、Edge、Vivaldi、Opera、Arc([@andrey-esipov](https://github.com/andrey-esipov))——并接入 macOS Keychain 和 Linux `pass(1)` 凭据来源。此外还有 `--as-of` 历史回溯([@chiyi-creator](https://github.com/chiyi-creator))、通过 uv 自动配置 Python 3.12([@buntysomroy](https://github.com/buntysomroy))、用于解读公司招聘页面的 `--hiring-signals`,以及多次运行之间的观察列表差异。
|
||||
|
||||
### v3 的核心能力仍然完整保留
|
||||
|
||||
v3 打下的基础都还在:真正调用 API 前先运行预研究模块,解析正确的账号、subreddit 和话题标签(由 [@j-sperling](https://github.com/j-sperling) 开发);Best Takes 评分在相关性之外也衡量幽默感和传播力;跨来源故事聚类;单次完成对比研究(例如 “CLI vs MCP” 只需 3 分钟,而不是 12 分钟);自动发现竞品的 `--competitors` 对比;GitHub 人物模式(`--github-user=steipete`);任何研究结束后可开启的 ELI5 模式(输入 “eli5 on”);以及可分享、自包含的 HTML 简报(`--emit=html`)。配置项详见 [CONFIGURATION.md](CONFIGURATION.md)。
|
||||
|
||||
## 安装
|
||||
|
||||
| 使用环境 | 安装方式 | 更新方式 |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code**(推荐) | `/plugin marketplace add mvanhorn/last30days-skill` | 通过 marketplace 自动更新,或运行 `claude plugin update last30days@last30days-skill` |
|
||||
| **Grok**(xAI Build CLI) | 先运行 `grok plugin marketplace add mvanhorn/last30days-skill`,再运行 `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex、Cursor、Copilot、Gemini CLI,或其他 50 多个支持 [Agent Skills](https://agentskills.io) 的宿主** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai**(网页) | [下载 `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill),然后在 claude.ai 中依次进入 Customize > Skills > + > Create skill > Upload a skill 上传 | 重新下载并上传 |
|
||||
| **Claude Desktop** | 从[最新版本](https://github.com/mvanhorn/last30days-skill/releases/latest)下载适用于你的平台的 `.mcpb`,拖入 Settings > Extensions | 重新下载新包并拖入 |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code(推荐)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
推荐这种方式,是因为 Claude Code marketplace 会替你处理更新:插件缓存按版本管理,每次发布新版本都会自动刷新。要强制检查更新,请运行 `claude plugin update last30days@last30days-skill`。
|
||||
|
||||
如果你更愿意在 Claude Code 中使用 Agent Skills 的安装方式,同样支持:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
原生插件和 `npx skills` 安装可以共存。但 Claude Code 不会对不同安装方式进行去重:若两者同时启用,`/last30days` 会出现两个条目。建议每台机器只选一种安装方式。
|
||||
|
||||
### Grok(xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces)(`grok`)可以将 last30days 安装为原生插件。直接安装会跟踪仓库更新:
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
也可以先把本仓库添加为 marketplace 来源,再按插件名安装:
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
加入 `--trust` 可跳过安装确认;使用 `grok plugin update last30days` 更新。为兼容旧机制,Grok 也会读取 Claude Code 的清单文件;原生 `.grok-plugin/` 文件是首选通路,也是 [xAI marketplace](https://github.com/xai-org/plugin-marketplace) 官方目录条目指向的对象。`npx skills add` 仍是有效的跨宿主备用方案。
|
||||
|
||||
### Codex、Cursor、Copilot、Gemini CLI 与其他 Agent Skills 宿主
|
||||
|
||||
通过开放的 [Agent Skills](https://agentskills.io) CLI 安装。它支持 50 多种运行环境,包括 `codex`、`cursor`、`github-copilot`、`gemini-cli`、`claude-code`、`windsurf`、`cline`、`continue`、`roo`、`aider-desk`、`opencode`、`goose` 等(完整列表见 [vercel-labs/skills 仓库](https://github.com/vercel-labs/skills))。
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
`-g`(全局)参数会把 Skill 安装到用户目录,因此所有项目均可使用。不加 `-g` 时,`npx skills` 会安装到当前项目的 `./.skills/` 中,并随仓库提交。对于一个用于研究整个世界的工具,全局安装通常更合适。
|
||||
|
||||
Codex 桌面版和其他以文件夹为工作区的宿主,不仅能在 Git 仓库中运行,也能在普通文件夹中工作。第一次研究前,请让宿主智能体从已加载的 Skill 目录运行随附的 `scripts/last30days.py --preflight`;若在源码仓库中,则运行等价命令 `python3 skills/last30days/scripts/last30days.py --preflight`。该命令会展示配置来源、浏览器 Cookie 方案、计划写入的文件、可选命令和被忽略的项目配置,但不会读取 Cookie、写入文件或执行研究。
|
||||
|
||||
默认情况下,`npx skills` 会安装到它自动检测到的宿主。若要指定一个或多个宿主:
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
日后可通过以下命令更新:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
也可以一次更新所有通过 `npx skills` 全局安装的 Skill:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
使用 `npx skills list -g` 查看列表,使用 `npx skills remove last30days -g` 卸载。
|
||||
|
||||
### claude.ai(网页)
|
||||
|
||||
1. 从最新版本[下载 `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill)
|
||||
2. 打开 [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. 在 Skills 面板点击 `+`,再选择 `Create skill` > `Upload a skill`,浏览或拖入文件
|
||||
|
||||
请先在 Capabilities 中启用 “Code execution and file creation”——否则 Skill 无法运行。
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop 通过 `.mcpb` 包(一种一键式 Model Context Protocol 软件包)将 `/last30days` 安装为 MCP 服务器。
|
||||
|
||||
1. 打开[最新版本](https://github.com/mvanhorn/last30days-skill/releases/latest),下载适用于你的平台的 `.mcpb`:
|
||||
- macOS Apple Silicon:`last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel:`last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64:`last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. 打开 Claude Desktop,进入 Settings > Extensions,将文件拖入。
|
||||
3. 出现提示时,粘贴你想启用的数据源所需的 API 密钥。所有字段均可留空——如果全部跳过,引擎会降级为纯 Web 模式。密钥存储在操作系统的钥匙串中。
|
||||
4. 重启 Claude Desktop。让 Claude “research Peter Steinberger” 或研究任意主题,它就会调用 `research` 工具。
|
||||
|
||||
**宿主要求:** PATH 中需要 Python 3.12+。软件包自带引擎源码,但使用本地 Python 解释器。Windows 用户可从 [python.org](https://www.python.org/downloads/) 安装;macOS 和大多数 Linux 发行版通常已提供兼容版本。
|
||||
|
||||
**密钥不会与 Code Skill 同步。** Claude Desktop 与 Claude Code 采用彼此独立的凭据存储,这是有意的设计。即使你已为 Code Skill 配置 `~/.config/last30days/.env`,仍需在这里重新输入一次相同的密钥。
|
||||
|
||||
Windows 支持需要等各平台的清单入口点确定后再实现,请关注后续 Issue。
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
如果你需要在 `/last30days` 研究之外执行 X/Twitter 操作,例如发布推文或回复、导出关注者、处理媒体、监控账号或抽奖,可使用 [TweetClaw](https://github.com/Xquik-dev/tweetclaw) 作为配套 OpenClaw 插件。TweetClaw 由 Xquik-dev 维护,这里仅将其列为可选配套方案;它不是 last30days 的依赖,也不代表本项目为其背书。
|
||||
|
||||
### 手动安装(开发者)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
这个符号链接会让安装内容随工作区代码实时同步,无需重复复制。若用于 `claude.ai`,可从源码构建 `.skill` 文件:运行 `bash skills/last30days/scripts/build-skill.sh`,产物位于 `dist/last30days.skill`。
|
||||
|
||||
Reddit(含评论)、Hacker News、Polymarket 和 GitHub 无需任何配置即可使用。首次运行 `/last30days` 后,配置向导会在 30 秒内解锁更多来源,包括免费的 arXiv 与 Techmeme CLI。
|
||||
|
||||
## 使用你自己的密钥
|
||||
|
||||
这些平台之间互不相通:X 不知道 Reddit 在讨论什么,YouTube 也看不到 TikTok。但只要接入你自己的 API 密钥和浏览器令牌,就能一次访问所有平台。
|
||||
|
||||
| 来源 | 你需要准备什么 | 成本 |
|
||||
|------|----------------|------|
|
||||
| Reddit(含评论)+ HN + Polymarket + GitHub + StockTwits | 无 | 免费 |
|
||||
| arXiv + Techmeme | 免费 CLI,由首次配置自动安装 | 免费 |
|
||||
| X / Twitter | 在任意浏览器中登录 x.com,或设置 `XQUIK_API_KEY` / `XAI_API_KEY` | 浏览器 Cookie 免费;密钥费用取决于服务商 |
|
||||
| YouTube | `brew install yt-dlp` | 免费 |
|
||||
| Bluesky | 来自 bsky.app 的应用密码 | 免费 |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + YouTube 评论 | ScrapeCreators 密钥 | 前 10,000 次调用免费,之后按量付费 |
|
||||
| 小红书(RED) | 运行已登录的 x-mcp 浏览器插件或 `xiaohongshu-mcp` 服务,并在单次运行中通过 `--search xhs` 启用,或在 `.env` 中设置 `INCLUDE_SOURCES=xiaohongshu`;last30days 会依次自动探测 `http://localhost:18060` 和 `http://host.docker.internal:18060`,也可通过 `XIAOHONGSHU_API_BASE` 指定自定义地址 | last30days 不需要 API 密钥;依赖本地浏览器会话服务 |
|
||||
| DripStack(付费金融通讯) | 每次运行通过 `--search dripstack` 启用,或在 `.env` 中设置 `INCLUDE_SOURCES=dripstack` | 无需密钥;公共搜索 API 免费 |
|
||||
| Perplexity Sonar / Search API / Deep Research | Perplexity 密钥,或作为 Sonar 回退方案的 OpenRouter 密钥 | 按量付费 |
|
||||
| Web 搜索 | Brave Search 密钥 | 每月 2,000 次免费查询 |
|
||||
|
||||
### macOS Keychain(可选)
|
||||
|
||||
在 macOS 上,你可以把密钥存入系统 Keychain,而不是 `.env` 文件。Skill 会自动将其作为最低优先级的密钥来源;发生冲突时,`.env` 文件和进程环境变量仍然优先。
|
||||
|
||||
```bash
|
||||
# 交互式配置——逐个询问已知密钥,留空即可跳过
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# 也可以手动存入单个密钥
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# 查看 / 清理
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
密钥项以 `last30days-<KEY>` 作为服务名称,归当前用户所有。在非 Darwin 平台上,加载器不会执行任何操作,因此 Linux/Windows 用户的行为不受影响。
|
||||
|
||||
如果已有密钥使用其他 Keychain 服务名称,可按 [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items) 中的说明设置不含秘密的 `LAST30DAYS_KEYCHAIN_ALIASES` 映射,无需复制密钥。
|
||||
|
||||
各来源的完整密钥矩阵、推理服务商优先级和 Web 搜索后端优先级,请参阅 [CONFIGURATION.md](CONFIGURATION.md)。
|
||||
|
||||
## 配置
|
||||
|
||||
第一天使用时,你大概最想知道以下两件事:
|
||||
|
||||
**研究文件保存在哪里。** `LAST30DAYS_MEMORY_DIR` 默认指向 `~/Documents/Last30Days/`(Windows:`C:\Users\<you>\Documents\Last30Days\`)。可以在 shell 中把该环境变量设为任意路径,也可以为单次运行传入 `--save-dir <path>`。若需要把渲染结果精确写入某个路径,请使用 `--output <file>`;文件格式由 `--emit` 决定。使用 `--save-suffix=<name>` 可分别保存同一主题的多个版本(例如按客户区分)。每次使用 `--save-dir` 都会生成 `<slug>-raw[-suffix].md`。研究前运行 `python3 skills/last30days/scripts/last30days.py --preflight`,可预览计划写入的内容。
|
||||
|
||||
**面向智能体和工作流的结构化输出。** 让 `/last30days` 输出机器可读的 JSON,即可获得稳定且带版本号的 agent profile。若在脚本或开发中直接调用引擎,可运行 `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json`;只有确实需要未版本化的内部 `Report` 转储时,才添加 `--json-profile=raw`。详见 [JSON 导出字段参考与版本策略](docs/reference/json-export.md)。
|
||||
|
||||
**无指定主题的趋势发现。** 输入 `/last30days what's trending in AI agents?`,会得到按排名整理的发现简报,而不是研究一个你已经知道的主题。在智能体宿主上,它会执行由宿主模型评审的三段式流程:模型命名主题、过滤垃圾、判断内容价值并撰写切入角度。在脚本或定时任务中直接调用引擎时,可运行 `python3 skills/last30days/scripts/last30days.py --discover "AI agents"`(单次执行:主题名称由确定性逻辑生成,不含内容角度);加入 `--emit=json` 可获得带版本号的发现数据契约。发现模式不能与位置参数主题或 `--drill` 同时使用。
|
||||
|
||||
**跨运行趋势监控。** 默认模式每次运行都会生成新的 Markdown 快照。若要长期积累结果,可添加 `--store` 写入 SQLite 数据库;随后使用 [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) 定时运行(发现新内容时可发送到 Slack 或 Webhook),使用 [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) 生成日报或周报。完整的周期配置见 [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings)。
|
||||
|
||||
**可订阅的研究资料库。** 让 `/last30days` 构建你的资料库信息流;在脚本和开发中也可以直接运行 `python3 skills/last30days/scripts/last30days.py library feed`。该命令会把已保存的简报整理成 `index.html`、本地 Atom `feed.xml` 和便于阅读的简报页面。仅在确实想托管 HTML 索引和简报页面时添加 `--publish`;发布必须显式开启,且默认公开。若要让 Atom 信息流可订阅,请将生成目录托管到 GitHub Pages 等静态站点服务。
|
||||
|
||||
**搜索你做过的所有研究。** 输入 `/last30days search my library for MCP servers` 或 `/last30days have I researched MCP servers before?`。直接调用引擎时,运行 `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`。搜索完全离线且结果确定:它增量索引资料库信息流使用的同一批简报,合并每次运行存储的匹配记录,并按主题和日期分组。新的研究若与历史内容重叠,还会显示精简的 **From your library** 章节;设置 `LAST30DAYS_LIBRARY_CONTEXT=off` 可关闭这种被动上下文。
|
||||
|
||||
各客户端包装脚本、自定义分类同类 subreddit,以及用于试验进行中定制功能的 beta 通道,也都记录在 [CONFIGURATION.md](CONFIGURATION.md) 中。
|
||||
|
||||
## 展示:社区研究信息流
|
||||
|
||||
你是否用 last30days 发布了定期 AI 动态、市场观察,或某个小众到可爱的长期专题?欢迎在[社区展示帖](https://github.com/mvanhorn/last30days-skill/issues/532)分享公开资料库 URL;若已将 `feed.xml` 托管到静态站点,也可分享 Atom URL。社区成员提交后,我们会在这里陆续添加链接;在此之前,该讨论帖就是统一的收集入口。
|
||||
|
||||
## 工作原理
|
||||
|
||||
1. **你输入一个主题。** 人物、公司、产品、技术、“X vs Y”——任何内容都可以。
|
||||
2. **智能体识别关键对象。** 找出 X 账号(包括创始人)、GitHub 仓库、subreddit、TikTok 话题标签和 YouTube 频道。搜索 “Kanye West” 时,它知道该查 r/hiphopheads、@kanyewest,以及 YouTube 上的 “bully review”;搜索 “OpenClaw” 时,它会定位 GitHub 上的 openclaw/openclaw 并获取实时 Star 数。
|
||||
3. **并行搜索所有来源。** 扩展多个查询,再按互动度、相关性和新鲜度评分。
|
||||
4. **提供其他工具没有的深度。** 获取反应视频的完整 YouTube 字幕、带赞同数的 Reddit 热门评论、TikTok 文案和 Polymarket 概率,而不只是标题和链接。
|
||||
5. **合并同一事件。** Wireless Festival 在 Reddit 官宣、在 X 上引发讨论、TikTok 出现票价信息——这些会合并为一个故事聚类,而不是三条重复结果。
|
||||
6. **综合成一份简报。** 用具体数据作依据,为来源添加引用,并按真实互动排序。不是“这是我找到的内容”,而是“这是最重要的内容”。
|
||||
7. **随后成为你的领域专家。** 运行一次后,当前 Claude 会话就掌握社区知道的一切。你可以继续追问,让它写提示词、起草邮件、规划旅行或设计系统架构——所有回答都基于此刻真实存在的信息。
|
||||
|
||||
## 用户怎么评价
|
||||
|
||||
> “我发现了一个 Claude Code Skill,可以研究任意主题过去 30 天在 Reddit、X、YouTube 和 HN 上的内容,然后替你写提示词。以前每写一篇内容,我都得手动在 Reddit 和 X 上做研究:一个标签页接一个标签页,一条讨论接一条讨论。光这一步就要 90 分钟。它彻底省掉了这些工作。” ——@itsjasonai
|
||||
|
||||
> “仅仅这一个 Skill,就取代了我的整套研究工作流。给它一个主题,它会抓取 Reddit、X 和 Web 上人们真正在谈论的内容。不是陈旧的博客,而是过去 30 天里真实发生的讨论。” ——@itswilsoncharles
|
||||
|
||||
> “今天 GitHub 的 10 个趋势仓库中,有 5 个是 Claude 工具。第一名:mvanhorn/last30days-skill。” ——@yieldhunter95
|
||||
|
||||
## 开源
|
||||
|
||||
采用 MIT 许可证。无跟踪、无分析,你的研究数据始终留在本机。拥有 2,700 多项测试。
|
||||
|
||||
项目基于 Python 3.12+、yt-dlp、Node.js(内置用于 X 搜索的 Bird 客户端)和 ScrapeCreators API 构建。v3 引擎架构由 [@j-sperling](https://github.com/j-sperling) 设计。
|
||||
|
||||
提交 PR 请参阅 [CONTRIBUTING.md](CONTRIBUTING.md),完整社区贡献者名单见 [CONTRIBUTORS.md](CONTRIBUTORS.md),版本历史见 [CHANGELOG.md](CHANGELOG.md)。
|
||||
|
||||
## Star 历史
|
||||
|
||||
<a href="https://star-history.com/#mvanhorn/last30days-skill&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
<img alt="Star 历史图" src="https://api.star-history.com/svg?repos=mvanhorn/last30days-skill&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
**@slashlast30days** · [github.com/mvanhorn/last30days-skill](https://github.com/mvanhorn/last30days-skill)
|
||||
@@ -0,0 +1,37 @@
|
||||
# Changelog fragments
|
||||
|
||||
Feature and fix PRs add a fragment here. **Do not edit `CHANGELOG.md` or bump version manifests** — the release workflow does that.
|
||||
|
||||
You do **not** need the towncrier CLI to contribute. Fragments are ordinary Markdown files; towncrier runs only when a release is prepared. See [CONTRIBUTING.md](../CONTRIBUTING.md).
|
||||
|
||||
## Create a fragment
|
||||
|
||||
```bash
|
||||
# Prefer the PR or issue number when you know it:
|
||||
# changelog.d/<number>.<type>.md
|
||||
# Orphan (no linked issue/PR yet):
|
||||
# changelog.d/+.<type>.md or changelog.d/+short-slug.<type>.md
|
||||
```
|
||||
|
||||
### Types (Keep a Changelog)
|
||||
|
||||
| Suffix | Section |
|
||||
|--------|---------|
|
||||
| `security` | Security |
|
||||
| `removed` | Removed |
|
||||
| `deprecated` | Deprecated |
|
||||
| `added` | Added |
|
||||
| `changed` | Changed |
|
||||
| `fixed` | Fixed |
|
||||
|
||||
### Content
|
||||
|
||||
One or a few sentences of what shipped — behavior, docs, or install impact someone would care about in release notes. Link issues in the fragment body if useful; towncrier also links the number from the filename.
|
||||
|
||||
```markdown
|
||||
General reports no longer promote unanchored fallback entity misses into synthesis.
|
||||
```
|
||||
|
||||
### Skip
|
||||
|
||||
Pure chores (typos in comments, CI pin bumps with nothing for release notes) can omit a fragment and check **Skip changelog** in the PR template, or add the `skip-changelog` label.
|
||||
@@ -0,0 +1,40 @@
|
||||
# feat(x): demote Grok CLI to opt-in backup
|
||||
|
||||
Stop using the Grok CLI as the default X backend. A leftover `~/.grok/auth.json` must never steal the X lane. Grok stays as a pin-only backup: off unless LAST30DAYS_X_BACKEND=grok or --x-backend grok.
|
||||
|
||||
### Requirements
|
||||
- R1. Unpinned auto chain is bird → xai → xurl → xquik. Bird is first. Grok is not a member. Presence of ~/.grok/auth.json (ok, expired, or error) must not change which backend an unpinned run uses.
|
||||
- R2. Grok remains a valid explicit selection: LAST30DAYS_X_BACKEND=grok and --x-backend grok. A pin forces grok with no failover. If grok is unusable, X is unconfigured and doctor/footer say so with the existing login hint.
|
||||
- R3. Doctor "will use: grok" only when grok is pinned and the probe is OK or DEGRADED. Unpinned, grok may appear as unused opt-in ("available, unused — pin LAST30DAYS_X_BACKEND=grok"), never as the predicted winner.
|
||||
- R4. get_x_source_status and get_x_source_with_method must prefer bird over xai/xurl/xquik when cookies are present. Grok wins only when the pin is grok.
|
||||
- R5. Host docs stop presenting grok as the default keyless X path. Document it as opt-in backup. Default story is bird first, then xai / xurl / xquik.
|
||||
- R6. Setup / first-run / prescriptions do not nag grok login as the fix for missing X. Cookie consent and paid keys remain the default prescriptions. Grok login is mentioned only as an optional pin.
|
||||
- R7. Do not delete scripts/lib/grok_x.py, retrieve-judge-retry, or expires_at honesty. Pinned grok still uses them.
|
||||
- R8. A machine with only a grok login (no cookies, no XAI/XQUIK, no xurl) has X unconfigured until the user pins grok. Footer: X skipped-unconfigured, not auth-failed-from-grok.
|
||||
- R9. Tests cover the cases above; docs/changelog updated.
|
||||
|
||||
### Implementation units
|
||||
U1 env.py: _X_BACKEND_ORDER = ("bird", "xai", "xurl", "xquik"); X_BACKEND_OPT_IN = ("grok",); X_BACKEND_KNOWN = ORDER + OPT_IN; pin uses KNOWN; unpinned walks ORDER only; get_x_source_status bird first, grok only if pinned; get_x_source_with_method bird before xai.
|
||||
|
||||
U2 backends.py / doctor.py / prescriptions.py: descriptor is auto ORDER then grok opt-in; unpinned collect-then-pick ignores opt-in; do not change _probe_grok honesty.
|
||||
|
||||
U3 SKILL.md, CONFIGURATION.md, README.md, README.pt-BR.md if needed, changelog.d: remove "sits ahead of the cookie path"; document bird → xai → xurl → xquik; pin grok to enable it.
|
||||
|
||||
U4 tests: unpinned grok-only empty; unpinned grok+bird → bird; unpinned bird+xai → bird; pin grok+store → ["grok"]; pin grok no store empty; doctor unpinned never predicts grok; descriptor parity treats grok as trailing opt-in.
|
||||
|
||||
### Tests T1–T7
|
||||
T1 unpinned grok AUTH_OK, no other creds → X unconfigured
|
||||
T2 unpinned grok AUTH_EXPIRED, no other creds → X unconfigured (not will-use grok)
|
||||
T3 unpinned grok AUTH_OK + cookies → bird
|
||||
T4 unpinned XAI_API_KEY + grok store, no cookies → xai
|
||||
T4b unpinned XAI_API_KEY + cookies → bird
|
||||
T5 pin grok AUTH_OK → grok no failover
|
||||
T6 pin grok no store → error / grok login prescription
|
||||
T7 docs match R5
|
||||
|
||||
### Keep-the-door-open (KTD)
|
||||
1. grok_x.py stays untouched
|
||||
2. x_judge.py stays untouched
|
||||
3. expires_at honesty stays untouched
|
||||
4. auth.x.ai is never called
|
||||
5. bird cookie extraction is unchanged
|
||||
@@ -0,0 +1,71 @@
|
||||
# Retrieve-Judge-Retry for X Search
|
||||
|
||||
**Date:** 2026-08-14
|
||||
**Status:** Completed
|
||||
|
||||
## Problem Statement
|
||||
|
||||
X search results become off-topic when multi-word search queries are phrase-quoted. The Rome failure (2026-08-14) demonstrated this:
|
||||
|
||||
1. Planner generated `search_query: "Rome Italy"` (phrase-quoted)
|
||||
2. Phrase-quoting returned thin hits with engagement bait (pretty-cities, geopolitics accounts)
|
||||
3. `entity_extract` ranked off-topic handles (PrettyCitiesX, visegrad24) by frequency
|
||||
4. `pipeline.py` promoted those handles to the FROM lane
|
||||
5. FROM lane filled the 40-slot X budget with off-topic timelines
|
||||
|
||||
## Solution
|
||||
|
||||
Implement retrieve-judge-retry for X search:
|
||||
|
||||
1. **Query Compilation (R2):** X now uses `raw_topic` like Reddit/YouTube, not the planner's `search_query`
|
||||
2. **Fanout Queries (R3):** Multi-word topics use unquoted AND as first variant; phrase-quote only for proper names
|
||||
3. **Corpus Judging (R7):** New `x_judge.py` module evaluates corpus on-topic ratio after retrieval
|
||||
4. **Retry (R1):** If off-topic flood detected (ratio < 0.4), retry ONCE with simplified keyword query inside the X stream (not `_retry_thin_sources`)
|
||||
5. **Split FROM Promotion (R4):**
|
||||
- Explicit handles (--x-handle): always FROM, no AND topic
|
||||
- Extracted handles: FROM only if ≥2 on-topic hits AND ≥50% ratio, and they DO AND the topic
|
||||
6. **First-Party Exemption (R5):** Floor immunity stays conservative (explicit handles only)
|
||||
7. **Status Reporting (R6):** Off-topic floods emit artifact warning, not `record_failure(PARTIAL)`
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### New Module: `x_judge.py`
|
||||
|
||||
- `judge_x_corpus(items, topic, ranking_query)`: Returns on_topic_ratio, is_off_topic_flood, on_topic_items, handle_stats
|
||||
- `promotable_handles(items, topic, extracted_handles, explicit_handles)`: Returns (explicit_promotable, extracted_promotable)
|
||||
- `should_retry_x_search(items, topic, depth)`: Returns True if retry warranted
|
||||
- `prune_off_topic_items(items, topic)`: Returns only on-topic items
|
||||
|
||||
### Key Changes
|
||||
|
||||
- `grok_x._fanout_queries()`: No phrase-quote for place/disambiguation strings
|
||||
- `grok_x._is_proper_name()`: Detects title-cased proper names for phrase-quoting
|
||||
- `grok_x.search_handles()`: Added `and_topic` parameter (default False)
|
||||
- `pipeline._fetch_x_backend()`: Accepts query directly, not subquery
|
||||
- `pipeline._retrieve_stream_impl()`: X source uses raw_topic, judges corpus, retries if needed
|
||||
- `pipeline._run_supplemental_searches()`: Uses `promotable_handles` for split FROM logic
|
||||
|
||||
### Tests
|
||||
|
||||
- `tests/test_x_judge.py`: New test file for x_judge module
|
||||
- `tests/test_grok_x.py`: Added fanout and and_topic tests
|
||||
- `tests/test_pipeline_v3.py`: Updated fixtures to have promotable content
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] `_fanout_queries("Rome Italy")` has no `"Rome Italy"` variant
|
||||
- [ ] `search_name("Peter Steinberger")` still phrase-quotes
|
||||
- [ ] `search_handles(["steipete"], "topic")` does not AND topic by default
|
||||
- [ ] `search_handles(["visegrad24"], "Rome", and_topic=True)` does AND Rome
|
||||
- [ ] Explicit --x-handle always gets FROM lane
|
||||
- [ ] Off-topic handles (visegrad24) not promoted to FROM lane
|
||||
- [ ] On-topic handles (mamboitaliano__) promoted to FROM lane
|
||||
- [ ] X source status is artifact warning, not PARTIAL failure
|
||||
- [ ] All tests pass
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- No live Grok calls in tests
|
||||
- No auth/doctor touch
|
||||
- No collision lexicon (AS Roma / Odunze still appear; judge + ranking_query drop them)
|
||||
- bird_x quote-preserving `build_topic_query` (follow-up if needed)
|
||||
@@ -0,0 +1,74 @@
|
||||
# Fix Amazon Review Budget by Starting at Search Time
|
||||
|
||||
**Date:** 2026-08-14
|
||||
**Status:** Implemented
|
||||
|
||||
## Problem (measured 2026-08-14 Bentgo run)
|
||||
|
||||
- Full multi-source run. Amazon search listings returned (12 products, stars, rating counts).
|
||||
- Review lane logged: "pulling up to 50 reviews for 3 products (budget 11s)"
|
||||
- "lane deadline 11s hit; dropped 3 straggling pull(s)"
|
||||
- Bright Data timed out after 11s x3. Zero review bodies. Credits spent.
|
||||
- Isolated Amazon-only re-run got the full 180s, finished in 124s, reviews landed.
|
||||
- Cause: `_remaining_lane_budget` = min(LANE_DEADLINE=180, max(0, FOREGROUND_CONTRACT=300 - elapsed - RENDER_MARGIN=20)). Enrichment runs after all other sources. Elapsed ~269s → 11s.
|
||||
|
||||
## Why the floor-up fix was wrong
|
||||
|
||||
`max(120, leftover)` at elapsed=269 means the run goes to ~389–449s. The host Bash contract is 300s (`SKILL.md` 300000ms). That kills the **whole** report. Do not do that.
|
||||
|
||||
The measured failure is **when** the review lane starts, not pull quality. Isolated Amazon-only already finishes in 124s of 180s. On a full run, search has already landed products, then reviews wait through every other source plus Phase 2/2b, then get 11s.
|
||||
|
||||
## What to build
|
||||
|
||||
1. Start `enrich_with_reviews` when Amazon search returns, inside `_retrieve_stream` (amazon branch ~4284), overlapping other source futures. Pass real `elapsed = time.monotonic() - run_started` (thread `run_started` into retrieve). After a 30–90s search, leftover is 190–250s → clamp to 180. Isolated Amazon-only unchanged.
|
||||
|
||||
2. Keep finalize (`_finalize_items_by_source` ~2956) as attach-if-missing only. `enrich_source_items` already no-ops if `top_comments` is set. Do not make finalize the only start. Do not enrich inline on the collect loop (that serializes other sources for 124s).
|
||||
|
||||
3. Leftover below a useful floor → budget **0**, skip the lane. Do not fire doomed 11s pulls (Bright Data `cli_timeout = max(5, timeout-10)` so budget 11s → CLI timeout 1s, still spends 3 credits). Suggested `MIN_USEFUL_REVIEW_BUDGET = 90`. Crumbs are a skip, not a short timeout.
|
||||
|
||||
4. If the lane is skipped or all pulls drop: Amazon `source_status` **PARTIAL** with detail like `review lane timed out` / `review lane skipped (budget 0s)`. Listings stay. Do not flip the whole source to `timeout` (search succeeded). Footer already shows ⚠ when state != ok — do not change render.py.
|
||||
|
||||
5. `depth=quick` still 0 pulls. `mock=True` still skips. No env knob. No raising LANE_DEADLINE or FOREGROUND_CONTRACT.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Do NOT change footer/render.py.
|
||||
- Do NOT change X search or Grok auth.
|
||||
- Do NOT add an env knob.
|
||||
- Do NOT reorder the whole source schedule (deferred).
|
||||
|
||||
## Implementation
|
||||
|
||||
### amazon.py
|
||||
|
||||
- Added `MIN_USEFUL_REVIEW_BUDGET = 90`
|
||||
- Changed `_remaining_lane_budget` to return 0 when below floor (not floor up)
|
||||
- Changed `enrich_with_reviews` to return `(products, status_detail)` tuple where status_detail is:
|
||||
- `None` for normal success
|
||||
- `"review lane skipped (budget 0s)"` when budget is below floor
|
||||
- `"review lane timed out"` when all pulls dropped
|
||||
|
||||
### pipeline.py
|
||||
|
||||
- Added `run_started` parameter to `_retrieve_stream`, `_retrieve_stream_impl`, and `_retry_thin_sources`
|
||||
- Updated all call sites to pass `run_started`
|
||||
- In the Amazon branch of `_retrieve_stream_impl`:
|
||||
- Run search as before
|
||||
- Calculate `elapsed = time.monotonic() - run_started`
|
||||
- Call `enrich_with_reviews` immediately after search returns
|
||||
- If enrichment reports a degraded status, add `_source_outcome` with `state=PARTIAL` to the artifact
|
||||
- Updated finalize comments to note it's now attach-if-missing only
|
||||
|
||||
### Tests (test_amazon.py)
|
||||
|
||||
Retargeted existing tests:
|
||||
- `test_lane_budget_shrinks_as_the_run_clock_advances` — now tests floor behavior
|
||||
- `test_dropped_straggler_keeps_its_product_with_search_stats` — uses patched short LANE_DEADLINE
|
||||
- `test_exhausted_wall_clock_skips_the_lane_entirely` — unchanged
|
||||
|
||||
New tests:
|
||||
- `test_lane_budget_floor_prevents_doomed_pulls` — verifies elapsed=269 returns 0
|
||||
- `test_lane_budget_constants_are_sane` — guards against constant drift
|
||||
- `test_crumb_budget_skips_not_fires_doomed_pulls` — regression test for the Bentgo bug
|
||||
- `test_early_elapsed_gets_full_budget` — verifies elapsed=40 gets 180s timeout
|
||||
- `test_all_pulls_dropped_reports_timed_out_status` — verifies PARTIAL status on all-dropped
|
||||
@@ -0,0 +1,63 @@
|
||||
# Plan: Fix Grok Auth Honesty
|
||||
|
||||
**Date:** 2026-08-14
|
||||
**Status:** Implemented
|
||||
**PR:** fix(grok): treat expired sessions as degraded, not ok
|
||||
|
||||
## Problem (measured 2026-08-14 Peter Steinberger run on the user's Mac)
|
||||
|
||||
- grok binary on PATH. Doctor cached grok status ok / will use grok because `~/.grok/auth.json` existed with token markers.
|
||||
- `stored_auth_status()` substring-scans for `refresh_token`/`access_token`/`auth_mode`. It never parses `expires_at`.
|
||||
- The file had `expires_at 2026-08-14T01:26:53Z`, hours dead.
|
||||
- A prior run at 07:47:26 UTC had `run_outcome` ok (2 items). Session was real.
|
||||
- At 08:43 grok loaded auth, `is_expired` true, OIDC refresh → `invalid_grant` "Refresh token has been revoked". grok deleted auth.json.
|
||||
- Engine exit 1 "Not signed in", fell back to bird (30 items via Safari cookies), lane flagged PARTIAL.
|
||||
- Host told the user "Grok CLI is not signed in" as if it never was.
|
||||
|
||||
## Three states to distinguish
|
||||
|
||||
1. **No grok CLI** — silent fallback. Fine. Do not waste the user's time. Do not nag install on every research run.
|
||||
2. **CLI installed, never logged in** — silent fallback. Fine.
|
||||
3. **CLI installed, WAS logged in, session dead** — currently reports ok then partial. **This is the bug.**
|
||||
|
||||
## What was built
|
||||
|
||||
1. **`stored_auth_status` parses `expires_at` locally** (no network, no subprocess). Added `AUTH_EXPIRED` distinct from `AUTH_OK` / `AUTH_MISSING` / `AUTH_ERROR`. Never echoes token values. Finds `expires_at` anywhere in the vendor-keyed JSON object via recursive search.
|
||||
|
||||
2. **Doctor / `_probe_grok` does NOT map `AUTH_EXPIRED` to `health.OK`** or "will use: grok". Reports `DEGRADED`/warn + expiry timestamp + "refresh happens at run; if refresh was revoked, `grok login --device-auth`".
|
||||
|
||||
3. **Research-time `is_available` STILL attempts grok when a `refresh_token` marker is present** even if `access` `expires_at` is past. Expiry of the access token is not proof refresh is dead. Does not skip a refresh that might work.
|
||||
|
||||
4. **Auth revocation detection**: If grok exits "Not signed in" / RefreshTokenRejected / auth.json vanished mid-run: does not retry grok in that run. Falls back once. Typed outcome `auth-failed` (via `is_auth_revoked_error()` and `classify_run_failure()`), not a generic PARTIAL that reads as "the product half-worked."
|
||||
|
||||
5. **Host-facing copy for case 3**: SKILL.md updated with guidance: "X used <fallback> after the Grok session expired" + login hint. Not "Grok CLI is not signed in" when `run_outcome` shows it worked earlier.
|
||||
|
||||
6. **Doctor --probe still does not call xAI or grok.** Whole-doctor-path test patches `subprocess.run` to raise and still passes. `active_backend` stays a prediction; when `run_outcome.at` is stale or not ok, doctor says "will use grok, unverified since <time>".
|
||||
|
||||
7. **Tests**: Fixture stores (missing file, future `expires_at`, past `expires_at`, unparseable JSON). No network.
|
||||
|
||||
8. **SKILL.md**: Host reads `sources.x.run_outcome` and grok expiry warn; does not treat `active_backend` as verified; does not spend a turn installing grok unless the user asked for first-party X.
|
||||
|
||||
9. **Changelog fragment**: `changelog.d/+grok-auth-expired.fixed.md`. Tests pass with `uv run pytest`.
|
||||
|
||||
## Scope boundaries (NOT in this PR)
|
||||
|
||||
- X query construction, fanout, `search_name`, retrieve-judge-retry, and handle promotion are unchanged. That is a separate PR.
|
||||
|
||||
## Success criteria (all met)
|
||||
|
||||
- Past `expires_at` fixture → not grok ok.
|
||||
- Future `expires_at` → still ok (not live-verified).
|
||||
- No grok binary → no extra user-facing failure.
|
||||
- Simulated "Not signed in" after prior ok `run_outcome` → typed `auth-failed` / fallback copy, not "never signed in."
|
||||
- No-subprocess doctor test still passes.
|
||||
|
||||
## Files changed
|
||||
|
||||
- `skills/last30days/scripts/lib/grok_x.py` — `AUTH_EXPIRED`, `stored_auth_status()` returns 3-tuple, `is_auth_revoked_error()`, `classify_run_failure()`, `_invoke()` sets `auth_revoked`, `_run_query()` returns 3-tuple, `search_x()` propagates `auth_revoked`
|
||||
- `skills/last30days/scripts/lib/backends.py` — `_probe_grok()` handles `AUTH_EXPIRED` as `DEGRADED`
|
||||
- `skills/last30days/scripts/lib/pipeline.py` — `_fetch_x_backend()` propagates `auth_revoked`, `_classify_source_failure()` recognizes grok markers
|
||||
- `skills/last30days/SKILL.md` — Grok session expiry handling guidance
|
||||
- `tests/test_grok_x.py` — expires_at and auth revocation tests
|
||||
- `tests/test_backend_descriptors.py` — grok expiry state tests
|
||||
- `changelog.d/+grok-auth-expired.fixed.md` — release notes fragment
|
||||
@@ -0,0 +1,87 @@
|
||||
# Research-quality eval harness
|
||||
|
||||
The eval suite measures the quality properties that ordinary unit tests do not: whether ranked evidence is grounded in retrieved inputs, stays inside the requested window, forms coherent clusters, accounts for every usable fixture source, and remains deterministic.
|
||||
|
||||
It runs the production pipeline offline. Recorded HTTP exchanges replay at `lib/http.py`; CLI-backed adapters such as yt-dlp, Digg, arXiv, Techmeme, and Trustpilot replay their parsed result at the source-module seam. Planning is supplied by each fixture manifest, and normalization, date filtering, scoring, fusion, clustering, source outcomes, and the versioned agent JSON export all run normally. The harness never calls an LLM or the network.
|
||||
|
||||
## Run it
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
uv run pytest tests/eval -x -s
|
||||
```
|
||||
|
||||
The `-s` keeps the score table visible. To print only the scored run and return a nonzero exit when a floor is missed:
|
||||
|
||||
```bash
|
||||
uv run python tests/eval/harness.py
|
||||
```
|
||||
|
||||
CI runs the pytest command in the `eval` job of `.github/workflows/validate.yml`, so every pull request gets a score table and a hard baseline check.
|
||||
|
||||
## Metrics
|
||||
|
||||
| Metric | Deterministic definition |
|
||||
|---|---|
|
||||
| Citation grounding | Fraction of exported result URLs that occur in the recorded fixture inputs. |
|
||||
| Recency compliance | Fraction of ranked source items whose known publication date is inside the report's inclusive date window. Undated evidence is not falsely classified as stale. |
|
||||
| Cluster coherence | Fraction of within-cluster candidate pairs meeting the production entity-overlap threshold (`0.45`). Singleton clusters are coherent by definition. |
|
||||
| Coverage | Fraction of fixture sources represented by usable report items or an explicit `Report.source_status` outcome. |
|
||||
| Determinism | `schema.to_dict()` equality for two runs with fixed time and identical recorded inputs. |
|
||||
|
||||
Aggregate floors live in `tests/eval/baseline.json`. The fixture matrix covers a tech product, a person, a comparison, breaking events, a niche technical topic, and a non-English CJK topic.
|
||||
|
||||
## Add or refresh a fixture
|
||||
|
||||
Fixture directories contain:
|
||||
|
||||
- `manifest.json`: topic archetype, fixed `as_of_date`, sources, safe dummy config, and a deterministic external query plan.
|
||||
- `http.json`: scrubbed HTTP exchanges and any CLI-backed source exchanges.
|
||||
|
||||
Use the direct engine invocation below only for development/fixture capture; `/last30days <topic>` remains the product interface:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py \
|
||||
"<topic>" \
|
||||
--quick \
|
||||
--as-of 2026-07-10 \
|
||||
--search grounding,hackernews \
|
||||
--plan /tmp/eval-plan.json \
|
||||
--record-fixtures tests/eval/fixtures/<fixture-name>
|
||||
```
|
||||
|
||||
`--record-fixtures` is intentionally hidden from `--help`. It records the live run's shared HTTP traffic and the bounded CLI-adapter seams, scrubs credential-shaped query/body/response fields, and writes `http.json`. It does not create the manifest because archetype, fixed date, source contract, and query plan are review decisions.
|
||||
|
||||
Before committing a recording:
|
||||
|
||||
1. Inspect `http.json` for cookies, keys, tokens, personal identifiers, and unnecessary long bodies.
|
||||
2. Truncate content to the smallest structure that exercises the adapter and pipeline.
|
||||
3. Replace irrelevant real usernames with obvious fixture identities.
|
||||
4. Add the manifest and run both commands above with networking unavailable.
|
||||
|
||||
The replay is fail-closed: an unrecorded request or an unused recorded exchange fails the run.
|
||||
|
||||
## Fixture flags
|
||||
|
||||
- `expects_clusters` (bool): fixtures whose topic historically forms multi-member clusters set this true; if cluster formation regresses to singletons on such a fixture, coherence scores 0.0 instead of a vacuous 1.0. Sparse topics (niche, non-english-cjk, tech-product) set it false because singletons are their legitimate shape.
|
||||
- Post-ranking enrichment (YouTube transcripts, Digg posts) is recorded and replayed by merging recorded `metadata` onto freshly computed items by item_id, so normalization/scoring/dedupe regressions stay visible to the eval rather than being overwritten by fixture state.
|
||||
- Post-rerank GitHub star enrichment records its repo->stars map and replays via `github.apply_star_map`, keeping runs offline even when `GITHUB_TOKEN` is set in CI. GitHub project-mode (`--github-repo`) and person-mode (`--github-user`) runs are not yet fixture-recordable; the network guard fails loudly if a fixture attempts them.
|
||||
|
||||
## Known seams
|
||||
|
||||
- Module-backed sources (yt-dlp, digg-pp-cli and other CLI adapters) record post-parse items at the module boundary, so replay does not re-exercise their parsing/normalization code the way HTTP-backed sources do (those replay raw responses through the real pipeline). A normalization regression in a module adapter is covered by that adapter's unit tests, not the eval. Recording raw CLI stdout is a possible future upgrade.
|
||||
- Cluster coherence shares `entity_extract` with production clustering. The pinned-predicate test (`test_entity_overlap_predicate_pinned`) guards against the shared predicate drifting permissive, and per-fixture floors in baseline.json catch a single archetype collapsing even when the cross-fixture average stays green.
|
||||
|
||||
## Move a baseline
|
||||
|
||||
Baseline edits are explicit quality-policy changes, not snapshot refreshes. Move a floor only when an intentional product change makes the old threshold invalid or when a new fixture legitimately changes the measured distribution.
|
||||
|
||||
Include in the review:
|
||||
|
||||
1. The old and new score tables.
|
||||
2. The reason the metric changed.
|
||||
3. A focused test proving the intended behavior.
|
||||
4. An explanation for any lower floor; never lower a floor solely to make CI green.
|
||||
|
||||
`test_intentional_out_of_window_regression_fails_recency_floor` is the standing negative control: it injects stale ranked evidence and proves the baseline check detects the regression.
|
||||
@@ -0,0 +1,133 @@
|
||||
# Agent JSON export
|
||||
|
||||
The agent JSON profile is the stable machine-readable research contract for downstream agents, scripts, dashboards, and workflow tools. Ask the slash command for machine-readable JSON:
|
||||
|
||||
```text
|
||||
/last30days AI coding agents — return the versioned agent JSON export
|
||||
```
|
||||
|
||||
For direct engine use in scripts, cron jobs, or development, use:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json
|
||||
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json --output results.json
|
||||
```
|
||||
|
||||
`--emit=json` defaults to `--json-profile=agent`. The full internal report remains available for debugging and power users:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json --json-profile=raw
|
||||
```
|
||||
|
||||
The raw profile is intentionally unversioned and may change when pipeline internals change. It preserves the JSON serialization used before the agent profile was introduced.
|
||||
|
||||
### Local corpus privacy
|
||||
|
||||
Evidence from `--corpus` / `LAST30DAYS_CORPUS_DIRS` is excluded from the versioned agent profile by default. The exclusion removes corpus results, corpus-only clusters, corpus source outcomes, freshness verdicts, and titles derived from a corpus representative. Set `LAST30DAYS_CORPUS_IN_EXPORT=1` only for a run whose JSON is intentionally allowed to contain local file contents. This opt-in does not change the schema shape or version; it permits `source: "corpus"` entries in the existing result fields. The unversioned `raw` profile is a complete local debug dump and may contain corpus paths and text.
|
||||
|
||||
## Discovery export
|
||||
|
||||
Discovery mode has a separate versioned contract so its topic results do not change the normal research export:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py --discover "AI agents" --emit=json
|
||||
```
|
||||
|
||||
Its top level contains `schema_version` (`1.1`), `kind` (`"discovery"`), `domain` (`""` for a global no-domain trending run), `generated_at`, `window_days`, `source_status`, `feeds`, `results`, `warnings`, `outcome` (`"ok"`, or `"nothing-solid"` when no topic cleared the confidence floor), and `weak_signal` (the closest sub-floor topic name on a nothing-solid run, else `null`). Each ranked result contains `rank`, `topic`, `why_spiking`, `momentum` (`new-this-week` or `building`), `velocity_score`, `sources`, per-source native `engagement`, a ready-to-run `command`, `evidence_urls`, `top_comment` (the strongest verbatim community comment from the topic's research pass, with attribution; `null` on shallow runs), `corroboration_count` (distinct confirming sources), `podcast_angle` (engine-generated podcast content hook; `null` when no reasoning provider produced one), `x_article_angle` (engine-generated X-article content hook; `null` when no reasoning provider produced one), `previously_surfaced_count` (topic-queue annotation: how many earlier sweeps surfaced this topic; `0` when the queue is off), `last_surfaced` (topic-queue annotation: date the topic last surfaced; `null` when the queue is off), and `covered` (topic-queue annotation: whether the topic was already covered; `false` when the queue is off). The discovery contract follows the same versioning policy below but evolves independently of the normal agent export. `--json-profile=raw` returns the unversioned internal `DiscoveryReport` dataclass instead.
|
||||
|
||||
When `LAST30DAYS_API_KEY` and `LAST30DAYS_API_BASE` route a run through a configured remote API, the server does not return the local `Report` needed to build this profile. In that mode, `--json-profile=agent` exits with status 2 instead of emitting a misleading shape; use `--json-profile=raw` to retain the remote backend's existing server-response JSON contract.
|
||||
|
||||
## Top-level fields
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `schema_version` | string | Agent export contract version. The current version is `1.2`. |
|
||||
| `query` | string | The research topic supplied to the engine. |
|
||||
| `generated_at` | string | UTC generation timestamp in RFC 3339 format. |
|
||||
| `window_days` | integer | Number of days between the report's start and end dates. |
|
||||
| `source_status` | object | Map of source name to the outcome observed during this run. |
|
||||
| `freshness_verdicts` | array | Per-claim act-time verdicts produced by `--verify-freshness`; empty when verification was not requested or no conservative claims were extractable. |
|
||||
| `clusters` | array | Ranked groups of related results. |
|
||||
| `results` | array | Ranked, flat evidence results for downstream processing. |
|
||||
|
||||
All top-level fields are always present. Empty runs contain empty `clusters` and `results` arrays. Sources appear in `source_status` when the run recorded an outcome for them.
|
||||
|
||||
## `freshness_verdicts`
|
||||
|
||||
Each entry identifies the grounded claim and candidate, its primary source item, the typed `verdict` (`current`, `stale`, `contradicted`, or `unsupported`), the original and re-derived values when applicable, and source/evidence URLs and timestamps. `stale` means a successful point re-fetch returned a moved value; `contradicted` means a newer item in the report window explicitly disagrees; `unsupported` means the datum could not be re-checked, including degraded `source_status` outcomes. Consumers can gate actions on `verdict == "current"` without treating an unreachable source as evidence that a claim moved.
|
||||
|
||||
## `source_status`
|
||||
|
||||
Each value distinguishes a clean empty result from incomplete coverage:
|
||||
|
||||
| State | Meaning |
|
||||
| --- | --- |
|
||||
| `ok` | The source completed and returned one or more items. |
|
||||
| `no-results` | The source completed successfully but found no matching items. |
|
||||
| `partial` | The source returned some items before a later failure. |
|
||||
| `rate-limited` | Retrieval was stopped by a provider rate limit. |
|
||||
| `auth-failed` | Credentials were missing, rejected, or expired during retrieval. |
|
||||
| `unreachable` | The source or network endpoint could not be reached. |
|
||||
| `timeout` | Retrieval exceeded its time limit. |
|
||||
| `schema-drift` | The provider response no longer matched the expected shape. |
|
||||
| `skipped-unconfigured` | The source was intentionally skipped because required configuration was absent. |
|
||||
| `error` | Retrieval failed for another reason. |
|
||||
|
||||
Consumers must not interpret failure states as evidence that a source had no discussion. Only `no-results` means the source completed cleanly with zero matches.
|
||||
|
||||
## Cluster fields
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `title` | string | Cluster headline. |
|
||||
| `summary` | string | Summary from the cluster's representative ranked result. |
|
||||
| `sources` | array of strings | Sources represented by the cluster. |
|
||||
| `engagement_total` | number | Sum of one headline native engagement counter per result. Known sources use their primary count (for example, Digg uses `postCount`); otherwise the largest counter-like field is used. Ranking, ratio, rating, and computed-score metadata are excluded. |
|
||||
|
||||
Cluster array order is ranking order. A result's `cluster` value is the zero-based index into this array.
|
||||
|
||||
## Result fields
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `candidate_id` | string | Stable identifier joining this result to `freshness_verdicts[].candidate_id`. Added in `1.2`. |
|
||||
| `title` | string | Result title. |
|
||||
| `source` | string | Primary source name, such as `reddit`, `x`, `youtube`, or `grounding`. |
|
||||
| `url` | string | Canonical result URL. It may be empty when the provider supplies no link. |
|
||||
| `published_at` | string | Primary source item's publication date or timestamp. Omitted when unknown. |
|
||||
| `summary` | string | Normalized snippet, with the relevance explanation or body used as fallback. |
|
||||
| `engagement` | object | Native engagement counters from the primary source item, such as Reddit `score` and `num_comments` or X `likes` and `reposts`. |
|
||||
| `relevance_score` | number | Engine final score normalized to the inclusive `0.0`–`1.0` range. |
|
||||
| `cluster` | integer | Zero-based index into `clusters`. Omitted when the result is not assigned to a cluster. |
|
||||
|
||||
Fields whose value is unknown are omitted rather than emitted as JSON `null`. Strings and collection fields otherwise remain present, including empty strings, objects, or arrays.
|
||||
|
||||
## Comparison runs
|
||||
|
||||
Comparison queries use an envelope so each entity keeps its own contract:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "1.2",
|
||||
"comparison": true,
|
||||
"entities": ["OpenAI", "Anthropic"],
|
||||
"reports": [
|
||||
{"entity": "OpenAI", "report": {"schema_version": "1.2", "query": "OpenAI"}},
|
||||
{"entity": "Anthropic", "report": {"schema_version": "1.2", "query": "Anthropic"}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The abbreviated reports above only illustrate the envelope; real reports contain every documented top-level field.
|
||||
|
||||
## Versioning policy
|
||||
|
||||
- `schema_version` uses `major.minor` numbering.
|
||||
- Any breaking field removal, rename, type change, semantic change, or envelope change requires a major-version bump.
|
||||
- Backward-compatible field additions may use a minor-version bump. Consumers should ignore fields they do not recognize.
|
||||
- The checked-in golden snapshot test locks the complete current shape. Contract changes must update the version and snapshot deliberately.
|
||||
- `1.2` added `candidate_id` to each `results` entry so verdicts can be joined to the result they annotate.
|
||||
- Discovery `1.1` added `podcast_angle`, `x_article_angle`, `previously_surfaced_count`, `last_surfaced`, and `covered` to each discovery `results` entry — a backward-compatible minor bump; the fields carry their defaults (`null`/`null`/`0`/`null`/`false`) until an angle generator or the topic queue populates them.
|
||||
- `--json-profile=raw` is outside this compatibility policy because it mirrors internal pipeline dataclasses.
|
||||
|
||||
`--preflight --emit=json` is a different machine contract for permission and configuration inspection. `--json-profile` does not alter preflight output.
|
||||
@@ -0,0 +1,205 @@
|
||||
# Original v3.0.0 First-Run NUX Wizard (reference capture)
|
||||
|
||||
Captured verbatim from `SKILL.md` at git commit `0a9ff16` (v3.0.0, 2026-04-08),
|
||||
the first-run setup wizard Matt built. Preserved here for provenance and as the
|
||||
source for the restored modal NUX (see docs/plans/2026-06-22-001-feat-restore-nux-wizard-plan.md).
|
||||
This is a historical snapshot - the live wizard in SKILL.md Step 0 uses the CURRENT
|
||||
source inventory (Digg, youtube_comments, SC backups) and omits Threads/Pinterest.
|
||||
|
||||
```markdown
|
||||
## Step 0: First-Run Setup Wizard
|
||||
|
||||
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever.
|
||||
|
||||
To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section **silently** and proceed to Step 1. **Do NOT say "Setup is complete" or any other status message — just move on.** The user doesn't need to be told setup is done every time they run the skill.
|
||||
|
||||
**When first run is detected, detect your platform first:**
|
||||
|
||||
**If you do NOT have WebSearch capability (OpenClaw, Codex, raw CLI):** Run the OpenClaw setup flow below.
|
||||
**If you DO have WebSearch (Claude Code):** Run the standard setup flow below.
|
||||
|
||||
---
|
||||
|
||||
### OpenClaw / Non-WebSearch Setup Flow
|
||||
|
||||
Run environment detection first:
|
||||
```bash
|
||||
python3 "${SKILL_ROOT}/scripts/last30days.py" setup --openclaw
|
||||
```
|
||||
|
||||
Read the JSON output. It tells you what's already configured. Display a status summary:
|
||||
|
||||
```
|
||||
👋 Welcome to /last30days!
|
||||
|
||||
Detected:
|
||||
{✅ or ❌} yt-dlp (YouTube search)
|
||||
{✅ or ❌} X/Twitter ({method} configured)
|
||||
{✅ or ❌} ScrapeCreators (TikTok, Instagram, Reddit backup)
|
||||
{✅ or ❌} Web search ({backend} configured)
|
||||
```
|
||||
|
||||
Then for each missing item, offer setup in priority order:
|
||||
|
||||
1. **ScrapeCreators** (if not configured): "ScrapeCreators adds TikTok and Instagram search (plus a Reddit backup if public Reddit gets rate-limited). 10,000 free calls, no credit card. (No referrals, no kickbacks - we don't get a cut.)"
|
||||
- Option A: "ScrapeCreators via GitHub (recommended)" -- Check if `gh` CLI was detected in the environment detection output above. If gh IS detected: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". Before running the command, display: "Registering via GitHub CLI..." If gh is NOT detected: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". Before running the command, display: "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 "${SKILL_ROOT}/scripts/last30days.py" setup --github`, parse JSON output. Tries PAT first (if `gh` is installed), falls back to device flow which copies a one-time code to your clipboard and opens your browser. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to .env.
|
||||
- Option B: "I have a key" -- accept paste, write to .env
|
||||
- Option C: "Skip for now"
|
||||
|
||||
2. **X/Twitter** (if not configured): "X search finds tweets and conversations. To unlock X: add FROM_BROWSER=auto (reads browser cookies, free), XAI_API_KEY (no browser access, api.x.ai), or AUTH_TOKEN+CT0 (manual cookies)."
|
||||
- Option A: "I have an xAI API key" (recommended for servers -- persistent, no expiry). Write XAI_API_KEY to .env.
|
||||
- Option B: "I have AUTH_TOKEN + CT0 from my browser" -- accept both, write to .env
|
||||
- Option C: "Skip for now"
|
||||
|
||||
3. **YouTube** (if yt-dlp not found): "YouTube search needs yt-dlp. Run: `pip install yt-dlp`"
|
||||
|
||||
4. **Web search** (if no Brave/Exa/Serper key): "A web search key enables smarter results. Brave Search is free for 2,000 queries/month at brave.com/search/api"
|
||||
|
||||
After setup, write `SETUP_COMPLETE=true` to .env and proceed to research.
|
||||
|
||||
**Skip to "END OF FIRST-RUN WIZARD" below after completing the OpenClaw flow.**
|
||||
|
||||
---
|
||||
|
||||
### Claude Code Setup Flow (Standard)
|
||||
|
||||
**You MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text -> (2) setup modal -> (3) run setup if chosen -> (4) optional ScrapeCreators modal -> (5) topic picker. You MUST start at step 1.**
|
||||
|
||||
**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion - do NOT repeat any of the welcome text inside the AskUserQuestion call.**
|
||||
|
||||
Welcome to /last30days!
|
||||
|
||||
I research any topic across Reddit, X, YouTube, and other sources - synthesizing what people are actually saying right now.
|
||||
|
||||
Auto setup gives you 5 core sources for free in 30 seconds:
|
||||
- X/Twitter - reads your x.com browser cookies to authenticate (not saved to disk). Chrome on macOS will prompt for Keychain access.
|
||||
- Reddit with comments - public JSON, no API key needed
|
||||
- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars)
|
||||
- Hacker News + Polymarket + GitHub (if `gh` CLI installed) - always on, zero config
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation.
|
||||
|
||||
**Then call AskUserQuestion with ONLY this question and these options - no additional text:**
|
||||
|
||||
Question: "How would you like to set up?"
|
||||
Options:
|
||||
- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp for YouTube"
|
||||
- "Manual setup - show me what to configure"
|
||||
- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if gh installed), Web"
|
||||
|
||||
**If the user picks 1 (Auto setup):**
|
||||
|
||||
**Before running the setup command, get cookie consent:**
|
||||
|
||||
Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`. If it does, skip the consent prompt and run setup directly.
|
||||
|
||||
If `BROWSER_CONSENT=true` is NOT present, **call AskUserQuestion:**
|
||||
Question: "Auto setup will scan your browser for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. Chrome on macOS will prompt for Keychain access. OK to proceed?"
|
||||
Options:
|
||||
- "Yes, scan my cookies for X" - Run setup as normal. Append `BROWSER_CONSENT=true` to .env after setup completes.
|
||||
- "Skip X, just set up YouTube" - Run setup with YouTube only (install yt-dlp). Do not scan cookies.
|
||||
- "I have an xAI API key instead" - Ask them to paste it, write XAI_API_KEY to .env. Then install yt-dlp.
|
||||
|
||||
Run the setup subcommand:
|
||||
```bash
|
||||
cd {SKILL_DIR} && python3 scripts/last30days.py setup
|
||||
```
|
||||
Show the user the results (what cookies were found, whether yt-dlp was installed).
|
||||
|
||||
**Then show the optional ScrapeCreators offer (plain text, then modal):**
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those platforms - 10,000 free calls, no credit card. It also serves as a Reddit backup if public Reddit ever gets rate-limited.
|
||||
|
||||
**Before showing the ScrapeCreators modal, check for `gh` CLI:** Run `which gh` via Bash silently. Store the result as gh_available (true if found, false if not).
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Want to add TikTok, Instagram, and Reddit backup via ScrapeCreators? (We don't get a cut.)"
|
||||
Options:
|
||||
- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". If NOT gh_available: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". After the user selects this option: If gh_available, display "Registering via GitHub CLI..." before running the command. If NOT gh_available, display "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V on Mac, Ctrl+V on Windows/Linux)." Then run `cd {SKILL_DIR} && python3 scripts/last30days.py setup --github` via Bash with a 5-minute timeout. This tries PAT auth first (if `gh` CLI is installed, zero browser needed), then falls back to GitHub device flow which copies a one-time code to your clipboard and opens GitHub in your browser. Parse the JSON stdout. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to `~/.config/last30days/.env`. If `method` is `pat`, show: "You're in! Registered via GitHub CLI - zero browser needed. 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is true, show: "You're in! (The authorization code was copied to your clipboard automatically.) 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is false, show: "You're in! 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `status` is `timeout` or `error`, show: "GitHub auth didn't complete. No worries - you can sign up at scrapecreators.com instead or try again later." Then offer the web signup option.
|
||||
- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env
|
||||
- "I have a key" - accept the key, write to .env
|
||||
- "Skip for now" - proceed without ScrapeCreators
|
||||
|
||||
**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:**
|
||||
|
||||
Your ScrapeCreators key powers TikTok, Instagram, Threads, Pinterest, and YouTube comments. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.)
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Which ScrapeCreators sources do you want on?"
|
||||
Options:
|
||||
- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "TikTok and Instagram are on, plus Reddit backup if public Reddit has issues. You can add threads, pinterest, youtube_comments to INCLUDE_SOURCES anytime."
|
||||
- "Everything - TikTok, Instagram, Threads, Pinterest, YouTube comments" - append `INCLUDE_SOURCES=tiktok,instagram,threads,pinterest,youtube_comments` to ~/.config/last30days/.env. Confirm: "All ScrapeCreators sources are on."
|
||||
- "Just the basics - let's run our first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will serve as Reddit backup. You can add sources to INCLUDE_SOURCES in your .env anytime."
|
||||
|
||||
**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:**
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "What do you want to research first?"
|
||||
Options:
|
||||
- "Claude Code vs Codex" - tech comparison
|
||||
- "Sam Altman" - person in the news
|
||||
- "Warriors Basketball" - sports
|
||||
- "AI Legal Prompting Techniques" - niche/professional
|
||||
- "Type my own topic"
|
||||
|
||||
If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly.
|
||||
|
||||
**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.**
|
||||
|
||||
**If the user picks 2 (Manual setup):**
|
||||
Show them this guide (present as plain text, not blockquoted):
|
||||
|
||||
The magic of /last30days is Reddit comments + X posts together - and both are free. Here's how to unlock each source.
|
||||
|
||||
Add these to `~/.config/last30days/.env`:
|
||||
|
||||
X/Twitter (pick one - this is the most important):
|
||||
- `FROM_BROWSER=auto` - free. Reads your x.com login cookies at search time to authenticate. Cookies are read live each run, not saved to disk. Chrome on macOS will prompt for Keychain access the first time. Firefox and Safari don't.
|
||||
- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers or if you don't want cookie scanning.
|
||||
- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com -> F12 -> Application -> Cookies)
|
||||
|
||||
Reddit (free, works out of the box):
|
||||
- Public JSON gives you threads + top comments with upvote counts. No setup required.
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - optional backup source if public Reddit gets rate-limited.
|
||||
- `OPENAI_API_KEY=xxx` - optional fallback if public Reddit search has trouble finding threads.
|
||||
|
||||
YouTube (free, open source):
|
||||
- Run `brew install yt-dlp` - free, open source, 190K+ GitHub stars. Enables YouTube search and transcripts.
|
||||
|
||||
Bonus: TikTok, Instagram, Threads, Pinterest, YouTube comments (ScrapeCreators):
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com.
|
||||
- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the most popular ones. Add threads, pinterest, youtube_comments for more.
|
||||
|
||||
GitHub Issues/PRs (free, no key needed):
|
||||
- If you have the `gh` CLI installed (`brew install gh`), GitHub search is automatic. No API key required.
|
||||
|
||||
Perplexity Sonar Pro (AI-synthesized research via OpenRouter):
|
||||
- `OPENROUTER_API_KEY=xxx` - adds AI-synthesized research with citations as an additive source alongside Reddit/X/YouTube. Returns structured narratives with specific dates, names, and numbers that social sources miss. ~$0.02/run.
|
||||
- After adding your key, set `INCLUDE_SOURCES=perplexity` (or append to existing, e.g. `INCLUDE_SOURCES=tiktok,instagram,perplexity`).
|
||||
- Use `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query) on topics that need serious investigation.
|
||||
- Bonus: also powers the planning and reranking engine if you don't have a Gemini/OpenAI/xAI key.
|
||||
|
||||
Other bonus sources (add anytime):
|
||||
- `EXA_API_KEY=xxx` - semantic web search, 1K free/month (exa.ai)
|
||||
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password)
|
||||
- `BRAVE_API_KEY=xxx` - Brave web search
|
||||
|
||||
Always add this last line: `SETUP_COMPLETE=true`
|
||||
|
||||
**CRITICAL: NEVER overwrite an existing .env file.** Before writing ANY key to `~/.config/last30days/.env`:
|
||||
1. Check if the file exists: `test -f ~/.config/last30days/.env`
|
||||
2. If it exists, READ it first, then APPEND only missing keys using `>>` (double redirect)
|
||||
3. NEVER use `>` (single redirect) which destroys existing content
|
||||
4. If it doesn't exist, create it: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env`
|
||||
|
||||
**Then call AskUserQuestion:**
|
||||
Question: "How do you want to add your keys?"
|
||||
Options:
|
||||
- "Open .env in my editor" - Creates the file with a commented template and opens it. You edit, save, and come back.
|
||||
- "Paste keys here" - Paste your API keys and I'll write the file for you.
|
||||
- "I'll do it myself" - I'll tell you the file path and you handle it.
|
||||
|
||||
**If the user picks "Open .env in editor":**
|
||||
Create `~/.config/last30days/.env` if it doesn't exist (check first!), pre-populated with this template:
|
||||
```
|
||||
```
|
||||
@@ -0,0 +1,29 @@
|
||||
## Residual Review Findings
|
||||
|
||||
Run context: ce-code-review `mode:agent` on branch `fix/github-qualifier-strip` (head `42c5ab5bebcb3d4bd4d8bfc11f89b4df4bc1da9b`), plan `docs/plans/2026-08-07-001-fix-github-qualifier-collision-plan.md`, run id `20260807-231856-17902`. Findings not applied in LFG step 5; filed for durability.
|
||||
|
||||
### Filed (tracker: GitHub Issues)
|
||||
|
||||
- **P1** — `skills/last30days/scripts/lib/github.py:237` — Qualifier-only topic classified as ERROR poisons retry eligibility — [mvanhorn/last30days-skill#951](https://github.com/mvanhorn/last30days-skill/issues/951) (settled-conflict: report-only per KTD-1)
|
||||
- **P2** — `skills/last30days/scripts/lib/github.py:186` — Quote-wrapped or paren-wrapped qualifiers bypass the strip — [mvanhorn/last30days-skill#952](https://github.com/mvanhorn/last30days-skill/issues/952)
|
||||
- **P2** — `skills/last30days/scripts/lib/github.py:231` — Empty or noise-plus-qualifier topics flip to hard ERROR — [mvanhorn/last30days-skill#953](https://github.com/mvanhorn/last30days-skill/issues/953) (settled-conflict: report-only per KTD-1)
|
||||
- **P3** — `skills/last30days/scripts/lib/github.py:229` — Repeated qualifier-only subqueries spam logs and error detail — [mvanhorn/last30days-skill#954](https://github.com/mvanhorn/last30days-skill/issues/954)
|
||||
|
||||
### Settled-conflict findings (report-only, not filed as apply requests)
|
||||
|
||||
- **P1** — `skills/last30days/scripts/lib/github.py:237` — Qualifier-only topic classified as ERROR poisons retry eligibility — conflicts with KTD-1 (session-settled plan decision: qualifier-only topics return the error envelope). Downstream ERROR/attempted classification blocks `_retry_thin_sources`; filed as #951 for durability, not for application.
|
||||
- **P2** — `skills/last30days/scripts/lib/github.py:231` — Empty or noise-plus-qualifier topics flip to hard ERROR — conflicts with KTD-1/R3 (error envelope for qualifier-only/empty topics). Filed as #953 for durability, not for application.
|
||||
|
||||
### No sink / failed
|
||||
|
||||
None.
|
||||
|
||||
### Proceeded-and-flagged settled-decision conflicts (from ce-work step 2)
|
||||
|
||||
None — ce-work returned no `settled_decision_conflicts`.
|
||||
|
||||
### Residual risks carried from the review
|
||||
|
||||
- Error-envelope `context["core"]` is unstripped in the qualifier-only path vs stripped in the success path; no current consumer is affected.
|
||||
- GitHub 422 behavior for unbalanced quotes is external API behavior, not exercised in tests.
|
||||
- Planner emission of comma-glued, quoted, or wrapped qualifier shapes is LLM behavior; exposure is unquantifiable from code.
|
||||
+380
@@ -0,0 +1,380 @@
|
||||
---
|
||||
title: "Checkpointed discovery protocol: five design conventions for host-side LLM judgment"
|
||||
date: 2026-07-21
|
||||
category: architecture-patterns
|
||||
module: discovery-checkpoint-protocol
|
||||
problem_type: architecture_pattern
|
||||
component: tooling
|
||||
severity: high
|
||||
applies_when:
|
||||
- "The product's primary consumer is a frontier reasoning model invoking the tool as an agent skill, not a traditional programmatic API client"
|
||||
- "A pipeline stage needs semantic judgment (naming, classification, worthiness scoring) that only an LLM can supply"
|
||||
- "Building a keyless or free-tier path where a silent heuristic fallback would degrade output quality without disclosing that an API key was assumed"
|
||||
- "A CLI or script needs to persist state across multiple invocations while the host model performs judgment in between (checkpoint-and-resume design)"
|
||||
- "An existing skill law or convention already establishes that host-side reasoning replaces engine-side API keys, and a new pipeline stage needs the same treatment"
|
||||
symptoms:
|
||||
- "v3.17.0 silently fell back to deterministic topic naming and junk heuristics for keyless users when the engine's own LLM judge was unavailable"
|
||||
- "An API key was the de facto front door to real discovery judgment, contradicting the skill's keyless-path promise and its own LAW 7 host-is-the-reasoning-model precedent"
|
||||
root_cause: wrong_api
|
||||
resolution_type: code_fix
|
||||
tags:
|
||||
- "discovery-protocol"
|
||||
- "host-judged-protocol"
|
||||
- "checkpoint-files"
|
||||
- "law-11"
|
||||
- "keyless-path"
|
||||
- "nominations-bundle"
|
||||
- "provenance-enforcement"
|
||||
- "bundle-id-ttl"
|
||||
related_components:
|
||||
- "skills/last30days/scripts/lib/discovery_handoff.py"
|
||||
- "skills/last30days/scripts/last30days.py"
|
||||
- "skills/last30days/SKILL.md"
|
||||
- "tests/test_discover_handoff.py"
|
||||
- "tests/test_discover_mode.py"
|
||||
---
|
||||
|
||||
# Checkpointed discovery protocol: five design conventions for host-side LLM judgment
|
||||
|
||||
## Context
|
||||
|
||||
An Agent Skill's primary consumer is a frontier reasoning model: the engine
|
||||
(`skills/last30days/scripts/last30days.py`) is invoked by Claude Code, Codex,
|
||||
Gemini, or another agent runtime that read SKILL.md. v3.17.0 (PR #852) forgot
|
||||
that and shipped judgment as an engine-side LLM pass: `lib/discovery_judge.py`
|
||||
(since deleted by PR #856; the path is historical) resolved a reasoning
|
||||
provider across Gemini/OpenAI/xAI/OpenRouter keys and,
|
||||
per its own contract, never raised - "No provider, a failed call, or a
|
||||
malformed payload logs a warning and returns None, and the caller falls back"
|
||||
to deterministic heuristics. Every keyless user silently got the degraded
|
||||
branch: heuristic topic names like "120k 1,600 ESP32s" and zero content
|
||||
angles, with no signal that a better path existed.
|
||||
|
||||
PR #856 (v3.18.0) deleted the engine judge outright (CHANGELOG.md:18-20) and
|
||||
replaced it with a three-command host-judged protocol, mandated by SKILL.md
|
||||
LAW 11 "YOU ARE THE JUDGE" (skills/last30days/SKILL.md:233): the pipeline
|
||||
pauses at its judgment points and persists versioned checkpoint files that
|
||||
the hosting model judges between invocations. Leg 1 (`--discover
|
||||
--nominate-only`) sweeps and writes the nominations bundle; the host writes a
|
||||
judgments file; leg 2 (`--discover --judgments <file>`) resumes, deep-enriches,
|
||||
and writes the pending report; the host writes an angles file; leg 3
|
||||
(`--discover --finalize [--angles <file>]`) renders offline. The contracts
|
||||
live in `skills/last30days/scripts/lib/discovery_handoff.py` (module
|
||||
docstring, lines 1-18), the leg handlers in
|
||||
`skills/last30days/scripts/last30days.py:1716-1986`, and the resumed pipeline
|
||||
math in `skills/last30days/scripts/lib/pipeline.py:1525-1697`.
|
||||
|
||||
This doc records the five conventions that make a checkpoint protocol safe:
|
||||
identity/TTL binding, the lossless-state-vs-capped-digest split, fail-closed
|
||||
parsing of empty state, provenance enforcement across invocations, and
|
||||
guarded writes plus stale-sibling invalidation.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Checkpoints are identity-bound and time-bound
|
||||
|
||||
Every host-authored file must echo the checkpoint's identity. Leg 1 mints a
|
||||
random `bundle_id` (`discovery_handoff.py:270`), prints it in the digest, and
|
||||
both host files (judgments, angles) must carry it back.
|
||||
`_require_bundle_binding` (`discovery_handoff.py:638-673`) enforces the echo
|
||||
and its error names BOTH ids and the cheap remedy:
|
||||
|
||||
```python
|
||||
raise HandoffContractError(
|
||||
f"The {label} file is bound to bundle_id {file_bundle_id!r} but the "
|
||||
f"{noun} is {bundle.bundle_id!r}. {location_label}:\n"
|
||||
f"{_searched_lines(searched)}\n"
|
||||
f"Correct the bundle_id field in your {label} file to "
|
||||
f"{bundle.bundle_id!r} and re-run this same leg."
|
||||
)
|
||||
```
|
||||
|
||||
WHY the remedy split matters: a mismatched echo means the host copied the
|
||||
wrong id into an otherwise-current file, so the fix is edit-one-field and
|
||||
retry THIS leg - never the expensive re-sweep (`_RESWEEP_REMEDY`,
|
||||
`discovery_handoff.py:43`) or resume (`_RESUME_REMEDY`, lines 48-51)
|
||||
remedies, which belong to missing/stale state. On the finalize leg the
|
||||
message deliberately names the pending report, not the bundle, so the host's
|
||||
retry is not misdirected (lines 653-664). `HandoffContractError` maps to
|
||||
exit 2 in one place (`last30days.py:1984-1986`).
|
||||
|
||||
Time binding is a dedicated module constant with a deliberate non-reuse
|
||||
comment (`discovery_handoff.py:32-36`):
|
||||
|
||||
```python
|
||||
# How long a nominations bundle stays valid. Deliberately a module constant
|
||||
# and NOT the LAST30DAYS_REPORT_CACHE_TTL_SECONDS env knob: a user who
|
||||
# lowered the report-cache TTL for drill freshness must not shrink the
|
||||
# window a host has to author judgments.
|
||||
DISCOVERY_HANDOFF_TTL_SECONDS = 3600.0
|
||||
```
|
||||
|
||||
Staleness is checked in the shared envelope validator via
|
||||
`env.is_timestamp_fresh` (`discovery_handoff.py:430-436`, `env.py:121`), and
|
||||
the pending report gets a FRESH TTL clock stamped at leg-2 write time
|
||||
(`last30days.py:1854-1856`) because leg 2 started a new authoring window.
|
||||
WHY: an unrelated cache knob silently shrinking the host's judging window is
|
||||
exactly the class of cross-feature coupling a checkpoint file must not have.
|
||||
|
||||
### 2. One checkpoint, two audiences, hard split: lossless resume state vs capped fenced digest
|
||||
|
||||
The nominations bundle serves the engine and the host, and the two halves
|
||||
have opposite rules.
|
||||
|
||||
Engine half: the FULL judge pool with complete seed items, serialized
|
||||
losslessly (`schema.py:848-852` states the contract;
|
||||
`schema.nomination_to_dict`, `schema.py:917-932`, round-trips every item).
|
||||
Leg 2's floor/velocity/entity math must score identically to a
|
||||
single-process run: `_floor_survivor_records` is "Shared verbatim by
|
||||
run_discover (one-shot) and run_discover_resume (protocol leg 2) so floor
|
||||
semantics can never drift between the paths" (`pipeline.py:1199-1215`), and
|
||||
velocity scores against the bundle's momentum window, never the resume-time
|
||||
clock (`pipeline.py:1558-1561`). A capped bundle would silently starve
|
||||
downgraded-topic scoring: host-junk rows, and heuristic-junk fallback rows
|
||||
below the seed-source floor, never
|
||||
get an enrichment pass, so their weak-signal velocity and the
|
||||
seed-source-corroboration floor count are computed purely from bundle seed
|
||||
items (`pipeline.py:1584-1593`, `rerank.py:110-136`) - truncate the items and
|
||||
those rows under-count sources and engagement with no error anywhere. Parity
|
||||
is test-pinned: `tests/test_discover_handoff.py:232`
|
||||
(`test_parity_floor_and_velocity_inputs_survive_round_trip`) asserts
|
||||
velocity, engagement totals, source sets, and entity-disambiguation inputs
|
||||
(title + snippet) recompute identically after the round trip.
|
||||
|
||||
Host half: `build_host_digest` (`discovery_handoff.py:915-976`) is capped
|
||||
(`_DIGEST_TITLE_MAX_CHARS`/`_DIGEST_SNIPPET_MAX_CHARS`/`_DIGEST_COMMENT_MAX_CHARS`,
|
||||
lines 66-68) and its evidence lines ride inside the untrusted-content fence:
|
||||
|
||||
```python
|
||||
if evidence_lines:
|
||||
lines.append("")
|
||||
lines.append(rerank._fenced_untrusted_content("\n".join(evidence_lines)))
|
||||
```
|
||||
|
||||
That is the exact fence the rerank judge uses (`rerank.py:305-312`), and the
|
||||
deleted engine judge fenced the same evidence the same way (v3.17.0's
|
||||
`discovery_judge.py` imported `_fenced_untrusted_content` for both of its
|
||||
prompts). Dropping the fencing during the rewrite was a caught regression;
|
||||
the fence is now pinned by
|
||||
`tests/test_discover_handoff.py:755`
|
||||
(`test_digest_fences_untrusted_evidence_like_the_engine_judge`): scraped
|
||||
titles/snippets/comments inside the fence, structural lines (ids, sources,
|
||||
signal, bundle path) outside it. Host-supplied text going the other
|
||||
direction is capped too - names at 96 chars, angles at 200
|
||||
(`discovery_handoff.py:53-62`), "ported from the retired engine-judge pass"
|
||||
because names become search queries and angles render verbatim on cards.
|
||||
|
||||
### 3. Engine-written checkpoints parse strict-at-top, lenient-per-row, but FAIL CLOSED on structurally empty state
|
||||
|
||||
The readers are strict at the top level (readable, JSON object, right kind,
|
||||
right schema version, bundle_id present, within TTL - all in
|
||||
`_parse_handoff_envelope`, `discovery_handoff.py:381-437`) and lenient per
|
||||
row: one corrupt nomination row is warned and skipped, never fatal
|
||||
(`discovery_handoff.py:469-487`). But leniency has a floor. A non-list
|
||||
`nominations` value raises (`discovery_handoff.py:460-466`), and zero valid
|
||||
parsed rows raises too (`discovery_handoff.py:506-514`):
|
||||
|
||||
```python
|
||||
if not nominations:
|
||||
# Leg 1 never writes an empty bundle (a zero-nomination sweep
|
||||
# short-circuits with no bundle file), so an empty or all-invalid
|
||||
# nominations array is corrupt state: fail closed, never hand the
|
||||
# resume leg a silently empty pool.
|
||||
raise HandoffContractError(...)
|
||||
```
|
||||
|
||||
WHY: without this, a corrupt bundle flows into leg 2 as an empty pool, which
|
||||
floors to zero survivors and renders an authoritative-looking "Nothing solid
|
||||
this window" brief - a green result manufactured from broken state. PR
|
||||
#856's review caught this
|
||||
empty-pool-renders-authoritative-empty-result failure; it is now pinned by
|
||||
`tests/test_discover_handoff.py:439` (all rows malformed) and `:457` (empty
|
||||
list). The invariant that makes fail-closed valid: leg 1 short-circuits a
|
||||
zero-nomination sweep to the nothing-solid brief and writes NO bundle
|
||||
(`last30days.py:1748-1750`), so an empty pool on disk is always corruption,
|
||||
never a legitimate outcome.
|
||||
|
||||
### 4. Cross-invocation state carries provenance, and the resume legs enforce it
|
||||
|
||||
Three kinds of provenance ride the checkpoint files:
|
||||
|
||||
Mock parity. Both checkpoints stamp `mock` at write time
|
||||
(`discovery_handoff.py:307`, `last30days.py:1861`), and every resume leg
|
||||
runs `_require_discover_mock_parity` (`last30days.py:1506-1532`): mock-born
|
||||
state is rejected by a real run and real state by a `--mock` run, in both
|
||||
cases exit 2 with a fix-the-flag or fresh-sweep remedy. WHY: "mock-born
|
||||
state finalized by a real run would fake a real brief from fixture data, and
|
||||
real state finalized by --mock would silently drop the round's queue write."
|
||||
|
||||
Sweep coverage. Leg 1 serializes the sweep's per-source outcome map into the
|
||||
bundle (`discovery_handoff.py:308-311`), leg 2 restores it into its report
|
||||
(`pipeline.py:1681-1692`), and leg 3 inherits it through the pending report,
|
||||
so degraded coverage "survives the protocol instead of silently reading as
|
||||
clean" (`discovery_handoff.py:115-118`). Every leg terminal - one-shot and
|
||||
all three legs - exits through the ONE shared strict-exit helper,
|
||||
`_discovery_strict_exit_code` (`last30days.py:1481-1503`, called at
|
||||
1703, 1750, 1797, 1839, 1895, 1967), which turns `LAST30DAYS_STRICT_EXIT`
|
||||
plus any non-clean source outcome into exit 3. The PR #856 review validated
|
||||
this as a P1: before the fix, protocol legs silently exited 0 on degraded
|
||||
sweeps because the status map was dropped between legs.
|
||||
|
||||
Store scoping. An explicit `--save-dir` is the SOLE handoff store.
|
||||
`_search_paths` (`discovery_handoff.py:211-227`) returns "ONLY the save dir
|
||||
when one was supplied, else the config dir", mirroring `_scoped_store_db`
|
||||
(`last30days.py:432-437`): "a handoff file in the config dir must never
|
||||
silently satisfy a save-dir run." The second validated P1 of the review:
|
||||
with a fallback chain, a missing pending file in the save dir would let a
|
||||
bare `--finalize` quietly consume the config-dir store's pending report and
|
||||
finalize another store's run.
|
||||
|
||||
### 5. Guard the write after the expensive work, and invalidate stale siblings on fresh rounds
|
||||
|
||||
Both engine checkpoint writes happen after minutes of paid-for work (a sweep;
|
||||
a deep enrichment pass), so an OSError there is converted to the typed
|
||||
contract error, never a traceback (`discovery_handoff.py:329-334` for the
|
||||
bundle; `last30days.py:1869-1877` for the pending report):
|
||||
|
||||
```python
|
||||
except OSError as exc:
|
||||
# A locked/read-only/full disk is the protocol's clean exit-2 path,
|
||||
# never a traceback.
|
||||
raise HandoffContractError(
|
||||
f"Could not write nominations bundle {path}: {exc}"
|
||||
) from exc
|
||||
```
|
||||
|
||||
This is the repo's guarded-write convention (same shape as the discovery
|
||||
queue's guarded end-of-run write) applied to checkpoints, and it is pinned by
|
||||
`tests/test_discover_handoff.py:472`.
|
||||
|
||||
Fresh rounds invalidate stale siblings. A new leg-1 bundle starts a NEW
|
||||
protocol round, so any pending report left by a prior round is deleted
|
||||
alongside it (`last30days.py:1783-1788`); a leg 2 that ends nothing-solid
|
||||
wrote no pending file this round, so it also unlinks any stale one
|
||||
(`last30days.py:1830-1837`). WHY: without the unlinks, an unbound bare
|
||||
`--finalize` inside the TTL could re-serve the PREVIOUS round's report as if
|
||||
it belonged to the current sweep. The deliberate exception proves the rule:
|
||||
a SUCCESSFUL finalize leaves the pending file in place
|
||||
(`last30days.py:1905-1911`) so a retry with a corrected angles file keeps
|
||||
working, and idempotency comes from replaying the leg-2 `run_ref` into the
|
||||
queue (`last30days.py:1958-1964`) rather than from deleting state.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The architecture smell this pattern removes: an external LLM API call inside
|
||||
an engine whose invoker IS an LLM. That shape fails three ways at once. It
|
||||
adds cost (a second metered model where a capable one is already in the
|
||||
loop). It forks quality silently between keyed and keyless users - v3.17.0's
|
||||
judge never raised on a missing provider, so keyless users got heuristic
|
||||
names and no angles with zero indication anything was degraded, on the
|
||||
skill's PRIMARY invocation path. And it produces a strictly worse judge: the
|
||||
budget-priced engine-side model (flash-lite class, batched, no session
|
||||
context) judged evidence the frontier host model could have judged directly.
|
||||
|
||||
The checkpoint protocol is the general remedy shape: pause the pipeline at
|
||||
each judgment point, persist versioned, identity-bound, TTL-bound state, let
|
||||
the host judge between invocations, and validate every resume so stale or
|
||||
mismatched state becomes a clean exit 2 with a named remedy instead of
|
||||
silent wrong output. LAW 11's framing (SKILL.md:233) is the contract in one
|
||||
line: "You do not need an API key ... you ARE the reasoning model." The
|
||||
one-shot path prints a loud note pointing at the protocol
|
||||
(`pipeline.py:1421-1433`) precisely so a reasoning-model host can never
|
||||
mistake heuristic output for a capability ceiling.
|
||||
|
||||
The five conventions are what make the pause safe. Splitting one pipeline
|
||||
into three processes creates every classic distributed-state hazard in
|
||||
miniature - stale state, cross-round state, cross-store state, fixture/real
|
||||
crosses, silently-empty state, lost coverage warnings - and each convention
|
||||
above closes one of them.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Apply this pattern when:
|
||||
|
||||
- A CLI or engine embedded in an Agent Skill needs semantic judgment
|
||||
(naming, junk filtering, scoring, prose authoring) in the middle of an
|
||||
otherwise deterministic pipeline - the host model is the judge; checkpoint
|
||||
around the judgment points.
|
||||
- You are about to add an LLM provider key, client, or "reasoning provider"
|
||||
resolution to an engine whose invoker is already a reasoning model - that
|
||||
is the smell; reach for the protocol instead.
|
||||
- An existing engine-side LLM pass has a "silent heuristic fallback" - the
|
||||
keyless majority is getting invisible degraded output today.
|
||||
|
||||
Do NOT apply it when:
|
||||
|
||||
- No reasoning model is in the loop. The one-shot cron/scripted path keeps
|
||||
the single-process pipeline deliberately (`run_discover`,
|
||||
`pipeline.py:1384`, and the degradation rule in SKILL.md:398): a
|
||||
checkpoint pause with nobody to judge is just a hang.
|
||||
- The judgment is expressible as a deterministic rule - the junk-shape
|
||||
heuristics and the confidence floor stayed engine-side because they need
|
||||
no model at all.
|
||||
|
||||
## Examples
|
||||
|
||||
The three-command sequence as SKILL.md ships it (skills/last30days/SKILL.md:318-399),
|
||||
one identical `--save-dir` threaded through all three legs:
|
||||
|
||||
```bash
|
||||
# Leg 1 - sweep and nominate (global trending; domain runs pass the domain
|
||||
# phrase as the --discover argument on this leg only):
|
||||
python3 scripts/last30days.py --discover --nominate-only \
|
||||
--save-dir="$HOME/Documents/Last30Days"
|
||||
# stdout: judging digest + bundle path + bundle_id. Host reads the bundle
|
||||
# file, then writes judgments.json:
|
||||
# {"bundle_id": "<echoed>", "judgments": [
|
||||
# {"id": "n1", "name": "Gemma 4 chat templates", "junk": false, "worthiness": 85},
|
||||
# {"id": "n2", "name": "Beginner asks how to deploy", "junk": true, "worthiness": 10}]}
|
||||
|
||||
# Leg 2 - resume with judgments; deep per-topic research (several minutes):
|
||||
python3 scripts/last30days.py --discover --judgments judgments.json \
|
||||
--save-dir="$HOME/Documents/Last30Days"
|
||||
# stdout ends with angle inputs keyed by surviving id. Host writes
|
||||
# angles.json: {"bundle_id": "<same>", "angles": [
|
||||
# {"id": "n1", "podcast": "<hook>", "x_article": "<hook>"}]}
|
||||
|
||||
# Leg 3 - finalize offline: apply angles, render, record the topic queue.
|
||||
python3 scripts/last30days.py --discover --finalize --angles angles.json \
|
||||
--emit=compact --save-dir="$HOME/Documents/Last30Days"
|
||||
```
|
||||
|
||||
Failure-mode walkthrough (mismatched then stale checkpoint):
|
||||
|
||||
1. The host echoes a bundle_id from an earlier round into judgments.json and
|
||||
runs leg 2. `_require_bundle_binding` raises; the CLI prints
|
||||
`[last30days] The judgments file is bound to bundle_id 'aaaa...' but the
|
||||
current nominations bundle is 'bbbb...'` plus the searched location, and
|
||||
exits 2. Remedy as printed: correct the `bundle_id` field and re-run leg 2.
|
||||
The expensive sweep is NOT redone - the bundle on disk is still current.
|
||||
2. The host instead waits 90 minutes before judging. The envelope check
|
||||
(`discovery_handoff.py:430-436`) finds `generated_at` outside the 3600s
|
||||
TTL and exits 2: the bundle "is stale ... the momentum window it captured
|
||||
has moved on. Run a fresh `--discover --nominate-only` re-sweep." Here the
|
||||
expensive leg IS the remedy, because the state itself expired - the
|
||||
protocol never asks for the expensive path when a cheap edit fixes the
|
||||
problem, and never accepts cheap edits when the data has aged out.
|
||||
|
||||
The deterministic end-to-end twin of the whole sequence is pinned in CI:
|
||||
`tests/test_discover_mode.py:2439`
|
||||
(`test_discovery_cli_full_mock_protocol_three_legs_end_to_end`).
|
||||
|
||||
## Related
|
||||
|
||||
- `docs/solutions/architecture-patterns/discovery-topic-queue-design-conventions.md` -
|
||||
same feature family, the queue side: the persistent topic queue leg 3
|
||||
writes into (idempotently, under the leg-2 `run_ref`), including the
|
||||
guarded-write convention this protocol reuses.
|
||||
- `docs/solutions/design-patterns/ranked-output-confidence-floor-honest-empty-state.md` -
|
||||
the confidence-floor semantics the protocol preserves verbatim across the
|
||||
process split (`_floor_survivor_records` shared by both paths), including
|
||||
seed-source corroboration for junk shapes.
|
||||
- `docs/solutions/logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md` -
|
||||
the enrichment wall-clock budget pattern leg 2's deep tier extends
|
||||
(`RESUME_DEEP_ENRICH_BUDGET_SECONDS` 450s via
|
||||
`LAST30DAYS_ENRICH_BUDGET_SECONDS`, `pipeline.py:1492-1508`; workers stay
|
||||
daemon threads and never touch disk - the pending report is ONE post-loop
|
||||
write from the main thread, `last30days.py:1867-1871`).
|
||||
- PR #856 (protocol, engine-judge removal), PR #852 (the v3.17.0 engine
|
||||
judge this replaced), CHANGELOG.md v3.18.0 / v3.17.0 entries.
|
||||
- SKILL.md LAW 11 and the Step 1 DISCOVERY branch (skills/last30days/SKILL.md:233, 314-399).
|
||||
@@ -0,0 +1,241 @@
|
||||
---
|
||||
title: "Persistent discovery topic queue: five interlocking design conventions"
|
||||
date: 2026-07-20
|
||||
category: architecture-patterns
|
||||
module: discovery-topic-queue
|
||||
problem_type: architecture_pattern
|
||||
component: database
|
||||
severity: high
|
||||
applies_when:
|
||||
- "Adding default-on local persistence (SQLite, JSON state) hooked onto the end of an expensive pipeline run"
|
||||
- "Building a fuzzy identity layer over LLM-named entities whose names drift across runs"
|
||||
- "Reading a feature toggle in an engine where .env-file values only reach code through env.get_config's keys allowlist"
|
||||
- "Recording per-item state in a loop where a later item could fuzzy-match a row written earlier in the same run"
|
||||
- "Persisting user-set status (covered, dismissed, read) that must survive entity renames"
|
||||
tags:
|
||||
- "discovery-topic-queue"
|
||||
- "fuzzy-matching"
|
||||
- "sqlite-persistence"
|
||||
- "env-allowlist-opt-out"
|
||||
- "two-phase-write"
|
||||
- "covered-status-inheritance"
|
||||
- "guarded-write-hook"
|
||||
- "scoped-db"
|
||||
- "llm-naming-drift"
|
||||
related_components:
|
||||
- "skills/last30days/scripts/store.py"
|
||||
- "skills/last30days/scripts/last30days.py"
|
||||
- "skills/last30days/scripts/lib/env.py"
|
||||
- "tests/test_store.py"
|
||||
- "tests/test_discover_mode.py"
|
||||
---
|
||||
|
||||
|
||||
# Persistent discovery topic queue: five interlocking design conventions
|
||||
|
||||
## Context
|
||||
|
||||
PR #852 shipped a persistent topic queue for `/last30days discover`: every real
|
||||
discovery run records which topics it surfaced into a `discovery_topics` table in
|
||||
research.db, so the podcast/X-article pipeline remembers what it has already seen
|
||||
("surfaced 3rd time") and what the user already produced content for ("marked
|
||||
covered"). This is the design record for that queue - five conventions that were
|
||||
each load-bearing in review, two of them caught as real bugs (one P0). The
|
||||
seed-source corroboration change that landed in the same PR is documented
|
||||
separately in
|
||||
`docs/solutions/design-patterns/ranked-output-confidence-floor-honest-empty-state.md`
|
||||
(section 2b); this doc does not cover it.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Default-on, disabled only via the config allowlist - never bare os.environ
|
||||
|
||||
The queue records every real (non-mock) run by default; the literal value `off`
|
||||
disables it. The knob is registered in `env.get_config`'s keys allowlist
|
||||
(`skills/last30days/scripts/lib/env.py:482`):
|
||||
|
||||
```python
|
||||
# Discovery topic queue (podcast/X-article pipeline memory). Default
|
||||
# ON; the literal value "off" disables queue writes and annotations.
|
||||
('LAST30DAYS_DISCOVERY_QUEUE', None),
|
||||
```
|
||||
|
||||
and read from the resolved config dict, never `os.environ`
|
||||
(`skills/last30days/scripts/last30days.py:1312-1314`):
|
||||
|
||||
```python
|
||||
queue_setting = str(config.get("LAST30DAYS_DISCOVERY_QUEUE") or "").strip().lower()
|
||||
if queue_setting == "off" or not report.topics:
|
||||
return report
|
||||
```
|
||||
|
||||
WHY: `.env`-file users' values only reach the engine through the `get_config`
|
||||
allowlist merge - a bare `os.environ` read silently ignores them, a documented
|
||||
invisible-failure class in this repo. Scoped runs (`--save-dir`) write the scoped
|
||||
research.db via `store.scoped_db(_scoped_store_db(args))`
|
||||
(`last30days.py:432-437`, `store.py:41-53`), never the global one; `--mock` runs
|
||||
stay 100% side-effect-free (`last30days.py:1505`).
|
||||
|
||||
### 2. Annotate-only fuzzy matching - a match stamps context, it never merges rows
|
||||
|
||||
`store.match_discovery_topic` tries exact normalized-name match first, then the
|
||||
best entity-overlap candidate - the better of full `entity_key` token overlap and
|
||||
anchor-token overlap - at a conservative floor
|
||||
(`skills/last30days/scripts/store.py:810`, `898-938`):
|
||||
|
||||
```python
|
||||
DISCOVERY_QUEUE_OVERLAP_THRESHOLD = 0.6
|
||||
...
|
||||
if best is not None and best_overlap >= DISCOVERY_QUEUE_OVERLAP_THRESHOLD:
|
||||
return dict(best)
|
||||
```
|
||||
|
||||
A fuzzy match only annotates the rendered card - the `Pipeline: surfaced Nth
|
||||
time, marked covered` line (`skills/last30days/scripts/lib/render.py:153-168`) -
|
||||
and never merges or rewrites queue rows (`store.py:806-809`, `906-907`).
|
||||
|
||||
WHY: with annotate-only semantics a false-positive match costs one noisy line on
|
||||
one card; a false merge would silently collapse two distinct stories into one
|
||||
row and hide one of them forever. The threshold is tunable precisely because
|
||||
mislabeling is recoverable and data loss is not.
|
||||
|
||||
### 3. Two-phase hook: match ALL topics before recording ANY
|
||||
|
||||
`_annotate_and_record_discovery_queue` computes priors for every topic first,
|
||||
then records surfacings, inside one `store.scoped_db` block
|
||||
(`skills/last30days/scripts/last30days.py:1323-1345`):
|
||||
|
||||
```python
|
||||
with store.scoped_db(_scoped_store_db(args)):
|
||||
store.init_db()
|
||||
# Phase 1: match EVERY topic before recording ANY. Interleaving
|
||||
# match+record in one loop lets topic N fuzzy-match a same-anchor
|
||||
# sibling row this very run recorded seconds earlier, falsely
|
||||
# annotating a first-ever topic as "surfaced 2nd time".
|
||||
priors = [store.match_discovery_topic(topic.name) for topic in report.topics]
|
||||
# Phase 2: record this run's surfacings. ...
|
||||
for topic, prior in zip(report.topics, priors):
|
||||
```
|
||||
|
||||
WHY: one report often contains same-anchor siblings ("Gemma 4 chat templates" /
|
||||
"Gemma 4 tool calling fixes"). Interleaved match+record lets topic N fuzzy-match
|
||||
the row topic N-1 wrote seconds earlier, falsely annotating a first-ever topic
|
||||
as a repeat. Caught in review; regression-tested.
|
||||
|
||||
### 4. Covered inheritance: fresh rows born covered, existing rows never mutated
|
||||
|
||||
`record_discovery_surfacing(inherit_covered_at=...)` makes a fresh row start in
|
||||
`covered` status when its fuzzy-matched prior is covered; the `ON CONFLICT`
|
||||
update path deliberately never touches `status`/`covered_at`
|
||||
(`skills/last30days/scripts/store.py:842-895`):
|
||||
|
||||
```python
|
||||
status = "covered" if inherit_covered_at else "surfaced"
|
||||
...
|
||||
ON CONFLICT(normalized_name) DO UPDATE SET
|
||||
surface_count = surface_count + 1,
|
||||
last_surfaced = excluded.last_surfaced,
|
||||
last_run_ref = excluded.last_run_ref,
|
||||
domain = CASE WHEN excluded.domain <> '' THEN excluded.domain ELSE domain END
|
||||
```
|
||||
|
||||
The caller passes it when a topic's prior is covered
|
||||
(`last30days.py:1334-1344`). Locked by the flip-flop regression test
|
||||
`test_covered_status_survives_judge_rename_across_runs`
|
||||
(`tests/test_store.py:1082-1101`) and by
|
||||
`tests/test_store.py:1060-1079` (ON CONFLICT ignores `inherit_covered_at`).
|
||||
|
||||
WHY: the LLM judge renames the same story across runs; without inheritance a
|
||||
rename forks a fresh uncovered row and the user's covered mark silently
|
||||
evaporates. Without the never-mutate rule, a stale inherit could flip a row the
|
||||
user just changed.
|
||||
|
||||
### 5. Guarded, synchronous end-of-run write - never crash a finished pipeline
|
||||
|
||||
The hook call in `_run_discover` is wrapped so a broken queue db degrades to a
|
||||
stderr warning and an unannotated report
|
||||
(`skills/last30days/scripts/last30days.py:1505-1515`):
|
||||
|
||||
```python
|
||||
if not args.mock:
|
||||
try:
|
||||
report = _annotate_and_record_discovery_queue(report, args, config)
|
||||
except (sqlite3.Error, OSError) as exc:
|
||||
# A broken queue db (locked, read-only dir, corrupt) must never
|
||||
# destroy a finished multi-minute pipeline run: warn and render
|
||||
# the report without queue annotations (fields keep defaults).
|
||||
sys.stderr.write(
|
||||
f"[last30days] Warning: discovery queue unavailable ({exc}); "
|
||||
"continuing without queue annotations.\n"
|
||||
)
|
||||
```
|
||||
|
||||
WHY: unguarded, a locked/read-only/corrupt research.db raises AFTER the
|
||||
multi-minute research pipeline finished and discards all of its output - the PR
|
||||
#852 code review's P0, empirically reproduced. The write also runs synchronously
|
||||
after the pipeline returns (`last30days.py:1308-1310` docstring): it touches
|
||||
disk, so the abandon-on-timeout daemon-thread pattern is forbidden here (see
|
||||
`docs/solutions/logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md`).
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Ranked by blast radius when a convention is violated:
|
||||
|
||||
- Unguarded end-of-run write (5): the whole run's output is destroyed by a
|
||||
bookkeeping failure, and only in degraded environments (locked db, read-only
|
||||
dir), so it ships green and detonates on exactly the machines you cannot see.
|
||||
This was the review's P0.
|
||||
- Interleaved match+record (3): the queue's core promise ("first time you've
|
||||
seen this") is wrong on day one - a first-ever topic gets annotated "surfaced
|
||||
2nd time" by its same-run sibling, and no cross-run test catches it because
|
||||
the corruption happens inside a single run.
|
||||
- Bare os.environ read (1): `.env`-file users cannot turn the queue off; the
|
||||
toggle works in the maintainer's shell and fails invisibly for everyone
|
||||
configuring via file.
|
||||
- Merging on fuzzy match (2): a 0.6-overlap false positive stops being one
|
||||
noisy line and becomes a hidden story - unrecoverable data loss from a
|
||||
heuristic.
|
||||
- Mutating rows or skipping inheritance (4): user covered marks flip-flop with
|
||||
judge naming drift, so the queue re-pitches stories the user already produced,
|
||||
which is the exact failure the queue exists to prevent.
|
||||
|
||||
## When to Apply
|
||||
|
||||
- Any default-on local persistence bolted onto the end of an expensive pipeline:
|
||||
the write must be guarded (degrade to a warning) and synchronous if it touches
|
||||
disk.
|
||||
- Any fuzzy identity layer over LLM-named entities: keep matching annotate-only,
|
||||
batch all matches before any writes in a run, and inherit user-set status onto
|
||||
fresh rows instead of mutating existing ones.
|
||||
- Any new engine toggle in this repo: register it in `env.get_config`'s keys
|
||||
allowlist and read it from the config dict, never bare `os.environ`.
|
||||
|
||||
## Examples
|
||||
|
||||
Covered flip-flop, the archetype 3-run scenario (mirrors
|
||||
`tests/test_store.py:1082-1101`):
|
||||
|
||||
1. Run 1 surfaces "Gemma 4 chat templates"; the user records an episode and
|
||||
runs `queue cover "Gemma 4 chat templates"` (row status: covered).
|
||||
2. Run 2's judge names the same story "Gemma 4 template fixes". Exact match
|
||||
misses; fuzzy match (anchor overlap `gemma`/`4` at >= 0.6) finds the covered
|
||||
prior, so the new row is recorded born covered and the card renders
|
||||
`Pipeline: surfaced 2nd time, marked covered` instead of pitching it fresh.
|
||||
3. Run 3 resurfaces "Gemma 4 template fixes"; it exact-matches its own covered
|
||||
row (`covered_at` still the run-1 date). Without convention 4, run 2 would
|
||||
have forked an uncovered row and run 3 would re-pitch a story the user
|
||||
already covered.
|
||||
|
||||
Queue failure behavior: with research.db locked by another process, a discovery
|
||||
run still prints the full rendered report; stderr shows
|
||||
`[last30days] Warning: discovery queue unavailable (database is locked);
|
||||
continuing without queue annotations.` and the cards simply lack Pipeline lines.
|
||||
|
||||
## Related
|
||||
|
||||
- PR #852 - judged topic names, junk gate, angles, topic queue (this design).
|
||||
- `docs/solutions/design-patterns/ranked-output-confidence-floor-honest-empty-state.md`
|
||||
section 2b - the seed-source corroboration rule from the same PR (not covered
|
||||
here).
|
||||
- `docs/solutions/logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md`
|
||||
- why abandon-on-timeout daemon threads are forbidden for disk writers.
|
||||
@@ -0,0 +1,185 @@
|
||||
---
|
||||
title: "Optional-value argparse flags: dispatch on 'is not None', never truthiness"
|
||||
date: 2026-07-12
|
||||
category: conventions
|
||||
module: last30days-cli
|
||||
problem_type: convention
|
||||
component: tooling
|
||||
severity: medium
|
||||
applies_when:
|
||||
- "Retrofitting an optional value onto an existing flag (nargs='?' + const), where old dispatch sites were written for a two-state flag"
|
||||
- "Any flag or config key where a falsy value ('', 0, []) is a meaningful present-value distinct from absence"
|
||||
- "Dependent/modifier flags whose behavior only applies when an anchor flag is present"
|
||||
tags:
|
||||
- argparse
|
||||
- nargs-optional
|
||||
- truthiness
|
||||
- cli-flags
|
||||
- dispatch
|
||||
- dependent-flags
|
||||
- silent-failure
|
||||
- sentinel-values
|
||||
related_components:
|
||||
- testing_framework
|
||||
---
|
||||
|
||||
# Optional-value argparse flags: dispatch on `is not None`, never truthiness
|
||||
|
||||
## Context
|
||||
|
||||
PR #816 retrofitted an optional value onto the `--discover` flag in `skills/last30days/scripts/last30days.py`:
|
||||
|
||||
```python
|
||||
parser.add_argument(
|
||||
"--discover",
|
||||
metavar="DOMAIN",
|
||||
nargs="?",
|
||||
const="",
|
||||
default=None,
|
||||
help=(
|
||||
"Sweep river listings and rank the topics accelerating in a domain; "
|
||||
"each survivor gets a full research pass. Bare --discover (no domain) "
|
||||
"runs global trending across every feed's hot list"
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
With `nargs="?"` plus `const=""` plus `default=None`, the flag is deliberately three-state:
|
||||
|
||||
- flag absent -> `args.discover is None` -> normal research run
|
||||
- bare `--discover` -> `args.discover == ""` -> global trending sweep (empty domain)
|
||||
- `--discover X` -> `args.discover == "X"` -> domain-scoped discovery
|
||||
|
||||
The near-miss: the pre-existing dispatch was `if args.discover:`. Under Python truthiness, `""` and `None` are both falsy, so bare `--discover` would have been indistinguishable from no flag at all. The headline new mode of the PR (global trending) would simply never fire - the run would silently route into the normal-research path with no error, no warning, and no failing test unless a test exercised the bare form specifically. This was caught during development and the dispatch was changed to key on flag presence.
|
||||
|
||||
A second, related trap was caught in PR review (Greptile P2): the dependent flag `--discover-shallow` was accepted without `--discover` and silently no-opped into a full research run - the user asked for a fast, thin sweep and got a slow, full one. It was fixed with an explicit guard that errors loudly (exit 2).
|
||||
|
||||
## Guidance
|
||||
|
||||
1. With `nargs="?"` + `const`, the None/const/value trichotomy IS the contract: absent = `None`, bare flag = the `const` value, valued = the user's string. Dispatch on `args.flag is not None` (flag present), never on truthiness. When retrofitting optionality onto an existing flag, grep EVERY existing reference to `args.<flag>` - the old references were written when the flag was two-state and any `if args.flag:` among them is now a latent misroute.
|
||||
|
||||
2. Put a comment at the dispatch site explaining why it uses `is not None`. `if args.flag:` looks like the idiomatic form, and a future "simplification" pass will happily rewrite the correct check back into the bug. The repo's dispatch carries exactly this comment (`skills/last30days/scripts/last30days.py`):
|
||||
|
||||
```python
|
||||
# Bare --discover (no domain) is global trending, so the dispatch keys on
|
||||
# "flag present" (is not None), never on the domain string's truthiness.
|
||||
if args.discover is not None:
|
||||
```
|
||||
|
||||
3. Dependent/modifier flags (`--x-modifier` that only applies with `--x`) must error loudly when their anchor flag is absent - never silently no-op. A silent no-op means the user asked for one mode and got another with zero feedback. Reject with a clear message and a nonzero exit:
|
||||
|
||||
```python
|
||||
if args.discover_shallow:
|
||||
# Without --discover this flag would silently no-op into a full
|
||||
# research run - reject it instead of ignoring the requested mode.
|
||||
sys.stderr.write(
|
||||
"[last30days] --discover-shallow only applies to --discover runs; "
|
||||
"add --discover [domain] or drop the flag.\n"
|
||||
)
|
||||
return 2
|
||||
```
|
||||
|
||||
4. Pin both behaviors with CLI-level subprocess tests. Unit tests of the parser alone would not have caught the misroute, because parsing was correct - the bug lived in dispatch. The tests must run the real entry point end to end: one asserting the bare form reaches the new mode, one asserting the orphaned dependent flag is rejected (see Examples).
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The failure mode is silent misrouting, which is the worst kind: the feature ships, `--help` documents the bare form, and every invocation of it quietly runs the wrong mode. There is no exception, no error message, no test failure - the output is a plausible-looking result from the wrong pipeline. Nothing surfaces the bug unless a test (or an alert user) exercises the bare form specifically and checks which mode actually ran. The same is true of the dependent-flag no-op: `--discover-shallow` without `--discover` produced a valid full research run, just not the one the user asked for.
|
||||
|
||||
## When to Apply
|
||||
|
||||
- Retrofitting an optional value onto an existing flag (`action="store_true"` or a plain valued option becoming `nargs="?"`): audit every dispatch site that reads the flag.
|
||||
- Any flag where a falsy value (`""`, `0`, `[]`) is a MEANINGFUL present-value distinct from absence - the sentinel-vs-truthiness distinction applies beyond argparse (env vars, config keys, JSON fields).
|
||||
- Dependent/modifier flags whose behavior only applies when an anchor flag is present.
|
||||
|
||||
## Examples
|
||||
|
||||
Before (the near-miss - conflates bare flag with no flag):
|
||||
|
||||
```python
|
||||
if args.discover: # '' and None are both falsy: bare --discover falls through
|
||||
return _run_discover(args, config)
|
||||
```
|
||||
|
||||
After (`skills/last30days/scripts/last30days.py`, with the drift-guard comment):
|
||||
|
||||
```python
|
||||
# Bare --discover (no domain) is global trending, so the dispatch keys on
|
||||
# "flag present" (is not None), never on the domain string's truthiness.
|
||||
if args.discover is not None:
|
||||
if topic:
|
||||
sys.stderr.write(
|
||||
"[last30days] --discover supplies the domain and cannot be combined "
|
||||
"with a positional topic.\n"
|
||||
)
|
||||
return 2
|
||||
if args.drill:
|
||||
sys.stderr.write("[last30days] --discover and --drill are mutually exclusive.\n")
|
||||
return 2
|
||||
return _run_discover(args, config)
|
||||
```
|
||||
|
||||
The dependent-flag guard immediately below the dispatch:
|
||||
|
||||
```python
|
||||
if args.discover_shallow:
|
||||
# Without --discover this flag would silently no-op into a full
|
||||
# research run - reject it instead of ignoring the requested mode.
|
||||
sys.stderr.write(
|
||||
"[last30days] --discover-shallow only applies to --discover runs; "
|
||||
"add --discover [domain] or drop the flag.\n"
|
||||
)
|
||||
return 2
|
||||
```
|
||||
|
||||
The two pinning tests in `tests/test_discover_mode.py`, both running the real CLI via subprocess:
|
||||
|
||||
```python
|
||||
def test_discovery_cli_bare_discover_is_global_trending():
|
||||
"""Bare --discover (no domain) must run global trending, not error."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"skills/last30days/scripts/last30days.py",
|
||||
"--discover",
|
||||
"--mock",
|
||||
"--emit=json",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["kind"] == "discovery"
|
||||
assert payload["domain"] == ""
|
||||
assert payload["outcome"] in {"ok", "nothing-solid"}
|
||||
|
||||
|
||||
def test_discovery_cli_rejects_shallow_without_discover():
|
||||
"""--discover-shallow on a normal topic run must error, not silently no-op
|
||||
into a full research pass (P2 from PR #816 review)."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"skills/last30days/scripts/last30days.py",
|
||||
"AI agents",
|
||||
"--discover-shallow",
|
||||
"--mock",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 2
|
||||
assert "--discover-shallow only applies to --discover runs" in result.stderr
|
||||
```
|
||||
|
||||
The first test asserts not just exit 0 but that the discovery pipeline actually ran (`payload["kind"] == "discovery"`, `payload["domain"] == ""`) - the exact property the truthiness bug would have violated. Source: PR #816 (last30days-skill).
|
||||
|
||||
## Related
|
||||
|
||||
- [Ranked-output confidence floor + honest empty state](../design-patterns/ranked-output-confidence-floor-honest-empty-state.md) - sibling lesson from the same PR #816 discover rebuild (ranking quality).
|
||||
- [Non-daemon executor threads defeat wall-clock budgets](../logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md) - sibling lesson from PR #816, same lesson class: a discover-mode defect that result-oriented unit tests structurally cannot catch (process lifetime there, bare-flag vs flag-absent conflation here).
|
||||
- [PR #816](https://github.com/mvanhorn/last30days-skill/pull/816) - the discovery rebuild that introduced the three-state `--discover` flag (released v3.14.0).
|
||||
@@ -0,0 +1,231 @@
|
||||
---
|
||||
title: "Ranked-output features need an explicit confidence floor with an honest empty state"
|
||||
date: 2026-07-12
|
||||
last_updated: 2026-07-20
|
||||
category: design-patterns
|
||||
module: discover-trending
|
||||
problem_type: design_pattern
|
||||
component: tooling
|
||||
severity: medium
|
||||
applies_when:
|
||||
- "Any feature that ranks and displays top-N results from variable-quality inputs (search, trending, recommendations, discovery)"
|
||||
- "Quiet or over-broad query domains where feeds return thin or noisy data"
|
||||
- "A gate measures corroboration or independence downstream of a stage of the same pipeline that amplifies that signal (enrichment, fan-out, retrieval expansion)"
|
||||
symptoms:
|
||||
- "Top-N ranker emits near-zero-engagement items (e.g., five 1-like tweets) as a trend list because top-N has no notion of 'none of this is good enough'"
|
||||
resolution_type: code_fix
|
||||
tags:
|
||||
- ranking
|
||||
- confidence-floor
|
||||
- empty-state
|
||||
- top-n
|
||||
- discovery
|
||||
- trending
|
||||
- signal-quality
|
||||
- corroboration
|
||||
- "seed-sources"
|
||||
- "junk-shape"
|
||||
- "source-independence"
|
||||
related_components:
|
||||
- "skills/last30days/scripts/lib/rerank.py"
|
||||
- "skills/last30days/scripts/lib/pipeline.py"
|
||||
- "tests/test_discover_floor.py"
|
||||
---
|
||||
|
||||
# Ranked-output features need an explicit confidence floor with an honest empty state
|
||||
|
||||
## Context
|
||||
|
||||
The `--discover` trending feature sweeps listing feeds (r/all, Hacker News front page, Digg, X), clusters what it finds into candidate topics, and emits the top topics by an engagement-velocity score. The selection was purely relative: `run_discover()` in `skills/last30days/scripts/lib/pipeline.py` computed `topic_limit = max(5, min(10, limit))` and took the top N clusters by score, regardless of whether the Nth-best (or even the 1st-best) was any good.
|
||||
|
||||
On quiet or overly broad domains the feeds return thin, noisy data, and a relative ranker has no way to say so. The named failure (2026-07-12): `/last30days --discover "sports"` returned five single-source tweets with 1 like each - a Wii Sports nostalgia thread, a kids-travel-sports burnout post, a manga review that mentioned sports - presented with straight-faced ranks 1 through 5 as a "trend list". Every stage worked as designed. The sweep ran, the clusterer clustered, the scorer scored. The problem is structural: a top-N ranker ranks noise against noise when inputs are thin. Relative ranking cannot express "nothing here is good enough to show a user"; that requires an absolute gate the pipeline did not have.
|
||||
|
||||
## Guidance
|
||||
|
||||
The pattern shipped in PR #816 (merged, v3.14.0) has four parts. Apply all four when building any ranked-output surface.
|
||||
|
||||
### 1. Put an absolute floor in front of the relative ranker
|
||||
|
||||
Before a candidate is allowed to compete on relative score, it must clear an absolute evidence bar. The floor lives in `skills/last30days/scripts/lib/rerank.py`:
|
||||
|
||||
```python
|
||||
FLOOR_MIN_ENGAGEMENT = 25.0
|
||||
FLOOR_MIN_SOURCES = 2
|
||||
FLOOR_SINGLE_SOURCE_ENGAGEMENT = 200.0
|
||||
|
||||
|
||||
def passes_discovery_floor(
|
||||
*,
|
||||
source_count: int,
|
||||
engagement_total: float,
|
||||
item_count: int,
|
||||
junk_shape: bool = False,
|
||||
seed_source_count: int | None = None,
|
||||
) -> bool:
|
||||
"""Whether a discovery topic's evidence is strong enough to show a user.
|
||||
|
||||
Below this floor the honest output is "nothing solid this window", not a
|
||||
ranked list of whatever survived the sweep.
|
||||
"""
|
||||
if item_count <= 0 or engagement_total < FLOOR_MIN_ENGAGEMENT:
|
||||
return False
|
||||
if junk_shape:
|
||||
corroboration = seed_source_count if seed_source_count is not None else source_count
|
||||
return corroboration >= FLOOR_MIN_SOURCES
|
||||
if source_count >= FLOOR_MIN_SOURCES:
|
||||
return True
|
||||
return engagement_total >= FLOOR_SINGLE_SOURCE_ENGAGEMENT
|
||||
```
|
||||
|
||||
(The `junk_shape` / `seed_source_count` branch landed in PR #852 - see section 2b.) The first check is the junk gate: `FLOOR_MIN_ENGAGEMENT = 25.0` means a 1-like tweet can never rank, no matter how empty the field is. The floor is judged per topic inside `run_discover()` (`skills/last30days/scripts/lib/pipeline.py`), before the topic is appended and before `topic_limit` is consulted - sub-floor evidence never enters the ranked list at all.
|
||||
|
||||
### 2. Make the clearing criteria composite: corroboration OR a genuinely strong spike
|
||||
|
||||
A single threshold is either too strict (kills real single-source stories) or too loose (lets corroborated-but-tiny noise through). The floor uses two independent ways to clear, after the junk gate:
|
||||
|
||||
- Cross-source corroboration: appearing on `FLOOR_MIN_SOURCES = 2` or more independent feeds clears with only modest engagement. Two feeds independently surfacing the same story is signal in itself.
|
||||
- A strong single-source spike: `FLOOR_SINGLE_SOURCE_ENGAGEMENT = 200.0`. A 1,600-point single-source HN thread is a real story; a 30-upvote single-source meme is not.
|
||||
|
||||
The regression tests in `tests/test_discover_floor.py` pin both edges of this policy directly (`test_passes_discovery_floor_policy`): `floor(source_count=2, engagement_total=30, item_count=2)` clears, `floor(source_count=1, engagement_total=100, item_count=3)` does not, `floor(source_count=1, engagement_total=1600, item_count=1)` does.
|
||||
|
||||
### 2b. Count corroboration on the layer your own pipeline does not amplify
|
||||
|
||||
PR #852 added a stricter path for junk-shaped topics (help-me posts, beginner asks, musings - flagged by the stage-1 judge or the `topic_shape` heuristics): they lose the single-source engagement bypass entirely (a 226-comment "help me choose" thread is a busy support thread, not a story) and must clear `FLOOR_MIN_SOURCES` via corroboration alone.
|
||||
|
||||
The subtle half of that change is WHICH source count the corroboration check reads. The original design counted sources in the topic's enriched corpus - and the adversarial code review proved that check would never bind: the enrichment stage deliberately fans every nominated topic out to Reddit, X, YouTube, and the web, so a single-subreddit junk thread enriches into 4-6 "sources" of mentions of itself. A gate reading the post-fan-out count is checking that enrichment works, not that the topic is corroborated. The shipped gate counts distinct sources among the nomination's own seed listing items - what the river sweep actually found - which enrichment cannot inflate (`skills/last30days/scripts/lib/pipeline.py`, floor call site):
|
||||
|
||||
```python
|
||||
junk_shape=nomination.junk_shape,
|
||||
# Junk corroboration counts distinct SEED listing sources, never
|
||||
# the enriched corpus - a successful enrichment pass is
|
||||
# multi-source for almost any topic, so it would never bind.
|
||||
seed_source_count=len({item.source for item in nomination.items}),
|
||||
```
|
||||
|
||||
The two archetypes, side by side:
|
||||
|
||||
| Topic | Seed listing sources | Enriched corpus sources | Enriched-count gate (never binds) | Seed-count gate (shipped) |
|
||||
|---|---|---|---|---|
|
||||
| Single-subreddit help-me thread (junk shape) | 1 | 4-6 | passes | fails |
|
||||
| Real story swept from Reddit AND Hacker News | 2 | 4-6 | passes | passes |
|
||||
|
||||
Generalized rule: when a gate requires corroboration or independence, measure it on the signal layer your own system does not amplify - corroboration is evidence only when the corroborating signals could have failed to appear. This applies to any "N independent confirmations" threshold downstream of your own search fan-out, enrichment, crawling, or retrieval expansion. It does NOT apply when the downstream layer is genuinely independent evidence your pipeline cannot manufacture (human review verdicts, third-party confirmations) - there, the enriched layer is exactly what to count.
|
||||
|
||||
Testing note: a unit test that feeds the gate's parameters directly cannot catch a never-binds design. At least one test must drive the full production path with the amplifier running and assert the gate still fires - `test_junk_corroboration_counts_seed_sources_not_enriched_corpus` in `tests/test_discover_floor.py` mocks enrichment to return a rich multi-source corpus and asserts the single-seed-source junk topic still fails, with the unit-level matrix in `test_passes_discovery_floor_junk_params` pinning that a high enriched `source_count` cannot rescue `seed_source_count=1`.
|
||||
|
||||
### 3. Make honest emptiness a first-class outcome, and name the nearest miss
|
||||
|
||||
When zero topics survive the floor, the pipeline does not error, does not pad, and does not lower the bar. `run_discover()` sets `outcome = "ok" if topics else "nothing-solid"` on the `DiscoveryReport`, and while filtering it remembers the highest-scoring sub-floor candidate as `weak_signal` so the empty result can still say what came closest:
|
||||
|
||||
```python
|
||||
if not rerank.passes_discovery_floor(
|
||||
source_count=len(sources),
|
||||
engagement_total=native_total,
|
||||
item_count=len(evidence_items),
|
||||
junk_shape=nomination.junk_shape,
|
||||
# Junk corroboration counts distinct SEED listing sources, never
|
||||
# the enriched corpus - a successful enrichment pass is
|
||||
# multi-source for almost any topic, so it would never bind.
|
||||
seed_source_count=len({item.source for item in nomination.items}),
|
||||
):
|
||||
# Sub-floor evidence never ranks; remember what came closest so a
|
||||
# nothing-solid brief can still name the strongest weak signal.
|
||||
# Junk-shaped failures are tracked separately: the brief prefers
|
||||
# the strongest NON-junk failure and names a junk one only when
|
||||
# every failure is junk-shaped (never empty when failures exist).
|
||||
if nomination.junk_shape:
|
||||
if junk_weak_signal is None or score > junk_weak_signal[0]:
|
||||
junk_weak_signal = (score, nomination.name)
|
||||
elif weak_signal is None or score > weak_signal[0]:
|
||||
weak_signal = (score, nomination.name)
|
||||
continue
|
||||
```
|
||||
|
||||
The renderer (`render_discovery()` in `skills/last30days/scripts/lib/render.py`) presents this as a deliberate answer, not a failure:
|
||||
|
||||
```python
|
||||
if report.outcome == "nothing-solid":
|
||||
lines.extend([
|
||||
"**Nothing solid this window.** No topic cleared the confidence "
|
||||
"floor - not enough cross-source confirmation or engagement to "
|
||||
"call anything a trend, and ranked noise would be worse than an "
|
||||
"honest empty result.",
|
||||
"",
|
||||
])
|
||||
if report.weak_signal:
|
||||
lines.extend([
|
||||
f"Closest weak signal: {report.weak_signal} (sub-floor; "
|
||||
"single-source or too little engagement).",
|
||||
"",
|
||||
])
|
||||
```
|
||||
|
||||
Naming the weak signal matters: it tells the user the sweep actually ran and looked at real data, and it gives them a thread to pull ("closest weak signal: X" often suggests the narrower query that would work). There is also a soft middle state - if some topics clear but fewer than five, `run_discover()` emits a warning ("Fewer than five topic clusters cleared the confidence floor this window") rather than padding the list to the minimum.
|
||||
|
||||
### 4. Pin the failing corpus as a regression test
|
||||
|
||||
The exact junk corpus that produced the bad output is frozen in `test_junk_corpus_returns_nothing_solid_not_ranked_noise` in `tests/test_discover_floor.py`: five single-source 1-like tweets on the "sports" domain, asserting `report.topics == []`, `report.outcome == "nothing-solid"`, a non-None `weak_signal`, and a "confidence floor" warning. Sibling tests pin the other side so the floor cannot silently become a wall: a 1,084-point HN thread ranks (`test_strong_single_source_spike_clears_floor`), a mixed corpus keeps the strong story and drops the junk (`test_mixed_corpus_emits_only_floor_clearing_topics`), and enriched topics are judged on their enriched evidence, not their thin seed (`test_enriched_evidence_is_judged_not_seed_evidence`).
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Trust in a ranked surface is asymmetric. Users cannot see the corpus behind the ranking; they can only judge the output. One junk trend list - five 1-like tweets dressed up with ranks, velocity scores, and momentum labels - teaches a user that the feature is garbage, and they generalize instantly: if it confidently ranked noise once, every future list is suspect. The presentation makes it worse, because ranking machinery (rank numbers, scores, "why spiking" prose) signals confidence the evidence never had.
|
||||
|
||||
Honest emptiness does the opposite. "Nothing solid this window" plus a named weak signal tells the user three things at once: the sweep ran, the bar is real, and here is roughly where the signal petered out. That preserves trust in every future non-empty list (topics that do appear are known to have cleared an absolute bar - the renderer even badges cross-source topics with "confirmed across N sources") and it invites a narrower, more productive follow-up query instead of a shrug. The empty state is a feature, not an apology.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Any top-N surface over variable-quality inputs, where the input pool can be thin, noisy, or empty and the ranker will still dutifully order whatever it gets:
|
||||
|
||||
- Search and retrieval results ("no good matches" beats ten irrelevant hits)
|
||||
- Trending / discovery feeds (this case)
|
||||
- Recommendation lists ("nothing new worth recommending" beats recycled filler)
|
||||
- Leaderboards and "top contributors" style rankings over sparse activity
|
||||
- LLM-generated shortlists, digests, and "best of" summaries, where the model will fill N slots on request regardless of evidence quality
|
||||
|
||||
The tell that you need this pattern: the code computes `top N by score` with no branch that can produce an empty result from a non-empty input. If the only way to get an empty list is an empty corpus, the ranker cannot say "nothing here is good enough" - and someday the corpus will be five 1-like tweets.
|
||||
|
||||
Design notes when applying it:
|
||||
|
||||
- The floor must be absolute (engagement counts, source counts, item counts), not relative (percentile of the current pool). A relative floor degrades with the pool, which is exactly the failure being prevented.
|
||||
- Prefer composite clearing criteria over a single threshold: independent corroboration OR a strong single-signal spike. Tune the constants to the domain and keep them named and commented as deliberately tunable (see the comment block above the constants in `skills/last30days/scripts/lib/rerank.py`).
|
||||
- The empty state must name the nearest miss. A bare "no results" reads as breakage; "nothing cleared the bar, closest was X" reads as judgment.
|
||||
|
||||
## Examples
|
||||
|
||||
Before (v3.13.x behavior, reconstructed from the pinned regression corpus): `--discover "sports"` on a quiet window returned a ranked list built from this corpus -
|
||||
|
||||
```
|
||||
x: "Wii Sports nostalgia thread about sports" 1 like, single source
|
||||
x: "kids travel sports burnout post" 1 like, single source
|
||||
x: "motorsports vs stick and ball sports" 1 like, single source
|
||||
x: "midjourney skateboarder sports prompt" 1 like, single source
|
||||
x: "manga review mentioning sports matches" 1 like, single source
|
||||
```
|
||||
|
||||
- rendered as topics 1-5 with velocity scores, because `topic_limit = max(5, min(10, limit))` took the top N unconditionally.
|
||||
|
||||
After (v3.14.0, PR #816): the same corpus produces `outcome="nothing-solid"`, an empty `topics` list, and the renderer's explicit empty state ("**Nothing solid this window.** No topic cleared the confidence floor ... Closest weak signal: ... (sub-floor; single-source or too little engagement)."). Verified live in the implementing session: `--discover "sports"` returned nothing-solid, while global trending (no domain) returned six real cross-source topics with community quotes - the floor removed the junk without starving the healthy path.
|
||||
|
||||
The strong-corpus side, from `tests/test_discover_floor.py`: a single 1,084-point, 577-comment HN thread clears the floor alone via the single-source-spike branch (`engagement_total >= FLOOR_SINGLE_SOURCE_ENGAGEMENT`) and ranks as a real topic; a 25-upvote single-source Reddit post stays buried. The decision logic, in full, is small enough to quote:
|
||||
|
||||
```python
|
||||
if item_count <= 0 or engagement_total < FLOOR_MIN_ENGAGEMENT:
|
||||
return False
|
||||
if junk_shape:
|
||||
corroboration = seed_source_count if seed_source_count is not None else source_count
|
||||
return corroboration >= FLOOR_MIN_SOURCES
|
||||
if source_count >= FLOOR_MIN_SOURCES:
|
||||
return True
|
||||
return engagement_total >= FLOOR_SINGLE_SOURCE_ENGAGEMENT
|
||||
```
|
||||
|
||||
A handful of lines of gate, placed before the ranker, are the difference between a feature that fills five slots no matter what and one whose non-empty answers can be believed.
|
||||
|
||||
## Related
|
||||
|
||||
- [Entity grounding: full-phrase false demotion](../logic-errors/entity-grounding-full-phrase-false-demotion.md) - sibling ranking-quality fix in the same rerank module, opposite failure direction (false demotion of good signal vs. junk promotion). Together they bracket the two ways a ranker fails.
|
||||
- [Search-quality eval: manual by default](../architecture/search-quality-eval-manual-by-default-2026-05-10.md) - how to validate a ranking-threshold change like this floor: manual eval run plus deterministic regression tests, not CI-gated quality scoring.
|
||||
- [Non-daemon executor threads defeat wall-clock budgets](../logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md) - sibling learning from the same PR #816 rebuild: the process-lifetime half (enrichment budget enforcement) vs this doc's ranking-quality half.
|
||||
- [argparse optional-value flag dispatch](../conventions/argparse-optional-value-flag-dispatch-truthiness.md) - third lesson from the same PR #816: the CLI flag semantics that route into this feature.
|
||||
- [PR #816](https://github.com/mvanhorn/last30days-skill/pull/816) - the discovery rebuild that introduced `passes_discovery_floor()` and the nothing-solid empty state (released v3.14.0).
|
||||
- [PR #852](https://github.com/mvanhorn/last30days-skill/pull/852) - the discovery content pipeline that added the junk-shape branch and seed-source corroboration (section 2b).
|
||||
@@ -0,0 +1,80 @@
|
||||
---
|
||||
title: Digg NUX must match printing-press-library install paths and agent subprocess PATH
|
||||
date: 2026-06-17
|
||||
category: docs/solutions/integration-issues
|
||||
module: lib/setup_wizard
|
||||
problem_type: integration_issue
|
||||
component: development_workflow
|
||||
severity: medium
|
||||
symptoms:
|
||||
- Digg source silently off after first-run setup reports success on Hermes or OpenClaw
|
||||
- Users who already installed pp-digg via printing-press-library still see Digg missing from --diagnose available_sources
|
||||
- Setup wizard probed ~/go/bin while the catalog installer writes to ~/.local/bin (printing-press-library 0.1.16+)
|
||||
- OpenClaw setup --openclaw path skipped Digg install entirely
|
||||
root_cause: config_error
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- lib/pipeline
|
||||
- lib/digg
|
||||
- CONFIGURATION.md
|
||||
tags:
|
||||
- digg
|
||||
- setup-wizard
|
||||
- printing-press-library
|
||||
- agent-path
|
||||
- hermes
|
||||
- openclaw
|
||||
- nux
|
||||
- optional-cli-sources
|
||||
---
|
||||
|
||||
# Digg NUX must match printing-press-library install paths and agent subprocess PATH
|
||||
|
||||
## Problem
|
||||
|
||||
First-run setup auto-install for `digg-pp-cli` could report success while the engine still omitted Digg, especially on Hermes and OpenClaw where the agent subprocess PATH often excludes `$HOME/.local/bin`. The initial PR also used the deprecated `@mvanhorn/printing-press` package and probed legacy `~/go/bin` fallbacks instead of the current Printing Press default install dir.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- `--diagnose` `available_sources` lacks `digg` even though pp-digg or setup "installed" the CLI.
|
||||
- Hermes/OpenClaw users with a prior `npx @mvanhorn/printing-press-library install digg --cli-only` run hit false failures or false "now active" messages depending on probe logic.
|
||||
- OpenClaw `setup --openclaw` never attempted Digg install (desktop NUX only).
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
- **Treating "binary exists somewhere" as installed** — `pipeline.available_sources()` and `digg._is_available()` gate on `shutil.which("digg-pp-cli")` only. Probing `~/go/bin` without PATH visibility produced false positives.
|
||||
- **Assuming Hermes vs OpenClaw use different binary locations** — both harnesses use the same printing-press-library default (`$HOME/.local/bin`); only the focused pp-digg *skill* wiring differs.
|
||||
- **Using `@mvanhorn/printing-press`** — superseded by `@mvanhorn/printing-press-library`; install defaults moved from `$GOPATH/bin` to `$HOME/.local/bin` in npm 0.1.16.
|
||||
|
||||
## Solution
|
||||
|
||||
Align setup wizard with the catalog installer and the engine PATH gate:
|
||||
|
||||
1. **Pin installer:** `npx -y @mvanhorn/printing-press-library@0.1.16 install digg --cli-only` (`--cli-only` only — last30days embeds Digg as an engine source, not pp-digg skill).
|
||||
2. **Split outcomes:** `already_installed` / `installed` only when `shutil.which` resolves; `installed_off_path` when the binary exists under known dirs (`~/.local/bin`, legacy `~/go/bin`, Windows PrintingPress bin) but is not PATH-visible; surface `digg_path` and PATH-restart guidance in status text.
|
||||
3. **OpenClaw parity:** `run_openclaw_setup()` runs the same `_install_digg_cli()` and returns `digg_cli`, `digg_action`, optional `digg_path`.
|
||||
4. **Docs:** CONFIGURATION.md, SKILL.md Step 0, HERMES_SETUP.md, AGENTS.md rule for CLI-gated sources.
|
||||
|
||||
Key helper shape in `setup_wizard.py`:
|
||||
|
||||
```python
|
||||
def _digg_on_path() -> Optional[str]:
|
||||
return shutil.which(DIGG_CLI_BIN) # engine gate
|
||||
|
||||
def _digg_off_path_binary() -> Optional[str]:
|
||||
for candidate in _digg_bin_candidate_paths(): # ~/.local/bin first
|
||||
if candidate.is_file() and os.access(candidate, os.X_OK):
|
||||
return str(candidate)
|
||||
return None
|
||||
```
|
||||
|
||||
## Why This Works
|
||||
|
||||
The engine never reads "is pp-digg skill installed?" — every research run shells out to `digg-pp-cli` by name on PATH. Printing Press already installs to a managed user bin dir and warns when that dir is off PATH; last30days setup must mirror that contract instead of inventing a separate success definition. Detecting off-PATH binaries lets setup reuse prior pp-digg installs without lying about activation.
|
||||
|
||||
## Prevention
|
||||
|
||||
- When adding NUX auto-install for a CLI-gated source, match the upstream installer's default bin dir and pin the npm semver.
|
||||
- Success messaging must use the same probe as `available_sources()` (`shutil.which`), with a separate off-PATH outcome when the binary exists on disk.
|
||||
- Cover Hermes/OpenClaw in tests with redirected `HOME` and mocked PATH; add OpenClaw JSON fields when server setup should mirror desktop NUX.
|
||||
- Search `docs/solutions/` for `digg`, `setup-wizard`, and `agent-path` before changing optional-source onboarding.
|
||||
@@ -0,0 +1,137 @@
|
||||
---
|
||||
title: "Wall-clock budget over ThreadPoolExecutor is illusory - non-daemon workers are joined at interpreter shutdown"
|
||||
category: logic-errors
|
||||
module: discover-enrichment
|
||||
date: 2026-07-12
|
||||
problem_type: logic_error
|
||||
component: background_job
|
||||
severity: high
|
||||
symptoms:
|
||||
- "CLI process stayed alive after the enrichment budget expired, even though the hung topic had already been dropped"
|
||||
- "as_completed(futures, timeout=budget) returned control on time, but one still-running sub-run kept the interpreter from exiting"
|
||||
- "shutdown(wait=False, cancel_futures=True) cancelled unstarted futures but could not stop an already-running worker thread"
|
||||
root_cause: thread_violation
|
||||
resolution_type: code_fix
|
||||
tags:
|
||||
- threadpoolexecutor
|
||||
- daemon-threads
|
||||
- wall-clock-budget
|
||||
- interpreter-shutdown
|
||||
- semaphore
|
||||
- concurrency
|
||||
- timeout
|
||||
- python
|
||||
related_components:
|
||||
- testing_framework
|
||||
---
|
||||
|
||||
# Wall-clock budget over ThreadPoolExecutor is illusory - non-daemon workers are joined at interpreter shutdown
|
||||
|
||||
## Problem
|
||||
|
||||
Discovery enrichment fans out one research sub-run per nominated topic under a wall-clock batch budget (`ENRICH_BUDGET_SECONDS`, `skills/last30days/scripts/lib/pipeline.py`), but the first implementation built the budget on `ThreadPoolExecutor` - whose worker threads are non-daemon and joined at interpreter shutdown - so the budget bounded the answer without bounding the process. One hung sub-run could keep the whole CLI alive indefinitely after its topic had already been downgraded to nomination-only.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- The CLI process stays alive past the enrichment budget - potentially indefinitely - whenever one sub-run hangs (e.g. a network fetch that stalls without tripping a timeout). The batch "finishes", results are assembled, but the process will not exit because Python joins the executor's non-daemon threads at interpreter shutdown.
|
||||
- Only visible under a genuinely hung worker. All unit tests passed: the slow-topic test observed exactly what `as_completed(timeout=...)` guarantees - the fast topic returned, the slow one was dropped from results - and the test process exited fine because the "slow" worker was merely sleeping a few seconds, not hung. The defect lives entirely in process-lifetime behavior that a result-oriented test never touches.
|
||||
- Caught by code review on PR #816 (Greptile P1 "Enrichment Budget Keeps Running") before release; shipped fixed in v3.14.0.
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
The first version of `enrich_nominations()`:
|
||||
|
||||
```python
|
||||
executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
futures = {executor.submit(_run_one, n): n for n in nominations}
|
||||
try:
|
||||
for future in as_completed(futures, timeout=budget_seconds):
|
||||
...collect result...
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
```
|
||||
|
||||
This looks like budget enforcement, and every knob in it does something real - just not the thing needed. Precisely why each one fails to stop a thread that is already running:
|
||||
|
||||
- `as_completed(futures, timeout=budget)` bounds only the consumer. When the timeout expires it raises `TimeoutError` in the collecting loop; it sends nothing to the workers. Python threads cannot be killed from outside, so a running sub-run keeps running.
|
||||
- `cancel_futures=True` cancels only futures still sitting in the executor's queue - futures whose callable has not started. A future in the RUNNING state is not cancellable (`Future.cancel()` returns False for it); the worker executing it is untouched.
|
||||
- `shutdown(wait=False)` merely skips joining the threads at shutdown-call time. It does not detach them. CPython's `ThreadPoolExecutor` creates its worker threads non-daemon and (since Python 3.9) registers a `threading._register_atexit` hook that joins every worker at interpreter shutdown. So even after `shutdown(wait=False)`, interpreter exit blocks until the hung worker returns - which for a stalled fetch may be never.
|
||||
|
||||
Net effect: the budget expired, the slow topic was correctly reported as nomination-only, and the process then sat there anyway, held open by the very thread the budget had supposedly abandoned.
|
||||
|
||||
## Solution
|
||||
|
||||
Replace the executor with plain daemon threads, a semaphore for the concurrency cap, a queue for results, and a monotonic deadline in the consumer. From `skills/last30days/scripts/lib/pipeline.py`, `enrich_nominations()`:
|
||||
|
||||
```python
|
||||
# Daemon threads + a semaphore instead of ThreadPoolExecutor: executor
|
||||
# threads are non-daemon and joined at interpreter shutdown, so one hung
|
||||
# sub-run could keep the whole process alive long after its topic was
|
||||
# downgraded to nomination-only. Daemon workers make the wall-clock budget
|
||||
# real - stragglers cannot delay process exit. Abandonment is safe because
|
||||
# internal_subrun passes write nothing to disk (no save, no library sync,
|
||||
# no store), and every fetch layer inside run() carries its own timeout.
|
||||
enriched: dict[str, EnrichedTopic] = {}
|
||||
results_queue: queue.Queue[tuple[Nomination, schema.Report | None, Exception | None]] = queue.Queue()
|
||||
slots = threading.Semaphore(max(1, max_workers))
|
||||
|
||||
def _worker(nomination: Nomination) -> None:
|
||||
with slots:
|
||||
try:
|
||||
results_queue.put((nomination, _run_one(nomination), None))
|
||||
except Exception as exc: # noqa: BLE001 - containment is the contract
|
||||
results_queue.put((nomination, None, exc))
|
||||
|
||||
for nomination in nominations:
|
||||
threading.Thread(
|
||||
target=_worker,
|
||||
args=(nomination,),
|
||||
name=f"discover-enrich-{nomination.name[:32]}",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
deadline = time.monotonic() + max(1.0, budget_seconds)
|
||||
pending = len(nominations)
|
||||
while pending and (remaining := deadline - time.monotonic()) > 0:
|
||||
try:
|
||||
nomination, report, exc = results_queue.get(timeout=min(remaining, 0.5))
|
||||
except queue.Empty:
|
||||
continue
|
||||
pending -= 1
|
||||
...record EnrichedTopic success or error...
|
||||
# Budget expired (or all done): unfinished topics fall through below as
|
||||
# nomination-only; their daemon workers are abandoned and cannot block exit.
|
||||
```
|
||||
|
||||
Topics still unfinished when the loop exits fall through with `error="enrichment budget exhausted"` and survive as nomination-only entries; the batch preserves nomination order and never raises. Defaults live beside the function: `ENRICH_MAX_WORKERS = 3`, `ENRICH_BUDGET_SECONDS = 240.0`.
|
||||
|
||||
Three tests in `tests/test_discover_enrich.py` pin the contract:
|
||||
|
||||
- `test_enrich_workers_are_daemon_threads` asserts `threading.current_thread().daemon` from inside every worker - the daemon property is tested directly, not inferred from process behavior.
|
||||
- `test_enrich_concurrency_capped_by_semaphore` runs 6 nominations with `max_workers=2` and asserts peak in-flight workers never exceeds 2.
|
||||
- `test_enrich_budget_expiry_drops_slow_topic_to_nomination_only` runs a fast and a 5-second topic under `budget_seconds=1.0` and asserts the fast one returns enriched while the slow one drops to nomination-only with a budget error.
|
||||
|
||||
## Why This Works
|
||||
|
||||
- Daemon threads are not joined at interpreter exit. CPython's shutdown sequence waits only for non-daemon threads; a daemon worker mid-fetch simply dies with the process. That is what makes the wall-clock budget real: expiry means the process can exit now, not "after the straggler finishes".
|
||||
- The monotonic deadline bounds the consumer independently of worker behavior. `results_queue.get(timeout=min(remaining, 0.5))` wakes at least twice a second to re-check the deadline, so the collecting loop exits within ~0.5s of budget expiry no matter what any worker is doing.
|
||||
- The semaphore preserves the executor's one useful property. `threading.Semaphore(max(1, max_workers))` acquired inside each worker caps in-flight sub-runs at `max_workers`, so upstream APIs see the same low parallelism as before; threads beyond the cap exist but block on the semaphore, costing almost nothing.
|
||||
- The write-free precondition is what makes abandonment safe, and it is documented in the code comment where the next editor will see it: enrichment sub-runs are `internal_subrun=True` passes that write nothing to disk - no save, no library sync, no store - and every fetch layer inside `run()` carries its own timeout. Killing such a worker at process exit can corrupt nothing. A worker that mutates shared state (files, databases, caches) must not be abandoned this way; it needs cooperative cancellation instead.
|
||||
|
||||
The general lesson: `as_completed(timeout=...)` plus `shutdown(wait=False, cancel_futures=True)` is answer-bounding, not process-bounding. If the requirement is "this batch may not extend the life of the process", non-daemon executor threads cannot deliver it (CPython behavior since 3.9, when executor threads moved from atexit-daemon handling to `threading._register_atexit` joining), and no combination of executor knobs changes that.
|
||||
|
||||
## Prevention
|
||||
|
||||
- Any "budget" or "timeout" over threaded work must state what happens to a RUNNING straggler. If the design doc or comment only says what happens to the result, the process-lifetime question is unanswered - and the default answer (non-daemon threads joined at exit) is usually wrong for a CLI.
|
||||
- Prefer explicit daemon threads for abandonable work. When stragglers are safe to drop, `threading.Thread(daemon=True)` + semaphore + queue + monotonic deadline is barely more code than an executor and actually enforces the budget. Reserve `ThreadPoolExecutor` for work you intend to wait for.
|
||||
- Require the write-free precondition in a comment next to the daemon flag. Daemon abandonment is only safe for workers that mutate no shared state and hold no resources needing cleanup. State the precondition where the code is (as `enrich_nominations()` does), so a future change that adds a disk write inside the worker trips over the warning.
|
||||
- Test daemon-ness explicitly. Process-hang bugs are invisible to result-oriented unit tests - the passing slow-topic test proved the wrong thing. Assert `threading.current_thread().daemon` inside the worker (see `test_enrich_workers_are_daemon_threads` in `tests/test_discover_enrich.py`); it is a one-line assertion that pins the property the budget depends on.
|
||||
- Per-request timeouts inside workers remain the first line of defense. Daemon abandonment is the backstop for the pathological case; every network call inside a worker should still carry its own timeout so hung workers are rare, not routine.
|
||||
- In review, treat `shutdown(wait=False, cancel_futures=True)` in a `finally` as a signal to ask the straggler question. It is the idiom people reach for when they want abandonment, and it does not provide it.
|
||||
|
||||
## Related Issues
|
||||
|
||||
- [Ranked-output confidence floor + honest empty state](../design-patterns/ranked-output-confidence-floor-honest-empty-state.md) - sibling learning from the same PR #816 discovery rebuild: the ranking-quality half vs this doc's process-lifetime half. Both live in `skills/last30days/scripts/lib/pipeline.py`.
|
||||
- [argparse optional-value flag dispatch](../conventions/argparse-optional-value-flag-dispatch-truthiness.md) - third lesson from the same PR #816: bare-flag vs flag-absent conflation, another defect class invisible to result-oriented tests.
|
||||
- [PR #816](https://github.com/mvanhorn/last30days-skill/pull/816) - the discovery rebuild that replaced the executor with daemon threads + semaphore + result queue + monotonic deadline in `enrich_nominations()` (released v3.14.0).
|
||||
- Note: `skills/last30days/scripts/lib/pipeline.py` still uses `ThreadPoolExecutor` at other call sites where work is genuinely waited for; the daemon-thread pattern was applied only to `enrich_nominations()`, whose stragglers are abandonable. Apply the straggler question, not the pattern, when touching those.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
title: Towncrier fragments + automated lockstep release PRs
|
||||
date: 2026-07-24
|
||||
category: docs/solutions/workflow-issues
|
||||
module: ci-release-engineering
|
||||
problem_type: workflow_issue
|
||||
component: release_workflow
|
||||
severity: medium
|
||||
applies_when:
|
||||
- multiple PRs edit CHANGELOG.md ## [Unreleased] and conflict on merge
|
||||
- a release must bump the same semver across skill, pyproject, and every plugin/marketplace manifest
|
||||
- agents (not humans) author most feature PRs and need a clear changelog rule
|
||||
symptoms:
|
||||
- Unreleased section merge conflicts on every release train
|
||||
- missed marketplace JSON version bumps when releasing by hand
|
||||
- agents invent release steps that drift from test_plugin_contract lockstep
|
||||
root_cause: missing_workflow_step
|
||||
resolution_type: workflow_change
|
||||
related_components:
|
||||
- development_workflow
|
||||
- documentation
|
||||
- github_actions
|
||||
tags:
|
||||
- changelog
|
||||
- towncrier
|
||||
- release-engineering
|
||||
- version-lockstep
|
||||
- agents
|
||||
- github-actions
|
||||
---
|
||||
|
||||
# Towncrier fragments + automated lockstep release PRs
|
||||
|
||||
## Context
|
||||
|
||||
Every feature PR used to edit `CHANGELOG.md` under `## [Unreleased]`, which produced constant merge conflicts. Separately, a correct release must bump the **same** semver across skill frontmatter + H1, `pyproject.toml`, `uv.lock`, Claude/Codex/Grok/Gemini plugin manifests, and both marketplace JSON files — enforced by `tests/test_plugin_contract.py`. Hand-rolled release PRs missed files; release-please would work only with a large `extra-files` surface and conventional-commit discipline that agent traffic does not reliably provide.
|
||||
|
||||
## Solution
|
||||
|
||||
1. **towncrier** — PRs add `changelog.d/<n>.<type>.md`; `CHANGELOG.md` is written only at release time.
|
||||
2. **`.github/scripts/prepare_release.py`** — runs `towncrier build` then bumps every lockstep path.
|
||||
3. **Actions → Prepare release** — opens the release PR; **Tag release** creates `vX.Y.Z` on merge; existing **Release** workflow attaches artifacts.
|
||||
4. **changelog-guard** — blocks non-release edits to `CHANGELOG.md` and version *strings*; requires a fragment (or `skip-changelog`) for engine/skill changes.
|
||||
5. **PR template** — changelog checklist, agent disclosure (AI review + security), and relationship disclosure for contributors tied to a vendor/product they are adding.
|
||||
|
||||
## Agent rules (short)
|
||||
|
||||
- Write fragments, not `CHANGELOG.md`.
|
||||
- Do not bump versions in feature PRs.
|
||||
- Cut releases via Prepare release, not by editing ten files.
|
||||
|
||||
## See also
|
||||
|
||||
- `AGENTS.md` § Changelog and releases
|
||||
- `changelog.d/README.md`
|
||||
- `tests/test_changelog_workflow.py`
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.3.2",
|
||||
"version": "3.21.0",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
|
||||
+197
-40
@@ -2,10 +2,25 @@
|
||||
set -euo pipefail
|
||||
|
||||
# Check last30days configuration status and show appropriate welcome message.
|
||||
# Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars
|
||||
# Priority for this status hook (mirrors lib/env.py):
|
||||
# process env > trusted .claude/last30days.env > ~/.config/last30days/.env > Keychain presence
|
||||
# Project-scoped config is loaded only when LAST30DAYS_TRUST_PROJECT_CONFIG is
|
||||
# truthy in the process environment or the global config file — never from the
|
||||
# project file itself (it cannot self-grant trust).
|
||||
|
||||
PROJECT_ENV=".claude/last30days.env"
|
||||
GLOBAL_ENV="$HOME/.config/last30days/.env"
|
||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||
GLOBAL_ENV="$LAST30DAYS_CONFIG_DIR/.env"
|
||||
else
|
||||
GLOBAL_ENV=""
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ensure LAST30DAYS_MEMORY_DIR exists for HTML-brief / raw-markdown saves.
|
||||
# SKILL.md and the engine default this via the same env-var fallback. Fresh
|
||||
# installs otherwise fail silently on first --emit=html run. See #395.
|
||||
mkdir -p "${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}" 2>/dev/null || true
|
||||
|
||||
# Helper: warn if file permissions are too open
|
||||
check_perms() {
|
||||
@@ -23,11 +38,29 @@ check_perms() {
|
||||
# every Linux session start and printed a false WARNING.
|
||||
perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
|
||||
if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
|
||||
echo "/last30days: WARNING — $file has permissions $perms (should be 600)."
|
||||
echo " Fix: chmod 600 $file"
|
||||
chmod 600 "$file" && echo "/last30days: WARNING — $file had permissions $perms — auto-fixed with chmod 600" || echo "/last30days: WARNING — $file has permissions $perms (should be 600). Fix: chmod 600 $file"
|
||||
fi
|
||||
}
|
||||
|
||||
trim_ws() {
|
||||
local s="$1"
|
||||
s="${s#"${s%%[![:space:]]*}"}"
|
||||
s="${s%"${s##*[![:space:]]}"}"
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
strip_outer_quotes() {
|
||||
local s="$1"
|
||||
if [[ ${#s} -ge 2 ]]; then
|
||||
if [[ "${s:0:1}" == '"' && "${s: -1}" == '"' ]]; then
|
||||
s="${s:1:${#s}-2}"
|
||||
elif [[ "${s:0:1}" == "'" && "${s: -1}" == "'" ]]; then
|
||||
s="${s:1:${#s}-2}"
|
||||
fi
|
||||
fi
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
# Load env file into variables for inspection (without exporting)
|
||||
load_env_vars() {
|
||||
local file="$1"
|
||||
@@ -36,8 +69,15 @@ load_env_vars() {
|
||||
# Skip comments, empty lines
|
||||
[[ "$key" =~ ^[[:space:]]*# ]] && continue
|
||||
[[ -z "$key" ]] && continue
|
||||
key=$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
value=$(echo "$value" | sed -e 's/^[[:space:]]*//;s/[[:space:]]*$//' -e 's/^["'\''"]//;s/["'\''"]$//')
|
||||
key="$(trim_ws "$key")"
|
||||
# Only plain identifiers may reach `printf -v`. printf -v uses assignment
|
||||
# semantics, so a key carrying an array subscript — e.g. `x[$(id)]` — has
|
||||
# that subscript arithmetic-evaluated, which runs the command inside it.
|
||||
# A project-scoped .claude/last30days.env is attacker-controlled as soon
|
||||
# as an untrusted repo is opened, so an unvalidated key here is arbitrary
|
||||
# code execution at session start.
|
||||
[[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
|
||||
value="$(strip_outer_quotes "$(trim_ws "$value")")"
|
||||
# Strip inline comments (# preceded by whitespace) to prevent
|
||||
# command substitution in backtick-containing comments
|
||||
value="${value%%[[:space:]]#*}"
|
||||
@@ -50,22 +90,108 @@ load_env_vars() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Determine which config file is active
|
||||
# Match lib/env.py::_truthy — process/global trust signal only.
|
||||
is_truthy() {
|
||||
local v
|
||||
v="$(trim_ws "$1")"
|
||||
case "$v" in
|
||||
1|[Tt][Rr][Uu][Ee]|[Yy][Ee][Ss]|[Oo][Nn]) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Project config cannot self-grant trust. Process env (including empty/0 deny)
|
||||
# wins when set; otherwise the global config file's trust flag is consulted.
|
||||
project_config_trusted() {
|
||||
if [[ "${LAST30DAYS_TRUST_PROJECT_CONFIG+x}" == "x" ]]; then
|
||||
is_truthy "$LAST30DAYS_TRUST_PROJECT_CONFIG"
|
||||
return $?
|
||||
fi
|
||||
is_truthy "${ENV_LAST30DAYS_TRUST_PROJECT_CONFIG:-}"
|
||||
}
|
||||
|
||||
# Mirror lib/env.py::_find_project_env: walk up from $PWD for
|
||||
# .claude/last30days.env, stopping at the git root, $HOME, or filesystem root.
|
||||
# Prints the absolute path on stdout when found; returns 1 when none.
|
||||
find_project_env() {
|
||||
local dir candidate parent
|
||||
dir="$PWD"
|
||||
while :; do
|
||||
candidate="${dir}/.claude/last30days.env"
|
||||
if [[ -f "$candidate" ]]; then
|
||||
printf '%s' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
# Stop at git root even if no project env was found there (matches env.py).
|
||||
if [[ -e "${dir}/.git" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "$dir" == "$HOME" ]]; then
|
||||
return 1
|
||||
fi
|
||||
parent="$(dirname "$dir")"
|
||||
if [[ "$parent" == "$dir" ]]; then
|
||||
return 1
|
||||
fi
|
||||
dir="$parent"
|
||||
done
|
||||
}
|
||||
|
||||
# Determine which config file(s) are active. Always load global first (when
|
||||
# present) so a trust signal there can unlock the project file — matching
|
||||
# lib/env.py, where the project file is never parsed before the trust check.
|
||||
CONFIG_FILE=""
|
||||
if [[ -f "$PROJECT_ENV" ]]; then
|
||||
CONFIG_FILE="$PROJECT_ENV"
|
||||
check_perms "$PROJECT_ENV"
|
||||
elif [[ -f "$GLOBAL_ENV" ]]; then
|
||||
if [[ -n "$GLOBAL_ENV" && -f "$GLOBAL_ENV" ]]; then
|
||||
CONFIG_FILE="$GLOBAL_ENV"
|
||||
check_perms "$GLOBAL_ENV"
|
||||
load_env_vars "$GLOBAL_ENV"
|
||||
fi
|
||||
|
||||
# Load config if found
|
||||
if [[ -n "$CONFIG_FILE" ]]; then
|
||||
load_env_vars "$CONFIG_FILE"
|
||||
PROJECT_ENV=""
|
||||
if project_config_trusted; then
|
||||
# `|| true` keeps set -e from aborting when no project env is in the walk.
|
||||
PROJECT_ENV="$(find_project_env)" || true
|
||||
fi
|
||||
if [[ -n "$PROJECT_ENV" && -f "$PROJECT_ENV" ]]; then
|
||||
CONFIG_FILE="$PROJECT_ENV"
|
||||
check_perms "$PROJECT_ENV"
|
||||
load_env_vars "$PROJECT_ENV"
|
||||
fi
|
||||
|
||||
# Check SETUP_COMPLETE (from file or env)
|
||||
# Load Keychain item presence for status checks without reading secret values.
|
||||
# Runtime credential resolution still happens in lib/env.py; this hook only
|
||||
# needs to avoid stale first-run/source-count messages.
|
||||
load_keychain_presence() {
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
Darwin*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
command -v security >/dev/null 2>&1 || return 0
|
||||
|
||||
local user key env_var current
|
||||
user="${USER:-}"
|
||||
if [[ -z "$user" ]]; then
|
||||
user="$(id -un 2>/dev/null || true)"
|
||||
fi
|
||||
[[ -n "$user" ]] || return 0
|
||||
|
||||
for key in SETUP_COMPLETE OPENAI_API_KEY SCRAPECREATORS_API_KEY AUTH_TOKEN CT0 XAI_API_KEY BSKY_HANDLE EXA_API_KEY; do
|
||||
env_var="ENV_${key}"
|
||||
current="${!env_var:-}"
|
||||
if [[ -z "$current" ]]; then
|
||||
current="${!key:-}"
|
||||
fi
|
||||
[[ -n "$current" ]] && continue
|
||||
if security find-generic-password -a "$user" -s "last30days-${key}" >/dev/null 2>&1; then
|
||||
printf -v "ENV_${key}" '%s' "keychain"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
load_keychain_presence
|
||||
|
||||
# Check SETUP_COMPLETE (from file, env, or Keychain presence)
|
||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||
|
||||
# Compute last-run summary line (if last-run.json exists)
|
||||
@@ -79,8 +205,11 @@ else
|
||||
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
||||
fi
|
||||
LAST_RUN_LINE=""
|
||||
# python3 -c, NOT a heredoc: bash 5.3 feeds heredocs to the child through a
|
||||
# pipe and can deadlock in heredoc_write inside command substitution, hanging
|
||||
# this hook forever at session start (observed on Homebrew bash 5.3.15).
|
||||
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 - <<'PY' 2>/dev/null || true
|
||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 -c '
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
@@ -101,31 +230,51 @@ try:
|
||||
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
||||
except Exception:
|
||||
pass
|
||||
PY
|
||||
)
|
||||
' 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
# Detect capability that doesn't need a config file: yt-dlp on PATH.
|
||||
# Done before the new-user early-exit so first-run users with yt-dlp
|
||||
# installed see YouTube is already available. See #394.
|
||||
HAS_YTDLP=""
|
||||
if command -v yt-dlp &>/dev/null; then
|
||||
HAS_YTDLP="yes"
|
||||
fi
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||
cat <<'EOF'
|
||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||
Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).
|
||||
|
||||
Reddit, Hacker News, and Polymarket work out of the box.
|
||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||
EOF
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${ENV_OPENAI_API_KEY:-${OPENAI_API_KEY:-}}" && -z "${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}" && -z "${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}" && -z "${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}" ]]; then
|
||||
# printf, NOT cat-with-heredoc: see the bash 5.3 heredoc deadlock note above.
|
||||
if [[ -n "$HAS_YTDLP" ]]; then
|
||||
# YouTube is already working via the on-system yt-dlp binary — don't list
|
||||
# it as something the wizard needs to unlock. See #394.
|
||||
printf '%s\n' \
|
||||
'/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.' \
|
||||
' Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).' \
|
||||
'' \
|
||||
'Reddit, Hacker News, Polymarket, and YouTube (yt-dlp detected) work out of the box.' \
|
||||
'The setup wizard can unlock X/Twitter and more.' \
|
||||
' Detected: yt-dlp is installed (YouTube transcripts ready, no setup needed).'
|
||||
else
|
||||
printf '%s\n' \
|
||||
'/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.' \
|
||||
' Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).' \
|
||||
'' \
|
||||
'Reddit, Hacker News, and Polymarket work out of the box.' \
|
||||
'The setup wizard can unlock X/Twitter, YouTube, and more.'
|
||||
fi
|
||||
if [[ -n "$LAST_RUN_LINE" ]]; then
|
||||
echo "$LAST_RUN_LINE"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Setup done but check for ScrapeCreators
|
||||
HAS_SCRAPECREATORS="${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}"
|
||||
HAS_X="${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}"
|
||||
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
|
||||
HAS_YTDLP=""
|
||||
if command -v yt-dlp &>/dev/null; then
|
||||
HAS_YTDLP="yes"
|
||||
HAS_X=""
|
||||
if [[ -n "${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}" && -n "${ENV_CT0:-${CT0:-}}" ]]; then
|
||||
HAS_X="yes"
|
||||
fi
|
||||
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
|
||||
HAS_BSKY="${ENV_BSKY_HANDLE:-${BSKY_HANDLE:-}}"
|
||||
HAS_EXA="${ENV_EXA_API_KEY:-${EXA_API_KEY:-}}"
|
||||
|
||||
@@ -147,16 +296,15 @@ if [[ -n "$HAS_BSKY" ]]; then
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
||||
# Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
|
||||
# whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
|
||||
# Normalise EXCLUDED by removing whitespace; case-insensitive matches below
|
||||
# mirror pipeline.py's .strip().lower() parsing without requiring sed/tr.
|
||||
SC_ADD=3
|
||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
||||
EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
|
||||
| sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
|
||||
if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
|
||||
EXCLUDED_NORM="${EXCLUDED//[[:space:]]/}"
|
||||
if [[ ",$EXCLUDED_NORM," == *",[Tt][Ii][Kk][Tt][Oo][Kk],"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
|
||||
if [[ ",$EXCLUDED_NORM," == *",[Ii][Nn][Ss][Tt][Aa][Gg][Rr][Aa][Mm],"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
||||
@@ -166,13 +314,22 @@ if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Fully configured — compact ready message
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
if [[ -n "$LAST_RUN_LINE" ]]; then
|
||||
echo "$LAST_RUN_LINE"
|
||||
fi
|
||||
else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
if [[ -n "$LAST_RUN_LINE" ]]; then
|
||||
echo "$LAST_RUN_LINE"
|
||||
fi
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
|
||||
# The branches above end with `[[ -n "$LAST_RUN_LINE" ]] && echo ...`. When
|
||||
# LAST_RUN_LINE is empty, that test returns 1 and is the script's last command,
|
||||
# leaking exit=1 to callers (e.g. SessionStart hook drivers) despite no error.
|
||||
exit 0
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# Mirror of skills/last30days/scripts/, populated by scripts/sync-engine.sh.
|
||||
# Source of truth lives in the Python skill; never commit the mirror.
|
||||
# Lives inside internal/engine/ because //go:embed cannot reach outside
|
||||
# its own package directory.
|
||||
internal/engine/vendored/*
|
||||
!internal/engine/vendored/.gitkeep
|
||||
|
||||
# Local build output: cross-compiled binaries and packaged .mcpb files.
|
||||
build/
|
||||
# Anchor to the mcp/ root so the cmd/last30days-pp-mcp/ package directory
|
||||
# is not also excluded (subdirs with the same name would otherwise match).
|
||||
/last30days-pp-mcp
|
||||
@@ -0,0 +1,36 @@
|
||||
# last30days-pp-mcp
|
||||
|
||||
Go MCP server that wraps the last30days Python engine for Claude Desktop. Packaged as a `.mcpb` bundle (drag-drop install into Claude Desktop).
|
||||
|
||||
The MCP server exposes a single `research` tool that mirrors the `/last30days <topic>` slash command available in Claude Code. At runtime the binary extracts the vendored Python engine into a per-user cache and shells out to `python3` to produce the synthesis input Claude renders.
|
||||
|
||||
## Architecture
|
||||
|
||||
- `cmd/last30days-pp-mcp/` - server entry point
|
||||
- `internal/engine/` - `embed.FS` of the Python engine + cache extractor + subprocess wrapper
|
||||
- `internal/tools/` - MCP tool handlers (currently `research`)
|
||||
- `internal/engine/vendored/` - mirror of `skills/last30days/scripts/`, generated by `scripts/sync-engine.sh` (gitignored). Lives inside the engine package because `//go:embed` cannot reach files outside its own package directory.
|
||||
- `manifest.json` - MCPB v0.3 manifest consumed by Claude Desktop and `printing-press bundle`
|
||||
|
||||
## Local build
|
||||
|
||||
```bash
|
||||
# Mirror the Python engine into vendored/.
|
||||
bash scripts/sync-engine.sh
|
||||
|
||||
# Build for the current host.
|
||||
go build -ldflags "-X main.Version=dev" -o build/last30days-pp-mcp ./cmd/last30days-pp-mcp
|
||||
|
||||
# Package as a .mcpb (requires the printing-press binary on PATH).
|
||||
printing-press bundle . --skip-build --binary build/last30days-pp-mcp
|
||||
```
|
||||
|
||||
The output `.mcpb` lands at `build/last30days-pp-mcp-<os>-<arch>.mcpb`. Drag it into Claude Desktop's Extensions panel to install.
|
||||
|
||||
## Runtime requirements
|
||||
|
||||
End users need Python 3.12+ on PATH. The bundle ships the engine source but relies on the host interpreter.
|
||||
|
||||
## Versioning
|
||||
|
||||
The MCPB `manifest.json` version is hand-bumped in the same PR that ships engine changes worth releasing. Release CI stamps the Go binary's `main.Version` from the tag.
|
||||
@@ -0,0 +1,39 @@
|
||||
// Package main is the entry point for the last30days MCP server bundled
|
||||
// as a .mcpb for Claude Desktop. The server registers a single research
|
||||
// tool (see internal/tools) and serves it over stdio. See mcp/README.md
|
||||
// for build and packaging instructions.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/tools"
|
||||
)
|
||||
|
||||
// Version is stamped at build time via -ldflags "-X main.Version=<tag>".
|
||||
// It namespaces the per-user cache directory in internal/engine so multiple
|
||||
// installed versions can coexist without clobbering each other.
|
||||
var Version = "dev"
|
||||
|
||||
const (
|
||||
serverName = "last30days"
|
||||
serverVersion = "1"
|
||||
)
|
||||
|
||||
func main() {
|
||||
s := server.NewMCPServer(
|
||||
serverName,
|
||||
serverVersion,
|
||||
server.WithToolCapabilities(false),
|
||||
)
|
||||
|
||||
tools.Register(s, tools.Config{Version: Version})
|
||||
|
||||
if err := server.ServeStdio(s); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "last30days-pp-mcp: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
module github.com/mvanhorn/last30days-skill/mcp
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/mark3labs/mcp-go v0.57.0
|
||||
|
||||
require (
|
||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/text v0.39.0 // indirect
|
||||
)
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
|
||||
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E=
|
||||
github.com/mark3labs/mcp-go v0.57.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
|
||||
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,26 @@
|
||||
// Package engine wraps the vendored Python last30days engine. The engine
|
||||
// is embedded at build time via //go:embed and extracted into a per-user
|
||||
// cache directory on first use, then invoked through python3 in a
|
||||
// subprocess. Consumers should call EnsureUserCache to materialize the
|
||||
// engine and Run to execute it.
|
||||
package engine
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// EngineSourceDir is the embed root inside the binary. scripts/sync-engine.sh
|
||||
// mirrors skills/last30days/scripts/ into this directory before each build.
|
||||
// The all: prefix preserves files starting with "." or "_" so the .gitkeep
|
||||
// anchor file survives - without it the embed would error before sync runs.
|
||||
//
|
||||
//go:embed all:vendored
|
||||
var vendored embed.FS
|
||||
|
||||
// EngineFS returns the embedded engine as a filesystem rooted at the
|
||||
// vendored/ directory contents (so callers see "last30days.py" at the
|
||||
// root, not "vendored/last30days.py").
|
||||
func EngineFS() (fs.FS, error) {
|
||||
return fs.Sub(vendored, "vendored")
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// SentinelFilename names the file Ensure writes inside the cache directory
|
||||
// after a successful extraction. Its contents are compared to the requested
|
||||
// version; a match short-circuits re-extraction on subsequent calls.
|
||||
const SentinelFilename = ".version"
|
||||
|
||||
// cacheSubdir namespaces our cache under the OS user cache directory so
|
||||
// multiple printing-press-style bundles can coexist.
|
||||
const cacheSubdir = "last30days-pp-mcp"
|
||||
|
||||
// CacheEnvOverride lets users redirect the cache directory when the default
|
||||
// OS cache location is read-only (locked-down corp images, ephemeral CI
|
||||
// containers). Pointed at by extract errors via the documented escape hatch.
|
||||
const CacheEnvOverride = "LAST30DAYS_CACHE_DIR"
|
||||
|
||||
// Ensure extracts src into baseDir/last30days-pp-mcp/<version> and returns
|
||||
// the cache path. If the sentinel file already records the same version the
|
||||
// directory is reused without rewriting. version must be non-empty so the
|
||||
// cache layout always namespaces by version.
|
||||
//
|
||||
// Extraction writes to a sibling .tmp directory and renames it on success
|
||||
// so a partial extraction can never be mistaken for a complete one. Concurrent
|
||||
// callers within the same process serialize behind a per-cache-dir sync.Once
|
||||
// so the rename happens exactly once.
|
||||
func Ensure(src fs.FS, baseDir, version string) (string, error) {
|
||||
if version == "" {
|
||||
return "", errors.New("engine: version is required")
|
||||
}
|
||||
cacheDir := filepath.Join(baseDir, cacheSubdir, version)
|
||||
|
||||
once := getOnce(cacheDir)
|
||||
var extractErr error
|
||||
once.Do(func() {
|
||||
extractErr = ensureLocked(src, cacheDir, version)
|
||||
})
|
||||
if extractErr != nil {
|
||||
// Reset the sync.Once so a follow-up call can retry rather than
|
||||
// permanently caching the error. Retry is the right default when
|
||||
// the failure is transient (e.g., disk full, parent dir restored).
|
||||
resetOnce(cacheDir)
|
||||
return "", extractErr
|
||||
}
|
||||
return cacheDir, nil
|
||||
}
|
||||
|
||||
// EnsureUserCache wraps Ensure with the OS user cache dir (or the
|
||||
// LAST30DAYS_CACHE_DIR override) as base. Production callers use this; tests
|
||||
// use Ensure with an explicit temp dir.
|
||||
func EnsureUserCache(src fs.FS, version string) (string, error) {
|
||||
if override := os.Getenv(CacheEnvOverride); override != "" {
|
||||
return Ensure(src, override, version)
|
||||
}
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("engine: resolve user cache dir (set %s to override): %w", CacheEnvOverride, err)
|
||||
}
|
||||
return Ensure(src, base, version)
|
||||
}
|
||||
|
||||
func ensureLocked(src fs.FS, cacheDir, version string) error {
|
||||
if sentinelMatches(cacheDir, version) {
|
||||
return nil
|
||||
}
|
||||
tmpDir := cacheDir + ".tmp"
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
return fmt.Errorf("engine: clean tmp cache: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(tmpDir, 0o755); err != nil {
|
||||
return fmt.Errorf("engine: create tmp cache (%s, set %s to override): %w", tmpDir, CacheEnvOverride, err)
|
||||
}
|
||||
if err := extractAll(src, tmpDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return err
|
||||
}
|
||||
sentinel := filepath.Join(tmpDir, SentinelFilename)
|
||||
if err := os.WriteFile(sentinel, []byte(version), 0o644); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: write sentinel: %w", err)
|
||||
}
|
||||
if err := os.RemoveAll(cacheDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: clean old cache: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpDir, cacheDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: promote tmp cache: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sentinelMatches(cacheDir, version string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(cacheDir, SentinelFilename))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return string(data) == version
|
||||
}
|
||||
|
||||
func extractAll(src fs.FS, dst string) error {
|
||||
return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
target := filepath.Join(dst, path)
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
return copyEmbeddedFile(src, path, target)
|
||||
})
|
||||
}
|
||||
|
||||
func copyEmbeddedFile(src fs.FS, srcPath, dst string) error {
|
||||
in, err := src.Open(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("engine: open %s: %w", srcPath, err)
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return fmt.Errorf("engine: ensure parent of %s: %w", dst, err)
|
||||
}
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("engine: create %s: %w", dst, err)
|
||||
}
|
||||
defer func() { _ = out.Close() }()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return fmt.Errorf("engine: write %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// onceRegistry serializes first-call extraction per cache directory so the
|
||||
// rename in ensureLocked happens exactly once across goroutines.
|
||||
var (
|
||||
onceMu sync.Mutex
|
||||
onceRegistry = map[string]*sync.Once{}
|
||||
)
|
||||
|
||||
func getOnce(cacheDir string) *sync.Once {
|
||||
onceMu.Lock()
|
||||
defer onceMu.Unlock()
|
||||
if o, ok := onceRegistry[cacheDir]; ok {
|
||||
return o
|
||||
}
|
||||
o := &sync.Once{}
|
||||
onceRegistry[cacheDir] = o
|
||||
return o
|
||||
}
|
||||
|
||||
func resetOnce(cacheDir string) {
|
||||
onceMu.Lock()
|
||||
defer onceMu.Unlock()
|
||||
delete(onceRegistry, cacheDir)
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func newTestFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
|
||||
"lib/__init__.py": &fstest.MapFile{Data: []byte(""), Mode: 0o644},
|
||||
"lib/env.py": &fstest.MapFile{Data: []byte("# env helpers\n"), Mode: 0o644},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureExtractsEngine(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
cacheDir, err := Ensure(src, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure: %v", err)
|
||||
}
|
||||
if cacheDir != filepath.Join(base, cacheSubdir, "v1") {
|
||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, filepath.Join(base, cacheSubdir, "v1"))
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
||||
mustReadFile(t, filepath.Join(cacheDir, "lib/env.py"), "# env helpers\n")
|
||||
mustReadFile(t, filepath.Join(cacheDir, SentinelFilename), "v1")
|
||||
}
|
||||
|
||||
func TestEnsureSkipsWhenSentinelMatches(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
cacheDir, err := Ensure(src, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("first Ensure: %v", err)
|
||||
}
|
||||
target := filepath.Join(cacheDir, "last30days.py")
|
||||
info1, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
|
||||
// Reset the sync.Once so a second call would re-extract if not for the
|
||||
// sentinel short-circuit. Without the reset, sync.Once would skip the
|
||||
// extraction regardless of sentinel state.
|
||||
resetOnce(cacheDir)
|
||||
|
||||
if _, err := Ensure(src, base, "v1"); err != nil {
|
||||
t.Fatalf("second Ensure: %v", err)
|
||||
}
|
||||
info2, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Fatalf("stat second: %v", err)
|
||||
}
|
||||
if !info2.ModTime().Equal(info1.ModTime()) {
|
||||
t.Fatalf("expected file untouched on sentinel match; got mtime %v -> %v", info1.ModTime(), info2.ModTime())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReExtractsOnVersionChange(t *testing.T) {
|
||||
v1 := fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("v1\n"), Mode: 0o644},
|
||||
}
|
||||
v2 := fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("v2\n"), Mode: 0o644},
|
||||
}
|
||||
base := t.TempDir()
|
||||
|
||||
cache1, err := Ensure(v1, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure v1: %v", err)
|
||||
}
|
||||
cache2, err := Ensure(v2, base, "v2")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure v2: %v", err)
|
||||
}
|
||||
if cache1 == cache2 {
|
||||
t.Fatalf("expected distinct cache dirs per version, got %q == %q", cache1, cache2)
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cache1, "last30days.py"), "v1\n")
|
||||
mustReadFile(t, filepath.Join(cache2, "last30days.py"), "v2\n")
|
||||
}
|
||||
|
||||
func TestEnsureConcurrentFirstCall(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
const goroutines = 10
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
results := make([]string, goroutines)
|
||||
errs := make([]error, goroutines)
|
||||
for i := 0; i < goroutines; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
results[i], errs[i] = Ensure(src, base, "v1")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("goroutine %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
for i := 1; i < goroutines; i++ {
|
||||
if results[i] != results[0] {
|
||||
t.Fatalf("goroutine 0 saw %q, goroutine %d saw %q", results[0], i, results[i])
|
||||
}
|
||||
}
|
||||
mustReadFile(t, filepath.Join(results[0], "last30days.py"), "# last30days entry\n")
|
||||
}
|
||||
|
||||
func TestEnsureRejectsEmptyVersion(t *testing.T) {
|
||||
if _, err := Ensure(newTestFS(), t.TempDir(), ""); err == nil {
|
||||
t.Fatal("expected error for empty version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReturnsErrorWhenCacheUnwritable(t *testing.T) {
|
||||
// Place the cache root at a path that cannot exist (a regular file).
|
||||
// MkdirAll will refuse and Ensure must surface a wrapped error.
|
||||
base := t.TempDir()
|
||||
blocker := filepath.Join(base, "blocker")
|
||||
if err := os.WriteFile(blocker, []byte("not a dir"), 0o644); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
_, err := Ensure(newTestFS(), blocker, "v1")
|
||||
if err == nil {
|
||||
t.Fatal("expected error when cache parent is not a directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureUserCacheHonorsOverride(t *testing.T) {
|
||||
override := t.TempDir()
|
||||
t.Setenv(CacheEnvOverride, override)
|
||||
|
||||
src := newTestFS()
|
||||
cacheDir, err := EnsureUserCache(src, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureUserCache: %v", err)
|
||||
}
|
||||
want := filepath.Join(override, cacheSubdir, "v1")
|
||||
if cacheDir != want {
|
||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, want)
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("%s: got %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultPythonBinary is the interpreter we look up unless RunOptions
|
||||
// overrides it. Windows installs may expose only "python"; we surface a
|
||||
// clear error in that case rather than silently picking the wrong binary.
|
||||
const DefaultPythonBinary = "python3"
|
||||
|
||||
// MinPythonVersion mirrors the engine's MIN_PYTHON constant in
|
||||
// last30days.py. Surfaced in errors so users know what they're missing.
|
||||
const MinPythonVersion = "3.12"
|
||||
|
||||
// PythonInstallURL is included in the missing-interpreter error so users
|
||||
// have a direct route from the failure to a fix.
|
||||
const PythonInstallURL = "https://www.python.org/downloads/"
|
||||
|
||||
// DefaultTimeout caps a single research subprocess. The engine's deep mode
|
||||
// can run several minutes; five minutes is a safe upper bound that still
|
||||
// fails fast when something hangs.
|
||||
const DefaultTimeout = 5 * time.Minute
|
||||
|
||||
// TimeoutEnvOverride lets operators override DefaultTimeout per install
|
||||
// (seconds, integer). Honored by Run when RunOptions.Timeout is zero.
|
||||
const TimeoutEnvOverride = "LAST30DAYS_MCP_TIMEOUT"
|
||||
|
||||
// RunOptions configures one invocation of the embedded Python engine.
|
||||
// PythonPath is exposed so tests can substitute a stub interpreter without
|
||||
// manipulating the process PATH.
|
||||
type RunOptions struct {
|
||||
PythonPath string // resolved python3 binary; empty means look up DefaultPythonBinary on PATH
|
||||
CacheDir string // engine.Ensure result; lib/ here is added to PYTHONPATH
|
||||
Args []string // arguments after last30days.py (topic, --emit=..., etc.)
|
||||
ExtraEnv []string // appended to os.Environ() for the child process
|
||||
Timeout time.Duration // zero means DefaultTimeout or TimeoutEnvOverride
|
||||
}
|
||||
|
||||
// RunResult captures the engine's full output. Stdout is what we surface to
|
||||
// the agent; Stderr is included in error messages so users can diagnose
|
||||
// engine failures without leaving Claude Desktop.
|
||||
type RunResult struct {
|
||||
Stdout []byte
|
||||
Stderr []byte
|
||||
ExitCode int
|
||||
TimedOut bool
|
||||
}
|
||||
|
||||
// Run shells out to python3 with last30days.py inside cacheDir. The child
|
||||
// receives the parent environment (so MCPB user_config env-injection
|
||||
// reaches the engine) plus ExtraEnv and a PYTHONPATH that points at the
|
||||
// cache so the engine's `from lib import ...` statements resolve.
|
||||
//
|
||||
// A missing interpreter, a non-zero exit, and a timeout each surface as
|
||||
// distinct errors so the tool handler can map them to user-facing
|
||||
// messages without re-parsing stderr.
|
||||
func Run(ctx context.Context, opts RunOptions) (*RunResult, error) {
|
||||
if opts.CacheDir == "" {
|
||||
return nil, errors.New("engine: CacheDir is required")
|
||||
}
|
||||
pythonPath, err := resolvePython(opts.PythonPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scriptPath := filepath.Join(opts.CacheDir, "last30days.py")
|
||||
if _, err := os.Stat(scriptPath); err != nil {
|
||||
return nil, fmt.Errorf("engine: last30days.py not found in cache %s: %w", opts.CacheDir, err)
|
||||
}
|
||||
|
||||
timeout := resolveTimeout(opts.Timeout)
|
||||
subCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
args := append([]string{scriptPath}, opts.Args...)
|
||||
cmd := exec.CommandContext(subCtx, pythonPath, args...)
|
||||
cmd.Env = buildEnv(opts.CacheDir, opts.ExtraEnv)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
err = cmd.Run()
|
||||
res := &RunResult{
|
||||
Stdout: stdout.Bytes(),
|
||||
Stderr: stderr.Bytes(),
|
||||
ExitCode: 0,
|
||||
TimedOut: errors.Is(subCtx.Err(), context.DeadlineExceeded),
|
||||
}
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
res.ExitCode = exitErr.ExitCode()
|
||||
if res.TimedOut {
|
||||
return res, fmt.Errorf("engine: subprocess exceeded %s timeout", timeout)
|
||||
}
|
||||
return res, fmt.Errorf("engine: subprocess exited with code %d", res.ExitCode)
|
||||
}
|
||||
return res, fmt.Errorf("engine: subprocess failed to start: %w", err)
|
||||
}
|
||||
|
||||
// resolvePython returns an absolute path to the interpreter or an error
|
||||
// naming the install URL. If the caller supplied a path we trust it - tests
|
||||
// rely on this to inject a stub. Otherwise we look up python3 on PATH.
|
||||
func resolvePython(override string) (string, error) {
|
||||
if override != "" {
|
||||
return override, nil
|
||||
}
|
||||
path, err := exec.LookPath(DefaultPythonBinary)
|
||||
if err == nil {
|
||||
return path, nil
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"engine: %s not found on PATH (need Python %s+, install from %s; current GOOS=%s)",
|
||||
DefaultPythonBinary, MinPythonVersion, PythonInstallURL, runtime.GOOS,
|
||||
)
|
||||
}
|
||||
|
||||
func resolveTimeout(explicit time.Duration) time.Duration {
|
||||
if explicit > 0 {
|
||||
return explicit
|
||||
}
|
||||
if raw := os.Getenv(TimeoutEnvOverride); raw != "" {
|
||||
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
// Accept bare integer seconds (e.g. "300") as documented.
|
||||
if secs, err := strconv.Atoi(raw); err == nil && secs > 0 {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
}
|
||||
return DefaultTimeout
|
||||
}
|
||||
|
||||
// buildEnv stitches PYTHONPATH onto os.Environ + ExtraEnv. Any pre-existing
|
||||
// PYTHONPATH in the parent environment is dropped before appending the
|
||||
// cache dir; otherwise the child sees two PYTHONPATH= entries and POSIX
|
||||
// getenv returns the first one, so the user's value wins and the engine's
|
||||
// `from lib import ...` fails with ModuleNotFoundError. The engine is
|
||||
// self-contained and does not need the user's Python module search path.
|
||||
func buildEnv(cacheDir string, extra []string) []string {
|
||||
const pyKey = "PYTHONPATH="
|
||||
parent := os.Environ()
|
||||
base := make([]string, 0, len(parent)+1+len(extra))
|
||||
for _, kv := range parent {
|
||||
if strings.HasPrefix(kv, pyKey) {
|
||||
continue
|
||||
}
|
||||
base = append(base, kv)
|
||||
}
|
||||
base = append(base, pyKey+cacheDir)
|
||||
base = append(base, extra...)
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// makeStubPython writes a shell script that simulates python3 and returns
|
||||
// its absolute path. The script honors a small env-driven protocol so each
|
||||
// test can shape its output:
|
||||
//
|
||||
// STUB_STDOUT - text printed to stdout
|
||||
// STUB_STDERR - text printed to stderr
|
||||
// STUB_EXIT_CODE - integer exit code (default 0)
|
||||
// STUB_SLEEP_SECS - sleep before exiting (for timeout tests)
|
||||
// STUB_ECHO_ENV - name of an env var; the stub prints "<NAME>=<VALUE>"
|
||||
// STUB_ECHO_ARG - integer index; the stub prints "ARG<i>=<args[i]>"
|
||||
//
|
||||
// The stub ignores its first argument (the script path), matching how a
|
||||
// real python3 invocation treats `python3 last30days.py ...`.
|
||||
func makeStubPython(t *testing.T) string {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("stub-python tests rely on POSIX shell")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "python3-stub.sh")
|
||||
script := `#!/usr/bin/env bash
|
||||
if [ -n "${STUB_SLEEP_SECS:-}" ]; then sleep "$STUB_SLEEP_SECS"; fi
|
||||
if [ -n "${STUB_STDOUT:-}" ]; then printf "%s" "$STUB_STDOUT"; fi
|
||||
if [ -n "${STUB_STDERR:-}" ]; then printf "%s" "$STUB_STDERR" >&2; fi
|
||||
if [ -n "${STUB_ECHO_ENV:-}" ]; then echo "${STUB_ECHO_ENV}=${!STUB_ECHO_ENV:-<unset>}"; fi
|
||||
if [ -n "${STUB_ECHO_ARG:-}" ]; then echo "ARG${STUB_ECHO_ARG}=${!STUB_ECHO_ARG:-<unset>}"; fi
|
||||
exit "${STUB_EXIT_CODE:-0}"
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// stageCache materializes a fake CacheDir with a no-op last30days.py so
|
||||
// the existence check in Run passes. The stub python3 ignores the script
|
||||
// contents, so the file just has to exist.
|
||||
func stageCache(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "last30days.py"), []byte("# stub\n"), 0o644); err != nil {
|
||||
t.Fatalf("stage cache: %v", err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestRunHappyPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_STDOUT", "synthesis output\n")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
Args: []string{"my topic", "--emit=compact"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if string(res.Stdout) != "synthesis output\n" {
|
||||
t.Fatalf("stdout = %q, want %q", res.Stdout, "synthesis output\n")
|
||||
}
|
||||
if res.ExitCode != 0 {
|
||||
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
||||
}
|
||||
if res.TimedOut {
|
||||
t.Fatal("TimedOut = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForwardsEnv(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("OPENAI_API_KEY", "sk-test-value")
|
||||
t.Setenv("STUB_ECHO_ENV", "OPENAI_API_KEY")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if got := strings.TrimSpace(string(res.Stdout)); got != "OPENAI_API_KEY=sk-test-value" {
|
||||
t.Fatalf("stdout = %q, want OPENAI_API_KEY=sk-test-value", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSetsPythonPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
want := "PYTHONPATH=" + cache
|
||||
if got := strings.TrimSpace(string(res.Stdout)); got != want {
|
||||
t.Fatalf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDropsPreExistingPythonPath guards the buildEnv dedup: when the
|
||||
// parent already sets PYTHONPATH (common on dev machines and CI runners
|
||||
// that touch Python), the child must NOT see two PYTHONPATH= entries.
|
||||
// POSIX getenv returns the first match, so a duplicate from os.Environ
|
||||
// would shadow our cache-dir entry and break `from lib import ...`.
|
||||
func TestRunDropsPreExistingPythonPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("PYTHONPATH", "/users-stale-pythonpath")
|
||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
got := strings.TrimSpace(string(res.Stdout))
|
||||
want := "PYTHONPATH=" + cache
|
||||
if got != want {
|
||||
t.Fatalf("stdout = %q, want %q (stale parent value leaked through)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildEnvDropsAllPreExistingPythonPath(t *testing.T) {
|
||||
// Direct unit test on buildEnv to catch the case where the parent has
|
||||
// PYTHONPATH set: the returned slice must contain exactly one
|
||||
// PYTHONPATH= entry, and it must be ours.
|
||||
t.Setenv("PYTHONPATH", "/parent/one")
|
||||
cache := "/cache/dir"
|
||||
out := buildEnv(cache, []string{"EXTRA=1"})
|
||||
|
||||
var pythonPaths []string
|
||||
for _, kv := range out {
|
||||
if strings.HasPrefix(kv, "PYTHONPATH=") {
|
||||
pythonPaths = append(pythonPaths, kv)
|
||||
}
|
||||
}
|
||||
if len(pythonPaths) != 1 {
|
||||
t.Fatalf("got %d PYTHONPATH entries, want 1: %v", len(pythonPaths), pythonPaths)
|
||||
}
|
||||
if pythonPaths[0] != "PYTHONPATH="+cache {
|
||||
t.Fatalf("PYTHONPATH = %q, want %q", pythonPaths[0], "PYTHONPATH="+cache)
|
||||
}
|
||||
// Confirm ExtraEnv still rides along.
|
||||
found := false
|
||||
for _, kv := range out {
|
||||
if kv == "EXTRA=1" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("EXTRA=1 missing from buildEnv output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSurfacesExitCode(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_STDERR", "engine boom\n")
|
||||
t.Setenv("STUB_EXIT_CODE", "2")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for non-zero exit")
|
||||
}
|
||||
if res == nil {
|
||||
t.Fatal("res is nil; want populated result alongside error")
|
||||
}
|
||||
if res.ExitCode != 2 {
|
||||
t.Fatalf("ExitCode = %d, want 2", res.ExitCode)
|
||||
}
|
||||
if !strings.Contains(string(res.Stderr), "engine boom") {
|
||||
t.Fatalf("stderr did not surface engine output: %q", res.Stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTimesOut(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_SLEEP_SECS", "3")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
Timeout: 200 * time.Millisecond,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
if !res.TimedOut {
|
||||
t.Fatal("TimedOut = false, want true")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "timeout") {
|
||||
t.Fatalf("error %q lacks 'timeout' marker", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingPython(t *testing.T) {
|
||||
cache := stageCache(t)
|
||||
// Empty PATH guarantees the lookup fails. PythonPath stays unset so Run
|
||||
// falls through to exec.LookPath.
|
||||
t.Setenv("PATH", "")
|
||||
|
||||
_, err := Run(context.Background(), RunOptions{CacheDir: cache})
|
||||
if err == nil {
|
||||
t.Fatal("expected lookup failure with empty PATH")
|
||||
}
|
||||
if !strings.Contains(err.Error(), DefaultPythonBinary) {
|
||||
t.Fatalf("error %q does not mention %s", err, DefaultPythonBinary)
|
||||
}
|
||||
if !strings.Contains(err.Error(), PythonInstallURL) {
|
||||
t.Fatalf("error %q does not include install URL", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingScript(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
// CacheDir exists but contains no last30days.py.
|
||||
cache := t.TempDir()
|
||||
|
||||
_, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when last30days.py missing")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "last30days.py") {
|
||||
t.Fatalf("error %q does not name missing script", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsEmptyCacheDir(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
_, err := Run(context.Background(), RunOptions{PythonPath: stub})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty CacheDir")
|
||||
}
|
||||
if !errors.Is(err, err) || !strings.Contains(err.Error(), "CacheDir") {
|
||||
t.Fatalf("error %q does not name CacheDir", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTimeoutHonorsEnv(t *testing.T) {
|
||||
t.Setenv(TimeoutEnvOverride, "750ms")
|
||||
if got := resolveTimeout(0); got != 750*time.Millisecond {
|
||||
t.Fatalf("resolveTimeout = %v, want 750ms", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "garbage")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("garbage value: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
if got := resolveTimeout(time.Minute); got != time.Minute {
|
||||
t.Fatalf("explicit value not honored: got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTimeoutBareIntegerSeconds(t *testing.T) {
|
||||
t.Setenv(TimeoutEnvOverride, "300")
|
||||
if got := resolveTimeout(0); got != 300*time.Second {
|
||||
t.Fatalf("bare integer 300: got %v, want 5m0s", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "1")
|
||||
if got := resolveTimeout(0); got != 1*time.Second {
|
||||
t.Fatalf("bare integer 1: got %v, want 1s", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "0")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("bare integer 0: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "-1")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("bare integer -1: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Populated at build time by scripts/sync-engine.sh.
|
||||
Source of truth: skills/last30days/scripts/.
|
||||
@@ -0,0 +1,189 @@
|
||||
// Package manifest holds tests for mcp/manifest.json. It contains no
|
||||
// production code - the manifest itself is the artifact, and these tests
|
||||
// guard structural invariants the bundling pipeline depends on.
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// envBinding is a minimal subset of the MCPB v0.3 manifest just covering
|
||||
// the fields these tests assert on. We deliberately do not depend on the
|
||||
// printing-press internal/pipeline types (that's an internal/ package and
|
||||
// not importable across modules) - the structural invariants below are
|
||||
// what actually matter for Claude Desktop install correctness.
|
||||
type manifestShape struct {
|
||||
ManifestVersion string `json:"manifest_version"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Server struct {
|
||||
Type string `json:"type"`
|
||||
EntryPoint string `json:"entry_point"`
|
||||
MCPConfig struct {
|
||||
Command string `json:"command"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"mcp_config"`
|
||||
} `json:"server"`
|
||||
UserConfig map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Sensitive bool `json:"sensitive"`
|
||||
Required bool `json:"required"`
|
||||
} `json:"user_config"`
|
||||
Compatibility struct {
|
||||
ClaudeDesktop string `json:"claude_desktop"`
|
||||
Platforms []string `json:"platforms"`
|
||||
} `json:"compatibility"`
|
||||
}
|
||||
|
||||
// loadManifest reads mcp/manifest.json relative to this test file so the
|
||||
// test passes regardless of where `go test` is invoked from.
|
||||
func loadManifest(t *testing.T) manifestShape {
|
||||
t.Helper()
|
||||
_, thisFile, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("runtime.Caller failed")
|
||||
}
|
||||
// manifest_test.go is at mcp/internal/manifest/; manifest.json at mcp/.
|
||||
manifestPath := filepath.Join(filepath.Dir(thisFile), "..", "..", "manifest.json")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
var m manifestShape
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
t.Fatalf("parse manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestManifestRequiredFields(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
if m.ManifestVersion != "0.3" {
|
||||
t.Errorf("manifest_version = %q, want 0.3", m.ManifestVersion)
|
||||
}
|
||||
if m.Name != "last30days-pp-mcp" {
|
||||
t.Errorf("name = %q, want last30days-pp-mcp", m.Name)
|
||||
}
|
||||
if m.Version == "" {
|
||||
t.Error("version is empty")
|
||||
}
|
||||
if m.Server.Type != "binary" {
|
||||
t.Errorf("server.type = %q, want binary", m.Server.Type)
|
||||
}
|
||||
if m.Server.EntryPoint != "bin/last30days-pp-mcp" {
|
||||
t.Errorf("server.entry_point = %q, want bin/last30days-pp-mcp", m.Server.EntryPoint)
|
||||
}
|
||||
if m.Compatibility.ClaudeDesktop == "" {
|
||||
t.Error("compatibility.claude_desktop is empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnvAndUserConfigCrossReference is the key invariant: every
|
||||
// ${user_config.<key>} substitution in server.mcp_config.env must point
|
||||
// at a real user_config entry, and every declared user_config must be
|
||||
// wired to an env var. A typo on either side silently disables a credential
|
||||
// at install time without the binary or Claude Desktop noticing.
|
||||
func TestEnvAndUserConfigCrossReference(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
|
||||
if len(m.Server.MCPConfig.Env) == 0 {
|
||||
t.Fatal("server.mcp_config.env is empty; expected user_config substitutions")
|
||||
}
|
||||
if len(m.UserConfig) == 0 {
|
||||
t.Fatal("user_config is empty; expected per-key declarations")
|
||||
}
|
||||
|
||||
for envName, value := range m.Server.MCPConfig.Env {
|
||||
key, ok := parseUserConfigRef(value)
|
||||
if !ok {
|
||||
t.Errorf("env[%s] = %q is not a ${user_config.<key>} reference", envName, value)
|
||||
continue
|
||||
}
|
||||
if _, declared := m.UserConfig[key]; !declared {
|
||||
t.Errorf("env[%s] references user_config[%q], which is not declared", envName, key)
|
||||
}
|
||||
// The user_config key must be the lowercased env var so Claude
|
||||
// Desktop's substitution rule matches PP's emitted shape.
|
||||
if got := strings.ToLower(envName); key != got {
|
||||
t.Errorf("env[%s] -> user_config[%q]; convention requires user_config[%q]", envName, key, got)
|
||||
}
|
||||
}
|
||||
|
||||
envValues := make(map[string]bool, len(m.Server.MCPConfig.Env))
|
||||
for _, value := range m.Server.MCPConfig.Env {
|
||||
if key, ok := parseUserConfigRef(value); ok {
|
||||
envValues[key] = true
|
||||
}
|
||||
}
|
||||
for key := range m.UserConfig {
|
||||
if !envValues[key] {
|
||||
t.Errorf("user_config[%q] is declared but never substituted into env", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserConfigShape(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
for key, slot := range m.UserConfig {
|
||||
if slot.Type != "string" {
|
||||
t.Errorf("user_config[%q].type = %q, want string", key, slot.Type)
|
||||
}
|
||||
if slot.Title == "" {
|
||||
t.Errorf("user_config[%q].title is empty", key)
|
||||
}
|
||||
if slot.Description == "" {
|
||||
t.Errorf("user_config[%q].description is empty", key)
|
||||
}
|
||||
if !slot.Sensitive {
|
||||
// API keys must be flagged sensitive so Claude Desktop masks
|
||||
// the input and prefers OS-keychain storage.
|
||||
t.Errorf("user_config[%q].sensitive = false; want true for API credentials", key)
|
||||
}
|
||||
if slot.Required {
|
||||
// The engine degrades to web-only mode without keys, so no
|
||||
// key is install-blocking.
|
||||
t.Errorf("user_config[%q].required = true; engine degrades without keys, so all keys are optional", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformsMatchShippingMatrix(t *testing.T) {
|
||||
// compatibility.platforms must list exactly what the release CI
|
||||
// actually packages. Listing a platform we don't ship would let
|
||||
// Claude Desktop start an install that has no matching binary inside
|
||||
// the bundle, producing a silent failure. The CI matrix in
|
||||
// .github/workflows/release.yml currently covers darwin (arm64 +
|
||||
// amd64) and linux/amd64; Windows is deferred.
|
||||
m := loadManifest(t)
|
||||
required := map[string]bool{"darwin": false, "linux": false}
|
||||
forbidden := map[string]bool{"win32": true}
|
||||
for _, p := range m.Compatibility.Platforms {
|
||||
if _, ok := required[p]; ok {
|
||||
required[p] = true
|
||||
}
|
||||
if forbidden[p] {
|
||||
t.Errorf("compatibility.platforms contains %q but the release matrix does not ship that platform; add it to the matrix or remove from the manifest", p)
|
||||
}
|
||||
}
|
||||
for p, found := range required {
|
||||
if !found {
|
||||
t.Errorf("compatibility.platforms missing %q", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseUserConfigRef(value string) (string, bool) {
|
||||
const prefix = "${user_config."
|
||||
const suffix = "}"
|
||||
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
||||
return "", false
|
||||
}
|
||||
return value[len(prefix) : len(value)-len(suffix)], true
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
func registerPreflightTool(s *server.MCPServer, cfg Config) {
|
||||
s.AddTool(
|
||||
mcplib.NewTool("preflight",
|
||||
mcplib.WithDescription(
|
||||
"Safely summarize what last30days would read, write, execute, and contact "+
|
||||
"without running research, saving files, or reading browser cookies.",
|
||||
),
|
||||
mcplib.WithString("format", mcplib.Description("Output shape: 'text' (default) for a concise summary or 'json' for structured details.")),
|
||||
mcplib.WithReadOnlyHintAnnotation(true),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(false),
|
||||
),
|
||||
makePreflightHandler(cfg),
|
||||
)
|
||||
}
|
||||
|
||||
func makePreflightHandler(cfg Config) server.ToolHandlerFunc {
|
||||
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
|
||||
format, err := preflightFormatArgument(req.GetArguments())
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
src, err := engine.EngineFS()
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
|
||||
}
|
||||
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf(
|
||||
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
|
||||
err, engine.CacheEnvOverride,
|
||||
)), nil
|
||||
}
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
Args: preflightRunArgs(format),
|
||||
})
|
||||
if runErr != nil {
|
||||
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
|
||||
}
|
||||
return mcplib.NewToolResultText(string(res.Stdout)), nil
|
||||
}
|
||||
}
|
||||
|
||||
func preflightRunArgs(format string) []string {
|
||||
runArgs := []string{"--preflight", "--preflight-report-on-save-dir", mcpSaveDir()}
|
||||
if format == "json" {
|
||||
runArgs = append(runArgs, "--emit=json")
|
||||
}
|
||||
return runArgs
|
||||
}
|
||||
|
||||
func preflightFormatArgument(args map[string]any) (string, error) {
|
||||
raw, ok := args["format"]
|
||||
if !ok {
|
||||
return "text", nil
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return "", errors.New("format must be a string")
|
||||
}
|
||||
switch value {
|
||||
case "", "text":
|
||||
return "text", nil
|
||||
case "json":
|
||||
return "json", nil
|
||||
default:
|
||||
return "", fmt.Errorf("format must be 'text' or 'json', got %q", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreflightRunArgsDefaultTextIsSafe(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
|
||||
args := preflightRunArgs("text")
|
||||
want := []string{
|
||||
"--preflight",
|
||||
"--preflight-report-on-save-dir",
|
||||
"~/Documents/Last30Days",
|
||||
}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightRunArgsJSONIsSafeAndStructured(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
|
||||
args := preflightRunArgs("json")
|
||||
want := []string{
|
||||
"--preflight",
|
||||
"--preflight-report-on-save-dir",
|
||||
"/tmp/last30days-reports",
|
||||
"--emit=json",
|
||||
}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightFormatArgumentDefaultsAndValidates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"missing defaults to text", map[string]any{}, "text", false},
|
||||
{"empty defaults to text", map[string]any{"format": ""}, "text", false},
|
||||
{"text passes", map[string]any{"format": "text"}, "text", false},
|
||||
{"json passes", map[string]any{"format": "json"}, "json", false},
|
||||
{"invalid rejected", map[string]any{"format": "xml"}, "", true},
|
||||
{"non-string rejected", map[string]any{"format": true}, "", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := preflightFormatArgument(tc.args)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
// Package tools owns the MCP tool surface for last30days.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
// Config carries the version string used to namespace the per-user cache.
|
||||
// main passes its ldflags-stamped Version here.
|
||||
type Config struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
// Register adds every tool this server exposes to s. The caller supplies a
|
||||
// Config so test harnesses can pin a version without touching globals.
|
||||
func Register(s *server.MCPServer, cfg Config) {
|
||||
registerPreflightTool(s, cfg)
|
||||
s.AddTool(
|
||||
mcplib.NewTool("research",
|
||||
mcplib.WithDescription(
|
||||
"Research what people are actually saying about any topic in the last 30 days. "+
|
||||
"Aggregates Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web, "+
|
||||
"scored by upvotes, likes, transcripts, and real-money prediction-market odds. "+
|
||||
"Returns the engine's compact output for the model to synthesize.",
|
||||
),
|
||||
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
|
||||
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
|
||||
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
|
||||
mcplib.WithReadOnlyHintAnnotation(false),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(true),
|
||||
),
|
||||
makeResearchHandler(cfg),
|
||||
)
|
||||
}
|
||||
|
||||
func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
|
||||
args := req.GetArguments()
|
||||
topic, err := requireString(args, "topic")
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
emit, err := emitArgument(args)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
save, err := boolArgument(args, "save")
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
src, err := engine.EngineFS()
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
|
||||
}
|
||||
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf(
|
||||
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
|
||||
err, engine.CacheEnvOverride,
|
||||
)), nil
|
||||
}
|
||||
|
||||
runArgs := researchRunArgs(topic, emit, save)
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
Args: runArgs,
|
||||
})
|
||||
if runErr != nil {
|
||||
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
|
||||
}
|
||||
return mcplib.NewToolResultText(string(res.Stdout)), nil
|
||||
}
|
||||
}
|
||||
|
||||
func researchRunArgs(topic, emit string, save bool) []string {
|
||||
runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"}
|
||||
if save {
|
||||
runArgs = append(runArgs, "--save-dir", mcpSaveDir())
|
||||
}
|
||||
return runArgs
|
||||
}
|
||||
|
||||
func mcpSaveDir() string {
|
||||
saveDir := os.Getenv("LAST30DAYS_MEMORY_DIR")
|
||||
if saveDir == "" {
|
||||
return "~/Documents/Last30Days"
|
||||
}
|
||||
return saveDir
|
||||
}
|
||||
|
||||
func requireString(args map[string]any, name string) (string, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is required", name)
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok || strings.TrimSpace(value) == "" {
|
||||
return "", fmt.Errorf("%s must be a non-empty string", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func emitArgument(args map[string]any) (string, error) {
|
||||
raw, ok := args["emit"]
|
||||
if !ok {
|
||||
return "compact", nil
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return "", errors.New("emit must be a string")
|
||||
}
|
||||
switch value {
|
||||
case "":
|
||||
return "compact", nil
|
||||
case "compact", "html":
|
||||
return value, nil
|
||||
default:
|
||||
return "", fmt.Errorf("emit must be 'compact' or 'html', got %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
func boolArgument(args map[string]any, name string) (bool, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
value, ok := raw.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%s must be a boolean", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// formatRunError flattens engine.Run's distinct error shapes into a single
|
||||
// user-facing message that includes the relevant stderr context.
|
||||
func formatRunError(runErr error, res *engine.RunResult) string {
|
||||
var msg strings.Builder
|
||||
msg.WriteString(runErr.Error())
|
||||
if res != nil && len(res.Stderr) > 0 {
|
||||
msg.WriteString("\nengine stderr:\n")
|
||||
msg.Write(res.Stderr)
|
||||
}
|
||||
return msg.String()
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
func newCallToolRequest(args map[string]any) mcplib.CallToolRequest {
|
||||
var req mcplib.CallToolRequest
|
||||
req.Params.Arguments = args
|
||||
return req
|
||||
}
|
||||
|
||||
// resultText pulls text content out of a tool result so tests can assert on
|
||||
// the body Claude will see. Returns empty string when the result is nil or
|
||||
// has no text content.
|
||||
func resultText(res *mcplib.CallToolResult) string {
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, item := range res.Content {
|
||||
if tc, ok := item.(mcplib.TextContent); ok {
|
||||
out.WriteString(tc.Text)
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRequireStringRejectsMissingAndBlank(t *testing.T) {
|
||||
if _, err := requireString(map[string]any{}, "topic"); err == nil {
|
||||
t.Fatal("expected error for missing topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": ""}, "topic"); err == nil {
|
||||
t.Fatal("expected error for empty topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": " "}, "topic"); err == nil {
|
||||
t.Fatal("expected error for whitespace-only topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": 42}, "topic"); err == nil {
|
||||
t.Fatal("expected error for non-string topic")
|
||||
}
|
||||
v, err := requireString(map[string]any{"topic": "OpenAI"}, "topic")
|
||||
if err != nil || v != "OpenAI" {
|
||||
t.Fatalf("requireString ok = %q, %v", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitArgumentDefaultsAndValidates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"missing defaults to compact", map[string]any{}, "compact", false},
|
||||
{"empty string defaults to compact", map[string]any{"emit": ""}, "compact", false},
|
||||
{"compact passes through", map[string]any{"emit": "compact"}, "compact", false},
|
||||
{"html passes through", map[string]any{"emit": "html"}, "html", false},
|
||||
{"invalid value rejected", map[string]any{"emit": "json"}, "", true},
|
||||
{"non-string rejected", map[string]any{"emit": 7}, "", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := emitArgument(tc.args)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoolArgument(t *testing.T) {
|
||||
v, err := boolArgument(map[string]any{}, "save")
|
||||
if err != nil || v {
|
||||
t.Fatalf("missing: %v, %v", v, err)
|
||||
}
|
||||
v, err = boolArgument(map[string]any{"save": true}, "save")
|
||||
if err != nil || !v {
|
||||
t.Fatalf("true: %v, %v", v, err)
|
||||
}
|
||||
v, err = boolArgument(map[string]any{"save": false}, "save")
|
||||
if err != nil || v {
|
||||
t.Fatalf("false: %v, %v", v, err)
|
||||
}
|
||||
if _, err := boolArgument(map[string]any{"save": "true"}, "save"); err == nil {
|
||||
t.Fatal("expected error for string value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) {
|
||||
args := researchRunArgs("OpenAI", "compact", false)
|
||||
want := []string{"OpenAI", "--emit=compact", "--no-browser-cookies"}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
|
||||
args := researchRunArgs("OpenAI", "html", true)
|
||||
got := strings.Join(args, "\x00")
|
||||
if strings.Contains(got, "--save\x00") || strings.HasSuffix(got, "--save") {
|
||||
t.Fatalf("args still include unsupported --save: %#v", args)
|
||||
}
|
||||
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "~/Documents/Last30Days"}
|
||||
if got != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsSaveUsesMemoryDirEnvOverride(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
|
||||
args := researchRunArgs("OpenAI", "html", true)
|
||||
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "/tmp/last30days-reports"}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
|
||||
// Validation failures are returned as MCP tool errors (not Go errors)
|
||||
// so Claude sees a structured failure with a readable message rather
|
||||
// than a transport-level fault.
|
||||
handler := makeResearchHandler(Config{Version: "test"})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
wantSub string
|
||||
}{
|
||||
{"missing topic", map[string]any{}, "topic is required"},
|
||||
{"blank topic", map[string]any{"topic": " "}, "non-empty string"},
|
||||
{"invalid emit", map[string]any{"topic": "OpenAI", "emit": "json"}, "must be 'compact' or 'html'"},
|
||||
{"non-bool save", map[string]any{"topic": "OpenAI", "save": "yes"}, "save must be a boolean"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := handler(context.Background(), newCallToolRequest(tc.args))
|
||||
if err != nil {
|
||||
t.Fatalf("handler should not return Go error for validation; got %v", err)
|
||||
}
|
||||
if res == nil || !res.IsError {
|
||||
t.Fatalf("expected IsError result, got %+v", res)
|
||||
}
|
||||
if !strings.Contains(resultText(res), tc.wantSub) {
|
||||
t.Fatalf("result text %q missing substring %q", resultText(res), tc.wantSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRunErrorIncludesStderr(t *testing.T) {
|
||||
res := &engine.RunResult{Stderr: []byte("engine exploded\n")}
|
||||
msg := formatRunError(errors.New("boom"), res)
|
||||
if !strings.Contains(msg, "boom") || !strings.Contains(msg, "engine exploded") {
|
||||
t.Fatalf("formatRunError missed pieces: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRunErrorHandlesNilResult(t *testing.T) {
|
||||
msg := formatRunError(errors.New("boom"), nil)
|
||||
if msg != "boom" {
|
||||
t.Fatalf("nil result: got %q, want %q", msg, "boom")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"manifest_version": "0.3",
|
||||
"name": "last30days-pp-mcp",
|
||||
"display_name": "Last30Days",
|
||||
"version": "3.6.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
||||
},
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"x",
|
||||
"youtube",
|
||||
"hacker-news",
|
||||
"polymarket",
|
||||
"github",
|
||||
"search",
|
||||
"synthesis"
|
||||
],
|
||||
"server": {
|
||||
"type": "binary",
|
||||
"entry_point": "bin/last30days-pp-mcp",
|
||||
"mcp_config": {
|
||||
"command": "${__dirname}/bin/last30days-pp-mcp",
|
||||
"args": [],
|
||||
"env": {
|
||||
"OPENAI_API_KEY": "${user_config.openai_api_key}",
|
||||
"XAI_API_KEY": "${user_config.xai_api_key}",
|
||||
"BRAVE_API_KEY": "${user_config.brave_api_key}",
|
||||
"EXA_API_KEY": "${user_config.exa_api_key}",
|
||||
"SERPER_API_KEY": "${user_config.serper_api_key}",
|
||||
"GOOGLE_API_KEY": "${user_config.google_api_key}",
|
||||
"GEMINI_API_KEY": "${user_config.gemini_api_key}",
|
||||
"GOOGLE_GENAI_API_KEY": "${user_config.google_genai_api_key}",
|
||||
"APIFY_API_TOKEN": "${user_config.apify_api_token}",
|
||||
"BSKY_APP_PASSWORD": "${user_config.bsky_app_password}",
|
||||
"PARALLEL_API_KEY": "${user_config.parallel_api_key}",
|
||||
"SCRAPECREATORS_API_KEY": "${user_config.scrapecreators_api_key}",
|
||||
"OPENROUTER_API_KEY": "${user_config.openrouter_api_key}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"user_config": {
|
||||
"openai_api_key": {
|
||||
"type": "string",
|
||||
"title": "OPENAI_API_KEY",
|
||||
"description": "OpenAI API key. Powers Reddit research via OpenAI's web_search tool. Get one at https://platform.openai.com/api-keys.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"xai_api_key": {
|
||||
"type": "string",
|
||||
"title": "XAI_API_KEY",
|
||||
"description": "xAI API key. Powers X / Twitter research via xAI's x_search tool. Get one at https://console.x.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"brave_api_key": {
|
||||
"type": "string",
|
||||
"title": "BRAVE_API_KEY",
|
||||
"description": "Brave Search API key. Used for grounded web search results. Get one at https://brave.com/search/api/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"exa_api_key": {
|
||||
"type": "string",
|
||||
"title": "EXA_API_KEY",
|
||||
"description": "Exa search API key. Alternative web search backend with semantic ranking. Get one at https://exa.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"serper_api_key": {
|
||||
"type": "string",
|
||||
"title": "SERPER_API_KEY",
|
||||
"description": "Serper API key. Google search via API. Get one at https://serper.dev/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"google_api_key": {
|
||||
"type": "string",
|
||||
"title": "GOOGLE_API_KEY",
|
||||
"description": "Google API key for YouTube transcript fetching and other Google services. Get one at https://console.cloud.google.com/apis/credentials.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"gemini_api_key": {
|
||||
"type": "string",
|
||||
"title": "GEMINI_API_KEY",
|
||||
"description": "Gemini API key. Used for synthesis fallback when other LLM providers are unavailable. Get one at https://aistudio.google.com/apikey.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"google_genai_api_key": {
|
||||
"type": "string",
|
||||
"title": "GOOGLE_GENAI_API_KEY",
|
||||
"description": "Alternative Google generative-AI API key. Same source as GEMINI_API_KEY; set whichever name your tooling expects.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"apify_api_token": {
|
||||
"type": "string",
|
||||
"title": "APIFY_API_TOKEN",
|
||||
"description": "Apify API token. Powers TikTok and Instagram Reels search via Apify actors. Get one at https://console.apify.com/account/integrations.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"bsky_app_password": {
|
||||
"type": "string",
|
||||
"title": "BSKY_APP_PASSWORD",
|
||||
"description": "Bluesky app password (not your main password). Powers AT Protocol post search. Create at https://bsky.app/settings/app-passwords.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"parallel_api_key": {
|
||||
"type": "string",
|
||||
"title": "PARALLEL_API_KEY",
|
||||
"description": "Parallel AI key. Powers parallel research runs across sources. Get one at https://parallel.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"scrapecreators_api_key": {
|
||||
"type": "string",
|
||||
"title": "SCRAPECREATORS_API_KEY",
|
||||
"description": "ScrapeCreators API key. Powers creator-focused social search across TikTok, Instagram, and YouTube. Get one at https://scrapecreators.com/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"openrouter_api_key": {
|
||||
"type": "string",
|
||||
"title": "OPENROUTER_API_KEY",
|
||||
"description": "OpenRouter API key. Alternative LLM provider gateway for synthesis. Get one at https://openrouter.ai/keys.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"compatibility": {
|
||||
"claude_desktop": ">=1.0.0",
|
||||
"platforms": [
|
||||
"darwin",
|
||||
"linux"
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mirrors skills/last30days/scripts/{last30days.py,lib/} into mcp/vendored/
|
||||
# so the Go binary's embed.FS captures the engine at build time.
|
||||
#
|
||||
# Source of truth: skills/last30days/scripts/. Never edit mcp/vendored/ directly.
|
||||
# Run before `go build` locally and in CI before `printing-press bundle`.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MCP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
REPO_ROOT="$(cd "${MCP_DIR}/.." && pwd)"
|
||||
ENGINE_SRC="${REPO_ROOT}/skills/last30days/scripts"
|
||||
# Embed path must live inside the consuming package (Go //go:embed cannot
|
||||
# reach outside its own directory tree), so vendored/ sits under engine/.
|
||||
VENDORED="${MCP_DIR}/internal/engine/vendored"
|
||||
|
||||
if [ ! -f "${ENGINE_SRC}/last30days.py" ]; then
|
||||
echo "sync-engine: ${ENGINE_SRC}/last30days.py not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${VENDORED}"
|
||||
# Clear stale content while keeping the .gitkeep that anchors the embed path.
|
||||
find "${VENDORED}" -mindepth 1 -not -name ".gitkeep" -delete
|
||||
|
||||
# Copy the entry script and the lib/ tree (modules + lib/vendor/).
|
||||
cp "${ENGINE_SRC}/last30days.py" "${VENDORED}/last30days.py"
|
||||
cp -R "${ENGINE_SRC}/lib" "${VENDORED}/lib"
|
||||
|
||||
# Strip caches so the embed.FS stays deterministic.
|
||||
find "${VENDORED}" -type d -name "__pycache__" -prune -exec rm -rf {} +
|
||||
find "${VENDORED}" -type f -name "*.pyc" -delete
|
||||
|
||||
echo "sync-engine: vendored engine at ${VENDORED}"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 8.6 MiB |
+48
-2
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.3.2"
|
||||
version = "3.21.0"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -8,10 +8,51 @@ dependencies = []
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3,<10",
|
||||
"pytest>=9.1.1,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
"pyyaml>=6.0.2,<7",
|
||||
"towncrier>=25.8.0,<26",
|
||||
]
|
||||
|
||||
[tool.towncrier]
|
||||
name = "last30days-skill"
|
||||
directory = "changelog.d"
|
||||
filename = "CHANGELOG.md"
|
||||
start_string = "<!-- towncrier release notes start -->\n"
|
||||
underlines = ["", "", ""]
|
||||
title_format = "## [{version}] - {project_date}"
|
||||
issue_format = "[#{issue}](https://github.com/mvanhorn/last30days-skill/issues/{issue})"
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "security"
|
||||
name = "Security"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "removed"
|
||||
name = "Removed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "deprecated"
|
||||
name = "Deprecated"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "added"
|
||||
name = "Added"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "changed"
|
||||
name = "Changed"
|
||||
showcontent = true
|
||||
|
||||
[[tool.towncrier.type]]
|
||||
directory = "fixed"
|
||||
name = "Fixed"
|
||||
showcontent = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
@@ -31,6 +72,11 @@ omit = [
|
||||
[tool.coverage.report]
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
# Coverage gate (issue #254). Floor intended to rise over time, not a ceiling.
|
||||
# Baseline measured 2026-07-03 on main before feat/hosted-api-mode
|
||||
# (source = scripts + tests): TOTAL 84.06%. Gate pinned at that baseline.
|
||||
# Do not lower without documenting why in the PR (see AGENTS.md Rules).
|
||||
fail_under = 84
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Hermes scans from this skill directory, not the repository root.
|
||||
# Keep non-runtime packaging/dev/eval artifacts out of install-time security scans.
|
||||
assets/
|
||||
agents/
|
||||
scripts/build-skill.sh
|
||||
scripts/compare.sh
|
||||
scripts/evaluate_search_quality.py
|
||||
scripts/test_device_auth.py
|
||||
scripts/test-v1-vs-v2.sh
|
||||
scripts/verify_v3.py
|
||||
|
||||
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
|
||||
scripts/lib/vendor/
|
||||
+659
-108
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,16 @@
|
||||
# Save shareable HTML brief
|
||||
|
||||
This reference file is loaded by the main `SKILL.md` when the user asked for an HTML brief (either explicitly via `--emit=html` / `--emit:html` / `--html`, or in natural language - "give me a shareable HTML brief", "for Slack", "for Notion", "export as HTML", etc.). The detection happens in `SKILL.md` so that the common no-HTML path stays short; the implementation lives here.
|
||||
This reference file is loaded by the main `SKILL.md` when the user asked for an HTML brief (either through an HTML-looking prompt argument like `--emit=html` / `--emit:html` / `--html`, or in natural language - "give me a shareable HTML brief", "give it to me in HTML", "for Slack", "for Notion", "export as HTML", etc.). The detection happens in `SKILL.md` so that the common no-HTML path stays short; the implementation lives here. Those prompt arguments are user intent signals for the skill; they are not the full Python CLI contract.
|
||||
|
||||
The contract: the synthesis still appears in chat as the primary output. The HTML is an additional artifact saved to disk for sharing. Both happen in the same turn.
|
||||
The contract has two modes:
|
||||
|
||||
- **HTML as the requested deliverable** (`--emit=html`, `--emit:html`, `--html`, or prose like "give it to me in HTML"): the HTML artifact is the primary output. Write the synthesis to the temp file, render the HTML, then give a concise artifact handoff in chat instead of pasting the full Markdown report again.
|
||||
- **Normal report plus HTML copy** (the user asks for the normal report and also wants an HTML copy): the synthesis still appears in chat as the primary output. The HTML is an additional artifact saved to disk for sharing. Both happen in the same turn.
|
||||
|
||||
## When to fire this flow
|
||||
|
||||
- After you have already emitted the full chat response: badge, "What I learned:" (or comparison title), bold-lead-in paragraphs with citations, KEY PATTERNS list, engine footer pass-through, invitation block.
|
||||
- For normal-report-plus-HTML mode: after you have already emitted the full chat response: badge, "What I learned:" (or comparison title), bold-lead-in paragraphs with citations, KEY PATTERNS list, engine footer pass-through, invitation block.
|
||||
- For HTML-as-deliverable mode: after you have drafted the synthesis that will go into the HTML, before emitting the final chat response.
|
||||
- BEFORE the WAIT FOR USER'S RESPONSE pause.
|
||||
- ONLY if the user asked. Do NOT save HTML when the user didn't ask for it.
|
||||
|
||||
@@ -15,14 +19,19 @@ The contract: the synthesis still appears in chat as the primary output. The HTM
|
||||
```bash
|
||||
# 1. Write your synthesis prose VERBATIM to a temp file. The synthesis is the
|
||||
# "What I learned:" prose label, the bold-lead-in paragraphs with their
|
||||
# inline citations as you wrote them in chat, and the "KEY PATTERNS from
|
||||
# the research:" numbered list. Do NOT include the badge or the engine
|
||||
# footer in the temp file - the engine adds those when it renders the HTML.
|
||||
# Use the EXACT text you just wrote in chat. Do not paraphrase, do not
|
||||
# summarize, do not reorder. The HTML must read identically to the chat
|
||||
# response in voice and citations.
|
||||
# inline citations, and the "KEY PATTERNS from the research:" numbered list.
|
||||
# Do NOT include the badge or the engine footer in the temp file - the engine
|
||||
# adds those when it renders the HTML.
|
||||
# - HTML-as-deliverable mode: use the exact synthesis draft you prepared for
|
||||
# the artifact. Do not paste it to chat first.
|
||||
# - Normal-report-plus-HTML mode: use the exact synthesis text you already
|
||||
# wrote in chat.
|
||||
# In both modes, do not paraphrase, summarize, or reorder. The HTML must read
|
||||
# identically to the intended report in voice and citations.
|
||||
SYNTHESIS_FILE="/tmp/last30days-synthesis-${CLAUDE_SESSION_ID}.md"
|
||||
cat > "$SYNTHESIS_FILE" <<'SYNTHESIS_EOF'
|
||||
# >| not >: fixed path may already exist on a same-session re-run; a plain >
|
||||
# is refused under `set -o noclobber`.
|
||||
cat >| "$SYNTHESIS_FILE" <<'SYNTHESIS_EOF'
|
||||
What I learned:
|
||||
|
||||
**{First headline}** - {body with [name](url) inline citations}
|
||||
@@ -39,12 +48,15 @@ SYNTHESIS_EOF
|
||||
|
||||
# 2. Convert the synthesis to a self-contained HTML file via the engine.
|
||||
# REPLAY THE SAME SCOPE FLAGS as your original run (--plan, --hiring-signals,
|
||||
# resolved --x-handle/--subreddits/etc). The engine re-runs the pipeline to
|
||||
# build the badge metadata line and footer; if you omit the scope flags it
|
||||
# takes the generic multi-source path and the artifact's footer/metadata will
|
||||
# describe a DIFFERENT dataset than your synthesis body (observed: a 74s
|
||||
# mismatched re-run on a --hiring-signals brief). Same flags = footer matches
|
||||
# the brief.
|
||||
# resolved --x-handle/--subreddits/etc). On a same-topic follow-up, the
|
||||
# engine reuses the structured last-report cache at
|
||||
# ~/.config/last30days/last-report.json to build badge metadata and footer
|
||||
# without re-running source fetchers. That cache is intentionally short-lived
|
||||
# (default: one hour; tune with LAST30DAYS_REPORT_CACHE_TTL_SECONDS, or set
|
||||
# it to 0 to disable reuse). If the cache is stale, missing, or for a
|
||||
# different topic, stderr says "No matching cached report data" and the
|
||||
# engine falls back to a fresh run; the same scope flags keep that fallback
|
||||
# aligned with the synthesis body.
|
||||
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/^-//;s/-$//')
|
||||
HTML_PATH="${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html"
|
||||
# Collision guard: the `> "$HTML_PATH"` redirect below OVERWRITES - the engine
|
||||
@@ -58,18 +70,95 @@ fi
|
||||
--emit=html \
|
||||
--synthesis-file "$SYNTHESIS_FILE" \
|
||||
"${SCOPE_FLAGS[@]}" \
|
||||
> "$HTML_PATH"
|
||||
>| "$HTML_PATH" # >| not >: noclobber-safe write to the collision-guarded path
|
||||
# where SCOPE_FLAGS is the same array you passed the first time, e.g.
|
||||
# SCOPE_FLAGS=(--hiring-signals --plan "$QUERY_PLAN_FILE" --x-handle=acme).
|
||||
# For a scoped --hiring-signals brief, --hiring-signals MUST be here too so
|
||||
# the footer reflects the jobs-scoped board, not a generic crawl.
|
||||
|
||||
# 3. Append ONE line to your already-emitted chat response, after the
|
||||
# invitation block. Use a paperclip emoji as a visible signal that an
|
||||
# artifact was produced:
|
||||
echo "📎 Shareable brief saved to $HTML_PATH"
|
||||
# 3. Finish with the artifact handoff described below. Do not print the saved
|
||||
# path from the shell block; the chat handoff is the single user-visible
|
||||
# completion message.
|
||||
```
|
||||
|
||||
## Optional hosted publishing
|
||||
|
||||
Only publish after the local HTML file has already been saved and the user chooses a publish option. The local HTML save is always first, and its absolute path is always shown before any publish/upload step.
|
||||
|
||||
Respect any existing user, project, or host preference for HTML publishing first. If the user already has a preferred publisher or internal sharing workflow, include that option. If multiple publishing options are available, show each as its own choice and include `ht-ml.app` as one option; label `ht-ml.app` as supporting optional password protection. If no preference exists, use `ht-ml.app` as the fallback publishing option.
|
||||
|
||||
Use this decision flow:
|
||||
|
||||
- Save the local HTML file.
|
||||
- Show the absolute saved path.
|
||||
- Then proactively present next-step choices:
|
||||
1. Open HTML file
|
||||
2. Publish to `<preferred/configured service>`; if `ht-ml.app` is shown, say password protection is available
|
||||
3. Done for now
|
||||
- Do not upload until the user chooses a publishing option.
|
||||
|
||||
When publishing to `ht-ml.app`, ask a second question:
|
||||
|
||||
- **Public link** - publish without a password.
|
||||
- **Password-protected link** - ask the user to type the shared password in free form, then publish with that password.
|
||||
|
||||
Before the `ht-ml.app` choice, tell the user that public pages may be crawled or indexed, and that password protection is available. If the user chooses password protection, use a unique shared password they provide for this report; do not use their own account password.
|
||||
|
||||
Agents should discover the current publishing mechanics for the selected service when needed, including by visiting the service site, rather than hard-coding detailed service-specific instructions in chat. For the built-in `ht-ml.app` path, the engine supports `--publish-html`; on the password-protected branch, pass the shared password through `LAST30DAYS_PUBLISH_PASSWORD` rather than command-line arguments.
|
||||
|
||||
When the user chooses the built-in `ht-ml.app` path, add `--publish-html` to the same `--emit=html` command. Use `--output "$HTML_PATH"` rather than shell redirection so the engine can write the `.publish.json` companion metadata next to the local HTML file. On the password-protected branch, set `LAST30DAYS_PUBLISH_PASSWORD` in the subprocess environment instead of passing `--publish-password` in the shell command.
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PUBLISH_PASSWORD="${PUBLISH_PASSWORD:-}" \
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "${TOPIC}" \
|
||||
--emit=html \
|
||||
--synthesis-file "$SYNTHESIS_FILE" \
|
||||
--output "$HTML_PATH" \
|
||||
--publish-html \
|
||||
"${SCOPE_FLAGS[@]}" \
|
||||
>/dev/null
|
||||
```
|
||||
|
||||
The hosted URL appears on stderr as `[last30days] Published HTML to https://...`. Confirm the result with the hosted URL. If the user chose password protection, also repeat the shared password they selected so they can send the URL and password together. The engine writes URL metadata to `<HTML_PATH>.publish.json`. The provider may return an `update_key`; treat it as secret. The engine deliberately does not write the update key to stdout, the HTML artifact, or `.publish.json` companion metadata.
|
||||
|
||||
## Chat handoff after saving
|
||||
|
||||
Use the mode that matches the request.
|
||||
|
||||
### HTML as the requested deliverable
|
||||
|
||||
When HTML is the requested deliverable - whether by `--emit=html`, `--emit:html`, `--html`, or natural-language phrasing - do **not** paste the full Markdown report back into chat after saving the artifact. The user asked for an HTML deliverable; repeating the Markdown makes the run feel like a normal report with an attachment bolted on.
|
||||
|
||||
Respond with a concise handoff that includes the next-step choices:
|
||||
|
||||
```text
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
|
||||
📎 Shareable brief saved to <absolute HTML path>
|
||||
|
||||
What do you want to do next?
|
||||
1. Open HTML file
|
||||
2. Publish to <available HTML publishing service> (<service-specific note, e.g. ht-ml.app supports optional password protection>)
|
||||
3. Done for now
|
||||
```
|
||||
|
||||
If the user chooses open, open the HTML file when the host can safely open local files, leave the saved-path line in chat, and add `Opened locally.` Let the host choose the correct OS-specific mechanism; do not print a menu of shell commands. If opening fails or the host is headless, do not treat that as a failed report; show the path and say the file is ready to open in a browser.
|
||||
|
||||
### Normal report plus HTML copy
|
||||
|
||||
When the user asked for a normal `/last30days` report and also asked for an HTML copy, keep the full chat synthesis and append this artifact block after the invitation:
|
||||
|
||||
```text
|
||||
📎 Shareable brief saved to <absolute HTML path>
|
||||
|
||||
What do you want to do next?
|
||||
1. Open HTML file
|
||||
2. Publish to <available HTML publishing service> (<service-specific note, e.g. ht-ml.app supports optional password protection>)
|
||||
3. Done for now
|
||||
```
|
||||
|
||||
If the user chooses open, open it when the host can safely open local files; otherwise the saved-path line is enough. Do not upload in this flow unless the user chooses a publishing option.
|
||||
|
||||
## What ends up in the HTML file
|
||||
|
||||
The engine's `--emit=html` renderer combines:
|
||||
@@ -89,15 +178,23 @@ Same flow when the topic is `X vs Y` (or `X vs Y vs Z`). The engine routes throu
|
||||
|
||||
## Follow-up turn
|
||||
|
||||
If the user runs `/last30days OpenClaw` normally, sees the synthesis in chat, and THEN says "save that as HTML" or "give me a shareable version" in a follow-up turn, do the same save flow on the synthesis you wrote in the previous turn. Do not re-research; the synthesis is already in the conversation history. Just write it to the temp file and call the engine with `--emit=html --synthesis-file`.
|
||||
If the user runs `/last30days OpenClaw` normally, sees the synthesis in chat, and THEN explicitly refers back to that visible synthesis ("save that as HTML", "make this shareable", "turn the above into HTML"), do the same save flow on the synthesis you wrote in the previous turn. Do not re-research; the synthesis is already in the conversation history. Just write it to the temp file and call the engine with `--emit=html --synthesis-file`, then use the normal-report-plus-HTML artifact block.
|
||||
|
||||
If the follow-up instead asks for a new HTML deliverable ("give it to me in HTML", `--emit=html`, `--html`) rather than referring back to an already-visible report, treat it as HTML-as-deliverable mode.
|
||||
|
||||
The engine will try to reuse `~/.config/last30days/last-report.json` for that second invocation when it is still within `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` (default: one hour). If stderr says it is reusing cached report data, continue normally. If stderr says no matching cache exists, the cache may be stale; let the command finish only if you supplied the same scope flags as the original run. Otherwise stop and re-run with the original flags so the HTML footer does not describe a different dataset.
|
||||
|
||||
## What NOT to do
|
||||
|
||||
- Do NOT save HTML if the user didn't ask. The sparse mode (no synthesis) produces a thin file; not useful as a shareable.
|
||||
- Do NOT add content to the temp file beyond your synthesis prose. The badge / footer / colophon come from the engine.
|
||||
- Do NOT change the file path convention. `${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html` is the canonical location.
|
||||
- Do NOT silently overwrite an existing file. The `--emit=html` output is written via a shell redirect (`> "$HTML_PATH"`), which OVERWRITES - the engine does NOT auto-date the brief (its date-suffix logic only applies to `--save-dir` raw files). The collision guard in step 2 handles this: if `{slug}-brief.html` already exists it date-suffixes to `{slug}-brief-YYYY-MM-DD.html`. Always print whichever path the redirect actually used. First save = clean `{slug}-brief.html` (no datestamp - that is expected); a datestamp appears only on a same-name collision.
|
||||
- Do NOT silently overwrite an existing file. The `--emit=html` output is written via a shell redirect (`>| "$HTML_PATH"`), which OVERWRITES the collision-guarded path — use `>|` not `>` because `set -o noclobber` refuses plain `>` when the file already exists. The collision guard in step 2 handles same-topic re-runs: if `{slug}-brief.html` already exists it date-suffixes to `{slug}-brief-YYYY-MM-DD.html`. Always report whichever path the redirect actually used in the chat handoff.
|
||||
- Do NOT include the data quality warning text in the temp file or in your final chat line. Warnings are an engine-stderr concern, not an artifact concern.
|
||||
- Do NOT publish, upload, or send the HTML to a third-party service as part of the local save flow.
|
||||
- Do NOT publish to any service merely because HTML was requested. Show the saved path and next-step choices first; publishing requires the user to choose a publish option.
|
||||
- Do NOT block a local HTML export on a hosting decision unless the user explicitly asked for a hosted URL.
|
||||
- Do NOT paste or store the `update_key` in chat, Markdown, HTML, raw output, or companion metadata.
|
||||
|
||||
## Edge cases
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ def generate_daily(since: str = None) -> dict:
|
||||
|
||||
# Extract top finding by engagement
|
||||
if findings:
|
||||
top = max(findings, key=lambda f: f.get("engagement_score", 0))
|
||||
top = max(findings, key=lambda f: f.get("engagement_score") or 0)
|
||||
topic_data["top_finding"] = {
|
||||
"title": top.get("source_title", ""),
|
||||
"source": top.get("source", ""),
|
||||
@@ -110,7 +110,7 @@ def generate_daily(since: str = None) -> dict:
|
||||
|
||||
top_overall = None
|
||||
if all_findings:
|
||||
top_overall = max(all_findings, key=lambda f: f.get("engagement_score", 0))
|
||||
top_overall = max(all_findings, key=lambda f: f.get("engagement_score") or 0)
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
@@ -172,8 +172,8 @@ def generate_weekly() -> dict:
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
this_engagement = sum(f.get("engagement_score", 0) for f in this_week)
|
||||
last_engagement = sum(f.get("engagement_score", 0) for f in last_week)
|
||||
this_engagement = sum(f.get("engagement_score") or 0 for f in this_week)
|
||||
last_engagement = sum(f.get("engagement_score") or 0 for f in last_week)
|
||||
|
||||
# Trend calculation
|
||||
if last_engagement > 0:
|
||||
@@ -188,7 +188,15 @@ def generate_weekly() -> dict:
|
||||
"this_week_engagement": this_engagement,
|
||||
"last_week_engagement": last_engagement,
|
||||
"engagement_change_pct": round(engagement_change, 1),
|
||||
"top_findings": this_week[:5], # Top 5 by engagement (already sorted)
|
||||
# get_new_findings returns first_seen DESC, so sort by engagement
|
||||
# before slicing — otherwise the digest headlines the most recent
|
||||
# items, not the highest-engagement ones (the daily path keys on
|
||||
# engagement too).
|
||||
"top_findings": sorted(
|
||||
this_week,
|
||||
key=lambda f: f.get("engagement_score") or 0,
|
||||
reverse=True,
|
||||
)[:5],
|
||||
})
|
||||
|
||||
result = {
|
||||
|
||||
@@ -28,7 +28,7 @@ echo ""
|
||||
# Run 1: public release
|
||||
echo "[1/2] Running /last30days (public release)..."
|
||||
echo " This takes 2-4 minutes..."
|
||||
claude -p --dangerously-skip-permissions "/last30days $TOPIC" > /dev/null 2>&1 || true
|
||||
claude -p "/last30days $TOPIC" > /dev/null 2>&1 || true
|
||||
RELEASE_FILE="$DIR/${SLUG}-raw.md"
|
||||
[ -f "$RELEASE_FILE" ] && echo " Done: $RELEASE_FILE" || echo " FAILED: no output file"
|
||||
echo ""
|
||||
@@ -39,7 +39,7 @@ sleep 30
|
||||
# Run 2: private beta
|
||||
echo "[2/2] Running /last30days-beta (private beta)..."
|
||||
echo " This takes 2-4 minutes..."
|
||||
claude -p --dangerously-skip-permissions "/last30days-beta $TOPIC" > /dev/null 2>&1 || true
|
||||
claude -p "/last30days-beta $TOPIC" > /dev/null 2>&1 || true
|
||||
BETA_FILE="$DIR/${SLUG}-raw-beta.md"
|
||||
[ -f "$BETA_FILE" ] && echo " Done: $BETA_FILE" || echo " FAILED: no output file"
|
||||
echo ""
|
||||
|
||||
@@ -46,6 +46,19 @@ DEFAULT_TOPICS = _load_default_topics()
|
||||
DEFAULT_SEARCH = ""
|
||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
EVAL_CREDENTIAL_ENV_KEYS = (
|
||||
"GOOGLE_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"SCRAPECREATORS_API_KEY",
|
||||
"BSKY_HANDLE",
|
||||
"BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN",
|
||||
"AUTH_TOKEN",
|
||||
"CT0",
|
||||
)
|
||||
|
||||
|
||||
def stable_item_key(item: dict[str, Any]) -> str:
|
||||
@@ -269,12 +282,30 @@ def get_judgments(
|
||||
) -> dict[str, int]:
|
||||
cache_file = output_dir / "judgments" / f"{slug}.json"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale_cache = False
|
||||
if cache_file.exists():
|
||||
payload = json.loads(cache_file.read_text())
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
# The cache key is the topic slug alone, but judgments are model-
|
||||
# specific. Only reuse the cache when it was produced by the same judge
|
||||
# model; otherwise re-judge, so a --judge-model change cannot return
|
||||
# stale grades that silently skew precision@k / nDCG. Caches written
|
||||
# before judge_model was recorded miss here and get refreshed once.
|
||||
if payload.get("judge_model") == judge_model:
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
stale_cache = True
|
||||
if not gemini_api_key or not items:
|
||||
if stale_cache:
|
||||
# Discarded a different-model cache but can't re-judge. Returning {}
|
||||
# scores every item as ungraded (zero precision@k / nDCG); say so
|
||||
# rather than letting the run report silently wrong numbers.
|
||||
sys.stderr.write(
|
||||
f"[Eval] Cached judgments for {slug!r} were graded by a different "
|
||||
f"judge model and no Gemini API key is set to re-judge; returning "
|
||||
f"no grades (metrics for this topic will be zero).\n"
|
||||
)
|
||||
return {}
|
||||
payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items))
|
||||
payload["judge_model"] = judge_model
|
||||
cache_file.write_text(json.dumps(payload, indent=2))
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
|
||||
@@ -289,19 +320,7 @@ def create_eval_env() -> dict[str, str]:
|
||||
"PYTHONUTF8": "1",
|
||||
"LAST30DAYS_CONFIG_DIR": "",
|
||||
}
|
||||
for key in (
|
||||
"GOOGLE_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"SCRAPECREATORS_API_KEY",
|
||||
"BSKY_HANDLE",
|
||||
"BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN",
|
||||
"AUTH_TOKEN",
|
||||
"CT0",
|
||||
):
|
||||
for key in EVAL_CREDENTIAL_ENV_KEYS:
|
||||
value = os.environ.get(key) or config.get(key)
|
||||
if value:
|
||||
passthrough[key] = value
|
||||
@@ -313,6 +332,12 @@ def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds:
|
||||
if not engine.exists():
|
||||
engine = repo_dir / "scripts" / "last30days.py"
|
||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
||||
# Current engines default to the stable agent export, while older revisions
|
||||
# used by the evaluator implicitly emit the raw report and do not recognize
|
||||
# --json-profile. Request raw explicitly whenever the checked-out engine
|
||||
# supports the selector.
|
||||
if not engine.exists() or "--json-profile" in engine.read_text(encoding="utf-8"):
|
||||
cmd.append("--json-profile=raw")
|
||||
if search:
|
||||
cmd.extend(["--search", search])
|
||||
if quick:
|
||||
@@ -330,7 +355,16 @@ def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds:
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"{repo_dir.name} failed for '{topic}' with exit {result.returncode}\n{result.stderr.strip()}")
|
||||
return json.loads(result.stdout)
|
||||
payload = json.loads(result.stdout)
|
||||
# Shape guard: the evaluator compares raw Report fields. If the engine
|
||||
# emitted the agent profile anyway (flag detection missed a future
|
||||
# spelling), fail loudly instead of scoring empty ranked_candidates.
|
||||
if "schema_version" in payload and "ranked_candidates" not in payload:
|
||||
raise RuntimeError(
|
||||
f"{repo_dir.name} emitted the agent JSON profile; the evaluator "
|
||||
"requires the raw Report (--json-profile=raw)."
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def create_worktree(rev: str) -> Path:
|
||||
|
||||
+2825
-178
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,877 @@
|
||||
"""Amazon product and review signals via the Bright Data CLI.
|
||||
|
||||
Two-stage source, following the digg discover-then-enrich shape:
|
||||
|
||||
1. **Discovery** -- one ``amazon_product_search`` per run turns a
|
||||
model-supplied product keyword into product records carrying live
|
||||
aggregate stats (rating, rating count, price). Cheap and fast.
|
||||
2. **Enrichment** -- ``amazon_product_reviews`` pulls a capped sample of
|
||||
written reviews for the top few surviving products, in parallel, under
|
||||
a lane deadline. Reviews ride on their product item as metadata
|
||||
comments and feed community-voice weaving.
|
||||
|
||||
The signature signal is the fusion of those two: an all-time rating from
|
||||
thousands of ratings, set against the average of just the reviews inside
|
||||
the last-30-day window. When those disagree, something changed this month,
|
||||
and the review text says what. No Amazon page shows that.
|
||||
|
||||
Metering (R13): one credit per pipeline request regardless of records
|
||||
returned, so the caps here bound paid-tier *records*, not credits. A
|
||||
default run is 1 search + up to 3 review pulls = 4 requests.
|
||||
|
||||
Field names and quirks below are verified against live payloads pulled
|
||||
2026-08-13; see the plan's schema block. Three fields arrive doubled
|
||||
(``review_posted_date``, ``review_header``, ``badge``) and are repaired
|
||||
here rather than downstream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import brightdata, log
|
||||
from .relevance import token_overlap_relevance
|
||||
|
||||
|
||||
SEARCH_PIPELINE = "amazon_product_search"
|
||||
REVIEWS_PIPELINE = "amazon_product_reviews"
|
||||
|
||||
DEFAULT_DOMAIN = "https://www.amazon.com"
|
||||
|
||||
# Reviews requested per pull. Uniform across topic shapes and depths by
|
||||
# decision: billing is per *request*, not per record, so a bigger cap is
|
||||
# free on the monthly credit tier, and the in-window sample is what the
|
||||
# drift signal rests on. Live-verified that latency does not scale with
|
||||
# this number (50 reviews in 22s vs 20 reviews in 115s on a slower SKU).
|
||||
#
|
||||
# It is a ceiling, never a quota -- a SKU with 31 total reviews returns 31.
|
||||
MAX_REVIEWS = 50
|
||||
|
||||
# How many products get a review pull, per depth. Quick spends one credit
|
||||
# on discovery only: aggregate stats with no recent window.
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 0,
|
||||
"default": 3,
|
||||
"deep": 5,
|
||||
}
|
||||
|
||||
SEARCH_TIMEOUT = 90
|
||||
REVIEW_TIMEOUT = 180
|
||||
|
||||
# Wall-clock ceiling for the whole parallel review lane. Pulls that miss it
|
||||
# are abandoned, and their products degrade to `quiet` rather than
|
||||
# disappearing (a slow SKU is real and unrelated to the cap: one live pull
|
||||
# took 115s).
|
||||
LANE_DEADLINE = 180
|
||||
|
||||
# The engine's foreground contract. The lane deadline is clamped against
|
||||
# whatever remains of it, minus room to render.
|
||||
FOREGROUND_CONTRACT = 300
|
||||
RENDER_MARGIN = 20
|
||||
|
||||
# Minimum useful budget for the review lane. Below this threshold, Bright
|
||||
# Data pulls reliably time out (cli_timeout = max(5, timeout-10), so budget
|
||||
# 11s → CLI timeout 1s). Crumbs are a skip, not a short timeout: firing
|
||||
# doomed pulls still spends 3 credits with no reviews returned.
|
||||
MIN_USEFUL_REVIEW_BUDGET = 90
|
||||
|
||||
# Minimum dated reviews inside the window before a drift arrow is honest.
|
||||
# Live census: a 50-cap pull returned 31 records of which only 5 were
|
||||
# inside 30 days, so an unguarded arrow would routinely publish a "trend"
|
||||
# computed from one or two reviews.
|
||||
MIN_DRIFT_SAMPLE = 5
|
||||
|
||||
RECENT_WINDOW_DAYS = 30
|
||||
|
||||
# Product names run long and pipe-delimited; the footer needs a scannable
|
||||
# handle, not a title.
|
||||
SHORT_NAME_MAX = 18
|
||||
|
||||
_STAR_FIELDS = (
|
||||
("one_star", 1),
|
||||
("two_star", 2),
|
||||
("three_star", 3),
|
||||
("four_star", 4),
|
||||
("five_star", 5),
|
||||
)
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("Amazon", msg, tty_only=False)
|
||||
|
||||
|
||||
def _today() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# --------------------------------------------------------------- parsing
|
||||
|
||||
|
||||
def undouble(text: str) -> str:
|
||||
"""Repair the CLI's doubled string fields.
|
||||
|
||||
Observed live: ``review_header`` arrives as ``"Best Box!Best Box!"`` and
|
||||
``badge`` as ``"Verified Purchase, Verified Purchase"``. Handles the
|
||||
exact-repeat case and the comma-joined repeat, and leaves anything else
|
||||
untouched -- a genuinely repetitive title must survive intact.
|
||||
"""
|
||||
value = (text or "").strip()
|
||||
if not value:
|
||||
return ""
|
||||
half, odd = divmod(len(value), 2)
|
||||
# Only treat an exact repeat as doubling when the halves are substantial
|
||||
# and look like a phrase rather than a syllable -- otherwise a real title
|
||||
# of "ByeBye" or "NoNo" gets silently truncated to half of itself. The
|
||||
# observed artifact doubles whole headlines, so requiring some length and
|
||||
# either whitespace or terminal punctuation keeps the repair targeted.
|
||||
if not odd and half >= 6 and value[:half] == value[half:]:
|
||||
first = value[:half]
|
||||
if " " in first or first[-1] in ".!?":
|
||||
return first.strip()
|
||||
parts = [p.strip() for p in value.split(",")]
|
||||
if len(parts) == 2 and parts[0] and parts[0] == parts[1]:
|
||||
return parts[0]
|
||||
return value
|
||||
|
||||
|
||||
_DATE_HEAD = re.compile(r"^([A-Z][a-z]+ \d{1,2}, \d{4})")
|
||||
|
||||
|
||||
def parse_review_date(raw: Any) -> Optional[str]:
|
||||
"""Pull the ISO date out of the CLI's prose-wrapped date field.
|
||||
|
||||
Live shape: ``"August 3, 2026Reviewed in the United States on August 3,
|
||||
2026"``. Only the leading ``%B %d, %Y`` is trustworthy; the tail is
|
||||
localized prose that varies by marketplace.
|
||||
|
||||
Returns ``YYYY-MM-DD`` or None.
|
||||
"""
|
||||
match = _DATE_HEAD.match(str(raw or "").strip())
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(match.group(1), "%B %d, %Y").date().isoformat()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def short_name(name: str, brand: str = "") -> str:
|
||||
"""Derive a scannable footer handle from a long product name.
|
||||
|
||||
Live names are pipe-delimited marketing strings with the brand carried
|
||||
in its own field rather than as a prefix ("Chill Max Leak-Proof XL
|
||||
Bento-Style Lunch Box | Included Ice Pack Keeps Food Cold"). Take the
|
||||
segment before the first delimiter, drop a leading brand token if one
|
||||
did sneak in, and clip to a scannable width on a word boundary.
|
||||
"""
|
||||
text = re.split(r"[|(–—]", str(name or ""), maxsplit=1)[0].strip(" -,")
|
||||
brand_token = str(brand or "").strip()
|
||||
if brand_token:
|
||||
# Word-boundary anchored: a bare startswith() eats into sub-brands and
|
||||
# coincidental prefixes ("AnkerWork" under brand "Anker" would become
|
||||
# "Work", "Chillax" under "Chill" would become "ax").
|
||||
stripped = re.sub(
|
||||
rf"^{re.escape(brand_token)}\b[\s\-,]*", "", text, count=1, flags=re.IGNORECASE
|
||||
)
|
||||
if stripped:
|
||||
text = stripped.strip(" -,")
|
||||
if len(text) <= SHORT_NAME_MAX:
|
||||
return text
|
||||
clipped = text[:SHORT_NAME_MAX].rsplit(" ", 1)[0].strip(" -,")
|
||||
return clipped or text[:SHORT_NAME_MAX].strip()
|
||||
|
||||
|
||||
def _as_float(value: Any) -> Optional[float]:
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _is_sponsored(value: Any) -> bool:
|
||||
"""The flag arrives as the string 'true'/'false', not a bool.
|
||||
|
||||
Recorded in metadata but never used to filter (R4): its distribution
|
||||
swings hard with keyword phrasing, so filtering on it can blank the
|
||||
lane on an unlucky query.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value or "").strip().lower() == "true"
|
||||
|
||||
|
||||
def _valid_product_url(url: str, domain: str) -> bool:
|
||||
"""Accept only https URLs on the configured Amazon host."""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
expected = urlparse(domain or DEFAULT_DOMAIN)
|
||||
except ValueError:
|
||||
return False
|
||||
if parsed.scheme != "https" or not parsed.netloc:
|
||||
return False
|
||||
host = parsed.netloc.lower().removeprefix("www.")
|
||||
want = (expected.netloc or "").lower().removeprefix("www.")
|
||||
return bool(want) and host == want
|
||||
|
||||
|
||||
# Amazon ASINs are a fixed shape. Validating it matters because the value
|
||||
# is interpolated into a URL that is then refetched through the CLI *and*
|
||||
# rendered as a link in the report -- two sinks, one unvalidated API field.
|
||||
_ASIN_RE = re.compile(r"^[A-Za-z0-9]{10}$")
|
||||
|
||||
|
||||
def _valid_asin(asin: str) -> bool:
|
||||
return bool(_ASIN_RE.match(asin or ""))
|
||||
|
||||
|
||||
def canonical_product_url(url: str, asin: str, domain: str) -> str:
|
||||
"""Strip Amazon's tracking tail down to a stable /dp/<asin> link.
|
||||
|
||||
Search records carry 200+ character URLs with session-scoped ``dib``
|
||||
tokens. Those work but are unreadable in a report and unstable across
|
||||
runs, which breaks dedupe on re-runs of the same topic.
|
||||
|
||||
Falls back to the (already host-validated) original URL if the ASIN is
|
||||
not well-formed, so a malformed record can never shape the rebuilt URL.
|
||||
"""
|
||||
if not _valid_asin(asin):
|
||||
return url
|
||||
base = (domain or DEFAULT_DOMAIN).rstrip("/")
|
||||
return f"{base}/dp/{asin}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------- discovery
|
||||
|
||||
|
||||
def search_products(
|
||||
keyword: str,
|
||||
*,
|
||||
domain: str = DEFAULT_DOMAIN,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = SEARCH_TIMEOUT,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run one product search. Never raises; returns the adapter envelope."""
|
||||
query = (keyword or "").strip()
|
||||
if not query:
|
||||
return {"records": []}
|
||||
# A leading dash would be parsed as a CLI option rather than a search
|
||||
# term. The keyword is model-supplied and can be influenced by
|
||||
# pre-research over untrusted web content, so reject rather than
|
||||
# sanitize -- a keyword starting with '-' is never a real product.
|
||||
if query.startswith("-"):
|
||||
_log(f"rejecting option-shaped keyword: {query!r}")
|
||||
return {"records": [], "error": "amazon keyword may not begin with '-'"}
|
||||
_log(f"search '{query}' on {domain}")
|
||||
response = brightdata.run_pipeline(
|
||||
SEARCH_PIPELINE, [query, domain or DEFAULT_DOMAIN],
|
||||
timeout=timeout, config=config,
|
||||
)
|
||||
if response.get("error"):
|
||||
_log(f"search failed: {response['error']}")
|
||||
else:
|
||||
_log(f"search returned {len(response.get('records') or [])} records")
|
||||
return response
|
||||
|
||||
|
||||
def parse_search_response(
|
||||
response: Dict[str, Any],
|
||||
keyword: str,
|
||||
*,
|
||||
domain: str = DEFAULT_DOMAIN,
|
||||
min_relevance: float = 0.15,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Turn raw search records into deduped, relevance-gated product dicts.
|
||||
|
||||
Dedupe is by ASIN: live payloads repeat a single product up to five
|
||||
times across the result set (64 unique of 66 records on one pull).
|
||||
Relevance is scored against the *supplied keyword*, not the run topic,
|
||||
because the model may search "June Oven" on a topic about a person.
|
||||
"""
|
||||
records = response.get("records") if isinstance(response, dict) else None
|
||||
if not isinstance(records, list):
|
||||
return []
|
||||
|
||||
today = _today().date().isoformat()
|
||||
seen: Dict[str, Dict[str, Any]] = {}
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
asin = str(record.get("asin") or "").strip()
|
||||
raw_url = str(record.get("url") or "").strip()
|
||||
if not _valid_asin(asin) or not _valid_product_url(raw_url, domain):
|
||||
continue
|
||||
|
||||
name = str(record.get("name") or "").strip()
|
||||
brand = str(record.get("brand") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
|
||||
relevance = token_overlap_relevance(keyword, f"{brand} {name}".strip())
|
||||
if relevance < min_relevance:
|
||||
continue
|
||||
|
||||
num_ratings = _as_int(record.get("num_ratings"))
|
||||
existing = seen.get(asin)
|
||||
# Duplicates of one ASIN can disagree on rating count (variant-level
|
||||
# records); keep the richest.
|
||||
if existing and _as_int(existing.get("num_ratings")) >= num_ratings:
|
||||
continue
|
||||
|
||||
seen[asin] = {
|
||||
"asin": asin,
|
||||
# Current-date stamped (KTD6, trustpilot precedent): a live
|
||||
# aggregate rating is a fact about now, not about the product's
|
||||
# launch date, so it must not be dropped by the 30-day filter.
|
||||
"date": today,
|
||||
"name": name,
|
||||
"short_name": short_name(name, brand),
|
||||
"brand": brand,
|
||||
"url": canonical_product_url(raw_url, asin, domain),
|
||||
"rating": _as_float(record.get("rating")),
|
||||
"num_ratings": num_ratings,
|
||||
"price": _as_float(record.get("final_price")),
|
||||
"currency": str(record.get("currency") or "").strip(),
|
||||
"badge": undouble(str(record.get("badge") or "")),
|
||||
"sponsored": _is_sponsored(record.get("sponsored")),
|
||||
"bought_past_month": _as_int(record.get("bought_past_month")),
|
||||
"rank_on_page": _as_int(record.get("rank_on_page")),
|
||||
"relevance": relevance,
|
||||
}
|
||||
|
||||
products = sorted(
|
||||
seen.values(),
|
||||
key=lambda p: (p["num_ratings"], p["relevance"]),
|
||||
reverse=True,
|
||||
)
|
||||
_log(f"{len(products)} unique on-keyword products after dedupe")
|
||||
return products
|
||||
|
||||
|
||||
def infer_brand(products: Sequence[Dict[str, Any]], keyword: str) -> str:
|
||||
"""Detect a brand topic by matching record brands against the keyword.
|
||||
|
||||
This is the guard against paying to review a competitor. Rival brands
|
||||
buy ads against a brand keyword and can outrank the brand's own catalog
|
||||
on raw rating count: on a live "bentgo lunch box" search a competitor
|
||||
held the top two slots and would have taken two of the three review
|
||||
pulls, putting a rival's reviews in a Bentgo report.
|
||||
|
||||
Matching the *keyword's own tokens*, rather than picking the most
|
||||
common brand in the results, is what keeps category topics unfiltered.
|
||||
"best bluetooth speaker" names no brand, so nothing is constrained and
|
||||
the top products across brands compete on merit -- which is exactly
|
||||
what that topic shape wants.
|
||||
"""
|
||||
normalized_keyword = " ".join(re.findall(r"[a-z0-9]+", (keyword or "").lower()))
|
||||
if not normalized_keyword:
|
||||
return ""
|
||||
keyword_tokens = set(normalized_keyword.split())
|
||||
|
||||
# Keyed by the lowercased brand so one vendor spelled two ways ("Bentgo"
|
||||
# and "BENTGO" in the same result set) reads as one candidate. Without
|
||||
# this the set has two members, the function bails, and the guard it
|
||||
# exists to provide silently turns off.
|
||||
candidates: Dict[str, str] = {}
|
||||
for product in products:
|
||||
brand = str(product.get("brand") or "").strip()
|
||||
if not brand:
|
||||
continue
|
||||
brand_tokens = re.findall(r"[a-z0-9]+", brand.lower())
|
||||
if not brand_tokens:
|
||||
continue
|
||||
# Multi-word brands ("Hydro Flask") can never match a single-token
|
||||
# test, so compare the brand's whole token sequence against the
|
||||
# keyword's -- otherwise the guard is off for every two-word brand.
|
||||
if len(brand_tokens) == 1:
|
||||
matched = brand_tokens[0] in keyword_tokens and len(brand_tokens[0]) > 2
|
||||
else:
|
||||
matched = " ".join(brand_tokens) in normalized_keyword
|
||||
if matched:
|
||||
# First spelling wins, so the result is deterministic across runs.
|
||||
candidates.setdefault(brand.lower(), brand)
|
||||
return next(iter(candidates.values())) if len(candidates) == 1 else ""
|
||||
|
||||
|
||||
def select_enrichment_targets(
|
||||
products: Sequence[Dict[str, Any]],
|
||||
*,
|
||||
limit: int,
|
||||
brand: str = "",
|
||||
keyword: str = "",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Pick which products get a review pull.
|
||||
|
||||
Ranked by rating count, which is a coarse signal: search records carry
|
||||
variant-level counts that can undercount badly (84 on a record whose
|
||||
review pull reported 8,446). The review pull's own
|
||||
``product_rating_count`` is authoritative once available.
|
||||
|
||||
Two filters run before the cut:
|
||||
|
||||
* **Brand**, supplied or inferred from the keyword (see ``infer_brand``).
|
||||
The record's own ``brand`` field does the work, which also solves
|
||||
accessory contamination outright -- a "grill brush for Weber" carries
|
||||
the brush maker's brand, not Weber. A front-anchored name match covers
|
||||
the few records where ``brand`` is null.
|
||||
* **Variant collapse.** Live results repeat one product across colors
|
||||
and sizes under distinct ASINs with near-identical names. Two of those
|
||||
would burn two of three pulls on the same product and render as
|
||||
duplicate footer entries, so only the best-ranked of each short-name
|
||||
group stays eligible.
|
||||
"""
|
||||
if limit <= 0:
|
||||
return []
|
||||
pool = list(products)
|
||||
|
||||
wanted = (brand or "").strip().lower() or infer_brand(pool, keyword).lower()
|
||||
if wanted:
|
||||
matched = [
|
||||
p for p in pool
|
||||
if (p.get("brand") or "").strip().lower() == wanted
|
||||
or (not (p.get("brand") or "").strip()
|
||||
and str(p.get("name") or "").strip().lower().startswith(wanted))
|
||||
]
|
||||
if matched:
|
||||
pool = matched
|
||||
|
||||
deduped: List[Dict[str, Any]] = []
|
||||
seen_names: set[str] = set()
|
||||
for product in pool:
|
||||
key = (product.get("short_name") or "").strip().lower()
|
||||
if key and key in seen_names:
|
||||
continue
|
||||
if key:
|
||||
seen_names.add(key)
|
||||
deduped.append(product)
|
||||
return deduped[:limit]
|
||||
|
||||
|
||||
# ------------------------------------------------------------ enrichment
|
||||
|
||||
|
||||
def fetch_reviews(
|
||||
product_url: str,
|
||||
*,
|
||||
max_reviews: int = MAX_REVIEWS,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = REVIEW_TIMEOUT,
|
||||
) -> Dict[str, Any]:
|
||||
"""Pull a capped review sample for one product. Never raises."""
|
||||
if not product_url:
|
||||
return {"records": []}
|
||||
return brightdata.run_pipeline(
|
||||
REVIEWS_PIPELINE, [product_url, str(max_reviews)],
|
||||
timeout=timeout, config=config,
|
||||
)
|
||||
|
||||
|
||||
def parse_reviews(response: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""Split a review payload into comment dicts and product-level stats.
|
||||
|
||||
Product-level fields (``product_rating``, ``product_rating_count``, the
|
||||
``product_rating_object`` star distribution) ride on *every* review
|
||||
record, so they are read off the first one.
|
||||
|
||||
Comments are built directly in the shared score/excerpt shape rather
|
||||
than routed through ``normalize._remap_comments``, which strips keys it
|
||||
does not know -- and rating, date, and verified are exactly the keys
|
||||
this source needs to keep. Sorted newest first so the woven sample
|
||||
favors recent voices.
|
||||
"""
|
||||
records = response.get("records") if isinstance(response, dict) else None
|
||||
if not isinstance(records, list) or not records:
|
||||
return [], {}
|
||||
|
||||
first = records[0]
|
||||
distribution = first.get("product_rating_object")
|
||||
stats: Dict[str, Any] = {
|
||||
"product_rating": _as_float(first.get("product_rating")),
|
||||
"product_rating_count": _as_int(first.get("product_rating_count")),
|
||||
"star_distribution": distribution if isinstance(distribution, dict) else {},
|
||||
}
|
||||
|
||||
comments: List[Dict[str, Any]] = []
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
body = str(record.get("review_text") or "").strip()
|
||||
header = undouble(str(record.get("review_header") or ""))
|
||||
excerpt = body or header
|
||||
if not excerpt:
|
||||
continue
|
||||
comments.append(
|
||||
{
|
||||
# Shared comment shape: downstream weaving reads score/excerpt.
|
||||
"score": _as_int(record.get("helpful_count")),
|
||||
"excerpt": excerpt,
|
||||
"author": str(record.get("author_name") or "").strip(),
|
||||
"rating": _as_int(record.get("rating")),
|
||||
"date": parse_review_date(record.get("review_posted_date")),
|
||||
"verified": bool(record.get("is_verified")),
|
||||
"vine": bool(record.get("is_amazon_vine")),
|
||||
"title": header,
|
||||
}
|
||||
)
|
||||
|
||||
# Newest first; undated records sink rather than disappear (R2a).
|
||||
comments.sort(key=lambda c: (c["date"] or "", c["score"]), reverse=True)
|
||||
return comments, stats
|
||||
|
||||
|
||||
def _remaining_lane_budget(elapsed: float) -> int:
|
||||
"""Compute the review lane's wall-clock budget.
|
||||
|
||||
Returns the lesser of LANE_DEADLINE and whatever remains of the foreground
|
||||
contract. If the remaining time is below MIN_USEFUL_REVIEW_BUDGET, returns
|
||||
0 (skip the lane entirely) rather than firing doomed short pulls that spend
|
||||
Bright Data credits without returning reviews.
|
||||
"""
|
||||
remaining = FOREGROUND_CONTRACT - elapsed - RENDER_MARGIN
|
||||
if remaining < MIN_USEFUL_REVIEW_BUDGET:
|
||||
return 0
|
||||
return int(min(LANE_DEADLINE, remaining))
|
||||
|
||||
|
||||
def enrich_with_reviews(
|
||||
products: Sequence[Dict[str, Any]],
|
||||
*,
|
||||
depth: str = "default",
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
elapsed: float = 0.0,
|
||||
max_reviews: int = MAX_REVIEWS,
|
||||
brand: str = "",
|
||||
keyword: str = "",
|
||||
fetcher=None,
|
||||
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
||||
"""Attach review samples to the top products, in parallel, under a deadline.
|
||||
|
||||
Every product is returned either way. A product whose pull is dropped
|
||||
by the deadline keeps its search-record stats and simply carries no
|
||||
review sample -- it renders as ``quiet`` rather than vanishing, because
|
||||
losing a top product entirely is a worse failure than losing its
|
||||
recent-window read. The dropped pull has spent its credit regardless.
|
||||
|
||||
Returns (enriched_products, status_detail). status_detail is None when
|
||||
enrichment succeeded normally, or a string describing a degraded outcome:
|
||||
- ``"review lane skipped (budget 0s)"`` -- crumb budget, lane did not run
|
||||
- ``"review lane timed out"`` -- all pulls dropped by the deadline
|
||||
"""
|
||||
enriched = [dict(p) for p in products]
|
||||
pull_count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
if pull_count <= 0:
|
||||
_log(f"depth={depth}: discovery only, no review pulls")
|
||||
return enriched, None
|
||||
|
||||
budget = _remaining_lane_budget(elapsed)
|
||||
if budget <= 0:
|
||||
_log(f"review lane skipped (budget {budget}s, floor {MIN_USEFUL_REVIEW_BUDGET}s)")
|
||||
return enriched, "review lane skipped (budget 0s)"
|
||||
|
||||
targets = select_enrichment_targets(
|
||||
enriched, limit=pull_count, brand=brand, keyword=keyword
|
||||
)
|
||||
if not targets:
|
||||
return enriched, None
|
||||
|
||||
by_asin = {p["asin"]: p for p in enriched}
|
||||
pull = fetcher or (
|
||||
lambda url: fetch_reviews(
|
||||
url, max_reviews=max_reviews, config=config,
|
||||
timeout=min(REVIEW_TIMEOUT, budget),
|
||||
)
|
||||
)
|
||||
|
||||
_log(f"pulling up to {max_reviews} reviews for {len(targets)} products (budget {budget}s)")
|
||||
started = time.monotonic()
|
||||
completed_count = 0
|
||||
dropped_count = 0
|
||||
# Not a `with` block on purpose. Every future is already running (one
|
||||
# worker per target), so `future.cancel()` can never succeed, and
|
||||
# ThreadPoolExecutor's context-manager exit calls shutdown(wait=True) --
|
||||
# which would block on the very straggler the deadline just declared
|
||||
# dropped, making the deadline advisory rather than real. Shutting down
|
||||
# without waiting lets the abandoned thread finish and discard its result
|
||||
# in the background while the run proceeds.
|
||||
pool = ThreadPoolExecutor(max_workers=max(1, len(targets)))
|
||||
try:
|
||||
futures = {pool.submit(pull, t["url"]): t["asin"] for t in targets}
|
||||
try:
|
||||
for future in as_completed(futures, timeout=budget):
|
||||
asin = futures[future]
|
||||
try:
|
||||
response = future.result()
|
||||
except Exception as exc: # never let one pull kill siblings
|
||||
_log(f"review pull failed for {asin}: {exc}")
|
||||
continue
|
||||
if response.get("error"):
|
||||
_log(f"review pull error for {asin}: {response['error']}")
|
||||
continue
|
||||
comments, stats = parse_reviews(response)
|
||||
product = by_asin.get(asin)
|
||||
if product is None:
|
||||
continue
|
||||
product["top_comments"] = comments
|
||||
product.update({k: v for k, v in stats.items() if v})
|
||||
completed_count += 1
|
||||
except TimeoutError:
|
||||
dropped_count = sum(1 for f in futures if not f.done())
|
||||
_log(f"lane deadline {budget}s hit; dropped {dropped_count} straggling pull(s)")
|
||||
finally:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
_log(f"review lane finished in {time.monotonic() - started:.0f}s")
|
||||
|
||||
# Report degraded outcome if all pulls dropped (none completed)
|
||||
status_detail = None
|
||||
if completed_count == 0 and dropped_count > 0:
|
||||
status_detail = "review lane timed out"
|
||||
|
||||
return enriched, status_detail
|
||||
|
||||
|
||||
def enrich_source_items(
|
||||
items: List[Any],
|
||||
*,
|
||||
depth: str = "default",
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
keyword: str = "",
|
||||
elapsed: float = 0.0,
|
||||
max_reviews: int = MAX_REVIEWS,
|
||||
fetcher=None,
|
||||
) -> List[Any]:
|
||||
"""Attach review samples to the amazon SourceItems that survived dedupe.
|
||||
|
||||
Reads product identity out of ``metadata`` and writes ``top_comments``
|
||||
plus the computed stat block back into it, in place. Runs from
|
||||
``pipeline._finalize_items_by_source`` so the review budget is spent on
|
||||
the products the brief will actually show, not on the top of the raw
|
||||
fanout (the digg enrichment precedent).
|
||||
"""
|
||||
products: List[Dict[str, Any]] = []
|
||||
by_asin: Dict[str, Any] = {}
|
||||
for item in items:
|
||||
if getattr(item, "source", None) != "amazon":
|
||||
continue
|
||||
metadata = getattr(item, "metadata", None) or {}
|
||||
asin = str(metadata.get("asin") or "").strip()
|
||||
if not asin or metadata.get("top_comments"):
|
||||
continue
|
||||
products.append(
|
||||
{
|
||||
"asin": asin,
|
||||
"url": getattr(item, "url", "") or metadata.get("url", ""),
|
||||
"name": metadata.get("name") or getattr(item, "title", ""),
|
||||
"short_name": metadata.get("short_name") or "",
|
||||
"brand": metadata.get("brand") or "",
|
||||
"num_ratings": metadata.get("num_ratings") or 0,
|
||||
"rating": metadata.get("rating"),
|
||||
}
|
||||
)
|
||||
by_asin[asin] = item
|
||||
|
||||
if not products:
|
||||
return items
|
||||
|
||||
enriched, _status = enrich_with_reviews(
|
||||
products, depth=depth, config=config, elapsed=elapsed,
|
||||
max_reviews=max_reviews, keyword=keyword, fetcher=fetcher,
|
||||
)
|
||||
for product in enriched:
|
||||
item = by_asin.get(product["asin"])
|
||||
if item is None:
|
||||
continue
|
||||
metadata = getattr(item, "metadata", None)
|
||||
if metadata is None:
|
||||
continue
|
||||
if product.get("top_comments"):
|
||||
metadata["top_comments"] = product["top_comments"]
|
||||
stats = product_stats(product)
|
||||
metadata["stats"] = stats
|
||||
# The review pull's product_rating_count supersedes the search
|
||||
# record's, which is variant-level and can undercount by orders of
|
||||
# magnitude (84 on a record whose pull reported 8,446). Normalization
|
||||
# ran before enrichment, so refresh the surfaces that already baked
|
||||
# the old number in -- otherwise one product shows two different
|
||||
# rating counts in the same report.
|
||||
for key in ("product_rating", "product_rating_count", "star_distribution"):
|
||||
if product.get(key):
|
||||
metadata[key] = product[key]
|
||||
authoritative = stats.get("ratings_total") or 0
|
||||
if authoritative and getattr(item, "engagement", None) is not None:
|
||||
item.engagement["ratings"] = authoritative
|
||||
metadata["num_ratings"] = authoritative
|
||||
_refresh_title(item, stats)
|
||||
return items
|
||||
|
||||
|
||||
def _refresh_title(item: Any, stats: Dict[str, Any]) -> None:
|
||||
"""Rewrite the trailing "- 4.4/5 (N ratings)" headline after enrichment."""
|
||||
title = getattr(item, "title", "") or ""
|
||||
rating = stats.get("all_time")
|
||||
total = stats.get("ratings_total") or 0
|
||||
if not title or rating is None or not total:
|
||||
return
|
||||
headline = f"{rating}/5 ({total:,} ratings)"
|
||||
base = title.rsplit(" - ", 1)[0] if " - " in title else title
|
||||
item.title = f"{base} - {headline}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ stats
|
||||
|
||||
|
||||
def stats_from_item(item: Any, *, today: Optional[datetime] = None) -> Dict[str, Any]:
|
||||
"""Compute the stat block for a rendered SourceItem.
|
||||
|
||||
Enrichment stores a precomputed block, but mock runs and replayed
|
||||
fixtures skip enrichment entirely, so render recomputes from metadata
|
||||
when it is absent. Cheap and pure -- all the inputs already live on
|
||||
the item.
|
||||
"""
|
||||
metadata = getattr(item, "metadata", None) or {}
|
||||
cached = metadata.get("stats")
|
||||
if isinstance(cached, dict) and cached:
|
||||
return cached
|
||||
return product_stats(
|
||||
{
|
||||
"short_name": metadata.get("short_name") or "",
|
||||
"name": metadata.get("name") or getattr(item, "title", ""),
|
||||
"url": getattr(item, "url", "") or "",
|
||||
"rating": metadata.get("rating"),
|
||||
"num_ratings": metadata.get("num_ratings") or 0,
|
||||
"product_rating": metadata.get("product_rating"),
|
||||
"product_rating_count": metadata.get("product_rating_count") or 0,
|
||||
"star_distribution": metadata.get("star_distribution") or {},
|
||||
"top_comments": metadata.get("top_comments") or [],
|
||||
},
|
||||
today=today,
|
||||
)
|
||||
|
||||
|
||||
def footer_entry(stats: Dict[str, Any], *, quote: str = "") -> str:
|
||||
"""Render one product's segment of the emoji-footer line (R1c).
|
||||
|
||||
Shapes, by drift state::
|
||||
|
||||
Chill Max XL 4.4★→3.8★ ↓ "the lid jams" negative drift (+ quote)
|
||||
Deluxe Bag 4.7★→5.0★ positive or flat drift
|
||||
Spirit E-325 4.4★ quiet too few in-window reviews
|
||||
BLUEY Set new no all-time baseline
|
||||
|
||||
The ``↓`` is asymmetric on purpose: a sagging product is the alarm
|
||||
worth catching at a glance, and a healthy one needs no decoration.
|
||||
"""
|
||||
name = stats.get("short_name") or "Product"
|
||||
all_time = stats.get("all_time")
|
||||
recent = stats.get("recent_avg")
|
||||
drift = stats.get("drift")
|
||||
|
||||
if drift == "new" or all_time is None:
|
||||
return f"{name} new"
|
||||
if drift == "quiet" or recent is None:
|
||||
return f"{name} {all_time}★ quiet"
|
||||
|
||||
entry = f"{name} {all_time}★→{recent}★"
|
||||
if drift == "down":
|
||||
entry += " ↓"
|
||||
if quote:
|
||||
entry += f' "{quote}"'
|
||||
return entry
|
||||
|
||||
|
||||
def five_star_share(distribution: Dict[str, Any]) -> Optional[float]:
|
||||
"""Share of ratings that are 5-star, from the star-distribution object."""
|
||||
if not isinstance(distribution, dict) or not distribution:
|
||||
return None
|
||||
total = sum(_as_int(distribution.get(key)) for key, _ in _STAR_FIELDS)
|
||||
if total <= 0:
|
||||
return None
|
||||
return _as_int(distribution.get("five_star")) / total
|
||||
|
||||
|
||||
def recent_window_stats(
|
||||
comments: Iterable[Dict[str, Any]],
|
||||
*,
|
||||
today: Optional[datetime] = None,
|
||||
window_days: int = RECENT_WINDOW_DAYS,
|
||||
) -> Dict[str, Any]:
|
||||
"""Average rating and sample size inside the recent window."""
|
||||
reference = (today or _today()).date()
|
||||
ratings: List[int] = []
|
||||
for comment in comments or []:
|
||||
iso = comment.get("date")
|
||||
if not iso:
|
||||
continue
|
||||
try:
|
||||
posted = datetime.strptime(iso, "%Y-%m-%d").date()
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if 0 <= (reference - posted).days <= window_days:
|
||||
rating = _as_int(comment.get("rating"))
|
||||
if rating:
|
||||
ratings.append(rating)
|
||||
if not ratings:
|
||||
return {"recent_n": 0, "recent_avg": None}
|
||||
return {"recent_n": len(ratings), "recent_avg": sum(ratings) / len(ratings)}
|
||||
|
||||
|
||||
def product_stats(
|
||||
product: Dict[str, Any],
|
||||
*,
|
||||
today: Optional[datetime] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Compute the render-facing stat block for one product.
|
||||
|
||||
``drift`` is one of:
|
||||
* ``"new"`` -- no all-time baseline to move away from
|
||||
* ``"quiet"`` -- baseline exists but the window has too few dated
|
||||
reviews to average honestly (below MIN_DRIFT_SAMPLE)
|
||||
* ``"up"`` / ``"down"`` / ``"flat"`` -- a real, sample-backed move
|
||||
|
||||
The engine owns every number here; the model owns the words (R1b).
|
||||
"""
|
||||
# The review pull's rating count supersedes the search record's, which
|
||||
# can be variant-level and badly low.
|
||||
all_time = product.get("product_rating")
|
||||
if all_time is None:
|
||||
all_time = product.get("rating")
|
||||
ratings_total = product.get("product_rating_count") or product.get("num_ratings") or 0
|
||||
|
||||
window = recent_window_stats(product.get("top_comments") or [], today=today)
|
||||
recent_avg = window["recent_avg"]
|
||||
recent_n = window["recent_n"]
|
||||
|
||||
if all_time is None:
|
||||
drift = "new"
|
||||
elif recent_n < MIN_DRIFT_SAMPLE or recent_avg is None:
|
||||
drift = "quiet"
|
||||
elif round(recent_avg, 1) > round(float(all_time), 1):
|
||||
drift = "up"
|
||||
elif round(recent_avg, 1) < round(float(all_time), 1):
|
||||
drift = "down"
|
||||
else:
|
||||
drift = "flat"
|
||||
|
||||
return {
|
||||
"short_name": product.get("short_name") or short_name(product.get("name", "")),
|
||||
"url": product.get("url", ""),
|
||||
"all_time": round(float(all_time), 1) if all_time is not None else None,
|
||||
"ratings_total": _as_int(ratings_total),
|
||||
"five_star_share": five_star_share(product.get("star_distribution") or {}),
|
||||
"recent_avg": round(recent_avg, 1) if recent_avg is not None else None,
|
||||
"recent_n": recent_n,
|
||||
"reviews_pulled": len(product.get("top_comments") or []),
|
||||
"drift": drift,
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
"""arXiv research-paper source for last30days.
|
||||
|
||||
Shells out to ``arxiv-pp-cli`` (open Atom API, no auth) to surface recent
|
||||
research papers relevant to a topic. arXiv carries no engagement signal, so
|
||||
ranking leans on relevance (the CLI's own relevance sort plus token overlap)
|
||||
and recency.
|
||||
|
||||
Activation gate: this source is only available when ``arxiv-pp-cli`` is on
|
||||
PATH. ``pipeline.available_sources`` checks ``shutil.which`` before including
|
||||
``arxiv``. The functions below also detect the missing-binary case defensively.
|
||||
|
||||
Default-on safety (two gates, both required):
|
||||
1. Query construction. arXiv is queried with a *quoted* phrase and
|
||||
``--sort-by relevance``. Sorting by submitted-date instead returns the
|
||||
newest cs.* papers regardless of topic -- topic-blind noise.
|
||||
2. Recency cutoff. Entries older than ``RECENCY_DAYS`` are dropped. Research
|
||||
does not trend on a 30-day clock, so this window is wider than the social
|
||||
sources' 30 days; it keeps arXiv current while dropping stale keyword
|
||||
matches (e.g. a 2017 sports-statistics paper that an off-topic query like
|
||||
"Golden State Warriors" would otherwise surface).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import log, subproc
|
||||
from .relevance import token_overlap_relevance
|
||||
|
||||
|
||||
CLI_BIN = "arxiv-pp-cli"
|
||||
|
||||
# Per-depth result counts.
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 5,
|
||||
"default": 10,
|
||||
"deep": 20,
|
||||
}
|
||||
|
||||
# Recency window for arXiv specifically. Papers do not trend daily; a year keeps
|
||||
# the source current (the off-topic 2017 paper still drops) without discarding
|
||||
# the genuinely-relevant work from the last few months.
|
||||
RECENCY_DAYS = 365
|
||||
|
||||
SEARCH_TIMEOUT = 30
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("arXiv", msg, tty_only=False)
|
||||
|
||||
|
||||
def _is_available() -> bool:
|
||||
"""True when the arxiv-pp-cli binary is on PATH."""
|
||||
return shutil.which(CLI_BIN) is not None
|
||||
|
||||
|
||||
def _today() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _build_search_query(topic: str, *, quoted: bool = True) -> str:
|
||||
"""Build the arXiv search-query string for ``topic``.
|
||||
|
||||
Quoted (default): phrase-scoped exact match across all fields. Precise
|
||||
for topics that genuinely appear as a phrase in a title/abstract, but a
|
||||
natural-language multi-word topic ("AI video generation advances") almost
|
||||
never appears verbatim, so it returns zero results (#908). Unquoted uses
|
||||
an AND-conjoined clause for every individual term as a fallback retry.
|
||||
|
||||
Inner double-quotes are stripped (arXiv has no phrase-escaping) either way.
|
||||
"""
|
||||
phrase = _clean_phrase(topic)
|
||||
if quoted:
|
||||
return f'all:"{phrase}"'
|
||||
return " AND ".join(f'all:"{term}"' for term in phrase.split())
|
||||
|
||||
|
||||
def _clean_phrase(topic: str) -> str:
|
||||
"""Strip quotes and collapse whitespace into a phrase for the query."""
|
||||
return " ".join(topic.replace('"', " ").split())
|
||||
|
||||
|
||||
def _build_search_args(topic: str, limit: int, *, quoted: bool = True) -> List[str]:
|
||||
return [
|
||||
CLI_BIN,
|
||||
"query",
|
||||
"--search-query",
|
||||
_build_search_query(topic, quoted=quoted),
|
||||
"--sort-by",
|
||||
"relevance",
|
||||
"--max-results",
|
||||
str(limit),
|
||||
"--agent",
|
||||
]
|
||||
|
||||
|
||||
def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
|
||||
"""Invoke arxiv-pp-cli and parse the JSON envelope.
|
||||
|
||||
arXiv returns ``{"meta": ..., "results": {"entries": [...]}}``. This
|
||||
normalizes to ``{"results": [...entries...]}`` so the parse step sees a
|
||||
flat list, matching the other sources' shape. Never raises.
|
||||
"""
|
||||
if not _is_available():
|
||||
return {"results": [], "error": f"{CLI_BIN} not on PATH"}
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=timeout)
|
||||
except subproc.SubprocTimeout as exc:
|
||||
_log(f"Timeout: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
except FileNotFoundError as exc:
|
||||
_log(f"Binary missing: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
except OSError as exc:
|
||||
_log(f"Spawn failed: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
|
||||
if result.returncode != 0:
|
||||
snippet = (result.stderr or "").strip().splitlines()[:1]
|
||||
first = snippet[0] if snippet else f"exit {result.returncode}"
|
||||
_log(f"CLI exit {result.returncode}: {first}")
|
||||
return {"results": [], "error": first}
|
||||
|
||||
stdout = result.stdout or ""
|
||||
if not stdout.strip():
|
||||
_log("CLI returned empty stdout")
|
||||
return {"results": [], "error": "empty stdout"}
|
||||
try:
|
||||
data = json.loads(stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
_log(f"JSON decode failed: {exc}")
|
||||
return {"results": [], "error": f"json decode: {exc}"}
|
||||
|
||||
if not _is_entry_envelope(data):
|
||||
_log("CLI returned an unrecognized JSON response")
|
||||
return {"results": [], "error": "unrecognized JSON response"}
|
||||
|
||||
return {"results": _extract_entries(data)}
|
||||
|
||||
|
||||
def _extract_entries(data: Any) -> List[Dict[str, Any]]:
|
||||
"""Pull the entries list out of arXiv's nested envelope.
|
||||
|
||||
Tolerates ``{"results": {"entries": [...]}}`` (current shape),
|
||||
``{"entries": [...]}``, and a bare list.
|
||||
"""
|
||||
if isinstance(data, list):
|
||||
return [e for e in data if isinstance(e, dict)]
|
||||
if isinstance(data, dict):
|
||||
results = data.get("results")
|
||||
if isinstance(results, dict):
|
||||
entries = results.get("entries")
|
||||
if isinstance(entries, list):
|
||||
return [e for e in entries if isinstance(e, dict)]
|
||||
if isinstance(results, list):
|
||||
return [e for e in results if isinstance(e, dict)]
|
||||
entries = data.get("entries")
|
||||
if isinstance(entries, list):
|
||||
return [e for e in entries if isinstance(e, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _is_entry_envelope(data: Any) -> bool:
|
||||
"""Return whether ``data`` has one of the supported entry-list shapes."""
|
||||
if isinstance(data, list):
|
||||
return True
|
||||
if not isinstance(data, dict):
|
||||
return False
|
||||
results = data.get("results")
|
||||
return (
|
||||
isinstance(results, list)
|
||||
or (isinstance(results, dict) and isinstance(results.get("entries"), list))
|
||||
or isinstance(data.get("entries"), list)
|
||||
)
|
||||
|
||||
|
||||
def search_arxiv(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search arXiv via arxiv-pp-cli using a quoted, relevance-sorted query.
|
||||
|
||||
Returns a dict with a flat ``results`` list of entry dicts. On failure,
|
||||
``results`` is empty and an ``error`` key carries a one-line description.
|
||||
"""
|
||||
if not topic or not topic.strip():
|
||||
return {"results": []}
|
||||
# A topic of only quote characters cleans to an empty phrase (all:""),
|
||||
# which is a topic-blind query; bail rather than search for nothing.
|
||||
if not _clean_phrase(topic):
|
||||
return {"results": []}
|
||||
limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
cmd = _build_search_args(topic, limit)
|
||||
_log(f"query '{topic}' (relevance, max={limit})")
|
||||
response = _run_cli(cmd, timeout=SEARCH_TIMEOUT)
|
||||
_log(f"found {len(response.get('results') or [])} entries")
|
||||
# Retry a clean zero-result phrase match with individually quoted AND terms.
|
||||
# CLI failures, malformed responses, and missing binaries skip the retry.
|
||||
if not response.get("error") and not response.get("results"):
|
||||
retry_cmd = _build_search_args(topic, limit, quoted=False)
|
||||
_log(f"quoted phrase matched nothing; retrying unquoted for '{topic}'")
|
||||
response = _run_cli(retry_cmd, timeout=SEARCH_TIMEOUT)
|
||||
_log(f"unquoted retry found {len(response.get('results') or [])} entries")
|
||||
return response
|
||||
|
||||
|
||||
def _parse_published(published: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse an arXiv ``published`` timestamp (ISO 8601, e.g.
|
||||
'2026-06-25T17:59:48Z') into an aware datetime. Returns None on failure."""
|
||||
if not published or not isinstance(published, str):
|
||||
return None
|
||||
text = published.strip().replace("Z", "+00:00")
|
||||
try:
|
||||
dt = datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def _alternate_url(entry: Dict[str, Any]) -> str:
|
||||
"""Return the human-facing abstract URL (rel=alternate), not the PDF."""
|
||||
links = entry.get("links")
|
||||
if isinstance(links, list):
|
||||
for link in links:
|
||||
if isinstance(link, dict) and link.get("rel") == "alternate":
|
||||
href = str(link.get("href") or "").strip()
|
||||
if href:
|
||||
return href
|
||||
# Fall back to the abstract URL derived from the entry id.
|
||||
entry_id = str(entry.get("id") or "").strip()
|
||||
if entry_id.startswith("http"):
|
||||
return entry_id
|
||||
return ""
|
||||
|
||||
|
||||
def _author_names(entry: Dict[str, Any]) -> List[str]:
|
||||
authors = entry.get("authors")
|
||||
out: List[str] = []
|
||||
if isinstance(authors, list):
|
||||
for a in authors:
|
||||
if isinstance(a, dict):
|
||||
name = str(a.get("name") or "").strip()
|
||||
if name:
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def parse_arxiv_response(
|
||||
response: Dict[str, Any],
|
||||
query: str = "",
|
||||
today: Optional[datetime] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Parse an arXiv envelope into normalized item dicts.
|
||||
|
||||
Applies the recency cutoff (drops entries older than ``RECENCY_DAYS`` and
|
||||
entries with an unparseable date) and computes a token-overlap relevance
|
||||
hint. Returns dicts ready for ``normalize._normalize_arxiv``.
|
||||
"""
|
||||
raw = response.get("results") if isinstance(response, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
|
||||
now = today or _today()
|
||||
items: List[Dict[str, Any]] = []
|
||||
for i, entry in enumerate(raw):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
title = " ".join(str(entry.get("title") or "").split()).strip()
|
||||
if not title:
|
||||
continue
|
||||
published = _parse_published(entry.get("published") or entry.get("updated"))
|
||||
if published is None:
|
||||
# No usable date -> cannot honor the recency contract; drop.
|
||||
continue
|
||||
age_days = (now - published).days
|
||||
# Allow a one-day grace on the future side: a paper announced later in
|
||||
# the same UTC day yields age_days == -1 (timedelta.days floors toward
|
||||
# negative); dropping it as "future" would discard the freshest work.
|
||||
if age_days > RECENCY_DAYS or age_days < -1:
|
||||
continue
|
||||
|
||||
summary = " ".join(str(entry.get("summary") or "").split()).strip()
|
||||
authors = _author_names(entry)
|
||||
url = _alternate_url(entry)
|
||||
|
||||
rank_decay = max(0.3, 1.0 - (i * 0.03))
|
||||
if query:
|
||||
content_score = token_overlap_relevance(query, f"{title} {summary}".strip())
|
||||
else:
|
||||
content_score = 0.5
|
||||
relevance = min(1.0, 0.6 * rank_decay + 0.4 * content_score)
|
||||
|
||||
primary_author = authors[0] if authors else ""
|
||||
author_label = primary_author
|
||||
if len(authors) > 1:
|
||||
author_label = f"{primary_author} et al."
|
||||
|
||||
items.append(
|
||||
{
|
||||
"id": str(entry.get("id") or url or f"AX{i + 1}"),
|
||||
"title": title,
|
||||
"url": url,
|
||||
"summary": summary,
|
||||
"author": author_label,
|
||||
"authors": authors,
|
||||
"date": published.date().isoformat(),
|
||||
"engagement": {},
|
||||
"relevance": round(relevance, 2),
|
||||
"why_relevant": (
|
||||
f"arXiv paper ({primary_author}, {published.date().isoformat()})"
|
||||
if primary_author
|
||||
else f"arXiv paper ({published.date().isoformat()})"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return items
|
||||
@@ -0,0 +1,700 @@
|
||||
"""Backend-chain descriptors with predicted selection (doctor, R4).
|
||||
|
||||
Chained sources declare their routing here ONCE — imported from the
|
||||
definitions ``lib/env.py`` already owns (chain order, pin var names) — and
|
||||
``resolve()`` turns side-effect-free probes into a truthful prediction of
|
||||
what the next run will do.
|
||||
|
||||
Two resolution modes:
|
||||
|
||||
- ``alternative`` (X, YouTube, web search): the pipeline tries genuinely
|
||||
interchangeable backends in a declared order. Resolution probes ALL
|
||||
candidates first, then picks (collect-then-pick): the first fully-usable
|
||||
backend wins the "will use" prediction; otherwise the best degraded
|
||||
candidate resolves with a warn tier; otherwise the source is an error
|
||||
carrying the highest-priority backend's prescription. Collecting before
|
||||
picking prevents an installed-but-unauthenticated preferred backend from
|
||||
shadowing a fully working fallback.
|
||||
|
||||
- ``conditional`` (Reddit): routing is per-query and outcome-dependent —
|
||||
public keyless composite by default, ScrapeCreators backfill only when
|
||||
results fall below the configured thinness floor (see the gating in
|
||||
``lib/pipeline.py``). No probe can pick one winner, so resolution renders
|
||||
honest conditional wording instead of an ``active_backend``. Reddit's
|
||||
internal keyless lanes (rss/listing/arctic/shreddit) are sub-probe detail
|
||||
inside the public composite, never chain entries.
|
||||
|
||||
``active_backend`` semantics: a PREDICTION — "the first backend the probes
|
||||
say the next run will try" — rendered as "will use". It is not an
|
||||
observation of what served a past run, and runtime failover can still
|
||||
diverge mid-run (a present-but-expired paid key passes a presence probe).
|
||||
|
||||
Paid lanes (xai, xquik, serper, and every other API-key backend, including
|
||||
ScrapeCreators) probe KEY PRESENCE ONLY: a dict lookup, never a network
|
||||
call or credential spend. Binary-backed lanes reuse the U1 dependency
|
||||
probe layer (``health.probe_dependency``) so a stale shim reads as BROKEN,
|
||||
not available (#692).
|
||||
|
||||
This module observes and predicts only. It must never alter which backend
|
||||
the pipeline actually uses; parity with the pipeline's pre-failover
|
||||
selection is asserted in ``tests/test_backend_descriptors.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from shutil import which
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from . import env, health, prescriptions
|
||||
|
||||
# Resolution modes.
|
||||
MODE_ALTERNATIVE = "alternative" # probe-ordered chain, first-usable wins
|
||||
MODE_CONDITIONAL = "conditional" # per-query routing; wording, never a winner
|
||||
|
||||
# Rollup tiers for a resolved chain (doctor maps these into its R1 table).
|
||||
TIER_OK = "ok"
|
||||
TIER_WARN = "warn"
|
||||
TIER_ERROR = "error"
|
||||
|
||||
# Web search backend order. grounding.web_search's auto branch owns the
|
||||
# runtime behavior (brave -> exa -> serper -> parallel -> keyless floor);
|
||||
# there is no importable constant there, so this declaration is guarded by
|
||||
# the grounding-auto parity test rather than an import.
|
||||
WEB_BACKEND_ORDER: Tuple[str, ...] = ("brave", "exa", "serper", "parallel", "keyless")
|
||||
|
||||
# YouTube backend order (pipeline: yt-dlp first, ScrapeCreators search
|
||||
# fallback when yt-dlp is absent or fails — see lib/pipeline.py).
|
||||
YOUTUBE_BACKEND_ORDER: Tuple[str, ...] = ("yt-dlp", "scrapecreators")
|
||||
|
||||
# Chain-failure fixes embed the registry's CLI forms (KTD 7): the command a
|
||||
# backend finding prescribes and the one doctor/quality-nudge render for the
|
||||
# same failure mode come from one entry and cannot drift.
|
||||
_SC_PRESCRIPTION = (
|
||||
"set SCRAPECREATORS_API_KEY (free 10,000-call signup: "
|
||||
f"{prescriptions.get('scrapecreators', 'key_missing').fix_cli})"
|
||||
)
|
||||
_X_COOKIES_PRESCRIPTION = (
|
||||
"run setup with browser-cookie consent: "
|
||||
f"{prescriptions.get('x', 'cookies_missing').fix_cli}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackendFinding:
|
||||
"""Side-effect-free probe outcome for one backend of a chained source.
|
||||
|
||||
``status`` uses the ``lib.health`` vocabulary (OK/DEGRADED/MISSING/
|
||||
BROKEN/TIMEOUT/ERROR). ``prescription`` is the fix when non-OK.
|
||||
``requires`` is the backend's requirement note for report rendering.
|
||||
"""
|
||||
|
||||
name: str
|
||||
status: str
|
||||
detail: str = ""
|
||||
prescription: str = ""
|
||||
requires: str = ""
|
||||
|
||||
@property
|
||||
def usable(self) -> bool:
|
||||
"""Fully or partially usable (OK/DEGRADED) — eligible for selection."""
|
||||
return self.status in (health.OK, health.DEGRADED)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackendSpec:
|
||||
"""One backend in a chain: name, probe, requirement note, paid flag.
|
||||
|
||||
``probe`` must be side-effect-free. When ``paid`` is True the probe is
|
||||
key-presence only: no subprocess, no network, no credential spend.
|
||||
``opt_in`` marks backends that are never auto-selected and require an
|
||||
explicit pin (grok).
|
||||
"""
|
||||
|
||||
name: str
|
||||
requires: str
|
||||
probe: Callable[[Dict[str, Any]], "BackendFinding"]
|
||||
paid: bool = False
|
||||
opt_in: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChainDescriptor:
|
||||
"""A chained source's declared routing: backends, mode, and pin knob."""
|
||||
|
||||
source: str
|
||||
mode: str
|
||||
backends: Tuple[BackendSpec, ...]
|
||||
pin_var: Optional[str] = None # env var pin (X, Reddit)
|
||||
pin_flag: Optional[str] = None # CLI flag pin (web: --web-backend)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackendResolution:
|
||||
"""Resolved routing for one chained source.
|
||||
|
||||
``active_backend`` is the will-use PREDICTION for alternative chains
|
||||
and always None for conditional mode (Reddit never gets a computed
|
||||
winner — ``conditional`` carries the honest wording instead).
|
||||
"""
|
||||
|
||||
source: str
|
||||
mode: str
|
||||
chain: List[str]
|
||||
findings: List[BackendFinding]
|
||||
active_backend: Optional[str] = None
|
||||
tier: str = TIER_OK
|
||||
pinned: bool = False
|
||||
pin: Optional[str] = None
|
||||
prescription: str = ""
|
||||
conditional: str = ""
|
||||
|
||||
@property
|
||||
def summary(self) -> str:
|
||||
"""One-line rendering: will-use prediction or conditional wording."""
|
||||
if self.mode == MODE_CONDITIONAL:
|
||||
return self.conditional
|
||||
if self.active_backend is None:
|
||||
line = f"no usable backend (chain: {' -> '.join(self.chain)})"
|
||||
if self.prescription:
|
||||
line += f"; fix: {self.prescription}"
|
||||
return line
|
||||
line = f"will use: {self.active_backend}"
|
||||
if self.pinned:
|
||||
line += f" (pinned via {self._pin_origin()})"
|
||||
return line
|
||||
|
||||
def _pin_origin(self) -> str:
|
||||
d = DESCRIPTORS.get(self.source)
|
||||
if d is None:
|
||||
return "pin"
|
||||
return d.pin_var or d.pin_flag or "pin"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probes. All side-effect-free; paid lanes are pure dict lookups.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _key_probe(name: str, key_var: str, requires: str, note: str = "") -> Callable:
|
||||
"""Key-presence probe for a paid API lane. Never touches the network."""
|
||||
|
||||
def probe(config: Dict[str, Any]) -> BackendFinding:
|
||||
if config.get(key_var):
|
||||
return BackendFinding(
|
||||
name=name,
|
||||
status=health.OK,
|
||||
detail=f"{key_var} present",
|
||||
requires=requires,
|
||||
)
|
||||
prescription = note or f"set {key_var} in ~/.config/last30days/.env"
|
||||
return BackendFinding(
|
||||
name=name,
|
||||
status=health.MISSING,
|
||||
detail=f"{key_var} not set",
|
||||
prescription=prescription,
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
return probe
|
||||
|
||||
|
||||
def _probe_bird(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""Bird = vendored X GraphQL client (node script) + browser-cookie creds.
|
||||
|
||||
Cookie presence is checked FIRST, mirroring ``env._x_backend_available``'s
|
||||
gating (``has_bird_creds and is_bird_installed()``): without cookies bird
|
||||
is unconfigured regardless of node/script state, and the fix is the
|
||||
cookie-consent flow — a broken node runtime must not turn an unconfigured
|
||||
backend into an error carrying a node prescription.
|
||||
"""
|
||||
from . import bird_x
|
||||
|
||||
requires = "X browser cookies (AUTH_TOKEN/CT0) + node"
|
||||
if not (config.get("AUTH_TOKEN") and config.get("CT0")):
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=health.MISSING,
|
||||
detail="X browser cookies (AUTH_TOKEN/CT0) not configured",
|
||||
prescription=_X_COOKIES_PRESCRIPTION,
|
||||
requires=requires,
|
||||
)
|
||||
if not bird_x.is_bird_installed():
|
||||
# Distinguish a missing/broken node runtime from a missing script.
|
||||
node = health.probe_dependency("node")
|
||||
if node.status != health.OK:
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=node.status,
|
||||
detail=node.detail,
|
||||
prescription=node.prescription,
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=health.MISSING,
|
||||
detail="vendored bird-search client not found",
|
||||
prescription="reinstall the skill (npx skills add . -g -y) to restore lib/vendor/bird-search",
|
||||
requires=requires,
|
||||
)
|
||||
node = health.probe_dependency("node")
|
||||
if node.status != health.OK:
|
||||
# Resolvable-but-broken node (stale shim) must not read as usable.
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=node.status,
|
||||
detail=node.detail,
|
||||
prescription=node.prescription,
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=health.OK,
|
||||
detail="browser-cookie auth (AUTH_TOKEN/CT0) configured",
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
|
||||
def _probe_grok(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""grok CLI = keyless X. LOCAL-ONLY probe, like _probe_xurl.
|
||||
|
||||
Deliberately does NOT call ``health.probe_dependency``: that helper runs
|
||||
``subprocess.run([name, "--version"])``, and the whole-doctor-path test
|
||||
patches ``subprocess.run`` to raise.
|
||||
|
||||
Consequence to be honest about: a grok binary that resolves on PATH but
|
||||
will not execute (the stale-shim class) reports OK here and fails only when
|
||||
a real run shells out. ``grok_x.is_available`` does not close that gap
|
||||
either -- it is also filesystem-only. ``health.probe_dependency("grok")``
|
||||
is the executing probe, and it runs in doctor's CLI-health block rather
|
||||
than on this no-subprocess path.
|
||||
"""
|
||||
from . import grok_x
|
||||
|
||||
requires = "grok CLI installed + signed in (no X credential)"
|
||||
if which("grok") is None:
|
||||
off_path = health._off_path_binary("grok")
|
||||
if off_path is not None:
|
||||
return BackendFinding(
|
||||
name="grok",
|
||||
status=health.MISSING,
|
||||
requires=requires,
|
||||
detail=f"grok is installed at {off_path} but that directory is not on this process's PATH",
|
||||
prescription=f'add {off_path.parent} to PATH (e.g. export PATH="{off_path.parent}:$PATH")',
|
||||
)
|
||||
return BackendFinding(
|
||||
name="grok",
|
||||
status=health.MISSING,
|
||||
requires=requires,
|
||||
detail="grok CLI not found on PATH",
|
||||
prescription=(
|
||||
"install the Grok CLI: curl -fsSL https://x.ai/cli/install.sh | bash, "
|
||||
"then run `grok login`"
|
||||
),
|
||||
)
|
||||
store_status, store_detail, expires_at = grok_x.stored_auth_status()
|
||||
if store_status == grok_x.AUTH_OK:
|
||||
return BackendFinding(
|
||||
name="grok",
|
||||
status=health.OK,
|
||||
requires=requires,
|
||||
detail=f"{store_detail} (not live-verified until a run)",
|
||||
)
|
||||
if store_status == grok_x.AUTH_EXPIRED:
|
||||
expiry_str = expires_at.isoformat() if expires_at else "unknown"
|
||||
return BackendFinding(
|
||||
name="grok",
|
||||
status=health.DEGRADED,
|
||||
requires=requires,
|
||||
detail=(
|
||||
f"Grok session expired at {expiry_str}; "
|
||||
"refresh happens at run time (if revoked, run `grok login --device-auth`)"
|
||||
),
|
||||
prescription="grok login --device-auth",
|
||||
)
|
||||
if store_status == grok_x.AUTH_ERROR:
|
||||
return BackendFinding(
|
||||
name="grok",
|
||||
status=health.ERROR,
|
||||
requires=requires,
|
||||
detail=store_detail,
|
||||
prescription="grok login",
|
||||
)
|
||||
return BackendFinding(
|
||||
name="grok",
|
||||
status=health.MISSING,
|
||||
requires=requires,
|
||||
detail="grok CLI installed but not signed in",
|
||||
prescription="grok login",
|
||||
)
|
||||
|
||||
|
||||
def _probe_xurl(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""xurl = official X API v2 CLI (OAuth2). Free lane; LOCAL-ONLY probe.
|
||||
|
||||
Doctor's no-network guarantee forbids the live ``xurl whoami`` check
|
||||
(``xurl_x.is_available()`` — an authenticated X API call, reserved for
|
||||
research time). This probe keys on local evidence instead: the binary
|
||||
on PATH plus xurl's on-disk token store (~/.xurl). Stored credentials
|
||||
read as OK with an explicit "not live-verified" caveat; an unreadable
|
||||
token store is a typed ERROR (broken, not unconfigured).
|
||||
"""
|
||||
from . import xurl_x
|
||||
|
||||
requires = "xurl CLI installed + OAuth2 login"
|
||||
if which("xurl") is None:
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.MISSING,
|
||||
detail="xurl CLI not found on PATH",
|
||||
prescription="npm install -g xurl && xurl auth oauth2 login",
|
||||
requires=requires,
|
||||
)
|
||||
store_status, store_detail = xurl_x.stored_auth_status()
|
||||
if store_status == xurl_x.AUTH_OK:
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.OK,
|
||||
detail=(
|
||||
"installed; stored OAuth2 credentials present; "
|
||||
"auth not live-verified (no network)"
|
||||
),
|
||||
requires=requires,
|
||||
)
|
||||
if store_status == xurl_x.AUTH_ERROR:
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.ERROR,
|
||||
detail=store_detail,
|
||||
prescription="xurl auth oauth2 login",
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.MISSING,
|
||||
detail="xurl installed but not authenticated",
|
||||
prescription="xurl auth oauth2 login",
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
|
||||
def _probe_ytdlp(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""yt-dlp via the U1 dependency-probe layer (missing/broken/timeout)."""
|
||||
dep = health.probe_dependency("yt-dlp")
|
||||
return BackendFinding(
|
||||
name="yt-dlp",
|
||||
status=dep.status,
|
||||
detail=dep.detail,
|
||||
prescription=dep.prescription,
|
||||
requires="yt-dlp on the agent-subprocess PATH",
|
||||
)
|
||||
|
||||
|
||||
def _probe_web_keyless(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""The keyless web-search floor: works keyless, but degraded quality."""
|
||||
requires = "no key; suppressed on native-search hosts"
|
||||
if env.keyless_web_allowed(config):
|
||||
return BackendFinding(
|
||||
name="keyless",
|
||||
status=health.DEGRADED,
|
||||
detail="keyless search floor (no paid key; lower quality)",
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="keyless",
|
||||
status=health.MISSING,
|
||||
detail="keyless floor suppressed: host has native web search",
|
||||
prescription="",
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
|
||||
def _probe_reddit_public(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""Public keyless Reddit composite; internal lanes are sub-probe detail."""
|
||||
return BackendFinding(
|
||||
name="public",
|
||||
status=health.OK,
|
||||
detail="public keyless composite (lanes: rss, listing, arctic, shreddit)",
|
||||
requires="none (public endpoints)",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry: routing declared once, from env.py's definitions where they exist.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_X_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = {
|
||||
"xai": _key_probe("xai", "XAI_API_KEY", "XAI_API_KEY (xAI/Grok live search)"),
|
||||
"grok": _probe_grok,
|
||||
"bird": _probe_bird,
|
||||
"xurl": _probe_xurl,
|
||||
"xquik": _key_probe("xquik", "XQUIK_API_KEY", "XQUIK_API_KEY (xquik.com)"),
|
||||
}
|
||||
_X_PAID = {"xai", "xquik"}
|
||||
# Opt-in backends: never auto-selected; require explicit pin.
|
||||
_X_OPT_IN = set(env.X_BACKEND_OPT_IN)
|
||||
|
||||
_WEB_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = {
|
||||
"brave": _key_probe("brave", "BRAVE_API_KEY", "BRAVE_API_KEY"),
|
||||
"exa": _key_probe("exa", "EXA_API_KEY", "EXA_API_KEY"),
|
||||
"serper": _key_probe("serper", "SERPER_API_KEY", "SERPER_API_KEY"),
|
||||
"parallel": _key_probe("parallel", "PARALLEL_API_KEY", "PARALLEL_API_KEY"),
|
||||
"keyless": _probe_web_keyless,
|
||||
}
|
||||
_WEB_KEYED = {"brave", "exa", "serper", "parallel"}
|
||||
|
||||
_SC_SPEC = BackendSpec(
|
||||
name="scrapecreators",
|
||||
requires="SCRAPECREATORS_API_KEY",
|
||||
probe=_key_probe(
|
||||
"scrapecreators", "SCRAPECREATORS_API_KEY", "SCRAPECREATORS_API_KEY",
|
||||
note=_SC_PRESCRIPTION,
|
||||
),
|
||||
paid=True,
|
||||
)
|
||||
|
||||
# X backend requirements, keyed by name.
|
||||
_X_REQUIRES: Dict[str, str] = {
|
||||
"xai": "XAI_API_KEY (xAI/Grok live search)",
|
||||
"grok": "grok CLI installed + signed in (opt-in only; pin to enable)",
|
||||
"bird": "X browser cookies (AUTH_TOKEN/CT0) + node",
|
||||
"xurl": "xurl CLI installed + OAuth2 login",
|
||||
"xquik": "XQUIK_API_KEY (xquik.com)",
|
||||
}
|
||||
|
||||
DESCRIPTORS: Dict[str, ChainDescriptor] = {
|
||||
# X: chain order and pin var imported from env.py (single source of truth).
|
||||
# Backends include the auto chain (X_BACKEND_ORDER) plus opt-in entries
|
||||
# (X_BACKEND_OPT_IN) for doctor visibility. Opt-in backends like grok
|
||||
# appear in findings but are never auto-selected; pin to enable.
|
||||
"x": ChainDescriptor(
|
||||
source="x",
|
||||
mode=MODE_ALTERNATIVE,
|
||||
backends=tuple(
|
||||
BackendSpec(
|
||||
name=name,
|
||||
requires=_X_REQUIRES[name],
|
||||
probe=_X_PROBES[name],
|
||||
paid=name in _X_PAID,
|
||||
opt_in=name in _X_OPT_IN,
|
||||
)
|
||||
for name in env.X_BACKEND_ORDER + env.X_BACKEND_OPT_IN
|
||||
),
|
||||
pin_var=env.X_BACKEND_PIN_VAR,
|
||||
),
|
||||
"youtube": ChainDescriptor(
|
||||
source="youtube",
|
||||
mode=MODE_ALTERNATIVE,
|
||||
backends=(
|
||||
BackendSpec(
|
||||
name="yt-dlp",
|
||||
requires="yt-dlp on the agent-subprocess PATH",
|
||||
probe=_probe_ytdlp,
|
||||
),
|
||||
_SC_SPEC,
|
||||
),
|
||||
pin_var=None, # no YouTube pin knob exists
|
||||
),
|
||||
"web": ChainDescriptor(
|
||||
source="web",
|
||||
mode=MODE_ALTERNATIVE,
|
||||
backends=tuple(
|
||||
BackendSpec(
|
||||
name=name,
|
||||
requires=(f"{name.upper()}_API_KEY" if name in _WEB_KEYED
|
||||
else "no key; suppressed on native-search hosts"),
|
||||
probe=_WEB_PROBES[name],
|
||||
paid=name in _WEB_KEYED,
|
||||
)
|
||||
for name in WEB_BACKEND_ORDER
|
||||
),
|
||||
pin_var=None, # pinned per-run via --web-backend, not an env var
|
||||
pin_flag="--web-backend",
|
||||
),
|
||||
"reddit": ChainDescriptor(
|
||||
source="reddit",
|
||||
mode=MODE_CONDITIONAL,
|
||||
backends=(
|
||||
BackendSpec(
|
||||
name="public",
|
||||
requires="none (public endpoints)",
|
||||
probe=_probe_reddit_public,
|
||||
),
|
||||
_SC_SPEC,
|
||||
),
|
||||
pin_var=env.REDDIT_BACKEND_PIN_VAR,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_descriptor(source: str) -> ChainDescriptor:
|
||||
"""Return the declared routing descriptor for ``source`` (KeyError if none)."""
|
||||
return DESCRIPTORS[source]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def resolve(
|
||||
source: str,
|
||||
config: Dict[str, Any],
|
||||
pin: Optional[str] = None,
|
||||
) -> BackendResolution:
|
||||
"""Resolve a chained source's routing into a truthful prediction.
|
||||
|
||||
``pin`` is an explicit per-run pin (the ``--web-backend`` flag); it
|
||||
takes precedence over the descriptor's env pin var. ``"auto"``/None
|
||||
mean unpinned. Probing is side-effect-free and collect-then-pick.
|
||||
|
||||
Time budget: backends are probed sequentially, so a chain's budget is
|
||||
ADDITIVE across its backends — each binary-backed probe is bounded by
|
||||
``health.PROBE_TIMEOUT`` and paid/key lanes are dict lookups that cost
|
||||
nothing, giving a worst case of roughly (binary probes in the chain) x
|
||||
``health.PROBE_TIMEOUT``. Deliberately no intra-chain concurrency:
|
||||
probes are memoized per process and the worst case only occurs when
|
||||
multiple binaries are simultaneously hung.
|
||||
"""
|
||||
descriptor = get_descriptor(source)
|
||||
findings = [
|
||||
_run_probe(spec, config) for spec in descriptor.backends
|
||||
]
|
||||
if descriptor.mode == MODE_CONDITIONAL:
|
||||
return _resolve_conditional(descriptor, config, findings)
|
||||
return _resolve_alternative(descriptor, config, findings, pin)
|
||||
|
||||
|
||||
def _run_probe(spec: BackendSpec, config: Dict[str, Any]) -> BackendFinding:
|
||||
"""Run one probe, isolating failures so one bad probe can't blank a chain."""
|
||||
try:
|
||||
finding = spec.probe(config)
|
||||
except Exception as exc: # a probe bug must not take the report down
|
||||
finding = BackendFinding(
|
||||
name=spec.name,
|
||||
status=health.ERROR,
|
||||
detail=f"probe failed: {type(exc).__name__}: {exc}",
|
||||
requires=spec.requires,
|
||||
)
|
||||
if not finding.requires:
|
||||
finding.requires = spec.requires
|
||||
return finding
|
||||
|
||||
|
||||
def _resolve_alternative(
|
||||
descriptor: ChainDescriptor,
|
||||
config: Dict[str, Any],
|
||||
findings: List[BackendFinding],
|
||||
pin: Optional[str],
|
||||
) -> BackendResolution:
|
||||
names = [spec.name for spec in descriptor.backends]
|
||||
by_name = {f.name: f for f in findings}
|
||||
# Track which backends are opt-in (never auto-selected).
|
||||
opt_in_names = {spec.name for spec in descriptor.backends if spec.opt_in}
|
||||
res = BackendResolution(
|
||||
source=descriptor.source,
|
||||
mode=MODE_ALTERNATIVE,
|
||||
chain=list(names),
|
||||
findings=findings,
|
||||
)
|
||||
|
||||
pin_name: Optional[str] = None
|
||||
if pin and pin not in ("auto", "none") and pin in by_name:
|
||||
pin_name = pin
|
||||
elif descriptor.pin_var:
|
||||
raw = (config.get(descriptor.pin_var) or "").lower()
|
||||
if raw in by_name:
|
||||
pin_name = raw
|
||||
|
||||
if pin_name:
|
||||
# A pin forces a single backend (no failover) — mirror
|
||||
# env.x_backend_chain's pin semantics exactly.
|
||||
res.pinned = True
|
||||
res.pin = pin_name
|
||||
finding = by_name[pin_name]
|
||||
if finding.status == health.OK:
|
||||
res.active_backend = pin_name
|
||||
res.tier = TIER_OK
|
||||
elif finding.status == health.DEGRADED:
|
||||
res.active_backend = pin_name
|
||||
res.tier = TIER_WARN
|
||||
else:
|
||||
res.tier = TIER_ERROR
|
||||
res.prescription = finding.prescription or (
|
||||
f"unpin {descriptor.pin_var or descriptor.pin_flag} or fix {pin_name}"
|
||||
)
|
||||
return res
|
||||
|
||||
# Collect-then-pick: first fully-usable wins; else best degraded; else
|
||||
# error carrying the highest-priority backend's prescription.
|
||||
# Opt-in backends are NEVER auto-selected; skip them entirely.
|
||||
auto_findings = [f for f in findings if f.name not in opt_in_names]
|
||||
for finding in auto_findings:
|
||||
if finding.status == health.OK:
|
||||
res.active_backend = finding.name
|
||||
res.tier = TIER_OK
|
||||
return res
|
||||
for finding in auto_findings:
|
||||
if finding.status == health.DEGRADED:
|
||||
res.active_backend = finding.name
|
||||
res.tier = TIER_WARN
|
||||
return res
|
||||
res.tier = TIER_ERROR
|
||||
# Prescription comes from the first auto-chain backend, not opt-in.
|
||||
res.prescription = auto_findings[0].prescription if auto_findings else ""
|
||||
return res
|
||||
|
||||
|
||||
def _reddit_sc_min_items(config: Dict[str, Any]) -> int:
|
||||
"""The thinness floor, parsed exactly as the pipeline parses it
|
||||
(lib/pipeline.py reddit fetch: int(... or 0), malformed -> 0)."""
|
||||
try:
|
||||
return int(config.get(env.REDDIT_SC_MIN_ITEMS_VAR) or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _resolve_conditional(
|
||||
descriptor: ChainDescriptor,
|
||||
config: Dict[str, Any],
|
||||
findings: List[BackendFinding],
|
||||
) -> BackendResolution:
|
||||
"""Reddit: render the real per-query semantics, never a computed winner."""
|
||||
res = BackendResolution(
|
||||
source=descriptor.source,
|
||||
mode=MODE_CONDITIONAL,
|
||||
chain=[spec.name for spec in descriptor.backends],
|
||||
findings=findings,
|
||||
active_backend=None, # conditional mode never picks a winner
|
||||
tier=TIER_OK, # the public keyless composite is always reachable
|
||||
)
|
||||
has_key = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||
raw_pin = (config.get(descriptor.pin_var) or "").lower() if descriptor.pin_var else ""
|
||||
pinned_sc = has_key and raw_pin == "scrapecreators"
|
||||
floor = _reddit_sc_min_items(config)
|
||||
|
||||
if pinned_sc:
|
||||
res.pinned = True
|
||||
res.pin = "scrapecreators"
|
||||
res.conditional = (
|
||||
f"ScrapeCreators primary (pinned via {descriptor.pin_var}); "
|
||||
"public keyless composite fallback"
|
||||
)
|
||||
return res
|
||||
|
||||
if has_key:
|
||||
if floor > 0:
|
||||
backfill = (
|
||||
f"ScrapeCreators backfill when results fall below the "
|
||||
f"{floor}-item floor"
|
||||
)
|
||||
else:
|
||||
backfill = "ScrapeCreators backfill when the free path returns nothing"
|
||||
res.conditional = f"public keyless composite (default); {backfill}"
|
||||
return res
|
||||
|
||||
res.conditional = "public keyless composite (default); no ScrapeCreators key for backfill"
|
||||
if raw_pin == "scrapecreators":
|
||||
# The pipeline ignores the pin without a key; say so honestly.
|
||||
res.conditional += (
|
||||
f" ({descriptor.pin_var} pin ignored: SCRAPECREATORS_API_KEY not set)"
|
||||
)
|
||||
return res
|
||||
@@ -12,7 +12,7 @@ import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log, subproc
|
||||
from . import env, health, http, log, subproc
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -24,6 +24,16 @@ MAX_JSON_DECODE_RETRIES = 2
|
||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
||||
|
||||
|
||||
def _leading_mentions(text: str) -> list:
|
||||
"""Leading-run @mention parse, shared with other X-shaped sources (xquik).
|
||||
|
||||
Thin wrapper over ``query.leading_mentions`` so bird and xquik share one
|
||||
implementation; kept here for existing call sites and tests.
|
||||
"""
|
||||
from .query import leading_mentions
|
||||
return leading_mentions(text)
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
for v in values:
|
||||
@@ -60,7 +70,7 @@ def _has_injected_credentials() -> bool:
|
||||
|
||||
def _has_process_credentials() -> bool:
|
||||
"""Return True when AUTH_TOKEN/CT0 are present in process env."""
|
||||
return bool(os.environ.get("AUTH_TOKEN") and os.environ.get("CT0"))
|
||||
return bool(env.read_secret_env("AUTH_TOKEN") and env.read_secret_env("CT0"))
|
||||
|
||||
|
||||
def _subprocess_env() -> Dict[str, str]:
|
||||
@@ -77,6 +87,19 @@ def _log(msg: str):
|
||||
log.source_log("Bird", msg, tty_only=False)
|
||||
|
||||
|
||||
def classify_run_failure(detail: str) -> str:
|
||||
"""Map Bird's subprocess-only failure shapes to run outcome states."""
|
||||
text = detail.lower()
|
||||
if any(marker in text for marker in ("interstitial", "non-json", "invalid json")):
|
||||
return health.SCHEMA_DRIFT
|
||||
if any(
|
||||
marker in text
|
||||
for marker in ("cookie expired", "expired cookie", "unauthorized", "forbidden", "login required")
|
||||
):
|
||||
return health.AUTH_FAILED
|
||||
return http.classify_failure(message=detail)
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for X search.
|
||||
|
||||
@@ -88,6 +111,53 @@ def _extract_core_subject(topic: str) -> str:
|
||||
return extract_core_subject(topic, max_words=5, strip_suffixes=True)
|
||||
|
||||
|
||||
def _plain_query_tokens(text: str) -> list[str]:
|
||||
"""Return lexical tokens without Bird query grouping syntax.
|
||||
|
||||
Strips phrase quotes as well as grouping characters. Used where a flat
|
||||
token list is wanted; use ``build_topic_query`` for the provider query,
|
||||
which preserves quoted phrases.
|
||||
"""
|
||||
separators = str.maketrans({char: " " for char in '\"“”()[]{}'})
|
||||
return [
|
||||
clean
|
||||
for token in text.translate(separators).split()
|
||||
if (clean := token.strip("'‘’"))
|
||||
]
|
||||
|
||||
|
||||
# Bird/X grouping syntax that carries no lexical meaning. Double quotes are
|
||||
# deliberately absent: X advanced search treats "..." as a phrase match, which
|
||||
# is exactly what the planner intended when it quoted a proper noun.
|
||||
_GROUPING_CHARS = "“”()[]{}"
|
||||
|
||||
|
||||
def build_topic_query(topic: str, from_date: str) -> str:
|
||||
"""Build the X topic query, preserving quoted proper-noun phrases.
|
||||
|
||||
Previously the topic went through ``_plain_query_tokens``, which stripped
|
||||
the quotes the planner had added, so an intended phrase match for
|
||||
'"Peter Steinberger"' degraded into `peter AND steinberger` -- narrower and
|
||||
noisier at once. X supports phrase queries natively, so the quotes are
|
||||
passed through.
|
||||
"""
|
||||
separators = str.maketrans({char: " " for char in _GROUPING_CHARS})
|
||||
cleaned = topic.translate(separators)
|
||||
# An unbalanced quote is worse than no quote: X reads the orphan as an
|
||||
# unterminated phrase and matches nothing. Upstream trimming (core-subject
|
||||
# extraction, retry shortening) can cut a topic mid-phrase, so verify the
|
||||
# quotes pair up and fall back to bare tokens when they do not.
|
||||
if cleaned.count('"') % 2:
|
||||
cleaned = cleaned.replace('"', " ")
|
||||
tokens = [
|
||||
clean
|
||||
for token in cleaned.split()
|
||||
if (clean := token.strip("'‘’"))
|
||||
]
|
||||
core = " ".join(tokens).strip()
|
||||
return f"{core} since:{from_date}" if core else f"since:{from_date}"
|
||||
|
||||
|
||||
def is_bird_installed() -> bool:
|
||||
"""Check if vendored Bird search module is available.
|
||||
|
||||
@@ -115,6 +185,40 @@ def is_bird_authenticated() -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
_probe_cache: Optional[Optional[bool]] = "unset" # "unset" | True | False | None
|
||||
|
||||
|
||||
def probe_works(timeout: int = 8) -> Optional[bool]:
|
||||
"""Cheap runtime check that X auth actually returns data.
|
||||
|
||||
Returns True when a 1-result probe comes back without an error, False on a
|
||||
clear failure (auth error / generic search failure), and None when the
|
||||
result is inconclusive (network timeout) so callers can fail open and keep
|
||||
the static credential-presence status rather than reporting a false-down.
|
||||
Cached per process so repeated diagnose calls don't re-probe.
|
||||
"""
|
||||
global _probe_cache
|
||||
if _probe_cache != "unset":
|
||||
return _probe_cache # type: ignore[return-value]
|
||||
if not (_has_injected_credentials() or _has_process_credentials()):
|
||||
_probe_cache = False
|
||||
return False
|
||||
from datetime import datetime, timedelta, timezone
|
||||
since = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
# @x (the platform's own account) posts frequently, so a no-error response
|
||||
# means auth works even if this particular window is quiet.
|
||||
resp = _run_bird_search(f"from:x since:{since}", count=1, timeout=timeout)
|
||||
if isinstance(resp, dict) and resp.get("error"):
|
||||
err = str(resp.get("error")).lower()
|
||||
if "timed out" in err or "timeout" in err:
|
||||
_probe_cache = None # inconclusive — don't downgrade on a transient timeout
|
||||
return None
|
||||
_probe_cache = False
|
||||
return False
|
||||
_probe_cache = True
|
||||
return True
|
||||
|
||||
|
||||
def check_npm_available() -> bool:
|
||||
"""Check if npm is available (kept for API compatibility).
|
||||
|
||||
@@ -226,11 +330,18 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
if terminal_error is not None:
|
||||
return terminal_error
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = result.stdout.strip()
|
||||
if result.returncode != 0:
|
||||
if not output:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
# Windows/Node 24: the vendored Bird CLI uses native fetch (undici),
|
||||
# and calling process.exit() while keep-alive sockets are still
|
||||
# closing trips a libuv assertion -> non-zero exit code AFTER it has
|
||||
# already written a complete, valid JSON result to stdout. Trust
|
||||
# stdout when it has content; only treat a non-zero exit as a real
|
||||
# failure when stdout is empty.
|
||||
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
@@ -252,10 +363,11 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
log.source_log(
|
||||
"X/bird",
|
||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
||||
tty_only=False,
|
||||
)
|
||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
||||
continue
|
||||
log.source_log("X/bird", log_msg)
|
||||
log.source_log("X/bird", log_msg, tty_only=False)
|
||||
return {
|
||||
"error": (
|
||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
||||
@@ -296,17 +408,19 @@ def search_x(
|
||||
timeout = 30 if depth == "quick" else 45 if depth == "default" else 60
|
||||
|
||||
# Extract core subject - X search is literal, not semantic
|
||||
core_topic = _extract_core_subject(topic)
|
||||
query = f"{core_topic} since:{from_date}"
|
||||
core_subject = _extract_core_subject(topic)
|
||||
core_words = _plain_query_tokens(core_subject)
|
||||
core_topic = " ".join(core_words)
|
||||
query = build_topic_query(core_subject, from_date)
|
||||
|
||||
_log(f"Searching: {query}")
|
||||
response = _run_bird_search(query, count, timeout)
|
||||
last_clean_response = response if not response.get("error") else None
|
||||
|
||||
# Check if we got results
|
||||
items = parse_bird_response(response, query=core_topic)
|
||||
|
||||
# Retry with OR groups for multi-word queries (X supports OR operator)
|
||||
core_words = core_topic.split()
|
||||
if not items and len(core_words) >= 2:
|
||||
from .query import extract_compound_terms
|
||||
compounds = extract_compound_terms(topic)
|
||||
@@ -316,6 +430,8 @@ def search_x(
|
||||
_log(f"0 results for '{core_topic}', retrying with OR groups: {or_parts}")
|
||||
query = f"({or_parts}) since:{from_date}"
|
||||
response = _run_bird_search(query, count, timeout)
|
||||
if not response.get("error"):
|
||||
last_clean_response = response
|
||||
items = parse_bird_response(response, query=core_topic)
|
||||
|
||||
# Retry with fewer keywords if still 0 results and query has 3+ words
|
||||
@@ -324,6 +440,8 @@ def search_x(
|
||||
_log(f"0 results for '{core_topic}', retrying with '{shorter}'")
|
||||
query = f"{shorter} since:{from_date}"
|
||||
response = _run_bird_search(query, count, timeout)
|
||||
if not response.get("error"):
|
||||
last_clean_response = response
|
||||
items = parse_bird_response(response, query=core_topic)
|
||||
|
||||
# Last-chance retry: use strongest remaining token (often the product name)
|
||||
@@ -335,11 +453,24 @@ def search_x(
|
||||
}
|
||||
candidates = [w for w in core_words if w not in low_signal]
|
||||
if candidates:
|
||||
# Keep an entity anchor (the first distinctive topic token) in the
|
||||
# retry so it can't collapse to a bare generic token like "compound"
|
||||
# and flood the X pool with off-topic noise. Add the strongest
|
||||
# (longest) distinctive token when it differs from the anchor;
|
||||
# otherwise query the anchor alone. Better to return 0 than to
|
||||
# over-broaden to an unanchored generic term.
|
||||
anchor = candidates[0]
|
||||
strongest = max(candidates, key=len)
|
||||
_log(f"0 results for '{core_topic}', retrying with strongest token '{strongest}'")
|
||||
query = f"{strongest} since:{from_date}"
|
||||
retry_terms = anchor if strongest == anchor else f"{anchor} {strongest}"
|
||||
_log(f"0 results for '{core_topic}', retrying anchored on '{retry_terms}'")
|
||||
query = f"{retry_terms} since:{from_date}"
|
||||
response = _run_bird_search(query, count, timeout)
|
||||
if not response.get("error"):
|
||||
last_clean_response = response
|
||||
|
||||
if response.get("error") and last_clean_response is not None:
|
||||
_log("Optional retry failed after a clean empty response; preserving no-results outcome")
|
||||
return last_clean_response
|
||||
return response
|
||||
|
||||
|
||||
@@ -351,12 +482,15 @@ def search_handles(
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search specific X handles for topic-related content.
|
||||
|
||||
Runs targeted Bird searches using `from:handle topic` syntax.
|
||||
Used in Phase 2 supplemental search after entity extraction.
|
||||
Pulls each handle's actual timeline via `from:handle since:` — the FROM
|
||||
lane (tweets BY the person), engagement-weighted downstream. The topic is
|
||||
used for relevance RANKING, never AND'd into the query: X search is literal,
|
||||
so `from:handle <their name>` only matched tweets where they wrote their own
|
||||
name and returned ~0. Used in Phase 2 after entity extraction.
|
||||
|
||||
Args:
|
||||
handles: List of X handles to search (without @)
|
||||
topic: Search topic (core subject), or None for unfiltered search
|
||||
topic: Search topic — used for relevance ranking only, not the query
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
count_per: Results to request per handle
|
||||
|
||||
@@ -367,10 +501,8 @@ def search_handles(
|
||||
|
||||
def _search_one_handle(handle: str) -> List[Dict[str, Any]]:
|
||||
handle = handle.lstrip("@")
|
||||
if core_topic:
|
||||
query = f"from:{handle} {core_topic} since:{from_date}"
|
||||
else:
|
||||
query = f"from:{handle} since:{from_date}"
|
||||
# Always unfiltered: pull the timeline, rank by topic relevance below.
|
||||
query = f"from:{handle} since:{from_date}"
|
||||
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -388,11 +520,14 @@ def search_handles(
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
if result.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
|
||||
output = result.stdout.strip()
|
||||
if result.returncode != 0:
|
||||
if not output:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
# Windows/Node 24: benign libuv assertion can cause non-zero exit
|
||||
# AFTER valid JSON is written to stdout. Trust stdout content.
|
||||
|
||||
if not output:
|
||||
return []
|
||||
|
||||
@@ -401,7 +536,11 @@ def search_handles(
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
return []
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
items = parse_bird_response(response, query=core_topic)
|
||||
# Log on success/empty too (not only on failure): a silent handle search
|
||||
# made the from: query look like it never ran and caused wrong diagnoses.
|
||||
_log(f"Searching: {query} -> {len(items)} results")
|
||||
return items
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
@@ -414,6 +553,77 @@ def search_handles(
|
||||
return all_items
|
||||
|
||||
|
||||
def search_mentions(
|
||||
handles: List[str],
|
||||
from_date: str,
|
||||
count_per: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search for tweets ABOUT/TO each handle — the mention lane.
|
||||
|
||||
Queries `@handle since:` (tweets that mention the account) and excludes the
|
||||
handle's OWN tweets (those belong to the FROM lane via search_handles), so
|
||||
this surfaces what OTHERS are saying about the person. Engagement-weighted
|
||||
downstream; deduped against the FROM lane by URL at normalize time.
|
||||
|
||||
Args:
|
||||
handles: List of X handles (without @)
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
count_per: Results to request per handle
|
||||
|
||||
Returns:
|
||||
List of raw item dicts (same format as parse_bird_response output).
|
||||
"""
|
||||
def _search_one(handle: str) -> List[Dict[str, Any]]:
|
||||
handle = handle.lstrip("@")
|
||||
query = f"@{handle} since:{from_date}"
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
query,
|
||||
"--count", str(count_per),
|
||||
"--json",
|
||||
]
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
||||
except subproc.SubprocTimeout:
|
||||
_log(f"Mention search timed out for @{handle}")
|
||||
return []
|
||||
except OSError as e:
|
||||
_log(f"Mention search error for @{handle}: {e}")
|
||||
return []
|
||||
if result.returncode != 0:
|
||||
_log(f"Mention search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return []
|
||||
try:
|
||||
response = json.loads(output)
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from mention search for @{handle}")
|
||||
return []
|
||||
items = parse_bird_response(response, query=None)
|
||||
# ABOUT lane = OTHERS mentioning the handle. Drop the handle's own tweets
|
||||
# (the FROM lane already covers those); identify by the status URL author.
|
||||
hl = handle.lower()
|
||||
# The Bird API may return either x.com or twitter.com permalinks, so
|
||||
# match both when excluding the handle's own tweets.
|
||||
def _is_own(url):
|
||||
u = (url or "").lower()
|
||||
return f"x.com/{hl}/status" in u or f"twitter.com/{hl}/status" in u
|
||||
about = [it for it in items if not _is_own(it.get("url"))]
|
||||
_log(f"Searching: {query} -> {len(about)} mentions")
|
||||
return about
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
all_items: List[Dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=min(5, len(handles))) as executor:
|
||||
futures = {executor.submit(_search_one, h): h for h in handles}
|
||||
for future in as_completed(futures):
|
||||
all_items.extend(future.result())
|
||||
return all_items
|
||||
|
||||
|
||||
def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
"""Parse Bird response to match xai_x output format.
|
||||
|
||||
@@ -489,11 +699,16 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
|
||||
engagement[key] = None
|
||||
|
||||
# Build normalized item
|
||||
text = str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500]
|
||||
item = {
|
||||
"id": f"X{i+1}",
|
||||
"text": str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500],
|
||||
"text": text,
|
||||
"url": url,
|
||||
"author_handle": author_handle.lstrip("@"),
|
||||
# Leading @mentions parsed from the post text identify who a reply is
|
||||
# directed at (X replies open with the target handle(s)). Used by the
|
||||
# interaction-signal classifier in rerank.
|
||||
"mentioned_handles": _leading_mentions(text),
|
||||
"date": date,
|
||||
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
|
||||
"why_relevant": "", # Bird doesn't provide relevance explanations
|
||||
|
||||
@@ -96,7 +96,7 @@ _TOKEN_MAX_AGE_SECONDS = 5400 # 90 minutes (conservative, tokens last ~2 hours)
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
log.source_log("Bluesky", msg)
|
||||
log.source_log("Bluesky", msg, tty_only=False)
|
||||
|
||||
|
||||
def _create_session(handle: str, app_password: str) -> Optional[str]:
|
||||
@@ -158,14 +158,8 @@ def _reset_session_cache() -> None:
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for Bluesky search."""
|
||||
from .query import extract_core_subject
|
||||
_BSKY_NOISE = frozenset({
|
||||
'best', 'top', 'good', 'great', 'awesome',
|
||||
'latest', 'new', 'news', 'update', 'updates',
|
||||
'trending', 'hottest', 'popular', 'viral',
|
||||
'practices', 'features', 'recommendations', 'advice',
|
||||
})
|
||||
return extract_core_subject(topic, noise=_BSKY_NOISE)
|
||||
from .query import SOCIAL_NOISE, extract_core_subject
|
||||
return extract_core_subject(topic, noise=SOCIAL_NOISE)
|
||||
|
||||
|
||||
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Bright Data CLI adapter for last30days.
|
||||
|
||||
Shells out to the ``brightdata`` CLI (``@brightdata/cli``) to run Bright
|
||||
Data Pipelines. The CLI owns authentication end to end -- ``brightdata
|
||||
login`` does a gh-style zero-click browser flow and stores credentials in
|
||||
a platform config directory -- so this module never handles a login, and
|
||||
never reads credential *contents*: the auth probe is presence-only.
|
||||
|
||||
Activation gate: two-way, mirroring the digg CLI-gated precedent but with
|
||||
an auth dimension the digg source does not have.
|
||||
|
||||
1. ``shutil.which("brightdata")`` must resolve on the **agent subprocess
|
||||
PATH** (not merely exist on disk -- Hermes/OpenClaw gateways often drop
|
||||
``~/.local/bin``).
|
||||
2. A credential signal must be present: either ``BRIGHTDATA_API_KEY``
|
||||
resolved through the normal config layering, or the CLI's own
|
||||
credentials file in the platform config dir.
|
||||
|
||||
The second check is deliberately offline. A stale token passes it and
|
||||
then 401s fast at call time; that path degrades to empty results with the
|
||||
CLI's own error line preserved in the envelope, which is the AE2 contract.
|
||||
|
||||
Metering note (R13): no pricing logic lives here. One pipeline request
|
||||
costs one credit against the account's monthly free tier regardless of how
|
||||
many records come back, so caps in the calling adapter bound *records*
|
||||
(paid-tier cost), not credits. Credit and auth warnings from the CLI are
|
||||
passed through verbatim rather than interpreted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from . import log, subproc
|
||||
|
||||
|
||||
CLI_BIN = "brightdata"
|
||||
|
||||
# Env var carrying an explicit API key. Registered in env.py so `.env` file
|
||||
# and keychain users pass the gate the same way process-env users do; when
|
||||
# it resolves from a non-process-env layer we hand it to the CLI via -k.
|
||||
API_KEY_ENV = "BRIGHTDATA_API_KEY"
|
||||
|
||||
# Credentials filename written by `brightdata login`. Probed for existence
|
||||
# only -- never opened, parsed, or logged.
|
||||
_CREDENTIALS_FILENAME = "credentials.json"
|
||||
_CONFIG_DIRNAME = "brightdata-cli"
|
||||
|
||||
# The CLI's own polling timeout sits below our subprocess timeout so the CLI
|
||||
# exits cleanly with its own error rather than being SIGTERM'd mid-poll. Its
|
||||
# timeout path throws with zero records (verified in its polling module --
|
||||
# never partial output), so a timed-out pull is a clean parseable failure.
|
||||
_CLI_TIMEOUT_MARGIN = 10
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("BrightData", msg, tty_only=False)
|
||||
|
||||
|
||||
def _config_dir() -> Path:
|
||||
"""Platform config directory the Bright Data CLI stores credentials in.
|
||||
|
||||
Mirrors the CLI's own credentials module: APPDATA on Windows, the
|
||||
Application Support tree on macOS, XDG_CONFIG_HOME (or ~/.config) on
|
||||
everything else.
|
||||
"""
|
||||
if sys.platform == "win32":
|
||||
base = os.environ.get("APPDATA")
|
||||
root = Path(base) if base else Path.home() / "AppData" / "Roaming"
|
||||
elif sys.platform == "darwin":
|
||||
root = Path.home() / "Library" / "Application Support"
|
||||
else:
|
||||
base = os.environ.get("XDG_CONFIG_HOME")
|
||||
root = Path(base) if base else Path.home() / ".config"
|
||||
return root / _CONFIG_DIRNAME
|
||||
|
||||
|
||||
def is_installed() -> bool:
|
||||
"""True when the brightdata binary resolves on the agent subprocess PATH."""
|
||||
return shutil.which(CLI_BIN) is not None
|
||||
|
||||
|
||||
def _api_key(config: Optional[Dict[str, Any]]) -> str:
|
||||
if not config:
|
||||
return ""
|
||||
return str(config.get(API_KEY_ENV) or "").strip()
|
||||
|
||||
|
||||
def has_credentials(config: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""True when some credential signal exists, without reading any secret.
|
||||
|
||||
Presence-only by design: an explicit API key resolved through config
|
||||
layering, or the existence of the CLI's credentials file. The file is
|
||||
never opened. This cannot distinguish a live token from an expired one
|
||||
-- that is what the fast 401 at call time is for.
|
||||
"""
|
||||
if _api_key(config):
|
||||
return True
|
||||
try:
|
||||
return (_config_dir() / _CREDENTIALS_FILENAME).exists()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def is_available(config: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""The full activation gate: binary on PATH *and* a credential signal."""
|
||||
return is_installed() and has_credentials(config)
|
||||
|
||||
|
||||
def gate_status(config: Optional[Dict[str, Any]] = None) -> Dict[str, bool]:
|
||||
"""Two-field probe for ``pipeline.diagnose`` (bird_installed precedent).
|
||||
|
||||
Network-free, so it is safe on the ``--diagnose`` / doctor path.
|
||||
"""
|
||||
installed = is_installed()
|
||||
return {
|
||||
"brightdata_installed": installed,
|
||||
"brightdata_authenticated": installed and has_credentials(config),
|
||||
}
|
||||
|
||||
|
||||
def _build_args(
|
||||
pipeline_type: str,
|
||||
params: Sequence[str],
|
||||
*,
|
||||
cli_timeout: int,
|
||||
) -> List[str]:
|
||||
"""Assemble the CLI invocation.
|
||||
|
||||
The API key is deliberately **absent** here -- it travels in the child's
|
||||
environment instead (see ``_child_env``). Process arguments are not a
|
||||
secret channel: ``/proc/<pid>/cmdline`` is world-readable under the
|
||||
default ``hidepid=0``, and a review pull lives for up to 180s, so a key
|
||||
on the command line is readable by any other local user and is captured
|
||||
verbatim by execve auditing, process accounting, and any monitoring
|
||||
agent that snapshots ``ps``. Mirrors the ``bird_x`` cookie-injection
|
||||
precedent.
|
||||
|
||||
Positional params are fenced behind ``--`` so a keyword that happens to
|
||||
begin with a dash is parsed as a search term rather than as an option.
|
||||
"""
|
||||
return [
|
||||
CLI_BIN,
|
||||
"pipelines",
|
||||
pipeline_type,
|
||||
"--json",
|
||||
"--timeout",
|
||||
str(cli_timeout),
|
||||
"--",
|
||||
*(str(p) for p in params),
|
||||
]
|
||||
|
||||
|
||||
def _child_env(api_key: str) -> Optional[Dict[str, str]]:
|
||||
"""Environment for the child process, carrying the key when we have one.
|
||||
|
||||
Returns None when there is nothing to inject, so the child simply
|
||||
inherits the parent environment (the common case: the CLI owns its own
|
||||
credentials file, or the key is already exported).
|
||||
"""
|
||||
if not api_key:
|
||||
return None
|
||||
return {**os.environ, API_KEY_ENV: api_key}
|
||||
|
||||
|
||||
def _scrub(text: str, secret: str) -> str:
|
||||
"""Remove a secret from text before it is logged or returned.
|
||||
|
||||
Defense in depth for the passthrough paths: the stderr lines this
|
||||
module deliberately surfaces are auth and quota failures, which are
|
||||
exactly the messages a CLI is most likely to echo the rejected
|
||||
credential back in.
|
||||
"""
|
||||
if not secret or not text:
|
||||
return text
|
||||
return text.replace(secret, "***")
|
||||
|
||||
|
||||
def _extract_records(payload: Any) -> List[Dict[str, Any]]:
|
||||
"""Pull the record list out of a parsed CLI payload.
|
||||
|
||||
Verified live (2026-08-13): both amazon pipelines return a **bare JSON
|
||||
array** of flat record dicts, not the ``{"results": [...]}`` envelope the
|
||||
digg CLI uses. The dict branches below are defensive against CLI churn,
|
||||
which is a live risk on a package this young.
|
||||
"""
|
||||
if isinstance(payload, list):
|
||||
return [r for r in payload if isinstance(r, dict)]
|
||||
if isinstance(payload, dict):
|
||||
for key in ("records", "results", "data"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, list):
|
||||
return [r for r in value if isinstance(r, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def run_pipeline(
|
||||
pipeline_type: str,
|
||||
params: Sequence[str],
|
||||
*,
|
||||
timeout: int,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run one Bright Data pipeline and return ``{"records", "error"}``.
|
||||
|
||||
Never raises. Every failure mode -- missing binary, spawn failure,
|
||||
subprocess timeout, non-zero exit, unparseable stdout -- returns empty
|
||||
records plus a one-line ``error`` string, so callers can record the
|
||||
failure in ``errors_by_source`` without branching on exception types.
|
||||
|
||||
The CLI's first stderr line is preserved verbatim as the error (auth
|
||||
401s and low-credit warnings are the cases that matter), and also
|
||||
mirrored to ``source_log`` so the failure is visible in non-TTY hosts.
|
||||
|
||||
Args:
|
||||
pipeline_type: pipeline name, e.g. ``amazon_product_search``.
|
||||
params: positional pipeline params, passed through in order.
|
||||
timeout: subprocess timeout in seconds. The CLI's own polling
|
||||
timeout is set just below this so it can fail cleanly first.
|
||||
config: resolved config dict, consulted only for the API key.
|
||||
|
||||
Returns:
|
||||
``{"records": [...]}`` on success, else ``{"records": [], "error": str}``.
|
||||
"""
|
||||
if not is_installed():
|
||||
return {"records": [], "error": f"{CLI_BIN} not on PATH"}
|
||||
|
||||
cli_timeout = max(5, int(timeout) - _CLI_TIMEOUT_MARGIN)
|
||||
key = _api_key(config)
|
||||
cmd = _build_args(pipeline_type, params, cli_timeout=cli_timeout)
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=timeout, env=_child_env(key))
|
||||
except subproc.SubprocTimeout as exc:
|
||||
_log(f"Timeout: {exc}")
|
||||
return {"records": [], "error": str(exc)}
|
||||
except FileNotFoundError as exc:
|
||||
_log(f"Binary missing: {exc}")
|
||||
return {"records": [], "error": str(exc)}
|
||||
except OSError as exc:
|
||||
_log(f"Spawn failed: {exc}")
|
||||
return {"records": [], "error": str(exc)}
|
||||
|
||||
stderr = _scrub(result.stderr or "", key)
|
||||
_passthrough_warnings(stderr)
|
||||
|
||||
if result.returncode != 0:
|
||||
lines = [ln.strip() for ln in stderr.strip().splitlines() if ln.strip()]
|
||||
# The CLI narrates polling progress on stderr, so the *last* line is
|
||||
# the actual failure; the first line is "Triggering pipeline...".
|
||||
first = lines[-1] if lines else f"exit {result.returncode}"
|
||||
_log(f"CLI exit {result.returncode}: {first}")
|
||||
return {"records": [], "error": first}
|
||||
|
||||
stdout = result.stdout or ""
|
||||
if not stdout.strip():
|
||||
return {"records": []}
|
||||
try:
|
||||
payload = json.loads(stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
_log(f"JSON decode failed: {exc}")
|
||||
return {"records": [], "error": f"json decode: {exc}"}
|
||||
|
||||
return {"records": _extract_records(payload)}
|
||||
|
||||
|
||||
# Substrings that mark a stderr line worth surfacing even on a successful
|
||||
# run -- credit exhaustion and auth trouble are the two the user must see.
|
||||
# Matched case-insensitively against the CLI's own wording, and echoed
|
||||
# verbatim rather than reworded (R13: no pricing logic, no interpretation).
|
||||
_WARNING_MARKERS = ("credit", "quota", "balance", "unauthor", "401", "expired", "login")
|
||||
|
||||
|
||||
def _passthrough_warnings(stderr: str) -> None:
|
||||
"""Echo credit/auth warning lines from the CLI verbatim.
|
||||
|
||||
Skips the routine polling narration so a normal run stays quiet.
|
||||
"""
|
||||
for line in (stderr or "").splitlines():
|
||||
text = line.strip()
|
||||
if not text or text.lower().startswith(("status:", "triggering", "triggered", "data received")):
|
||||
continue
|
||||
lowered = text.lower()
|
||||
if any(marker in lowered for marker in _WARNING_MARKERS):
|
||||
_log(text)
|
||||
@@ -23,6 +23,7 @@ before `ai_chat_model` so "gpt image 2" matches the image-gen category.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List, Optional, TypedDict
|
||||
|
||||
|
||||
@@ -126,6 +127,8 @@ CATEGORY_PEERS: dict[str, _CategoryEntry] = {
|
||||
},
|
||||
"ai_agent_framework": {
|
||||
"patterns": [
|
||||
"ai agent",
|
||||
"ai agents",
|
||||
"agent framework",
|
||||
"agentic framework",
|
||||
"langchain",
|
||||
@@ -264,7 +267,10 @@ def detect_category(topic: Optional[str]) -> Optional[str]:
|
||||
lowered = topic.lower()
|
||||
for category_id, entry in CATEGORY_PEERS.items():
|
||||
for pattern in entry["patterns"]:
|
||||
if pattern in lowered:
|
||||
# Word-boundary match: "ai agent" must not fire on "Dubai agents"
|
||||
# or "Thai agents". Substring matching classified those as
|
||||
# ai_agent_framework and routed discovery to LangChain subreddits.
|
||||
if re.search(rf"(?<![a-z0-9]){re.escape(pattern)}(?![a-z0-9])", lowered):
|
||||
return category_id
|
||||
return None
|
||||
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
"""Chrome and Brave cookie extraction for macOS.
|
||||
"""Chromium-family cookie extraction for macOS.
|
||||
|
||||
Extracts cookies from Chromium-based browser SQLite databases using only
|
||||
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
||||
dependencies.
|
||||
|
||||
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
Chrome and Brave share the same algorithm; only the DB path and Keychain
|
||||
service name differ.
|
||||
Every Chromium-based browser (Chrome, Brave, Edge, Vivaldi, Opera, Arc,
|
||||
Chromium) shares the same algorithm; only the profile directory and Keychain
|
||||
service name differ, so they all run through the same decryption core.
|
||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
@@ -21,9 +23,33 @@ from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _lock_temp_cookie_copy(path: str) -> None:
|
||||
"""Restrict copied cookie DB temp files to the current user on POSIX."""
|
||||
if os.name == "nt":
|
||||
return
|
||||
Path(path).chmod(0o600)
|
||||
|
||||
# Cookie DB locations on macOS
|
||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||
BRAVE_BASE_DIR = Path.home() / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser"
|
||||
_APP_SUPPORT = Path.home() / "Library" / "Application Support"
|
||||
CHROME_BASE_DIR = _APP_SUPPORT / "Google" / "Chrome"
|
||||
# Kept for backward compatibility; resolution now goes through the profile
|
||||
# finder (which also handles the modern Network/Cookies layout).
|
||||
CHROME_COOKIES_DB = CHROME_BASE_DIR / "Default" / "Cookies"
|
||||
BRAVE_BASE_DIR = _APP_SUPPORT / "BraveSoftware" / "Brave-Browser"
|
||||
|
||||
# Other Chromium-based browsers, keyed by FROM_BROWSER name. Each maps to
|
||||
# (profile base directory, macOS Keychain service name). Chrome and Brave keep
|
||||
# their dedicated helpers below for backward compatibility; everything here is
|
||||
# resolved generically by extract_chromium_browser_cookies_macos(). Keychain
|
||||
# service names follow Chromium's "<Browser> Safe Storage" convention.
|
||||
CHROMIUM_BROWSER_PROFILES: dict[str, tuple[Path, str]] = {
|
||||
"edge": (_APP_SUPPORT / "Microsoft Edge", "Microsoft Edge Safe Storage"),
|
||||
"vivaldi": (_APP_SUPPORT / "Vivaldi", "Vivaldi Safe Storage"),
|
||||
"opera": (_APP_SUPPORT / "com.operasoftware.Opera", "Opera Safe Storage"),
|
||||
"arc": (_APP_SUPPORT / "Arc" / "User Data", "Arc Safe Storage"),
|
||||
"chromium": (_APP_SUPPORT / "Chromium", "Chromium Safe Storage"),
|
||||
}
|
||||
|
||||
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
||||
CHROME_SALT = b"saltysalt"
|
||||
@@ -178,6 +204,7 @@ def _extract_chromium_cookies_macos(
|
||||
keychain_service: str,
|
||||
domain: str,
|
||||
cookie_names: list[str],
|
||||
key_cache: Optional[dict[str, Optional[bytes]]] = None,
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from any Chromium-based browser on macOS.
|
||||
|
||||
@@ -198,15 +225,16 @@ def _extract_chromium_cookies_macos(
|
||||
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
||||
return None
|
||||
|
||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
|
||||
# Copy DB to temp file (browser locks the original while running)
|
||||
tmp_fd = None
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
# mkstemp creates the file 0600. copy2 would copy the source DB's
|
||||
# permission bits onto the temp file before the chmod below runs,
|
||||
# briefly exposing live cookies when the source DB is looser.
|
||||
shutil.copyfile(str(db_path), tmp_path)
|
||||
_lock_temp_cookie_copy(tmp_path)
|
||||
except Exception as e:
|
||||
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
||||
if tmp_path:
|
||||
@@ -236,12 +264,28 @@ def _extract_chromium_cookies_macos(
|
||||
cursor.execute(query, params)
|
||||
|
||||
results: dict[str, str] = {}
|
||||
aes_key = None
|
||||
key_fetched = False
|
||||
for name, value, encrypted_value in cursor.fetchall():
|
||||
if value:
|
||||
results[name] = value
|
||||
continue
|
||||
|
||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||
if not key_fetched:
|
||||
# Fetch the Keychain key lazily — only once we actually have
|
||||
# an encrypted cookie to decrypt. This avoids a macOS
|
||||
# Keychain prompt for browsers that don't hold the requested
|
||||
# cookie, which matters for FROM_BROWSER=auto across several
|
||||
# installed Chromium browsers.
|
||||
if key_cache is not None and keychain_service in key_cache:
|
||||
aes_key = key_cache[keychain_service]
|
||||
else:
|
||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
if key_cache is not None:
|
||||
key_cache[keychain_service] = aes_key
|
||||
key_fetched = True
|
||||
if aes_key is None:
|
||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||
continue
|
||||
@@ -275,38 +319,115 @@ def _extract_chromium_cookies_macos(
|
||||
|
||||
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS."""
|
||||
return _extract_chromium_cookies_macos(
|
||||
CHROME_COOKIES_DB, "Chrome Safe Storage", domain, cookie_names
|
||||
"""Extract cookies from Chrome on macOS.
|
||||
|
||||
Resolves the cookie DB through the shared profile finder so Chrome gets the
|
||||
same modern ``Default/Network/Cookies`` (Chromium >= 96) and legacy
|
||||
``Default/Cookies`` probing as the rest of the Chromium family.
|
||||
"""
|
||||
return _extract_chromium_cookies_any_profile(
|
||||
CHROME_BASE_DIR, "Chrome Safe Storage", domain, cookie_names
|
||||
)
|
||||
|
||||
|
||||
def _find_brave_cookies_db() -> Optional[Path]:
|
||||
"""Find Brave's Cookies database on macOS.
|
||||
def _profile_cookie_db(profile_dir: Path) -> Optional[Path]:
|
||||
"""Return the Cookies DB inside a profile dir, or None.
|
||||
|
||||
Tries the Default profile first, then scans numbered Profile directories
|
||||
by most-recently-modified. Brave creates extra profiles as "Profile 1",
|
||||
"Profile 2", etc. alongside Default; the most recently used one is the
|
||||
likeliest to hold current cookies. Lexicographic sort would visit
|
||||
"Profile 10" before "Profile 2", which can return the wrong profile.
|
||||
Prefers the modern ``Network/Cookies`` location (Chromium >= 96 moved the
|
||||
cookie store into a per-profile ``Network/`` subdirectory) and falls back
|
||||
to the legacy flat ``Cookies`` file. Different browsers and versions use
|
||||
different layouts, so both are probed.
|
||||
"""
|
||||
default = BRAVE_BASE_DIR / "Default" / "Cookies"
|
||||
if default.exists():
|
||||
return default
|
||||
for rel in ("Network/Cookies", "Cookies"):
|
||||
candidate = profile_dir / rel
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _find_chromium_cookies_db(base_dir: Path) -> Optional[Path]:
|
||||
"""Find a Chromium-based browser's Cookies database under base_dir.
|
||||
|
||||
Checks the Default profile first, then the base dir itself (Opera's flat
|
||||
layout), then numbered "Profile N" directories by most-recently-modified.
|
||||
Each location is probed for both the modern ``Network/Cookies`` and legacy
|
||||
``Cookies`` paths (see _profile_cookie_db). Chromium browsers create extra
|
||||
profiles as "Profile 1", "Profile 2", etc. alongside Default; the most
|
||||
recently used one is the likeliest to hold current cookies. Lexicographic
|
||||
sort would visit "Profile 10" before "Profile 2", which can return the
|
||||
wrong profile, so we sort by mtime.
|
||||
|
||||
Kept for backward compatibility; new code should use
|
||||
_find_all_chromium_cookies_dbs() to search across all profiles.
|
||||
"""
|
||||
dbs = _find_all_chromium_cookies_dbs(base_dir)
|
||||
return dbs[0] if dbs else None
|
||||
|
||||
|
||||
def _find_all_chromium_cookies_dbs(base_dir: Path) -> list[Path]:
|
||||
"""Return ALL candidate Cookies DBs under base_dir, best-guess order first.
|
||||
|
||||
Order: Default, the base dir itself (Opera's flat layout), then numbered
|
||||
"Profile N" dirs by most-recently-modified. Unlike _find_chromium_cookies_db
|
||||
(which returns the first DB that merely EXISTS), this returns every profile
|
||||
so the caller can pick the one that actually holds the target domain's
|
||||
cookies. Needed because a logged-in session often lives in a non-Default
|
||||
profile while Default still has a (guest-only) cookie DB.
|
||||
"""
|
||||
paths: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
|
||||
def add(p: Optional[Path]) -> None:
|
||||
if p is not None and p not in seen:
|
||||
seen.add(p)
|
||||
paths.append(p)
|
||||
|
||||
add(_profile_cookie_db(base_dir / "Default"))
|
||||
add(_profile_cookie_db(base_dir))
|
||||
try:
|
||||
candidates = [
|
||||
child for child in BRAVE_BASE_DIR.iterdir()
|
||||
child for child in base_dir.iterdir()
|
||||
if child.is_dir() and child.name.startswith("Profile ")
|
||||
]
|
||||
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||
candidate = child / "Cookies"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
add(_profile_cookie_db(child))
|
||||
except OSError:
|
||||
pass
|
||||
return paths
|
||||
|
||||
return None
|
||||
|
||||
def _extract_chromium_cookies_any_profile(
|
||||
base_dir: Path, keychain_service: str, domain: str, cookie_names: list[str]
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Try every profile under base_dir and return the best cookie match.
|
||||
|
||||
Returns the first profile that yields ALL requested cookie_names. If no
|
||||
profile has the complete set, returns the first partial match found, or
|
||||
None if no profile yielded any. This fixes the single-profile limitation
|
||||
where a guest-only Default profile shadowed a logged-in "Profile N".
|
||||
"""
|
||||
db_paths = _find_all_chromium_cookies_dbs(base_dir)
|
||||
if not db_paths:
|
||||
logger.info("%s cookies database not found under %s", keychain_service, base_dir)
|
||||
return None
|
||||
best: Optional[dict[str, str]] = None
|
||||
key_cache: dict[str, Optional[bytes]] = {}
|
||||
for db_path in db_paths:
|
||||
got = _extract_chromium_cookies_macos(
|
||||
db_path, keychain_service, domain, cookie_names, key_cache=key_cache
|
||||
)
|
||||
if got:
|
||||
if all(name in got for name in cookie_names):
|
||||
logger.debug("Found complete cookie set for %s in %s", domain, db_path)
|
||||
return got
|
||||
if best is None:
|
||||
best = got
|
||||
return best
|
||||
|
||||
|
||||
def _find_brave_cookies_db() -> Optional[Path]:
|
||||
"""Find Brave's Cookies database on macOS (Default, then Profile N)."""
|
||||
return _find_chromium_cookies_db(BRAVE_BASE_DIR)
|
||||
|
||||
|
||||
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
@@ -315,8 +436,25 @@ def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optiona
|
||||
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
||||
path and Keychain service name differ.
|
||||
"""
|
||||
db_path = _find_brave_cookies_db()
|
||||
if db_path is None:
|
||||
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
||||
return _extract_chromium_cookies_any_profile(
|
||||
BRAVE_BASE_DIR, "Brave Safe Storage", domain, cookie_names
|
||||
)
|
||||
|
||||
|
||||
def extract_chromium_browser_cookies_macos(
|
||||
browser: str, domain: str, cookie_names: list[str]
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from a registry-defined Chromium browser on macOS.
|
||||
|
||||
Covers every browser in CHROMIUM_BROWSER_PROFILES (Edge, Vivaldi, Opera,
|
||||
Arc, Chromium). They all reuse Chrome's v10 AES-128-CBC encryption; only
|
||||
the profile directory and Keychain service name differ.
|
||||
"""
|
||||
spec = CHROMIUM_BROWSER_PROFILES.get(browser)
|
||||
if spec is None:
|
||||
logger.debug("Unknown Chromium browser: %s", browser)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
||||
base_dir, keychain_service = spec
|
||||
return _extract_chromium_cookies_any_profile(
|
||||
base_dir, keychain_service, domain, cookie_names
|
||||
)
|
||||
|
||||
@@ -2,59 +2,25 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from . import dedupe, entity_extract, schema
|
||||
|
||||
from . import dedupe, schema
|
||||
|
||||
def _cluster_sort_key(candidate: schema.Candidate) -> tuple:
|
||||
"""Sort key that partitions stale candidates below fresh ones.
|
||||
|
||||
Stale items (all dated source_items outside the window) must never lead
|
||||
cluster representatives or render as the cluster title.
|
||||
"""
|
||||
return (
|
||||
1 if schema.candidate_out_of_window(candidate) else 0,
|
||||
-candidate.final_score,
|
||||
)
|
||||
|
||||
CLUSTERABLE_INTENTS = {"breaking_news", "opinion", "comparison", "prediction"}
|
||||
|
||||
# Words too common to signal shared topic between clusters.
|
||||
_ENTITY_STOPWORDS = frozenset({
|
||||
"the", "a", "an", "to", "for", "how", "is", "in", "of", "on", "and",
|
||||
"with", "from", "by", "at", "this", "that", "it", "what", "are", "do",
|
||||
"can", "his", "her", "he", "she", "its", "was", "has", "new", "just",
|
||||
"says", "said", "will", "about", "after", "now", "all", "been", "here",
|
||||
"not", "out", "up", "more", "also", "but", "who", "year", "first",
|
||||
"make", "being", "making", "over", "into", "than", "they", "their",
|
||||
"would", "could", "get", "got", "some", "like", "back", "going",
|
||||
"breaking", "https", "http", "www", "com",
|
||||
})
|
||||
|
||||
|
||||
def _candidate_text(candidate: schema.Candidate) -> str:
|
||||
return " ".join(part for part in [candidate.title, candidate.snippet] if part).strip()
|
||||
|
||||
|
||||
def _extract_entities(text: str) -> set[str]:
|
||||
"""Extract significant words (proper nouns, numbers, capitalized words) from text.
|
||||
|
||||
Used for cross-source cluster merging where phrasing differs but entities overlap.
|
||||
"""
|
||||
# Normalize but preserve word boundaries
|
||||
words = re.sub(r"[^\w\s]", " ", text).split()
|
||||
entities = set()
|
||||
for word in words:
|
||||
lower = word.lower()
|
||||
if lower in _ENTITY_STOPWORDS or len(word) <= 2:
|
||||
continue
|
||||
# Keep words that are: capitalized, ALL CAPS, contain digits, or 4+ chars
|
||||
if word[0].isupper() or word.isupper() or any(c.isdigit() for c in word) or len(word) >= 4:
|
||||
entities.add(lower)
|
||||
return entities
|
||||
|
||||
|
||||
def _entity_overlap(entities_a: set[str], entities_b: set[str]) -> float:
|
||||
"""Jaccard-style overlap on extracted entities."""
|
||||
if not entities_a or not entities_b:
|
||||
return 0.0
|
||||
intersection = entities_a & entities_b
|
||||
smaller = min(len(entities_a), len(entities_b))
|
||||
# Use overlap coefficient (intersection / min) instead of Jaccard,
|
||||
# because a short tweet about the same event as a long Reddit post
|
||||
# will have fewer total entities but high overlap with the larger set.
|
||||
return len(intersection) / smaller if smaller > 0 else 0.0
|
||||
|
||||
|
||||
def _mmr_representatives(
|
||||
candidates: list[schema.Candidate],
|
||||
text_cache: dict[str, dedupe._PreparedText],
|
||||
@@ -66,7 +32,7 @@ def _mmr_representatives(
|
||||
remaining = list(candidates)
|
||||
while remaining and len(selected) < limit:
|
||||
if not selected:
|
||||
best = max(remaining, key=lambda candidate: candidate.final_score)
|
||||
best = min(remaining, key=_cluster_sort_key)
|
||||
selected.append(best)
|
||||
remaining_set.discard(best.candidate_id)
|
||||
remaining = [c for c in remaining if c.candidate_id in remaining_set]
|
||||
@@ -74,12 +40,16 @@ def _mmr_representatives(
|
||||
|
||||
selected_preps = [text_cache[c.candidate_id] for c in selected]
|
||||
|
||||
def score(candidate: schema.Candidate) -> float:
|
||||
def score(candidate: schema.Candidate) -> tuple:
|
||||
prep = text_cache[candidate.candidate_id]
|
||||
diversity_penalty = max(
|
||||
dedupe.prepared_similarity(prep, sp) for sp in selected_preps
|
||||
)
|
||||
return (diversity_lambda * candidate.final_score) - ((1 - diversity_lambda) * diversity_penalty * 100)
|
||||
base_score = (diversity_lambda * candidate.final_score) - ((1 - diversity_lambda) * diversity_penalty * 100)
|
||||
return (
|
||||
0 if schema.candidate_out_of_window(candidate) else 1,
|
||||
base_score,
|
||||
)
|
||||
|
||||
best = max(remaining, key=score)
|
||||
selected.append(best)
|
||||
@@ -135,7 +105,7 @@ def cluster_candidates(
|
||||
|
||||
clusters: list[schema.Cluster] = []
|
||||
for index, group in enumerate(groups, start=1):
|
||||
group.sort(key=lambda candidate: candidate.final_score, reverse=True)
|
||||
group.sort(key=_cluster_sort_key)
|
||||
cluster_id = f"cluster-{index}"
|
||||
representatives = _mmr_representatives(group, text_cache)
|
||||
for candidate in group:
|
||||
@@ -153,7 +123,11 @@ def cluster_candidates(
|
||||
)
|
||||
|
||||
# Second pass: merge small clusters that share entities across sources.
|
||||
clusters = _merge_entity_clusters(clusters, candidates)
|
||||
clusters = _merge_entity_clusters(
|
||||
clusters,
|
||||
candidates,
|
||||
min_shared_entities=2 if "discover-mode" in plan.notes else 1,
|
||||
)
|
||||
|
||||
return sorted(clusters, key=lambda cluster: cluster.score, reverse=True)
|
||||
|
||||
@@ -161,6 +135,8 @@ def cluster_candidates(
|
||||
def _merge_entity_clusters(
|
||||
clusters: list[schema.Cluster],
|
||||
all_candidates: list[schema.Candidate],
|
||||
*,
|
||||
min_shared_entities: int = 1,
|
||||
) -> list[schema.Cluster]:
|
||||
"""Merge small clusters that cover the same story across different sources.
|
||||
|
||||
@@ -182,7 +158,7 @@ def _merge_entity_clusters(
|
||||
for cid in cl.candidate_ids:
|
||||
cand = candidate_map.get(cid)
|
||||
if cand:
|
||||
entities |= _extract_entities(_candidate_text(cand))
|
||||
entities |= entity_extract.extract_text_entities(_candidate_text(cand))
|
||||
cluster_entities.append(entities)
|
||||
|
||||
# Only merge clusters with <= 3 items (don't merge already-large clusters)
|
||||
@@ -207,8 +183,9 @@ def _merge_entity_clusters(
|
||||
if poly_i != poly_j:
|
||||
continue
|
||||
|
||||
overlap = _entity_overlap(cluster_entities[i], cluster_entities[j])
|
||||
if overlap >= 0.45:
|
||||
shared_entities = cluster_entities[i] & cluster_entities[j]
|
||||
overlap = entity_extract.entity_overlap(cluster_entities[i], cluster_entities[j])
|
||||
if len(shared_entities) >= min_shared_entities and overlap >= 0.45:
|
||||
merged_into[j] = i
|
||||
|
||||
if not merged_into:
|
||||
@@ -236,7 +213,7 @@ def _merge_entity_clusters(
|
||||
|
||||
# Pick representatives from combined pool
|
||||
combined_candidates = [candidate_map[cid] for cid in combined_cids if cid in candidate_map]
|
||||
combined_candidates.sort(key=lambda c: c.final_score, reverse=True)
|
||||
combined_candidates.sort(key=_cluster_sort_key)
|
||||
merge_text_cache = {
|
||||
c.candidate_id: dedupe._PreparedText(_candidate_text(c))
|
||||
for c in combined_candidates
|
||||
|
||||
@@ -13,13 +13,21 @@ the caller's requested count.
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from . import dates, grounding
|
||||
from . import dates, grounding, log
|
||||
from .resolve import _has_backend
|
||||
|
||||
# Peer cap vs total vs-entity cap (main + peers).
|
||||
COMPETITORS_MIN = 1
|
||||
COMPETITORS_MAX = 6
|
||||
COMPETITORS_DEFAULT = 2
|
||||
COMPARISON_ENTITY_MAX = COMPETITORS_MAX + 1
|
||||
# Discovery SERP fan-out is small (3 queries today) but still needs a ceiling
|
||||
# so a future query expansion cannot open one worker per query unbounded.
|
||||
MAX_DISCOVERY_WORKERS = 3
|
||||
|
||||
# A "brand-shaped" token starts with uppercase OR is camelCase with an
|
||||
# uppercase letter later. Catches "Anthropic", "OpenAI", "xAI", "iPhone",
|
||||
# "eBay", "Hugging", "Face".
|
||||
@@ -62,7 +70,7 @@ _STOPWORD_TOKENS: frozenset[str] = frozenset(
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
print(f"[Competitors] {msg}", file=sys.stderr)
|
||||
log.source_log("Competitors", msg, tty_only=False)
|
||||
|
||||
|
||||
def _topic_tokens(topic: str) -> set[str]:
|
||||
@@ -173,7 +181,7 @@ def discover_competitors(
|
||||
items, _artifact = grounding.web_search(query, date_range, config)
|
||||
return label, items
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(queries)) as executor:
|
||||
with ThreadPoolExecutor(max_workers=min(len(queries), MAX_DISCOVERY_WORKERS)) as executor:
|
||||
futures = {
|
||||
executor.submit(_search, label, q): label
|
||||
for label, q in queries.items()
|
||||
|
||||
@@ -11,6 +11,7 @@ Only uses Python stdlib — no external dependencies.
|
||||
import configparser
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import sqlite3
|
||||
@@ -21,6 +22,13 @@ from typing import Dict, List, Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _lock_temp_cookie_copy(path: str) -> None:
|
||||
"""Restrict copied cookie DB temp files to the current user on POSIX."""
|
||||
if os.name == "nt":
|
||||
return
|
||||
Path(path).chmod(0o600)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _is_wsl() -> bool:
|
||||
"""Detect if running under Windows Subsystem for Linux.
|
||||
@@ -61,7 +69,16 @@ def _get_firefox_profiles_dir() -> Optional[Path]:
|
||||
if system == "Darwin":
|
||||
path = Path.home() / "Library" / "Application Support" / "Firefox"
|
||||
elif system == "Linux":
|
||||
# Default location for most distros
|
||||
path = Path.home() / ".mozilla" / "firefox"
|
||||
if path.is_dir():
|
||||
return path
|
||||
# Some distros (e.g. Fedora) honour $XDG_CONFIG_HOME
|
||||
xdg_config = os.environ.get("XDG_CONFIG_HOME")
|
||||
if xdg_config and os.path.isabs(xdg_config):
|
||||
path = Path(xdg_config) / "mozilla" / "firefox"
|
||||
else:
|
||||
path = Path.home() / ".config" / "mozilla" / "firefox"
|
||||
else:
|
||||
# Windows: %APPDATA%\Mozilla\Firefox — best-effort
|
||||
appdata = Path.home() / "AppData" / "Roaming" / "Mozilla" / "Firefox"
|
||||
@@ -149,7 +166,13 @@ def _query_cookies_db(
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
# mkstemp creates the file 0600. copy2 would copy the source's mode
|
||||
# (Firefox cookies.sqlite is commonly 0644, looser on WSL /mnt/c) onto
|
||||
# the temp file, leaving live session secrets world-readable in shared
|
||||
# /tmp until the chmod below runs. copyfile writes content only and
|
||||
# leaves the 0600 perms intact, closing that window.
|
||||
shutil.copyfile(str(db_path), tmp_path)
|
||||
_lock_temp_cookie_copy(tmp_path)
|
||||
|
||||
conn = sqlite3.connect(tmp_path)
|
||||
try:
|
||||
@@ -191,12 +214,36 @@ def _query_cookies_db(
|
||||
|
||||
|
||||
def _try_firefox_dir(profiles_dir: Path, domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Try to extract cookies from a Firefox profiles directory."""
|
||||
profile_path = _find_default_profile(profiles_dir)
|
||||
if profile_path is None:
|
||||
logger.debug("No Firefox profile found in %s", profiles_dir)
|
||||
return None
|
||||
return _query_cookies_db(profile_path / "cookies.sqlite", domain, cookie_names)
|
||||
"""Try to extract cookies from a Firefox profiles directory.
|
||||
|
||||
Tries the default profile first, then falls back to scanning all
|
||||
profiles for matching cookies. This handles multi-profile setups
|
||||
where the user is logged into x.com on a non-default profile.
|
||||
"""
|
||||
default_profile = _find_default_profile(profiles_dir)
|
||||
profiles_tried = 0
|
||||
if default_profile is not None:
|
||||
result = _query_cookies_db(default_profile / "cookies.sqlite", domain, cookie_names)
|
||||
if result is not None:
|
||||
return result
|
||||
profiles_tried = 1
|
||||
# Fallback: scan every profile directory for matching cookies
|
||||
try:
|
||||
for child in sorted(profiles_dir.iterdir()):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
if default_profile is not None and child == default_profile:
|
||||
continue
|
||||
db = child / "cookies.sqlite"
|
||||
if db.is_file():
|
||||
result = _query_cookies_db(db, domain, cookie_names)
|
||||
if result is not None:
|
||||
return result
|
||||
profiles_tried += 1
|
||||
except OSError:
|
||||
pass
|
||||
logger.debug("No matching cookies found in %d Firefox profile(s)", profiles_tried)
|
||||
return None
|
||||
|
||||
|
||||
def extract_firefox_cookies(
|
||||
@@ -280,6 +327,51 @@ def extract_brave_cookies(
|
||||
return None
|
||||
|
||||
|
||||
def _extract_chromium_family_cookies(
|
||||
browser: str, domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from a non-Chrome/Brave Chromium browser on macOS.
|
||||
|
||||
macOS only — Edge, Vivaldi, Opera, Arc, and Chromium all reuse Chrome's
|
||||
v10 AES-128-CBC encryption, with their own profile path and Keychain
|
||||
service name (see chrome_cookies.CHROMIUM_BROWSER_PROFILES).
|
||||
"""
|
||||
if platform.system() != "Darwin":
|
||||
logger.debug("%s cookie extraction only supported on macOS", browser)
|
||||
return None
|
||||
try:
|
||||
from .chrome_cookies import extract_chromium_browser_cookies_macos
|
||||
return extract_chromium_browser_cookies_macos(browser, domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("%s cookie extraction failed: %s", browser, exc)
|
||||
return None
|
||||
|
||||
|
||||
def extract_edge_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Microsoft Edge for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("edge", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_vivaldi_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Vivaldi for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("vivaldi", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_opera_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Opera for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("opera", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_arc_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Arc for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("arc", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_chromium_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from open-source Chromium for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("chromium", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_safari_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
@@ -307,9 +399,10 @@ def extract_cookies(
|
||||
"""Extract cookies from the specified browser.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'edge', 'vivaldi',
|
||||
'opera', 'arc', 'chromium', 'safari', or 'auto'.
|
||||
'auto' tries browsers in platform-appropriate order:
|
||||
- macOS: Chrome -> Brave -> Firefox -> Safari
|
||||
- macOS: Chrome -> Brave -> Edge -> Vivaldi -> Opera -> Arc -> Chromium -> Firefox -> Safari
|
||||
- Linux: Firefox only
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
@@ -358,7 +451,8 @@ def extract_cookies_with_source(
|
||||
so callers can track the source.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'edge', 'vivaldi',
|
||||
'opera', 'arc', 'chromium', 'safari', or 'auto'.
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
|
||||
@@ -370,6 +464,11 @@ def extract_cookies_with_source(
|
||||
"firefox": extract_firefox_cookies,
|
||||
"chrome": extract_chrome_cookies,
|
||||
"brave": extract_brave_cookies,
|
||||
"edge": extract_edge_cookies,
|
||||
"vivaldi": extract_vivaldi_cookies,
|
||||
"opera": extract_opera_cookies,
|
||||
"arc": extract_arc_cookies,
|
||||
"chromium": extract_chromium_cookies,
|
||||
"safari": extract_safari_cookies,
|
||||
}
|
||||
|
||||
@@ -383,10 +482,14 @@ def extract_cookies_with_source(
|
||||
result = extractor(domain, cookie_names)
|
||||
return (result, browser) if result is not None else None
|
||||
|
||||
# Auto mode: try browsers in platform-appropriate order
|
||||
# Auto mode: try browsers in platform-appropriate order.
|
||||
# Note: the skill's own entry point (env.extract_browser_credentials) builds
|
||||
# its own list that tries the SILENT browsers (Firefox, Safari) first to
|
||||
# avoid macOS Keychain prompts. This standalone "auto" is Chromium-first; the
|
||||
# two orderings are intentional for their respective callers.
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
order = ["chrome", "brave", "firefox", "safari"]
|
||||
order = ["chrome", "brave", "edge", "vivaldi", "opera", "arc", "chromium", "firefox", "safari"]
|
||||
elif system == "Linux":
|
||||
order = ["firefox"]
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
"""Deterministic, local-only document corpus source.
|
||||
|
||||
The corpus adapter deliberately has no HTTP dependency. It scans explicitly
|
||||
registered directories, extracts small text documents (and PDFs only when the
|
||||
local ``pdftotext`` binary is available), and returns normalized ``SourceItem``
|
||||
objects for the shared relevance/fusion pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from shutil import which
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import entity_extract, log, relevance, schema
|
||||
|
||||
SOURCE = "corpus"
|
||||
SUPPORTED_SUFFIXES = {".md", ".txt", ".pdf"}
|
||||
IGNORED_DIRECTORIES = {".git", "node_modules"}
|
||||
MAX_FILES = 500
|
||||
MAX_TEXT_CHARS = 1_000_000
|
||||
MAX_CACHE_TEXT_CHARS = MAX_TEXT_CHARS
|
||||
MAX_CACHE_BYTES = 50 * 1024 * 1024
|
||||
MAX_CACHE_ENTRIES = 2_000
|
||||
CACHE_FILENAME = "corpus-cache.json"
|
||||
CACHE_SCHEMA_VERSION = "last30days-corpus-cache/v2"
|
||||
|
||||
_CACHE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CorpusScanResult:
|
||||
"""One bounded scan, including non-fatal extraction notes."""
|
||||
|
||||
items: list[schema.SourceItem]
|
||||
notes: list[str] = field(default_factory=list)
|
||||
files_scanned: int = 0
|
||||
cache_hits: int = 0
|
||||
|
||||
|
||||
def resolve_directories(
|
||||
cli_directories: Iterable[str] | None,
|
||||
configured: str | Iterable[str] | None,
|
||||
) -> list[Path]:
|
||||
"""Merge repeatable CLI paths with ``os.pathsep``-separated config paths."""
|
||||
raw: list[str] = [str(value) for value in (cli_directories or []) if str(value).strip()]
|
||||
if isinstance(configured, str):
|
||||
raw.extend(value for value in configured.split(os.pathsep) if value.strip())
|
||||
elif configured:
|
||||
raw.extend(str(value) for value in configured if str(value).strip())
|
||||
|
||||
resolved: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
for value in raw:
|
||||
path = Path(value.strip()).expanduser().resolve()
|
||||
key = os.path.normcase(str(path))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
resolved.append(path)
|
||||
return resolved
|
||||
|
||||
|
||||
def _safe_error(exc: BaseException) -> str:
|
||||
"""Describe an error without str(exc), which embeds absolute paths.
|
||||
|
||||
These notes travel into source_status detail and render in coverage
|
||||
diagnostics outside the private corpus block.
|
||||
"""
|
||||
reason = getattr(exc, "strerror", None)
|
||||
return str(reason) if reason else exc.__class__.__name__
|
||||
|
||||
|
||||
def search(
|
||||
topic: str,
|
||||
directories: Iterable[Path | str],
|
||||
*,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
all_time: bool = False,
|
||||
limit: int = 12,
|
||||
cache_dir: Path | None = None,
|
||||
) -> CorpusScanResult:
|
||||
"""Search registered directories without making any network calls."""
|
||||
roots = resolve_directories([str(path) for path in directories], None)
|
||||
notes: list[str] = []
|
||||
cache_path = cache_dir / CACHE_FILENAME if cache_dir is not None else None
|
||||
with _CACHE_LOCK:
|
||||
cache = _load_cache(cache_path)
|
||||
cache_entries = cache.setdefault("entries", {})
|
||||
cache_entry_sizes = {
|
||||
path: _cache_entry_fragment_size(path, value)
|
||||
for path, value in cache_entries.items()
|
||||
}
|
||||
|
||||
candidates: list[tuple[float, int, schema.SourceItem]] = []
|
||||
seen_files: set[str] = set()
|
||||
files_scanned = 0
|
||||
cache_hits = 0
|
||||
pdf_available = which("pdftotext")
|
||||
pdf_unavailable_noted = False
|
||||
|
||||
readable_roots: list[Path] = []
|
||||
for root in roots:
|
||||
if not root.is_dir():
|
||||
notes.append(f"Skipped corpus root '{Path(root).name}': not a readable directory")
|
||||
continue
|
||||
readable_roots.append(root)
|
||||
|
||||
per_root_limit, extra_slots = divmod(MAX_FILES, len(readable_roots) or 1)
|
||||
scan_limit_reached = False
|
||||
for root_index, root in enumerate(readable_roots):
|
||||
root_limit = per_root_limit + (1 if root_index < extra_slots else 0)
|
||||
root_files_scanned = 0
|
||||
for path in _iter_files(root, notes=notes):
|
||||
if root_files_scanned >= root_limit:
|
||||
scan_limit_reached = True
|
||||
break
|
||||
key = os.path.normcase(str(path))
|
||||
if key in seen_files:
|
||||
continue
|
||||
seen_files.add(key)
|
||||
root_files_scanned += 1
|
||||
files_scanned += 1
|
||||
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError as exc:
|
||||
notes.append(f"Skipped {_display_path(path, root)}: {_safe_error(exc)}")
|
||||
continue
|
||||
published_at = datetime.fromtimestamp(
|
||||
stat.st_mtime, tz=timezone.utc
|
||||
).date().isoformat()
|
||||
if not all_time and not (from_date <= published_at <= to_date):
|
||||
continue
|
||||
|
||||
cached = cache_entries.get(str(path))
|
||||
if (
|
||||
isinstance(cached, dict)
|
||||
and cached.get("mtime_ns") == stat.st_mtime_ns
|
||||
and cached.get("size") == stat.st_size
|
||||
and isinstance(cached.get("text"), str)
|
||||
):
|
||||
text = cached["text"]
|
||||
cache_hits += 1
|
||||
else:
|
||||
if path.suffix.lower() == ".pdf" and not pdf_available:
|
||||
if not pdf_unavailable_noted:
|
||||
notes.append("Skipped PDF files because pdftotext is not on PATH")
|
||||
pdf_unavailable_noted = True
|
||||
continue
|
||||
try:
|
||||
text = _extract_text(path, pdftotext=pdf_available)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
notes.append(f"Skipped {_display_path(path, root)}: {_safe_error(exc)}")
|
||||
continue
|
||||
_cache_entry_put(cache_entries, cache_entry_sizes, str(path), {
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"size": stat.st_size,
|
||||
"text": text[:MAX_CACHE_TEXT_CHARS],
|
||||
})
|
||||
|
||||
title = _path_title(path)
|
||||
score = _match_score(topic, f"{title}\n{text}")
|
||||
if score < 0.15:
|
||||
continue
|
||||
relative_path = str(path.relative_to(root))
|
||||
path_digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()
|
||||
item = schema.SourceItem(
|
||||
item_id=f"C{path_digest[:12]}",
|
||||
source=SOURCE,
|
||||
title=title,
|
||||
body=text,
|
||||
url=f"corpus://{path_digest}",
|
||||
container=str(path.parent),
|
||||
published_at=published_at,
|
||||
date_confidence="high",
|
||||
relevance_hint=score,
|
||||
why_relevant=f"Matched local file {relative_path}",
|
||||
# Leave empty so extract_best_snippet derives the matching
|
||||
# window; a file-prefix snippet is preserved verbatim and can
|
||||
# show unrelated intro text (and draw entity-miss demotion).
|
||||
snippet="",
|
||||
metadata={
|
||||
"path": str(path),
|
||||
"relative_path": relative_path,
|
||||
"extension": path.suffix.lower(),
|
||||
"local_only": True,
|
||||
},
|
||||
)
|
||||
candidates.append((score, stat.st_mtime_ns, item))
|
||||
if scan_limit_reached:
|
||||
notes.append(f"Stopped after the {MAX_FILES}-file corpus scan limit")
|
||||
|
||||
cache["schema_version"] = CACHE_SCHEMA_VERSION
|
||||
cache["entries"] = _bounded_entries(cache_entries)
|
||||
with _CACHE_LOCK:
|
||||
_write_cache(cache_path, cache, notes)
|
||||
|
||||
candidates.sort(key=lambda row: (-row[0], -row[1], row[2].title.casefold()))
|
||||
items = [item for _score, _mtime, item in candidates[: max(0, limit)]]
|
||||
log.source_log(
|
||||
"Corpus",
|
||||
f"scanned {files_scanned} file(s), {cache_hits} cache hit(s), {len(items)} match(es)",
|
||||
tty_only=False,
|
||||
)
|
||||
return CorpusScanResult(
|
||||
items=items,
|
||||
notes=notes,
|
||||
files_scanned=files_scanned,
|
||||
cache_hits=cache_hits,
|
||||
)
|
||||
|
||||
|
||||
def _display_path(path: Path | str, root: Path | None = None) -> str:
|
||||
"""Render a note-safe path: never the absolute local path.
|
||||
|
||||
Corpus notes flow into source_status detail and the Partial Coverage
|
||||
block, which render OUTSIDE the private corpus markers - an absolute
|
||||
path like /home/user/private/notes/foo.md must not escape there.
|
||||
"""
|
||||
candidate = Path(path)
|
||||
if root is not None:
|
||||
try:
|
||||
return str(Path(root).name / candidate.relative_to(root))
|
||||
except ValueError:
|
||||
pass
|
||||
return candidate.name
|
||||
|
||||
|
||||
def _iter_files(root: Path, notes: list[str] | None = None) -> Iterable[Path]:
|
||||
# Bounded newest-first selection: keep only the newest MAX_FILES paths in a
|
||||
# heap while walking, so registering a huge tree does not materialize every
|
||||
# path before the caller's extraction cap applies.
|
||||
import heapq
|
||||
|
||||
heap: list[tuple[int, str]] = []
|
||||
walk_errors = 0
|
||||
|
||||
def _on_walk_error(error: OSError) -> None:
|
||||
nonlocal walk_errors
|
||||
walk_errors += 1
|
||||
if notes is not None and walk_errors <= 3:
|
||||
unreadable = _display_path(error.filename, root) if error.filename else Path(root).name
|
||||
notes.append(f"corpus: could not read {unreadable}: {error.strerror}")
|
||||
|
||||
for current, directory_names, file_names in os.walk(
|
||||
root, followlinks=False, onerror=_on_walk_error
|
||||
):
|
||||
directory_names[:] = sorted(
|
||||
name
|
||||
for name in directory_names
|
||||
if name not in IGNORED_DIRECTORIES and not name.startswith(".")
|
||||
)
|
||||
current_path = Path(current)
|
||||
for name in sorted(file_names):
|
||||
if name.startswith("."):
|
||||
continue
|
||||
path = current_path / name
|
||||
if path.suffix.lower() in SUPPORTED_SUFFIXES and not path.is_symlink():
|
||||
entry = (_safe_mtime_ns(path), str(path))
|
||||
if len(heap) < MAX_FILES:
|
||||
heapq.heappush(heap, entry)
|
||||
else:
|
||||
heapq.heappushpop(heap, entry)
|
||||
if notes is not None and walk_errors > 3:
|
||||
notes.append(f"corpus: {walk_errors - 3} more unreadable directories suppressed")
|
||||
ordered = sorted(heap, key=lambda item: (-item[0], item[1].casefold()))
|
||||
for _mtime, raw_path in ordered:
|
||||
yield Path(raw_path)
|
||||
|
||||
|
||||
def _safe_mtime_ns(path: Path) -> int:
|
||||
try:
|
||||
return path.stat().st_mtime_ns
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_text(path: Path, *, pdftotext: str | None) -> str:
|
||||
if path.suffix.lower() == ".pdf":
|
||||
if not pdftotext:
|
||||
return ""
|
||||
completed = subprocess.run(
|
||||
[pdftotext, str(path), "-"],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
)
|
||||
return completed.stdout[:MAX_TEXT_CHARS]
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return handle.read(MAX_TEXT_CHARS)
|
||||
|
||||
|
||||
def _path_title(path: Path) -> str:
|
||||
title = path.stem.replace("_", " ").replace("-", " ")
|
||||
return " ".join(title.split()) or path.name
|
||||
|
||||
|
||||
def _match_score(topic: str, text: str) -> float:
|
||||
lexical = relevance.token_overlap_relevance(topic, text)
|
||||
topic_entities = entity_extract.extract_text_entities(topic)
|
||||
text_entities = entity_extract.extract_text_entities(text)
|
||||
entity_score = entity_extract.entity_overlap(topic_entities, text_entities)
|
||||
return round(max(lexical, entity_score * 0.9), 4)
|
||||
|
||||
|
||||
def _load_cache(path: Path | None) -> dict[str, Any]:
|
||||
if path is None:
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
try:
|
||||
if path.stat().st_size > MAX_CACHE_BYTES:
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
if not isinstance(payload, dict) or payload.get("schema_version") != CACHE_SCHEMA_VERSION:
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
if not isinstance(payload.get("entries"), dict):
|
||||
payload["entries"] = {}
|
||||
payload["entries"] = _bounded_entries(payload["entries"])
|
||||
return payload
|
||||
|
||||
|
||||
def _bounded_entries(entries: Any) -> dict[str, Any]:
|
||||
if not isinstance(entries, dict):
|
||||
return {}
|
||||
ordered = sorted(
|
||||
(
|
||||
(path, value)
|
||||
for path, value in entries.items()
|
||||
if (
|
||||
isinstance(path, str)
|
||||
and isinstance(value, dict)
|
||||
and isinstance(value.get("text"), str)
|
||||
)
|
||||
),
|
||||
key=lambda row: int(row[1].get("mtime_ns") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
base_bytes = len(
|
||||
json.dumps(
|
||||
{"schema_version": CACHE_SCHEMA_VERSION, "entries": {}},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
)
|
||||
used_bytes = base_bytes
|
||||
bounded: dict[str, Any] = {}
|
||||
for path, value in ordered[:MAX_CACHE_ENTRIES]:
|
||||
normalized = {
|
||||
"mtime_ns": value.get("mtime_ns"),
|
||||
"size": value.get("size"),
|
||||
"text": value["text"][:MAX_CACHE_TEXT_CHARS],
|
||||
}
|
||||
fragment = json.dumps({path: normalized}, ensure_ascii=False).encode("utf-8")
|
||||
fragment_bytes = len(fragment) - 2 + (2 if bounded else 0)
|
||||
if used_bytes + fragment_bytes > MAX_CACHE_BYTES:
|
||||
continue
|
||||
bounded[path] = normalized
|
||||
used_bytes += fragment_bytes
|
||||
return bounded
|
||||
|
||||
|
||||
def _cache_entry_fragment_size(path: str, value: dict[str, Any]) -> int:
|
||||
return len(json.dumps({path: value}, ensure_ascii=False).encode("utf-8")) - 2
|
||||
|
||||
|
||||
def _cache_entry_put(
|
||||
entries: dict[str, Any],
|
||||
sizes: dict[str, int],
|
||||
path: str,
|
||||
value: dict[str, Any],
|
||||
) -> None:
|
||||
entries[path] = value
|
||||
sizes[path] = _cache_entry_fragment_size(path, value)
|
||||
while (
|
||||
len(entries) > MAX_CACHE_ENTRIES
|
||||
or _cache_payload_size(sizes) > MAX_CACHE_BYTES
|
||||
):
|
||||
oldest = min(
|
||||
entries,
|
||||
key=lambda candidate: (
|
||||
int(entries[candidate].get("mtime_ns") or 0),
|
||||
candidate,
|
||||
),
|
||||
)
|
||||
del entries[oldest]
|
||||
del sizes[oldest]
|
||||
|
||||
|
||||
def _cache_payload_size(sizes: dict[str, int]) -> int:
|
||||
base_bytes = len(
|
||||
json.dumps(
|
||||
{"schema_version": CACHE_SCHEMA_VERSION, "entries": {}},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
)
|
||||
separators = max(0, len(sizes) - 1) * 2
|
||||
return base_bytes + sum(sizes.values()) + separators
|
||||
|
||||
|
||||
def _write_cache(path: Path | None, payload: dict[str, Any], notes: list[str]) -> None:
|
||||
if path is None:
|
||||
return
|
||||
try:
|
||||
_ensure_private_directory(path.parent)
|
||||
payload["entries"] = _bounded_entries(payload.get("entries", {}))
|
||||
encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
try:
|
||||
fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
temporary.unlink()
|
||||
fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(encoded)
|
||||
temporary.replace(path)
|
||||
path.chmod(0o600)
|
||||
except OSError as exc:
|
||||
notes.append(f"Corpus cache unavailable: {_safe_error(exc)}")
|
||||
|
||||
|
||||
def _ensure_private_directory(path: Path) -> None:
|
||||
missing: list[Path] = []
|
||||
current = path
|
||||
while not current.exists():
|
||||
missing.append(current)
|
||||
current = current.parent
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
for directory in missing:
|
||||
directory.chmod(0o700)
|
||||
@@ -4,15 +4,55 @@ from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
def get_date_range(days: int = 30) -> Tuple[str, str]:
|
||||
"""Get the date range for the last N days.
|
||||
def parse_as_of_date(as_of_date: Optional[str]) -> Optional[str]:
|
||||
"""Validate and normalize an --as-of date.
|
||||
|
||||
Args:
|
||||
as_of_date: Date string in YYYY-MM-DD format.
|
||||
|
||||
Returns:
|
||||
Tuple of (from_date, to_date) as YYYY-MM-DD strings
|
||||
Normalized YYYY-MM-DD string, or None when no date was provided.
|
||||
|
||||
Raises:
|
||||
ValueError: If the date is not in YYYY-MM-DD format.
|
||||
"""
|
||||
today = datetime.now(timezone.utc).date()
|
||||
from_date = today - timedelta(days=days)
|
||||
return from_date.isoformat(), today.isoformat()
|
||||
if as_of_date is None:
|
||||
return None
|
||||
|
||||
if not as_of_date.strip():
|
||||
raise ValueError("--as-of must be in YYYY-MM-DD format.")
|
||||
|
||||
try:
|
||||
parsed = datetime.strptime(as_of_date, "%Y-%m-%d").date()
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"Invalid --as-of date: {as_of_date}. Expected YYYY-MM-DD."
|
||||
) from exc
|
||||
|
||||
return parsed.isoformat()
|
||||
|
||||
|
||||
def get_date_range(days: int = 30, as_of_date: Optional[str] = None) -> Tuple[str, str]:
|
||||
"""Get the date range for the last N days.
|
||||
|
||||
When as_of_date is provided, the range ends at that date instead of today.
|
||||
|
||||
Args:
|
||||
days: Number of days to look back.
|
||||
as_of_date: Optional end date in YYYY-MM-DD format.
|
||||
|
||||
Returns:
|
||||
Tuple of (from_date, to_date) as YYYY-MM-DD strings.
|
||||
"""
|
||||
normalized_as_of = parse_as_of_date(as_of_date)
|
||||
|
||||
if normalized_as_of:
|
||||
to_date = datetime.strptime(normalized_as_of, "%Y-%m-%d").date()
|
||||
else:
|
||||
to_date = datetime.now(timezone.utc).date()
|
||||
|
||||
from_date = to_date - timedelta(days=days)
|
||||
return from_date.isoformat(), to_date.isoformat()
|
||||
|
||||
|
||||
def parse_date(date_str: Optional[str]) -> Optional[datetime]:
|
||||
@@ -86,9 +126,10 @@ def get_date_confidence(date_str: Optional[str], from_date: str, to_date: str) -
|
||||
return 'low'
|
||||
|
||||
|
||||
def days_ago(date_str: Optional[str]) -> Optional[int]:
|
||||
"""Calculate how many days ago a date is.
|
||||
def days_ago(date_str: Optional[str], reference_date: Optional[str] = None) -> Optional[int]:
|
||||
"""Calculate how many days before the reference date a date is.
|
||||
|
||||
If reference_date is None, use real today for backward compatibility.
|
||||
Returns None if date is invalid or missing.
|
||||
"""
|
||||
if not date_str:
|
||||
@@ -96,24 +137,32 @@ def days_ago(date_str: Optional[str]) -> Optional[int]:
|
||||
|
||||
try:
|
||||
dt = datetime.strptime(date_str, "%Y-%m-%d").date()
|
||||
today = datetime.now(timezone.utc).date()
|
||||
if reference_date:
|
||||
today = datetime.strptime(reference_date, "%Y-%m-%d").date()
|
||||
else:
|
||||
today = datetime.now(timezone.utc).date()
|
||||
delta = today - dt
|
||||
return delta.days
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def recency_score(date_str: Optional[str], max_days: int = 30) -> int:
|
||||
def recency_score(
|
||||
date_str: Optional[str],
|
||||
max_days: int = 30,
|
||||
reference_date: Optional[str] = None,
|
||||
) -> int:
|
||||
"""Calculate recency score (0-100).
|
||||
|
||||
0 days ago = 100, max_days ago = 0, clamped.
|
||||
0 days before reference_date = 100, max_days before reference_date = 0.
|
||||
If reference_date is None, use real today for backward compatibility.
|
||||
"""
|
||||
age = days_ago(date_str)
|
||||
age = days_ago(date_str, reference_date=reference_date)
|
||||
if age is None:
|
||||
return 0 # Unknown date gets worst score
|
||||
return 0
|
||||
|
||||
if age < 0:
|
||||
return 100 # Future date (treat as today)
|
||||
return 100
|
||||
if age >= max_days:
|
||||
return 0
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ import json
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import log, subproc
|
||||
from .relevance import token_overlap_relevance
|
||||
@@ -53,7 +54,7 @@ POSTS_TIMEOUT = 15
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("Digg", msg)
|
||||
log.source_log("Digg", msg, tty_only=False)
|
||||
|
||||
|
||||
def _is_available() -> bool:
|
||||
@@ -296,6 +297,20 @@ def parse_digg_response(
|
||||
return items
|
||||
|
||||
|
||||
def _is_safe_http_url(url: str) -> bool:
|
||||
"""True iff ``url`` parses with an http or https scheme.
|
||||
|
||||
Used to reject upstream-supplied post URLs whose scheme would be
|
||||
dangerous in a rendered ``<a href>`` (``javascript:``, ``data:``,
|
||||
``file:``, ``vbscript:``, ``about:``).
|
||||
"""
|
||||
try:
|
||||
scheme = urlparse(url).scheme.lower()
|
||||
except ValueError:
|
||||
return False
|
||||
return scheme in ("http", "https")
|
||||
|
||||
|
||||
def _parse_post(raw_post: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Reduce a digg post payload into the small dict render uses.
|
||||
|
||||
@@ -316,6 +331,17 @@ def _parse_post(raw_post: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
x_url = str(raw_post.get("xUrl") or "").strip()
|
||||
if not x_url:
|
||||
return None
|
||||
if not _is_safe_http_url(x_url):
|
||||
# Security-class drop: an upstream-supplied URL with a dangerous
|
||||
# scheme. Force tty_only=False so the rejection is visible in
|
||||
# non-interactive runs (Claude Code), which is the actual attack
|
||||
# surface — the default tty_only=True would suppress it there.
|
||||
log.source_log(
|
||||
"Digg",
|
||||
f"dropped post with unsafe xUrl scheme: {x_url!r}",
|
||||
tty_only=False,
|
||||
)
|
||||
return None
|
||||
return {
|
||||
"username": username,
|
||||
"display_name": str(author.get("display_name") or "").strip() or username,
|
||||
|
||||
@@ -0,0 +1,976 @@
|
||||
"""File contracts for the three-command host-judged discovery protocol.
|
||||
|
||||
Leg 1 (``--discover --nominate-only``) writes the nominations bundle: the
|
||||
FULL judge pool, each nomination with its complete seed item set, serialized
|
||||
losslessly so leg 2 can recompute floor/velocity/entity-token disambiguation
|
||||
exactly as an in-memory run would. Leg 2 (``--discover --judgments <file>``)
|
||||
reads host judgments (names/junk/worthiness) bound to the bundle by
|
||||
bundle_id. Leg 3 (``--discover --finalize [--angles <file>]``) applies
|
||||
host-written content angles.
|
||||
|
||||
This module owns the handoff contracts - bundle writer/reader, judgments
|
||||
reader, pending-report reader (the leg-2 output leg 3 finalizes from),
|
||||
angles reader - plus the host-facing digest and the post-judgment
|
||||
name-collision resolver. Readers are strict at the top level (typed
|
||||
``HandoffContractError``, mapped to exit 2 by the CLI layer) and lenient per
|
||||
row: a malformed or omitted row falls back to the bundle's heuristics rather
|
||||
than failing the run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterator, Sequence
|
||||
|
||||
from . import env, log, pipeline, rerank, schema
|
||||
|
||||
|
||||
# How long a nominations bundle stays valid. Deliberately a module constant
|
||||
# and NOT the LAST30DAYS_REPORT_CACHE_TTL_SECONDS env knob: a user who
|
||||
# lowered the report-cache TTL for drill freshness must not shrink the
|
||||
# window a host has to author judgments.
|
||||
DISCOVERY_HANDOFF_TTL_SECONDS = 3600.0
|
||||
|
||||
NOMINATIONS_BUNDLE_FILENAME = "discover-nominations.json"
|
||||
PENDING_REPORT_FILENAME = "discover-pending.json"
|
||||
|
||||
_VALID_TIERS = ("deep", "shallow")
|
||||
|
||||
_RESWEEP_REMEDY = "Run a fresh `--discover --nominate-only` re-sweep."
|
||||
|
||||
# Leg-3 remedy: the pending report is leg-2 output, so the first fix is to
|
||||
# re-run the resume leg; only when the bundle itself has also gone stale does
|
||||
# the whole protocol restart.
|
||||
_RESUME_REMEDY = (
|
||||
"Re-run the resume leg (`--discover --judgments <file>`), or the full "
|
||||
"protocol from `--discover --nominate-only` if the bundle is stale too."
|
||||
)
|
||||
|
||||
# Defensive caps on host-supplied text, ported from the retired engine-judge
|
||||
# pass: names become search queries and the /last30days handoff, angles
|
||||
# render verbatim on trend cards, so a runaway (or adversarial) value never
|
||||
# yields an unbounded string.
|
||||
_NAME_MAX_CHARS = 96
|
||||
_ANGLE_MAX_CHARS = 200
|
||||
|
||||
# Unified trailing-punctuation charset for word-boundary truncation: names
|
||||
# and angle sentences share it so the strip sets cannot drift.
|
||||
_TRUNCATE_STRIP_CHARS = " \"'`.,;:!?-"
|
||||
|
||||
# Digest evidence caps: the surface the engine judge used to see per
|
||||
# nomination (leader title, leader snippet, strongest community comment).
|
||||
_DIGEST_TITLE_MAX_CHARS = 220
|
||||
_DIGEST_SNIPPET_MAX_CHARS = 420
|
||||
_DIGEST_COMMENT_MAX_CHARS = 340
|
||||
|
||||
|
||||
class HandoffContractError(Exception):
|
||||
"""A handoff file failed its contract: unreadable, invalid JSON, wrong
|
||||
shape or schema version, stale, or not bound to the current bundle.
|
||||
The CLI layer maps this to exit code 2."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PoolEntry:
|
||||
"""One judge-pool nomination as handed to the bundle writer (leg 1).
|
||||
|
||||
``heuristic_name`` and ``heuristic_junk`` are the deterministic
|
||||
topic_shape fallbacks, kept alongside the nomination so leg 2 can fill
|
||||
any row the host omitted without re-deriving them.
|
||||
"""
|
||||
|
||||
nomination: pipeline.Nomination
|
||||
cluster_id: str
|
||||
heuristic_name: str
|
||||
heuristic_junk: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BundleNomination:
|
||||
"""One nomination read back from a bundle, with its stable id."""
|
||||
|
||||
nomination_id: str
|
||||
nomination: pipeline.Nomination
|
||||
cluster_id: str
|
||||
heuristic_name: str
|
||||
heuristic_junk: bool
|
||||
sources: list[str]
|
||||
engagement_by_source: dict[str, dict[str, float | int]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NominationsBundle:
|
||||
"""A parsed leg-1 nominations bundle (also returned by the writer).
|
||||
|
||||
``source_status`` is the leg-1 sweep's finalized per-source outcome map:
|
||||
legs 2 and 3 restore it so degraded sweep coverage survives the protocol
|
||||
instead of silently reading as clean. ``mock`` is the writing run's
|
||||
provenance - mock-born state must never be finalized by a real run (and
|
||||
vice versa); files written before either field existed read as an empty
|
||||
map and a real run."""
|
||||
|
||||
schema_version: str
|
||||
bundle_id: str
|
||||
generated_at: str
|
||||
from_date: str
|
||||
to_date: str
|
||||
domain: str
|
||||
tier: str
|
||||
enrichment_source_boundary: list[str] | None
|
||||
requested_sources: list[str] | None
|
||||
lookback_days: int
|
||||
nominations: list[BundleNomination]
|
||||
source_status: dict[str, schema.SourceOutcome] = field(default_factory=dict)
|
||||
mock: bool = False
|
||||
path: Path | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostJudgment:
|
||||
"""One host verdict row. ``None`` on any field means the host left it
|
||||
absent for that row and the caller falls back to the bundle's heuristic
|
||||
value (name/junk) or to no worthiness signal."""
|
||||
|
||||
name: str | None
|
||||
junk: bool | None
|
||||
worthiness: int | None
|
||||
|
||||
|
||||
# The per-row-absent marker: what ``judgment_for`` returns for a nomination
|
||||
# the host omitted entirely. Every field falls back to the bundle heuristics.
|
||||
ROW_ABSENT = HostJudgment(name=None, junk=None, worthiness=None)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostAngles:
|
||||
"""One host-written angle row; either field may be absent."""
|
||||
|
||||
podcast: str | None
|
||||
x_article: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PendingReport:
|
||||
"""A parsed leg-2 pending report: the floored/folded/ranked discovery
|
||||
report (as its raw ``schema.to_dict`` payload - leg 3 rebuilds it via
|
||||
``schema.discovery_report_from_dict``) plus the angle inputs keyed by
|
||||
surviving nomination id. ``run_ref`` is the leg-2 run identity the
|
||||
finalize leg replays into the topic queue so retries stay idempotent."""
|
||||
|
||||
schema_version: str
|
||||
bundle_id: str
|
||||
generated_at: str
|
||||
run_ref: str
|
||||
report: dict[str, Any]
|
||||
angle_inputs: dict[str, dict[str, str]]
|
||||
# Leg-2 provenance: True when a --mock resume wrote this file. Files
|
||||
# written before the flag existed read as real (False).
|
||||
mock: bool = False
|
||||
path: Path | None = None
|
||||
|
||||
|
||||
def _warn(message: str) -> None:
|
||||
log.source_log("Discover", message, tty_only=False)
|
||||
|
||||
|
||||
def handoff_state_dir(
|
||||
save_dir: str | Path | None,
|
||||
config_dir: Path | None,
|
||||
) -> Path | None:
|
||||
"""Resolve the handoff state directory: ``save_dir`` when provided, else
|
||||
the config dir (mirrors the report-cache convention in last30days.py).
|
||||
Both are accepted as arguments so this module never imports the CLI
|
||||
layer above it. Returns None when neither location is available."""
|
||||
if save_dir:
|
||||
return Path(save_dir).expanduser().resolve()
|
||||
if config_dir is not None:
|
||||
return Path(config_dir)
|
||||
return None
|
||||
|
||||
|
||||
def nominations_bundle_path(state_dir: str | Path) -> Path:
|
||||
"""The nominations bundle file inside a handoff state directory."""
|
||||
return Path(state_dir) / NOMINATIONS_BUNDLE_FILENAME
|
||||
|
||||
|
||||
def pending_report_path(state_dir: str | Path) -> Path:
|
||||
"""The leg-2 pending-report file inside a handoff state directory."""
|
||||
return Path(state_dir) / PENDING_REPORT_FILENAME
|
||||
|
||||
|
||||
def _search_paths(
|
||||
save_dir: str | Path | None,
|
||||
config_dir: Path | None,
|
||||
path_fn: Callable[[Path], Path],
|
||||
) -> list[Path]:
|
||||
"""Candidate handoff-file locations: ONLY the save dir when one was
|
||||
supplied, else the config dir. An explicit save dir is the protocol's
|
||||
single handoff store (mirroring ``_scoped_store_db`` and SKILL.md's "a
|
||||
different or missing save dir on a later leg means the leg cannot find
|
||||
them" contract), so a handoff file in the config dir must never silently
|
||||
satisfy a save-dir run. ``path_fn`` picks which handoff file (bundle vs
|
||||
pending)."""
|
||||
if save_dir:
|
||||
return [path_fn(Path(save_dir).expanduser().resolve())]
|
||||
if config_dir is not None:
|
||||
return [path_fn(Path(config_dir))]
|
||||
return []
|
||||
|
||||
|
||||
def _searched_lines(searched: list[Path]) -> str:
|
||||
if not searched:
|
||||
return " (no --save-dir and no config directory available)"
|
||||
return "\n".join(f" - {path}" for path in searched)
|
||||
|
||||
|
||||
def write_nominations_bundle(
|
||||
entries: Sequence[PoolEntry],
|
||||
*,
|
||||
domain: str,
|
||||
tier: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
lookback_days: int,
|
||||
enrichment_source_boundary: list[str] | None,
|
||||
requested_sources: list[str] | None,
|
||||
source_status: dict[str, schema.SourceOutcome] | None = None,
|
||||
mock: bool = False,
|
||||
save_dir: str | Path | None = None,
|
||||
config_dir: Path | None = None,
|
||||
) -> NominationsBundle:
|
||||
"""Write the leg-1 nominations bundle and return its parsed form.
|
||||
|
||||
Nomination ids are assigned ``n1, n2, ...`` in pool order. The leg-1
|
||||
invocation context (enrichment source boundary, requested discovery
|
||||
sources, lookback days) rides along so leg 2 resumes with identical
|
||||
settings. ``None`` boundaries are preserved as null - "no boundary" and
|
||||
"empty boundary" are different contracts. ``source_status`` is the
|
||||
sweep's finalized per-source outcome map (serialized via the same
|
||||
``schema.to_dict`` round trip every report uses) so degraded coverage
|
||||
survives into legs 2-3; ``mock`` stamps the writing run's provenance.
|
||||
"""
|
||||
if tier not in _VALID_TIERS:
|
||||
raise ValueError(f"tier must be one of {_VALID_TIERS}, got {tier!r}")
|
||||
state_dir = handoff_state_dir(save_dir, config_dir)
|
||||
if state_dir is None:
|
||||
raise HandoffContractError(
|
||||
"No handoff location available to write the nominations bundle: "
|
||||
"pass --save-dir or configure ~/.config/last30days/."
|
||||
)
|
||||
bundle_id = secrets.token_hex(8)
|
||||
generated_at = schema._utc_now()
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
nominations: list[BundleNomination] = []
|
||||
for index, entry in enumerate(entries, start=1):
|
||||
nomination_id = f"n{index}"
|
||||
sources = sorted({item.source for item in entry.nomination.items})
|
||||
engagement = pipeline._discovery_engagement(entry.nomination.items)
|
||||
rows.append({
|
||||
"id": nomination_id,
|
||||
"cluster_id": entry.cluster_id,
|
||||
"heuristic_name": entry.heuristic_name,
|
||||
"heuristic_junk": bool(entry.heuristic_junk),
|
||||
"sources": sources,
|
||||
"engagement_by_source": engagement,
|
||||
"nomination": schema.nomination_to_dict(entry.nomination),
|
||||
})
|
||||
nominations.append(BundleNomination(
|
||||
nomination_id=nomination_id,
|
||||
nomination=entry.nomination,
|
||||
cluster_id=entry.cluster_id,
|
||||
heuristic_name=entry.heuristic_name,
|
||||
heuristic_junk=bool(entry.heuristic_junk),
|
||||
sources=sources,
|
||||
engagement_by_source=engagement,
|
||||
))
|
||||
|
||||
payload = {
|
||||
"schema_version": schema.DISCOVERY_NOMINATIONS_SCHEMA_VERSION,
|
||||
"kind": schema.DISCOVERY_NOMINATIONS_KIND,
|
||||
"bundle_id": bundle_id,
|
||||
"generated_at": generated_at,
|
||||
"from_date": from_date,
|
||||
"to_date": to_date,
|
||||
"domain": domain,
|
||||
"tier": tier,
|
||||
"mock": bool(mock),
|
||||
"source_status": {
|
||||
source: schema.to_dict(outcome)
|
||||
for source, outcome in (source_status or {}).items()
|
||||
},
|
||||
"context": {
|
||||
"enrichment_source_boundary": (
|
||||
list(enrichment_source_boundary)
|
||||
if enrichment_source_boundary is not None
|
||||
else None
|
||||
),
|
||||
"requested_sources": (
|
||||
list(requested_sources) if requested_sources is not None else None
|
||||
),
|
||||
"lookback_days": int(lookback_days),
|
||||
},
|
||||
"nominations": rows,
|
||||
}
|
||||
path = nominations_bundle_path(state_dir)
|
||||
try:
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
except OSError as exc:
|
||||
# A locked/read-only/full disk is the protocol's clean exit-2 path,
|
||||
# never a traceback.
|
||||
raise HandoffContractError(
|
||||
f"Could not write nominations bundle {path}: {exc}"
|
||||
) from exc
|
||||
return NominationsBundle(
|
||||
schema_version=schema.DISCOVERY_NOMINATIONS_SCHEMA_VERSION,
|
||||
bundle_id=bundle_id,
|
||||
generated_at=generated_at,
|
||||
from_date=from_date,
|
||||
to_date=to_date,
|
||||
domain=domain,
|
||||
tier=tier,
|
||||
enrichment_source_boundary=(
|
||||
list(enrichment_source_boundary)
|
||||
if enrichment_source_boundary is not None
|
||||
else None
|
||||
),
|
||||
requested_sources=(
|
||||
list(requested_sources) if requested_sources is not None else None
|
||||
),
|
||||
lookback_days=int(lookback_days),
|
||||
nominations=nominations,
|
||||
source_status=dict(source_status or {}),
|
||||
mock=bool(mock),
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def read_nominations_bundle(
|
||||
*,
|
||||
save_dir: str | Path | None = None,
|
||||
config_dir: Path | None = None,
|
||||
) -> NominationsBundle:
|
||||
"""Locate and parse the nominations bundle for legs 2 and 3.
|
||||
|
||||
The bundle lives in the save dir when one was supplied, else the config
|
||||
dir - never both (no cross-store fallback). Raises HandoffContractError
|
||||
(naming the searched location and the re-sweep remedy) when no bundle
|
||||
exists, and for any top-level contract violation in the file found.
|
||||
"""
|
||||
searched = _search_paths(save_dir, config_dir, nominations_bundle_path)
|
||||
path = next((candidate for candidate in searched if candidate.exists()), None)
|
||||
if path is None:
|
||||
raise HandoffContractError(
|
||||
"No discovery nominations bundle found. Searched:\n"
|
||||
f"{_searched_lines(searched)}\n{_RESWEEP_REMEDY}"
|
||||
)
|
||||
return _parse_bundle_file(path)
|
||||
|
||||
|
||||
def _parse_handoff_envelope(
|
||||
path: Path,
|
||||
*,
|
||||
label: str,
|
||||
kind: str,
|
||||
schema_version: str,
|
||||
remedy: str,
|
||||
missing_id_context: str,
|
||||
stale_context: str,
|
||||
) -> tuple[dict[str, Any], str, Any]:
|
||||
"""Shared strict top-level validation for the two engine-written handoff
|
||||
files (nominations bundle, pending report): readable, valid JSON object,
|
||||
right kind and schema version, bundle_id present, within TTL. Returns
|
||||
(payload, bundle_id, generated_at)."""
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise HandoffContractError(
|
||||
f"Could not read {label.lower()} {path}: {exc}"
|
||||
) from exc
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HandoffContractError(
|
||||
f"{label} {path} is not valid JSON: {exc}"
|
||||
) from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise HandoffContractError(
|
||||
f"{label} {path} must be a top-level JSON object, "
|
||||
f"got {type(payload).__name__}."
|
||||
)
|
||||
version = payload.get("schema_version")
|
||||
if version != schema_version:
|
||||
raise HandoffContractError(
|
||||
f"{label} {path} has schema version {version!r}; this "
|
||||
f"build reads {schema_version!r}. {remedy}"
|
||||
)
|
||||
file_kind = payload.get("kind")
|
||||
if file_kind != kind:
|
||||
raise HandoffContractError(
|
||||
f"{label} {path} has kind {file_kind!r}; expected "
|
||||
f"{kind!r}. {remedy}"
|
||||
)
|
||||
bundle_id = str(payload.get("bundle_id") or "")
|
||||
if not bundle_id:
|
||||
raise HandoffContractError(
|
||||
f"{label} {path} is missing its bundle_id; "
|
||||
f"{missing_id_context}. {remedy}"
|
||||
)
|
||||
generated_at = payload.get("generated_at")
|
||||
if not env.is_timestamp_fresh(generated_at, DISCOVERY_HANDOFF_TTL_SECONDS):
|
||||
raise HandoffContractError(
|
||||
f"{label} {path} is stale (generated_at="
|
||||
f"{generated_at!r}, TTL {int(DISCOVERY_HANDOFF_TTL_SECONDS)}s): "
|
||||
f"{stale_context}. {remedy}"
|
||||
)
|
||||
return payload, bundle_id, generated_at
|
||||
|
||||
|
||||
def _parse_bundle_file(path: Path) -> NominationsBundle:
|
||||
payload, bundle_id, generated_at = _parse_handoff_envelope(
|
||||
path,
|
||||
label="Nominations bundle",
|
||||
kind=schema.DISCOVERY_NOMINATIONS_KIND,
|
||||
schema_version=schema.DISCOVERY_NOMINATIONS_SCHEMA_VERSION,
|
||||
remedy=_RESWEEP_REMEDY,
|
||||
missing_id_context="judgments cannot bind to it",
|
||||
stale_context="the momentum window it captured has moved on",
|
||||
)
|
||||
version = payload.get("schema_version")
|
||||
|
||||
context = payload.get("context") or {}
|
||||
boundary = context.get("enrichment_source_boundary")
|
||||
requested = context.get("requested_sources")
|
||||
try:
|
||||
lookback_days = int(context.get("lookback_days") or 30)
|
||||
except (TypeError, ValueError):
|
||||
lookback_days = 30
|
||||
|
||||
rows_raw = payload.get("nominations")
|
||||
if not isinstance(rows_raw, list):
|
||||
raise HandoffContractError(
|
||||
f"Nominations bundle {path} must carry a top-level "
|
||||
f"\"nominations\" list, got {type(rows_raw).__name__}. "
|
||||
f"{_RESWEEP_REMEDY}"
|
||||
)
|
||||
|
||||
nominations: list[BundleNomination] = []
|
||||
for position, row in enumerate(rows_raw, start=1):
|
||||
# Lenient per row: the bundle is engine-written, but one corrupted
|
||||
# row must not discard the rest of the pool.
|
||||
if not isinstance(row, dict):
|
||||
_warn(
|
||||
f"skipping malformed nomination row {position} in "
|
||||
f"{path.name} (not an object)"
|
||||
)
|
||||
continue
|
||||
try:
|
||||
nomination = pipeline.Nomination(
|
||||
**schema.nomination_kwargs_from_dict(row.get("nomination") or {})
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
_warn(
|
||||
f"skipping unparseable nomination row {position} in "
|
||||
f"{path.name}: {type(exc).__name__}: {exc}"
|
||||
)
|
||||
continue
|
||||
engagement_raw = row.get("engagement_by_source")
|
||||
engagement = {
|
||||
str(source): dict(metrics)
|
||||
for source, metrics in (
|
||||
engagement_raw.items() if isinstance(engagement_raw, dict) else ()
|
||||
)
|
||||
if isinstance(metrics, dict)
|
||||
}
|
||||
nominations.append(BundleNomination(
|
||||
nomination_id=str(row.get("id") or f"n{position}"),
|
||||
nomination=nomination,
|
||||
cluster_id=str(row.get("cluster_id") or ""),
|
||||
heuristic_name=str(row.get("heuristic_name") or ""),
|
||||
heuristic_junk=bool(row.get("heuristic_junk")),
|
||||
sources=[str(source) for source in row.get("sources") or []],
|
||||
engagement_by_source=engagement,
|
||||
))
|
||||
|
||||
if not nominations:
|
||||
# Leg 1 never writes an empty bundle (a zero-nomination sweep
|
||||
# short-circuits with no bundle file), so an empty or all-invalid
|
||||
# nominations array is corrupt state: fail closed, never hand the
|
||||
# resume leg a silently empty pool.
|
||||
raise HandoffContractError(
|
||||
f"Nominations bundle {path} contains no readable nominations "
|
||||
f"(leg 1 never writes an empty pool). {_RESWEEP_REMEDY}"
|
||||
)
|
||||
|
||||
# Sweep status is advisory coverage context: restore it through the same
|
||||
# deserializer every report uses, but degrade a malformed map to empty
|
||||
# rather than discarding an otherwise-valid pool.
|
||||
try:
|
||||
source_status = schema._source_status_from_dict(payload)
|
||||
except (AttributeError, KeyError, TypeError, ValueError):
|
||||
_warn(f"ignoring malformed source_status map in {path.name}")
|
||||
source_status = {}
|
||||
|
||||
return NominationsBundle(
|
||||
schema_version=str(version),
|
||||
bundle_id=bundle_id,
|
||||
generated_at=str(generated_at or ""),
|
||||
from_date=str(payload.get("from_date") or ""),
|
||||
to_date=str(payload.get("to_date") or ""),
|
||||
domain=str(payload.get("domain") or ""),
|
||||
tier=str(payload.get("tier") or "deep"),
|
||||
enrichment_source_boundary=(
|
||||
[str(source) for source in boundary]
|
||||
if isinstance(boundary, list) else None
|
||||
),
|
||||
requested_sources=(
|
||||
[str(source) for source in requested]
|
||||
if isinstance(requested, list) else None
|
||||
),
|
||||
lookback_days=lookback_days,
|
||||
nominations=nominations,
|
||||
source_status=source_status,
|
||||
mock=bool(payload.get("mock")),
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def read_pending_report(
|
||||
*,
|
||||
save_dir: str | Path | None = None,
|
||||
config_dir: Path | None = None,
|
||||
) -> PendingReport:
|
||||
"""Locate and parse the leg-2 pending report for the finalize leg.
|
||||
|
||||
Same strictness family as the bundle reader: missing file (the searched
|
||||
location named - save dir when supplied, else config dir, never a
|
||||
cross-store fallback), unreadable, invalid JSON, wrong kind or schema version,
|
||||
missing bundle_id, or stale TTL all raise HandoffContractError (mapped to
|
||||
exit 2 by the CLI layer). Staleness is measured from the PENDING report's
|
||||
own generated_at - the leg-2 write started a fresh authoring window - and
|
||||
the remedy is the resume leg, not a full re-sweep.
|
||||
"""
|
||||
searched = _search_paths(save_dir, config_dir, pending_report_path)
|
||||
path = next((candidate for candidate in searched if candidate.exists()), None)
|
||||
if path is None:
|
||||
raise HandoffContractError(
|
||||
"No pending discovery report found. Searched:\n"
|
||||
f"{_searched_lines(searched)}\n{_RESUME_REMEDY}"
|
||||
)
|
||||
return _parse_pending_file(path)
|
||||
|
||||
|
||||
def _parse_pending_file(path: Path) -> PendingReport:
|
||||
payload, bundle_id, generated_at = _parse_handoff_envelope(
|
||||
path,
|
||||
label="Pending discovery report",
|
||||
kind=schema.DISCOVERY_PENDING_KIND,
|
||||
schema_version=schema.DISCOVERY_PENDING_SCHEMA_VERSION,
|
||||
remedy=_RESUME_REMEDY,
|
||||
missing_id_context="angles cannot bind to it",
|
||||
stale_context="the judged window it captured has moved on",
|
||||
)
|
||||
version = payload.get("schema_version")
|
||||
report = payload.get("report")
|
||||
if not isinstance(report, dict):
|
||||
raise HandoffContractError(
|
||||
f"Pending discovery report {path} must carry a top-level "
|
||||
f"\"report\" object. {_RESUME_REMEDY}"
|
||||
)
|
||||
# Lenient per row (engine-written, but one corrupt row must not discard
|
||||
# the rest): keep only well-shaped angle-input entries.
|
||||
angle_inputs_raw = payload.get("angle_inputs")
|
||||
angle_inputs = {
|
||||
str(nomination_id): {
|
||||
str(key): str(value) for key, value in info.items()
|
||||
}
|
||||
for nomination_id, info in (
|
||||
angle_inputs_raw.items() if isinstance(angle_inputs_raw, dict) else ()
|
||||
)
|
||||
if isinstance(info, dict)
|
||||
}
|
||||
return PendingReport(
|
||||
schema_version=str(version),
|
||||
bundle_id=bundle_id,
|
||||
generated_at=str(generated_at or ""),
|
||||
run_ref=str(payload.get("run_ref") or ""),
|
||||
report=report,
|
||||
angle_inputs=angle_inputs,
|
||||
mock=bool(payload.get("mock")),
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def _load_host_file(path: str | Path, label: str) -> dict[str, Any]:
|
||||
"""Load a host-authored handoff file with strict top-level checks."""
|
||||
file_path = Path(path).expanduser()
|
||||
try:
|
||||
raw = file_path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise HandoffContractError(
|
||||
f"Could not read {label} file {file_path}: {exc}"
|
||||
) from exc
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HandoffContractError(
|
||||
f"{label.capitalize()} file {file_path} is not valid JSON: {exc}"
|
||||
) from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise HandoffContractError(
|
||||
f"{label.capitalize()} file {file_path} must be a top-level JSON "
|
||||
f"object, got {type(payload).__name__}."
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _require_bundle_binding(
|
||||
payload: dict[str, Any],
|
||||
bundle: NominationsBundle | PendingReport,
|
||||
*,
|
||||
label: str,
|
||||
save_dir: str | Path | None,
|
||||
config_dir: Path | None,
|
||||
) -> None:
|
||||
"""Enforce bundle-id binding between a host file and the current bundle
|
||||
(or, on the finalize leg, the pending report that inherited its id).
|
||||
The mismatch message names the file actually validated against - the
|
||||
pending report on the finalize leg - so a host's retry is not misdirected
|
||||
at the nominations bundle. A mismatch means the host echoed the wrong id
|
||||
into an otherwise-current file, so the remedy is the cheap one - correct
|
||||
the bundle_id field and re-run this same leg - never the expensive
|
||||
re-sweep/resume remedies (those belong to missing/stale state)."""
|
||||
file_bundle_id = str(payload.get("bundle_id") or "")
|
||||
if file_bundle_id == bundle.bundle_id:
|
||||
return
|
||||
if isinstance(bundle, PendingReport):
|
||||
searched = _search_paths(save_dir, config_dir, pending_report_path)
|
||||
noun = "current pending discovery report"
|
||||
location_label = "Pending-report locations searched"
|
||||
else:
|
||||
searched = _search_paths(save_dir, config_dir, nominations_bundle_path)
|
||||
noun = "current nominations bundle"
|
||||
location_label = "Bundle locations searched"
|
||||
if not searched and bundle.path is not None:
|
||||
searched = [bundle.path]
|
||||
raise HandoffContractError(
|
||||
f"The {label} file is bound to bundle_id {file_bundle_id!r} but the "
|
||||
f"{noun} is {bundle.bundle_id!r}. {location_label}:\n"
|
||||
f"{_searched_lines(searched)}\n"
|
||||
f"Correct the bundle_id field in your {label} file to "
|
||||
f"{bundle.bundle_id!r} and re-run this same leg."
|
||||
)
|
||||
|
||||
|
||||
def _truncate_at_word(text: str, max_chars: int) -> str:
|
||||
"""Cap ``text`` at ``max_chars``, cutting back to a word boundary and
|
||||
stripping trailing punctuation. Text within the cap passes through
|
||||
untouched."""
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
return text[:max_chars].rsplit(" ", 1)[0].rstrip(_TRUNCATE_STRIP_CHARS)
|
||||
|
||||
|
||||
def _sanitized_name(raw: object) -> str | None:
|
||||
"""One whitespace-collapsed, punctuation-stripped, length-capped topic
|
||||
name, or None for anything unusable (non-strings, and names that
|
||||
sanitize to empty - e.g. emoji-only - count as per-row-absent)."""
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
name = " ".join(raw.split()).strip(_TRUNCATE_STRIP_CHARS)
|
||||
name = _truncate_at_word(name, _NAME_MAX_CHARS)
|
||||
if not any(char.isalnum() for char in name):
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def _sanitized_angle(raw: object) -> str | None:
|
||||
"""One whitespace-collapsed, length-capped angle sentence, or None for
|
||||
anything unusable. Non-strings are rejected outright, never coerced."""
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
text = _truncate_at_word(" ".join(raw.split()), _ANGLE_MAX_CHARS)
|
||||
return text or None
|
||||
|
||||
|
||||
def _known_rows(
|
||||
rows: list[Any],
|
||||
known: set[str],
|
||||
*,
|
||||
row_label: str,
|
||||
unknown_label: str,
|
||||
) -> Iterator[tuple[str, dict[str, Any]]]:
|
||||
"""Shared lenient per-row gate for host-authored files: skip non-object
|
||||
rows, rows with no nomination id, and rows for unknown ids - warning on
|
||||
each - and yield (row_id, row) for the rest."""
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
_warn(f"skipping malformed {row_label} row (not an object)")
|
||||
continue
|
||||
row_id = str(row.get("id") or "").strip()
|
||||
if not row_id:
|
||||
_warn(f"skipping {row_label} row with no nomination id")
|
||||
continue
|
||||
if row_id not in known:
|
||||
_warn(f"ignoring {unknown_label} for unknown nomination id {row_id!r}")
|
||||
continue
|
||||
yield row_id, row
|
||||
|
||||
|
||||
def _clamped_worthiness(raw: object) -> int | None:
|
||||
"""Worthiness clamped to 0-100 integers; anything non-numeric is absent."""
|
||||
if isinstance(raw, bool):
|
||||
return None
|
||||
try:
|
||||
value = float(raw) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return max(0, min(100, round(value)))
|
||||
|
||||
|
||||
def read_judgments(
|
||||
path: str | Path,
|
||||
bundle: NominationsBundle,
|
||||
*,
|
||||
save_dir: str | Path | None = None,
|
||||
config_dir: Path | None = None,
|
||||
) -> dict[str, HostJudgment]:
|
||||
"""Read the host judgments file for leg 2, keyed by nomination id.
|
||||
|
||||
Strict at the top level (readable, valid JSON object, ``judgments`` list,
|
||||
bundle_id bound to ``bundle``), lenient per row: an unknown id is warned
|
||||
and ignored, a missing/unusable name or junk field is per-row-absent, and
|
||||
worthiness is clamped to 0-100 integers. Nominations with no row at all
|
||||
are simply missing from the mapping - use ``judgment_for`` to get the
|
||||
ROW_ABSENT marker for them.
|
||||
"""
|
||||
payload = _load_host_file(path, "judgments")
|
||||
_require_bundle_binding(
|
||||
payload, bundle, label="judgments", save_dir=save_dir, config_dir=config_dir,
|
||||
)
|
||||
rows = payload.get("judgments")
|
||||
if not isinstance(rows, list):
|
||||
raise HandoffContractError(
|
||||
f"Judgments file {path} must carry a top-level \"judgments\" list."
|
||||
)
|
||||
known = {entry.nomination_id for entry in bundle.nominations}
|
||||
judgments: dict[str, HostJudgment] = {}
|
||||
for row_id, row in _known_rows(
|
||||
rows, known, row_label="judgments", unknown_label="judgment"
|
||||
):
|
||||
# Only a real JSON boolean is a junk verdict: null, "false", 0, or
|
||||
# any other non-bool value is per-row-absent (bundle heuristic),
|
||||
# never coerced - bool("false") is True.
|
||||
raw_junk = row.get("junk")
|
||||
judgments[row_id] = HostJudgment(
|
||||
name=_sanitized_name(row.get("name")),
|
||||
junk=raw_junk if isinstance(raw_junk, bool) else None,
|
||||
worthiness=_clamped_worthiness(row.get("worthiness")),
|
||||
)
|
||||
return judgments
|
||||
|
||||
|
||||
def judgment_for(
|
||||
judgments: dict[str, HostJudgment],
|
||||
nomination_id: str,
|
||||
) -> HostJudgment:
|
||||
"""The host's verdict for one nomination, or ROW_ABSENT when the host
|
||||
omitted the row (caller falls back to the bundle's heuristic name/junk)."""
|
||||
return judgments.get(nomination_id, ROW_ABSENT)
|
||||
|
||||
|
||||
def read_angles(
|
||||
path: str | Path | None,
|
||||
bundle: NominationsBundle | PendingReport,
|
||||
*,
|
||||
save_dir: str | Path | None = None,
|
||||
config_dir: Path | None = None,
|
||||
) -> dict[str, HostAngles]:
|
||||
"""Read the host angles file for leg 3, keyed by nomination id.
|
||||
|
||||
``bundle`` is the binding target: the finalize leg passes the pending
|
||||
report (the bundle_id echo validates against it, and the known ids are
|
||||
its surviving ``angle_inputs`` ids), while a NominationsBundle binds
|
||||
against the full pool. A missing angles file is legal: ``path=None``
|
||||
returns an empty mapping and every topic ships without angles. When a
|
||||
path is given the same strict-top-level / lenient-per-row rules as
|
||||
judgments apply; angle sentences are word-boundary capped at 200 chars.
|
||||
"""
|
||||
if path is None:
|
||||
return {}
|
||||
payload = _load_host_file(path, "angles")
|
||||
_require_bundle_binding(
|
||||
payload, bundle, label="angles", save_dir=save_dir, config_dir=config_dir,
|
||||
)
|
||||
rows = payload.get("angles")
|
||||
if not isinstance(rows, list):
|
||||
raise HandoffContractError(
|
||||
f"Angles file {path} must carry a top-level \"angles\" list."
|
||||
)
|
||||
known = (
|
||||
set(bundle.angle_inputs)
|
||||
if isinstance(bundle, PendingReport)
|
||||
else {entry.nomination_id for entry in bundle.nominations}
|
||||
)
|
||||
angles: dict[str, HostAngles] = {}
|
||||
for row_id, row in _known_rows(
|
||||
rows, known, row_label="angles", unknown_label="angles"
|
||||
):
|
||||
podcast = _sanitized_angle(row.get("podcast"))
|
||||
x_article = _sanitized_angle(row.get("x_article"))
|
||||
if podcast is None and x_article is None:
|
||||
# No usable hook at all: treat the row as absent.
|
||||
continue
|
||||
angles[row_id] = HostAngles(podcast=podcast, x_article=x_article)
|
||||
return angles
|
||||
|
||||
|
||||
def resolve_name_collisions(
|
||||
pairs: Sequence[tuple[pipeline.Nomination, str]],
|
||||
) -> list[str]:
|
||||
"""Re-run the nominate-stage casefold/entity-token collision rules over
|
||||
host-applied names, returning one collision-free name per input pair in
|
||||
order.
|
||||
|
||||
Short host-judged names collide far more often than raw titles; a
|
||||
colliding name gets the later nomination's strongest non-shared entity
|
||||
token appended (``pipeline._disambiguated_topic_name``, fed synthetic
|
||||
per-nomination clusters built from the seed items). Unlike the nominate
|
||||
stage, a collision can never DROP a nomination here - the pool already
|
||||
de-duplicated same-story clusters at leg 1 - so when no distinguishing
|
||||
entity token exists the name falls back to an ordinal suffix.
|
||||
"""
|
||||
candidate_map: dict[str, schema.Candidate] = {}
|
||||
clusters: list[schema.Cluster] = []
|
||||
for index, (nomination, _applied) in enumerate(pairs):
|
||||
candidate_ids: list[str] = []
|
||||
for item_index, item in enumerate(nomination.items):
|
||||
candidate_id = f"handoff-{index}-{item_index}"
|
||||
candidate_map[candidate_id] = schema.Candidate(
|
||||
candidate_id=candidate_id,
|
||||
item_id=item.item_id,
|
||||
source=item.source,
|
||||
title=item.title,
|
||||
url=item.url,
|
||||
snippet=item.snippet,
|
||||
subquery_labels=[],
|
||||
native_ranks={},
|
||||
local_relevance=0.0,
|
||||
freshness=0,
|
||||
engagement=None,
|
||||
source_quality=0.0,
|
||||
rrf_score=0.0,
|
||||
)
|
||||
candidate_ids.append(candidate_id)
|
||||
clusters.append(schema.Cluster(
|
||||
cluster_id=f"handoff-n{index}",
|
||||
title=nomination.name,
|
||||
candidate_ids=candidate_ids,
|
||||
representative_ids=candidate_ids[:1],
|
||||
sources=sorted({item.source for item in nomination.items}),
|
||||
score=nomination.seed_score,
|
||||
))
|
||||
|
||||
resolved_names: list[str] = []
|
||||
taken: dict[str, schema.Cluster] = {}
|
||||
entity_counts_cache: dict[str, Counter] = {}
|
||||
for index, (_nomination, applied) in enumerate(pairs):
|
||||
cluster = clusters[index]
|
||||
name = applied
|
||||
key = name.casefold()
|
||||
if key in taken:
|
||||
resolved = pipeline._disambiguated_topic_name(
|
||||
name, cluster, taken[key], candidate_map, entity_counts_cache,
|
||||
taken,
|
||||
)
|
||||
if resolved is None:
|
||||
# Indistinguishable by content: keep the nomination anyway
|
||||
# (distinct stories at leg 1) under an ordinal suffix.
|
||||
suffix = 2
|
||||
while f"{name} {suffix}".casefold() in taken:
|
||||
suffix += 1
|
||||
resolved = f"{name} {suffix}"
|
||||
name = resolved
|
||||
key = name.casefold()
|
||||
taken[key] = cluster
|
||||
resolved_names.append(name)
|
||||
return resolved_names
|
||||
|
||||
|
||||
def _one_line(text: str) -> str:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def build_host_digest(bundle: NominationsBundle) -> str:
|
||||
"""The host-facing judging digest for a nominations bundle: plain,
|
||||
promptable text with one structural line per nomination (id, seed source
|
||||
names, velocity/engagement signal) plus capped evidence lines (leader
|
||||
title, leader snippet, strongest community comment - the surface the
|
||||
engine judge used to see). Names the bundle file and instructs the host
|
||||
to read its full evidence before judging.
|
||||
|
||||
The evidence lines are scraped third-party text, so they are fenced the
|
||||
way the deleted engine judge fenced its candidate block (the exact
|
||||
``rerank._fenced_untrusted_content`` fence: a security-notice header
|
||||
stating the fenced content is data, never instructions, around
|
||||
``<untrusted_content>`` tags). The structural lines - nomination ids,
|
||||
sources, signal, bundle path, judging instructions - stay outside the
|
||||
fence."""
|
||||
location = str(bundle.path) if bundle.path is not None else (
|
||||
NOMINATIONS_BUNDLE_FILENAME
|
||||
)
|
||||
domain_label = bundle.domain or "global trending (no domain filter)"
|
||||
lines = [
|
||||
f"Discovery nominations awaiting host judgment "
|
||||
f"({len(bundle.nominations)} topics).",
|
||||
f"Domain: {domain_label} | window {bundle.from_date} -> "
|
||||
f"{bundle.to_date} | tier {bundle.tier}",
|
||||
f"Bundle file: {location} (bundle_id {bundle.bundle_id})",
|
||||
"Read the bundle file's per-nomination evidence before judging; the "
|
||||
"lines below are only a digest.",
|
||||
"",
|
||||
]
|
||||
evidence_lines: list[str] = []
|
||||
for entry in bundle.nominations:
|
||||
items = entry.nomination.items
|
||||
leader = items[0] if items else None
|
||||
title = _one_line((leader.title if leader else "") or entry.nomination.name)
|
||||
sources = ", ".join(entry.sources) if entry.sources else "unknown"
|
||||
native_total = sum(
|
||||
rerank.discovery_engagement_total(item) for item in items
|
||||
)
|
||||
lines.append(
|
||||
f"{entry.nomination_id} | sources: {sources} | "
|
||||
f"signal: seed velocity {entry.nomination.seed_score:.1f}, "
|
||||
f"{native_total:,.0f} native interactions"
|
||||
)
|
||||
evidence_lines.append(f"- id: {entry.nomination_id}")
|
||||
evidence_lines.append(f" title: {title[:_DIGEST_TITLE_MAX_CHARS]}")
|
||||
snippet_text = _one_line(
|
||||
(leader.snippet if leader else "") or entry.nomination.summary
|
||||
)
|
||||
if snippet_text:
|
||||
evidence_lines.append(
|
||||
f" snippet: {snippet_text[:_DIGEST_SNIPPET_MAX_CHARS]}"
|
||||
)
|
||||
top_comment = pipeline._best_community_comment(items)
|
||||
if top_comment:
|
||||
evidence_lines.append(
|
||||
f" top comment: "
|
||||
f"{_one_line(top_comment)[:_DIGEST_COMMENT_MAX_CHARS]}"
|
||||
)
|
||||
if evidence_lines:
|
||||
lines.append("")
|
||||
lines.append(rerank._fenced_untrusted_content("\n".join(evidence_lines)))
|
||||
return "\n".join(lines)
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user