Compare commits
585 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7cdf1b8209 | |||
| 1eb23657e9 | |||
| 98860333cd | |||
| 82f3e029a4 | |||
| 6b91a9927b | |||
| 4f209ee600 | |||
| fd8f3d2f32 | |||
| b644dbade9 | |||
| 909d3ffdd8 | |||
| 386cc92688 | |||
| 5ea09dc0ae | |||
| 090d3af15e | |||
| fdee7e61b6 | |||
| 92b664e742 | |||
| 27e17245b6 | |||
| f9a3c919bc | |||
| 249c7a4c04 | |||
| 2626d0ff2f | |||
| 50d44bf208 | |||
| 691c21234c | |||
| e71ac74cf0 | |||
| b58c2e7947 | |||
| 4488271f79 | |||
| d5a8f2836a | |||
| 224e84a043 | |||
| 665f9a893b | |||
| 9585ec4b84 | |||
| cb44afeb19 | |||
| 39cca461aa | |||
| 3176b07842 | |||
| 1d927db51d | |||
| b8cfb9d03c | |||
| 5c4fe1ed1d | |||
| 86c5303c1a | |||
| aab5395778 | |||
| ed49713e4d | |||
| de3b692b1c | |||
| 586b7e29f9 | |||
| e4f7e9b175 | |||
| 7650a68638 | |||
| 4e14c9a259 | |||
| d1efed83d4 | |||
| 1d55d8bef1 | |||
| 2f4374ba9f | |||
| 943e18d1c3 | |||
| 4b027919c7 | |||
| af5f0ec4e3 | |||
| d31d5f3b3a | |||
| d1ecc2f062 | |||
| c245847046 | |||
| d3cafd2980 | |||
| 22e095fa0a | |||
| dd69abe669 | |||
| 0a1a5799ae | |||
| c410a2d141 | |||
| f462577eaa | |||
| 9bf0a79a70 | |||
| 4ce7a0ebc9 | |||
| 3b0b021e2c | |||
| 9c020b1393 | |||
| a06e221552 | |||
| 6b62b867b5 | |||
| cb27c689b5 | |||
| 298310ca34 | |||
| ffa2e31e9b | |||
| 0270c12352 | |||
| 7abcb9d108 | |||
| 021bdbf60e | |||
| 06b68a883a | |||
| 91e606a369 | |||
| 24c6567731 | |||
| f1beee7e81 | |||
| fc45949239 | |||
| c74fbd1b0d | |||
| 856da6dc8e | |||
| f53bde5420 | |||
| 5f4e7e73c3 | |||
| 602dbde2db | |||
| 98dd5dce29 | |||
| f5a0a18ec0 | |||
| 6287cf0aeb | |||
| 6340fadf31 | |||
| 961572de22 | |||
| 541717af30 | |||
| ae8c32327f | |||
| c4e70ee40a | |||
| 8f31ed3179 | |||
| ee04fba116 | |||
| 030a2c8fe6 | |||
| 4bbfee4055 | |||
| 750176314a | |||
| 1e82659bb4 | |||
| 16bfbd8ce9 | |||
| a5b3ca1f3c | |||
| 9e00fe5b6d | |||
| f5bbc03aab | |||
| 51424be41d | |||
| 6a1d93bdb5 | |||
| 6318f7449e | |||
| 76215629b9 | |||
| 47b924c87f | |||
| cc620449ce | |||
| 443b05f330 | |||
| 9b9bd42e0f | |||
| f857a5101f | |||
| 784624c002 | |||
| 3e78f45438 | |||
| a0bbac22e6 | |||
| a1ab6fc4b6 | |||
| a28668ad30 | |||
| cea36329af | |||
| 344731b95a | |||
| 6e87b83d78 | |||
| a9d2674147 | |||
| 7d2868b9d1 | |||
| 3f7c8c5022 | |||
| 433ac8d2ed | |||
| 545a8bdc33 | |||
| 732d4ae76b | |||
| ad52872ecf | |||
| 283dbae20c | |||
| 9a550c0ccb | |||
| 62072dacfe | |||
| e0775b9d18 | |||
| 34365e68fe | |||
| d4d603d540 | |||
| d5f3083b82 | |||
| 4bd20f3f4b | |||
| c09e67ce76 | |||
| 5b8b021318 | |||
| a61852b34e | |||
| f168eecd67 | |||
| 65951b3bd7 | |||
| 80cc0aae13 | |||
| d979c3640a | |||
| 97c7636126 | |||
| 287327df60 | |||
| 6db296ca68 | |||
| 1e365e573b | |||
| 8a0dea2884 | |||
| e219c65ea7 | |||
| 0afd78dabc | |||
| 91dba43d1f | |||
| eefeba9537 | |||
| 979e6dece3 | |||
| fcf047c3f3 | |||
| a824fa3c3a | |||
| c966c93653 | |||
| 2c149b73e4 | |||
| 0abd39abaa | |||
| f067f90151 | |||
| 811f044367 | |||
| 7bd11506bd | |||
| d3b0a45a94 | |||
| 7a4a0ee223 | |||
| 3494926ef2 | |||
| e389a0ecc4 | |||
| 39cfd722f8 | |||
| 0e427ab98a | |||
| 9a9d735ef9 | |||
| 25b86de81c | |||
| 945f11d048 | |||
| 0cd5772834 | |||
| e64ed92b10 | |||
| 030dda7399 | |||
| 3e36224927 | |||
| 8bec337c99 | |||
| 42eb5f5273 | |||
| 6e91e8106c | |||
| 51b6762f4b | |||
| b582bffcb3 | |||
| bd7e46fbfd | |||
| 770b91e023 | |||
| 46269c46a5 | |||
| 963a6c8a15 | |||
| fcf428266d | |||
| 824a4a9595 | |||
| eee3359248 | |||
| 50d062b88b | |||
| 0aed01555a | |||
| d9c9edffb8 | |||
| f74169fdaa | |||
| 22a2e1d254 | |||
| 76fb9de86b | |||
| ead5ce5466 | |||
| 3f0d1b410b | |||
| db8a2ad5ab | |||
| f48f6d337e | |||
| 66428a8a8e | |||
| edbb3daa3b | |||
| bdb4827bc7 | |||
| ba5c8b6aeb | |||
| ae229e1f3f | |||
| d1398ef3a1 | |||
| 82bc473e29 | |||
| e3a481df37 | |||
| c65bd764dd | |||
| 50d8101013 | |||
| 5f49ad2b0c | |||
| e12330df3f | |||
| cc5598de84 | |||
| a431340fd2 | |||
| 136b4f90c1 | |||
| 6c9f1ada5c | |||
| 5eea498e7d | |||
| 9b9439d25b | |||
| 55f7aa5850 | |||
| 89a5685953 | |||
| dd5bdbb707 | |||
| 36d6f29684 | |||
| 9961d4396f | |||
| 21c0a9291a | |||
| 207cfa1e74 | |||
| f6edf563b4 | |||
| 834535ec15 | |||
| 472eb88bba | |||
| 91d5887199 | |||
| 0f98304f6e | |||
| 5f86484c0a | |||
| c5ff5505b3 | |||
| 25cfced305 | |||
| 565bb03b63 | |||
| c8ab309f45 | |||
| c3649f30f5 | |||
| ed42d8aa26 | |||
| dab238ea7f | |||
| 56fd24bf14 | |||
| f00162ca89 | |||
| 977f0beed5 | |||
| aecab67c81 | |||
| 4d69ae6a1f | |||
| 9849de1396 | |||
| 6291ac7ec2 | |||
| e65914a9c1 | |||
| fda62bd215 | |||
| 0fa5542dd4 | |||
| 1b832a26bc | |||
| 561e224248 | |||
| 220928f664 | |||
| f092c9f5ba | |||
| 10e19f00fd | |||
| 9400673608 | |||
| f9a016e130 | |||
| a8506fc315 | |||
| 8c2b1c3985 | |||
| 5a74d17559 | |||
| 9124035d7e | |||
| 9f77d3ac31 | |||
| 7f6fd4efd4 | |||
| a8472599b1 | |||
| 8979bc74e4 | |||
| 8fdab71bd6 | |||
| 9450f1f29b | |||
| 64ed8aaf29 | |||
| 22a21e8f2f | |||
| b8122a28d8 | |||
| 03848365dd | |||
| e0102f817d | |||
| e5354f702a | |||
| 7b4ae7afb8 | |||
| e14af82f12 | |||
| ef193c093c | |||
| 8d1c08568d | |||
| b371c0d31f | |||
| 859a63146f | |||
| bed9679699 | |||
| afba3ba528 | |||
| 0a5bdee16d | |||
| 75badd9117 | |||
| 634b95b787 | |||
| 96ab2e276b | |||
| 891797248d | |||
| cffca3aeb8 | |||
| b6e79032c2 | |||
| c4e5d7b50c | |||
| c165fbfab5 | |||
| 87d2be30b1 | |||
| faf8782dd4 | |||
| e603c7b4f0 | |||
| a6a7a2fb65 | |||
| f03d8a184c | |||
| 0381df5025 | |||
| d39d25f8d6 | |||
| 3fa91fce90 | |||
| 214a8fa059 | |||
| 5634bb3126 | |||
| e1f5e4686d | |||
| a1bda56650 | |||
| 73c9a84c88 | |||
| fbc828ad4a | |||
| ad69960683 | |||
| d9380c629b | |||
| 5b168c7382 | |||
| fce934bd31 | |||
| 7e21004236 | |||
| 7a5ded08ec | |||
| ae65f9aad1 | |||
| 6ef466d43f | |||
| 073ce4278d | |||
| c320908ee2 | |||
| 2d2803f62e | |||
| c2d7412411 | |||
| 885fea5ba5 | |||
| 5b5ed6ddd1 | |||
| ff22e1cd18 | |||
| 9eff7be21c | |||
| 91987294cd | |||
| 334c813cef | |||
| 30c84d7878 | |||
| 9fff90b62b | |||
| ccca10bbaf | |||
| 6a07685c3e | |||
| fa0f840a89 | |||
| faecb03b7f | |||
| 87ef4db772 | |||
| be8438459a | |||
| d434947e03 | |||
| d31f26eb17 | |||
| 09c163cec8 | |||
| ef8cc69fdd | |||
| 9d0fb3180b | |||
| 0283cf2d25 | |||
| b7e92d49c0 | |||
| 3360ef3b23 | |||
| 8333a9a1aa | |||
| 6bed7a3dc5 | |||
| 9a59d88b11 | |||
| 402b1eedf4 | |||
| 1cc1b2efa1 | |||
| 8d10d34010 | |||
| d6a4dd0822 | |||
| ffe4d3477d | |||
| 3bad700f0b | |||
| 3e0f5ebe1e | |||
| 29bb548772 | |||
| 3ecfc9126c | |||
| b70f52f426 | |||
| 7eaa4e0b11 | |||
| 07e6b59e62 | |||
| 76a7ceb553 | |||
| 84b21434dd | |||
| 7f95ca7c21 | |||
| a86c8a162b | |||
| 92c104b911 | |||
| b8e9ba92c7 | |||
| b6fa5b13e6 | |||
| 7d90275d57 | |||
| b034c7373b | |||
| a4d837d74e | |||
| 5f499d4eec | |||
| cd15598e62 | |||
| 884e82bd4f | |||
| 3a300a91b6 | |||
| 019be0f853 | |||
| 557eaec9ee | |||
| 46fe7570ad | |||
| 76b9e4b98f | |||
| 1c6cece934 | |||
| 36af50f1b6 | |||
| 9e2f0bab7d | |||
| 9e19c7e8e7 | |||
| 6b12caa39e | |||
| 4f0cff17a8 | |||
| 6fcf1792d6 | |||
| ca3fbbe377 | |||
| 3c7edd69de | |||
| e8187a36da | |||
| 358525d38b | |||
| 2cc88ecaf1 | |||
| b43c7f43e2 | |||
| 627d36c0aa | |||
| bad3ba97fa | |||
| c591ca3330 | |||
| a3445c0914 | |||
| 7a0800fd74 | |||
| fd6e64dfae | |||
| 8f843ad665 | |||
| 8c11b3a259 | |||
| 9662054e08 | |||
| 50df263261 | |||
| a2a4839987 | |||
| 7d366e0cc7 | |||
| e25a550e90 | |||
| b1b9b8a787 | |||
| a0f7789893 | |||
| ad7d1bf928 | |||
| 6c6876dc63 | |||
| a9749334d7 | |||
| e0a4e907c0 | |||
| 026639614d | |||
| 2ee0fc5c22 | |||
| c1ca1a4e9d | |||
| 57860aff1c | |||
| 4402dd317f | |||
| 6a92f63a56 | |||
| fd0e47d99f | |||
| c5b4a82917 | |||
| 09cd085b47 | |||
| 2655d8f4c8 | |||
| 122158415a | |||
| 1bdc14878c | |||
| 26da1e157c | |||
| 4aae93ee5d | |||
| 41f5e480ab | |||
| 82f9340c91 | |||
| 367904f156 | |||
| 7f211acbdb | |||
| 509fd3a553 | |||
| 8d3a9e4368 | |||
| 1e03af19e0 | |||
| f032e25e51 | |||
| 84a19cf44d | |||
| 861462689e | |||
| e74b0e1e93 | |||
| c5c0239dc9 | |||
| 269dda9f6c | |||
| a35677da77 | |||
| a78ab69ffe | |||
| 0bb01c2d6a | |||
| 444e07d141 | |||
| 850c7e0185 | |||
| d53121f035 | |||
| daca71f89e | |||
| d51e91ea26 | |||
| 170b570cbc | |||
| 4bae05e7fa | |||
| a4f1f94802 | |||
| 16ce073d0c | |||
| 5994b4f76a | |||
| bb5e6efbf9 | |||
| 2502a19d46 | |||
| 0f280245ac | |||
| 236c17a3b8 | |||
| b0b9e6ac5b | |||
| 5ede01745c | |||
| 19f8019323 | |||
| b2a762aa24 | |||
| 6b0261ff0e | |||
| 378be7cfd5 | |||
| 76b8df40d3 | |||
| eb2d8b55e0 | |||
| 1a8ffd4847 | |||
| 1814bb1967 | |||
| f236cff86a | |||
| 44971a6aae | |||
| a8e462c978 | |||
| d9a0ac31f2 | |||
| 3e60c0817d | |||
| d530c90239 | |||
| aef6f35460 | |||
| 77bd235f64 | |||
| e1017e95c9 | |||
| 71b1e8a411 | |||
| 618458eb7e | |||
| 8cccd3e982 | |||
| 17fb17222b | |||
| 8ccd778366 | |||
| a3f173dc8a | |||
| 5a3ac8ca37 | |||
| e8eb15102f | |||
| fd6e70c539 | |||
| d0b990e211 | |||
| 0f03a67166 | |||
| 5a625fda9f | |||
| b296a65515 | |||
| f2737fc035 | |||
| 9ce7264d43 | |||
| f458e0f5af | |||
| 8f565ee241 | |||
| dd7e6a1562 | |||
| 65313ce542 | |||
| 5ab8c3ba76 | |||
| 38bfb504e1 | |||
| e8f23b4205 | |||
| b78ce34922 | |||
| 0656b868e7 | |||
| 16a4fa9c39 | |||
| 3b75ff1537 | |||
| 321975e144 | |||
| 5f9c637bbe | |||
| 19132b0b5e | |||
| 1e4150ad78 | |||
| 6a15afd8e8 | |||
| 0ab7051bc5 | |||
| ec0b126af6 | |||
| d9e8a046ef | |||
| 87bf3debcc | |||
| 2f4b023db8 | |||
| 261ea5895c | |||
| 2692e0f4a2 | |||
| 10f35f82fe | |||
| 5b29b8f427 | |||
| 6a5a122195 | |||
| 7bda02169d | |||
| 6b40d2c46f | |||
| 4a99c4f557 | |||
| 5c802b0daa | |||
| 0e353ae03f | |||
| 0a5102e193 | |||
| cfde1dbbe4 | |||
| 608381a818 | |||
| af4cf7c03d | |||
| 5b0308b9e4 | |||
| 5817ef8387 | |||
| a87c1ba058 | |||
| 7214dd6051 | |||
| 6acf2fdbe2 | |||
| 07a3bdb3cf | |||
| 87577ff126 | |||
| 400fc4cc00 | |||
| e9ecce0b1c | |||
| f3df47c381 | |||
| c9cf3ef92f | |||
| 46cf2328aa | |||
| 7506cbd542 | |||
| a6bd481e61 | |||
| aba6172032 | |||
| d07e4698e3 | |||
| 4c0282dd55 | |||
| 99909fca67 | |||
| 36c43d50b7 | |||
| ecf68347db | |||
| e2d9d705f6 | |||
| 9c09a67ac2 | |||
| 32da0bd6cb | |||
| 863c3bc145 | |||
| 9f39d10bc5 | |||
| 03043da407 | |||
| 8bab997854 | |||
| 375fd0bcc0 | |||
| 92d65723e4 | |||
| f794f82af5 | |||
| 791c0a57a0 | |||
| 211df0deaa | |||
| d7b3995da1 | |||
| e217db77cc | |||
| 8ea207b348 | |||
| b04212680d | |||
| 2c2cfb9e7e | |||
| 3276496f49 | |||
| 68ae74ff4f | |||
| 27c90504c0 | |||
| f4eb0af104 | |||
| 79b5d049ce | |||
| 0e2059661a | |||
| b1c5f8db82 | |||
| 89c5cb9d5d | |||
| 5d4f9ef2c5 | |||
| 01262f78c6 | |||
| 96a4a78faa | |||
| eb2d7a0f37 | |||
| 719cdef2fb | |||
| ac04b56acc | |||
| 42bfc6c76c | |||
| 5a2fe5279b | |||
| a717dd2b2c | |||
| 9f08bb68b5 | |||
| 602de1ebda | |||
| bf3a82a87e | |||
| c010feb8f8 | |||
| edea402b7c | |||
| 4d4ac97ffb | |||
| cd34966b4f | |||
| 85255be350 | |||
| 1aa120a420 | |||
| 306d8c2d73 | |||
| d0dcf751f1 | |||
| afd4b04d6d | |||
| 14d8f62e02 | |||
| 0fd532d249 | |||
| 8867a007ea | |||
| 8af8f06b06 | |||
| 01b5f3dc1e | |||
| 2e39ee8ce4 | |||
| 37033164da | |||
| 9fe4b8f130 | |||
| 73dc6b9996 | |||
| 1fd763e09f | |||
| e0f6ef845a | |||
| c918e18465 | |||
| 6fe0aca7ee | |||
| 74a387b093 | |||
| 095bcae915 | |||
| 4f6b86c456 | |||
| ed455ca036 |
@@ -1,14 +1,14 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
"displayName": "last30days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
"source": "url",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill.git"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
"description": "Marketplace hosting the last30days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.2.3",
|
||||
"version": "3.17.0",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.2.3",
|
||||
"version": "3.17.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
@@ -10,5 +10,21 @@
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
"keywords": [
|
||||
"competitor research",
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"youtube",
|
||||
"tiktok",
|
||||
"instagram",
|
||||
"trends",
|
||||
"prompts",
|
||||
"polymarket",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"hacker-news"
|
||||
]
|
||||
}
|
||||
|
||||
+65
-12
@@ -1,18 +1,71 @@
|
||||
# Exclude binary assets and dev/test artifacts from ClawHub bundle
|
||||
assets/
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
release-notes.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
SKILL-original.md
|
||||
# ClawHub/Hermes packaging exclusions for repository-root scans.
|
||||
# Mirrors .skillignore so non-runtime docs/dev artifacts stay out of the
|
||||
# public bundle and install-time skill security scan.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
skills/last30days/assets/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
work/
|
||||
print/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
mcp/
|
||||
gemini-extension.json
|
||||
greptile.json
|
||||
pyproject.toml
|
||||
|
||||
# Non-runtime docs, plans, release notes, fixtures, and tests
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
media/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
CONCEPTS.md
|
||||
CONFIGURATION.md
|
||||
CONTRIBUTORS.md
|
||||
HERMES_SETUP.md
|
||||
release-notes.md
|
||||
SKILL-original.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
|
||||
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
|
||||
skills/last30days/scripts/build-skill.sh
|
||||
skills/last30days/scripts/compare.sh
|
||||
skills/last30days/scripts/evaluate_search_quality.py
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
skills/last30days/scripts/setup-pass.sh
|
||||
skills/last30days/scripts/test_device_auth.py
|
||||
skills/last30days/scripts/test-v1-vs-v2.sh
|
||||
skills/last30days/scripts/verify_v3.py
|
||||
|
||||
# Keep visible: optional runtime watchlist/store/briefing feature scripts
|
||||
# (`watchlist.py`, `store.py`, and `briefing.py`).
|
||||
|
||||
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
|
||||
skills/last30days/scripts/lib/vendor/
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.17.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and the web.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"email": "mvanhorn@gmail.com",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"competitor research",
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"youtube",
|
||||
"tiktok",
|
||||
"instagram",
|
||||
"trends",
|
||||
"prompts",
|
||||
"polymarket",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"hacker-news"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "last30days",
|
||||
"shortDescription": "Research what people are saying about a topic now.",
|
||||
"longDescription": "last30days adds a Codex skill for researching any topic based on recent discussion and engagement signals across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and the web.",
|
||||
"developerName": "Matt Van Horn",
|
||||
"category": "Research",
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Read",
|
||||
"Write"
|
||||
],
|
||||
"websiteURL": "https://github.com/mvanhorn/last30days-skill",
|
||||
"defaultPrompt": [
|
||||
"TikTok shop trends",
|
||||
"Codex vs Cursor",
|
||||
"best travel credit cards"
|
||||
],
|
||||
"brandColor": "#6F42C1"
|
||||
}
|
||||
}
|
||||
@@ -23,13 +23,10 @@ assets/ export-ignore
|
||||
# claude.ai-bundle-specific exclusions live in scripts/build-skill.sh.
|
||||
|
||||
# Historical + repo-only manifests
|
||||
SKILL-original.md export-ignore
|
||||
SPEC.md export-ignore
|
||||
TASKS.md export-ignore
|
||||
test-run.log export-ignore
|
||||
CONTRIBUTORS.md export-ignore
|
||||
HERMES_SETUP.md export-ignore
|
||||
release-notes.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
uv.lock export-ignore
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ body:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
||||
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
This file contains Copilot-specific additions. See AGENTS.md for the shared cross-tool governance layer.
|
||||
|
||||
# Copilot-specific guidance
|
||||
|
||||
## Test generation
|
||||
|
||||
- Prefer unittest.TestCase for generated tests to match the existing test suite.
|
||||
- Mock external calls with unittest.mock.patch.
|
||||
|
||||
## Pull request reminders
|
||||
|
||||
Before suggesting a pull request:
|
||||
|
||||
- Confirm that pytest passes.
|
||||
- If changes were made anywhere under skills/last30days/, confirm the install copy has been refreshed with:
|
||||
|
||||
npx skills add . -g -y
|
||||
|
||||
## Vendor exclusion zone
|
||||
|
||||
- Never suggest changes to skills/last30days/scripts/lib/vendor/.
|
||||
- Treat skills/last30days/scripts/lib/vendor/ as a no-touch zone.
|
||||
|
||||
## CI expectations
|
||||
|
||||
GitHub CI runs:
|
||||
|
||||
- pytest
|
||||
- ruff
|
||||
|
||||
Generated changes should pass both before review is requested.
|
||||
|
||||
## CLI examples
|
||||
|
||||
When suggesting CLI usage examples for safe local testing, default to:
|
||||
|
||||
--emit=compact --mock
|
||||
@@ -0,0 +1,22 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: uv
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: gomod
|
||||
directory: /mcp
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
@@ -0,0 +1,31 @@
|
||||
name: OSV-Scanner
|
||||
|
||||
# Scheduled OSV-Scanner workflow for vulnerability drift detection.
|
||||
# Scans the repository lockfiles (uv.lock, mcp/go.sum) on a weekly schedule
|
||||
# and uploads results to GitHub code scanning, so newly disclosed CVEs in
|
||||
# the dependency tree are visible even between PRs.
|
||||
#
|
||||
# Advisory-first: fail-on-vuln is false until maintainers confirm a clean
|
||||
# baseline, matching the pattern in security.yml.
|
||||
#
|
||||
# Separate from the pip-audit job in security.yml (which runs on every PR
|
||||
# and push) and from the dependency-review gate (which blocks on new
|
||||
# vulnerable deps at PR time). This workflow fills the scheduled-drift gap.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Weekly, Mondays at 12:30 UTC.
|
||||
- cron: "30 12 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
scan-scheduled:
|
||||
uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
with:
|
||||
# Advisory-first: surface results in code scanning without blocking.
|
||||
fail-on-vuln: false
|
||||
@@ -5,8 +5,8 @@ on:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
permissions: {}
|
||||
|
||||
|
||||
jobs:
|
||||
# Build the existing .skill artifact (Claude Code / Codex / Cursor install
|
||||
@@ -14,19 +14,29 @@ jobs:
|
||||
# so the .mcpb matrix can run in parallel.
|
||||
build-skill:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Attest .skill artifact provenance
|
||||
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
|
||||
with:
|
||||
subject-path: dist/last30days.skill
|
||||
|
||||
- name: Upload skill artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: last30days-skill
|
||||
path: dist/last30days.skill
|
||||
@@ -36,6 +46,13 @@ jobs:
|
||||
# zip layout; we only supply the pre-built binary via --skip-build.
|
||||
build-mcpb:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
env:
|
||||
MCPB_OUTPUT: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
MCPB_PLATFORM: ${{ matrix.platform }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -51,12 +68,20 @@ jobs:
|
||||
platform: linux/amd64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: stable
|
||||
# printing-press v4.8.0 declares `go >= 1.26.3`, newer than the
|
||||
# engine's own floor in mcp/go.mod. Install a 1.26.x toolchain so the
|
||||
# PP `go install` below is satisfied without a runtime toolchain
|
||||
# download (which GOSUMDB=off would block). Building the MCP binary
|
||||
# with a newer toolchain than mcp/go.mod declares is backward-safe.
|
||||
go-version: "1.26"
|
||||
cache: false
|
||||
|
||||
- name: Install printing-press
|
||||
# Pin to a known-good PP release so the bundle command's behavior
|
||||
@@ -77,10 +102,11 @@ jobs:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: "0"
|
||||
RELEASE_VERSION: ${{ github.ref_name }}
|
||||
run: |
|
||||
mkdir -p mcp/build
|
||||
go -C mcp build \
|
||||
-ldflags "-X main.Version=${{ github.ref_name }}" \
|
||||
-ldflags "-X main.Version=${RELEASE_VERSION}" \
|
||||
-o build/last30days-pp-mcp \
|
||||
./cmd/last30days-pp-mcp
|
||||
|
||||
@@ -93,33 +119,50 @@ jobs:
|
||||
printing-press bundle mcp \
|
||||
--skip-build \
|
||||
--binary mcp/build/last30days-pp-mcp \
|
||||
--platform ${{ matrix.platform }} \
|
||||
--output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
--platform "${MCPB_PLATFORM}" \
|
||||
--output "${MCPB_OUTPUT}"
|
||||
|
||||
- name: Attest .mcpb artifact provenance
|
||||
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
|
||||
with:
|
||||
subject-path: ${{ env.MCPB_OUTPUT }}
|
||||
|
||||
- name: Upload .mcpb artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
path: ${{ env.MCPB_OUTPUT }}
|
||||
|
||||
# Gather every platform artifact and attach to one GitHub release.
|
||||
# release-notes generation reads commits since the prior tag.
|
||||
release:
|
||||
needs: [build-skill, build-mcpb]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
# gh release create --verify-tag shells out to git, so the job needs a
|
||||
# checkout with the tag present; without it the step fails with
|
||||
# "fatal: not a git repository".
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: Create GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
dist/last30days.skill
|
||||
dist/last30days-pp-mcp-*.mcpb
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
gh release create "${RELEASE_TAG}" \
|
||||
dist/last30days.skill \
|
||||
dist/last30days-pp-mcp-*.mcpb \
|
||||
--generate-notes \
|
||||
--verify-tag
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
name: Scorecard
|
||||
|
||||
# OpenSSF Scorecard tracks broader repo security-health drift (branch
|
||||
# protection, token permissions, pinned actions, dangerous workflows, CI
|
||||
# tests, maintenance signals) on a schedule, complementing the per-diff
|
||||
# dependency-audit and secret-scan jobs in security.yml.
|
||||
#
|
||||
# Advisory-first: this workflow only measures and publishes a score, and it
|
||||
# never blocks merges. It runs on the default branch (Scorecard needs repo-level
|
||||
# data and a token, so it is not meaningful on PR forks) plus a weekly schedule
|
||||
# so regressions in security health surface even when no code changes.
|
||||
|
||||
on:
|
||||
branch_protection_rule:
|
||||
schedule:
|
||||
# Weekly, Mondays at 07:00 UTC.
|
||||
- cron: '0 7 * * 1'
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
# Top-level token is read-only; the analysis job widens only what it needs.
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
analysis:
|
||||
name: Scorecard analysis
|
||||
runs-on: ubuntu-latest
|
||||
# Job-level permissions fully replace the top-level block (unlisted scopes
|
||||
# default to none), so the reads checkout and Scorecard need are explicit.
|
||||
permissions:
|
||||
# Needed by actions/checkout to clone the repo, and by Scorecard to read
|
||||
# workflow files for its Dangerous-Workflow / Token-Permissions checks.
|
||||
contents: read
|
||||
actions: read
|
||||
# Needed to upload the SARIF results to the code-scanning dashboard.
|
||||
security-events: write
|
||||
# Needed to publish results and obtain a badge (uses OIDC, no secrets).
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run OpenSSF Scorecard
|
||||
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
|
||||
with:
|
||||
results_file: scorecard.sarif
|
||||
results_format: sarif
|
||||
# Publishes results to the OpenSSF REST API for the public badge and
|
||||
# trend tracking. Set to false if maintainers prefer to keep the
|
||||
# score private (the SARIF upload below still works either way).
|
||||
publish_results: true
|
||||
|
||||
# Retain the raw SARIF as a build artifact for offline inspection.
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: scorecard-sarif
|
||||
path: scorecard.sarif
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload SARIF to code-scanning
|
||||
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
with:
|
||||
sarif_file: scorecard.sarif
|
||||
@@ -0,0 +1,89 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
|
||||
# Block known vulnerabilities in the locked Python dependency graph.
|
||||
- name: Run uv audit against locked dependencies
|
||||
run: uv audit --locked
|
||||
|
||||
dependency-review:
|
||||
name: Dependency review
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Review dependency changes
|
||||
uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0
|
||||
|
||||
secret-scan:
|
||||
name: Secret scan
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
# The action derives the commit range from the GitHub event and fails on
|
||||
# verified secrets. Keep output limited to verified findings to avoid noisy
|
||||
# unverified annotations.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
uses: trufflesecurity/trufflehog@00155c9dc586f34d189adc83d3ac2698c2ec551f # v3.95.8
|
||||
with:
|
||||
version: 3.95.5
|
||||
extra_args: --results=verified
|
||||
|
||||
sast-scan:
|
||||
name: SAST scan
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
container:
|
||||
image: semgrep/semgrep@sha256:06938c1f365d3f67b8cedd8bc117607ae64253f88a0e768e9da9408548927dd6 # v1.167.0
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Advisory-first: visibility before enforcement. Semgrep CE scans
|
||||
# the repository with the community rule set (Python, shell, YAML,
|
||||
# JavaScript, Go) to catch source-level security bugs before they
|
||||
# reach production. Set continue-on-error: false once a clean baseline
|
||||
# is confirmed.
|
||||
- name: Run Semgrep SAST scan
|
||||
continue-on-error: true
|
||||
env:
|
||||
SEMGREP_SEND_METRICS: off
|
||||
run: semgrep scan --config=auto
|
||||
@@ -6,21 +6,63 @@ on:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
plugin-contract:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run plugin contract tests
|
||||
run: uv run pytest tests/test_plugin_contract.py tests/test_version_consistency.py
|
||||
- name: Run test suite
|
||||
run: uv run pytest --cov --cov-report=term-missing
|
||||
|
||||
eval:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Score research quality
|
||||
run: uv run pytest tests/eval -x -s
|
||||
|
||||
mcp-tests:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "1.25.5"
|
||||
cache-dependency-path: mcp/go.sum
|
||||
|
||||
- name: Run MCP Go tests
|
||||
run: go test -race ./...
|
||||
working-directory: mcp
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: GitHub Actions Security Analysis with zizmor 🌈
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
zizmor:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor 🌈
|
||||
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
|
||||
+12
@@ -1,5 +1,6 @@
|
||||
# Private benchmark / evaluation artifacts — never push to upstream
|
||||
docs/comparison-results/
|
||||
test-run.log
|
||||
scripts/evaluate-synthesis.py
|
||||
scripts/generate-synthesis-inputs.py
|
||||
fixtures/polymarket_sample.json
|
||||
@@ -7,6 +8,7 @@ docs/v2.1-tweets.md
|
||||
docs/30-day-anniversary-thread.md
|
||||
docs/30-day-anniversary-tweets.md
|
||||
variants/open/references/research.md
|
||||
docs/investigations/
|
||||
|
||||
# OS / tool files
|
||||
.DS_Store
|
||||
@@ -20,6 +22,11 @@ mise.toml
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Local secrets/config. Keep tracked examples if added later.
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Root vendor/ is accidental - real vendored client lives at scripts/lib/vendor/bird-search/
|
||||
/vendor/
|
||||
|
||||
@@ -37,3 +44,8 @@ htmlcov/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
docs/brainstorms/
|
||||
.context/
|
||||
|
||||
/work
|
||||
/print
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"owner": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"description": "Marketplace for the last30days research plugin",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.17.0",
|
||||
"category": "productivity",
|
||||
"source": {
|
||||
"source": "url",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill.git"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"keywords": [
|
||||
"last30days",
|
||||
"last 30 days"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.17.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"email": "mvanhorn@gmail.com",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"last30days",
|
||||
"last 30 days"
|
||||
],
|
||||
"skills": "./skills/"
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
# Hermes install-time scanner/package exclusions for repository-root scans.
|
||||
# Keep the public bundle focused on the runtime skill under skills/last30days/.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
assets/
|
||||
skills/last30days/assets/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
work/
|
||||
print/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
mcp/
|
||||
gemini-extension.json
|
||||
greptile.json
|
||||
pyproject.toml
|
||||
|
||||
# Non-runtime docs, plans, release notes, fixtures, and tests
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
media/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
CONCEPTS.md
|
||||
CONFIGURATION.md
|
||||
CONTRIBUTORS.md
|
||||
HERMES_SETUP.md
|
||||
release-notes.md
|
||||
SKILL-original.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
|
||||
# Dev/eval scripts shipped inside the skill tree but not needed at runtime
|
||||
skills/last30days/scripts/build-skill.sh
|
||||
skills/last30days/scripts/compare.sh
|
||||
skills/last30days/scripts/evaluate_search_quality.py
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
skills/last30days/scripts/setup-pass.sh
|
||||
skills/last30days/scripts/test_device_auth.py
|
||||
skills/last30days/scripts/test-v1-vs-v2.sh
|
||||
skills/last30days/scripts/verify_v3.py
|
||||
|
||||
# Keep visible: optional runtime watchlist/store/briefing feature scripts
|
||||
# (`watchlist.py`, `store.py`, and `briefing.py`).
|
||||
|
||||
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
|
||||
skills/last30days/scripts/lib/vendor/
|
||||
@@ -1 +1,112 @@
|
||||
@CLAUDE.md
|
||||
# last30days Skill
|
||||
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, Grok (xAI), and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition / runtime spec the model reads when the slash command fires
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||
- `CONFIGURATION.md` — user-facing knobs (env vars, flags, per-host install patterns); keep in sync per the rules below
|
||||
- `CHANGELOG.md` — structured release history (launch copy lives in GitHub Releases)
|
||||
- `HERMES_SETUP.md` — install instructions for the Hermes harness specifically
|
||||
|
||||
## Orientation
|
||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
||||
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
|
||||
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
|
||||
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only).
|
||||
# Saves to $LAST30DAYS_MEMORY_DIR when set in shell or ~/.config/last30days/.env;
|
||||
# add --save-dir <path> for a one-off override. Mirrors LAST30DAYS_STORE convention.
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # copies skill into ~/.agents/skills/<name>/ (frozen at install time); re-run to sync working-tree edits — see Rules below
|
||||
|
||||
# Tests (pytest, ~89 files under tests/, configured in pyproject.toml)
|
||||
uv run pytest # full suite
|
||||
uv run pytest tests/test_dedupe_v3.py # single file
|
||||
uv run pytest tests/test_dedupe_v3.py -k some_case # single case
|
||||
uv run pytest --cov # with coverage (skips lib/vendor/)
|
||||
```
|
||||
|
||||
Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` copies the skill into `~/.agents/skills/<name>/` (real directory) and, for harnesses that support symlinked skill dirs, drops a per-host symlink pointing at that copy. **Working-tree edits do NOT propagate automatically** — the `~/.agents/skills/<name>/` copy is frozen at install time. To sync after edits, re-run `npx skills add . -g -y`. For live-edit on a dev machine, replace the install copy with a symlink to the working tree: `ln -sfn "$PWD/skills/last30days" ~/.agents/skills/last30days` (run from the repo root).
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
- Do not reduce `fail_under` in `pyproject.toml` (`[tool.coverage.report]`) without documenting why in the PR. The coverage gate is a floor meant to rise over time, not to be relaxed when new code is under-tested.
|
||||
- Every `lib/*.py` call to `log.source_log(...)` must pass `tty_only=False`. The default is `True`, which silently drops every line when stderr isn't a TTY (Claude Code, Codex, CI, captured output) — turning source observability into invisible failure. Enforced by `tests/test_source_log_visibility.py`.
|
||||
- **CLI-gated optional sources** (Digg via `digg-pp-cli`, YouTube via `yt-dlp`) activate only when `shutil.which` resolves the binary on the **agent subprocess PATH** — not merely when the file exists on disk. First-run setup installs Digg through `@mvanhorn/printing-press-library` (default `$HOME/.local/bin`); Hermes/OpenClaw gateways often need that directory on PATH. Setup must distinguish PATH-visible installs from off-PATH binaries and must not claim "now active" unless the engine gate would pass. See `docs/solutions/integration-issues/digg-cli-agent-path-setup-wizard.md`.
|
||||
- **First-run onboarding is consent-driven, model-led, and host-split.** The setup subprocess does only mechanical work (cookie reads, tool installs, GitHub device-auth, and emitting the engine-owned welcome via `--welcome`) — it cannot prompt, so consent lives in `SKILL.md` Step 0. Two flows avoid model-authored prose that Claude Code folds or the model skips: in the **Modal Flow** the welcome pitch is embedded in the setup modal's question (the AskUserQuestion modal is the only always-fully-visible surface — a separate welcome message or `--welcome` Bash run gets buried behind "ctrl+o to expand"); the **Non-Modal Prose Flow** still uses `last30days.py --welcome` (relayed verbatim) since it has no modal. The GitHub device code is surfaced by a two-command split — `setup --github-start` returns the code fast (foreground, copies to clipboard) and `setup --github-poll` waits for authorization (`setup --github` still chains both for back-compat). Step 0 has TWO branches: a **Claude Code Modal Flow** (the restored v3.0.0 `AskUserQuestion`-driven NUX — welcome, Auto/Manual/Skip, cookie consent, ScrapeCreators offer, `INCLUDE_SOURCES` opt-in, first-topic picker) for hosts with modals, and a **Non-Modal Prose Flow** for hosts without (OpenClaw, Codex, Cursor, Gemini CLI, Grok). Both ask before reading cookies, surface the macOS Full Disk Access fix on permission-denied, and offer the ScrapeCreators GitHub signup (10,000 free calls) on every first run. A successful `setup --github` persists `SCRAPECREATORS_API_KEY` automatically (via `setup_wizard.write_api_key`, 0o600) and masks the key in stdout. Do NOT collapse the modal flow back into a bare silent `setup` call or flatten it to prose-only — the guided modals are the feature (they eroded once and were restored). The onboarding contract is locked by `tests/test_onboarding_contract.py`. The Step 5 source opt-in is two tiers, both comment-enabled: **Recommended** (TikTok + Instagram posts AND top comments, plus YouTube comments — `INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`) and **Everything** (also Threads + Pinterest). Comments are on by default (posts on → comments on for all three platforms); **Threads and Pinterest are the only opt-in extras**, appearing only in the Step 5 Everything option, never in the welcome or the Step 4 offer. Instagram comments are fetched via ScrapeCreators (`/v2/instagram/post/comments`, ranked by `comment_like_count`) with full vote-weighting parity to YouTube/TikTok (a dedicated `_instagram_engagement` carve-out, the `_VOTE_LOG_REFERENCE`/label/threshold entries). The cross-platform "Top Community Comments" list (`render._render_top_comments`) selects **round-robin by within-platform rank** (every platform's #1, then #2, then #3) so a viral platform can't crowd out a smaller one, and drops the per-platform absolute floor so a less-watched video's killer low-vote comment still surfaces.
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
|
||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||
|
||||
## Maintaining CONFIGURATION.md
|
||||
|
||||
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
||||
|
||||
Update `CONFIGURATION.md` when:
|
||||
|
||||
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
||||
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Grok, Hermes…)
|
||||
- adding a new optional source that requires its own credential
|
||||
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
||||
|
||||
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
|
||||
|
||||
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
||||
|
||||
## Plugin manifests (Grok)
|
||||
|
||||
The repo doubles as a native Grok Build plugin via `.grok-plugin/plugin.json` + `.grok-plugin/marketplace.json`. Grok also reads `.claude-plugin/*` for compatibility; the native pair is the first-class lane and what an official xAI marketplace listing points at. The self-hosted catalog uses a bare Git URL source (`{"source":"url","url":"https://github.com/mvanhorn/last30days-skill.git"}`) so `grok plugin marketplace add mvanhorn/last30days-skill` tracks HEAD — not a self-referential local `path: "."` (Grok does not enumerate those). Version lockstep with Claude/Codex/Gemini manifests is enforced by `tests/test_plugin_contract.py`. Validate with `grok plugin validate .`.
|
||||
|
||||
## Submitting to the xAI plugin marketplace
|
||||
|
||||
Getting last30days into xAI's official catalog (`xai-org/plugin-marketplace`) is an outbound PR to *their* repo — an index that only points at our source, so nothing of last30days is vendored there. Do this **after** the change you want to ship has merged to `main`: the entry pins a commit that must already exist.
|
||||
|
||||
1. Fork `xai-org/plugin-marketplace` and branch from `main`.
|
||||
2. Get the commit to pin — a full 40-char lowercase SHA; a branch, tag, or short SHA is rejected by their validator:
|
||||
```bash
|
||||
git ls-remote https://github.com/mvanhorn/last30days-skill.git HEAD
|
||||
```
|
||||
3. Add one entry to their `.grok-plugin/marketplace.json` under `plugins[]`, a remote source pinned to that SHA:
|
||||
```json
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"category": "productivity",
|
||||
"source": {
|
||||
"source": "url",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill.git",
|
||||
"sha": "<full-40-char-sha-from-step-2>"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"keywords": ["last30days", "last 30 days"]
|
||||
}
|
||||
```
|
||||
4. Regenerate their component index (never hand-edit it) and validate exactly as their CI does:
|
||||
```bash
|
||||
python3 scripts/generate-plugin-index.py
|
||||
python3 scripts/validate-catalog.py
|
||||
python3 scripts/generate-plugin-index.py --check
|
||||
```
|
||||
5. Open the PR, fill in their template, and wait for code-owner review.
|
||||
|
||||
To roll out a later update in their catalog, bump the pinned `sha` in the existing entry — never open a second, parallel entry.
|
||||
|
||||
Do not confuse this with our own `.grok-plugin/marketplace.json`: that file makes this repo directly addable as a Grok marketplace (`grok plugin marketplace add mvanhorn/last30days-skill`) and uses a **bare URL** source (no SHA) so it tracks HEAD; the xAI entry above lives in *their* repo and uses a **remote** source pinned to a SHA.
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
|
||||
+494
-8
@@ -9,16 +9,502 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Changed
|
||||
|
||||
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
|
||||
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
|
||||
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
|
||||
|
||||
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
|
||||
- Discovery is now a three-command host-judged protocol (SKILL.md LAW 11: "YOU ARE THE JUDGE"): `--discover --nominate-only` writes a nominations bundle and a fenced judging digest, the hosting model writes a judgments file (short names, junk flags, worthiness) and later an angles file, and `--discover --judgments <file>` / `--discover --finalize [--angles <file>]` complete the run. No API key is ever needed for host-judged trending. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
- Discovery protocol runs enrich at the normal-research tier (default depth, 4 workers, `LAST30DAYS_ENRICH_BUDGET_SECONDS` default 450s) instead of the 240s quick sweep; one-shot `--discover` keeps the quick tier unchanged. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
- Displayed discovery ranks now descend by the card's velocity score, and survivors sharing evidence (same top comment or 2+ shared URLs) fold into the higher-velocity story. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
|
||||
- The engine-side discovery LLM judge (`lib/discovery_judge.py` and all reasoning-provider resolution in the discovery path). One-shot cron runs use deterministic heuristic names, velocity-only order, and no angles, with one loud stderr note pointing at the host-judged protocol. Keyed one-shot users lose provider naming/angles by design - the protocol replaces them. ([#856](https://github.com/mvanhorn/last30days-skill/pull/856))
|
||||
|
||||
## [3.17.0] - 2026-07-21
|
||||
|
||||
### Added
|
||||
|
||||
- Discovery trend cards now lead with short judged topic names: a stage-1 LLM judge gives each nominated cluster a 2-6 word searchable name (with a deterministic fallback namer), replacing raw post titles as card headings, and blends a 0-100 content-worthiness score into the ranking. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Junk-shape gate in discovery: help-me / beginner / personal-musing post shapes flagged by the judge (or the deterministic classifier at the seed-source floor) lose the single-source ranking bypass and need cross-source corroboration to rank. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Stage-2 angle pass: every discovery trend card carries engine-owned `**Podcast angle:**` and `**X article angle:**` lines, so the brief doubles as a content-pipeline worksheet. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Persistent discovery topic queue: `--discover` runs record surfaced topics in research.db (on by default; `LAST30DAYS_DISCOVERY_QUEUE=off` opts out, `--mock` never writes, `--save-dir` scopes the store), re-surfaced or covered topics get a `**Pipeline:**` annotation on their card, and `queue list` / `queue cover "<name>"` manage the queue from the CLI. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
- Discovery JSON export schema 1.1: per-topic `podcast_angle` / `x_article_angle` plus the queue fields `previously_surfaced_count`, `last_surfaced`, and `covered` join the discovery export contract; every existing key is preserved. ([#852](https://github.com/mvanhorn/last30days-skill/pull/852))
|
||||
|
||||
## [3.16.0] - 2026-07-15
|
||||
|
||||
### Added
|
||||
|
||||
- YouTube comments now fetch free via yt-dlp first; ScrapeCreators is a failure-only fallback, dropping the paid-key requirement for comment enrichment. ([#827](https://github.com/mvanhorn/last30days-skill/pull/827))
|
||||
- `GITHUB_TOKEN` is registered end-to-end (.env, keychain, setup scripts, doctor) so the GitHub source stops rate-limiting keyed users. ([#793](https://github.com/mvanhorn/last30days-skill/pull/793))
|
||||
- Opt-in overridable per-source result caps for high-volume topics; defaults unchanged when unset. ([#717](https://github.com/mvanhorn/last30days-skill/pull/717))
|
||||
- `OPENROUTER_BASE_URL` override, mirroring the existing OPENAI/XAI base-URL knobs. ([#703](https://github.com/mvanhorn/last30days-skill/pull/703))
|
||||
- `LAST30DAYS_MCP_TIMEOUT` accepts bare integer seconds as documented, not just Go duration strings. ([#765](https://github.com/mvanhorn/last30days-skill/pull/765))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Keyless web search now works on hosts where DuckDuckGo anomaly-blocks the egress IP (a 202 challenge page with no results — common on datacenter/VPS IPs). Added Startpage as a second keyless rung (DuckDuckGo → Startpage → configured SearXNG), so the web floor still returns results there. Also hardened `_strip_html` to drop `<style>`/`<script>` contents so inline CSS can't leak into a title or snippet.
|
||||
- Web/grounding results are no longer discarded when one of them is a reddit.com URL whose enrichment fetch fails. Reddit enrichment is a best-effort secondary fetch; its HTTP failures (e.g. a 403 on a datacenter IP) were being attributed to the whole web source, which then reported "0 items — HTTP 403" despite having retrieved good results. Its failures are now isolated from the source's outcome.
|
||||
- Very long topic names no longer crash `save_output` (ENAMETOOLONG): slugify truncates at 180 chars with a stable hash suffix so distinct topics stay distinct. ([#786](https://github.com/mvanhorn/last30days-skill/pull/786))
|
||||
- Quick depth honors the plan's explicit sources instead of trimming them away. ([#664](https://github.com/mvanhorn/last30days-skill/pull/664))
|
||||
- X search on Windows/Node 24: valid Bird CLI JSON on stdout is trusted even when the process exits non-zero. ([#813](https://github.com/mvanhorn/last30days-skill/pull/813))
|
||||
- 17 `.get(key, 0)` sites are now None-safe, fixing sort/math crashes on stored data with null fields. ([#822](https://github.com/mvanhorn/last30days-skill/pull/822))
|
||||
- Non-ASCII characters in URLs are percent-encoded component-wise before urllib, fixing the latin-1 encode crash. ([#822](https://github.com/mvanhorn/last30days-skill/pull/822), supersedes [#821](https://github.com/mvanhorn/last30days-skill/pull/821))
|
||||
- `LAST30DAYS_DEBUG` is registered and resolved lazily; fixes the `http.DEBUG` AttributeError in xai_x. ([#770](https://github.com/mvanhorn/last30days-skill/pull/770))
|
||||
- `DEGRADED_TRANSCRIPT_THRESHOLD` set in .env is picked up. ([#807](https://github.com/mvanhorn/last30days-skill/pull/807))
|
||||
- One bad video no longer marks the whole ScrapeCreators transcript source failed. ([#830](https://github.com/mvanhorn/last30days-skill/pull/830))
|
||||
- Chromium cookie temp copies keep 0600 permissions for their whole lifetime. ([#764](https://github.com/mvanhorn/last30days-skill/pull/764))
|
||||
- Thin-source retries forward pinned subreddits/hashtags/creators instead of retrying generically. ([#795](https://github.com/mvanhorn/last30days-skill/pull/795))
|
||||
|
||||
## [3.15.0] - 2026-07-14
|
||||
|
||||
### Added
|
||||
|
||||
- `doctor` is now a four-state audit instead of a flat config prediction: every source is grouped into **WORKING** (verified this run, last run, or keyless-always-on), **TURNED ON - UNVERIFIED** (configured/opted-in but no run evidence), **NOT WORKING** (configured but failing, or the last run errored), or **COULD BE ON** (an available capability not yet configured). Each source renders on its own labeled line, so GitHub (and every other source) is no longer buried in a cluster.
|
||||
- `doctor --postmortem`: reads the last run's `last-report.json` (any age, labeled) and reports, per source, what actually happened - Failed / Partial / Succeeded / Skipped with details and fix hints - so "what broke on that run?" is answerable after the fact.
|
||||
- `doctor --probe`: a bounded live test that verifies WORKING instead of guessing. It also auto-fires when there is no fresh run. Each source is probed concurrently under a per-source deadline (`LAST30DAYS_DOCTOR_PROBE_TIMEOUT`, default 10s) so a slow source can never hang the command. Scope is free HTTP endpoints + keyless CLIs only; credit-gated sources (X, TikTok, Instagram, Threads, …) are never live-probed and stay UNVERIFIED.
|
||||
- `doctor` now surfaces **CLI health**: sources needing a downloaded binary (`yt-dlp`, `digg-pp-cli`, `techmeme-pp-cli`, `arxiv-pp-cli`, `trustpilot-pp-cli`, optional `gh`) carry an inline `[CLI: name ✓]` marker and a dedicated CLI-health block, visibly distinct from keyless sources.
|
||||
- `doctor` now audits **techmeme, arXiv, and trustpilot** (they run in research but were previously absent from the health surface), and surfaces **backup lanes** (Reddit ScrapeCreators backfill, YouTube SC transcript/search backstop used when yt-dlp is rate-limited, X cookie-vs-`XAI_API_KEY` dual path) and **comment lanes** (youtube/tiktok/instagram) as indented sub-lines.
|
||||
- `doctor --json` gains `audit_state`, `cli`, `backups`, `comments`, and `run_outcome` per source plus a top-level `mode`, all additive - every existing key is preserved.
|
||||
|
||||
### Fixed
|
||||
|
||||
- `doctor` no longer reports Threads as Ready when it will not run: SC-gated opt-in sources now honor `INCLUDE_SOURCES` (mirrors the correct LinkedIn gating), so Threads shows COULD BE ON until opted in. TikTok/Instagram stay on-by-default.
|
||||
|
||||
## [3.14.0] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Global trending: bare `--discover` (no domain) sweeps every river feed's own hot list (r/all, Hacker News front page, Digg) with no keyword gate - `/last30days trending` now works. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
- Discovery is now two-stage: a listing sweep nominates candidate topics, then each nomination gets a full research pass (Reddit with comments, X, YouTube, Techmeme, arXiv, HN, Polymarket, web) before ranking - Techmeme and arXiv reach discovery for the first time, and every trend card can carry a verbatim community-voice quote with attribution plus a cross-source corroboration badge. `--discover-shallow` skips the research passes for a faster, thinner sweep. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
- Discovery confidence floor: every topic must clear cross-source confirmation or a genuinely strong single-source spike; when nothing clears, the run reports an honest "Nothing solid this window" (JSON `outcome: nothing-solid` with the closest `weak_signal` named) instead of ranking noise. The discovery JSON contract gains `outcome`, `weak_signal`, and per-topic `top_comment` / `corroboration_count`. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Discovery no longer emits ranked junk on quiet or over-broad domains (the "sports" sweep that returned five 1-like tweets): sub-floor evidence never ranks. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
- An explicit `--search` source boundary now holds through discovery's research passes, not just the listing sweep; `--discover-shallow` without `--discover` errors instead of silently running a full research pass; enrichment stragglers can no longer keep the process alive past the wall-clock budget. ([#816](https://github.com/mvanhorn/last30days-skill/pull/816))
|
||||
|
||||
## [3.13.1] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Doctor `library` line: reports how many saved research briefs the local library holds (cheap glob, never a full parse), so the report's "From your library" block is explained on the health surface. The block itself now carries a one-line explainer with the `LAST30DAYS_LIBRARY_CONTEXT=off` opt-out. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Doctor no longer reports X as `Off` when the bird CLI plus browser-cookie consent serve X fine at runtime: the cookie-backed path now reads **Ready**, with an honest note that the session is verified only at run time and `XAI_API_KEY` is the key-backed alternative. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
- Doctor's YouTube note no longer reads as broken when yt-dlp is healthy: it affirms search + transcripts work, scopes the transcription key to caption-free videos, and correctly attributes comment text to ScrapeCreators (key + `youtube_comments` opt-in) with an actionable fix line - never to yt-dlp. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
- Doctor's Web line on Claude Code now says host-native web search is active instead of `degraded ... keyless`, and names the host rather than an env var the user never set. Messaging only; engine web behavior unchanged. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
- The report footer no longer prints `no results` lines for zero-item sources; failure signal stays in the Source Coverage / Partial Coverage evidence blocks, and the `Raw results saved to` line still renders when every source is empty. ([#815](https://github.com/mvanhorn/last30days-skill/pull/815))
|
||||
|
||||
## [3.13.0] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Xiaohongshu (RED) documented as a first-class requested-only source, with auto-detection of a logged-in local browser-session service: last30days probes `http://localhost:18060` then `http://host.docker.internal:18060` when the source is opted in; `XIAOHONGSHU_API_BASE` remains the explicit override. Zero probing and zero behavior change for users who have not opted in. ([#766](https://github.com/mvanhorn/last30days-skill/pull/766), thanks @yuzhiyang1)
|
||||
- DripStack as an opt-in source: premium financial newsletter and analyst-writeup search (free public API, no key), complementing StockTwits retail sentiment and Polymarket odds with professional analyst signal. Ships default-off; requests route through the shared HTTP layer and honor the 30-day window. ([#791](https://github.com/mvanhorn/last30days-skill/pull/791), thanks @zimoo354)
|
||||
- Persistent opt-in for both new sources via `INCLUDE_SOURCES=xiaohongshu` / `INCLUDE_SOURCES=dripstack` in `.env`, matching the LinkedIn/Perplexity pattern; per-run `--search` still works. ([#812](https://github.com/mvanhorn/last30days-skill/pull/812))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Whitespace in comma-separated `INCLUDE_SOURCES` values no longer silently breaks any source's persisted opt-in. ([#812](https://github.com/mvanhorn/last30days-skill/pull/812))
|
||||
- DripStack article bodies (subtitle/lede) now reach ranking and synthesis instead of only the capped snippet; the Xiaohongshu doctor prescription no longer recommends an env pin that disables auto-probing. ([#811](https://github.com/mvanhorn/last30days-skill/pull/811))
|
||||
- Release hygiene: SKILL.md body header and uv.lock are regenerated with the version bump (both were missed in the 3.12.0 cut and hotfixed on main).
|
||||
|
||||
## [3.12.0] - 2026-07-12
|
||||
|
||||
### Added
|
||||
|
||||
- Typed per-run source outcomes: every run records what actually happened per source (`ok`, `no-results`, `partial`, `rate-limited`, `auth-failed`, `unreachable`, `timeout`, `schema-drift`, `skipped-unconfigured`, `error`) in `source_status`, with doctor-aligned states and fix hints - silence is never mistaken for coverage. ([#797](https://github.com/mvanhorn/last30days-skill/pull/797))
|
||||
- Versioned agent JSON export profile: `--emit=json --json-profile=agent` returns a stable machine contract (`schema_version` 1.2) with `source_status`, clusters, ranked results with joinable `candidate_id`, and freshness verdicts; `--json-profile=raw` keeps the legacy dump byte-identical. ([#798](https://github.com/mvanhorn/last30days-skill/pull/798), [#810](https://github.com/mvanhorn/last30days-skill/pull/810))
|
||||
- Research-quality eval harness: recorded-fixture regression suite scoring runs on citation grounding, recency compliance, cluster coherence, coverage, and determinism against per-fixture floors, in CI. ([#799](https://github.com/mvanhorn/last30days-skill/pull/799))
|
||||
- `--drill`: re-research one cluster of the cached report in depth without a full re-run. ([#800](https://github.com/mvanhorn/last30days-skill/pull/800))
|
||||
- `--discover`: topic-less trending sweeps over listing feeds with velocity-ranked story clusters and ready-to-run research commands. ([#801](https://github.com/mvanhorn/last30days-skill/pull/801))
|
||||
- `library feed`: renders every saved brief into a browsable HTML library with a topic-grouped index and a subscribable Atom feed; hand-written pages are preserved with backups. ([#802](https://github.com/mvanhorn/last30days-skill/pull/802))
|
||||
- `library search`: SQLite FTS5 full-text search across saved briefs and store sightings, plus a passive "From your library" section when new runs overlap past research; scoped `--save-dir` libraries stay fully isolated from the shared store. ([#803](https://github.com/mvanhorn/last30days-skill/pull/803))
|
||||
- `--register` audience templates: `exec`, `dev`, and `creator` presets reshape section order and budgets for the reader; `eli5` is unified into the same mechanism. ([#804](https://github.com/mvanhorn/last30days-skill/pull/804))
|
||||
- `--verify-freshness`: typed per-claim act-time verdicts (`current` / `stale` / `contradicted` / `unsupported`) with point re-fetch of Polymarket lines, GitHub stars, and StockTwits sentiment, inline or post-hoc over the cached report; closes the recency-promise audit gap. ([#805](https://github.com/mvanhorn/last30days-skill/pull/805), closes [#769](https://github.com/mvanhorn/last30days-skill/issues/769))
|
||||
- `--corpus`: register local directories as a private, offline, deterministic source; matching notes rank alongside social evidence under a LOCAL ONLY badge and are excluded from hosted publishing and agent JSON by default. ([#808](https://github.com/mvanhorn/last30days-skill/pull/808))
|
||||
- Native Grok Build (xAI) plugin and marketplace lane: `.grok-plugin/plugin.json` + `.grok-plugin/marketplace.json` so `grok plugin install mvanhorn/last30days-skill` and `grok plugin marketplace add mvanhorn/last30days-skill` work as first-class install paths. The self-hosted catalog uses a bare Git URL source (tracks HEAD); submitting to the official `xai-org/plugin-marketplace` remains a post-merge SHA-pinned outbound PR documented in `AGENTS.md`.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Session-start hook no longer deadlocks under Homebrew bash 5.3: removed every heredoc from `check-config.sh` (bash 5.3 can block forever in `heredoc_write` inside command substitution). ([#809](https://github.com/mvanhorn/last30days-skill/pull/809))
|
||||
- Trustpilot transient-error retries keep their domain parameters. ([#794](https://github.com/mvanhorn/last30days-skill/pull/794))
|
||||
- Hosted same-day saves no longer overwrite earlier reports, and `save_output` never silently overwrites date-stamped files. ([#784](https://github.com/mvanhorn/last30days-skill/pull/784), [#785](https://github.com/mvanhorn/last30days-skill/pull/785))
|
||||
- `.env` reads as UTF-8 (with BOM tolerance and locale fallback) on Windows. ([#780](https://github.com/mvanhorn/last30days-skill/pull/780), [#715](https://github.com/mvanhorn/last30days-skill/pull/715))
|
||||
- `FUN_LEVEL` and `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` are registered in `env.py` so `.env` values are no longer silently ignored; doctor detects `GITHUB_TOKEN` from the process environment. ([#708](https://github.com/mvanhorn/last30days-skill/pull/708), [#732](https://github.com/mvanhorn/last30days-skill/pull/732), [#782](https://github.com/mvanhorn/last30days-skill/pull/782))
|
||||
- File descriptors close promptly across the engine (`open()` wrapped in `with`). ([#775](https://github.com/mvanhorn/last30days-skill/pull/775))
|
||||
|
||||
## [3.11.0] - 2026-07-05
|
||||
|
||||
### Added
|
||||
|
||||
- `last30days doctor`: a unified health command that aggregates every source's probe state into a single grouped report with copy-pasteable fix prescriptions. Layered design: dependency probes (missing/broken/timeout detection), backend-chain descriptors (predict-then-report, never a network call), a centralized prescription registry shared by doctor and quality nudges, and an aggregator with grouped rendering. Replaces the fragmented health knowledge previously spread across `--diagnose`, `--preflight`, `lib/health.py`, and post-run nudges. ([#753](https://github.com/mvanhorn/last30days-skill/pull/753))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Techmeme: `search` results are now windowed to each record's own ISO date instead of stamping every record with today's date, so years-old archive headlines can no longer surface as current news. Dated in-window records take result-cap slots first; undated records (old `techmeme-pp-cli` binary or upstream markup change) degrade gracefully with a logged upgrade hint. The sync machinery is removed because `search` never read the local cache. ([#752](https://github.com/mvanhorn/last30days-skill/pull/752))
|
||||
- LinkedIn now renders in the emoji-tree footer (👔 with likes/comments), the `## Stats` engagement summary, and with the correct "LinkedIn" label. Previously LinkedIn items were counted in `## Stats` but silently dropped from the footer because `_FOOTER_SOURCES`, `ENGAGEMENT_DISPLAY`, and `SOURCE_LABELS` all omitted the source - an 8-item LinkedIn run looked like the source never ran. ([#758](https://github.com/mvanhorn/last30days-skill/pull/758))
|
||||
|
||||
## [3.10.0] - 2026-07-04
|
||||
|
||||
### Added
|
||||
|
||||
- Instagram comments as a first-class ScrapeCreators source: `instagram.enrich_with_comments` fetches top comments via `GET /v2/instagram/post/comments` (ranked by `comment_like_count`), gated by `SCRAPECREATORS_API_KEY` + `instagram_comments` in `INCLUDE_SOURCES`. Full vote-weighting parity with YouTube/TikTok - a dedicated `_instagram_engagement` gives IG posts the same top-comment ranking carve-out, and IG comments render with a "likes" label. ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
|
||||
- Comments are now on by default: the first-run Step 5 Recommended tier enables top comments for TikTok, Instagram, and YouTube (`INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`); the Everything tier adds Threads + Pinterest. Comments were previously an opt-in "Everything" feature. ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
|
||||
|
||||
### Changed
|
||||
|
||||
- The cross-platform "Top Community Comments" list now selects **round-robin by within-platform rank** (every platform's #1, then #2, then #3) instead of a global vote-magnitude sort, so the top-3-of-each-platform outranks the 4th-of-any and each platform's #1 is guaranteed a slot - a viral platform can no longer sweep the list. The list also drops the per-platform absolute vote floor so a less-watched video's high-signal low-vote comment still surfaces (the per-candidate card keeps its floor). ([#751](https://github.com/mvanhorn/last30days-skill/pull/751))
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run wizard: the welcome pitch is embedded directly in the setup modal (the only always-visible surface) instead of a separate `--welcome` message that Claude Code folds behind "ctrl+o to expand"; the cookie-consent and ScrapeCreators-offer copy now name every installed CLI (yt-dlp, Digg, arXiv, Techmeme) and describe the key's real reach (auto Reddit enrichment + YouTube search backstop), with the GitHub device code auto-copied to the clipboard. ([#750](https://github.com/mvanhorn/last30days-skill/pull/750))
|
||||
|
||||
## [3.9.4] - 2026-07-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run wizard: the welcome message and the ScrapeCreators GitHub device code are now engine-driven instead of model-authored, because a real cold run showed the model skipping the welcome and never surfacing the device code no matter how forceful the SKILL.md prose. The welcome is printed by a new `last30days.py --welcome` command that Step 1 relays verbatim (single source of truth; it can't be skipped or drift), and the GitHub device flow is split into `setup --github-start` (submits, copies the code to the clipboard, prints it to stdout, opens the browser, returns immediately) and `setup --github-poll` (waits for authorization and persists the key). The one-shot `setup --github` still chains both. The code now always appears in the command output, and the "on your clipboard" claim is only made when the copy actually succeeded. ([#748](https://github.com/mvanhorn/last30days-skill/pull/748))
|
||||
|
||||
## [3.9.3] - 2026-07-04
|
||||
|
||||
### Added
|
||||
|
||||
- Optional remote research API backend (env-driven). When both `LAST30DAYS_API_KEY` and `LAST30DAYS_API_BASE` are set in the process environment (never read from `.env`), a search runs through the configured remote endpoint (submit -> poll with stderr progress -> render) instead of local sources; with either unset, behavior is byte-identical to local-only. Opt-in and inert by default (no built-in endpoint); the key is confined to the `Authorization` header and never logged or persisted. Handles the clarify gate and 401/402/429 paths. ([#747](https://github.com/mvanhorn/last30days-skill/pull/747))
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run wizard: the welcome message is now mandated before the setup modal (it was being skipped), the Auto-setup option lists every installed CLI (yt-dlp, Digg, arXiv, Techmeme, not just two), and the ScrapeCreators GitHub signup reliably surfaces the device code with an "it's on your clipboard, just paste" hint as a required step instead of leaving the user staring at a spinner. ([#746](https://github.com/mvanhorn/last30days-skill/pull/746))
|
||||
- ScrapeCreators GitHub signup: an already-linked account whose `.env` is cold no longer fails with the misleading "GitHub auth didn't complete." The `Authorized but failed to fetch API key` case now gets an honest branch (auth worked; the account is likely already linked -- get your key from scrapecreators.com and paste it), and `fetch_api_key` logs the `/profile` response field names (never values) so a full auto-fetch can follow. ([#746](https://github.com/mvanhorn/last30days-skill/pull/746))
|
||||
|
||||
## [3.9.2] - 2026-07-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Trustpilot source returned 0 items on company topics: the engine passed raw topic names to a domain-keyed CLI (`info ThriftBooks` -> HTTP 404) and parallel subqueries raced concurrent Chrome WAF-cookie harvests. Company names now resolve to their Trustpilot review-page domain via the CLI's search (per-topic cache; name-match mandatory, ambiguous cases fall back rather than misattributing another company's reviews), a new `--trustpilot-domain` flag pins the domain explicitly (verbatim, bypasses the brand-shape gate, per-entity `trustpilot_domain` in `--competitors-plan`), the WAF session warms once per 240s window behind a lock at first fetch, Trustpilot is capped to one fetch per run and excluded from the thin-source retry, and headless `--auto-resolve` fills a verified domain hint. SKILL.md Step 0.5d documents the resolution flow. ([#745](https://github.com/mvanhorn/last30days-skill/pull/745))
|
||||
|
||||
## [3.9.1] - 2026-07-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- First-run setup wizard: the browser-cookie scan now tries Chrome/Chromium first (Keychain, no Full Disk Access) before Safari, so macOS users logged into X in Chrome authenticate in ~2s instead of hitting the Safari Full Disk Access dead-end. The winning browser is pinned for later runs only when it is Firefox/Safari, so Chrome never re-triggers the Keychain prompt. Consent copy leads with Chrome and the one-time "Always Allow" cue. ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
|
||||
- ScrapeCreators GitHub signup now surfaces the device code immediately (emitted to stdout so a backgrounded caller shows it at once, instead of a spinner until the process exits), validates the `XXXX-XXXX` code shape before copying/labeling it, short-circuits an already-registered account without a fresh device dance, and masks the API key on every status (not just success). Removed the false "GitHub CLI ~2 seconds, no browser" promise. ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
|
||||
- ScrapeCreators source opt-in is now two real tiers. The Step 5 choices were previously identical — a key auto-ran TikTok, Instagram, Threads, and YouTube comments regardless of `INCLUDE_SOURCES`, and Pinterest's opt-in silently ignored a persisted `INCLUDE_SOURCES`. Threads, YouTube comments, and Pinterest are now genuine `INCLUDE_SOURCES` opt-ins: **Recommended** = TikTok + Instagram + the rate-limit backups; **Everything** = also Threads, Pinterest, and YouTube/TikTok/Instagram comments. "ScrapeCreators backups" is now defined inline (keeps Reddit/YouTube working at rate limits). ([#744](https://github.com/mvanhorn/last30days-skill/pull/744))
|
||||
|
||||
## [3.9.0] - 2026-07-03
|
||||
|
||||
### Added
|
||||
|
||||
- StockTwits as a source, gated to ticker/crypto topics only. Surfaces a retail sentiment ratio (self-reported Bullish/Bearish tags) and message volume on a resolved symbol. Inert on non-financial topics: an unambiguous finance-vocabulary gate (cashtags, "stock", "earnings", "dividend", "crypto", named coins) keeps it from injecting stock chatter into general runs, and it degrades to an empty lane if the public API fails without touching other sources. ([#658](https://github.com/mvanhorn/last30days-skill/pull/658), thanks @wtiwana)
|
||||
- LinkedIn as a source via ScrapeCreators, surfacing articles as high-signal results with date-range filtering, gated behind `INCLUDE_SOURCES`. ([#702](https://github.com/mvanhorn/last30days-skill/pull/702))
|
||||
- arXiv and Techmeme sources (default-on) plus Trustpilot (opt-in). ([#709](https://github.com/mvanhorn/last30days-skill/pull/709))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Runtime preflight now auto-provisions a uv-managed CPython 3.12 on hosts that have `uv` but no system Python 3.12+ (most agent sandboxes), instead of hard-failing the version gate. The install is bounded by a 30s HTTP timeout, matches an existing managed `>=3.12` interpreter before downloading, and announces the one-time ~28MB download on stderr rather than installing silently; hosts without `uv` still get the original clear error. Setup invocations now honor `LAST30DAYS_PYTHON` so first-run setup works on the same hosts. ([#738](https://github.com/mvanhorn/last30days-skill/pull/738), thanks @buntysomroy; setup-interpreter fix adapted from #699 by @SeanGearin)
|
||||
- Setup wizard summary now displays the install status of the arXiv/Techmeme pp_sources CLIs, so users can see whether they landed on PATH. ([#741](https://github.com/mvanhorn/last30days-skill/pull/741), thanks @23241a6749)
|
||||
- `--diagnose` / `--preflight` no longer falsely reports X as unreachable when X auth comes from `FROM_BROWSER` browser cookies. These modes run in `plan_only` and skip cookie extraction for privacy (no Keychain access), so X was dropped from `available_sources` even though a real run authenticates fine. A new side-effect-free `env.x_pending_browser_auth` predicate now reports X as available-pending-browser-auth (and surfaces an `x_pending_browser_auth` flag in `--diagnose`) by keying only on the already-resolved browser list — no cookie is read. Covers every configured browser, including Chrome. ([#692](https://github.com/mvanhorn/last30days-skill/issues/692); first reported and fixed by @23241a6749 in #700)
|
||||
|
||||
### Internal
|
||||
|
||||
- Tightened Hermes `.skillignore` regression coverage: the test now fails if an ignored path is deleted without updating the ignore list, or if a runtime-contract file is accidentally ignored. ([#739](https://github.com/mvanhorn/last30days-skill/pull/739), thanks @SyntaxSawdust)
|
||||
|
||||
## [3.8.3] - 2026-06-25
|
||||
|
||||
### Added
|
||||
|
||||
- Free Reddit gets dedicated-subreddit lanes: entity-home subs (e.g. r/Kanye for "Kanye West", via the new `--dedicated-subreddits` flag) are pulled in full from top+hot+new listings and exempt from the relevance floor, since the whole sub is the topic. Fixes the over-aggressive floor that dropped on-topic posts whose titles lacked the entity name.
|
||||
- `reddit_arctic` resolves upvote counts for threads found only via RSS search (which carries no score) using the free, keyless arctic-shift archive — batched, paced, cached, and graceful-degrading. Reddit now gets headlines-with-points and best-comments-with-points entirely for free, at parity with ScrapeCreators.
|
||||
- `LAST30DAYS_REDDIT_SC_MIN_ITEMS` (default 0 = unchanged empty-only behavior): set above 0 to let the ScrapeCreators backup backfill a thin free Reddit run instead of sitting idle. Backfilled items merge deduped by post id.
|
||||
|
||||
### Removed
|
||||
|
||||
- The permanently-403 `search.json` Tier 0 is gone from the keyless Reddit path; discovery is RSS breadth + shreddit listing partials (real scores) + the dedicated-sub lanes, with no wasted 403 calls.
|
||||
|
||||
## [3.8.2] - 2026-06-25
|
||||
|
||||
### Added
|
||||
|
||||
- Advisory Semgrep SAST scan runs on every push/PR as part of the Security workflow, catching source-level security bugs using Semgrep CE community rules ([#563](https://github.com/mvanhorn/last30days-skill/issues/563))
|
||||
- Scheduled OSV-Scanner vulnerability-drift workflow scans repository lockfiles weekly and uploads SARIF results to GitHub code scanning, catching newly disclosed CVEs in the dependency tree even between PRs ([#571](https://github.com/mvanhorn/last30days-skill/issues/571))
|
||||
- `LAST30DAYS_REDDIT_BACKEND=scrapecreators` makes ScrapeCreators the primary Reddit backend with the public path as fallback. Users with a ScrapeCreators key who were getting shallow public data will now get full nested comment trees by setting this flag ([#589](https://github.com/mvanhorn/last30days-skill/issues/589))
|
||||
- MCP Go tests (`mcp/`) now run in CI on every push/PR alongside the Python test suite, so MCP server regressions are caught before merge ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
- PR dependency review gate blocks merges that introduce new vulnerable dependencies ([#551](https://github.com/mvanhorn/last30days-skill/issues/551))
|
||||
|
||||
### Changed
|
||||
|
||||
- Citations are now renderer-aware (LAW 8). On hidden-link hosts (Claude Code) every citation stays an inline `[name](url)` link as before; on visible-URL hosts (Codex, Cursor, Gemini CLI, raw CLI) citations render as plain source labels so the narrative no longer turns into `label (https://...)` URL soup. The host is detected deterministically from the `CLAUDECODE` environment variable, and full URLs remain reachable through the engine footer and the saved raw file.
|
||||
|
||||
### Fixed
|
||||
|
||||
- The query-plan invocation guidance now warns against wrapping the heredoc in `bash -lc '...'` / `zsh -lc '...'`, whose single quotes terminate at the first apostrophe in a ranking string and abort the engine run with `unmatched "` on Codex. The quoted `<<'PLAN_EOF'` heredoc is already apostrophe-safe; the `-lc` wrapper was the hazard.
|
||||
- Firefox profile detection on Linux now checks `$XDG_CONFIG_HOME/mozilla/firefox` (or its default `~/.config/mozilla/firefox`) in addition to `~/.mozilla/firefox`, fixing cookie extraction on distros that honour the XDG Base Directory Specification ([#667](https://github.com/mvanhorn/last30days-skill/issues/667))
|
||||
|
||||
## [3.8.1] - 2026-06-22
|
||||
|
||||
### Added
|
||||
- **Restored the v3.0.0 first-run NUX wizard (Claude Code Modal Flow).** Step 0 now restores the original guided, `AskUserQuestion`-driven onboarding that eroded over time: a welcome message, an Auto/Manual/Skip setup modal, a cookie-consent modal, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker. It is gated to hosts with modals; hosts without (OpenClaw, Codex, Cursor, Gemini CLI) get the equivalent **Non-Modal Prose Flow**. Digg is threaded into the install messaging alongside yt-dlp everywhere it appears, the ScrapeCreators credit count is `10,000 free calls`, and the flow is locked against re-erosion by `tests/test_onboarding_contract.py`. Builds on the consent-driven foundation from #659/#660. Original wizard captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
- **Consent-driven first-run onboarding.** Step 0 now drives an in-chat consent flow instead of a silent `setup` run: the model asks before reading browser cookies (decline runs with `FROM_BROWSER=off` — still installs yt-dlp + Digg), surfaces the macOS Full Disk Access fix when a cookie read is permission-denied, and offers the ScrapeCreators GitHub signup on every first run. A successful `setup --github` now **persists `SCRAPECREATORS_API_KEY` automatically** (`setup_wizard.write_api_key`, 0o600) and masks the key in stdout so the secret never lands in the host model's captured output. Follows the first-run gate fix (#659).
|
||||
|
||||
### Fixed
|
||||
- **First-run setup no longer runs silently.** The prior Step 0 told the model to run `setup` and "follow the wizard's prompts end-to-end", but the wizard has no prompts — so onboarding extracted cookies, installed tools, and wrote `SETUP_COMPLETE` with zero interaction and never offered the ScrapeCreators signup. Reproduced 2026-06-22 (Fredy Montero, fresh macOS).
|
||||
|
||||
## [3.8.0] - 2026-06-21
|
||||
|
||||
### Added
|
||||
|
||||
- **Single X source with backend failover.** X is now one source backed by an ordered chain of interchangeable backends (xai, bird, xurl, xquik) with runtime failover, rather than separate sources. The key-based xquik backend reaches parity with bird, gaining the X-quality ranking and FROM/ABOUT handle lanes, so hosts that cannot supply browser cookies (OpenClaw, CI/cron, headless harnesses) get real X coverage from an xquik key alone. Handle lanes run via the first handle-capable backend in the chain even when a non-capable backend (xai/xurl) is primary. (#622)
|
||||
|
||||
## [3.7.1] - 2026-06-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- GitHub repo stars are no longer mislabeled as "reactions" in the report footer. Repo cards use a distinct `stars` engagement key, velocity cards use `merged_prs`, and genuine issue/PR reaction counts keep their own `reactions` key. (#645, closes #642)
|
||||
- Hacker News returned zero stories on every run: the Algolia query sent `points>2`, which the HN index no longer accepts as a filterable attribute, so every request 400'd. Dropped the server-side `points` filter; low-engagement demotion still happens at parse time. (#639)
|
||||
- Polymarket surfaced off-topic markets and rendered a mangled footer. The relevance filter was fed the per-subquery string instead of the stable topic, and market labels were truncated mid-article into fragments like "an Anthropic Claude model score at: an 19%". Now filters on the stable topic and cleans the labels. (#640)
|
||||
|
||||
## [3.7.0] - 2026-06-20
|
||||
|
||||
### Added
|
||||
|
||||
- **Direct Perplexity API support.** When `PERPLEXITY_API_KEY` is set it is preferred over OpenRouter for the Perplexity source, unlocking first-party Search API results and async Deep Research. Adds `LAST30DAYS_PERPLEXITY_MODE=sonar|search|both` plus model, search-context, domain/language/country, recency, and reasoning-effort knobs. OpenRouter stays the Sonar compatibility fallback when no direct key is set. Async Deep Research preserves request id, status, idempotency key, poll count, lifecycle timestamps, and failure metadata in raw artifacts. (#629, by @sk-holmes)
|
||||
|
||||
### Changed
|
||||
|
||||
- `check-config.sh` now parses env files in pure bash (no `sed` / `tr`), which also fixes the YouTube-availability hint breaking in minimal environments that lack those tools. (#629)
|
||||
|
||||
## [3.6.1] - 2026-06-20
|
||||
|
||||
### Added
|
||||
|
||||
- **ScrapeCreators transcript fallback.** When `SCRAPECREATORS_API_KEY` is set, YouTube transcripts fall back to the ScrapeCreators transcript endpoint after the keyless yt-dlp cascade fails (fetched server-side, so no 429 / cookies / PO tokens). yt-dlp stays primary and a credit is only spent on a genuine failure, never on success and never on a video proven to have no captions. With a key, yt-dlp also fails over fast (one short-timeout attempt) so a 429 hands off to ScrapeCreators in roughly 17s instead of roughly 90s. (#637, idea from #595)
|
||||
- **YouTube comments default-on.** Comment enrichment now activates whenever a ScrapeCreators key is present (bounded to the top ~3 videos by engagement, ~3 credits per run) instead of requiring `INCLUDE_SOURCES=youtube_comments`. Suppress with `EXCLUDE_SOURCES=youtube_comments`. TikTok/Instagram comments remain `INCLUDE_SOURCES` opt-ins. (#637)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Salvage partial YouTube transcripts on non-zero yt-dlp exit.** With the default `en,es,pt` languages an English video wrote `en.vtt` then 429'd on `es`/`pt`, and the already-written transcript was discarded and retried back into the rate limit. Any VTT on disk is now read before the failure is classified, which fixes the dominant `0/N transcripts` case. (#636)
|
||||
- **Windows transcript crash on subprocess timeout.** Guarded the SIGKILL escalation path in `run_with_timeout` against `os.killpg` / `os.getpgid` raising `AttributeError` on Windows (they are POSIX-only), mirroring the primary path's guard. (#638, reported in #588)
|
||||
|
||||
## [3.6.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- **First-party X posts are no longer buried.** A post authored by one of the run's resolved handles (`--x-handle`, `--x-related`, the GitHub user) is now treated as first-class evidence: it is exempt from the entity-miss demotion (a post never repeats its own author's name, so the body-text grounding check used to zero out the subject's own highest-signal posts) and gets a small authorship credit. Third-party collision-noise suppression is unchanged.
|
||||
- **Engagement rescue for on-topic X posts.** A high-engagement X post that is first-party or entity-grounded gets a `final_score` floor scaled by its engagement percentile within the run's X pool, so a viral on-topic post can't sit at ~0. Off-topic name-collision posts are explicitly excluded.
|
||||
- **First-party interaction signal.** A first-party post directed at another account (a reply / leading @mention) is floated into the visible band regardless of like-count and tagged `interaction:→@handle` in the EVIDENCE block, so the synthesis reads it as a relationship signal rather than low-engagement noise. New **LAW 10** in SKILL.md teaches the model to surface first-party posts and read the interaction tag.
|
||||
|
||||
### Changed
|
||||
|
||||
- The X FROM lane (the subject's own timeline) now pulls up to 8 posts per handle (was 3); the about/related lanes stay modest.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Secrets `.env` and its parent config directory are now auto-tightened to `0o600`/`0o700` after creation, and `check-config.sh`'s `check_perms` now auto-fixes loose permissions with `chmod 600` instead of warning only ([#573](https://github.com/mvanhorn/last30days-skill/issues/573))
|
||||
|
||||
## [3.5.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- **X surfaces tweets FROM and ABOUT a person, both engagement-weighted.** The handle search now pulls the person's real timeline (`from:handle since:`, topic used for ranking only — never AND'd into the query, which previously matched only tweets where they wrote their own name and returned ~0), and a new mention lane (`@handle since:`) surfaces what others say to/about them, excluding their own tweets and deduping against the FROM lane ([#610](https://github.com/mvanhorn/last30days-skill/pull/610)).
|
||||
- **`## Top Community Comments` block.** The engine now surfaces vote-ranked community comments across all candidates (not just the top-cluster representatives), per-platform-normalized, into the EVIDENCE-for-synthesis block, so the funniest/sharpest crowd reactions reach the synthesizing model even when no LLM fun-scorer is available. Paired with a new SKILL.md **LAW 9** that requires weaving ≥2 verbatim attributed comments, copying URLs verbatim, and never narrating the tooling in the deliverable ([#608](https://github.com/mvanhorn/last30days-skill/pull/608)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`--diagnose` honesty.** X status now reflects a real 1-tweet probe (downgrades from green when X is effectively dead; fail-open on a transient timeout) and reports the true auth lane (browser / env / keychain) instead of a hardcoded `env AUTH_TOKEN`. Handle/mention searches log query + result count on success, not only on failure ([#609](https://github.com/mvanhorn/last30days-skill/pull/609)).
|
||||
- **X column de-pollution.** The last-chance keyword retry no longer collapses a multi-word subquery to a bare generic token (e.g. `compound`); it keeps an entity anchor ([#607](https://github.com/mvanhorn/last30days-skill/pull/607)).
|
||||
- **Mandatory person-aware subquery disambiguation.** Collision-prone person names (Kevin Rose vs Kevin Warsh, Lan Xuezhao vs Lanzhou) must anchor every subquery with the resolved company/role/domain context ([#611](https://github.com/mvanhorn/last30days-skill/pull/611)).
|
||||
|
||||
## [3.4.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- **Crowd-vote weighting in the fun judge (Best Takes).** The fun judge now factors how many upvotes/likes each top comment earned. Comment vote counts are fed into the LLM prompt (as traction, not funniness), and Best-Takes selection ranks by an effective score — `fun_score` plus a bounded, per-platform-normalized, relevance-confidence-scaled crowd nudge — so genuinely funny, crowd-loved, on-topic comments surface while off-topic virality and high-voted-but-unfunny rants are excluded. `FUN_LEVEL=medium` stays the default and applies the signal as a meaningful factor ([#592](https://github.com/mvanhorn/last30days-skill/pull/592)).
|
||||
- **Digg added to first-run setup.** The free, keyless `digg-pp-cli` is now auto-installed during the first-run wizard (best-effort via the Printing Press installer, with a recommend-only fallback), so the already-built Digg AI-news source activates automatically for new users instead of silently never appearing ([#590](https://github.com/mvanhorn/last30days-skill/pull/590)).
|
||||
|
||||
- **`LAST30DAYS_YOUTUBE_SSH_HOST` transcript routing** — yt-dlp transcript fetch runs on the remote SSH host via a mktemp + cat pipeline ([#422](https://github.com/mvanhorn/last30days-skill/pull/422)).
|
||||
- Browser-cookie auth for X/Twitter now covers the full Chromium family on macOS - Brave, Microsoft Edge, Vivaldi, Opera, Arc, and Chromium - alongside the existing Chrome, Firefox, and Safari. They all share Chrome's v10 AES-128-CBC decryption, differing only in profile path and Keychain service name, so they run through one shared decryption core. The profile finder probes both the modern `Default/Network/Cookies` layout (Chromium >= 96) and the legacy flat `Default/Cookies`, and Chrome now resolves through that same finder so it picks up the modern layout too. Set `FROM_BROWSER=auto` to try every browser, or `FROM_BROWSER=<name>` (e.g. `brave`, `edge`, `arc`) to target one. Verified end-to-end on real Brave and Edge installs ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
|
||||
- **First-party positioning research + pitch-vs-pulse synthesis (company / product / service topics).** A new mandatory research step captures each entity's current stated positioning from first-party sources (homepage, docs, pricing) rather than from memory. The fetched pitch grounds `What it is` descriptions (entities described as they pitch themselves today), helps reject unrelated brand-name noise, and feeds an evidence-triggered prose beat: when the month's conversation directly supports a specific claim, cuts against one, or is squarely about the pitched ground, the synthesis says so anchored to the top thread — and stays silent when the pulse is orthogonal to the pitch, because a manufactured connection is worse than omission. Claims are tested at matched altitude (specific claims against specific threads; broad taglines are never graded against individual items), and statements stay windowed to the 30 days — no trend verdicts. Scoped to entities with an identifiable first party: people are always excluded (even founders whose companies qualify), as are events, abstract concepts, and ownerless topics like Bitcoin; the beat requires positioning fetched during the run, never from memory.
|
||||
|
||||
### Changed
|
||||
|
||||
- Updated "Unlock X" promo message to mention Chrome/macOS support and Windows Firefox-only limitation instead of generic "Firefox or Safari" ([#387](https://github.com/mvanhorn/last30days-skill/issues/387))
|
||||
|
||||
### Fixed
|
||||
|
||||
- **SSH routing failures no longer present as "0 results"** — `search_youtube` surfaces non-zero SSH exit codes as an explicit `error` field ([#422](https://github.com/mvanhorn/last30days-skill/pull/422)).
|
||||
- `extract_browser_credentials()` silently ignored Brave even though the lower-level `cookie_extract` layer already supported it: `FROM_BROWSER=brave` fell back to Firefox/Safari and `FROM_BROWSER=auto` never tried Brave. The env wiring now passes Brave - and the rest of the Chromium family - through to the extractor ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
|
||||
- Chromium cookie extraction now fetches the macOS Keychain key lazily - only when an encrypted cookie actually needs decrypting. Previously the key was fetched as soon as the cookie DB existed, so `FROM_BROWSER=auto` could trigger a Keychain prompt for every installed Chromium browser. Now only the browser that actually holds the requested cookie prompts ([#572](https://github.com/mvanhorn/last30days-skill/pull/572)).
|
||||
- YouTube transcript budget prioritises recent videos (by a combination of views and recency) instead of views alone, preventing transcript slots from being consumed by old high-view-count videos that would be discarded by strict_recent freshness pruning ([#531](https://github.com/mvanhorn/last30days-skill/issues/531))
|
||||
- YouTube items with successfully extracted transcripts are no longer pruned by title-only relevance scoring; the transcript content proves substantive topical coverage even when the video title has low lexical overlap with the query ([#468](https://github.com/mvanhorn/last30days-skill/issues/468))
|
||||
- First-run setup wizard in SKILL.md now references the existing Python setup wizard (`last30days.py setup`) instead of the missing `nux-wizard.md` file, so first-run setup actually runs on new installs. ([#574](https://github.com/mvanhorn/last30days-skill/issues/574))
|
||||
- `check-config.sh` no longer exits 1 on the ScrapeCreators-configured path when no prior run exists (empty `LAST_RUN_LINE`) — swapped `&&` guard for an `if` block that always exits cleanly ([#463](https://github.com/mvanhorn/last30days-skill/issues/463))
|
||||
- `check-config.sh` no longer exits 1 when a `.env` value contains an unbalanced quote — replaced `xargs` (which interprets quotes) with `sed` for whitespace trimming in `load_env_vars` ([#506](https://github.com/mvanhorn/last30days-skill/issues/506))
|
||||
- X/Twitter `.env` template now includes `CT0` alongside `AUTH_TOKEN` in the example skeleton ([CONFIGURATION.md](CONFIGURATION.md)), and the just-in-time unlock wizard offers AUTH_TOKEN/CT0 cookie entry ([#396](https://github.com/mvanhorn/last30days-skill/issues/396))
|
||||
- `check-config.sh` no longer counts X as an active source when only `AUTH_TOKEN` is set without `CT0` — both cookies are now required to credit X in the source count ([#396](https://github.com/mvanhorn/last30days-skill/issues/396))
|
||||
- Firefox cookie extraction now falls back to scanning non-default profiles when the default profile has no matching X cookies, fixing multi-profile setups where login lives on a non-default profile ([#498](https://github.com/mvanhorn/last30days-skill/issues/498))
|
||||
- `subproc.py` `run_with_timeout()` now guards `os.killpg` / `os.getpgid` with `hasattr`, preventing an uncaught `AttributeError` crash when a subprocess times out on Windows where these functions don't exist ([#527](https://github.com/mvanhorn/last30days-skill/issues/527))
|
||||
- Entity-grounding rerank demotion now keys on the head token of the primary entity instead of requiring the full multi-word phrase as a contiguous substring. A high-engagement on-entity item (e.g. a 323-pt HN thread titled "Stripe is friendly to 'friendly fraud'") is no longer demoted to score 0 on a `Stripe payments` query just because it lacks the trailing search-hint word. The intended demotion still fires for items that never name the brand at all. The keyless Reddit comment-enrichment slot selection (`_slot_priority`), which mirrors this signal, was updated to the same head-token grounding so the two paths stay consistent.
|
||||
- `--plan` / `--competitors-plan` file reads now specify `encoding="utf-8"` and catch `UnicodeDecodeError`, preventing crashes on non-ASCII content like accented entity names on Windows (cp1252). `check_perms()` in `check-config.sh` now skips the POSIX 600-permission check on MSYS/MinGW/Cygwin where `stat` runs in noacl mode. `skill_meta.py` `read_skill_version()` now passes `encoding="utf-8"` so SKILL.md emoji doesn't break version detection on Windows. ([#549](https://github.com/mvanhorn/last30days-skill/issues/549))
|
||||
|
||||
|
||||
## [3.3.2] - 2026-06-06
|
||||
|
||||
### Fixed
|
||||
|
||||
- YouTube transcript extraction now falls back through `en,es,pt` (configurable via `LAST30DAYS_YT_SUB_LANGS`) instead of English-only, so non-English videos with auto-captions in any of those three languages now contribute transcripts to the brief ([#469](https://github.com/mvanhorn/last30days-skill/issues/469))
|
||||
- Keyless Reddit comment enrichment now spends its limited slots on entity-matching posts first (mirroring rerank's entity-miss demotion signal) instead of raw upvote order, so off-topic high-upvote threads from broad subreddits no longer consume the comment budget only to be demoted afterward ([#484](https://github.com/mvanhorn/last30days-skill/pull/484))
|
||||
|
||||
## [3.3.1] - 2026-05-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Removed the redundant `commands/last30days.md` wrapper so the plugin exposes only the skill ([#461](https://github.com/mvanhorn/last30days-skill/issues/461)). Previously the plugin shipped both a command wrapper and the skill under the same name, so `/last30` surfaced two `last30days` entries with two different descriptions. The skill already carries its own `argument-hint`, so the `/last30days <topic>` picker UX is unchanged.
|
||||
- Corrected the README install note that claimed Claude Code dedupes the slash command across install methods; it does not, so having both the marketplace plugin and the `npx skills` copy active shows two entries.
|
||||
|
||||
## [3.3.0] - 2026-05-17
|
||||
|
||||
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
|
||||
|
||||
### Added
|
||||
|
||||
**Emit modes and sources**
|
||||
|
||||
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
|
||||
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
|
||||
|
||||
**Configuration knobs**
|
||||
|
||||
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
|
||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
|
||||
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
|
||||
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
|
||||
|
||||
**Pipeline and storage**
|
||||
|
||||
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
|
||||
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
|
||||
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
|
||||
|
||||
**Tests and CI**
|
||||
|
||||
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
|
||||
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
|
||||
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
|
||||
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
|
||||
|
||||
**Docs**
|
||||
|
||||
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
|
||||
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
|
||||
|
||||
### Changed
|
||||
|
||||
**Install story modernized**
|
||||
|
||||
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
|
||||
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
|
||||
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
|
||||
|
||||
**Skill semantics and multi-harness reframe**
|
||||
|
||||
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
|
||||
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
|
||||
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
|
||||
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
|
||||
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
|
||||
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
|
||||
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
|
||||
|
||||
**Dependencies and tooling**
|
||||
|
||||
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
|
||||
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
|
||||
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
|
||||
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
|
||||
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
|
||||
|
||||
### Fixed
|
||||
|
||||
**Reddit**
|
||||
|
||||
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
|
||||
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
|
||||
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
|
||||
|
||||
**Authentication and credentials**
|
||||
|
||||
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
|
||||
|
||||
**Windows compatibility**
|
||||
|
||||
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
|
||||
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
|
||||
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
|
||||
|
||||
**xAI / X / xurl**
|
||||
|
||||
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
|
||||
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
|
||||
|
||||
**YouTube**
|
||||
|
||||
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
|
||||
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
|
||||
|
||||
**bird_x / HTTP**
|
||||
|
||||
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
|
||||
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
|
||||
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
|
||||
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
|
||||
|
||||
**Planner and sources**
|
||||
|
||||
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
|
||||
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
|
||||
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
|
||||
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
|
||||
|
||||
**Render, version display, hosting paths**
|
||||
|
||||
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
|
||||
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
|
||||
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
|
||||
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
|
||||
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
|
||||
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
|
||||
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
|
||||
|
||||
**Hooks**
|
||||
|
||||
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
|
||||
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
|
||||
|
||||
**Sync and version metadata**
|
||||
|
||||
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
|
||||
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
|
||||
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
|
||||
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
|
||||
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
|
||||
|
||||
### Contributors
|
||||
|
||||
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||
|
||||
- Dave Morin — portable test-harness paths
|
||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||
- Eric Oberhofer — multi-key rotation restored
|
||||
- gujishh — Windows process cleanup
|
||||
- Franco Carballar — Reddit browser-like headers
|
||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||
- Kaustav Mishra — xAI error surfacing
|
||||
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
|
||||
|
||||
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
@@ -45,7 +531,7 @@ Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`,
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
||||
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
|
||||
@@ -1,25 +1 @@
|
||||
# last30days Skill
|
||||
|
||||
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
@AGENTS.md
|
||||
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
# Concepts
|
||||
|
||||
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
|
||||
|
||||
## The package
|
||||
|
||||
### Skill
|
||||
|
||||
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
|
||||
|
||||
### Engine
|
||||
|
||||
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
|
||||
|
||||
### Harness
|
||||
|
||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
||||
|
||||
## Research pipeline
|
||||
|
||||
### Primary entity
|
||||
|
||||
The brand or proper-noun core of a research topic — the topic with its Intent modifier stripped. It is what the research is *about*, as distinct from how the user phrased the search.
|
||||
|
||||
### Intent modifier
|
||||
|
||||
A trailing word or phrase in a topic that expresses what the user wants to know rather than what the topic is ("review", "use cases", "pricing"). Stripped when deriving the Primary entity.
|
||||
|
||||
### Entity grounding
|
||||
|
||||
The check that a candidate item plausibly mentions the Primary entity before final ranking. Grounding keys on the head token (first word) of the Primary entity rather than the full phrase — trailing words are usually search descriptors, so requiring them falsely demotes on-entity items.
|
||||
|
||||
An item that fails grounding receives a decisive entity-miss demotion, designed so engagement cannot rescue off-entity content. Because the demotion is decisive, the grounding bar is deliberately conservative: its failure modes degrade toward "no penalty," never toward burying on-entity signal.
|
||||
|
||||
### Keyless path
|
||||
|
||||
The research flow available with no API keys: source data is gathered by scraping and RSS rather than authenticated APIs, and ranking falls back to local scoring instead of LLM-based reranking. This is the free tier of the Skill; lexical quality safeguards like Entity grounding matter most here, because no LLM is available to judge relevance semantically.
|
||||
|
||||
### Comment-enrichment slots
|
||||
|
||||
The small, depth-dependent budget of Reddit posts whose comments get fetched in the Keyless path. Slot selection is relevance-aware: posts that pass Entity grounding claim slots first, so the budget is not spent on high-engagement posts that final ranking will demote anyway.
|
||||
|
||||
## Discovery
|
||||
|
||||
### Discovery
|
||||
|
||||
The topic-less research mode: instead of researching a named topic, it finds what is worth researching. On a reasoning-model host it runs as a three-leg host-judged protocol: leg 1 sweeps the river listings and writes a nominations bundle, the host judges every Nomination (name, junk, worthiness) into a judgments file, leg 2 resumes from the bundle and runs the Enrichment passes, and leg 3 applies host-written content angles and renders the brief. Headless/cron runs keep the one-shot form - same sweep and enrichment, deterministic heuristics in place of the judge, no angles. Either way every surviving topic must clear the Confidence floor before it is shown. Global Discovery (no domain given) sweeps every river feed's own hot list with no keyword gate; domain Discovery scopes and keyword-gates the sweep.
|
||||
|
||||
### Nomination
|
||||
|
||||
A named candidate topic produced by Discovery's listing sweep: clustered items from the river feeds, given a short searchable name plus a Junk shape flag and a content-worthiness score that blends into its seed rank. On protocol runs the hosting model judges all three via the judgments file - the engine's deterministic heuristics only fill rows the host left absent; on headless one-shot runs deterministic distillation supplies the name and junk flag and no worthiness signal exists. A Nomination is only a candidate - its blended seed rank decides which topics deserve an Enrichment pass and the display order of survivors; the Confidence floor judgment and the displayed velocity score are computed from the enriched evidence, never the seed score. The Nomination's name doubles as its Enrichment pass search query and its research handoff, so naming happens before enrichment, never at render time.
|
||||
|
||||
### Enrichment pass
|
||||
|
||||
A full research-pipeline run executed on one Nomination's topic name during Discovery. This is what gives a trend card the whole multi-source corpus (community comments, prediction markets, keyword-driven sources that have no hot-list of their own) instead of thin listing evidence. Enrichment passes run in parallel against a wall-clock budget; a pass that fails or outruns the budget downgrades its topic to nomination-only evidence, never fails the run.
|
||||
|
||||
### Confidence floor
|
||||
|
||||
The absolute evidence bar every Discovery topic must clear before it may rank: an engagement junk-gate first, then either independent cross-source corroboration or a genuinely strong single-source spike. Topics with a Junk shape get a stricter read: the single-source spike bypass is off, and their corroboration is counted against the seed listing sources the sweep actually found - never the enriched corpus, because an Enrichment pass makes almost any topic look multi-source. The floor is absolute, not relative to the current pool - a relative bar would degrade with the pool, which is the failure it exists to prevent. Its thresholds are deliberately tunable; the behavior contract is only that sub-floor evidence never ranks.
|
||||
|
||||
### Nothing-solid
|
||||
|
||||
The honest empty outcome of a Discovery run in which zero topics cleared the Confidence floor. A first-class result, not an error: the run reports that nothing in the window was strong enough to call a trend, and names the closest sub-floor candidate (the weak signal, preferring a non-junk-shaped one) so the user knows where the signal petered out. Rendering junk instead of Nothing-solid is the named failure this outcome replaced.
|
||||
|
||||
### Junk shape
|
||||
|
||||
A classification applied to a Nomination whose leading item reads as a help-me question, beginner ask, or personal musing rather than a story - the post shapes that engagement alone cannot distinguish from news. Junk shape does not exclude a topic outright; it removes the Confidence floor's single-source bypass so the topic surfaces only with independent seed-source corroboration.
|
||||
|
||||
### Topic queue
|
||||
|
||||
The persistent memory of what Discovery has surfaced: each surfaced topic is recorded per research store, so later runs can annotate repeats ("surfaced Nth time") and the user can mark stories Covered. On by default for every real Discovery run, with an engine toggle to disable; mock runs never write it.
|
||||
|
||||
Identity in the queue is annotate-only: a new topic name that closely matches an earlier row (exact normalized match, else entity overlap) annotates the rendered card but never merges or rewrites rows - a false match costs one noisy line, never a hidden story. Queue annotations always describe the state before the current run, and a failed queue write degrades to a warning; it must never destroy a finished run's output.
|
||||
|
||||
### Covered
|
||||
|
||||
The user-set status on a Topic queue row meaning "I already produced content for this story." Set by marking a topic covered by its exact name; surfaced is the only other status. A resurfacing never un-covers a row, and a new name that fuzzily matches a Covered row is born Covered - so the mark survives the LLM judge renaming the same story across runs instead of silently re-pitching it.
|
||||
|
||||
## Flagged ambiguities
|
||||
|
||||
- "Enrichment" is used for two distinct things: Comment-enrichment slots (fetching comments for already-ranked Reddit posts in the Keyless path) and Discovery's Enrichment pass (a full research run per Nomination). Context disambiguates; prefer the full term when writing.
|
||||
@@ -0,0 +1,607 @@
|
||||
# Configuration
|
||||
|
||||
Everything you can tune in `/last30days` without editing the engine source.
|
||||
Three layers, in order of how often you'll touch them:
|
||||
|
||||
1. **Per-run flags** - what you pass on the command line.
|
||||
2. **Environment variables and `.env`** - what's enabled across all runs.
|
||||
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
|
||||
|
||||
Per-client patterns and the experimental beta channel are at the bottom.
|
||||
|
||||
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
|
||||
|
||||
## Why this document exists
|
||||
|
||||
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
|
||||
|
||||
---
|
||||
|
||||
## Where output is saved
|
||||
|
||||
| Platform | Default path | Override |
|
||||
|---|---|---|
|
||||
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
|
||||
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
|
||||
|
||||
Each run produces one file per topic, slug-named:
|
||||
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
||||
|
||||
### Recommended `.env` entry
|
||||
|
||||
`.env` files don't travel between machines or harnesses, so set `LAST30DAYS_MEMORY_DIR` explicitly in `~/.config/last30days/.env` once per host. The `/last30days` slash command works without it (the SKILL.md wrapper has its own default), but **bare engine invocations** — `python3 scripts/last30days.py ...` from cron jobs, scripts, or agents that bypass the wrapper — silently no-op the file save unless the engine sees the env var. Mirrors the `LAST30DAYS_STORE` env-or-flag convention.
|
||||
|
||||
```bash
|
||||
# ~/.config/last30days/.env (pick ONE — uncomment the line that matches your OS)
|
||||
LAST30DAYS_MEMORY_DIR=~/Documents/Last30Days # POSIX — defaults to this path when unset
|
||||
# LAST30DAYS_MEMORY_DIR=C:\Users\<user>\Documents\Last30Days # Windows
|
||||
# LAST30DAYS_LIBRARY_OWNER=Your Name # Optional Atom feed author
|
||||
# LAST30DAYS_LIBRARY_CONTEXT=off # Disable prior-run context (default: on)
|
||||
```
|
||||
|
||||
The engine's `.env` reader doesn't expand `$HOME` — only the tilde, via `Path().expanduser()` downstream. Use `~/...` or an absolute path; **don't** write the literal string `$HOME/...` into your `.env` (it gets stored verbatim and breaks path resolution).
|
||||
|
||||
**Per-run overrides:**
|
||||
|
||||
- `--save-dir <path>` - one-off output location. **Flag wins over env var.** If neither flag nor env var is set, the engine does not write a file (DB persistence is independent — see `LAST30DAYS_STORE` below).
|
||||
- `--output <file>` - write the rendered output to an exact file path, using the format selected by `--emit`.
|
||||
- `--json-profile {agent,raw}` - select the research JSON shape used with `--emit=json`. `agent` is the default, versioned workflow contract; `raw` preserves the full internal `Report` dump for debugging and power users. See the [JSON export reference](docs/reference/json-export.md).
|
||||
- `--corpus <dir>` - add a local `.md`/`.txt` directory as a private ranked source; repeat the flag for multiple directories. PDFs are extracted only when `pdftotext` is on PATH and otherwise skip with a note. File modification time supplies recency, so the normal research window applies.
|
||||
- `--corpus-all-time` - include relevant registered files whose modification time is older than the current research window. Without this flag, a 30-day run includes only files modified in those 30 days.
|
||||
- `--register {default,exec,dev,creator,eli5}` - shape a standard single-topic Markdown or HTML research brief for its audience. `exec` is decisions-first with five core findings and numbers up top; `dev` gives GitHub, code, and technical signals more room; `creator` leads with hooks, Best Takes, community reactions, and virality metrics; `eli5` keeps the established evidence layout and asks the synthesizing agent for accessible language. Registers do not change retrieval, JSON exports, discovery, drill, library feed/search, or comparison output.
|
||||
- `--discover [domain]` - trending discovery, two-stage: a river-listing sweep NOMINATES candidate topics, then each nomination gets a full research pass (Reddit with comments, X, YouTube, Techmeme, arXiv, HN, Polymarket, web) before ranking. Bare `--discover` (no domain) is **global trending**: every feed's own hot list (r/all rising/top-week, Hacker News front/best, Digg clusters when `digg-pp-cli` is on PATH) with no keyword gate; with a domain, the sweep is category-scoped and keyword-gated, and broad X activity joins when an X backend is authenticated. Every topic must clear a confidence floor (cross-source confirmation or a genuinely strong single-source spike); when nothing clears it the run reports "Nothing solid this window" instead of ranked noise. Run without a positional topic; it is mutually exclusive with `--drill`. `--emit=json` uses the separate versioned discovery contract (now with `outcome`, `weak_signal`, per-topic `top_comment` and `corroboration_count`) documented in the [JSON export reference](docs/reference/json-export.md).
|
||||
- `--discover-shallow` - skip discovery's per-topic research passes and rank on listing evidence only. Faster and thinner; the confidence floor still applies. An explicit `--search` source list bounds both the sweep and the research passes. On a protocol run (below), adding it to the `--nominate-only` leg marks the bundle quick-tier so the resume leg uses the faster shallow research pass.
|
||||
- `--nominate-only` - leg 1 of the three-command host-judged discovery protocol (agent hosts; SKILL.md drives it - one-shot `--discover` stays the scripting/cron form with deterministic topic names and no angles). With `--discover [domain]`: sweep the listings, write the nominations bundle (`discover-nominations.json` in the save dir, TTL one hour) for host judgment, print a judging digest, and stop - no enrichment, no queue writes. A zero-nomination sweep prints the nothing-solid brief directly.
|
||||
- `--judgments <path>` - leg 2: resume from the nominations bundle, applying the host judgments file (`{"bundle_id": "...", "judgments": [{"id", "name", "junk", "worthiness"}, ...]}`, bound to the bundle by `bundle_id`). Runs the per-topic research passes (deep tier by default; budget tunable via `LAST30DAYS_ENRICH_BUDGET_SECONDS` below), writes the pending report (`discover-pending.json`), and prints per-topic angle inputs. Requires `--discover`.
|
||||
- `--finalize` - leg 3: apply optional host angles to the pending report, render the final discovery brief, save artifacts, and record the topic queue (retries are idempotent - the pending file stays in place within its TTL). Offline; requires `--discover`.
|
||||
- `--angles <path>` - optional host angles file for `--discover --finalize` (`{"bundle_id": "...", "angles": [{"id", "podcast", "x_article"}, ...]}`, sentences capped at 200 chars); omitting it ships the brief without angle lines. All three protocol legs must share one `--save-dir` (handoff files live there, else in `~/.config/last30days/`); contract failures (missing/stale/unbound handoff files) exit 2 with the remedy on stderr, and `--mock` protocol legs require `--save-dir` to stay side-effect-free.
|
||||
- `--drill <target>` - deep follow-up over the fresh `~/.config/last30days/last-report.json` cache. Accepts a 1-based index (`--drill "cluster 3"` or `--drill "3"`) or a fuzzy cluster title/entity description. It re-fetches only sources that contributed to the matched cluster, enables their deep comment/transcript enrichment paths, merges/dedupes the evidence, and replaces the cache so drills can chain. Run it without a positional topic; if the cache is absent or expired, run a normal research pass first.
|
||||
- `--verify-freshness` - opt into an act-time verification pass for conservatively extracted, source-grounded claims (Polymarket odds/end dates, GitHub stars, StockTwits sentiment ratios, and explicit status assertions). With a topic, verification runs after research; without a topic, it re-verifies the fresh `last-report.json` cache without repeating research. Verdicts are `current`, `stale`, `contradicted`, or `unsupported` and include evidence timestamps. Set `LAST30DAYS_VERIFY_FRESHNESS=on` in `.env` to make the pass default for normal research runs.
|
||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||
- `--no-browser-cookies` - hard-disable browser-cookie extraction for this run, even when `FROM_BROWSER` is configured. MCP and folder-mode hosts use this for safe defaults.
|
||||
- `--publish-html` - with `--emit=html`, publish the rendered HTML to `ht-ml.app` after local output/save-dir writes. This is explicit opt-in only; pages are public by default.
|
||||
- `library feed` - scan `LAST30DAYS_MEMORY_DIR` plus `~/.local/share/last30days/briefs/`, then write a self-contained `index.html`, valid Atom `feed.xml`, and browser-ready pages under `briefs/`. The index is reverse-chronological and grouped by topic. For direct engine use: `python3 skills/last30days/scripts/last30days.py library feed`; use `--save-dir <path>` to scan and write another library directory.
|
||||
- `library feed --publish` - publish each rendered brief and the HTML index through `ht-ml.app`. The generated `feed.xml` remains a first-class local artifact because this HTML host does not serve Atom with an XML content type. Host the output directory on any static host (for example, GitHub Pages) to make `feed.xml` subscribable. Publishing is explicit opt-in and pages are public by default; public pages may be crawled or indexed.
|
||||
- `library search "<query>"` - incrementally sync `LAST30DAYS_MEMORY_DIR` and `~/.local/share/last30days/briefs/` through the shared library scanner, then run offline SQLite FTS5 across those briefs plus dated per-run sightings in `~/.local/share/last30days/research.db`. Results are grouped by topic run. The sibling search index lives at `~/.local/share/last30days/library.db`; hand edits, renames, and deletes are picked up on sync, and a corrupt index is rebuilt automatically.
|
||||
- `LAST30DAYS_LIBRARY_OWNER=<name>` - optional feed-level Atom author. Defaults to `last30days research library`.
|
||||
- `LAST30DAYS_LIBRARY_CONTEXT=on|off` - controls passive prior-run context on fresh research reports. It defaults to `on`; matching saved research appears in a short `From your library` section. Set `off` to skip the local index read and leave reports unchanged. Mock runs, eval replays, and internal fan-out subruns do not load library context, keeping fixtures deterministic.
|
||||
- `--publish-password <password>` - optional shared password for `--publish-html` or `library feed --publish`. Prefer `LAST30DAYS_PUBLISH_PASSWORD=<password>` instead so the password is not visible in the process list or shell history. Use a unique non-personal password; never reuse the user's own password. The provider's update key is treated as secret and is not written to stdout, HTML, raw output, or `.publish.json` metadata.
|
||||
- `--preflight` - print a human-readable permission preflight. It reports config source, project config trust/ignore state, browser-cookie plan, planned writes, optional commands, source availability, and endpoint overrides without reading browser cookies, writing setup/config/report files, or running research. Add `--emit=json` for the separate machine-readable preflight contract (`--json-profile` does not change it); use `--diagnose` when you need the full source diagnostic JSON.
|
||||
- `--welcome` - print the first-run welcome text (engine-owned; the skill relays it verbatim on first run). Safe: prints and exits, no reads or writes.
|
||||
- `--record-fixtures <dir>` - developer-only, hidden flag that records scrubbed source responses for the offline research-quality eval harness. It writes `<dir>/http.json`; see the [eval reference](docs/reference/eval.md) before recording or committing fixtures.
|
||||
- `setup --github-start` / `setup --github-poll` - the two-command ScrapeCreators GitHub device-auth split. `--github-start` submits the device flow, copies the code to the clipboard, opens the browser, and returns the code immediately (foreground); `--github-poll` waits for you to authorize and persists the key. `setup --github` still runs both in one shot for back-compat.
|
||||
|
||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||
|
||||
Every completed research pass writes a structured `last-report.json` cache beside `last-run.json`. HTML follow-up renders use it so `--emit=html --synthesis-file` can reuse report metadata/footer without fetching sources again; `--drill <target>` uses it as the grounded starting point for targeted re-research; bare `--verify-freshness` updates only the cached report's claim verdicts. Reuse is intentionally short-lived: `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` defaults to `3600` (one hour). Set it to another integer number of seconds to tune the window, or `0` to disable report-cache reuse and post-run follow-ups.
|
||||
|
||||
---
|
||||
|
||||
## First-run onboarding
|
||||
|
||||
On the very first `/last30days` run (no `~/.config/last30days/.env`, or `SETUP_COMPLETE` not set), the skill runs a consent-driven onboarding the model drives in chat. It takes one of two forms depending on the host:
|
||||
|
||||
- **Claude Code Modal Flow** - the restored v3.0.0 guided NUX, used on hosts with `AskUserQuestion` (Claude Code). A welcome message, then modals for Auto/Manual/Skip setup, cookie consent, the ScrapeCreators signup offer, a TikTok/Instagram `INCLUDE_SOURCES` opt-in, and a first-topic picker.
|
||||
- **Non-Modal Prose Flow** - the same work done conversationally on hosts without modals (OpenClaw, Codex, Cursor, Gemini CLI, Grok, raw CLI).
|
||||
|
||||
Both share the same consent points:
|
||||
|
||||
1. **Browser cookies** - the model asks before reading anything. On yes it runs `setup --allow-browser-cookies`, which extracts Firefox/Safari cookies (never Chrome unless `FROM_BROWSER=auto` or a named Chromium browser is explicitly configured) to unlock X/Twitter and other logged-in sources, and installs yt-dlp + the keyless Digg CLI. On no it runs setup without `--allow-browser-cookies` (or with `FROM_BROWSER=off`), which skips all cookie reads and still installs the tools.
|
||||
2. **Full Disk Access (macOS)** - if a cookie read is permission-denied, the model surfaces the System Settings > Privacy & Security > Full Disk Access fix and offers one retry.
|
||||
3. **ScrapeCreators GitHub signup** - offered on every first run (10,000 free calls). On consent it runs `setup --github`, which opens a browser for GitHub device-auth (or registers instantly via the `gh` CLI when installed) and, on success, **persists `SCRAPECREATORS_API_KEY` automatically** (0o600, masked in output) so TikTok, Instagram, and the SC Reddit/YouTube backups activate on the next run. Decline anytime; you can run it later by asking to set up ScrapeCreators. The Step 5 opt-in has two tiers, both comment-enabled: **Recommended** (TikTok + Instagram posts AND top comments, plus YouTube comments — `INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments`) and **Everything**, which also adds Threads + Pinterest. Comments are on by default; Threads and Pinterest are the only opt-in extras.
|
||||
|
||||
Re-run onboarding by deleting `~/.config/last30days/.env`. The mechanical work lives in `scripts/lib/setup_wizard.py`; the consent conversation and both host flows are specified in `skills/last30days/SKILL.md` Step 0. The original v3.0.0 wizard is captured at `docs/reference/old-nux-wizard-v3.0.0.md`.
|
||||
|
||||
---
|
||||
|
||||
## API keys (`.env`)
|
||||
|
||||
The skill reads keys from a `.env` file. Two locations are supported:
|
||||
|
||||
1. **`~/.config/last30days/.env`** at the user level (global default) - loaded by default.
|
||||
2. **`.claude/last30days.env`** in the current project directory (project-scoped) - loaded only when trusted by setting `LAST30DAYS_TRUST_PROJECT_CONFIG=1` in the process environment or global config.
|
||||
|
||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||
|
||||
The project-scoped file is useful for **intentional per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), then opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or `~/.config/last30days/.env`. Folder-mode hosts such as Codex desktop do not trust hidden project config by default, and discovery stops at the git root so unrelated parent folders cannot silently influence runs.
|
||||
|
||||
**`LAST30DAYS_API_KEY`** + **`LAST30DAYS_API_BASE`** - optional remote-API backend. Set BOTH to route research through a remote API endpoint instead of running the local sources: `LAST30DAYS_API_BASE` is the endpoint (there is no built-in default), and `LAST30DAYS_API_KEY` is the bearer key for it. When both are set (and `--mock` is not passed), the engine submits the topic to that endpoint, polls with progress on stderr, and prints the server's report; none of the per-source keys below are used for that run. A configured local corpus is the privacy exception: the engine bypasses the hosted backend and runs locally rather than forwarding file-derived input. Non-default `--register` selections are forwarded with the request so server-side synthesis uses the same audience preset. Leave either unset to run local sources exactly as normal. Unlike the other keys here, these two are read only from the **process environment** (export them in your shell or host config) - they are deliberately not loaded from the `.env` files above, so a project-scoped `.env` can never silently redirect research to a remote endpoint. The remote endpoint does not return the local `Report` needed for the versioned agent JSON profile; use `--emit=json --json-profile=raw` for its existing server-response JSON contract.
|
||||
|
||||
### Local corpus (your files)
|
||||
|
||||
Register persistent directories with `LAST30DAYS_CORPUS_DIRS`. Separate paths with `:` on macOS/Linux (the platform path separator is `;` on Windows):
|
||||
|
||||
```bash
|
||||
# ~/.config/last30days/.env
|
||||
LAST30DAYS_CORPUS_DIRS=~/notes:~/meeting-transcripts
|
||||
# LAST30DAYS_CORPUS_IN_EXPORT=1 # explicit agent-JSON opt-in; off by default
|
||||
```
|
||||
|
||||
The slash-command experience remains primary: ask `/last30days` to include your registered notes. For direct engine scripting or development, the equivalent one-off invocation is:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "MCP servers" \
|
||||
--corpus ~/notes --corpus ~/meeting-transcripts
|
||||
```
|
||||
|
||||
**Privacy:** corpus files are read locally, never sent through a source HTTP client, never forwarded to `LAST30DAYS_API_BASE`, never included in remote reranker/fun-scoring prompts, and do not consume network-source concurrency or retry budget. Matches appear in a badged **From your files** section. Corpus candidates are removed from `--publish-html`, `library feed --publish`, and the versioned agent JSON export by default, including corpus-derived cluster titles and source outcomes. Set `LAST30DAYS_CORPUS_IN_EXPORT=1` only when you intentionally want corpus results in the agent JSON written to local stdout/files. The unversioned `--json-profile=raw` debug dump remains a full local report and can contain corpus text; do not redirect it to an external system unless that is intentional. Extracted text is cached by file mtime in `~/.config/last30days/corpus-cache.json` with mode `0600`; a corpus-bearing `last-report.json` cache is also tightened to `0600`. Delete either cache at any time to clear it.
|
||||
|
||||
**Source-by-source** - what each key unlocks:
|
||||
|
||||
| Source | Key(s) | Required for | Free tier |
|
||||
|---|---|---|---|
|
||||
| Local corpus | `--corpus <dir>` or `LAST30DAYS_CORPUS_DIRS` | private `.md`/`.txt`; `.pdf` when `pdftotext` is on PATH | yes (offline) |
|
||||
| Reddit (public) | none (default); `SCRAPECREATORS_API_KEY` + `LAST30DAYS_REDDIT_BACKEND=scrapecreators` to pin SC primary with public fallback | always on; SC pin requires `SCRAPECREATORS_API_KEY` | yes |
|
||||
| Hacker News | none | always on | yes |
|
||||
| Polymarket | none | always on | yes |
|
||||
| StockTwits | none | auto-on for ticker/crypto topics only (gated by symbol detection); never registered for non-financial topics | yes (public API, ~200 req/hr per IP) |
|
||||
| DripStack | none | opt-in only: per run with `--search dripstack`, or persistently with `INCLUDE_SOURCES=dripstack` in `.env`. Searches premium financial newsletters and analyst writeups via a free, public search API — no key needed. Never active without the opt-in. | yes when opted in (public API, no auth) |
|
||||
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
||||
| YouTube | `yt-dlp` CLI installed; `SCRAPECREATORS_API_KEY` adds a server-side transcript fallback used only when yt-dlp fails (429 / bot-gate) | always on if `yt-dlp` present; SC transcript fallback default-on when key set (no credit spent unless yt-dlp fails) | yes |
|
||||
| YouTube comments | `yt-dlp` CLI installed — **free and keyless, no API key and no opt-in needed**. Falls back to `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` containing `youtube_comments` only when yt-dlp is absent. Suppress with `EXCLUDE_SOURCES=youtube_comments`. | top comments (by likes) on the top ~3 videos by engagement | yes — free via yt-dlp (no credits spent) |
|
||||
| TikTok comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok_comments` (**on by default** — Step 5 Recommended tier) | top comments (by `digg_count`) on the top ~3 TikTok posts | ~3 calls/run; 10K free calls |
|
||||
| Instagram comments | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram_comments` (**on by default** — Step 5 Recommended tier) | top comments (by `comment_like_count`) on the top ~3 Instagram posts, via `/v2/instagram/post/comments` | ~3 calls/run; 10K free calls |
|
||||
| Digg | `digg-pp-cli` on PATH (auto-installed during first-run setup via `npx -y @mvanhorn/printing-press-library@0.1.16 install digg --cli-only`; binary defaults to `$HOME/.local/bin` — Hermes/OpenClaw agent subprocesses must inherit that dir on PATH for Digg to activate; prior pp-digg installs use the same path) | always on if `digg-pp-cli` on PATH | yes (free, keyless, read-only) |
|
||||
| arXiv | `arxiv-pp-cli` on PATH (auto-installed during first-run setup via `npx -y @mvanhorn/printing-press-library@0.1.16 install arxiv --cli-only`) | always on if `arxiv-pp-cli` on PATH; fires on research/technical topics and stays quiet otherwise (relevance + 365-day recency gating) | yes (free, keyless) |
|
||||
| Techmeme | `techmeme-pp-cli` on PATH (auto-installed via `... install techmeme --cli-only`) | always on if `techmeme-pp-cli` on PATH; searches Techmeme's live archive and keeps only headlines dated within the research window (undated headlines flow through as low-confidence) | yes (free, keyless) |
|
||||
| Trustpilot | `trustpilot-pp-cli` on PATH (NOT auto-installed; install on demand via `npx -y @mvanhorn/printing-press-library@0.1.16 install trustpilot --cli-only`) + `INCLUDE_SOURCES` contains `trustpilot` | **opt-in, off by default**; when enabled, activates only on company/brand topics — or on any topic when `--trustpilot-domain=<domain>` pins the review page explicitly (bypasses the brand-shape gate; also the per-entity `trustpilot_domain` key in `--competitors-plan`). Bare company names auto-resolve to the review-page domain via the CLI's search. The session warms once before the search fan-out; a stale session does a ~10s headless-Chrome WAF-cookie harvest (set `LAST30DAYS_TRUSTPILOT_NO_BROWSER=1` to disable in cron/CI) | yes (no API key; cookie-replay after the one-time harvest) |
|
||||
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `XQUIK_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; Xquik / xAI / ScrapeCreators = key-based |
|
||||
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
||||
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
||||
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
||||
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
||||
| LinkedIn | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `linkedin` | LinkedIn posts + articles (articles rank as high signal on person topics) | 10K free calls; power-user opt-in, not offered during first-run onboarding |
|
||||
| Xiaohongshu (RED) | logged-in x-mcp browser plugin or `xiaohongshu-mcp` service; optional `XIAOHONGSHU_API_BASE` for custom URLs | requested-only via `--search xhs` or `--search xiaohongshu`; auto-probes `http://localhost:18060` then `http://host.docker.internal:18060` | no last30days API key; depends on your local browser-session service |
|
||||
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
||||
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
||||
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
||||
| Perplexity Sonar / Search API / Deep Research | `PERPLEXITY_API_KEY` (preferred) or `OPENROUTER_API_KEY` (Sonar fallback) | `INCLUDE_SOURCES=perplexity`; `--deep-research` flag (~$0.90/query) | no |
|
||||
| Caption-free transcription | `GROQ_API_KEY` (free tier, preferred) or `OPENAI_API_KEY` (paid backstop); requires `ffmpeg` | Whisper transcription for audio/video without captions (groundwork: module shipped, not yet auto-invoked by the engine) | Groq free tier is generous; needs ffmpeg installed |
|
||||
| Jobs / careers pages | none for public ATS pages; web backend improves fallback discovery | `--hiring-signals` and strong Hiring Signals in standard company reports | yes |
|
||||
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
||||
|
||||
**X on cookie-less hosts.** Bird (the free X source) scrapes X using your logged-in browser cookies (`AUTH_TOKEN`/`CT0`), which agent hosts like OpenClaw, CI, or headless runs often can't supply — and scraping carries some account risk. On those, set `XQUIK_API_KEY` (or `XAI_API_KEY`) for full, ranked X coverage from a single API key: the same engagement-based ranking, first-party authorship, and handle (from/mentions) lanes the native X source gets. `--diagnose` reports whether the key is working (and flags an unpaid key).
|
||||
|
||||
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
||||
|
||||
```bash
|
||||
# Reasoning + planning (one provider; see priority below)
|
||||
GOOGLE_API_KEY=<your-gemini-key>
|
||||
|
||||
# Web search backend (one is enough; Brave is the cheapest)
|
||||
BRAVE_API_KEY=<your-brave-key>
|
||||
|
||||
# Optional sources
|
||||
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
# Xiaohongshu is requested-only: run with --search xhs after starting a local
|
||||
# browser-session service. Defaults probe localhost, then host.docker.internal.
|
||||
# XIAOHONGSHU_API_BASE=http://localhost:18060
|
||||
# Add perplexity to INCLUDE_SOURCES when you want the paid Perplexity source.
|
||||
# PERPLEXITY_API_KEY=<your-perplexity-key>
|
||||
# INCLUDE_SOURCES=tiktok,instagram,perplexity
|
||||
# LAST30DAYS_PERPLEXITY_MODE=sonar # sonar | search | both
|
||||
# LAST30DAYS_PERPLEXITY_MODEL=sonar-pro # sonar | sonar-pro | sonar-reasoning-pro
|
||||
|
||||
# X authentication (one option only)
|
||||
AUTH_TOKEN=<your-auth-token>
|
||||
CT0=<your-ct0-token>
|
||||
# OR xAI API key (paid)
|
||||
# XAI_API_KEY=<your-xai-key>
|
||||
# OR Xquik key-based X search
|
||||
# XQUIK_API_KEY=<your-xquik-key>
|
||||
# OR cookie-jar (free; logs in via your browser session).
|
||||
# Unset = no browser-cookie reads. FROM_BROWSER=auto tries Firefox/Safari and
|
||||
# the Chromium family (Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium); it
|
||||
# only prompts for macOS Keychain access on the browser that actually holds your
|
||||
# X cookies. Or name a single browser, e.g. brave/edge. On Windows only Firefox
|
||||
# is supported.
|
||||
# FROM_BROWSER=firefox
|
||||
|
||||
# Bluesky
|
||||
BSKY_HANDLE=<your-handle>.bsky.social
|
||||
BSKY_APP_PASSWORD=<your-app-password>
|
||||
```
|
||||
|
||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --preflight` for a human permission summary or `python3 scripts/last30days.py --diagnose` for full JSON diagnostics. Both are safe: they report source availability, config source, browser-cookie plan, external command availability, write destinations, and ignored untrusted project config without reading browser cookies or running live provider probes.
|
||||
|
||||
### Perplexity source modes
|
||||
|
||||
Perplexity is a paid opt-in source. A direct `PERPLEXITY_API_KEY` unlocks first-party Perplexity features. `OPENROUTER_API_KEY` remains a Sonar compatibility fallback only; Perplexity Search API and async Deep Research call Perplexity directly.
|
||||
|
||||
`LAST30DAYS_PERPLEXITY_MODE` controls normal `perplexity` source runs:
|
||||
|
||||
| Value | Behavior | Calls |
|
||||
|---|---|---|
|
||||
| `sonar` (default) | Sonar synthesis plus citations. | one Sonar call |
|
||||
| `search` | Raw ranked Search API rows; best when you want source aggregation over prose. | one Search API call |
|
||||
| `both` | Sonar synthesis plus raw ranked Search API rows, deduped by URL. | one Search API call and one Sonar call |
|
||||
|
||||
`--deep-research` ignores `LAST30DAYS_PERPLEXITY_MODE` and uses `sonar-deep-research`. With `PERPLEXITY_API_KEY`, it submits to Perplexity's async Sonar endpoint and polls with a hard wall-clock timeout. The async request uses a deterministic idempotency key derived from the request body. If the request is still running at timeout, fails remotely, or polling hits a transport/rate-limit error after the async id exists, the raw artifact records the async request id, idempotency key, last status, lifecycle timestamps returned by Perplexity, poll count, and timeout/error fields so you can inspect or resume by id outside the run. With only `OPENROUTER_API_KEY`, it keeps the OpenRouter synchronous fallback.
|
||||
|
||||
Perplexity-specific env vars:
|
||||
|
||||
| Env var | Default | Applies to | Notes |
|
||||
|---|---|---|---|
|
||||
| `LAST30DAYS_PERPLEXITY_MODE` | `sonar` | normal Perplexity source runs | `sonar`, `search`, or `both`; `search` and `both` require `PERPLEXITY_API_KEY`. |
|
||||
| `LAST30DAYS_PERPLEXITY_MODEL` | `sonar-pro` | direct Sonar only | Supported: `sonar`, `sonar-pro`, `sonar-reasoning-pro`. `--deep-research` forces `sonar-deep-research`. |
|
||||
| `LAST30DAYS_PERPLEXITY_MAX_RESULTS` | `10` | Search API | Clamped to Perplexity's 1..20 range. |
|
||||
| `LAST30DAYS_PERPLEXITY_SEARCH_CONTEXT_SIZE` | provider default | Search API | `low`, `medium`, or `high`; omitted unless set. |
|
||||
| `LAST30DAYS_PERPLEXITY_SEARCH_MODE` | provider default | direct Sonar | `web`, `academic`, or `sec`. |
|
||||
| `LAST30DAYS_PERPLEXITY_DOMAIN_FILTER` | unset | Search API and direct Sonar | Comma-separated domains, max 20. |
|
||||
| `LAST30DAYS_PERPLEXITY_LANGUAGE_FILTER` | unset | Search API and direct Sonar | Comma-separated ISO 639-1 language codes, max 20. |
|
||||
| `LAST30DAYS_PERPLEXITY_COUNTRY` | unset | Search API | Two-letter country code such as `US`. |
|
||||
| `LAST30DAYS_PERPLEXITY_RECENCY_FILTER` | unset | Search API and direct Sonar | `hour`, `day`, `week`, `month`, or `year`. |
|
||||
| `LAST30DAYS_PERPLEXITY_REASONING_EFFORT` | unset | direct Sonar | `minimal`, `low`, `medium`, or `high`. |
|
||||
| `LAST30DAYS_PERPLEXITY_DEEP_TIMEOUT_SECONDS` | `600` | direct async Deep Research | Wall-clock polling deadline. |
|
||||
|
||||
### Encrypted credential sources (Keychain / pass)
|
||||
|
||||
If you'd rather not keep keys in a plaintext `.env`, the loader has two
|
||||
encrypted sources that decrypt secrets transiently at call time (never written
|
||||
to disk, never logged). Both are **lowest-priority and additive** — an explicit
|
||||
`.env` or process-env value always overrides them, so you can mix and match. The
|
||||
`pass` source is only consulted for keys still missing after the higher-priority
|
||||
sources, so a box that merely has `pass` installed pays no decrypt cost when
|
||||
everything is already in `.env`.
|
||||
|
||||
Effective credential priority is: process env > trusted project config
|
||||
(`.claude/last30days.env`) > global config (`~/.config/last30days/.env`) >
|
||||
macOS Keychain > `pass`(1). The SessionStart status hook also checks for
|
||||
Keychain item **presence** under `last30days-<KEY>` without reading secret
|
||||
values, so a Keychain-only setup is treated as configured instead of showing the
|
||||
first-run welcome again.
|
||||
|
||||
| Platform | Source | Store keys with | Lookup convention |
|
||||
|---|---|---|---|
|
||||
| macOS | Keychain | `scripts/setup-keychain.sh` | service name `last30days-<KEY>` |
|
||||
| Linux / Unix (anywhere `pass` exists, incl. macOS) | [`pass`(1)](https://www.passwordstore.org/) | `scripts/setup-pass.sh` | pass path `last30days/<KEY>` |
|
||||
|
||||
```bash
|
||||
# macOS Keychain
|
||||
./scripts/setup-keychain.sh # interactive; --list / --delete KEY
|
||||
|
||||
# pass(1) — Linux/Unix analog
|
||||
./scripts/setup-pass.sh # interactive; --list / --delete KEY
|
||||
./scripts/setup-pass.sh SCRAPECREATORS_API_KEY # just one key
|
||||
```
|
||||
|
||||
The `pass` source honors `PASSWORD_STORE_DIR`. If your store organizes secrets
|
||||
under a different prefix, point the loader at it with `LAST30DAYS_PASS_PREFIX`
|
||||
(works from your `.env` too, and must match where `setup-pass.sh` wrote them).
|
||||
The prefix is used verbatim, so keep the trailing separator:
|
||||
|
||||
```bash
|
||||
export LAST30DAYS_PASS_PREFIX="secrets/last30days/" # default: last30days/
|
||||
```
|
||||
|
||||
Both sources cover the same key set as the `.env` skeleton above.
|
||||
|
||||
#### Reusing existing macOS Keychain items
|
||||
|
||||
If you already have keys stored under another Keychain naming convention, you
|
||||
can reference them without copying the secret by setting non-secret alias
|
||||
metadata in `LAST30DAYS_KEYCHAIN_ALIASES`. The loader still checks
|
||||
`last30days-<KEY>` first; aliases are fallback lookups only.
|
||||
|
||||
```bash
|
||||
# ~/.config/last30days/.env
|
||||
LAST30DAYS_KEYCHAIN_ALIASES={"XAI_API_KEY":{"account":"keychain-user","service":"existing-xai-api-key"},"BRAVE_API_KEY":"existing-brave-api-key"}
|
||||
```
|
||||
|
||||
Each JSON key must be one of the supported env-var names (`XAI_API_KEY`,
|
||||
`SCRAPECREATORS_API_KEY`, `BRAVE_API_KEY`, etc). A string value means "use this
|
||||
service name with the current user account"; an object can specify both
|
||||
`account` and `service`. Lists are allowed for fallback order:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_KEYCHAIN_ALIASES={"XAI_API_KEY":[{"account":"keychain-user","service":"existing-xai-api-key"},{"service":"last-resort-xai"}]}
|
||||
```
|
||||
|
||||
The alias value contains no secret material; it is safe to keep in `.env` as
|
||||
configuration. The secret itself remains in its original Keychain item and is
|
||||
read directly by the engine process.
|
||||
|
||||
Write `LAST30DAYS_KEYCHAIN_ALIASES` as a single-line JSON value in `.env`.
|
||||
Multiline JSON formatting is not supported because `.env` files are parsed
|
||||
line-by-line.
|
||||
|
||||
### Bluesky app-password format and search host
|
||||
|
||||
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
||||
|
||||
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
|
||||
|
||||
```bash
|
||||
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||
```
|
||||
|
||||
### Default source set (`LAST30DAYS_DEFAULT_SEARCH`)
|
||||
|
||||
By default the engine decides the source set per query (everything available, minus `EXCLUDE_SOURCES`). To pin a **fixed** source set for every run without passing `--search` each time — and without patching `SKILL.md`, which a release would overwrite — set:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_DEFAULT_SEARCH=reddit,x,youtube,hn
|
||||
```
|
||||
|
||||
Accepts the same comma-separated names and aliases as `--search` (`web` → grounding, `hn` → hackernews, `bsky` → bluesky, `xhs` → xiaohongshu). Precedence: an explicit `--search` on the command line always wins; `LAST30DAYS_DEFAULT_SEARCH` applies only when the flag is omitted; when neither is set, per-query behavior is unchanged. `INCLUDE_SOURCES` / `EXCLUDE_SOURCES` keep their existing additive/subtractive roles on whichever set is selected.
|
||||
|
||||
### Audience register (`LAST30DAYS_REGISTER`)
|
||||
|
||||
The default standard brief stays balanced and byte-compatible with prior releases. To keep a named audience preset across runs, set one of the supported values:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_REGISTER=exec # default | exec | dev | creator | eli5
|
||||
```
|
||||
|
||||
An explicit `--register` wins over `LAST30DAYS_REGISTER`; the environment/config value defaults to `default`. Presets are intentionally named and bounded - arbitrary prompt or template files are not accepted. Existing `ELI5_MODE=true` configurations continue to resolve to the `eli5` register when no explicit register is selected, but new configuration should use `LAST30DAYS_REGISTER=eli5`.
|
||||
|
||||
---
|
||||
|
||||
## Reasoning provider priority
|
||||
|
||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||
|
||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||
2. **OpenAI** - `OPENAI_API_KEY` only. Codex ChatGPT auth at `~/.codex/auth.json` is intentionally not used as an OpenAI provider credential.
|
||||
3. **xAI** - `XAI_API_KEY`
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (Sonar fallback for the Perplexity source / `--deep-research`; also usable as a reasoning provider)
|
||||
5. **Local / deterministic** - always available, lowest quality
|
||||
|
||||
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
||||
|
||||
---
|
||||
|
||||
## Web search backend priority
|
||||
|
||||
The search-source preference ladder, strict best-to-floor:
|
||||
|
||||
1. **Host web search** - whatever web-search capability the agent session already has: built-in search, a deferred web-search tool that must be loaded first, or an installed connector such as Brave, Firecrawl, Exa, Serper, or another provider. Best results; used automatically on hosts that have it. A failed lookup for one specific tool name is not fatal when another web-search capability is available. Signalled to the engine via `LAST30DAYS_NATIVE_SEARCH=1` (the skill sets this for you when your agent session has web search) so the engine does not run a worse search underneath it.
|
||||
2. **Paid engine backend** - one of `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY`, auto-detected in that order. Override per-run with `--web-backend=<name>`.
|
||||
3. **Keyless engine floor** - zero-key web search (DuckDuckGo, plus an optional SearXNG instance) and zero-key page fetch (Jina Reader). Runs only when the agent session has **no** host web search **and** no paid key is set, so headless/cron and hosts without a search tool still get general-web coverage. Force it explicitly with `--web-backend=keyless`.
|
||||
|
||||
Relevant env vars:
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_NATIVE_SEARCH=1` | Tells the engine your agent session has host-side web search; suppresses the keyless floor. Set automatically by the skill when web search is available. Leave unset when the agent has no web-search tool so the floor runs. |
|
||||
| `LAST30DAYS_SEARXNG_URL=<base-url>` | Optional. A SearXNG instance used as the keyless-search fallback rung when DuckDuckGo returns nothing. |
|
||||
| `LAST30DAYS_TRUSTPILOT_NO_BROWSER=1` | Optional. Truthy value disables the Trustpilot source's one-time headless-Chrome WAF-cookie harvest, so an automated/headless run (cron, CI, the eval harness) never spawns a browser. Trustpilot still degrades to empty gracefully. |
|
||||
|
||||
Privacy note: the keyless floor sends the query (to DuckDuckGo / your SearXNG instance) and any fetched URL (to Jina Reader) to those third parties. It is intended for public-research use; results may be cached snapshots. It never runs when native search or a paid backend is in play.
|
||||
|
||||
Visible quality difference between hosts with vs without native search or a configured backend. If your client setup produces thinner results than yours, this is usually why.
|
||||
|
||||
---
|
||||
|
||||
### `--hiring-signals` flag
|
||||
|
||||
Use `--hiring-signals` for a focused company hiring-signal report:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "Listen Labs" --hiring-signals
|
||||
```
|
||||
|
||||
The engine treats public jobs/careers postings as evidence of focus or priority shifts, not exact roadmap predictions. Standard company runs may include Hiring Signals automatically when multiple current roles support the same interpretation; weak or unavailable hiring evidence is omitted.
|
||||
|
||||
---
|
||||
|
||||
## Health check (`doctor`)
|
||||
|
||||
One command answers "what could be on, what's turned on, what's working, and what isn't" — a four-state audit (WORKING / TURNED ON - UNVERIFIED / NOT WORKING / COULD BE ON), one line per source, with a CLI-health block for sources that need a downloaded binary, indented backup/comment sub-lanes, the backend the next run will use (for chained sources), and an exact fix on anything that isn't working:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py doctor # four-state audit (text)
|
||||
python3 skills/last30days/scripts/last30days.py doctor --json # machine contract
|
||||
python3 skills/last30days/scripts/last30days.py doctor --cached # serve the cached report while fresh
|
||||
python3 skills/last30days/scripts/last30days.py doctor --postmortem # what actually broke on the last run
|
||||
python3 skills/last30days/scripts/last30days.py doctor --probe # bounded live test (free/CLI sources)
|
||||
```
|
||||
|
||||
Slash-command form: `/last30days doctor`. Reporting problems is a successful run — the exit code is always 0, no browser cookies are read, and no secret values appear anywhere (key presence is booleans only). Backends within a chained source are probed sequentially with a 5-second budget per binary probe, so a chained source's worst-case check time is additive across its backends (only reached when several binaries hang at once).
|
||||
|
||||
`doctor --postmortem` reads the last run's `last-report.json` (any age, labeled) and reports what actually happened per source — Failed / Partial / Succeeded / Skipped, with details and fix hints — so a run that returned less than expected can be diagnosed after the fact. It makes no network calls.
|
||||
|
||||
**Network note:** plain `doctor` with a fresh run, `--cached`, and `--json` make **no** network calls. `doctor --probe` — and a plain `doctor` when there is **no** fresh run to learn from — run a **bounded** live test to verify WORKING instead of guessing. The probe is scoped to free HTTP endpoints (Reddit, Hacker News, Polymarket, GitHub) plus keyless CLIs; credit-gated sources (X, TikTok, Instagram, Threads, …) are never probed, so no ScrapeCreators credits are spent and no auth rate limits are tripped. Each source is probed concurrently under a per-source deadline so a slow source can never hang the command.
|
||||
|
||||
Every live run writes its JSON result to `~/.config/last30days/doctor-cache.json` (beside `last-run.json`; honors `LAST30DAYS_CONFIG_DIR`). `doctor --cached` returns that stored report when it is younger than the TTL, and falls through to a live run — rewriting the cache — when it is stale, absent, or corrupt. The cache also self-invalidates on configuration change: the payload carries a schema stamp plus a fingerprint of non-secret config signals (which credentials are present as booleans, the `LAST30DAYS_X_BACKEND` / `LAST30DAYS_REDDIT_BACKEND` pin values, and `INCLUDE_SOURCES`), so adding or removing a key, changing a pin, or toggling an opt-in source makes the next `--cached` call run live — no raw secret ever enters the fingerprint or the file. Every report also carries `from_cache` (true/false) and `generated_at` (when the report was built), in the `--json` top level and as a final `generated: … (cached|live)` text line, so you can always tell how old a cached answer is. A failed cache write is never fatal — doctor prints a one-line stderr warning and continues. An explicit `doctor` without `--cached` always runs live and refreshes the cache.
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_DOCTOR_TTL` | Freshness window for `doctor --cached`, in **seconds**. Defaults to `900` (15 minutes). `0` makes every `--cached` call run live. |
|
||||
| `LAST30DAYS_DOCTOR_PROBE_TIMEOUT` | Per-source deadline (**seconds**) for `doctor --probe` live checks. Defaults to `10`. Caps each concurrent probe so a slow source cannot hang the command. |
|
||||
| `LAST30DAYS_X_BACKEND` | Pins the X backend (`xai` / `bird` / `xurl` / `xquik`); doctor renders the pin and predicts "will use" accordingly. |
|
||||
| `LAST30DAYS_REDDIT_BACKEND` | `scrapecreators` makes ScrapeCreators the primary Reddit backend; doctor renders Reddit's conditional routing with the pin applied. |
|
||||
|
||||
Web search has **no** env pin — pin it per-run with `--web-backend=<name>` only (see [Web search backend priority](#web-search-backend-priority)).
|
||||
|
||||
### Strict exit for degraded runs
|
||||
|
||||
By default a research run exits `0` even when a source failed mid-run (rate-limited, auth-failed, unreachable, timeout, schema-drift) — the report still renders, with the failure annotated in the per-source footer and a partial-coverage warning. Wrappers that need to distinguish degraded coverage from success (cron briefs, CI, downstream agents) can opt in:
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_STRICT_EXIT` | Truthy (`1`/`true`/`yes`/`on`): the engine exits `3` when any source outcome is neither `ok`, `no-results`, nor `skipped-unconfigured`. A one-line `strict-exit: degraded sources: ...` note goes to stderr. Default (unset): exit `0`, unchanged behavior. |
|
||||
|
||||
Exit codes with the flag on: `0` clean run, `3` completed-but-degraded (report was produced), non-zero others unchanged (hard failures). Same hybrid pattern as `LAST30DAYS_DEBUG` — works shell-exported or in `.env`.
|
||||
|
||||
---
|
||||
|
||||
## Debug mode (`--debug`)
|
||||
|
||||
Add `--debug` to any run to emit verbose `[DEBUG]` log lines to stderr from the source modules (X API, HTTP, etc.). Helpful for diagnosing API errors or unexpected behavior.
|
||||
|
||||
**Always-on alternative:** set `LAST30DAYS_DEBUG=true` in your `.env` or export it from your shell. The flag still works as before; the env var is purely additive — works whether shell-exported or set in `.env`.
|
||||
|
||||
---
|
||||
|
||||
## Trend monitoring (`--store` + watchlist + briefings)
|
||||
|
||||
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
||||
|
||||
### `--store` flag
|
||||
|
||||
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
|
||||
|
||||
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
|
||||
|
||||
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
||||
|
||||
### Discovery topic queue (`LAST30DAYS_DISCOVERY_QUEUE`)
|
||||
|
||||
`--discover` runs remember what they surfaced (table `discovery_topics` in the same research.db). Re-surfaced topics get a `**Pipeline:**` line on their card ("surfaced 2nd time", "marked covered") so the discovery brief doubles as a podcast / X-article content pipeline. On by default for real runs; `--mock` runs never write. With `--save-dir`, queue rows land in that directory's scoped `research.db`, never the global one.
|
||||
|
||||
| Var | Effect |
|
||||
| --- | --- |
|
||||
| `LAST30DAYS_DISCOVERY_QUEUE` | Set to `off` to disable queue writes and card annotations. Any other value (or unset) keeps the queue on. Works shell-exported or in `.env`. |
|
||||
| `LAST30DAYS_ENRICH_BUDGET_SECONDS` | Wall-clock budget (seconds) for the deep-tier per-topic research batch on the discovery resume leg (`--discover --judgments <file>`). Default `450`; unset/invalid/non-positive values fall back to it. The one-shot `--discover` path keeps its fixed quick-tier 240s budget regardless. Works shell-exported or in `.env`. |
|
||||
|
||||
Manage the queue from the engine CLI:
|
||||
|
||||
```bash
|
||||
# Uncovered surfaced topics (name, domain, surface_count, last_surfaced, status)
|
||||
python3 skills/last30days/scripts/last30days.py queue list
|
||||
|
||||
# Mark a topic done after you record the episode / publish the article.
|
||||
# Requires the exact topic name; unknown names exit 2 instead of no-opping.
|
||||
python3 skills/last30days/scripts/last30days.py queue cover "Gemma 4 chat templates"
|
||||
```
|
||||
|
||||
Both respect `--save-dir` scoping.
|
||||
|
||||
### `watchlist.py` - recurring topics
|
||||
|
||||
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
||||
|
||||
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
|
||||
|
||||
```bash
|
||||
# 1. Add the topic to the watchlist
|
||||
# Default schedule daily 8am; --weekly switches to Mondays 8am
|
||||
python3 scripts/watchlist.py add "british airways middle east" --weekly
|
||||
|
||||
# 2. Configure delivery and budget (optional)
|
||||
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
|
||||
python3 scripts/watchlist.py config budget 5.00
|
||||
|
||||
# 3. Trigger via cron / Task Scheduler / GitHub Actions
|
||||
python3 scripts/watchlist.py run-one "british airways middle east"
|
||||
# or run every enabled topic, gated by daily_budget
|
||||
python3 scripts/watchlist.py run-all
|
||||
```
|
||||
|
||||
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
|
||||
|
||||
### `briefing.py` - daily / weekly digests
|
||||
|
||||
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
|
||||
|
||||
### Recommended cadence pattern
|
||||
|
||||
| Step | Cadence | Command |
|
||||
|---|---|---|
|
||||
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
|
||||
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
|
||||
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
|
||||
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
|
||||
|
||||
---
|
||||
|
||||
## Per-client patterns
|
||||
|
||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||
|
||||
**Codex note:** the repository includes `.codex-plugin/plugin.json` so Codex can treat the existing
|
||||
`skills/last30days/SKILL.md` tree as plugin metadata without maintaining a separate Codex copy.
|
||||
The Codex marketplace catalog points at the repository root URL: Codex clones the repo, reads the
|
||||
root `.codex-plugin/plugin.json`, and loads skills from `./skills/`. The Agent Skills install
|
||||
command documented in the README remains the broadest cross-host path.
|
||||
|
||||
**Grok note:** the repository includes `.grok-plugin/plugin.json` and `.grok-plugin/marketplace.json`
|
||||
so xAI's Grok Build CLI (`grok`) can install last30days as a native plugin. Grok also reads the
|
||||
Claude Code manifests for compatibility; the native pair is the first-class lane. The Grok
|
||||
marketplace catalog uses a bare Git URL source (no commit pin) so `grok plugin marketplace add
|
||||
mvanhorn/last30days-skill` tracks HEAD — the same pattern as the Codex catalog. `npx skills add`
|
||||
remains a valid cross-host fallback.
|
||||
|
||||
### 1. Trusted per-client `.claude/last30days.env`
|
||||
|
||||
When each client has its own working directory, drop a `.claude/last30days.env` into the client folder and opt in with `LAST30DAYS_TRUST_PROJECT_CONFIG=1` from your shell or global `~/.config/last30days/.env`. The skill loads the project file only after that trust signal. Typical contents:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||
```
|
||||
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
|
||||
### 2. Per-client save dir + suffix wrapper
|
||||
|
||||
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
|
||||
|
||||
PowerShell example:
|
||||
|
||||
```powershell
|
||||
function Run-L30D-Client {
|
||||
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
|
||||
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
|
||||
/last30days @Args --save-suffix=$ClientSlug
|
||||
}
|
||||
# Usage: Run-L30D-Client acme "british airways middle east"
|
||||
```
|
||||
|
||||
Bash example:
|
||||
|
||||
```bash
|
||||
l30d-client() {
|
||||
local client=$1; shift
|
||||
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
|
||||
/last30days "$@" --save-suffix="$client"
|
||||
}
|
||||
# Usage: l30d-client acme "british airways middle east"
|
||||
```
|
||||
|
||||
### 3. Custom category-peer subreddits
|
||||
|
||||
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
|
||||
|
||||
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
|
||||
|
||||
### 4. Pre-built `--competitors-plan` JSON
|
||||
|
||||
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
|
||||
|
||||
```json
|
||||
{
|
||||
"Competitor B": {
|
||||
"x_handle": "competitor_b_handle",
|
||||
"subreddits": ["sub1", "sub2"],
|
||||
"github_user": "competitor-b-org",
|
||||
"context": "Founded 2019, focused on ..."
|
||||
},
|
||||
"Competitor C": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
|
||||
|
||||
---
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
|
||||
|
||||
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
|
||||
|
||||
---
|
||||
|
||||
## Cross-references
|
||||
|
||||
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
||||
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
||||
- Shared package vocabulary and engine/harness terminology: [`CONCEPTS.md`](CONCEPTS.md)
|
||||
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
||||
+2
-1
@@ -23,7 +23,7 @@ v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos
|
||||
### @thinkun
|
||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
|
||||
|
||||
### @thomasmktong
|
||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||
@@ -54,6 +54,7 @@ v3 has durable watchlist with multi-source storage and extended time windows.
|
||||
|
||||
## Past Contributors
|
||||
|
||||
- [@23241a6749](https://github.com/23241a6749) - Windows cp1252 fixes ([#549](https://github.com/mvanhorn/last30days-skill/pull/549)); Windows killpg guard ([#552](https://github.com/mvanhorn/last30days-skill/pull/552)); browser promo clarity ([#387](https://github.com/mvanhorn/last30days-skill/pull/561)); setup wizard fix ([#574](https://github.com/mvanhorn/last30days-skill/pull/578)); check-config xargs fix ([#506](https://github.com/mvanhorn/last30days-skill/issues/506)); check-config clean-exit on missing last-run ([#463](https://github.com/mvanhorn/last30days-skill/issues/463)); Firefox multi-profile cookies ([#498](https://github.com/mvanhorn/last30days-skill/issues/498)); X/Twitter CT0 template ([#396](https://github.com/mvanhorn/last30days-skill/issues/396)); .env permission auto-fix ([#573](https://github.com/mvanhorn/last30days-skill/pull/599)); MCP Go tests in CI ([#621](https://github.com/mvanhorn/last30days-skill/issues/621))
|
||||
- [@JosephOIbrahim](https://github.com/JosephOIbrahim) - Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
|
||||
- [@levineam](https://github.com/levineam) - Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
|
||||
- [@jonthebeef](https://github.com/jonthebeef) - Early testing and feedback
|
||||
|
||||
+8
-4
@@ -4,17 +4,19 @@ This guide covers installing last30days on Hermes AI Agent.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Hermes installed** - See https://github.com/mercurial-tf/hermes
|
||||
1. **Hermes installed** - See https://github.com/NousResearch/hermes-agent
|
||||
2. **Python 3.12+** - `brew install python@3.12` or similar
|
||||
3. **yt-dlp** (optional, for YouTube) - `brew install yt-dlp`
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
hermes skills install mvanhorn/last30days-skill/skills/last30days --force
|
||||
```
|
||||
|
||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
||||
The explicit `skills/last30days` path fetches the skill straight from this repo's current default branch and deploys it under `~/.hermes/skills/`. `--force` is required because Hermes's install-time security scanner returns a `caution` verdict for this skill — it flags benign patterns such as reading your own API keys from the environment and calling `subprocess` to run `yt-dlp`/`bird`. `--force` accepts the caution verdict and installs (it also reinstalls over any existing copy).
|
||||
|
||||
**Why the explicit path?** The shorter `hermes skills install mvanhorn/last30days-skill` currently resolves through the skills.sh index, which is serving an older cached snapshot of this repo (from before the skill moved under `skills/last30days/`). Use the explicit `.../skills/last30days` path above until the index re-crawls — tracked in [vercel-labs/skills#1602](https://github.com/vercel-labs/skills/issues/1602).
|
||||
|
||||
### Developer / live-edit alternative
|
||||
|
||||
@@ -47,11 +49,12 @@ On first run, the skill will guide you through setup:
|
||||
1. **Auto setup** (~30 seconds)
|
||||
- Scans browser cookies for X/Twitter
|
||||
- Checks/installs yt-dlp for YouTube
|
||||
- Best-effort install of `digg-pp-cli` for Digg AI-news clusters (via `@mvanhorn/printing-press-library`; binary lands in `$HOME/.local/bin` — ensure your Hermes gateway PATH includes it, or Digg stays off even after install)
|
||||
- Configures free sources (Reddit, HN, Polymarket)
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
- Adds TikTok, Instagram, Reddit backup
|
||||
- 10,000 free API calls
|
||||
- 100 free credits (no expiration)
|
||||
- Sign up at scrapecreators.com
|
||||
|
||||
3. **Optional: API Keys**
|
||||
@@ -65,6 +68,7 @@ On first run, the skill will guide you through setup:
|
||||
- **Hacker News** - Tech discussions via Algolia
|
||||
- **Polymarket** - Prediction markets
|
||||
- **YouTube** - Search and transcripts (requires yt-dlp)
|
||||
- **Digg** - AI-news story clusters (requires `digg-pp-cli` on the agent PATH; auto-installed to `$HOME/.local/bin` during setup when `npx` is available)
|
||||
|
||||
### Requires API Key
|
||||
- **X/Twitter** - xAI API key or browser cookies
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
# /last30days
|
||||
|
||||
<p align="center">
|
||||
<img src="media/pr-assets/last30days-ad.gif" width="720" alt="last30days - an AI agent-led search engine that searches people, not editors" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mvanhorn/last30days-skill">
|
||||
<img src="https://img.shields.io/badge/%231-Repository%20Of%20The%20Day-6f42c1?style=for-the-badge&logo=github&label=GITHUB%20TRENDING" alt="GitHub Trending #1 Repository Of The Day" />
|
||||
@@ -12,11 +16,12 @@
|
||||
|
||||
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
||||
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
|
||||
**Claude Code (recommended — auto-updates via marketplace):**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
||||
@@ -27,7 +32,7 @@ npx skills add mvanhorn/last30days-skill -g
|
||||
|
||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
||||
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, arXiv, Techmeme, and more in 30 seconds.
|
||||
|
||||
---
|
||||
|
||||
@@ -57,7 +62,7 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
|
||||
| Source | What the people tell you |
|
||||
|--------|--------------------------|
|
||||
| **Reddit** | The unfiltered take. Top comments with upvote counts, free via public JSON. The real opinions that Google buries. |
|
||||
| **Reddit** | The unfiltered take. Top comments with real upvote counts, free, no API key. The real opinions that Google buries. |
|
||||
| **X / Twitter** | The hot take, the expert thread, the breaking reaction. First to know, first to argue. |
|
||||
| **YouTube** | The 45-minute deep dive. Full transcripts searched for the 5 quotable sentences that matter. |
|
||||
| **TikTok** | The creator reaching 3.6M people with a take you'll never find on Google. |
|
||||
@@ -66,13 +71,18 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
||||
| **arXiv** | The papers behind the hype. New research in the window, free, no API key. Auto-enabled when `arxiv-pp-cli` is on PATH (first-run setup installs it). |
|
||||
| **Techmeme** | The tech-news editorial layer, date-windowed to your 30 days. Free, no API key. Auto-enabled when `techmeme-pp-cli` is on PATH (first-run setup installs it). |
|
||||
| **LinkedIn** | The professional signal. Posts and articles, with articles weighted as high signal. |
|
||||
| **StockTwits** | Trader sentiment. Auto-activates when your topic is a ticker or crypto. |
|
||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||
| **Xiaohongshu (RED)** | Chinese lifestyle, product, and creator signals. Requested explicitly with `--search xhs` when a logged-in x-mcp browser plugin or `xiaohongshu-mcp` service is running locally. |
|
||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||
| **Perplexity** | Grounded web search with citations via Sonar Pro. |
|
||||
| **Perplexity** | Grounded Sonar synthesis, raw Search API rows, and Deep Research. |
|
||||
| **Web** | The editorial coverage, the blog comparisons. One signal of many, not the only one. |
|
||||
|
||||
Community contributors keep adding more. Truth Social, Xiaohongshu (RED), and others are in the engine with more on the way.
|
||||
Community contributors keep adding more. Truth Social and other niche sources are in the engine with more on the way.
|
||||
|
||||
A Reddit thread with 1,500 upvotes is a stronger signal than a blog post nobody read. A TikTok with 3.6M views tells you more about what's culturally relevant than a press release. Polymarket odds backed by $66K in volume are harder to argue with than a pundit's guess.
|
||||
|
||||
@@ -82,6 +92,10 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
**Before a meeting.** `/last30days Peter Steinberger` - joined OpenAI's Codex team, fighting Anthropic's ban on third-party agents, 23 PRs merged at 85% merge rate on GitHub, building LobsterOS for cross-device agent control. r/ClaudeCode: "Ever since OpenClaw released, it was widely known that if you run it through anything other than the API, you were gonna get banned eventually" (227 upvotes). That's not on LinkedIn.
|
||||
|
||||
**To read hiring signals.** `/last30days Listen Labs --hiring-signals` - current jobs and careers pages become cited evidence for focus shifts: hiring into enterprise security, customer success, infrastructure, or product expansion. The report says what the hiring appears to signal, not what the roadmap will ship.
|
||||
|
||||
**To find the topic before it peaks.** Ask `/last30days what's exploding in AI agents?` and the skill switches to discovery mode: the engine sweeps Reddit category listings, Hacker News front/best stories, Digg's AI 1000 feed, and X when authenticated; your agent judges the nominations (names, junk filtering, content-worthiness) and writes podcast / X-article angles; then you get 5-10 velocity-ranked topics. Every result includes cross-source numbers, a momentum label, and a ready-to-run `/last30days "<topic>"` follow-up.
|
||||
|
||||
**When something drops.** `/last30days Kanye West` - UK blocked his visa, Wireless Festival canceled, sponsors fled. But BULLY debuted #2 on Billboard. Fantano came back from his "Yay sabbatical" to review it (653K views). SoFi Homecoming brought out Lauryn Hill and Travis Scott for 44 songs. Polymarket: "Will Kanye tweet again?" 86% Yes. 23 Reddit threads, 17 YouTube videos, 86K upvotes.
|
||||
|
||||
**To compare tools.** `/last30days OpenClaw vs Hermes vs Paperclip` - "These aren't competitors, they're layers." OpenClaw is the executor (351K GitHub stars, live), Hermes is the self-improving brain (31K stars), Paperclip is the org chart (49K stars). Star counts pulled live from the GitHub API, not stale blog posts. Side-by-side table with architecture, memory, security, best-for. Per @IMJustinBrooke: "OpenClaw = Charmander, Hermes = Charizard."
|
||||
@@ -92,84 +106,58 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
**To learn something fast.** `/last30days Nano Banana Pro prompting` - JSON-structured prompts are replacing tag soup. @pictsbyai's nested format prevents "concept bleeding." Edit-first workflow beats regeneration. Then it writes you a production prompt using exactly what the community said works.
|
||||
|
||||
## What v3 Changed
|
||||
## What's new
|
||||
|
||||
### Shareable HTML briefs
|
||||
Since the v3.3 announcement in May, as of v3.11.1 (July 2026): 175 merged PRs - 122 of them from 52 community contributors - across 15 releases. This is what landed.
|
||||
|
||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
||||
### First-class on OpenAI Codex
|
||||
|
||||
```
|
||||
/last30days OpenClaw --emit=html
|
||||
```
|
||||
/last30days is now a native Codex plugin with guided setup - not a port, a first-class citizen. Renderer-aware citations mean Codex output reads like a brief instead of URL soup (#694), and the same engine runs on Claude Code, Cursor, Copilot, Gemini CLI, Claude Desktop, OpenClaw, and 50+ Agent Skills hosts. Codex plugin manifest by [@rfoust](https://github.com/rfoust) (#686), Codex auth fix by [@tmchow](https://github.com/tmchow) (#698).
|
||||
|
||||
or just ask in plain language:
|
||||
### arXiv, Techmeme, and Digg - free, no API keys
|
||||
|
||||
```
|
||||
/last30days OpenClaw, give me a shareable HTML brief
|
||||
/last30days Cursor IDE for slack
|
||||
/last30days Anthropic earnings export as html
|
||||
```
|
||||
arXiv brings the papers behind the hype and Techmeme brings the editorial tech-news layer - free, zero keys, and first-run setup installs their CLIs so they activate automatically (#709). Digg's AI 1000 story clusters arrive without X auth the same way - setup installs the free Digg CLI for you (#590). Trustpilot ships opt-in for consumer-brand research.
|
||||
|
||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
||||
### Free Reddit grew real scores and top comments
|
||||
|
||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
||||
Reddit's public .json API died; the free path came back stronger. Keyless RSS + shreddit scraping (#457), dedicated-subreddit discovery with real upvote counts via arctic-shift (#696), and a relevance floor so a viral off-topic post can't hijack your brief (#488, thanks [@rzachsmith](https://github.com/rzachsmith)). No API key. Real scores. Top comments included.
|
||||
|
||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
||||
### The best comments in every brief
|
||||
|
||||
### Intelligent search: the killer feature
|
||||
Comments are now a default-on layer across sources: Instagram comments with rank-based diversity so five hot takes don't all come from one post (#751), YouTube comments plus a ScrapeCreators transcript backup for when yt-dlp strikes out (#637), and crowd-voted comments weighted into Best Takes so the community's funniest lines survive scoring (#592, #608).
|
||||
|
||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
### One doctor command
|
||||
|
||||
This is why v3 finds content v2 never could. "Paperclip" resolves @dotta. "Dave Morin" resolves @davemorin plus @OpenClaw plus the TWiST podcast. "Peter Steinberger" resolves @steipete on X and steipete on GitHub. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags - resolved before a single API call fires.
|
||||
Ask for a health check and the doctor runs every source, then prescribes exact fixes - which key is missing, which CLI is off PATH, which cookie expired (#753). No more guessing why X came back thin.
|
||||
|
||||
### Best Takes
|
||||
### X search, rebuilt
|
||||
|
||||
Reddit and X people are funny. The old engine buried their best stuff because it scored for relevance, not cleverness. v3 has a second judge that scores every result for humor, wit, and virality alongside the relevance score. Tommy Lloyd's "My Michael Jordan is Steve Kerr" scores low on relevance to "Arizona Basketball" but off the charts on fun. Now every brief ends with a "Best Takes" section - the cleverest one-liners, the most viral quotes, the reactions that make you want to share the research. Built in, not a toggle.
|
||||
The X pipeline got a ground-up overhaul: FROM and ABOUT lanes so a person's own posts and the conversation about them both rank (#610), person-aware subquery disambiguation (#611), first-party authorship grounding with interaction-signal ranking (#613), and a single X source with automatic backend failover (#622). Plus an honest `--diagnose` that actually probes auth (#609).
|
||||
|
||||
### Cross-source cluster merging
|
||||
### More sources joined
|
||||
|
||||
When the same story appears on Reddit, X, and YouTube, v3 merges them into one cluster instead of showing three separate items. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
LinkedIn via ScrapeCreators, with articles as high signal ([@ravstr](https://github.com/ravstr), #702). StockTwits auto-activates for ticker and crypto topics ([@wtiwana](https://github.com/wtiwana), #658). Perplexity grew direct API modes and async Deep Research ([@sk-holmes](https://github.com/sk-holmes), #629).
|
||||
|
||||
### Single-pass comparisons
|
||||
### Hardened by the community
|
||||
|
||||
"CLI vs MCP" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides simultaneously. Same depth, 3 minutes.
|
||||
The security wave was almost entirely community work: stored-XSS fixes in the HTML renderer ([@iliaal](https://github.com/iliaal), [@aaronjmars](https://github.com/aaronjmars)), locked-down cookie temp files, supply-chain-hardened CI with OpenSSF Scorecard and build provenance attestation ([@shaanmajid](https://github.com/shaanmajid), [@hammadxcm](https://github.com/hammadxcm), [@aniruddh909](https://github.com/aniruddh909)), Semgrep and OSV-Scanner scans plus a PR dependency-review gate ([@23241a6749](https://github.com/23241a6749)), a test-coverage floor introduced at 60% and since raised to 84% ([@gourab5139014](https://github.com/gourab5139014)), and a Hermes security scan cleared of every CRITICAL finding (#768).
|
||||
|
||||
### Auto-discovered competitor comparisons
|
||||
### Reaches further
|
||||
|
||||
`/last30days OpenAI --competitors` tells the hosting reasoning model to discover the top 2 peers via WebSearch (Anthropic, xAI), run Step 0.55 per entity, and invoke the engine with `"OpenAI vs Anthropic vs xAI"` and a per-entity `--competitors-plan` JSON. The engine fans out 3 full pipelines in parallel, saves a `*-raw.md` file per entity, and merges them into a 3-way comparison. Same mechanics power `/last30days "OpenAI vs Anthropic vs xAI"` directly.
|
||||
Hebrew and non-Latin languages ([@dudyme](https://github.com/dudyme)). CJK-aware tokenization for Chinese sources ([@An-idd](https://github.com/An-idd)). A Windows compatibility wave. Cookie extraction across the full Chromium family - Brave, Edge, Vivaldi, Opera, Arc ([@andrey-esipov](https://github.com/andrey-esipov)) - plus macOS Keychain and Linux pass(1) credential sources. `--as-of` historical lookback ([@chiyi-creator](https://github.com/chiyi-creator)). Auto-provisioned Python 3.12 via uv ([@buntysomroy](https://github.com/buntysomroy)). `--hiring-signals` for reading a company's job pages. Watchlist deltas between runs.
|
||||
|
||||
### GitHub person-mode
|
||||
### Still in the box from v3
|
||||
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. Instead of "who mentioned this name in an issue body," it answers: what are they shipping and where is it landing?
|
||||
|
||||
`/last30days Peter Steinberger --github-user=steipete` shows 22 PRs merged across 3 repos at 85% merge rate. Own projects with README summaries, star counts, and top feature requests. Release notes for what shipped this month. The synthesizer weaves it into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
### ELI5 mode
|
||||
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations - just clearer. "Arizona wins by being physical" instead of "Arizona's identity is paint scoring (50%+ shooting, 9th nationally)." Say "eli5 off" to go back.
|
||||
|
||||
### Everything else in v3
|
||||
|
||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||
- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock.
|
||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||
- **Polymarket odds, not dollars.** The % odds are the magic. Dollar volumes removed from display.
|
||||
- **Per-author cap.** Max 3 items per author prevents any single voice from dominating your brief.
|
||||
- **Entity disambiguation.** When the engine resolves handles, the synthesis trusts them. No more Mallorca resorts winning over Washington athletic clubs.
|
||||
- **OpenClaw first-class citizen.** Auto-resolve for engine-side pre-research. Device auth for frictionless ScrapeCreators signup.
|
||||
- **1,012 tests passing.**
|
||||
The v3 foundations are all still here: the pre-research brain that resolves the right handles, subreddits, and hashtags before a single API call fires (built by [@j-sperling](https://github.com/j-sperling)); Best Takes scoring for humor and virality alongside relevance; cross-source cluster merging; single-pass comparisons ("CLI vs MCP" in 3 minutes, not 12); auto-discovered `--competitors` comparisons; GitHub person-mode (`--github-user=steipete`); ELI5 mode ("eli5 on" after any run); and shareable, self-contained HTML briefs (`--emit=html`). Configuration knobs live in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install | Updates |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
||||
| **Grok** (xAI Build CLI) | `grok plugin marketplace add mvanhorn/last30days-skill` then `grok plugin install last30days` | `grok plugin update last30days` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via claude.ai > Customize > Skills > + > Create skill > Upload a skill | Re-download and re-upload |
|
||||
| **Claude Desktop** | [Download the `.mcpb` for your platform](https://github.com/mvanhorn/last30days-skill/releases/latest) and drag into Settings > Extensions | Re-download and drag the new bundle in |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
@@ -187,7 +175,24 @@ If you'd rather use the agent-skills install path on Claude Code, that's also su
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
|
||||
The native plugin and the `npx skills` install can coexist. Note that Claude Code does not dedupe across install methods: if you have both the marketplace plugin and the `npx skills` copy active, `/last30days` will show two entries. Use one install method per machine.
|
||||
|
||||
### Grok (xAI Build CLI)
|
||||
|
||||
[Grok Build](https://docs.x.ai/build/features/skills-plugins-marketplaces) (`grok`) installs last30days as a native plugin. Direct install tracks the repository:
|
||||
|
||||
```bash
|
||||
grok plugin install mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Or add this repo as a marketplace source, then install by plugin name:
|
||||
|
||||
```bash
|
||||
grok plugin marketplace add mvanhorn/last30days-skill
|
||||
grok plugin install last30days
|
||||
```
|
||||
|
||||
Add `--trust` to skip the install confirmation. Update with `grok plugin update last30days`. Grok also reads the Claude Code manifests for compatibility; the native `.grok-plugin/` pair is the first-class lane (and what an official [xAI marketplace](https://github.com/xai-org/plugin-marketplace) listing points at). `npx skills add` remains a valid cross-host fallback.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
||||
|
||||
@@ -199,6 +204,8 @@ npx skills add mvanhorn/last30days-skill -g
|
||||
|
||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
||||
|
||||
Codex desktop and other folder-mode hosts can work in ordinary folders as well as Git repos. Before first research, ask the host agent to run the bundled `scripts/last30days.py --preflight` from the loaded skill directory; in a source checkout, the equivalent command is `python3 skills/last30days/scripts/last30days.py --preflight`. It shows the config source, browser-cookie plan, planned writes, optional commands, and ignored project config without reading cookies, writing files, or running research.
|
||||
|
||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
||||
|
||||
```bash
|
||||
@@ -225,8 +232,8 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) from the latest release
|
||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||
3. Click the `+` button in the Skills panel and drop the file in
|
||||
2. Go to [claude.ai > Customize > Skills](https://claude.ai/customize/skills)
|
||||
3. Click the `+` button in the Skills panel > click on `Create skill` > `Upload a skill` and browse/drop the file in
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
||||
|
||||
@@ -254,6 +261,12 @@ Windows support is deferred until per-platform manifest entry points are sorted
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
For X/Twitter action workflows outside `/last30days` research, such as posting
|
||||
tweets or replies, follower export, media handling, monitors, and giveaway
|
||||
draws, use [TweetClaw](https://github.com/Xquik-dev/tweetclaw) as the companion
|
||||
OpenClaw plugin. TweetClaw is maintained by Xquik-dev and is listed only as an
|
||||
optional companion path, not a last30days dependency or endorsement.
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
```bash
|
||||
@@ -263,7 +276,7 @@ ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
|
||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds, including the free arXiv and Techmeme CLIs.
|
||||
|
||||
## Bring your own keys
|
||||
|
||||
@@ -271,14 +284,61 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
|
||||
| Sources | What you need | Cost |
|
||||
|---------|---------------|------|
|
||||
| Reddit (with comments) + HN + Polymarket + GitHub | Nothing | Free |
|
||||
| X / Twitter | Log into x.com in any browser | Free |
|
||||
| Reddit (with comments) + HN + Polymarket + GitHub + StockTwits | Nothing | Free |
|
||||
| arXiv + Techmeme | Free CLIs, auto-installed by first-run setup | Free |
|
||||
| X / Twitter | Log into x.com in any browser, or set `XQUIK_API_KEY` / `XAI_API_KEY` | Browser cookies are free; keys are provider-specific |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls |
|
||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||
| TikTok + Instagram + Threads + Pinterest + LinkedIn + YouTube comments | ScrapeCreators key | 10,000 free calls, then PAYG |
|
||||
| Xiaohongshu (RED) | Run a logged-in x-mcp browser plugin or `xiaohongshu-mcp` service and opt in with `--search xhs` per run or `INCLUDE_SOURCES=xiaohongshu` in `.env`; last30days auto-probes `http://localhost:18060` then `http://host.docker.internal:18060`, or use `XIAOHONGSHU_API_BASE` for a custom URL | No last30days API key; depends on your local browser-session service |
|
||||
| DripStack (premium financial newsletters) | Opt-in: `--search dripstack` per run, or `INCLUDE_SOURCES=dripstack` in `.env` | No key; free public search API |
|
||||
| Perplexity Sonar / Search API / Deep Research | Perplexity key, or OpenRouter key as Sonar fallback | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
|
||||
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||
|
||||
Already have keys under different Keychain service names? Set the non-secret `LAST30DAYS_KEYCHAIN_ALIASES` mapping described in [CONFIGURATION.md](CONFIGURATION.md#reusing-existing-macos-keychain-items) instead of copying secrets.
|
||||
|
||||
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
||||
|
||||
## Configuration
|
||||
|
||||
Two things you'll likely want to know on day one:
|
||||
|
||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--output <file>` when you need the rendered result at an exact path, using the format selected by `--emit`. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each `--save-dir` run produces `<slug>-raw[-suffix].md`. Run `python3 skills/last30days/scripts/last30days.py --preflight` to review planned writes before a research run.
|
||||
|
||||
**Structured output for agents and workflows.** Ask `/last30days` for machine-readable JSON to receive the stable, versioned agent profile. For direct engine use in scripts or development, run `python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json`; add `--json-profile=raw` only when you need the unversioned internal `Report` dump. See the [JSON export field reference and versioning policy](docs/reference/json-export.md).
|
||||
|
||||
**Topic-less discovery.** Ask `/last30days what's trending in AI agents?` to get a ranked discovery brief instead of researching a topic you already know - on an agent host this runs the three-command host-judged protocol (the model names topics, filters junk, scores worthiness, and writes the content angles). For direct engine use in scripts or cron, run `python3 skills/last30days/scripts/last30days.py --discover "AI agents"` (one-shot: deterministic topic names, no angles); add `--emit=json` for the versioned discovery contract. Discovery is mutually exclusive with a positional topic and `--drill`.
|
||||
|
||||
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
**A subscribable research library.** Ask `/last30days` to build your library feed, or use `python3 skills/last30days/scripts/last30days.py library feed` directly for scripting and development. It turns saved briefs into `index.html`, a local Atom `feed.xml`, and readable brief pages. Add `--publish` only when you want the HTML index and brief pages hosted; publishing is explicit opt-in and public by default. To make the Atom feed subscribable, host the generated output directory on a static host such as GitHub Pages.
|
||||
|
||||
**Search everything you've researched.** Ask `/last30days search my library for MCP servers` or `/last30days have I researched MCP servers before?`. For direct engine use, run `python3 skills/last30days/scripts/last30days.py library search "MCP servers"`. Search is offline and deterministic: it incrementally indexes the same saved briefs used by the library feed, merges matching per-run store sightings, and groups results by topic and date. Fresh runs also surface a compact **From your library** section when prior research overlaps the current topic; set `LAST30DAYS_LIBRARY_CONTEXT=off` to disable that passive context.
|
||||
|
||||
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## Showcase: community research feeds
|
||||
|
||||
Published a recurring AI update, market watch, or wonderfully narrow obsession with last30days? Share the public library URL—or the Atom URL after hosting `feed.xml` on a static host—in [the community showcase thread](https://github.com/mvanhorn/last30days-skill/issues/532). Community feeds will be linked here as their owners submit them; the thread is the collection point in the meantime.
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||
@@ -299,11 +359,11 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
|
||||
## Open source
|
||||
|
||||
MIT license. No tracking. No analytics. Your research stays on your machine. 1,012 tests.
|
||||
MIT license. No tracking. No analytics. Your research stays on your machine. 2,700+ tests.
|
||||
|
||||
Built with Python 3.12+, yt-dlp, Node.js (vendored Bird client for X search), and ScrapeCreators API. v3 engine architecture by [@j-sperling](https://github.com/j-sperling).
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list of community contributors and [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
|
||||
## Star History
|
||||
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
context: fork
|
||||
agent: Explore
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -1,77 +0,0 @@
|
||||
# last30days Skill Specification
|
||||
|
||||
## Overview
|
||||
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||
|
||||
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||
|
||||
## Architecture
|
||||
|
||||
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||
|
||||
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||
- **dates.py**: Date range calculation and confidence scoring
|
||||
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||
- **http.py**: stdlib-only HTTP client with retry logic
|
||||
- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching
|
||||
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||
- **xai_x.py**: xAI Responses API + x_search for X
|
||||
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||
- **normalize.py**: Convert raw API responses to canonical schema
|
||||
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||
- **render.py**: Generate markdown and JSON outputs
|
||||
- **schema.py**: Type definitions and validation
|
||||
|
||||
## Embedding in Other Skills
|
||||
|
||||
Other skills can import the research context in several ways:
|
||||
|
||||
### Inline Context Injection
|
||||
```markdown
|
||||
## Recent Research Context
|
||||
!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context
|
||||
```
|
||||
|
||||
### Read from File
|
||||
```markdown
|
||||
## Research Context
|
||||
!cat ~/.local/share/last30days/out/last30days.context.md
|
||||
```
|
||||
|
||||
### Get Path for Dynamic Loading
|
||||
```bash
|
||||
CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path)
|
||||
cat "$CONTEXT_PATH"
|
||||
```
|
||||
|
||||
### JSON for Programmatic Use
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json
|
||||
```
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py <topic> [options]
|
||||
|
||||
Options:
|
||||
--refresh Bypass cache and fetch fresh data
|
||||
--mock Use fixtures instead of real API calls
|
||||
--emit=MODE Output mode: compact|json|md|context|path (default: compact)
|
||||
--sources=MODE Source selection: auto|reddit|x|both (default: auto)
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
All outputs are written to `~/.local/share/last30days/out/`:
|
||||
|
||||
- `report.md` - Human-readable full report
|
||||
- `report.json` - Normalized data with scores
|
||||
- `last30days.context.md` - Compact reusable snippet for other skills
|
||||
- `raw_openai.json` - Raw OpenAI API response
|
||||
- `raw_xai.json` - Raw xAI API response
|
||||
- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data
|
||||
@@ -1,47 +0,0 @@
|
||||
# last30days Implementation Tasks
|
||||
|
||||
## Setup & Configuration
|
||||
- [x] Create directory structure
|
||||
- [x] Write SPEC.md
|
||||
- [x] Write TASKS.md
|
||||
- [x] Write SKILL.md with proper frontmatter
|
||||
|
||||
## Core Library Modules
|
||||
- [x] scripts/lib/env.py - Environment and API key loading
|
||||
- [x] scripts/lib/dates.py - Date range and confidence utilities
|
||||
- [x] scripts/lib/cache.py - TTL-based caching
|
||||
- [x] scripts/lib/http.py - HTTP client with retry
|
||||
- [x] scripts/lib/models.py - Auto model selection
|
||||
- [x] scripts/lib/schema.py - Data structures
|
||||
- [x] scripts/lib/openai_reddit.py - OpenAI Responses API
|
||||
- [x] scripts/lib/xai_x.py - xAI Responses API
|
||||
- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher
|
||||
- [x] scripts/lib/normalize.py - Schema normalization
|
||||
- [x] scripts/lib/score.py - Popularity scoring
|
||||
- [x] scripts/lib/dedupe.py - Near-duplicate detection
|
||||
- [x] scripts/lib/render.py - Output rendering
|
||||
|
||||
## Main Script
|
||||
- [x] scripts/last30days.py - CLI orchestrator
|
||||
|
||||
## Fixtures
|
||||
- [x] fixtures/openai_sample.json
|
||||
- [x] fixtures/xai_sample.json
|
||||
- [x] fixtures/reddit_thread_sample.json
|
||||
- [x] fixtures/models_openai_sample.json
|
||||
- [x] fixtures/models_xai_sample.json
|
||||
|
||||
## Tests
|
||||
- [x] tests/test_dates.py
|
||||
- [x] tests/test_cache.py
|
||||
- [x] tests/test_models.py
|
||||
- [x] tests/test_score.py
|
||||
- [x] tests/test_dedupe.py
|
||||
- [x] tests/test_normalize.py
|
||||
- [x] tests/test_render.py
|
||||
|
||||
## Validation
|
||||
- [x] Run tests in mock mode
|
||||
- [x] Demo --emit=compact
|
||||
- [x] Demo --emit=context
|
||||
- [x] Verify file tree
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
description: Research what people actually say about any topic in the last 30 days across Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.
|
||||
argument-hint: <topic> — e.g. "nvidia earnings reaction" or "best noise cancelling headphones"
|
||||
allowed-tools: [Bash, Read, Write, AskUserQuestion, WebSearch]
|
||||
---
|
||||
|
||||
Invoke the `last30days` skill with the user's arguments: $ARGUMENTS
|
||||
|
||||
Use the skill's canonical pipeline (plan → retrieve → normalize → fuse → rerank → cluster → render). If the user provided no arguments, ask them for a topic before proceeding.
|
||||
+12
-11
@@ -142,7 +142,7 @@ The repo vendors a search-only subset of Bird's Twitter GraphQL client and shell
|
||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||
| Replies/quotes | Real | Real |
|
||||
| Author handle | Real | Real |
|
||||
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
||||
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
|
||||
|
||||
### Depth settings
|
||||
|
||||
@@ -183,13 +183,14 @@ After both searches complete:
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
||||
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
||||
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `scripts/lib/models.py` | Auto-select best available model |
|
||||
| `scripts/lib/env.py` | API key loading, source detection |
|
||||
| `scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `scripts/lib/score.py` | Relevance scoring |
|
||||
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
|
||||
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
|
||||
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
|
||||
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
|
||||
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
|
||||
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
|
||||
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
|
||||
@@ -1,303 +0,0 @@
|
||||
---
|
||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# feat: --competitors flag for auto-discovered comparison fan-out
|
||||
|
||||
## Overview
|
||||
|
||||
Add a `--competitors` flag to the last30days engine that auto-discovers 2-4 peer entities for the topic, runs the full retrieval pipeline on each in parallel, and renders a multi-entity comparison. Invoking `last30days Kanye West --competitors` should resolve to "Kanye vs Drake vs Kendrick Lamar" and emit a comparison report covering all three. Invoking `last30days OpenAI --competitors` should resolve to "OpenAI vs Anthropic vs xAI vs Gemini" and emit a four-way comparison.
|
||||
|
||||
Discovery mirrors the existing `resolve.auto_resolve()` pattern used for X handles and subreddits at pipeline start — web search (Brave / Exa / Serper) plus deterministic extraction. Not an internal LLM call.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
Users who want a comparison today must type "OpenAI vs Anthropic vs xAI" themselves. The `planner._comparison_entities()` path already handles explicit multi-entity topics and `render._render_comparison_scaffold()` already emits a 9-axis comparison table. What is missing is the discovery half — a user who types a single entity with `--competitors` should get the comparison for free.
|
||||
|
||||
This is also the natural next step after the Step 0.55 category-peer subreddit work (PR #305, merged 2026-04-22). That feature widens the subreddit set within a single topic; this feature widens the entity set into peer entities.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. New `--competitors` boolean flag that triggers competitor discovery and multi-entity fan-out.
|
||||
- R2. New `--competitors-list="A,B,C"` to explicitly skip discovery (mirrors `--plan`, `--subreddits`, `--x-handle` overrides).
|
||||
- R3. New `--competitors=N` short form to set competitor count inline (N in 1..6).
|
||||
- R4. Default count is 3 competitors (original + 3 = 4-way comparison).
|
||||
- R5. Competitor retrieval depth inherits the main run's depth (`--quick` / `--deep`); all entities run in parallel so wall clock stays close to a single run.
|
||||
- R6. Discovery mirrors `resolve.auto_resolve()`: web search for peers, deterministic text extraction. No internal LLM dependency.
|
||||
- R7. If no web search backend is configured and no `--competitors-list` was passed, engine emits a LAW 7-style stderr telling the host agent to pass `--competitors-list` and exits non-zero.
|
||||
- R8. Output rendering is a single comparison report covering all entities, reusing the existing 9-axis scaffold from `render._render_comparison_scaffold()` where applicable.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- Synthesis prompt changes beyond wiring N reports into the existing comparison scaffold are out of scope.
|
||||
- `--competitors` does not replace the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`; both paths coexist.
|
||||
- No caching layer for discovery results in v1.
|
||||
- No UI/SKILL.md rewrite of the entire comparison section; only the new flag is documented.
|
||||
- No new web search backend.
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Caching of competitor lookups: separate follow-up once hit rate justifies it.
|
||||
- Disambiguation UX for topics with multiple common entities ("Amazon" the company vs the river): separate brainstorm.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py:168-249` — `build_parser()` argparse definitions. Existing depth flags (`--quick`, `--deep`) and override flags (`--plan`, `--subreddits`, `--x-handle`, `--auto-resolve`) set the convention to mirror.
|
||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the reference pattern: web search fan-out via `ThreadPoolExecutor`, per-query extraction functions, graceful empty-dict return when no backend is available.
|
||||
- `scripts/lib/resolve.py:98-140` — `_extract_x_handle()` and sibling extractors show the deterministic text-mining style competitor extraction should mirror.
|
||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature is the fan-out target. One call per entity, each returning a `schema.Report`.
|
||||
- `scripts/lib/planner.py:430-564` — Existing comparison-intent handling and `_comparison_entities()` entity extraction. The new flag feeds the same mental model but populates entities from discovery instead of from the topic string.
|
||||
- `scripts/lib/render.py:333-392` — `_render_comparison_scaffold()` already emits a 9-axis markdown comparison table. The new multi-report renderer should reuse this helper by assembling a synthetic "A vs B vs C" topic header for it.
|
||||
- `scripts/lib/grounding.py` + `scripts/lib/providers.py` — Web search backend resolution (Brave / Exa / Serper). Reused as-is.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- No existing `docs/solutions/` entries for competitor discovery or multi-entity fan-out.
|
||||
- Recent plan `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md` established the precedent of deterministic peer expansion; this plan extends that idea from subreddits to entities.
|
||||
|
||||
### External References
|
||||
|
||||
- None gathered — local patterns are strong. `resolve.auto_resolve()` is a direct template.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Discovery mirrors auto_resolve, not plan_query.** Web search + regex extraction, not an LLM call. Matches the user's explicit direction ("use the python brain the same way it searches for X handles"). Cheaper, no provider credential requirement, deterministic.
|
||||
- **Orchestration lives in `last30days.py` main, not inside `pipeline.run()`.** The fan-out is a top-level concern — one pipeline run per entity, each independent. Keeps `pipeline.run()` single-entity and unchanged except for sharing a `ThreadPoolExecutor` factory.
|
||||
- **Sub-runs inherit main depth and run in parallel.** Wall clock ≈ single run; token cost scales linearly with N. User-controlled via the existing `--quick`/`--deep` flags.
|
||||
- **New module `scripts/lib/competitors.py` instead of adding to `resolve.py`.** Keeps resolve focused on single-entity entity-bundle discovery (handles/subreddits/github); competitors.py owns peer-entity discovery. Similar shape, different responsibility.
|
||||
- **Multi-report render is additive in `render.py`.** New `render_comparison_multi(reports: list[Report]) -> str` composes a synthetic "A vs B vs C" topic and delegates to the existing scaffold + synthesis path where possible. No rewrite of the single-entity render path.
|
||||
- **Default count = 3 competitors (4-way comparison).** Hard cap at 6.
|
||||
- **LAW 7-style stderr when no backend and no list.** Matches how `planner.plan_query()` already tells the hosting agent to pass `--plan`.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Discovery mechanism:** Web search via `grounding.web_search()`, not an internal LLM. User confirmed the auto_resolve pattern is the target.
|
||||
- **Default competitor count:** 3 (original + 3 = 4-way).
|
||||
- **Sub-run depth:** Inherit main depth, parallel execution.
|
||||
- **Flag naming:** `--competitors` (standard argparse double-dash). `--competitors=N` for inline count. `--competitors-list="A,B,C"` to skip discovery.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact extraction heuristics for competitor names across Brave / Exa / Serper result shapes. The SERP text varies (listicles, comparison pages, "vs" pages); the initial implementation will start with listicle parsing plus a "X vs Y" pattern match, and harden against real results in the test phase.
|
||||
- Handling of topic ambiguity ("Amazon", "Apple"). Initial behavior: trust whatever web search returns for the topic verbatim; disambiguation is a separate concern.
|
||||
- Merge strategy when two entities return overlapping URLs (e.g., an "OpenAI vs Anthropic" article shows up in both runs). Likely dedupe at the clustering step, but defer the exact policy until we see how often it happens.
|
||||
- Whether to expose competitor discovery artifacts (the raw web search results) as a debug emit. Follow the existing `--debug` conventions.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: CLI flag parsing and validation**
|
||||
|
||||
**Goal:** Add `--competitors`, `--competitors=N`, and `--competitors-list` to the argparse surface, validate values, and thread them into the main orchestration.
|
||||
|
||||
**Requirements:** R1, R2, R3, R4
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
- Test: `tests/test_cli_competitors.py`
|
||||
|
||||
**Approach:**
|
||||
- Add three mutually cooperative flags near line 205 in `build_parser()`:
|
||||
- `--competitors` with `nargs="?"` and `const=3` so bare `--competitors` defaults to 3, `--competitors=4` is honored, and `--competitors=0` is rejected
|
||||
- `--competitors-list` free-text CSV
|
||||
- Normalize in `main()`: if `--competitors-list` is present, skip discovery and use the list. If `--competitors` is set and no list, trigger discovery with count = the flag value. Clamp count to 1..6 with a stderr warning at boundary.
|
||||
- Thread the resulting entity list into the orchestrator added in Unit 3.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `--plan` argument at `scripts/last30days.py:187` — same skip-discovery-when-explicit shape.
|
||||
- `--subreddits` / `--x-handle` at `scripts/last30days.py:180,189` — same override semantics.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: bare `--competitors` parses to count=3, empty list.
|
||||
- Happy path: `--competitors=4` parses to count=4.
|
||||
- Happy path: `--competitors-list="A,B,C"` parses to count=3, list=["A","B","C"], and is preferred over any discovery signal.
|
||||
- Edge case: `--competitors=0` and `--competitors=-1` are rejected with a clear error.
|
||||
- Edge case: `--competitors=99` clamps to 6 with a stderr warning.
|
||||
- Edge case: `--competitors` combined with `--competitors-list` uses the list and logs that discovery was skipped.
|
||||
- Edge case: `--competitors-list` value with whitespace ("A, B , C") normalizes correctly.
|
||||
|
||||
**Verification:**
|
||||
- Running the binary with each flag variation produces the expected post-parse state without calling out to the network.
|
||||
|
||||
- [ ] **Unit 2: `scripts/lib/competitors.py` discovery module**
|
||||
|
||||
**Goal:** Discover peer entities for a topic using web search + deterministic extraction, mirroring `resolve.auto_resolve()`.
|
||||
|
||||
**Requirements:** R6, R7
|
||||
|
||||
**Dependencies:** None (pure module; wired by Unit 3)
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/lib/competitors.py`
|
||||
- Test: `tests/test_competitors.py`
|
||||
|
||||
**Approach:**
|
||||
- Public entry point `discover_competitors(topic: str, count: int, config: dict) -> list[str]`.
|
||||
- Early return `[]` when `_has_backend(config)` is false (reuse the helper from `resolve.py`; factor if needed).
|
||||
- Fan out 2-3 web searches in a `ThreadPoolExecutor`:
|
||||
- `"{topic} competitors"`
|
||||
- `"{topic} alternatives"`
|
||||
- `"{topic} vs"` (captures "X vs Y" articles)
|
||||
- Feed results into a deterministic `_extract_peer_entities(results, topic)` that:
|
||||
- Mines titles and snippets for capitalized noun phrases other than the topic itself
|
||||
- Scores by frequency across results
|
||||
- Filters stopwords and the topic's own tokens
|
||||
- Returns top `count` unique entities ordered by score
|
||||
- Emit a single-line stderr log mirroring the `resolve._log` format.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/lib/resolve.py:179-258` for the function shape, executor usage, and empty-result fallback.
|
||||
- `scripts/lib/resolve.py:98-140` for extractor style (small, deterministic, no external state).
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: canned SERP fixtures for "OpenAI" return ["Anthropic", "xAI", "Google"] or close peers in the top 3.
|
||||
- Happy path: canned SERP fixtures for "Kanye West" return rap peers (Drake, Kendrick) in the top 3.
|
||||
- Edge case: empty SERP results return `[]` without raising.
|
||||
- Edge case: extractor filters out the topic itself (case- and punctuation-insensitive).
|
||||
- Edge case: near-duplicate entities ("OpenAI" vs "Open AI") dedupe to one slot.
|
||||
- Error path: web search backend raises — the failure is logged and the function returns `[]`.
|
||||
- Edge case: count=1 returns a single-element list; count=6 returns up to six entities.
|
||||
|
||||
**Verification:**
|
||||
- Unit tests pass with fixtures committed under `tests/fixtures/competitors-*.json`.
|
||||
- Manual run against a live backend for one topic confirms sensible output (recorded as a notes file, not a test assertion).
|
||||
|
||||
- [ ] **Unit 3: Parallel fan-out orchestrator**
|
||||
|
||||
**Goal:** Run `pipeline.run()` once per entity (topic + discovered competitors) in parallel, collect `schema.Report` per entity, and hand them to the comparison renderer.
|
||||
|
||||
**Requirements:** R5, R7
|
||||
|
||||
**Dependencies:** Unit 1, Unit 2
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
- Possibly create: `scripts/lib/fanout.py` if the orchestrator grows past ~60 lines
|
||||
- Test: `tests/test_competitor_fanout.py`
|
||||
|
||||
**Approach:**
|
||||
- After arg parsing and before the existing `pipeline.run()` call, branch on `args.competitors`:
|
||||
- If a list was provided or discovery returned entities, build `entities = [topic, *competitors]`.
|
||||
- Spawn one `pipeline.run()` per entity via `ThreadPoolExecutor(max_workers=len(entities))`, passing the same `config`, `depth`, and all sub-run-relevant args (mock, plan, etc.). Respect `--plan` — if a plan is passed it applies to the main topic only; competitors use the internal planner fallback for v1.
|
||||
- Collect `{entity: Report}` mapping. A per-entity failure logs a stderr warning and drops that entity from the comparison; the run continues as long as 2 entities succeed.
|
||||
- If fewer than 2 entities survive, exit with a clear error.
|
||||
- LAW 7-style stderr:
|
||||
- If `args.competitors` is set, no list was passed, no web search backend is configured, emit a LAW 7 stderr message pointing to the `--competitors-list` override and exit non-zero. Reuse the tone from `planner.plan_query()` fallback (`scripts/lib/planner.py:125-135`).
|
||||
|
||||
**Execution note:** Start with a failing integration test that exercises the full main → orchestrator → mocked pipeline.run path; the orchestrator is where bugs hide.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/lib/resolve.py:225-239` for ThreadPoolExecutor + as_completed + per-future error handling.
|
||||
- `scripts/lib/pipeline.py:310+` for how ThreadPoolExecutor is already used inside a single run (same idiom, outer layer).
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: main + 2 competitors, all three `pipeline.run()` calls succeed (mocked), orchestrator returns 3 Reports.
|
||||
- Happy path: discovery returns the competitor list; orchestrator fans out accordingly.
|
||||
- Edge case: one of three competitor pipelines raises — the run continues with the surviving 2 and emits a warning.
|
||||
- Edge case: all competitors fail but the main topic succeeds — orchestrator exits non-zero with a clear error rather than silently degrading to a single-entity render.
|
||||
- Edge case: `--competitors` set, no backend, no list — orchestrator emits the LAW 7 stderr and exits non-zero before any pipeline call.
|
||||
- Integration: wall-clock time for 3 mocked pipelines in parallel is close to the slowest single run, not the sum (timing assertion with generous margin).
|
||||
|
||||
**Verification:**
|
||||
- End-to-end test with mocked `pipeline.run()` and mocked competitors discovery produces 3 Reports and hands them to a stubbed renderer.
|
||||
|
||||
- [ ] **Unit 4: Multi-report comparison renderer**
|
||||
|
||||
**Goal:** Compose N `schema.Report`s into a single comparison-mode output, reusing the existing 9-axis scaffold.
|
||||
|
||||
**Requirements:** R8
|
||||
|
||||
**Dependencies:** Unit 3
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/render.py`
|
||||
- Test: `tests/test_render_comparison_multi.py`
|
||||
|
||||
**Approach:**
|
||||
- Add `render_comparison_multi(reports: list[schema.Report], *, emit: str) -> str`.
|
||||
- Build a synthetic comparison topic: `f"{entity_a} vs {entity_b} vs {entity_c}"`.
|
||||
- Reuse `_render_comparison_scaffold()` for the table skeleton. Each entity column is populated from its own Report's top clusters and citations.
|
||||
- For the narrative synthesis block, concatenate per-entity highlights, clearly labeled by entity, under a shared "Comparison" header.
|
||||
- Preserve existing emit modes (`compact`, `md`, `json`, `context`). In `json` emit, return a `{"entities": [...], "reports": [...]}` shape; single-Report consumers remain unaffected because the single-report render path is untouched.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/lib/render.py:333-392` (`_parse_comparison_entities`, `_render_comparison_scaffold`) — the scaffold is the contract.
|
||||
- `scripts/lib/render.py` single-report rendering — for per-entity narrative blocks.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: 3 Reports with distinct clusters render into a 3-column table and a "Comparison" section that mentions each entity at least once.
|
||||
- Happy path: 2 Reports render as a 2-column table without breaking the scaffold.
|
||||
- Edge case: a Report with an empty cluster list renders as "(no significant discussion this month)" in its column rather than crashing.
|
||||
- Edge case: Reports with overlapping URLs (same article cited by two entities) dedupe citations at the footer but keep both column entries.
|
||||
- Emit variants: `--emit=compact`, `--emit=md`, `--emit=json`, `--emit=context` each produce valid output with all entities represented.
|
||||
- Integration: end-to-end snapshot test using fixture Reports, checked against a stored expected output (with a clear update path when the scaffold intentionally evolves).
|
||||
|
||||
**Verification:**
|
||||
- Snapshot tests pass. Manual review of one real 3-way comparison confirms readability.
|
||||
|
||||
- [ ] **Unit 5: Docs, SKILL.md mention, and sync**
|
||||
|
||||
**Goal:** Document the new flag so the hosting agent and human users both know it exists, and run the sync script.
|
||||
|
||||
**Requirements:** R1-R8 (surfaces them to users)
|
||||
|
||||
**Dependencies:** Units 1-4
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md`
|
||||
- Modify: `README.md` (brief flag reference)
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
|
||||
**Approach:**
|
||||
- Add a compact "Competitor mode" subsection under the existing comparison docs in `SKILL.md`. Document the flag, the default count, the override flag, and the LAW 7 fallback stderr.
|
||||
- Keep `README.md` addition to a single example line.
|
||||
- CHANGELOG entry mirrors the voice of recent entries (imperative, outcome-first).
|
||||
- Sync via `scripts/sync.sh` per CLAUDE.md rules so `~/.claude/`, `~/.agents/`, `~/.codex/` pick up the new SKILL.md.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — documentation and sync only. Verification is by inspection and by running `sync.sh` and confirming target directories updated.
|
||||
|
||||
**Verification:**
|
||||
- `sync.sh` completes without errors.
|
||||
- `SKILL.md` rendered preview mentions `--competitors` in the comparison section.
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** `last30days.py main()` now orchestrates multiple `pipeline.run()` calls instead of one. No other callers of `pipeline.run()` are affected (it remains single-entity).
|
||||
- **Error propagation:** Per-entity failures degrade gracefully as long as ≥2 entities survive; fewer survivors exits non-zero. Discovery failure with `--competitors` and no list is fatal.
|
||||
- **State lifecycle risks:** Each sub-run uses its own `pipeline.run()` state; no shared mutable config. The `config` dict is read-only in `pipeline.run()` today — verify before committing to shared-reference passing, else deep-copy per sub-run.
|
||||
- **API surface parity:** `--competitors` coexists with the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`. Both produce comparable output formats; the only difference is where the entity list came from.
|
||||
- **Integration coverage:** The fan-out orchestrator crosses CLI → discovery → N pipelines → render; integration tests in Unit 3 and Unit 4 must exercise the full path end to end, not just unit-level.
|
||||
- **Unchanged invariants:** `pipeline.run()` signature and single-entity semantics are unchanged. The single-entity render path in `render.py` is unchanged. No changes to `planner.plan_query()`. No changes to existing flags.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Competitor discovery returns garbage entities for niche topics. | `--competitors-list` override lets the user (or hosting agent) correct it. Unit tests with edge-case fixtures. Log discovery output to stderr under `--debug`. |
|
||||
| Token cost scales linearly with N sub-runs. | Default count capped at 3, hard max 6, inherit `--quick` to let users throttle. Wall clock stays parallel. Emit a cost hint to stderr when N ≥ 4. |
|
||||
| Merge conflicts against the single-entity render path during refactoring. | Keep the multi-report renderer strictly additive; do not modify the single-Report code path. |
|
||||
| Config dict mutation inside sub-runs could leak state between entities. | Verify read-only usage before sharing references. If any sub-component mutates, deep-copy per sub-run before spawning threads. |
|
||||
| A SERP extractor that works on Brave fixtures breaks on Exa/Serper result shapes. | Test fixtures for all three backends. Extractor operates on a normalized shape from `grounding.web_search()` (already the case), not raw provider output. |
|
||||
| Hosting agent (Claude Code, Codex) unaware of the new flag when it could usefully pass `--competitors-list`. | SKILL.md updated in Unit 5 documents the flag in the same style as `--plan` and `--auto-resolve`. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first: per `CLAUDE.md`, experimental changes go to `mvanhorn/last30days-skill-private` on the `/last30days-beta` command. Land this on the private repo first, shake out on real topics for a day or two, then cherry-pick to public.
|
||||
- After land-merge: run `scripts/sync.sh` to deploy SKILL.md + scripts to `~/.claude/`, `~/.agents/`, `~/.codex/`.
|
||||
- Release notes entry in CHANGELOG.md follows the v3.0.9 voice — outcome-first, one paragraph.
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Related code: `scripts/lib/resolve.py:179` (`auto_resolve`), `scripts/lib/pipeline.py:162` (`pipeline.run`), `scripts/lib/planner.py:80` (`plan_query` LAW 7 fallback), `scripts/lib/render.py:333` (comparison scaffold)
|
||||
- Related PRs: #305 (Step 0.55 category-peer subreddit expansion — the precedent for deterministic peer expansion, merged 2026-04-22)
|
||||
- Related plan: `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md`
|
||||
@@ -1,349 +0,0 @@
|
||||
---
|
||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
||||
type: fix
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
origin: docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md
|
||||
---
|
||||
|
||||
# fix: per-entity resolution, default-2, and stale-path guard for --competitors
|
||||
|
||||
## Overview
|
||||
|
||||
Three test runs of v3.0.11 `--competitors` surfaced four real bugs plus one product tweak. This plan fixes all of them in a single follow-up:
|
||||
|
||||
1. Competitor sub-runs get no Step 0.55 resolution (no X handle, no subreddits, no GitHub repo). Drake / Kendrick / Travis ran with deterministic-fallback single-word queries while Kanye had the full targeting package. User called it "lazy" and was right.
|
||||
2. Two of three test windows (Linear, Coinbase) never invoked the new flag at all. They loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (a Claude-Code-managed git clone pinned to origin/main, which predates PR #308) instead of `plugins/cache/last30days-skill/last30days/3.0.11/`, so `--help` showed no `--competitors` flag and the model fell back to the manual comparison path.
|
||||
3. Each competitor sub-run emits a scary `[Planner] No --plan passed... deterministic fallback` stderr line because LAW 7 targets the hosting-model path, not internal fan-out sub-runs.
|
||||
4. Default competitor count is 3 (→ 4-way comparison). User wants default 2 (→ 3-way: original + 2 peers). Flag keeps `--competitors=N` to customize.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The 3 test runs (Kanye, Linear, Coinbase) showed a pattern:
|
||||
|
||||
| Window | Loaded SKILL.md from | Invoked --competitors? | Per-entity resolution? | Outcome |
|
||||
|--------|----------------------|-----------------------|------------------------|---------|
|
||||
| Kanye | cache/3.0.11/ (correct) | Yes | Only for main topic (Kanye) | Drake/Kendrick/Travis thin; Reddit 403 fallbacks |
|
||||
| Linear | marketplaces/ (stale) | No — fell back to manual comparison | No | Thin run with noisy subreddits |
|
||||
| Coinbase | marketplaces/ (stale) | No — fell back to manual comparison | Main only; keyword-search poisoned pool | Top subs: r/survivor, r/Airpodsmax (noise) |
|
||||
|
||||
Root causes:
|
||||
- **Per-entity resolution gap:** `scripts/lib/fanout.py` calls `pipeline.run()` with topic + depth + web_backend + lookback_days only. It does not call `resolve.auto_resolve()` per entity, so sub-runs have no X handle, subreddit, or GitHub targeting. The original plan (`2026-04-22-002`) acknowledged this as a deliberate v1 simplification ("competitor sub-runs use planner defaults"). In practice this produces visibly asymmetric output and triggers downstream retrieval issues (403 fallbacks, keyword-search noise).
|
||||
- **Stale-path loading:** Claude Code's skill loader alphabetizes `find` results with `marketplaces/` before `cache/`, and the model reads the first plausible SKILL.md it sees. SKILL.md line 823's `SKILL_ROOT` resolver is the correct path but only fires in engine-invocation blocks, not in the skill-load step.
|
||||
- **LAW 7 in sub-runs:** LAW 7 exists because the *hosting reasoning model* is supposed to pass `--plan`. For competitor sub-runs, there is no hosting-model planning — it's an engine-internal fan-out. The warning is a false positive there.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Default `--competitors` count is 2 peers (3-way comparison: original + 2).
|
||||
- R2. Each competitor sub-run performs Step 0.55 resolution (X handle, subreddits, GitHub user/repos, news context) before its pipeline runs — not just the main topic.
|
||||
- R3. Sub-runs do not emit the LAW 7 `No --plan passed` warning; they are internal fan-out, not hosting-model calls.
|
||||
- R4. The rendered comparison output includes a visible "Resolved entities" block showing per-entity handles/subs/github for debug transparency (answers "did it resolve everyone?" without the user having to read stderr).
|
||||
- R5. SKILL.md has a canonical-path self-check at the top: if the reader loaded it from anywhere other than `plugins/cache/last30days-skill/last30days/{VERSION}/`, re-read from the versioned path before proceeding.
|
||||
- R6. Version bumps to 3.0.12; CHANGELOG entry; `scripts/sync.sh` deploys.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No new discovery strategy. The web-search + regex extraction in `scripts/lib/competitors.py` stays as-is.
|
||||
- No new CLI flags beyond the behavior changes above. Specifically: no per-entity override flags like `--competitor-handles`. The hosting-model escape hatch remains `--competitors-list`.
|
||||
- No changes to the explicit `A vs B` comparison path (topic-string parsing in `planner._comparison_entities`).
|
||||
- No marketplace-clone auto-restore fix — that's Claude Code harness behavior. This plan only guards against the symptom on the skill side.
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Caching of per-entity resolution results: separate follow-up once hit rate justifies it.
|
||||
- Fan-out rate-limiting tuning (currently `max_workers=len(entities)+1`, capped at 6): defer until we see real-world quota exhaustion.
|
||||
- Pre-flight cost hint when N ≥ 4 (noted in `2026-04-22-002` risks): defer.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py:205-219` — `--competitors` / `--competitors-list` argparse definition (const=3 today; changing to 2).
|
||||
- `scripts/last30days.py:220-290` — `resolve_competitors_args()` validator; update `COMPETITORS_DEFAULT`.
|
||||
- `scripts/last30days.py:438-520` — main() fan-out orchestration; currently passes only topic/depth to each `_competitor_runner`.
|
||||
- `scripts/lib/fanout.py:40-95` — `run_competitor_fanout()` signature. The `competitor_runner` callable is where per-entity resolution needs to happen.
|
||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the exact per-entity resolver to reuse. Already does X handle + subreddits + GitHub user/repos + news context in parallel via ThreadPoolExecutor.
|
||||
- `scripts/lib/planner.py:80-135` — `plan_query()` emits the LAW 7 stderr. A `quiet: bool` keyword or `internal_subrun: bool` flag will suppress it.
|
||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature. Needs a new keyword to propagate quiet-mode down to the planner.
|
||||
- `scripts/lib/render.py:render_comparison_multi` — where the "Resolved entities" block is inserted.
|
||||
- `SKILL.md` line 823 — canonical `SKILL_ROOT` resolver already exists but fires in engine bash, not at skill-load time.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md` acknowledged the per-entity-resolution gap as a v1 tradeoff. This plan closes that gap.
|
||||
- Kanye run stderr: `[Planner] No --plan passed... deterministic fallback` × 3 (once per competitor sub-run). That's the LAW 7 noise R3 targets.
|
||||
- Linear / Coinbase runs loaded `plugins/marketplaces/last30days-skill/CLAUDE.md` as the first hit. That's the stale-path issue R5 targets.
|
||||
|
||||
### External References
|
||||
|
||||
- None. All patterns are in-repo.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Per-entity resolve happens inside fanout, not in SKILL.md.** The user-facing promise of `--competitors` is "one flag, engine does the work." Pushing resolution onto the hosting model creates another path-of-least-resistance trap (model skips it, output looks lazy). Auto-resolve inside each sub-run when a web backend is available makes the feature self-contained.
|
||||
- **Stale-path guard is a SKILL.md self-check, not a code change.** We cannot stop Claude Code from auto-restoring the marketplace clone. But we can put a 3-line banner at the top of SKILL.md that forces any path-mismatched read to re-read from the versioned cache. Both the marketplace copy (once main catches up) and the cache copy carry the guard.
|
||||
- **LAW 7 suppression is opt-in via `internal_subrun=True` keyword.** Do not remove the warning from the default path — it's load-bearing for the hosting-model contract. Add an explicit bypass for engine-internal fan-out only.
|
||||
- **Default 2, hard max 6 unchanged.** "Original + 2" matches the Kanye/Drake/Kendrick mental model from the feature description. Still allow `--competitors=N` from 1 to 6.
|
||||
- **Resolved block is inside the EVIDENCE envelope, not above it.** Keeps the rendered output structure stable for the synthesis contract (LAW 1–8). The block is context, not output.
|
||||
- **Skip auto-resolve when `--mock` or no web backend.** Mirrors the existing `resolve.auto_resolve()` fast-fail and keeps the mock test path deterministic.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Where does per-entity resolve live?** Inside `fanout.run_competitor_fanout`, not in `main()`. Each sub-run calls `auto_resolve()` just before `pipeline.run()`.
|
||||
- **Should the hosting model still be able to override?** Yes — `--competitors-list` remains the escape hatch. When an explicit list is passed, the engine still does auto-resolve per entity; the user's list just skips discovery.
|
||||
- **Should sub-runs run auto-resolve in parallel with each other?** Yes. The existing `ThreadPoolExecutor` in fanout already parallelizes sub-runs; auto-resolve happens inside each sub-run's thread, so resolve calls for different entities run concurrently.
|
||||
- **Default count:** 2 peers (3-way). Confirmed.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Whether to expose a `--no-auto-resolve-competitors` flag for power users who want the fast, shallow behavior. Probably not needed v2; ship auto-resolve always-on and revisit if someone complains about cost.
|
||||
- Whether to surface the per-entity resolution context back into the main topic's planner (cross-entity context sharing). Stays deferred.
|
||||
- Whether the Resolved block should be collapsible or always inline. Start inline; revisit based on output length feedback.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Default `--competitors` to 2 peers**
|
||||
|
||||
**Goal:** Change the bare `--competitors` default from 3 to 2 per user feedback. `--competitors=N` still overrides; range 1..6 unchanged.
|
||||
|
||||
**Requirements:** R1
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (`COMPETITORS_DEFAULT`, `--competitors` const, stderr messages if any reference 3)
|
||||
- Modify: `SKILL.md` Competitor mode section ("discovered 2-6" wording, bare-flag default line)
|
||||
- Modify: `README.md` auto-discovered example line (if it references count)
|
||||
- Test: `tests/test_cli_competitors.py`
|
||||
|
||||
**Approach:**
|
||||
- Change `COMPETITORS_DEFAULT = 3` → `2` in `scripts/last30days.py`.
|
||||
- Change argparse `--competitors` `const=3` → `const=2`.
|
||||
- Update any SKILL.md / README copy referencing "3 peers" to "2 peers" (default) or "2-6 peers" (range).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing default constants in `scripts/last30days.py` argparse block.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: bare `--competitors` yields count=2, enabled=True, empty explicit_list.
|
||||
- Edge case: `--competitors=3` still works (explicit override).
|
||||
- Edge case: existing `test_bare_flag_defaults_to_three` test is updated to `test_bare_flag_defaults_to_two` and asserts count=2.
|
||||
- Edge case: `--competitors=5` with a `--competitors-list` of length 2 still logs the mismatch warning and uses the list.
|
||||
|
||||
**Verification:**
|
||||
- `pytest tests/test_cli_competitors.py -v` passes with the updated default.
|
||||
|
||||
- [ ] **Unit 2: Per-entity Step 0.55 resolution inside fanout**
|
||||
|
||||
**Goal:** Each competitor sub-run auto-resolves its own X handle, subreddits, GitHub user/repos, and news context via `resolve.auto_resolve()` before its `pipeline.run()` call — just like the main topic.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** None (but Unit 3 should land together so sub-runs don't emit LAW 7 stderr while the resolution context is being passed)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/fanout.py`
|
||||
- Modify: `scripts/last30days.py` (`_competitor_runner` closure builds the resolved args)
|
||||
- Test: `tests/test_competitor_fanout.py`
|
||||
- Test: `tests/test_competitors_resolve_integration.py` (new; covers the auto-resolve path)
|
||||
|
||||
**Approach:**
|
||||
- `_competitor_runner(entity)` in main() does:
|
||||
1. Call `resolve.auto_resolve(entity, config)` when `not args.mock` and a web backend is configured (reuse `_has_backend`).
|
||||
2. Extract resolved x_handle, subreddits, github_user, github_repos, context.
|
||||
3. Pass them to `pipeline.run()` for that sub-run.
|
||||
4. Inject resolved context into a per-entity config copy (so `_auto_resolve_context` does not leak across sub-runs — deep-copy the config or use a local dict).
|
||||
5. Store the resolved block on the Report's `artifacts` so the renderer can surface it (Unit 4).
|
||||
- When `args.mock` is True or no backend is available, skip auto-resolve (fall through to planner defaults, matching the existing `auto_resolve()` early-return contract).
|
||||
- Update `fanout.run_competitor_fanout` docstring to note that auto-resolve happens inside the caller-provided runner.
|
||||
|
||||
**Execution note:** Start with a failing integration test that exercises two-entity fanout + auto-resolve via a mocked `resolve.auto_resolve` and asserts that `pipeline.run` receives the resolved x_handle/subreddits for each entity.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/last30days.py` main topic branch (`if args.auto_resolve and not external_plan`) already calls `resolve.auto_resolve` and propagates results — mirror the shape for competitors.
|
||||
- Config isolation: `scripts/lib/pipeline.py:162-220` reads config as-is; use `dict(config)` to avoid cross-sub-run mutation of `_auto_resolve_context`.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: 3 entities, mocked `auto_resolve` returns distinct handles per entity; `pipeline.run` receives `x_handle=@drake` for Drake, `x_handle=@kendricklamar` for Kendrick, etc.
|
||||
- Happy path: the main topic still uses the user-supplied `--x-handle` / `--subreddits` overrides (not overwritten by auto-resolve for the main). Competitors use their own auto-resolved values.
|
||||
- Edge case: `--mock` skips auto-resolve entirely for all sub-runs (no `resolve.auto_resolve` calls).
|
||||
- Edge case: `resolve.auto_resolve` returns empty dicts for one entity (low-signal topic) — the sub-run still executes with planner defaults; doesn't crash.
|
||||
- Edge case: no web backend configured — auto-resolve returns empty for every entity, sub-runs fall through to planner defaults, no stack trace.
|
||||
- Error path: `resolve.auto_resolve` raises — the sub-run logs a warning and continues with planner defaults (does not fail the whole comparison).
|
||||
- Integration: config `_auto_resolve_context` from entity A does not leak into entity B's `pipeline.run`. Assert each sub-run gets its own context string.
|
||||
|
||||
**Verification:**
|
||||
- New integration test passes.
|
||||
- End-to-end smoke (mock mode + explicit list): each sub-run's stderr shows `[AutoResolve]` lines per entity with distinct values.
|
||||
|
||||
- [ ] **Unit 3: Suppress LAW 7 warning for engine-internal sub-runs**
|
||||
|
||||
**Goal:** The `[Planner] No --plan passed... deterministic fallback` warning does not fire during competitor sub-runs. LAW 7 is load-bearing for hosting-model contracts and must stay on the default path; this is an opt-in bypass for internal fan-out only.
|
||||
|
||||
**Requirements:** R3
|
||||
|
||||
**Dependencies:** Unit 2 (so the sub-run call site is already being modified)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/planner.py` (`plan_query` signature + conditional stderr)
|
||||
- Modify: `scripts/lib/pipeline.py` (`run` signature + propagation)
|
||||
- Modify: `scripts/last30days.py` or `scripts/lib/fanout.py` (pass `internal_subrun=True` for competitor runners)
|
||||
- Test: `tests/test_planner_v3.py` (or new `tests/test_planner_quiet_mode.py`)
|
||||
- Test: `tests/test_competitor_fanout.py` (assert sub-runs don't emit LAW 7 stderr)
|
||||
|
||||
**Approach:**
|
||||
- Add a keyword `internal_subrun: bool = False` to `planner.plan_query`. When True, skip the two `print(..., file=sys.stderr)` blocks that emit the LAW 7 banner and the `[Planner] No --plan passed` capability message.
|
||||
- Add the same keyword to `pipeline.run()`; pass through to `plan_query`.
|
||||
- In main()/fanout, set `internal_subrun=True` for every competitor sub-run's pipeline.run call. The main topic's pipeline.run keeps the default (LAW 7 stays on for the hosting-model path).
|
||||
- Also suppress the LAW 7-triggered degraded-run warning block in the render layer for sub-reports when the envelope is going to be merged into a comparison output (or accept that the block is per-entity and surfaces once per entity).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing keyword-only parameters on `pipeline.run` (`mock`, `x_handle`, etc.).
|
||||
- `planner.plan_query` signature is already keyword-only.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `plan_query(..., internal_subrun=True, provider=None, model=None)` returns the deterministic fallback plan WITHOUT writing the LAW 7 stderr block.
|
||||
- Happy path: `plan_query(...)` with default `internal_subrun=False` still writes the LAW 7 warning (unchanged behavior).
|
||||
- Integration: end-to-end competitor fanout; assert captured stderr contains zero occurrences of `No --plan passed` and zero of `YOU ARE the planner`.
|
||||
- Integration: main topic is not part of competitor mode; if the user invokes bare `/last30days OpenAI` without `--plan`, LAW 7 stderr fires exactly once (regression test).
|
||||
|
||||
**Verification:**
|
||||
- Running the Kanye-style smoke test shows zero `[Planner] No --plan passed` lines for Drake / Kendrick / Travis sub-runs.
|
||||
|
||||
- [ ] **Unit 4: "Resolved entities" block in comparison output**
|
||||
|
||||
**Goal:** The rendered comparison output includes a visible block listing per-entity handles, subreddits, GitHub user, and resolved context. Answers "did it resolve everyone?" at a glance without reading stderr.
|
||||
|
||||
**Requirements:** R4
|
||||
|
||||
**Dependencies:** Unit 2 (needs resolved data on report artifacts)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/render.py` (`render_comparison_multi` and `render_comparison_multi_context`)
|
||||
- Test: `tests/test_render_comparison_multi.py`
|
||||
|
||||
**Approach:**
|
||||
- When each entity's `Report.artifacts` contains a `resolved` dict (populated by Unit 2), `render_comparison_multi` emits a `## Resolved Entities` block early in the EVIDENCE envelope:
|
||||
```
|
||||
## Resolved Entities
|
||||
- **Kanye West**: X @kanyewest | Subs r/Kanye, r/hiphopheads | GitHub: — | Context: BULLY released, UK ban…
|
||||
- **Drake**: X @Drake | Subs r/DrakeTheType, r/hiphopheads | GitHub: — | Context: ICEMAN rollout…
|
||||
- **Kendrick Lamar**: X @kendricklamar | Subs r/KendrickLamar | GitHub: — | Context: Grammy wins, dormant…
|
||||
```
|
||||
- Missing fields render as `—` not empty.
|
||||
- When no entity has a `resolved` payload (mock mode, no web backend), omit the block entirely rather than emit an empty section.
|
||||
- Context strings are truncated at 120 chars to keep the block scannable.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing `render_comparison_multi` envelope structure (lines ~395-480 in render.py).
|
||||
- Existing per-entity evidence block format (`## {label}`) for consistency.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: 3 entities each with a `resolved` artifact → block lists all 3 with their fields.
|
||||
- Happy path: 2 entities, one with full resolution, one with partial (x_handle only) → missing fields render as `—`.
|
||||
- Edge case: no entity has a resolved artifact → block is omitted entirely.
|
||||
- Edge case: context string > 120 chars → truncated with ellipsis.
|
||||
- Integration: rendered output passes through the same EVIDENCE envelope comments and synthesis contract (LAW 1–8 unchanged).
|
||||
|
||||
**Verification:**
|
||||
- Snapshot tests confirm the block appears in the right spot with the right formatting.
|
||||
- End-to-end smoke shows a realistic 3-entity Resolved block in the rendered output.
|
||||
|
||||
- [ ] **Unit 5: SKILL.md canonical-path self-check**
|
||||
|
||||
**Goal:** A top-of-file SKILL.md directive forces any reader (Claude Code, Codex, Hermes, Gemini) to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` before proceeding. If loaded from `marketplaces/` or any other path, re-read from the pinned versioned cache.
|
||||
|
||||
**Requirements:** R5
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md` (prepend a STEP 0 block before the existing STEP 0 / LAW list)
|
||||
|
||||
**Approach:**
|
||||
- Add a numbered first step at the top (before or bundled with existing "STEP 0: ToolSearch preload"):
|
||||
```
|
||||
## STEP 0: Canonical Path Self-Check (must run first)
|
||||
|
||||
Before reading anything else below, verify you loaded this SKILL.md from
|
||||
the versioned cache, not the marketplace clone:
|
||||
|
||||
CANONICAL=$HOME/.claude/plugins/cache/last30days-skill/last30days/
|
||||
CANONICAL_LATEST=$(ls -d "$CANONICAL"*/ 2>/dev/null | sort -V | tail -1)
|
||||
|
||||
If the SKILL.md you just read is not under $CANONICAL_LATEST, STOP. Re-read
|
||||
$CANONICAL_LATEST/SKILL.md and restart from here. Marketplace clones
|
||||
(`plugins/marketplaces/last30days-skill/`) are pinned to origin/main and
|
||||
can be stale; the versioned cache is the ground truth.
|
||||
```
|
||||
- Reinforce in the existing LAW 7 block that `--help` output must be read from the same pinned `SKILL_ROOT` to avoid flag-list skew.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing STEP 0 ToolSearch preload (top of SKILL.md) for tone / imperative voice.
|
||||
- Existing `SKILL_ROOT` resolver snippet (line ~823).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — SKILL.md is documentation; no unit test, verified by follow-up user invocation.
|
||||
|
||||
**Verification:**
|
||||
- In a fresh Claude Code window, `/last30days Test --competitors` loads SKILL.md, the model executes the STEP 0 self-check, and (if it had loaded from marketplaces/) switches to the cache path before running `--help` or the engine. Observable via the model's announced reasoning / task list.
|
||||
|
||||
- [ ] **Unit 6: Version bump, CHANGELOG, sync**
|
||||
|
||||
**Goal:** Ship 3.0.12 and deploy to all local targets.
|
||||
|
||||
**Requirements:** R6
|
||||
|
||||
**Dependencies:** Units 1-5
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json` (version 3.0.11 → 3.0.12)
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG entry under `## [3.0.12]` dated 2026-04-22 covering the four fixes (Fixed: per-entity resolution; Fixed: LAW 7 sub-run noise; Changed: default count 3→2; Added: Resolved entities block; Added: canonical-path self-check in SKILL.md).
|
||||
- `sync.sh` deploys to `~/.claude/plugins/cache/last30days-skill-private/...`, `~/.agents/`, `~/.codex/`, Hermes.
|
||||
- Manual hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.12/` so the public `/last30days` slash command picks up the new version before PR merge (matches the 3.0.11 testing pattern).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging only. Verification is by inspection.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` returns `3.0.12`.
|
||||
- `sync.sh` exits 0 with "Import check: OK" for each target.
|
||||
- Hot-copied 3.0.12 directory contains the new files and `/last30days` picks up the new version (highest-version resolver).
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** Fanout sub-runs now call `resolve.auto_resolve` per entity. Each sub-run is independent; no shared mutable state with other sub-runs or with the main topic.
|
||||
- **Error propagation:** `auto_resolve` failures inside a sub-run log a warning and degrade to planner defaults; do not propagate up to abort the comparison. Same contract as today for the main topic.
|
||||
- **State lifecycle risks:** Config dict is mutated by `auto_resolve` (via `config["_auto_resolve_context"]`). Must deep-copy per sub-run or scope context to a local mapping — otherwise two sub-runs' context strings race.
|
||||
- **API surface parity:** `pipeline.run` gains a keyword (`internal_subrun`); callers that don't pass it get the existing behavior. `planner.plan_query` gains the same. Backward compatible.
|
||||
- **Integration coverage:** New integration test for the fanout + auto-resolve + render chain. Existing snapshot tests update to include the Resolved block.
|
||||
- **Unchanged invariants:** Single-entity `/last30days` invocations (no `--competitors`) behave identically. Explicit `A vs B` comparison topics behave identically. LAW 7 still fires on the default hosting-model path. `render_compact` path is untouched.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Auto-resolving per competitor triples the WebSearch call volume (4 queries × 3 competitors = 12 extra web searches). | Fast-fail when no backend; user can pass `--competitors-list` to skip discovery but still get auto-resolve. Cost note in CHANGELOG. |
|
||||
| Config mutation across sub-runs via `_auto_resolve_context`. | Unit 2 deep-copies config per sub-run before each `auto_resolve` + `pipeline.run` call. Integration test asserts no cross-entity leak. |
|
||||
| LAW 7 suppression leaks onto the hosting-model path via a wrong default. | Default `internal_subrun=False`. Only fanout's competitor sub-runs set True. Unit test asserts bare-topic invocation still emits LAW 7. |
|
||||
| SKILL.md STEP 0 banner gets ignored by the model (same failure mode as line 823 today). | Put it in the guaranteed-read top band (before LAW 1, above all other content), imperative voice, concrete `STOP` verb. Still not bulletproof but strictly better than current. |
|
||||
| Default count change breaks assumptions in downstream tools or existing user muscle memory. | Changelog calls it out as Changed; `--competitors=3` still works for users who want the old default. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first: merge behind `/last30days-beta` via the private repo before cherry-picking to public. Follows the same process as 3.0.11.
|
||||
- Version 3.0.12 is a fix release; no marketing post required.
|
||||
- After merge, add a line to the PR description pointing at this plan.
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Origin plan: `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
||||
- Related PR: #308 (v3.0.11 shipping --competitors)
|
||||
- Test windows that surfaced the bugs: Kanye, Linear, Coinbase (2026-04-22 session)
|
||||
- Related code: `scripts/lib/fanout.py`, `scripts/lib/resolve.py` (`auto_resolve`), `scripts/lib/planner.py` (`plan_query`), `scripts/lib/render.py` (`render_comparison_multi`)
|
||||
-394
@@ -1,394 +0,0 @@
|
||||
---
|
||||
title: "fix: --competitors runs a full last30days per entity with hosting-model pre-resolve"
|
||||
type: fix
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
origin: docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md
|
||||
---
|
||||
|
||||
# fix: --competitors runs a full last30days per entity with hosting-model pre-resolve
|
||||
|
||||
## Overview
|
||||
|
||||
User intent confirmed 2026-04-22: `--competitors` should run a full single-entity `last30days` pipeline for the main topic AND for each discovered peer — three independent full-depth passes, each with its own Step 0.55 resolution, own X handle primary weight, own subreddit targeting, own GitHub repo scoping. Then merge them into the comparison output.
|
||||
|
||||
3.0.12 already built the N-parallel-pipelines orchestration (`scripts/lib/fanout.py`). What it got wrong: it tried to do per-entity Step 0.55 engine-side via `resolve.auto_resolve()`, which requires a web search backend key (BRAVE/EXA/SERPER/PARALLEL/OPENROUTER). Matt runs from Claude Code, which has its own WebSearch tool. The engine has none of those keys, so per-entity auto_resolve silently no-ops and all peer sub-runs fall through to deterministic single-word planner queries.
|
||||
|
||||
Four 2026-04-22 test runs (Warriors, Seattle, Arizona Wildcats, Kanye West) confirmed this via engine receipts:
|
||||
|
||||
- Compact Resolved Entities block shows peers as `X - | Subs - | GitHub - | Context: -`.
|
||||
- Sub-run planner lines show `source=deterministic, subqueries=1` — the "I gave up and keyword-searched" shape.
|
||||
- Engine footer keeps nudging `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY`, which is wrong advice for a Claude Code user who already has WebSearch.
|
||||
- Kanye run leaked main topic's `--subreddits` into Drake's and Kendrick's sub-runs (regression bug).
|
||||
|
||||
The fix is to flip the resolution responsibility: the hosting model (Claude Code, Codex, Hermes, Gemini) does Step 0.55 via its own WebSearch tool for every entity, then passes the resolved targeting to the engine via a new `--competitors-plan` JSON flag. Engine fan-out remains — each peer still runs a full `pipeline.run()`. The difference is the peers now arrive with full targeting, equivalent to the main topic, so retrieval is apples-to-apples.
|
||||
|
||||
Why not just reuse vs-mode? vs-mode is a SINGLE `pipeline.run()` with a comparison-optimized plan. It pre-resolves Step 0.55 per entity but merges everything into one retrieval pool with lower-weight `--x-related` for peers, merged subreddits, and cross-entity keyword noise. That is not "three full passes." The user explicitly wants three full passes.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
3.0.12's architecture was correct; its data dependency was wrong.
|
||||
|
||||
| Capability | 3.0.12 path | Target path (this plan) |
|
||||
|---|---|---|
|
||||
| Fan out to N parallel pipelines | Yes (`fanout.run_competitor_fanout`) | Same — keep |
|
||||
| Per-entity Step 0.55 resolution | Engine-internal `resolve.auto_resolve()` — needs BRAVE/EXA/SERPER/PARALLEL key | Hosting model does it via its own WebSearch, passes to engine |
|
||||
| Per-entity targeting threaded into `pipeline.run()` | Main topic only via outer flags; peers via auto_resolve (failing) or nothing | Main topic via outer flags; peers via `--competitors-plan` JSON |
|
||||
| Footer nudge | Unconditional BRAVE/SERPER | Suppressed when `--plan` or `--competitors-plan` present |
|
||||
| Resolved Entities block in raw save file | Stdout only | Also in `--save-dir` raw file |
|
||||
| Override-leak from main into peers | Present (Kanye receipt) | Fixed via explicit per-entity kwargs scrub |
|
||||
| Polymarket noise on ambiguous topics | Present (Warriors, Arizona receipts) | `--polymarket-keywords` + auto-skip for single-token-ambiguous |
|
||||
|
||||
The key architectural change is who owns per-entity resolution. The engine stops trying to do it itself; the hosting model does it upstream (it already has WebSearch) and passes results in.
|
||||
|
||||
This is the same pattern `--plan` already uses for the main topic: hosting model generates the plan via its own reasoning, passes it in, engine accepts. We apply the pattern to peers.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. New `--competitors-plan` JSON flag accepting per-entity targeting: `x_handle`, `x_related`, `subreddits`, `github_user`, `github_repos`, `context`. Implies `--competitors`. Per-entity values thread into that entity's `pipeline.run()`. Bypasses engine-internal `auto_resolve` for covered entities.
|
||||
- R2. SKILL.md "Competitor mode" rewritten to make the hosting-model path canonical: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers) via WebSearch, (c) assemble `--competitors-plan` JSON, (d) invoke engine. Engine-internal auto_resolve remains as headless fallback.
|
||||
- R3. The LAW 7-style stderr emitted when `--competitors` has no list, no plan, no backend is reframed: leads with "hosting reasoning model, use your WebSearch to run Step 0.55 per entity and pass `--competitors-plan`." Does not lead with BRAVE_API_KEY.
|
||||
- R4. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` OR `--competitors-plan` was passed. Signal: hosting model is driving and already has WebSearch.
|
||||
- R5. Override-leak fix: competitor sub-runs do not inherit main topic's `--subreddits`, `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`. Sub-runs use only their own per-entity targeting (from `--competitors-plan` if provided, else engine-internal auto_resolve if backend, else planner defaults).
|
||||
- R6. The `## Resolved Entities` block is also appended to the saved raw file when `--save-dir` is in use. Each entity's effective targeting (whatever was actually passed to its `pipeline.run()`) is visible on audit.
|
||||
- R6b. When `--save-dir` is in use with a comparison run, each entity's sub-run ALSO saves its own standalone raw file — same format as a single-entity run. `/last30days Kanye West --competitors` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md` (one per entity) plus the merged comparison file. Matches the historical vs-mode behavior when it ran as N passes.
|
||||
- R7. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
||||
- R8. Default `--competitors` count remains 2 (3-way: main + 2 peers). Unchanged from 3.0.12.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No changes to `scripts/lib/fanout.py` architecture. N parallel pipelines stays. Only the data each sub-run receives changes.
|
||||
- No changes to the vs-mode (topic contains "vs" / "versus") behavior. That path is independent.
|
||||
- No new emit modes. Comparison output format unchanged.
|
||||
- No deprecation of `--competitors-list`. Stays as the minimum escape hatch for hosting models that skip per-entity Step 0.55 (names-only).
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Cache layer for hosting-model competitor resolution: separate plan once cost evidence exists.
|
||||
- Cross-source disambiguation beyond Polymarket: separate plan.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py` — `--competitors` / `--competitors-list` argparse block, `resolve_competitors_args` validator, `_main_runner` closure, `_competitor_runner` closure, the `[Competitors] --competitors requires...` stderr block. Primary file for this plan.
|
||||
- `scripts/lib/fanout.py` — `run_competitor_fanout` orchestrator. Signature unchanged; `_competitor_runner` closure now builds kwargs from `--competitors-plan`.
|
||||
- `scripts/lib/pipeline.py` — `pipeline.run()` signature; no changes required (all per-entity flags already exist as kwargs).
|
||||
- `scripts/lib/planner.py` — existing `--plan` parsing and validation, pattern to mirror for `--competitors-plan`.
|
||||
- `scripts/lib/render.py` `_render_resolved_entities_block` (added in 3.0.12) — already reads `report.artifacts["resolved"]`; no change needed.
|
||||
- `scripts/last30days.py` `save_output` / `render.render_full` — the save path. Needs to include the Resolved Entities block for comparison runs.
|
||||
- `scripts/lib/quality_nudge.py` — where the BRAVE/SERPER footer nudge is emitted. Needs a context-aware suppression check.
|
||||
- `scripts/lib/polymarket.py` — source adapter. Entry point for `--polymarket-keywords` filter and single-token-ambiguous auto-skip.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- 3.0.11 plan (`2026-04-22-002`): built the initial fanout, deferred per-entity resolve as "v1 simplification."
|
||||
- 3.0.12 plan (`2026-04-22-003`): tried to close the gap via engine-internal `auto_resolve`. Works only with backend keys. Fails silently without.
|
||||
- 2026-04-22 test session receipts: confirmed all four fixes in this plan are real, reproducible bugs.
|
||||
- User's architectural steer 2026-04-22: "runs a full last30days on all 3 topics" — this plan encodes that explicitly as N full `pipeline.run()` calls with pre-resolved targeting per entity.
|
||||
|
||||
### External References
|
||||
|
||||
- None. All patterns in-repo.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **`--competitors-plan` is a single JSON flag, not a fan of separate flags.** Mirrors `--plan`. Stable schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Accept inline JSON or a file path (matches `--plan`).
|
||||
- **Hosting-model-driven resolution is the documented default.** Engine-internal `auto_resolve` is the headless / cron fallback. SKILL.md routes hosting models to the JSON-flag path; engine keeps auto_resolve alive for BRAVE/EXA/SERPER users running CI.
|
||||
- **Override-leak fix is call-site scrubbing, not a signature change.** `_competitor_runner` builds an explicit kwargs dict per entity from `_subrun_kwargs(entity, plan_entry)`. No closure-default fallthrough from main scope. The 3.0.12 `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag.
|
||||
- **Footer nudge becomes context-aware.** Suppressed when `--plan` or `--competitors-plan` present. Not suppressed for bare `--competitors-list` or bare invocations. Headless cron without keys still sees the nudge.
|
||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known list of single-token-ambiguous names (states, common nouns). Stderr notes the skip so it is observable and overridable.
|
||||
- **Per-entity sub-runs get the full `pipeline.run()` pass.** Same depth, same sources, same API cost per entity as a single-topic run. This is the explicit user intent — three full passes, not one merged pass.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **JSON or multi-flag?** JSON. Matches `--plan`.
|
||||
- **Default count?** 2 peers (3-way comparison). Unchanged from 3.0.12.
|
||||
- **Does engine-internal auto_resolve stay alive?** Yes, for entities not covered by `--competitors-plan` when a backend is configured. Headless/cron users with keys keep the current 3.0.12 behavior.
|
||||
- **vs-mode or fanout?** Fanout. User's explicit ask: three full passes, not one merged pass. vs-mode merges into one pipeline with lower peer weighting, which is not what the user wants.
|
||||
- **Does the save file need per-entity clusters?** Start with the Resolved block appended. Per-entity cluster sections can follow in a separate task; they are nice-to-have, not blocking.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact trace of override-leak source. Candidates: closure capture of `subreddits` in `_competitor_runner`, shared `_auto_resolve_context` leak, Reddit adapter inheriting global config. Test-first; trace at implementation time.
|
||||
- Heuristic for "single-token-ambiguous topic" auto-skip. Start with a short hard-coded list (US state names, US city names, common nouns like "Warriors", "Suns", "Jets"); revisit after dogfood.
|
||||
- Whether per-entity coverage warnings fire when `--competitors-plan` under-resolves an entity (e.g., only `x_handle`, no subreddits). Start with stderr logging; revisit UX.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: `--competitors-plan` JSON flag + per-entity kwargs threading**
|
||||
|
||||
**Goal:** New CLI flag accepting per-entity targeting JSON. Each covered entity's `pipeline.run()` receives its own `x_handle` / `x_related` / `subreddits` / `github_user` / `github_repos` / `context`. Skips engine-internal `auto_resolve` for covered entities.
|
||||
|
||||
**Requirements:** R1, R5 (primary leak fix site)
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner`)
|
||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; verify)
|
||||
- Test: `tests/test_cli_competitors.py` (extend)
|
||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON OR a file path (mirror `--plan`).
|
||||
- Validation: parse JSON; must be a dict; each value must be a dict; unknown fields log warnings; malformed input exits 2.
|
||||
- Schema per entity: optional fields `x_handle` (str), `x_related` (list), `subreddits` (list), `github_user` (str), `github_repos` (list), `context` (str).
|
||||
- Case-insensitive matching against `--competitors-list` / discovered entities.
|
||||
- Build `_subrun_kwargs(entity, plan_entry)` helper. Returns a complete, explicit kwargs dict for `pipeline.run()` with no closure-default fallthrough from main scope. This helper is the single source of truth for per-entity call args. It also fixes the override-leak (R5) by scrubbing all per-entity flags to None unless the plan (or auto_resolve) sets them.
|
||||
- `_competitor_runner(entity)`:
|
||||
1. Look up `plan_entry` from `--competitors-plan` (if any).
|
||||
2. If plan covers entity fully, build kwargs from it; skip `auto_resolve`.
|
||||
3. If plan partially covers or is absent, fall back to `auto_resolve` (3.0.12 behavior) when a backend is configured. Plan values win over auto_resolve values on conflict.
|
||||
4. If neither plan nor backend, fall through to `pipeline.run()` with per-entity kwargs all None — engine uses planner defaults for that entity only (no leak).
|
||||
- Deep-copy config per sub-run (already done in 3.0.12); merge per-entity `context` into `entity_config["_auto_resolve_context"]` only.
|
||||
|
||||
**Execution note:** Test-first for the override-leak regression (pass `--subreddits=A,B` on main + a peer, assert peer's `pipeline.run(subreddits=...)` is None or peer-specific).
|
||||
|
||||
**Patterns to follow:**
|
||||
- `--plan` parsing at `scripts/last30days.py` (inline JSON or file path).
|
||||
- 3.0.12's `_competitor_runner` closure for scope; extract the kwargs-build into `_subrun_kwargs` helper.
|
||||
- `entity_config = dict(config)` deep-copy pattern from 3.0.12.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors-plan '{"Drake": {"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's `pipeline.run` receives `x_handle="Drake"` and `subreddits=["Drizzy"]`; no `auto_resolve` call for Drake.
|
||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered entities skip auto_resolve; third falls back to auto_resolve.
|
||||
- Happy path: plan file path accepted like `--plan` file path.
|
||||
- Happy path: case-insensitive entity match (`Drake` in plan, `drake` in list).
|
||||
- Edge case: unknown fields in plan entry → logged, ignored, run continues.
|
||||
- Edge case: plan entry for entity not in list → ignored with warning.
|
||||
- Error path: malformed JSON → exit 2.
|
||||
- Error path: top-level JSON is list not dict → exit 2.
|
||||
- Regression (leak fix): main `--subreddits=A,B` + `--competitors-list "Drake"` + no plan → Drake's `pipeline.run` receives `subreddits=None` (no leak).
|
||||
- Regression (leak fix): same for `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`.
|
||||
- Regression (leak fix): main `--x-handle=kanyewest` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanyewest"`.
|
||||
- Integration: full main + 2 peers run via `--competitors-plan`; assert each sub-run's effective kwargs match expected per-entity values.
|
||||
|
||||
**Verification:**
|
||||
- All new and regression tests pass.
|
||||
- Smoke run (mock mode + `--competitors-plan`): stderr shows `[Competitors] Drake: x=@Drake subs=Drizzy` line per entity; no `[AutoResolve]` calls for plan-covered entities; no leak of main topic's flags.
|
||||
|
||||
- [ ] **Unit 2: Reframe LAW 7-style stderr for hosting-model context**
|
||||
|
||||
**Goal:** When `--competitors` has no `--competitors-list`, no `--competitors-plan`, and no backend, stderr tells the hosting reasoning model to use its WebSearch tool for Step 0.55 per entity and pass `--competitors-plan`. Stops leading with BRAVE_API_KEY.
|
||||
|
||||
**Requirements:** R3
|
||||
|
||||
**Dependencies:** Unit 1 (flag must exist)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (the existing `[Competitors] --competitors requires...` block)
|
||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Rewrite stderr in this order:
|
||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent runtime with a WebSearch tool), YOU should: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers), (c) assemble a `--competitors-plan` JSON, (d) re-invoke. Skip this step and quality degrades — peer entities will run with planner defaults."
|
||||
2. "If you are running headless (cron, CI, no hosting model), set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
||||
3. "Minimum escape hatch: `--competitors-list "A,B,C"` skips discovery but does not pre-resolve peers. Use only for quick tests."
|
||||
- Exits non-zero as today.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing LAW 7 stderr in `planner.plan_query` for tone.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: stderr leads with "If you are the hosting reasoning model" and names `--competitors-plan` before any backend key.
|
||||
- Happy path: stderr explicitly names `--competitors-plan` as the preferred override.
|
||||
- Happy path: stderr does NOT say "requires either a configured web search backend OR an explicit --competitors-list" (the current 3.0.12 wording).
|
||||
|
||||
**Verification:**
|
||||
- Test asserts ordering and required phrases.
|
||||
|
||||
- [ ] **Unit 3: Suppress BRAVE/SERPER footer nudge when hosting-model-driven**
|
||||
|
||||
**Goal:** The `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY` footer is suppressed when `--plan` or `--competitors-plan` was passed (signal: hosting model is driving and already has WebSearch).
|
||||
|
||||
**Requirements:** R4
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever nudge is emitted; verify during implementation)
|
||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Locate the nudge emission point.
|
||||
- Add a suppression check: if `--plan` OR `--competitors-plan` was passed, skip the nudge. Otherwise, current behavior.
|
||||
- Don't suppress the nudge for bare `--competitors-list` alone — that path isn't necessarily hosting-model-driven.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--plan` passed, no backend → nudge does NOT fire.
|
||||
- Happy path: `--competitors-plan` passed, no backend → nudge does NOT fire.
|
||||
- Happy path: `--competitors-list` only, no backend → nudge fires (current behavior).
|
||||
- Happy path: no `--competitors`, no `--plan`, no backend → nudge fires (current behavior unchanged).
|
||||
|
||||
**Verification:**
|
||||
- All four scenarios produce expected nudge presence/absence.
|
||||
|
||||
- [ ] **Unit 4: Per-entity save files + Resolved block in each**
|
||||
|
||||
**Goal:** When `--save-dir` is in use with a comparison run, each entity's sub-run saves its own standalone raw file (same format as a single-entity run), and each file includes the `## Resolved Entities` block so audits can see what targeting that entity received. Matches the historical vs-mode behavior when it was N passes.
|
||||
|
||||
**Requirements:** R6, R6b
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (`save_output`, the save loop after fanout completes)
|
||||
- Possibly modify: `scripts/lib/render.py` (`render_full` branch to include Resolved block when artifact is present)
|
||||
- Test: `tests/test_save_raw_competitor_files.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]`. For each `(entity, entity_report)` tuple, call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)` — same path a single-entity run takes.
|
||||
- Each saved file uses its entity's slug as the filename (`drake-raw.md`, `kendrick-lamar-raw.md`). Main topic keeps the existing `kanye-west-raw.md` filename.
|
||||
- Each file includes its own `## Resolved Entities` block (single-entity variant: one row for that entity only). This makes each sub-run's file self-describing — you can see what targeting was used without opening the comparison file.
|
||||
- The merged comparison output (stdout) still includes the 3-row Resolved Entities block.
|
||||
- Optional: also save a comparison summary file (e.g., `kanye-west-comparison-raw.md`) holding the merged multi-entity render. Start with per-entity files only; comparison summary is a follow-up if stdout-plus-individual-files is insufficient.
|
||||
- Single-entity runs unchanged (no additional files, no block change).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing `save_output` invocation for single-entity runs (line 501 of current `scripts/last30days.py`).
|
||||
- Existing slug generation (`slugify(topic)`) for filename consistency.
|
||||
- `_render_resolved_entities_block` from 3.0.12 for the single-entity variant.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors-list "Drake,Kendrick Lamar"` + `--save-dir=/tmp/x` → `/tmp/x/kanye-west-raw.md`, `/tmp/x/drake-raw.md`, `/tmp/x/kendrick-lamar-raw.md` all exist.
|
||||
- Happy path: each peer file's first sections include that entity's Resolved Entities block with its own row only.
|
||||
- Happy path: single-entity run with `--save-dir` → one file, unchanged from today's behavior.
|
||||
- Edge case: entity slug collides with existing file → overwrite (matches single-entity behavior).
|
||||
- Edge case: `--save-suffix=v3` → all 3 files get the suffix (`kanye-west-raw-v3.md`, `drake-raw-v3.md`, `kendrick-lamar-raw-v3.md`).
|
||||
- Edge case: comparison run with one peer whose sub-run failed → that entity's file is NOT saved; others are.
|
||||
- Integration: stderr after save shows three `[last30days] Saved output to <path>` lines, one per entity.
|
||||
|
||||
**Verification:**
|
||||
- After `/last30days Kanye West --competitors-list "Drake,Kendrick Lamar" --save-dir=/tmp/x`: `ls /tmp/x/*-raw.md` shows 3 files. Each contains its entity's Resolved block.
|
||||
|
||||
- [ ] **Unit 5: SKILL.md "Competitor mode" rewrite — hosting-model Step 0.55 canonical**
|
||||
|
||||
**Goal:** SKILL.md documents the hosting-model-driven path as canonical: discover N peers via WebSearch, run Step 0.55 per entity, assemble `--competitors-plan`, invoke engine. Engine-internal `auto_resolve` is labeled the headless fallback.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 1 (flag must exist before documented)
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md` (Competitor mode subsection)
|
||||
- Modify: `README.md` (one-line example update)
|
||||
|
||||
**Approach:**
|
||||
- Replace the 3.0.12 Competitor mode subsection with a clear flow:
|
||||
1. User invokes with `--competitors` or `--competitors=N`.
|
||||
2. Hosting model runs WebSearch for "[topic] competitors" / "[topic] alternatives" → picks top N peers.
|
||||
3. Hosting model runs Step 0.55 for main + each peer (x_handle, subreddits, github_user, github_repos, context) — same protocol as vs-mode per SKILL.md §679.
|
||||
4. Hosting model assembles a `--competitors-plan` JSON object.
|
||||
5. Hosting model invokes the engine with `--competitors-list "A,B,C" --competitors-plan '{...}'`.
|
||||
6. Engine fans out N full pipelines (main + peers), each with its own full Step 0.55-grade targeting. Each entity also saves its own `*-raw.md` file when `--save-dir` is set (three full passes → three save files, matching the historical vs-mode behavior). Comparison output merges them for display.
|
||||
- Concrete JSON example in SKILL.md showing the schema.
|
||||
- Failure-mode warning: a `## Resolved Entities` block with dashes for any entity means hosting model skipped Step 0.55 for that one. Re-run with corrected plan.
|
||||
- "Headless fallback" sub-subsection: when BRAVE/EXA/SERPER/PARALLEL/OPENROUTER is set, engine's internal `auto_resolve` handles peers and `--competitors-plan` is optional.
|
||||
|
||||
**Patterns to follow:**
|
||||
- SKILL.md "Step 0.55" section for per-entity resolve protocol.
|
||||
- SKILL.md "If QUERY_TYPE = COMPARISON" section for the same-protocol-as-vs-mode reference.
|
||||
- Tone of existing 3.0.12 Competitor mode prose.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — documentation. Verification is a fresh Claude Code window dogfood run.
|
||||
|
||||
**Verification:**
|
||||
- `/last30days Kanye West --competitors` in a new window: hosting model does Step 0.55 for Kanye + 2 discovered peers; passes `--competitors-plan`; rendered Resolved block shows non-empty fields for all 3; top voices include at least one peer-specific handle.
|
||||
|
||||
- [ ] **Unit 6: Polymarket disambiguation guard**
|
||||
|
||||
**Goal:** Support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
||||
|
||||
**Requirements:** R7
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` argparse (`--polymarket-keywords`)
|
||||
- Modify: `scripts/lib/polymarket.py`
|
||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Add `--polymarket-keywords "kw1,kw2"` flag. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
||||
- Auto-skip rule: if topic is one token AND token matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords` provided, skip Polymarket with a stderr note.
|
||||
- SKILL.md Step 0.55 protocol gets a small addition: for ambiguous topics, hosting model passes `--polymarket-keywords` with topic-specific qualifiers.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing Polymarket adapter match logic.
|
||||
- Single-token detection heuristic.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr notes the skip.
|
||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs; matches filtered.
|
||||
- Happy path: topic "OpenAI" (no ambiguity) → Polymarket runs as before.
|
||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
||||
|
||||
**Verification:**
|
||||
- Warriors smoke run → Polymarket footer absent OR filtered to nba/gsw markets.
|
||||
|
||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
||||
|
||||
**Goal:** Ship 3.0.13 to all local targets.
|
||||
|
||||
**Requirements:** Closes R1-R7
|
||||
|
||||
**Dependencies:** Units 1-6
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json`
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG entry groups the fixes: Added `--competitors-plan` JSON flag for per-entity hosting-model pre-resolve. Fixed override-leak from main into peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when `--plan` / `--competitors-plan` is present. Added: Resolved Entities block persists to saved raw file. Added: `--polymarket-keywords` + auto-skip for ambiguous single-token topics.
|
||||
- Beta channel first per CLAUDE.md.
|
||||
- Hot-copy so public `/last30days` picks up 3.0.13 immediately.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` returns 3.0.13.
|
||||
- `sync.sh` exits 0.
|
||||
- Hot-copy contains the new files with competitors.py, fanout.py, the updated SKILL.md, and plugin.json 3.0.13.
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** `_competitor_runner` becomes the single source of truth for sub-run kwargs via `_subrun_kwargs(entity, plan_entry)`. Every per-entity flag flows through one helper. No closure-default leaks.
|
||||
- **Error propagation:** `--competitors-plan` JSON parse errors exit 2 with stderr (same as `--plan`). Per-entity plan entries with malformed values log warnings and fall back; don't abort the whole run.
|
||||
- **State lifecycle risks:** `entity_config = dict(config)` already deep-copies for `_auto_resolve_context`; extend the isolation discipline to every per-entity flag. Verified in Unit 1 regression tests.
|
||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors` and `--competitors-list` unchanged. `--plan` unchanged. `--polymarket-keywords` additive.
|
||||
- **Integration coverage:** New regression tests for override-leak. New integration test for plan-driven sub-run threading. New nudge-suppression test. New Polymarket disambiguation test.
|
||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. `planner.plan_query` LAW 7 behavior for the default path unchanged. Single-entity render path unchanged. vs-mode behavior unchanged.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Hosting model takes the lazy path and uses `--competitors-list` names-only. | Unit 2 stderr explicitly steers to `--competitors-plan` with Step 0.55 protocol named. Unit 5 SKILL.md docs. Resolved Entities dashes in output make the gap visible. |
|
||||
| JSON gets verbose for the hosting model to construct repeatedly. | Schema is small (≤6 fields per entity). Hosting model already runs Step 0.55 for main topic in every comparison run; peers use the same protocol. One JSON block replaces N CLI flags. |
|
||||
| Override-leak source is deeper than `_competitor_runner` closure. | Test-first per Unit 1. Receipts from 2026-04-22 Kanye run are reproducible. Trace methodically from call site. |
|
||||
| Plan-covered entity bypasses auto_resolve but plan data is incomplete (e.g., no subreddits). | Hosting model's own SKILL.md contract says Step 0.55 must cover all fields. Stderr logs per-entity coverage so under-resolved entities are visible. Next-run correction, not engine-side rescue. |
|
||||
| Polymarket auto-skip false-positives on legitimate ambiguous topics with real markets. | Conservative match (single-token + known list). `--polymarket-keywords` override is explicit and unambiguous. Stderr notes the skip. |
|
||||
| Footer nudge suppression hides the message from headless users who genuinely need it. | Suppression only fires when `--plan` or `--competitors-plan` is present. Cron / CI runs that pass neither still see the nudge. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first per CLAUDE.md (private repo `/last30days-beta`).
|
||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
||||
- CHANGELOG voice should call this out as the feedback-driven follow-up to 3.0.12. Reader should see "we tried engine-internal resolve in 3.0.12; it needs backend keys we don't have; we moved resolution to the hosting model in 3.0.13."
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Origin plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
||||
- Earlier plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
||||
- 2026-04-22 test session receipts: Warriors, Seattle, Arizona Wildcats, Kanye West
|
||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol
|
||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
||||
@@ -1,451 +0,0 @@
|
||||
---
|
||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
origin: docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded
|
||||
---
|
||||
|
||||
# feat: vs mode runs N full passes and --competitors is vs with auto-discovery
|
||||
|
||||
## Overview
|
||||
|
||||
Architectural unification driven by user correction 2026-04-22: vs mode and `--competitors` are the same thing. A user typing `/last30days OpenAI vs Anthropic vs xAI` should get a full single-entity last30days pass for each of the three entities — three full pipelines, three saved `*-raw.md` files, merged into one comparison output. A user typing `/last30days OpenAI --competitors` should get the same output after the hosting model auto-picks 2 peers; i.e., `--competitors` is a thin shortcut that expands "topic + `--competitors`" into "topic vs peer1 vs peer2" and then runs the unified vs pipeline.
|
||||
|
||||
Current state diverges from this:
|
||||
|
||||
- **vs mode today**: one `pipeline.run()` with a comparison-optimized plan that merges all entities' targeting into a single retrieval pool. Lower-weight `--x-related` for peers, merged subreddits, cross-entity keyword noise. One saved file.
|
||||
- **`--competitors` today (3.0.12)**: N parallel `pipeline.run()` calls via `scripts/lib/fanout.py`, but per-entity Step 0.55 depends on an engine-side web backend key Matt doesn't have. Silently degrades to planner defaults for peers. One saved file (main topic only). Override-leak from main into peers.
|
||||
|
||||
After this plan:
|
||||
|
||||
- **vs mode**: N parallel `pipeline.run()` calls, one per entity, each with its own full Step 0.55-grade targeting, each saving its own `*-raw.md`. Merged into one comparison output.
|
||||
- **`--competitors`**: SKILL.md shortcut. Hosting model discovers N peers, builds `"topic vs peer1 vs peer2"`, and invokes the same vs pipeline. No separate orchestration path.
|
||||
- **Same fanout machinery (`scripts/lib/fanout.py`)** serves both. One fix, both behaviors improve.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The product insight from 2026-04-22 test runs is simple: the user wants three full last30days reports plus a comparison merge. Not one comparison pass with N-way targeting merged into a single retrieval pool. Not one save file. Not "main gets Step 0.55, peers get planner defaults." Three full passes. Three save files. Merged output.
|
||||
|
||||
The historical vs mode did that (it ran as 3 passes, saving 3 files). SKILL.md §551 currently says:
|
||||
|
||||
> "When the user asks 'X vs Y', run ONE research pass with a comparison-optimized plan that covers both entities AND their rivalry. This replaces the old 3-pass approach (which took 13+ minutes and produced tangential content)."
|
||||
|
||||
That change was a latency optimization that removed the user-visible behavior the user wants. The fix is to revert the architectural direction: N passes per entity, in parallel rather than serial (parallelism lowers wall-clock to ~1× a single pass, not N×), with per-entity save files.
|
||||
|
||||
The 3.0.11 `--competitors` flag already introduced parallel N-pass machinery (`fanout.run_competitor_fanout`). The 3.0.12 follow-up tried to wire per-entity Step 0.55 into it but failed when no web backend was configured. The elegant move: stop maintaining two architectures. vs-mode and `--competitors` both use `fanout.py`. `--competitors` becomes a SKILL.md-level shortcut that discovers 2 peers and hands off to vs-mode.
|
||||
|
||||
Four 2026-04-22 test receipts (Warriors, Seattle, Arizona Wildcats, Kanye West) all confirmed the user's pain points:
|
||||
|
||||
- Peers thin because they ran without per-entity handle/sub targeting.
|
||||
- Only one `*-raw.md` per run — no per-entity audit.
|
||||
- Kanye peers leaked main topic's `--subreddits`.
|
||||
- Engine footer nudging `BRAVE_API_KEY` to Claude Code users who already have WebSearch.
|
||||
- Polymarket noise on ambiguous topics (Warriors → Glasgow rugby; Arizona → Diamondbacks).
|
||||
|
||||
This plan closes all of them by unifying the architecture and making hosting-model-driven Step 0.55 per entity the canonical path.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. vs mode (any topic containing ` vs ` / ` versus `) runs N full `pipeline.run()` calls in parallel, one per entity. Each sub-run uses its entity's own Step 0.55 targeting (from the hosting model's pre-resolution, passed via a new `--competitors-plan` JSON).
|
||||
- R2. `--competitors` (and `--competitors=N`) becomes a SKILL.md-level shortcut: the hosting model (a) discovers N peers via WebSearch, (b) runs Step 0.55 per entity (main + peers), (c) rewrites the topic to `"main vs peer1 vs peer2"`, (d) invokes the engine with `--competitors-plan` containing each entity's targeting.
|
||||
- R3. New `--competitors-plan` JSON flag. Schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Implies vs mode when present with a single-entity topic. Applies per-entity targeting to each sub-run. Accepts inline JSON or a file path (matches `--plan`).
|
||||
- R4. Each entity's sub-run saves its own `*-raw.md` file when `--save-dir` is in use. Example: `/last30days "Kanye West vs Drake vs Kendrick Lamar" --save-dir=~/Documents/Last30Days` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`. Same filenames a single-entity run of each topic would produce. Matches historical vs-mode behavior.
|
||||
- R5. Each per-entity saved file includes its own single-row `## Resolved Entities` block so the audit survives. The merged comparison stdout still shows the full 3-row block.
|
||||
- R6. Override-leak fix: no main-topic flags (`--subreddits`, `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`) leak into peer sub-runs. Every per-entity kwarg is scrubbed at the sub-run call site.
|
||||
- R7. LAW 7-style stderr for `--competitors` invocations with no list, no plan, no backend is reframed for hosting-model context: leads with "use your WebSearch to discover peers, resolve Step 0.55 per entity, re-invoke with `topic vs peer1 vs peer2 --competitors-plan '...'`." Does not lead with BRAVE_API_KEY.
|
||||
- R8. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` or `--competitors-plan` was passed.
|
||||
- R9. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
||||
- R10. Default `--competitors` count stays 2 peers (3-way comparison). Unchanged from 3.0.12.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No changes to single-entity `pipeline.run()` semantics. Each sub-run in vs mode behaves identically to a bare `/last30days {entity}` invocation.
|
||||
- No changes to the planner's comparison-intent logic for single-entity-containing topics. The `_should_force_deterministic_plan` shortcut for vs-topics routes to fanout, not to its current single-pipeline path.
|
||||
- No new emit modes. Comparison output format unchanged.
|
||||
- No removal of `--competitors-list`. Stays as a minimum escape hatch (names-only, no per-entity targeting) for scripted headless use.
|
||||
- No removal of engine-internal `resolve.auto_resolve()` in fanout. Remains as headless / cron fallback for users with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys. The dominant Claude Code path bypasses it via `--competitors-plan`.
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Explicit "head-to-head" rivalry pass in vs-mode (a supplemental subquery like `"A vs B"` that catches rivalry articles missing from pure entity-scoped passes). Start with N independent passes; add a head-to-head supplemental pass if the rivalry-content gap shows up in dogfood.
|
||||
- Cache layer for hosting-model pre-resolution.
|
||||
- Cross-source disambiguation (not just Polymarket).
|
||||
- Latency knob for users who want the old one-pass vs behavior (probably not needed; parallel N-pass is ~1× wall clock).
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py` — main(), `_main_runner`, `_competitor_runner`, the competitor enable/discovery branch. Primary file.
|
||||
- `scripts/lib/fanout.py` — existing orchestrator (3.0.11). Reused as-is; `competitor_runner` closure is where per-entity kwargs apply.
|
||||
- `scripts/lib/planner.py` — `_should_force_deterministic_plan` detects vs-topics via regex. Current path synthesizes ONE comparison plan; new path routes to fanout.
|
||||
- `scripts/lib/render.py` — `render_comparison_multi` (3.0.12) + `_render_resolved_entities_block`. Both reused. `render_full` needs a per-entity variant when saving sub-run files.
|
||||
- `scripts/last30days.py` `save_output` — where raw files are written. Needs to iterate per entity when competitor_reports artifact present.
|
||||
- `scripts/lib/quality_nudge.py` — BRAVE/SERPER nudge emission.
|
||||
- `scripts/lib/polymarket.py` — source adapter for `--polymarket-keywords` and ambiguous-topic auto-skip.
|
||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol — the hosting-model contract that drives per-entity pre-resolution for both vs mode and `--competitors`.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- 3.0.11 plan (`2026-04-22-002`): built fanout.
|
||||
- 3.0.12 plan (`2026-04-22-003`): tried engine-internal per-entity auto_resolve; failed without backend keys.
|
||||
- 3.0.13 plan draft (`2026-04-22-004-...superseded`): proposed `--competitors-plan` JSON + vs-mode-shortcut path but kept them separate. User's 2026-04-22 correction unifies them.
|
||||
- 2026-04-22 test receipts: Warriors, Seattle, Arizona Wildcats, Kanye West runs all reproduced the per-entity resolve gap.
|
||||
- User's architectural steer: "vs mode should work that way too" + "--competitors is just vs mode with auto-discovery." This plan encodes that.
|
||||
|
||||
### External References
|
||||
|
||||
- None. All patterns in-repo.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Unify vs-mode and --competitors on one orchestrator.** `fanout.run_competitor_fanout` serves both. vs-mode is "topic contains ' vs '" detection → fanout. `--competitors` is "SKILL.md shortcut → hosting model rewrites topic to vs form → fanout." One code path.
|
||||
- **Per-entity targeting via `--competitors-plan` JSON.** Schema `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Mirrors `--plan`. Applies to both vs-mode and `--competitors` paths. Hosting model passes it after running Step 0.55 per entity.
|
||||
- **N save files, one per entity.** Each sub-run writes a `{entity-slug}-raw.md` file when `--save-dir` is set. Matches historical vs-mode behavior. Single-entity runs unchanged.
|
||||
- **Revert the "one pass for latency" optimization that removed per-entity passes.** Parallel execution via `ThreadPoolExecutor` means wall-clock is ~max(per-entity-latency), not sum. The old latency concern (13+ minutes for 3 serial passes) does not apply to a parallel fan-out.
|
||||
- **Override-leak fix at the call site.** `_subrun_kwargs(entity, plan_entry)` helper returns fully explicit per-entity kwargs; no closure-default fallthrough from main scope.
|
||||
- **LAW 7 stderr reframed, not just updated.** Current message treats BRAVE_API_KEY as the solution. New message treats hosting-model Step 0.55 as the solution, with backend keys listed only as the headless fallback.
|
||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known-ambiguous single-token list.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **vs mode N passes or single-pass?** N passes. User's architectural correction.
|
||||
- **Should --competitors still be an engine flag at all?** Yes, kept for headless / cron contexts with backend keys. Dominant Claude Code path is SKILL.md shortcut → vs-mode fanout. Engine flag stays as compatibility surface.
|
||||
- **`--competitors-plan` JSON or multi-flag?** JSON. Matches `--plan`.
|
||||
- **Default count?** 2 peers → 3-way comparison. Unchanged.
|
||||
- **Saved-file naming?** `{entity-slug}-raw.md` per entity, same as single-entity runs would produce.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact trace of override-leak path (closure capture vs shared config vs Reddit adapter fallback). Test-first per Unit 2; patch at the right layer.
|
||||
- Heuristic for single-token-ambiguous Polymarket auto-skip. Start with a short hard-coded list; iterate.
|
||||
- Whether to include a head-to-head rivalry supplemental pass in vs-mode. Ship N-independent passes first; revisit after dogfood if rivalry content is missing.
|
||||
- Exact filename convention when the comparison merged output is saved (if saved at all). Not blocking — per-entity files are the primary save artifact.
|
||||
|
||||
## High-Level Technical Design
|
||||
|
||||
> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.*
|
||||
|
||||
```
|
||||
User invokes:
|
||||
/last30days "OpenAI vs Anthropic vs xAI"
|
||||
OR
|
||||
/last30days OpenAI --competitors (hosting model rewrites to vs form)
|
||||
OR
|
||||
/last30days OpenAI --competitors-list "Anthropic,xAI"
|
||||
OR
|
||||
/last30days "OpenAI vs Anthropic vs xAI" --competitors-plan '{...per-entity...}'
|
||||
|
||||
↓
|
||||
|
||||
scripts/last30days.py main():
|
||||
- Detect: topic has " vs " OR --competitors enabled
|
||||
- If --competitors and no list/plan: emit LAW 7-style stderr with hosting-model instruction
|
||||
- If --competitors with list or discovery: rewrite topic to vs form, continue
|
||||
- Parse --competitors-plan JSON, map to entities
|
||||
|
||||
↓
|
||||
|
||||
fanout.run_competitor_fanout (shared path):
|
||||
- For each entity (main + peers):
|
||||
- entity_config = dict(config) [deep copy to prevent leak]
|
||||
- kwargs = _subrun_kwargs(entity, plan_entry) [explicit; no main-topic leak]
|
||||
- If plan_entry missing a field AND backend available: auto_resolve() fill
|
||||
- pipeline.run(topic=entity, **kwargs, internal_subrun=True)
|
||||
- Parallel ThreadPoolExecutor
|
||||
- Collect per-entity Reports
|
||||
- Attach resolved targeting to each Report.artifacts["resolved"]
|
||||
|
||||
↓
|
||||
|
||||
scripts/last30days.py after fanout:
|
||||
- If --save-dir: save each entity's Report as {entity-slug}-raw.md
|
||||
Each file includes its own single-row Resolved Entities block
|
||||
- emit_comparison_output → render_comparison_multi (merged stdout)
|
||||
Includes full N-row Resolved Entities block
|
||||
```
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: vs-topic detection routes to fanout (not single-pipeline)**
|
||||
|
||||
**Goal:** A topic containing ` vs ` / ` versus ` triggers `fanout.run_competitor_fanout` with the parsed entities. Each entity runs a full `pipeline.run()`. Replace the current single-pipeline-with-comparison-plan behavior.
|
||||
|
||||
**Requirements:** R1
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (main() — detect vs-topic, route to fanout)
|
||||
- Modify: `scripts/lib/planner.py` (remove / bypass the `_should_force_deterministic_plan` special case for vs topics; vs topics no longer go through `plan_query` as a single comparison plan)
|
||||
- Test: `tests/test_vs_mode_fanout.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Parse the incoming topic: if it contains ` vs ` or ` versus ` (case-insensitive), split into entities (reuse `planner._comparison_entities`-style logic or move that utility into main()).
|
||||
- When vs-entities are detected, route to the same fanout branch `--competitors` uses today. The entity list comes from the topic string; no discovery step needed.
|
||||
- Each entity runs `pipeline.run()` with its own plan (either from `--competitors-plan[entity]` or from the engine's per-entity fallback path).
|
||||
- For back-compat, if the user passes both a vs-topic AND `--plan`, honor `--plan` for the main (first) entity and use per-entity defaults for peers unless `--competitors-plan` is also provided.
|
||||
|
||||
**Execution note:** Start with an integration test that runs `"A vs B"` via mock mode and asserts fanout was called with two entities + two pipeline.run calls.
|
||||
|
||||
**Patterns to follow:**
|
||||
- 3.0.11 fanout wiring in `scripts/last30days.py`'s `--competitors` branch.
|
||||
- `planner._comparison_entities` for the split logic.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: topic `"A vs B"` → two pipeline.run calls, two Reports returned, merged render.
|
||||
- Happy path: topic `"A vs B vs C"` → three pipeline.run calls.
|
||||
- Happy path: topic `"A versus B"` → matches the same regex, two pipelines.
|
||||
- Edge case: topic `"OpenAI vs"` (trailing empty entity) → treated as single-entity `"OpenAI"`, not vs mode.
|
||||
- Edge case: topic contains "vs." (dot, no trailing space) → existing regex tolerates it; verify.
|
||||
- Edge case: topic `"A vs B"` plus `--plan` → plan applies to first entity only, peers use per-entity defaults.
|
||||
- Integration: full vs-mode run end-to-end in mock mode; verify rendered output, stderr has one `[Competitors] Comparing: A vs B vs ...` line.
|
||||
|
||||
**Verification:**
|
||||
- Test assertions pass.
|
||||
- Mock-mode smoke of `/last30days "OpenAI vs Anthropic"` shows fanout invocation, per-entity Reports, merged comparison output.
|
||||
|
||||
- [ ] **Unit 2: `--competitors-plan` JSON flag + `_subrun_kwargs` helper + override-leak fix**
|
||||
|
||||
**Goal:** New JSON flag threads per-entity targeting into each sub-run's `pipeline.run()`. A `_subrun_kwargs(entity, plan_entry)` helper is the single source of truth for per-entity kwargs, eliminating override-leak.
|
||||
|
||||
**Requirements:** R3, R6
|
||||
|
||||
**Dependencies:** None (can land alongside or before Unit 1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner` + `_subrun_kwargs` helper)
|
||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; the competitor_runner contract is unchanged)
|
||||
- Test: `tests/test_cli_competitors.py` (extend)
|
||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
||||
- Test: `tests/test_competitor_subrun_isolation.py` (new, regression)
|
||||
|
||||
**Approach:**
|
||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON or file path (mirror `--plan`).
|
||||
- Validation: top-level dict; each value is a dict; unknown fields log warnings; malformed input exits 2. Case-insensitive entity matching.
|
||||
- Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`.
|
||||
- Build `_subrun_kwargs(entity, plan_entry)` — returns an explicit dict with every per-entity flag. No closure-default fallthrough. This is the leak fix.
|
||||
- `_competitor_runner(entity)`:
|
||||
1. Get `plan_entry` from `--competitors-plan` if present.
|
||||
2. Build base kwargs with `_subrun_kwargs(entity, plan_entry)`.
|
||||
3. Fill missing fields via `resolve.auto_resolve(entity, entity_config)` only if backend is configured (3.0.12 fallback path).
|
||||
4. Call `pipeline.run(topic=entity, internal_subrun=True, **kwargs)`.
|
||||
5. Attach `resolved` dict to `report.artifacts`.
|
||||
- Verify no per-entity flag from main() leaks via closure. The helper is the only source of per-entity values.
|
||||
|
||||
**Execution note:** Test-first for the override-leak regression. Use the Kanye 2026-04-22 receipt as the failing test input (main `--subreddits=Kanye,hiphopheads` + `--competitors-list "Drake"` → assert Drake's pipeline.run receives `subreddits=None`).
|
||||
|
||||
**Patterns to follow:**
|
||||
- `--plan` parsing block in `scripts/last30days.py`.
|
||||
- 3.0.12's `entity_config = dict(config)` deep-copy pattern.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors-plan '{"Drake":{"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's pipeline.run receives `x_handle="Drake"`, `subreddits=["Drizzy"]`. No auto_resolve call for Drake.
|
||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered skip auto_resolve; third falls back.
|
||||
- Happy path: plan file path accepted like `--plan`.
|
||||
- Happy path: case-insensitive entity match.
|
||||
- Edge case: unknown fields → warn, ignore.
|
||||
- Edge case: plan entry for entity not in list → warn, ignore.
|
||||
- Error path: malformed JSON → exit 2.
|
||||
- Error path: top-level JSON is list → exit 2.
|
||||
- Regression (leak): main `--subreddits=A,B` + `--competitors-list "X"` + no plan → X's pipeline.run gets `subreddits=None`.
|
||||
- Regression (leak): same for `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`.
|
||||
- Regression (leak): main `--x-handle=kanye` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanye"`.
|
||||
|
||||
**Verification:**
|
||||
- All regression tests pass.
|
||||
- Smoke run (mock mode + plan): stderr shows per-entity `[Competitors] {entity}: x=... subs=...` line; no leak from main topic's flags.
|
||||
|
||||
- [ ] **Unit 3: Per-entity save files**
|
||||
|
||||
**Goal:** When `--save-dir` is set in a vs-mode or `--competitors` run, each entity's sub-run saves its own `{entity-slug}-raw.md` file — same format as a single-entity run would produce.
|
||||
|
||||
**Requirements:** R4, R5
|
||||
|
||||
**Dependencies:** Unit 1, Unit 2
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (`save_output` iteration after fanout)
|
||||
- Modify: `scripts/lib/render.py` (`render_full` includes single-row Resolved Entities block when that entity's `artifacts["resolved"]` is present)
|
||||
- Test: `tests/test_save_raw_per_entity.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]` (or equivalent). For each `(entity, entity_report)`:
|
||||
- Call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)`.
|
||||
- Uses entity's `slugify(entity)` for the filename. Same pattern a single-entity run uses.
|
||||
- Each saved file invokes `render_full` (or the save-variant). `render_full` now checks for `report.artifacts["resolved"]` and prepends a single-row Resolved Entities block.
|
||||
- Stderr logs one `[last30days] Saved output to <path>` line per entity.
|
||||
- Single-entity runs unchanged (no extra files, render_full unchanged for them).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing `save_output` invocation in main() for single-entity runs.
|
||||
- `slugify(topic)` for filename.
|
||||
- 3.0.12's `_render_resolved_entities_block` (reused, single-row mode).
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `/last30days "A vs B vs C" --save-dir=/tmp/x` → `/tmp/x/a-raw.md`, `/tmp/x/b-raw.md`, `/tmp/x/c-raw.md` exist.
|
||||
- Happy path: `--competitors-list "Drake,Kendrick" --save-dir=/tmp/x` on topic Kanye → three files: `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`.
|
||||
- Happy path: each file includes a single-row Resolved Entities block for its entity.
|
||||
- Happy path: single-entity run with `--save-dir` → one file, no Resolved block (unchanged).
|
||||
- Edge case: `--save-suffix=v3` → all N files get the suffix.
|
||||
- Edge case: one entity sub-run failed → its file is NOT saved; the others are.
|
||||
- Integration: `ls {save-dir}/*-raw.md` returns N files after a vs-mode run.
|
||||
|
||||
**Verification:**
|
||||
- Test assertions pass.
|
||||
- Manual vs-mode smoke saves N files.
|
||||
|
||||
- [ ] **Unit 4: LAW 7-style stderr reframe + footer-nudge suppression**
|
||||
|
||||
**Goal:** The `--competitors`-with-no-backend stderr tells the hosting model to do Step 0.55 per entity and pass `--competitors-plan`. The BRAVE/SERPER footer nudge is suppressed when `--plan` or `--competitors-plan` is present.
|
||||
|
||||
**Requirements:** R7, R8
|
||||
|
||||
**Dependencies:** Unit 2 (flag must exist)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (the `[Competitors] --competitors requires...` stderr block)
|
||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever footer nudge emits; verify during implementation)
|
||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Rewrite stderr in this order:
|
||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent with WebSearch), the recommended path: (a) discover N peers via WebSearch, (b) run Step 0.55 for main + each peer, (c) re-invoke as `/last30days 'topic vs peer1 vs peer2' --competitors-plan '{...}'`. See SKILL.md 'Competitor mode'."
|
||||
2. "Headless / cron path: set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
||||
3. "Minimum escape hatch: `--competitors-list 'A,B,C'` skips discovery but does not pre-resolve peers."
|
||||
- Suppress footer nudge when `external_plan` OR `competitors_plan` was passed.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors` with no backend, no list, no plan → stderr leads with "If you are the hosting reasoning model" and references `--competitors-plan` before naming API keys.
|
||||
- Happy path: `--plan` passed → footer nudge does NOT fire.
|
||||
- Happy path: `--competitors-plan` passed → footer nudge does NOT fire.
|
||||
- Happy path: `--competitors-list` only (no plan, no backend) → footer nudge still fires (hosting model didn't fully engage).
|
||||
- Happy path: no `--competitors`, no `--plan` → footer nudge unchanged.
|
||||
|
||||
**Verification:**
|
||||
- Tests pass.
|
||||
|
||||
- [ ] **Unit 5: Polymarket disambiguation guard**
|
||||
|
||||
**Goal:** `--polymarket-keywords "kw1,kw2"` filters market matches; auto-skip Polymarket on single-token-ambiguous topics without override.
|
||||
|
||||
**Requirements:** R9
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (argparse)
|
||||
- Modify: `scripts/lib/polymarket.py`
|
||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Add `--polymarket-keywords "kw1,kw2"`. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
||||
- Auto-skip: if topic is one token AND matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords`, skip Polymarket with stderr note.
|
||||
- SKILL.md update (small): mention `--polymarket-keywords` in Step 0.55 instructions for ambiguous topics.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr note.
|
||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs, filtered.
|
||||
- Happy path: topic "OpenAI" → Polymarket runs as before.
|
||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
||||
|
||||
**Verification:**
|
||||
- Warriors smoke → Polymarket footer absent or filtered.
|
||||
|
||||
- [ ] **Unit 6: SKILL.md rewrite — vs mode is the canonical path, `--competitors` is a shortcut**
|
||||
|
||||
**Goal:** SKILL.md documents the unified architecture. vs mode runs N full passes. `--competitors` is a SKILL.md-level shortcut that discovers 2 peers and invokes vs mode with `--competitors-plan`.
|
||||
|
||||
**Requirements:** R1, R2, R10 (surfaces them)
|
||||
|
||||
**Dependencies:** Units 1-4
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md` (§551 "If QUERY_TYPE = COMPARISON" rewrite; Competitor mode subsection rewrite)
|
||||
- Modify: `README.md` (one-line example)
|
||||
|
||||
**Approach:**
|
||||
- Rewrite §551 to describe the N-pass architecture: "When the user asks 'X vs Y' (or 'X vs Y vs Z'), run Step 0.55 per entity, then invoke the engine. The engine fans out N full pipelines in parallel. Each entity gets its own single-entity-grade coverage. Wall clock is close to a single run."
|
||||
- Remove the "ONE research pass with a comparison-optimized plan that replaces the old 3-pass approach" language.
|
||||
- Add a `--competitors-plan` JSON example.
|
||||
- Rewrite the Competitor mode subsection: "`--competitors` is a shortcut. The hosting model: (1) runs WebSearch to discover N=2 peers, (2) runs Step 0.55 for main + each peer, (3) rewrites topic to `'main vs peer1 vs peer2'`, (4) invokes engine with `--competitors-plan '{...}'`. Engine flag `--competitors` and `--competitors-list` remain for headless fallback."
|
||||
- Cross-reference §679 (per-entity Step 0.55 protocol).
|
||||
- Warning: a thin `## Resolved Entities` block (dashes for any entity) means the hosting model skipped Step 0.55 for that one.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing §679 per-entity Step 0.55 protocol for tone.
|
||||
- 3.0.12 Competitor mode prose for terseness.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — documentation. Verification is dogfood.
|
||||
|
||||
**Verification:**
|
||||
- `/last30days "OpenAI vs Anthropic vs xAI"` in a fresh Claude Code window produces 3 save files with populated Resolved blocks and non-dash per-entity targeting.
|
||||
- `/last30days OpenAI --competitors` produces same after discovery step.
|
||||
|
||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
||||
|
||||
**Goal:** Ship 3.0.13 to all local targets.
|
||||
|
||||
**Requirements:** Closes R1-R10
|
||||
|
||||
**Dependencies:** Units 1-6
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json`
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG: group the changes. "Changed: vs mode now runs N full passes in parallel, one per entity — reverting the one-pass optimization to restore per-entity depth. Added: --competitors-plan JSON for per-entity Step 0.55 targeting (applies to vs mode and --competitors). Changed: --competitors is now a SKILL.md shortcut for vs-with-discovery. Added: per-entity *-raw.md save files. Fixed: override-leak from main to peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when --plan / --competitors-plan present. Added: --polymarket-keywords + auto-skip for ambiguous topics."
|
||||
- Beta channel first per CLAUDE.md.
|
||||
- Hot-copy so public `/last30days` picks up 3.0.13.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` → 3.0.13.
|
||||
- `sync.sh` exits 0.
|
||||
- Hot-copy contains the new files.
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** vs-mode and `--competitors` share one orchestrator (`fanout.run_competitor_fanout`). `_subrun_kwargs` is the single source of per-entity kwargs. Save loop iterates per entity.
|
||||
- **Error propagation:** Per-entity sub-run failure → logged, dropped, continue (3.0.11 behavior unchanged). `--competitors-plan` JSON parse errors exit 2 (same shape as `--plan`).
|
||||
- **State lifecycle risks:** `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag (Unit 2 fix). No cross-entity context leak.
|
||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors`, `--competitors-list`, `--plan` unchanged. `--polymarket-keywords` additive. vs-mode keeps its topic-string surface.
|
||||
- **Integration coverage:** New vs-mode-fanout integration test. New override-leak regression test. New plan-threading test. New nudge-suppression test. New per-entity-save test. New Polymarket disambiguation test.
|
||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. Single-entity render path unchanged. LAW 7 on the default path unchanged (still fires when a single-entity run lacks `--plan`).
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| vs-mode N-pass latency feels slower for users who remember the one-pass shortcut. | Parallel execution keeps wall-clock ~= max(per-entity-latency), not sum. `--quick` on a vs-topic still applies to each sub-run. CHANGELOG calls out the revert + parallelism. |
|
||||
| API cost scales linearly with N (per source). | Default count 2 caps it. Hard max 6 on `--competitors`. vs-mode users opted into N entities explicitly. |
|
||||
| Rivalry content ("A vs B" articles) missed in N-independent passes. | Deferred to separate task (head-to-head supplemental pass). Start shipping and observe whether this is actually a gap. |
|
||||
| Hosting model skips `--competitors-plan` and uses `--competitors-list` only. | Unit 4 stderr reframe steers explicitly. SKILL.md Unit 6 makes the plan-path canonical. Thin Resolved block in output makes skipped-Step-0.55 visible. |
|
||||
| Override-leak fix misses a subtle closure path. | Unit 2 is test-first with the Kanye receipt as the failing input. Regression test asserts every per-entity flag is None unless plan provides it. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first per CLAUDE.md.
|
||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
||||
- CHANGELOG explicitly frames the vs-mode change as an architectural revert-with-parallelism, not a regression to the old serial N-pass.
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Superseded plan: `docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded`
|
||||
- Previous plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
||||
- Initial plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
||||
- 2026-04-22 test session receipts (Warriors, Seattle, Arizona Wildcats, Kanye West)
|
||||
- SKILL.md §551 + §679 — the per-entity Step 0.55 protocol the hosting model uses for both paths
|
||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/planner.py` vs-topic special-case, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
||||
@@ -1,87 +0,0 @@
|
||||
---
|
||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
||||
type: fix
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# fix: comparison title says (/Last30Days) instead of (Last 30 Days)
|
||||
|
||||
## Overview
|
||||
|
||||
User feedback 2026-04-22 on the 3.0.13 release runs (Kanye vs Drake, Mercer Island, Figma): the comparison title currently reads `# Kanye West vs Drake: What the Community Says (Last 30 Days)`. It should read `# Kanye West vs Drake: What the Community Says (/Last30Days)` — attributing the output to the slash command rather than describing the date range generically.
|
||||
|
||||
Single-line change in SKILL.md, three occurrences. No code change.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Comparison title pattern in SKILL.md changes from `(Last 30 Days)` to `(/Last30Days)` so synthesis outputs read `... What the Community Says (/Last30Days)`.
|
||||
- R2. Both the rule statement (line 113) and the COMPARISON-exception statement (line 131) and the synthesis template example (line 1208) all use the new suffix.
|
||||
- R3. Version bumps to 3.0.14, CHANGELOG entry, sync, hot-copy. Public cache picks up the new title pattern.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No changes to the single-entity output title (no `(/Last30Days)` suffix there — only comparison topics carry it).
|
||||
- No changes to engine code. Pure SKILL.md content.
|
||||
- No changes to anything else surfaced in the test runs.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Replace all three occurrences of the suffix string in one pass.** They are identical strings; changing one without the others would cause synthesis-time confusion when the model reaches a different reference.
|
||||
- **Ship as 3.0.14, not 3.0.13.x.** Patch-level bump matches the small scope and keeps the release log clean.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Replace `(Last 30 Days)` → `(/Last30Days)` in SKILL.md**
|
||||
|
||||
**Goal:** All three SKILL.md references to the comparison title use the new suffix.
|
||||
|
||||
**Requirements:** R1, R2
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md`
|
||||
|
||||
**Approach:**
|
||||
- `replace_all` swap of `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Three occurrences, no other strings overlap.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — pure documentation. Verification by inspection + dogfood run.
|
||||
|
||||
**Verification:**
|
||||
- `grep -c "What the Community Says (/Last30Days)" SKILL.md` returns 3.
|
||||
- `grep -c "What the Community Says (Last 30 Days)" SKILL.md` returns 0.
|
||||
|
||||
- [ ] **Unit 2: Version 3.0.14 + CHANGELOG + sync + hot-copy**
|
||||
|
||||
**Goal:** Ship 3.0.14 to all local targets.
|
||||
|
||||
**Requirements:** R3
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json`
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.14/`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG: "Changed: comparison-mode title attribution — `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range."
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` → 3.0.14.
|
||||
- Hot-copy contains the updated SKILL.md.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Hosting model has the old title pattern memorized from a prior run and re-emits `(Last 30 Days)`. | SKILL.md is read top-to-bottom each invocation. STEP 0 canonical-path self-check (3.0.12) ensures the model loads the new SKILL.md, not the marketplace stale copy. |
|
||||
|
||||
## Sources & References
|
||||
|
||||
- 2026-04-22 dogfood runs (Kanye West vs Drake, Mercer Island --competitors, Figma --competitors)
|
||||
- Related code: `SKILL.md` lines 113, 131, 1208
|
||||
@@ -0,0 +1,87 @@
|
||||
# Research-quality eval harness
|
||||
|
||||
The eval suite measures the quality properties that ordinary unit tests do not: whether ranked evidence is grounded in retrieved inputs, stays inside the requested window, forms coherent clusters, accounts for every usable fixture source, and remains deterministic.
|
||||
|
||||
It runs the production pipeline offline. Recorded HTTP exchanges replay at `lib/http.py`; CLI-backed adapters such as yt-dlp, Digg, arXiv, Techmeme, and Trustpilot replay their parsed result at the source-module seam. Planning is supplied by each fixture manifest, and normalization, date filtering, scoring, fusion, clustering, source outcomes, and the versioned agent JSON export all run normally. The harness never calls an LLM or the network.
|
||||
|
||||
## Run it
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
uv run pytest tests/eval -x -s
|
||||
```
|
||||
|
||||
The `-s` keeps the score table visible. To print only the scored run and return a nonzero exit when a floor is missed:
|
||||
|
||||
```bash
|
||||
uv run python tests/eval/harness.py
|
||||
```
|
||||
|
||||
CI runs the pytest command in the `eval` job of `.github/workflows/validate.yml`, so every pull request gets a score table and a hard baseline check.
|
||||
|
||||
## Metrics
|
||||
|
||||
| Metric | Deterministic definition |
|
||||
|---|---|
|
||||
| Citation grounding | Fraction of exported result URLs that occur in the recorded fixture inputs. |
|
||||
| Recency compliance | Fraction of ranked source items whose known publication date is inside the report's inclusive date window. Undated evidence is not falsely classified as stale. |
|
||||
| Cluster coherence | Fraction of within-cluster candidate pairs meeting the production entity-overlap threshold (`0.45`). Singleton clusters are coherent by definition. |
|
||||
| Coverage | Fraction of fixture sources represented by usable report items or an explicit `Report.source_status` outcome. |
|
||||
| Determinism | `schema.to_dict()` equality for two runs with fixed time and identical recorded inputs. |
|
||||
|
||||
Aggregate floors live in `tests/eval/baseline.json`. The fixture matrix covers a tech product, a person, a comparison, breaking events, a niche technical topic, and a non-English CJK topic.
|
||||
|
||||
## Add or refresh a fixture
|
||||
|
||||
Fixture directories contain:
|
||||
|
||||
- `manifest.json`: topic archetype, fixed `as_of_date`, sources, safe dummy config, and a deterministic external query plan.
|
||||
- `http.json`: scrubbed HTTP exchanges and any CLI-backed source exchanges.
|
||||
|
||||
Use the direct engine invocation below only for development/fixture capture; `/last30days <topic>` remains the product interface:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py \
|
||||
"<topic>" \
|
||||
--quick \
|
||||
--as-of 2026-07-10 \
|
||||
--search grounding,hackernews \
|
||||
--plan /tmp/eval-plan.json \
|
||||
--record-fixtures tests/eval/fixtures/<fixture-name>
|
||||
```
|
||||
|
||||
`--record-fixtures` is intentionally hidden from `--help`. It records the live run's shared HTTP traffic and the bounded CLI-adapter seams, scrubs credential-shaped query/body/response fields, and writes `http.json`. It does not create the manifest because archetype, fixed date, source contract, and query plan are review decisions.
|
||||
|
||||
Before committing a recording:
|
||||
|
||||
1. Inspect `http.json` for cookies, keys, tokens, personal identifiers, and unnecessary long bodies.
|
||||
2. Truncate content to the smallest structure that exercises the adapter and pipeline.
|
||||
3. Replace irrelevant real usernames with obvious fixture identities.
|
||||
4. Add the manifest and run both commands above with networking unavailable.
|
||||
|
||||
The replay is fail-closed: an unrecorded request or an unused recorded exchange fails the run.
|
||||
|
||||
## Fixture flags
|
||||
|
||||
- `expects_clusters` (bool): fixtures whose topic historically forms multi-member clusters set this true; if cluster formation regresses to singletons on such a fixture, coherence scores 0.0 instead of a vacuous 1.0. Sparse topics (niche, non-english-cjk, tech-product) set it false because singletons are their legitimate shape.
|
||||
- Post-ranking enrichment (YouTube transcripts, Digg posts) is recorded and replayed by merging recorded `metadata` onto freshly computed items by item_id, so normalization/scoring/dedupe regressions stay visible to the eval rather than being overwritten by fixture state.
|
||||
- Post-rerank GitHub star enrichment records its repo->stars map and replays via `github.apply_star_map`, keeping runs offline even when `GITHUB_TOKEN` is set in CI. GitHub project-mode (`--github-repo`) and person-mode (`--github-user`) runs are not yet fixture-recordable; the network guard fails loudly if a fixture attempts them.
|
||||
|
||||
## Known seams
|
||||
|
||||
- Module-backed sources (yt-dlp, digg-pp-cli and other CLI adapters) record post-parse items at the module boundary, so replay does not re-exercise their parsing/normalization code the way HTTP-backed sources do (those replay raw responses through the real pipeline). A normalization regression in a module adapter is covered by that adapter's unit tests, not the eval. Recording raw CLI stdout is a possible future upgrade.
|
||||
- Cluster coherence shares `entity_extract` with production clustering. The pinned-predicate test (`test_entity_overlap_predicate_pinned`) guards against the shared predicate drifting permissive, and per-fixture floors in baseline.json catch a single archetype collapsing even when the cross-fixture average stays green.
|
||||
|
||||
## Move a baseline
|
||||
|
||||
Baseline edits are explicit quality-policy changes, not snapshot refreshes. Move a floor only when an intentional product change makes the old threshold invalid or when a new fixture legitimately changes the measured distribution.
|
||||
|
||||
Include in the review:
|
||||
|
||||
1. The old and new score tables.
|
||||
2. The reason the metric changed.
|
||||
3. A focused test proving the intended behavior.
|
||||
4. An explanation for any lower floor; never lower a floor solely to make CI green.
|
||||
|
||||
`test_intentional_out_of_window_regression_fails_recency_floor` is the standing negative control: it injects stale ranked evidence and proves the baseline check detects the regression.
|
||||
@@ -0,0 +1,133 @@
|
||||
# Agent JSON export
|
||||
|
||||
The agent JSON profile is the stable machine-readable research contract for downstream agents, scripts, dashboards, and workflow tools. Ask the slash command for machine-readable JSON:
|
||||
|
||||
```text
|
||||
/last30days AI coding agents — return the versioned agent JSON export
|
||||
```
|
||||
|
||||
For direct engine use in scripts, cron jobs, or development, use:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json
|
||||
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json --output results.json
|
||||
```
|
||||
|
||||
`--emit=json` defaults to `--json-profile=agent`. The full internal report remains available for debugging and power users:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "AI coding agents" --emit=json --json-profile=raw
|
||||
```
|
||||
|
||||
The raw profile is intentionally unversioned and may change when pipeline internals change. It preserves the JSON serialization used before the agent profile was introduced.
|
||||
|
||||
### Local corpus privacy
|
||||
|
||||
Evidence from `--corpus` / `LAST30DAYS_CORPUS_DIRS` is excluded from the versioned agent profile by default. The exclusion removes corpus results, corpus-only clusters, corpus source outcomes, freshness verdicts, and titles derived from a corpus representative. Set `LAST30DAYS_CORPUS_IN_EXPORT=1` only for a run whose JSON is intentionally allowed to contain local file contents. This opt-in does not change the schema shape or version; it permits `source: "corpus"` entries in the existing result fields. The unversioned `raw` profile is a complete local debug dump and may contain corpus paths and text.
|
||||
|
||||
## Discovery export
|
||||
|
||||
Discovery mode has a separate versioned contract so its topic results do not change the normal research export:
|
||||
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py --discover "AI agents" --emit=json
|
||||
```
|
||||
|
||||
Its top level contains `schema_version` (`1.1`), `kind` (`"discovery"`), `domain` (`""` for a global no-domain trending run), `generated_at`, `window_days`, `source_status`, `feeds`, `results`, `warnings`, `outcome` (`"ok"`, or `"nothing-solid"` when no topic cleared the confidence floor), and `weak_signal` (the closest sub-floor topic name on a nothing-solid run, else `null`). Each ranked result contains `rank`, `topic`, `why_spiking`, `momentum` (`new-this-week` or `building`), `velocity_score`, `sources`, per-source native `engagement`, a ready-to-run `command`, `evidence_urls`, `top_comment` (the strongest verbatim community comment from the topic's research pass, with attribution; `null` on shallow runs), `corroboration_count` (distinct confirming sources), `podcast_angle` (engine-generated podcast content hook; `null` when no reasoning provider produced one), `x_article_angle` (engine-generated X-article content hook; `null` when no reasoning provider produced one), `previously_surfaced_count` (topic-queue annotation: how many earlier sweeps surfaced this topic; `0` when the queue is off), `last_surfaced` (topic-queue annotation: date the topic last surfaced; `null` when the queue is off), and `covered` (topic-queue annotation: whether the topic was already covered; `false` when the queue is off). The discovery contract follows the same versioning policy below but evolves independently of the normal agent export. `--json-profile=raw` returns the unversioned internal `DiscoveryReport` dataclass instead.
|
||||
|
||||
When `LAST30DAYS_API_KEY` and `LAST30DAYS_API_BASE` route a run through a configured remote API, the server does not return the local `Report` needed to build this profile. In that mode, `--json-profile=agent` exits with status 2 instead of emitting a misleading shape; use `--json-profile=raw` to retain the remote backend's existing server-response JSON contract.
|
||||
|
||||
## Top-level fields
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `schema_version` | string | Agent export contract version. The current version is `1.2`. |
|
||||
| `query` | string | The research topic supplied to the engine. |
|
||||
| `generated_at` | string | UTC generation timestamp in RFC 3339 format. |
|
||||
| `window_days` | integer | Number of days between the report's start and end dates. |
|
||||
| `source_status` | object | Map of source name to the outcome observed during this run. |
|
||||
| `freshness_verdicts` | array | Per-claim act-time verdicts produced by `--verify-freshness`; empty when verification was not requested or no conservative claims were extractable. |
|
||||
| `clusters` | array | Ranked groups of related results. |
|
||||
| `results` | array | Ranked, flat evidence results for downstream processing. |
|
||||
|
||||
All top-level fields are always present. Empty runs contain empty `clusters` and `results` arrays. Sources appear in `source_status` when the run recorded an outcome for them.
|
||||
|
||||
## `freshness_verdicts`
|
||||
|
||||
Each entry identifies the grounded claim and candidate, its primary source item, the typed `verdict` (`current`, `stale`, `contradicted`, or `unsupported`), the original and re-derived values when applicable, and source/evidence URLs and timestamps. `stale` means a successful point re-fetch returned a moved value; `contradicted` means a newer item in the report window explicitly disagrees; `unsupported` means the datum could not be re-checked, including degraded `source_status` outcomes. Consumers can gate actions on `verdict == "current"` without treating an unreachable source as evidence that a claim moved.
|
||||
|
||||
## `source_status`
|
||||
|
||||
Each value distinguishes a clean empty result from incomplete coverage:
|
||||
|
||||
| State | Meaning |
|
||||
| --- | --- |
|
||||
| `ok` | The source completed and returned one or more items. |
|
||||
| `no-results` | The source completed successfully but found no matching items. |
|
||||
| `partial` | The source returned some items before a later failure. |
|
||||
| `rate-limited` | Retrieval was stopped by a provider rate limit. |
|
||||
| `auth-failed` | Credentials were missing, rejected, or expired during retrieval. |
|
||||
| `unreachable` | The source or network endpoint could not be reached. |
|
||||
| `timeout` | Retrieval exceeded its time limit. |
|
||||
| `schema-drift` | The provider response no longer matched the expected shape. |
|
||||
| `skipped-unconfigured` | The source was intentionally skipped because required configuration was absent. |
|
||||
| `error` | Retrieval failed for another reason. |
|
||||
|
||||
Consumers must not interpret failure states as evidence that a source had no discussion. Only `no-results` means the source completed cleanly with zero matches.
|
||||
|
||||
## Cluster fields
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `title` | string | Cluster headline. |
|
||||
| `summary` | string | Summary from the cluster's representative ranked result. |
|
||||
| `sources` | array of strings | Sources represented by the cluster. |
|
||||
| `engagement_total` | number | Sum of one headline native engagement counter per result. Known sources use their primary count (for example, Digg uses `postCount`); otherwise the largest counter-like field is used. Ranking, ratio, rating, and computed-score metadata are excluded. |
|
||||
|
||||
Cluster array order is ranking order. A result's `cluster` value is the zero-based index into this array.
|
||||
|
||||
## Result fields
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `candidate_id` | string | Stable identifier joining this result to `freshness_verdicts[].candidate_id`. Added in `1.2`. |
|
||||
| `title` | string | Result title. |
|
||||
| `source` | string | Primary source name, such as `reddit`, `x`, `youtube`, or `grounding`. |
|
||||
| `url` | string | Canonical result URL. It may be empty when the provider supplies no link. |
|
||||
| `published_at` | string | Primary source item's publication date or timestamp. Omitted when unknown. |
|
||||
| `summary` | string | Normalized snippet, with the relevance explanation or body used as fallback. |
|
||||
| `engagement` | object | Native engagement counters from the primary source item, such as Reddit `score` and `num_comments` or X `likes` and `reposts`. |
|
||||
| `relevance_score` | number | Engine final score normalized to the inclusive `0.0`–`1.0` range. |
|
||||
| `cluster` | integer | Zero-based index into `clusters`. Omitted when the result is not assigned to a cluster. |
|
||||
|
||||
Fields whose value is unknown are omitted rather than emitted as JSON `null`. Strings and collection fields otherwise remain present, including empty strings, objects, or arrays.
|
||||
|
||||
## Comparison runs
|
||||
|
||||
Comparison queries use an envelope so each entity keeps its own contract:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "1.2",
|
||||
"comparison": true,
|
||||
"entities": ["OpenAI", "Anthropic"],
|
||||
"reports": [
|
||||
{"entity": "OpenAI", "report": {"schema_version": "1.2", "query": "OpenAI"}},
|
||||
{"entity": "Anthropic", "report": {"schema_version": "1.2", "query": "Anthropic"}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The abbreviated reports above only illustrate the envelope; real reports contain every documented top-level field.
|
||||
|
||||
## Versioning policy
|
||||
|
||||
- `schema_version` uses `major.minor` numbering.
|
||||
- Any breaking field removal, rename, type change, semantic change, or envelope change requires a major-version bump.
|
||||
- Backward-compatible field additions may use a minor-version bump. Consumers should ignore fields they do not recognize.
|
||||
- The checked-in golden snapshot test locks the complete current shape. Contract changes must update the version and snapshot deliberately.
|
||||
- `1.2` added `candidate_id` to each `results` entry so verdicts can be joined to the result they annotate.
|
||||
- Discovery `1.1` added `podcast_angle`, `x_article_angle`, `previously_surfaced_count`, `last_surfaced`, and `covered` to each discovery `results` entry — a backward-compatible minor bump; the fields carry their defaults (`null`/`null`/`0`/`null`/`false`) until an angle generator or the topic queue populates them.
|
||||
- `--json-profile=raw` is outside this compatibility policy because it mirrors internal pipeline dataclasses.
|
||||
|
||||
`--preflight --emit=json` is a different machine contract for permission and configuration inspection. `--json-profile` does not alter preflight output.
|
||||
@@ -0,0 +1,205 @@
|
||||
# Original v3.0.0 First-Run NUX Wizard (reference capture)
|
||||
|
||||
Captured verbatim from `SKILL.md` at git commit `0a9ff16` (v3.0.0, 2026-04-08),
|
||||
the first-run setup wizard Matt built. Preserved here for provenance and as the
|
||||
source for the restored modal NUX (see docs/plans/2026-06-22-001-feat-restore-nux-wizard-plan.md).
|
||||
This is a historical snapshot - the live wizard in SKILL.md Step 0 uses the CURRENT
|
||||
source inventory (Digg, youtube_comments, SC backups) and omits Threads/Pinterest.
|
||||
|
||||
```markdown
|
||||
## Step 0: First-Run Setup Wizard
|
||||
|
||||
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever.
|
||||
|
||||
To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section **silently** and proceed to Step 1. **Do NOT say "Setup is complete" or any other status message — just move on.** The user doesn't need to be told setup is done every time they run the skill.
|
||||
|
||||
**When first run is detected, detect your platform first:**
|
||||
|
||||
**If you do NOT have WebSearch capability (OpenClaw, Codex, raw CLI):** Run the OpenClaw setup flow below.
|
||||
**If you DO have WebSearch (Claude Code):** Run the standard setup flow below.
|
||||
|
||||
---
|
||||
|
||||
### OpenClaw / Non-WebSearch Setup Flow
|
||||
|
||||
Run environment detection first:
|
||||
```bash
|
||||
python3 "${SKILL_ROOT}/scripts/last30days.py" setup --openclaw
|
||||
```
|
||||
|
||||
Read the JSON output. It tells you what's already configured. Display a status summary:
|
||||
|
||||
```
|
||||
👋 Welcome to /last30days!
|
||||
|
||||
Detected:
|
||||
{✅ or ❌} yt-dlp (YouTube search)
|
||||
{✅ or ❌} X/Twitter ({method} configured)
|
||||
{✅ or ❌} ScrapeCreators (TikTok, Instagram, Reddit backup)
|
||||
{✅ or ❌} Web search ({backend} configured)
|
||||
```
|
||||
|
||||
Then for each missing item, offer setup in priority order:
|
||||
|
||||
1. **ScrapeCreators** (if not configured): "ScrapeCreators adds TikTok and Instagram search (plus a Reddit backup if public Reddit gets rate-limited). 10,000 free calls, no credit card. (No referrals, no kickbacks - we don't get a cut.)"
|
||||
- Option A: "ScrapeCreators via GitHub (recommended)" -- Check if `gh` CLI was detected in the environment detection output above. If gh IS detected: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". Before running the command, display: "Registering via GitHub CLI..." If gh is NOT detected: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". Before running the command, display: "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V / Ctrl+V)." Then run `python3 "${SKILL_ROOT}/scripts/last30days.py" setup --github`, parse JSON output. Tries PAT first (if `gh` is installed), falls back to device flow which copies a one-time code to your clipboard and opens your browser. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to .env.
|
||||
- Option B: "I have a key" -- accept paste, write to .env
|
||||
- Option C: "Skip for now"
|
||||
|
||||
2. **X/Twitter** (if not configured): "X search finds tweets and conversations. To unlock X: add FROM_BROWSER=auto (reads browser cookies, free), XAI_API_KEY (no browser access, api.x.ai), or AUTH_TOKEN+CT0 (manual cookies)."
|
||||
- Option A: "I have an xAI API key" (recommended for servers -- persistent, no expiry). Write XAI_API_KEY to .env.
|
||||
- Option B: "I have AUTH_TOKEN + CT0 from my browser" -- accept both, write to .env
|
||||
- Option C: "Skip for now"
|
||||
|
||||
3. **YouTube** (if yt-dlp not found): "YouTube search needs yt-dlp. Run: `pip install yt-dlp`"
|
||||
|
||||
4. **Web search** (if no Brave/Exa/Serper key): "A web search key enables smarter results. Brave Search is free for 2,000 queries/month at brave.com/search/api"
|
||||
|
||||
After setup, write `SETUP_COMPLETE=true` to .env and proceed to research.
|
||||
|
||||
**Skip to "END OF FIRST-RUN WIZARD" below after completing the OpenClaw flow.**
|
||||
|
||||
---
|
||||
|
||||
### Claude Code Setup Flow (Standard)
|
||||
|
||||
**You MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text -> (2) setup modal -> (3) run setup if chosen -> (4) optional ScrapeCreators modal -> (5) topic picker. You MUST start at step 1.**
|
||||
|
||||
**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion - do NOT repeat any of the welcome text inside the AskUserQuestion call.**
|
||||
|
||||
Welcome to /last30days!
|
||||
|
||||
I research any topic across Reddit, X, YouTube, and other sources - synthesizing what people are actually saying right now.
|
||||
|
||||
Auto setup gives you 5 core sources for free in 30 seconds:
|
||||
- X/Twitter - reads your x.com browser cookies to authenticate (not saved to disk). Chrome on macOS will prompt for Keychain access.
|
||||
- Reddit with comments - public JSON, no API key needed
|
||||
- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars)
|
||||
- Hacker News + Polymarket + GitHub (if `gh` CLI installed) - always on, zero config
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation.
|
||||
|
||||
**Then call AskUserQuestion with ONLY this question and these options - no additional text:**
|
||||
|
||||
Question: "How would you like to set up?"
|
||||
Options:
|
||||
- "Auto setup (~30 seconds) - scans browser cookies for X + installs yt-dlp for YouTube"
|
||||
- "Manual setup - show me what to configure"
|
||||
- "Skip for now - Reddit (with comments), HN, Polymarket, GitHub (if gh installed), Web"
|
||||
|
||||
**If the user picks 1 (Auto setup):**
|
||||
|
||||
**Before running the setup command, get cookie consent:**
|
||||
|
||||
Check if `BROWSER_CONSENT=true` already exists in `~/.config/last30days/.env`. If it does, skip the consent prompt and run setup directly.
|
||||
|
||||
If `BROWSER_CONSENT=true` is NOT present, **call AskUserQuestion:**
|
||||
Question: "Auto setup will scan your browser for x.com cookies to authenticate X search. Cookies are read live, not saved to disk. Chrome on macOS will prompt for Keychain access. OK to proceed?"
|
||||
Options:
|
||||
- "Yes, scan my cookies for X" - Run setup as normal. Append `BROWSER_CONSENT=true` to .env after setup completes.
|
||||
- "Skip X, just set up YouTube" - Run setup with YouTube only (install yt-dlp). Do not scan cookies.
|
||||
- "I have an xAI API key instead" - Ask them to paste it, write XAI_API_KEY to .env. Then install yt-dlp.
|
||||
|
||||
Run the setup subcommand:
|
||||
```bash
|
||||
cd {SKILL_DIR} && python3 scripts/last30days.py setup
|
||||
```
|
||||
Show the user the results (what cookies were found, whether yt-dlp was installed).
|
||||
|
||||
**Then show the optional ScrapeCreators offer (plain text, then modal):**
|
||||
|
||||
Want TikTok and Instagram too? ScrapeCreators adds those platforms - 10,000 free calls, no credit card. It also serves as a Reddit backup if public Reddit ever gets rate-limited.
|
||||
|
||||
**Before showing the ScrapeCreators modal, check for `gh` CLI:** Run `which gh` via Bash silently. Store the result as gh_available (true if found, false if not).
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Want to add TikTok, Instagram, and Reddit backup via ScrapeCreators? (We don't get a cut.)"
|
||||
Options:
|
||||
- "ScrapeCreators via GitHub (fastest, recommended)" - If gh_available: description should say "Registers directly via GitHub CLI in ~2 seconds - no browser needed". If NOT gh_available: description should say "Copies a one-time code to your clipboard and opens GitHub to authorize". After the user selects this option: If gh_available, display "Registering via GitHub CLI..." before running the command. If NOT gh_available, display "I'll copy a one-time code to your clipboard and open GitHub. When GitHub asks for a device code, just paste (Cmd+V on Mac, Ctrl+V on Windows/Linux)." Then run `cd {SKILL_DIR} && python3 scripts/last30days.py setup --github` via Bash with a 5-minute timeout. This tries PAT auth first (if `gh` CLI is installed, zero browser needed), then falls back to GitHub device flow which copies a one-time code to your clipboard and opens GitHub in your browser. Parse the JSON stdout. If `status` is `success`, write `SCRAPECREATORS_API_KEY={api_key}` to `~/.config/last30days/.env`. If `method` is `pat`, show: "You're in! Registered via GitHub CLI - zero browser needed. 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is true, show: "You're in! (The authorization code was copied to your clipboard automatically.) 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `method` is `device` and `clipboard_ok` is false, show: "You're in! 10,000 free calls. TikTok, Instagram, and Reddit backup are now active." If `status` is `timeout` or `error`, show: "GitHub auth didn't complete. No worries - you can sign up at scrapecreators.com instead or try again later." Then offer the web signup option.
|
||||
- "Open scrapecreators.com (Google sign-in)" - run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env
|
||||
- "I have a key" - accept the key, write to .env
|
||||
- "Skip for now" - proceed without ScrapeCreators
|
||||
|
||||
**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:**
|
||||
|
||||
Your ScrapeCreators key powers TikTok, Instagram, Threads, Pinterest, and YouTube comments. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.)
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "Which ScrapeCreators sources do you want on?"
|
||||
Options:
|
||||
- "TikTok + Instagram (recommended)" - append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "TikTok and Instagram are on, plus Reddit backup if public Reddit has issues. You can add threads, pinterest, youtube_comments to INCLUDE_SOURCES anytime."
|
||||
- "Everything - TikTok, Instagram, Threads, Pinterest, YouTube comments" - append `INCLUDE_SOURCES=tiktok,instagram,threads,pinterest,youtube_comments` to ~/.config/last30days/.env. Confirm: "All ScrapeCreators sources are on."
|
||||
- "Just the basics - let's run our first search" - don't write the flag. Confirm: "Got it. ScrapeCreators will serve as Reddit backup. You can add sources to INCLUDE_SOURCES in your .env anytime."
|
||||
|
||||
**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:**
|
||||
|
||||
**Call AskUserQuestion:**
|
||||
Question: "What do you want to research first?"
|
||||
Options:
|
||||
- "Claude Code vs Codex" - tech comparison
|
||||
- "Sam Altman" - person in the news
|
||||
- "Warriors Basketball" - sports
|
||||
- "AI Legal Prompting Techniques" - niche/professional
|
||||
- "Type my own topic"
|
||||
|
||||
If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly.
|
||||
|
||||
**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.**
|
||||
|
||||
**If the user picks 2 (Manual setup):**
|
||||
Show them this guide (present as plain text, not blockquoted):
|
||||
|
||||
The magic of /last30days is Reddit comments + X posts together - and both are free. Here's how to unlock each source.
|
||||
|
||||
Add these to `~/.config/last30days/.env`:
|
||||
|
||||
X/Twitter (pick one - this is the most important):
|
||||
- `FROM_BROWSER=auto` - free. Reads your x.com login cookies at search time to authenticate. Cookies are read live each run, not saved to disk. Chrome on macOS will prompt for Keychain access the first time. Firefox and Safari don't.
|
||||
- `XAI_API_KEY=xxx` - no browser access needed. Get a key at api.x.ai. Best for servers or if you don't want cookie scanning.
|
||||
- `AUTH_TOKEN=xxx` + `CT0=xxx` - paste your X cookies manually (x.com -> F12 -> Application -> Cookies)
|
||||
|
||||
Reddit (free, works out of the box):
|
||||
- Public JSON gives you threads + top comments with upvote counts. No setup required.
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - optional backup source if public Reddit gets rate-limited.
|
||||
- `OPENAI_API_KEY=xxx` - optional fallback if public Reddit search has trouble finding threads.
|
||||
|
||||
YouTube (free, open source):
|
||||
- Run `brew install yt-dlp` - free, open source, 190K+ GitHub stars. Enables YouTube search and transcripts.
|
||||
|
||||
Bonus: TikTok, Instagram, Threads, Pinterest, YouTube comments (ScrapeCreators):
|
||||
- `SCRAPECREATORS_API_KEY=xxx` - 10,000 free calls at scrapecreators.com.
|
||||
- After adding your key, set `INCLUDE_SOURCES=tiktok,instagram` to turn on the most popular ones. Add threads, pinterest, youtube_comments for more.
|
||||
|
||||
GitHub Issues/PRs (free, no key needed):
|
||||
- If you have the `gh` CLI installed (`brew install gh`), GitHub search is automatic. No API key required.
|
||||
|
||||
Perplexity Sonar Pro (AI-synthesized research via OpenRouter):
|
||||
- `OPENROUTER_API_KEY=xxx` - adds AI-synthesized research with citations as an additive source alongside Reddit/X/YouTube. Returns structured narratives with specific dates, names, and numbers that social sources miss. ~$0.02/run.
|
||||
- After adding your key, set `INCLUDE_SOURCES=perplexity` (or append to existing, e.g. `INCLUDE_SOURCES=tiktok,instagram,perplexity`).
|
||||
- Use `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query) on topics that need serious investigation.
|
||||
- Bonus: also powers the planning and reranking engine if you don't have a Gemini/OpenAI/xAI key.
|
||||
|
||||
Other bonus sources (add anytime):
|
||||
- `EXA_API_KEY=xxx` - semantic web search, 1K free/month (exa.ai)
|
||||
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` - Bluesky (free app password)
|
||||
- `BRAVE_API_KEY=xxx` - Brave web search
|
||||
|
||||
Always add this last line: `SETUP_COMPLETE=true`
|
||||
|
||||
**CRITICAL: NEVER overwrite an existing .env file.** Before writing ANY key to `~/.config/last30days/.env`:
|
||||
1. Check if the file exists: `test -f ~/.config/last30days/.env`
|
||||
2. If it exists, READ it first, then APPEND only missing keys using `>>` (double redirect)
|
||||
3. NEVER use `>` (single redirect) which destroys existing content
|
||||
4. If it doesn't exist, create it: `mkdir -p ~/.config/last30days && touch ~/.config/last30days/.env`
|
||||
|
||||
**Then call AskUserQuestion:**
|
||||
Question: "How do you want to add your keys?"
|
||||
Options:
|
||||
- "Open .env in my editor" - Creates the file with a commented template and opens it. You edit, save, and come back.
|
||||
- "Paste keys here" - Paste your API keys and I'll write the file for you.
|
||||
- "I'll do it myself" - I'll tell you the file path and you handle it.
|
||||
|
||||
**If the user picks "Open .env in editor":**
|
||||
Create `~/.config/last30days/.env` if it doesn't exist (check first!), pre-populated with this template:
|
||||
```
|
||||
```
|
||||
@@ -1,6 +1,6 @@
|
||||
# Search Quality Eval
|
||||
|
||||
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
|
||||
What it does:
|
||||
|
||||
@@ -18,13 +18,13 @@ What it does:
|
||||
Recommended usage:
|
||||
|
||||
```bash
|
||||
uv run python scripts/evaluate_search_quality.py
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py
|
||||
```
|
||||
|
||||
Useful flags:
|
||||
|
||||
```bash
|
||||
uv run python scripts/evaluate_search_quality.py \
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py \
|
||||
--baseline-rev origin/main \
|
||||
--candidate-rev HEAD \
|
||||
--no-default-topics \
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
---
|
||||
title: "Persistent discovery topic queue: five interlocking design conventions"
|
||||
date: 2026-07-20
|
||||
category: architecture-patterns
|
||||
module: discovery-topic-queue
|
||||
problem_type: architecture_pattern
|
||||
component: database
|
||||
severity: high
|
||||
applies_when:
|
||||
- "Adding default-on local persistence (SQLite, JSON state) hooked onto the end of an expensive pipeline run"
|
||||
- "Building a fuzzy identity layer over LLM-named entities whose names drift across runs"
|
||||
- "Reading a feature toggle in an engine where .env-file values only reach code through env.get_config's keys allowlist"
|
||||
- "Recording per-item state in a loop where a later item could fuzzy-match a row written earlier in the same run"
|
||||
- "Persisting user-set status (covered, dismissed, read) that must survive entity renames"
|
||||
tags:
|
||||
- "discovery-topic-queue"
|
||||
- "fuzzy-matching"
|
||||
- "sqlite-persistence"
|
||||
- "env-allowlist-opt-out"
|
||||
- "two-phase-write"
|
||||
- "covered-status-inheritance"
|
||||
- "guarded-write-hook"
|
||||
- "scoped-db"
|
||||
- "llm-naming-drift"
|
||||
related_components:
|
||||
- "skills/last30days/scripts/store.py"
|
||||
- "skills/last30days/scripts/last30days.py"
|
||||
- "skills/last30days/scripts/lib/env.py"
|
||||
- "tests/test_store.py"
|
||||
- "tests/test_discover_mode.py"
|
||||
---
|
||||
|
||||
|
||||
# Persistent discovery topic queue: five interlocking design conventions
|
||||
|
||||
## Context
|
||||
|
||||
PR #852 shipped a persistent topic queue for `/last30days discover`: every real
|
||||
discovery run records which topics it surfaced into a `discovery_topics` table in
|
||||
research.db, so the podcast/X-article pipeline remembers what it has already seen
|
||||
("surfaced 3rd time") and what the user already produced content for ("marked
|
||||
covered"). This is the design record for that queue - five conventions that were
|
||||
each load-bearing in review, two of them caught as real bugs (one P0). The
|
||||
seed-source corroboration change that landed in the same PR is documented
|
||||
separately in
|
||||
`docs/solutions/design-patterns/ranked-output-confidence-floor-honest-empty-state.md`
|
||||
(section 2b); this doc does not cover it.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Default-on, disabled only via the config allowlist - never bare os.environ
|
||||
|
||||
The queue records every real (non-mock) run by default; the literal value `off`
|
||||
disables it. The knob is registered in `env.get_config`'s keys allowlist
|
||||
(`skills/last30days/scripts/lib/env.py:482`):
|
||||
|
||||
```python
|
||||
# Discovery topic queue (podcast/X-article pipeline memory). Default
|
||||
# ON; the literal value "off" disables queue writes and annotations.
|
||||
('LAST30DAYS_DISCOVERY_QUEUE', None),
|
||||
```
|
||||
|
||||
and read from the resolved config dict, never `os.environ`
|
||||
(`skills/last30days/scripts/last30days.py:1312-1314`):
|
||||
|
||||
```python
|
||||
queue_setting = str(config.get("LAST30DAYS_DISCOVERY_QUEUE") or "").strip().lower()
|
||||
if queue_setting == "off" or not report.topics:
|
||||
return report
|
||||
```
|
||||
|
||||
WHY: `.env`-file users' values only reach the engine through the `get_config`
|
||||
allowlist merge - a bare `os.environ` read silently ignores them, a documented
|
||||
invisible-failure class in this repo. Scoped runs (`--save-dir`) write the scoped
|
||||
research.db via `store.scoped_db(_scoped_store_db(args))`
|
||||
(`last30days.py:432-437`, `store.py:41-53`), never the global one; `--mock` runs
|
||||
stay 100% side-effect-free (`last30days.py:1505`).
|
||||
|
||||
### 2. Annotate-only fuzzy matching - a match stamps context, it never merges rows
|
||||
|
||||
`store.match_discovery_topic` tries exact normalized-name match first, then the
|
||||
best entity-overlap candidate - the better of full `entity_key` token overlap and
|
||||
anchor-token overlap - at a conservative floor
|
||||
(`skills/last30days/scripts/store.py:810`, `898-938`):
|
||||
|
||||
```python
|
||||
DISCOVERY_QUEUE_OVERLAP_THRESHOLD = 0.6
|
||||
...
|
||||
if best is not None and best_overlap >= DISCOVERY_QUEUE_OVERLAP_THRESHOLD:
|
||||
return dict(best)
|
||||
```
|
||||
|
||||
A fuzzy match only annotates the rendered card - the `Pipeline: surfaced Nth
|
||||
time, marked covered` line (`skills/last30days/scripts/lib/render.py:153-168`) -
|
||||
and never merges or rewrites queue rows (`store.py:806-809`, `906-907`).
|
||||
|
||||
WHY: with annotate-only semantics a false-positive match costs one noisy line on
|
||||
one card; a false merge would silently collapse two distinct stories into one
|
||||
row and hide one of them forever. The threshold is tunable precisely because
|
||||
mislabeling is recoverable and data loss is not.
|
||||
|
||||
### 3. Two-phase hook: match ALL topics before recording ANY
|
||||
|
||||
`_annotate_and_record_discovery_queue` computes priors for every topic first,
|
||||
then records surfacings, inside one `store.scoped_db` block
|
||||
(`skills/last30days/scripts/last30days.py:1323-1345`):
|
||||
|
||||
```python
|
||||
with store.scoped_db(_scoped_store_db(args)):
|
||||
store.init_db()
|
||||
# Phase 1: match EVERY topic before recording ANY. Interleaving
|
||||
# match+record in one loop lets topic N fuzzy-match a same-anchor
|
||||
# sibling row this very run recorded seconds earlier, falsely
|
||||
# annotating a first-ever topic as "surfaced 2nd time".
|
||||
priors = [store.match_discovery_topic(topic.name) for topic in report.topics]
|
||||
# Phase 2: record this run's surfacings. ...
|
||||
for topic, prior in zip(report.topics, priors):
|
||||
```
|
||||
|
||||
WHY: one report often contains same-anchor siblings ("Gemma 4 chat templates" /
|
||||
"Gemma 4 tool calling fixes"). Interleaved match+record lets topic N fuzzy-match
|
||||
the row topic N-1 wrote seconds earlier, falsely annotating a first-ever topic
|
||||
as a repeat. Caught in review; regression-tested.
|
||||
|
||||
### 4. Covered inheritance: fresh rows born covered, existing rows never mutated
|
||||
|
||||
`record_discovery_surfacing(inherit_covered_at=...)` makes a fresh row start in
|
||||
`covered` status when its fuzzy-matched prior is covered; the `ON CONFLICT`
|
||||
update path deliberately never touches `status`/`covered_at`
|
||||
(`skills/last30days/scripts/store.py:842-895`):
|
||||
|
||||
```python
|
||||
status = "covered" if inherit_covered_at else "surfaced"
|
||||
...
|
||||
ON CONFLICT(normalized_name) DO UPDATE SET
|
||||
surface_count = surface_count + 1,
|
||||
last_surfaced = excluded.last_surfaced,
|
||||
last_run_ref = excluded.last_run_ref,
|
||||
domain = CASE WHEN excluded.domain <> '' THEN excluded.domain ELSE domain END
|
||||
```
|
||||
|
||||
The caller passes it when a topic's prior is covered
|
||||
(`last30days.py:1334-1344`). Locked by the flip-flop regression test
|
||||
`test_covered_status_survives_judge_rename_across_runs`
|
||||
(`tests/test_store.py:1082-1101`) and by
|
||||
`tests/test_store.py:1060-1079` (ON CONFLICT ignores `inherit_covered_at`).
|
||||
|
||||
WHY: the LLM judge renames the same story across runs; without inheritance a
|
||||
rename forks a fresh uncovered row and the user's covered mark silently
|
||||
evaporates. Without the never-mutate rule, a stale inherit could flip a row the
|
||||
user just changed.
|
||||
|
||||
### 5. Guarded, synchronous end-of-run write - never crash a finished pipeline
|
||||
|
||||
The hook call in `_run_discover` is wrapped so a broken queue db degrades to a
|
||||
stderr warning and an unannotated report
|
||||
(`skills/last30days/scripts/last30days.py:1505-1515`):
|
||||
|
||||
```python
|
||||
if not args.mock:
|
||||
try:
|
||||
report = _annotate_and_record_discovery_queue(report, args, config)
|
||||
except (sqlite3.Error, OSError) as exc:
|
||||
# A broken queue db (locked, read-only dir, corrupt) must never
|
||||
# destroy a finished multi-minute pipeline run: warn and render
|
||||
# the report without queue annotations (fields keep defaults).
|
||||
sys.stderr.write(
|
||||
f"[last30days] Warning: discovery queue unavailable ({exc}); "
|
||||
"continuing without queue annotations.\n"
|
||||
)
|
||||
```
|
||||
|
||||
WHY: unguarded, a locked/read-only/corrupt research.db raises AFTER the
|
||||
multi-minute research pipeline finished and discards all of its output - the PR
|
||||
#852 code review's P0, empirically reproduced. The write also runs synchronously
|
||||
after the pipeline returns (`last30days.py:1308-1310` docstring): it touches
|
||||
disk, so the abandon-on-timeout daemon-thread pattern is forbidden here (see
|
||||
`docs/solutions/logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md`).
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Ranked by blast radius when a convention is violated:
|
||||
|
||||
- Unguarded end-of-run write (5): the whole run's output is destroyed by a
|
||||
bookkeeping failure, and only in degraded environments (locked db, read-only
|
||||
dir), so it ships green and detonates on exactly the machines you cannot see.
|
||||
This was the review's P0.
|
||||
- Interleaved match+record (3): the queue's core promise ("first time you've
|
||||
seen this") is wrong on day one - a first-ever topic gets annotated "surfaced
|
||||
2nd time" by its same-run sibling, and no cross-run test catches it because
|
||||
the corruption happens inside a single run.
|
||||
- Bare os.environ read (1): `.env`-file users cannot turn the queue off; the
|
||||
toggle works in the maintainer's shell and fails invisibly for everyone
|
||||
configuring via file.
|
||||
- Merging on fuzzy match (2): a 0.6-overlap false positive stops being one
|
||||
noisy line and becomes a hidden story - unrecoverable data loss from a
|
||||
heuristic.
|
||||
- Mutating rows or skipping inheritance (4): user covered marks flip-flop with
|
||||
judge naming drift, so the queue re-pitches stories the user already produced,
|
||||
which is the exact failure the queue exists to prevent.
|
||||
|
||||
## When to Apply
|
||||
|
||||
- Any default-on local persistence bolted onto the end of an expensive pipeline:
|
||||
the write must be guarded (degrade to a warning) and synchronous if it touches
|
||||
disk.
|
||||
- Any fuzzy identity layer over LLM-named entities: keep matching annotate-only,
|
||||
batch all matches before any writes in a run, and inherit user-set status onto
|
||||
fresh rows instead of mutating existing ones.
|
||||
- Any new engine toggle in this repo: register it in `env.get_config`'s keys
|
||||
allowlist and read it from the config dict, never bare `os.environ`.
|
||||
|
||||
## Examples
|
||||
|
||||
Covered flip-flop, the archetype 3-run scenario (mirrors
|
||||
`tests/test_store.py:1082-1101`):
|
||||
|
||||
1. Run 1 surfaces "Gemma 4 chat templates"; the user records an episode and
|
||||
runs `queue cover "Gemma 4 chat templates"` (row status: covered).
|
||||
2. Run 2's judge names the same story "Gemma 4 template fixes". Exact match
|
||||
misses; fuzzy match (anchor overlap `gemma`/`4` at >= 0.6) finds the covered
|
||||
prior, so the new row is recorded born covered and the card renders
|
||||
`Pipeline: surfaced 2nd time, marked covered` instead of pitching it fresh.
|
||||
3. Run 3 resurfaces "Gemma 4 template fixes"; it exact-matches its own covered
|
||||
row (`covered_at` still the run-1 date). Without convention 4, run 2 would
|
||||
have forked an uncovered row and run 3 would re-pitch a story the user
|
||||
already covered.
|
||||
|
||||
Queue failure behavior: with research.db locked by another process, a discovery
|
||||
run still prints the full rendered report; stderr shows
|
||||
`[last30days] Warning: discovery queue unavailable (database is locked);
|
||||
continuing without queue annotations.` and the cards simply lack Pipeline lines.
|
||||
|
||||
## Related
|
||||
|
||||
- PR #852 - judged topic names, junk gate, angles, topic queue (this design).
|
||||
- `docs/solutions/design-patterns/ranked-output-confidence-floor-honest-empty-state.md`
|
||||
section 2b - the seed-source corroboration rule from the same PR (not covered
|
||||
here).
|
||||
- `docs/solutions/logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md`
|
||||
- why abandon-on-timeout daemon threads are forbidden for disk writers.
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
title: Search-quality eval is manual by default, not a CI gate on every PR
|
||||
date: 2026-05-10
|
||||
category: docs/solutions/architecture
|
||||
module: skills/last30days/scripts/evaluate_search_quality.py
|
||||
problem_type: design_decision
|
||||
component: ci_policy
|
||||
severity: low
|
||||
applies_when:
|
||||
- a contributor proposes wiring search-quality eval into PR CI
|
||||
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
|
||||
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
|
||||
related_components:
|
||||
- search_quality_evaluation
|
||||
- ci_workflow
|
||||
- llm_judging
|
||||
tags:
|
||||
- ci-policy
|
||||
- eval
|
||||
- design-decision
|
||||
- cost-vs-signal
|
||||
- non-determinism
|
||||
- manual-gates
|
||||
---
|
||||
|
||||
# Search-quality eval is manual by default, not a CI gate on every PR
|
||||
|
||||
## Context
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
|
||||
|
||||
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
|
||||
|
||||
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
|
||||
|
||||
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
|
||||
|
||||
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
|
||||
|
||||
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Keep search-quality eval available, just not automatic
|
||||
|
||||
The script stays runnable by maintainers and contributors. The pattern is:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PYTHON=python3.13 \
|
||||
python3 skills/last30days/scripts/evaluate_search_quality.py \
|
||||
--baseline main --candidate HEAD
|
||||
```
|
||||
|
||||
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
|
||||
|
||||
### 2. Standard PR CI gates remain deterministic and contract-shaped
|
||||
|
||||
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
|
||||
|
||||
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
|
||||
|
||||
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
|
||||
|
||||
### 4. Revisit if the eval can ever be made offline-deterministic
|
||||
|
||||
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
|
||||
|
||||
## What this means in practice
|
||||
|
||||
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
|
||||
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
|
||||
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
|
||||
|
||||
## Links
|
||||
|
||||
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
|
||||
- `docs/search-quality-eval.md` — user-facing usage documentation
|
||||
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
|
||||
|
||||
---
|
||||
|
||||
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
|
||||
@@ -0,0 +1,185 @@
|
||||
---
|
||||
title: "Optional-value argparse flags: dispatch on 'is not None', never truthiness"
|
||||
date: 2026-07-12
|
||||
category: conventions
|
||||
module: last30days-cli
|
||||
problem_type: convention
|
||||
component: tooling
|
||||
severity: medium
|
||||
applies_when:
|
||||
- "Retrofitting an optional value onto an existing flag (nargs='?' + const), where old dispatch sites were written for a two-state flag"
|
||||
- "Any flag or config key where a falsy value ('', 0, []) is a meaningful present-value distinct from absence"
|
||||
- "Dependent/modifier flags whose behavior only applies when an anchor flag is present"
|
||||
tags:
|
||||
- argparse
|
||||
- nargs-optional
|
||||
- truthiness
|
||||
- cli-flags
|
||||
- dispatch
|
||||
- dependent-flags
|
||||
- silent-failure
|
||||
- sentinel-values
|
||||
related_components:
|
||||
- testing_framework
|
||||
---
|
||||
|
||||
# Optional-value argparse flags: dispatch on `is not None`, never truthiness
|
||||
|
||||
## Context
|
||||
|
||||
PR #816 retrofitted an optional value onto the `--discover` flag in `skills/last30days/scripts/last30days.py`:
|
||||
|
||||
```python
|
||||
parser.add_argument(
|
||||
"--discover",
|
||||
metavar="DOMAIN",
|
||||
nargs="?",
|
||||
const="",
|
||||
default=None,
|
||||
help=(
|
||||
"Sweep river listings and rank the topics accelerating in a domain; "
|
||||
"each survivor gets a full research pass. Bare --discover (no domain) "
|
||||
"runs global trending across every feed's hot list"
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
With `nargs="?"` plus `const=""` plus `default=None`, the flag is deliberately three-state:
|
||||
|
||||
- flag absent -> `args.discover is None` -> normal research run
|
||||
- bare `--discover` -> `args.discover == ""` -> global trending sweep (empty domain)
|
||||
- `--discover X` -> `args.discover == "X"` -> domain-scoped discovery
|
||||
|
||||
The near-miss: the pre-existing dispatch was `if args.discover:`. Under Python truthiness, `""` and `None` are both falsy, so bare `--discover` would have been indistinguishable from no flag at all. The headline new mode of the PR (global trending) would simply never fire - the run would silently route into the normal-research path with no error, no warning, and no failing test unless a test exercised the bare form specifically. This was caught during development and the dispatch was changed to key on flag presence.
|
||||
|
||||
A second, related trap was caught in PR review (Greptile P2): the dependent flag `--discover-shallow` was accepted without `--discover` and silently no-opped into a full research run - the user asked for a fast, thin sweep and got a slow, full one. It was fixed with an explicit guard that errors loudly (exit 2).
|
||||
|
||||
## Guidance
|
||||
|
||||
1. With `nargs="?"` + `const`, the None/const/value trichotomy IS the contract: absent = `None`, bare flag = the `const` value, valued = the user's string. Dispatch on `args.flag is not None` (flag present), never on truthiness. When retrofitting optionality onto an existing flag, grep EVERY existing reference to `args.<flag>` - the old references were written when the flag was two-state and any `if args.flag:` among them is now a latent misroute.
|
||||
|
||||
2. Put a comment at the dispatch site explaining why it uses `is not None`. `if args.flag:` looks like the idiomatic form, and a future "simplification" pass will happily rewrite the correct check back into the bug. The repo's dispatch carries exactly this comment (`skills/last30days/scripts/last30days.py`):
|
||||
|
||||
```python
|
||||
# Bare --discover (no domain) is global trending, so the dispatch keys on
|
||||
# "flag present" (is not None), never on the domain string's truthiness.
|
||||
if args.discover is not None:
|
||||
```
|
||||
|
||||
3. Dependent/modifier flags (`--x-modifier` that only applies with `--x`) must error loudly when their anchor flag is absent - never silently no-op. A silent no-op means the user asked for one mode and got another with zero feedback. Reject with a clear message and a nonzero exit:
|
||||
|
||||
```python
|
||||
if args.discover_shallow:
|
||||
# Without --discover this flag would silently no-op into a full
|
||||
# research run - reject it instead of ignoring the requested mode.
|
||||
sys.stderr.write(
|
||||
"[last30days] --discover-shallow only applies to --discover runs; "
|
||||
"add --discover [domain] or drop the flag.\n"
|
||||
)
|
||||
return 2
|
||||
```
|
||||
|
||||
4. Pin both behaviors with CLI-level subprocess tests. Unit tests of the parser alone would not have caught the misroute, because parsing was correct - the bug lived in dispatch. The tests must run the real entry point end to end: one asserting the bare form reaches the new mode, one asserting the orphaned dependent flag is rejected (see Examples).
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The failure mode is silent misrouting, which is the worst kind: the feature ships, `--help` documents the bare form, and every invocation of it quietly runs the wrong mode. There is no exception, no error message, no test failure - the output is a plausible-looking result from the wrong pipeline. Nothing surfaces the bug unless a test (or an alert user) exercises the bare form specifically and checks which mode actually ran. The same is true of the dependent-flag no-op: `--discover-shallow` without `--discover` produced a valid full research run, just not the one the user asked for.
|
||||
|
||||
## When to Apply
|
||||
|
||||
- Retrofitting an optional value onto an existing flag (`action="store_true"` or a plain valued option becoming `nargs="?"`): audit every dispatch site that reads the flag.
|
||||
- Any flag where a falsy value (`""`, `0`, `[]`) is a MEANINGFUL present-value distinct from absence - the sentinel-vs-truthiness distinction applies beyond argparse (env vars, config keys, JSON fields).
|
||||
- Dependent/modifier flags whose behavior only applies when an anchor flag is present.
|
||||
|
||||
## Examples
|
||||
|
||||
Before (the near-miss - conflates bare flag with no flag):
|
||||
|
||||
```python
|
||||
if args.discover: # '' and None are both falsy: bare --discover falls through
|
||||
return _run_discover(args, config)
|
||||
```
|
||||
|
||||
After (`skills/last30days/scripts/last30days.py`, with the drift-guard comment):
|
||||
|
||||
```python
|
||||
# Bare --discover (no domain) is global trending, so the dispatch keys on
|
||||
# "flag present" (is not None), never on the domain string's truthiness.
|
||||
if args.discover is not None:
|
||||
if topic:
|
||||
sys.stderr.write(
|
||||
"[last30days] --discover supplies the domain and cannot be combined "
|
||||
"with a positional topic.\n"
|
||||
)
|
||||
return 2
|
||||
if args.drill:
|
||||
sys.stderr.write("[last30days] --discover and --drill are mutually exclusive.\n")
|
||||
return 2
|
||||
return _run_discover(args, config)
|
||||
```
|
||||
|
||||
The dependent-flag guard immediately below the dispatch:
|
||||
|
||||
```python
|
||||
if args.discover_shallow:
|
||||
# Without --discover this flag would silently no-op into a full
|
||||
# research run - reject it instead of ignoring the requested mode.
|
||||
sys.stderr.write(
|
||||
"[last30days] --discover-shallow only applies to --discover runs; "
|
||||
"add --discover [domain] or drop the flag.\n"
|
||||
)
|
||||
return 2
|
||||
```
|
||||
|
||||
The two pinning tests in `tests/test_discover_mode.py`, both running the real CLI via subprocess:
|
||||
|
||||
```python
|
||||
def test_discovery_cli_bare_discover_is_global_trending():
|
||||
"""Bare --discover (no domain) must run global trending, not error."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"skills/last30days/scripts/last30days.py",
|
||||
"--discover",
|
||||
"--mock",
|
||||
"--emit=json",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["kind"] == "discovery"
|
||||
assert payload["domain"] == ""
|
||||
assert payload["outcome"] in {"ok", "nothing-solid"}
|
||||
|
||||
|
||||
def test_discovery_cli_rejects_shallow_without_discover():
|
||||
"""--discover-shallow on a normal topic run must error, not silently no-op
|
||||
into a full research pass (P2 from PR #816 review)."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"skills/last30days/scripts/last30days.py",
|
||||
"AI agents",
|
||||
"--discover-shallow",
|
||||
"--mock",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 2
|
||||
assert "--discover-shallow only applies to --discover runs" in result.stderr
|
||||
```
|
||||
|
||||
The first test asserts not just exit 0 but that the discovery pipeline actually ran (`payload["kind"] == "discovery"`, `payload["domain"] == ""`) - the exact property the truthiness bug would have violated. Source: PR #816 (last30days-skill).
|
||||
|
||||
## Related
|
||||
|
||||
- [Ranked-output confidence floor + honest empty state](../design-patterns/ranked-output-confidence-floor-honest-empty-state.md) - sibling lesson from the same PR #816 discover rebuild (ranking quality).
|
||||
- [Non-daemon executor threads defeat wall-clock budgets](../logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md) - sibling lesson from PR #816, same lesson class: a discover-mode defect that result-oriented unit tests structurally cannot catch (process lifetime there, bare-flag vs flag-absent conflation here).
|
||||
- [PR #816](https://github.com/mvanhorn/last30days-skill/pull/816) - the discovery rebuild that introduced the three-state `--discover` flag (released v3.14.0).
|
||||
@@ -0,0 +1,231 @@
|
||||
---
|
||||
title: "Ranked-output features need an explicit confidence floor with an honest empty state"
|
||||
date: 2026-07-12
|
||||
last_updated: 2026-07-20
|
||||
category: design-patterns
|
||||
module: discover-trending
|
||||
problem_type: design_pattern
|
||||
component: tooling
|
||||
severity: medium
|
||||
applies_when:
|
||||
- "Any feature that ranks and displays top-N results from variable-quality inputs (search, trending, recommendations, discovery)"
|
||||
- "Quiet or over-broad query domains where feeds return thin or noisy data"
|
||||
- "A gate measures corroboration or independence downstream of a stage of the same pipeline that amplifies that signal (enrichment, fan-out, retrieval expansion)"
|
||||
symptoms:
|
||||
- "Top-N ranker emits near-zero-engagement items (e.g., five 1-like tweets) as a trend list because top-N has no notion of 'none of this is good enough'"
|
||||
resolution_type: code_fix
|
||||
tags:
|
||||
- ranking
|
||||
- confidence-floor
|
||||
- empty-state
|
||||
- top-n
|
||||
- discovery
|
||||
- trending
|
||||
- signal-quality
|
||||
- corroboration
|
||||
- "seed-sources"
|
||||
- "junk-shape"
|
||||
- "source-independence"
|
||||
related_components:
|
||||
- "skills/last30days/scripts/lib/rerank.py"
|
||||
- "skills/last30days/scripts/lib/pipeline.py"
|
||||
- "tests/test_discover_floor.py"
|
||||
---
|
||||
|
||||
# Ranked-output features need an explicit confidence floor with an honest empty state
|
||||
|
||||
## Context
|
||||
|
||||
The `--discover` trending feature sweeps listing feeds (r/all, Hacker News front page, Digg, X), clusters what it finds into candidate topics, and emits the top topics by an engagement-velocity score. The selection was purely relative: `run_discover()` in `skills/last30days/scripts/lib/pipeline.py` computed `topic_limit = max(5, min(10, limit))` and took the top N clusters by score, regardless of whether the Nth-best (or even the 1st-best) was any good.
|
||||
|
||||
On quiet or overly broad domains the feeds return thin, noisy data, and a relative ranker has no way to say so. The named failure (2026-07-12): `/last30days --discover "sports"` returned five single-source tweets with 1 like each - a Wii Sports nostalgia thread, a kids-travel-sports burnout post, a manga review that mentioned sports - presented with straight-faced ranks 1 through 5 as a "trend list". Every stage worked as designed. The sweep ran, the clusterer clustered, the scorer scored. The problem is structural: a top-N ranker ranks noise against noise when inputs are thin. Relative ranking cannot express "nothing here is good enough to show a user"; that requires an absolute gate the pipeline did not have.
|
||||
|
||||
## Guidance
|
||||
|
||||
The pattern shipped in PR #816 (merged, v3.14.0) has four parts. Apply all four when building any ranked-output surface.
|
||||
|
||||
### 1. Put an absolute floor in front of the relative ranker
|
||||
|
||||
Before a candidate is allowed to compete on relative score, it must clear an absolute evidence bar. The floor lives in `skills/last30days/scripts/lib/rerank.py`:
|
||||
|
||||
```python
|
||||
FLOOR_MIN_ENGAGEMENT = 25.0
|
||||
FLOOR_MIN_SOURCES = 2
|
||||
FLOOR_SINGLE_SOURCE_ENGAGEMENT = 200.0
|
||||
|
||||
|
||||
def passes_discovery_floor(
|
||||
*,
|
||||
source_count: int,
|
||||
engagement_total: float,
|
||||
item_count: int,
|
||||
junk_shape: bool = False,
|
||||
seed_source_count: int | None = None,
|
||||
) -> bool:
|
||||
"""Whether a discovery topic's evidence is strong enough to show a user.
|
||||
|
||||
Below this floor the honest output is "nothing solid this window", not a
|
||||
ranked list of whatever survived the sweep.
|
||||
"""
|
||||
if item_count <= 0 or engagement_total < FLOOR_MIN_ENGAGEMENT:
|
||||
return False
|
||||
if junk_shape:
|
||||
corroboration = seed_source_count if seed_source_count is not None else source_count
|
||||
return corroboration >= FLOOR_MIN_SOURCES
|
||||
if source_count >= FLOOR_MIN_SOURCES:
|
||||
return True
|
||||
return engagement_total >= FLOOR_SINGLE_SOURCE_ENGAGEMENT
|
||||
```
|
||||
|
||||
(The `junk_shape` / `seed_source_count` branch landed in PR #852 - see section 2b.) The first check is the junk gate: `FLOOR_MIN_ENGAGEMENT = 25.0` means a 1-like tweet can never rank, no matter how empty the field is. The floor is judged per topic inside `run_discover()` (`skills/last30days/scripts/lib/pipeline.py`), before the topic is appended and before `topic_limit` is consulted - sub-floor evidence never enters the ranked list at all.
|
||||
|
||||
### 2. Make the clearing criteria composite: corroboration OR a genuinely strong spike
|
||||
|
||||
A single threshold is either too strict (kills real single-source stories) or too loose (lets corroborated-but-tiny noise through). The floor uses two independent ways to clear, after the junk gate:
|
||||
|
||||
- Cross-source corroboration: appearing on `FLOOR_MIN_SOURCES = 2` or more independent feeds clears with only modest engagement. Two feeds independently surfacing the same story is signal in itself.
|
||||
- A strong single-source spike: `FLOOR_SINGLE_SOURCE_ENGAGEMENT = 200.0`. A 1,600-point single-source HN thread is a real story; a 30-upvote single-source meme is not.
|
||||
|
||||
The regression tests in `tests/test_discover_floor.py` pin both edges of this policy directly (`test_passes_discovery_floor_policy`): `floor(source_count=2, engagement_total=30, item_count=2)` clears, `floor(source_count=1, engagement_total=100, item_count=3)` does not, `floor(source_count=1, engagement_total=1600, item_count=1)` does.
|
||||
|
||||
### 2b. Count corroboration on the layer your own pipeline does not amplify
|
||||
|
||||
PR #852 added a stricter path for junk-shaped topics (help-me posts, beginner asks, musings - flagged by the stage-1 judge or the `topic_shape` heuristics): they lose the single-source engagement bypass entirely (a 226-comment "help me choose" thread is a busy support thread, not a story) and must clear `FLOOR_MIN_SOURCES` via corroboration alone.
|
||||
|
||||
The subtle half of that change is WHICH source count the corroboration check reads. The original design counted sources in the topic's enriched corpus - and the adversarial code review proved that check would never bind: the enrichment stage deliberately fans every nominated topic out to Reddit, X, YouTube, and the web, so a single-subreddit junk thread enriches into 4-6 "sources" of mentions of itself. A gate reading the post-fan-out count is checking that enrichment works, not that the topic is corroborated. The shipped gate counts distinct sources among the nomination's own seed listing items - what the river sweep actually found - which enrichment cannot inflate (`skills/last30days/scripts/lib/pipeline.py`, floor call site):
|
||||
|
||||
```python
|
||||
junk_shape=nomination.junk_shape,
|
||||
# Junk corroboration counts distinct SEED listing sources, never
|
||||
# the enriched corpus - a successful enrichment pass is
|
||||
# multi-source for almost any topic, so it would never bind.
|
||||
seed_source_count=len({item.source for item in nomination.items}),
|
||||
```
|
||||
|
||||
The two archetypes, side by side:
|
||||
|
||||
| Topic | Seed listing sources | Enriched corpus sources | Enriched-count gate (never binds) | Seed-count gate (shipped) |
|
||||
|---|---|---|---|---|
|
||||
| Single-subreddit help-me thread (junk shape) | 1 | 4-6 | passes | fails |
|
||||
| Real story swept from Reddit AND Hacker News | 2 | 4-6 | passes | passes |
|
||||
|
||||
Generalized rule: when a gate requires corroboration or independence, measure it on the signal layer your own system does not amplify - corroboration is evidence only when the corroborating signals could have failed to appear. This applies to any "N independent confirmations" threshold downstream of your own search fan-out, enrichment, crawling, or retrieval expansion. It does NOT apply when the downstream layer is genuinely independent evidence your pipeline cannot manufacture (human review verdicts, third-party confirmations) - there, the enriched layer is exactly what to count.
|
||||
|
||||
Testing note: a unit test that feeds the gate's parameters directly cannot catch a never-binds design. At least one test must drive the full production path with the amplifier running and assert the gate still fires - `test_junk_corroboration_counts_seed_sources_not_enriched_corpus` in `tests/test_discover_floor.py` mocks enrichment to return a rich multi-source corpus and asserts the single-seed-source junk topic still fails, with the unit-level matrix in `test_passes_discovery_floor_junk_params` pinning that a high enriched `source_count` cannot rescue `seed_source_count=1`.
|
||||
|
||||
### 3. Make honest emptiness a first-class outcome, and name the nearest miss
|
||||
|
||||
When zero topics survive the floor, the pipeline does not error, does not pad, and does not lower the bar. `run_discover()` sets `outcome = "ok" if topics else "nothing-solid"` on the `DiscoveryReport`, and while filtering it remembers the highest-scoring sub-floor candidate as `weak_signal` so the empty result can still say what came closest:
|
||||
|
||||
```python
|
||||
if not rerank.passes_discovery_floor(
|
||||
source_count=len(sources),
|
||||
engagement_total=native_total,
|
||||
item_count=len(evidence_items),
|
||||
junk_shape=nomination.junk_shape,
|
||||
# Junk corroboration counts distinct SEED listing sources, never
|
||||
# the enriched corpus - a successful enrichment pass is
|
||||
# multi-source for almost any topic, so it would never bind.
|
||||
seed_source_count=len({item.source for item in nomination.items}),
|
||||
):
|
||||
# Sub-floor evidence never ranks; remember what came closest so a
|
||||
# nothing-solid brief can still name the strongest weak signal.
|
||||
# Junk-shaped failures are tracked separately: the brief prefers
|
||||
# the strongest NON-junk failure and names a junk one only when
|
||||
# every failure is junk-shaped (never empty when failures exist).
|
||||
if nomination.junk_shape:
|
||||
if junk_weak_signal is None or score > junk_weak_signal[0]:
|
||||
junk_weak_signal = (score, nomination.name)
|
||||
elif weak_signal is None or score > weak_signal[0]:
|
||||
weak_signal = (score, nomination.name)
|
||||
continue
|
||||
```
|
||||
|
||||
The renderer (`render_discovery()` in `skills/last30days/scripts/lib/render.py`) presents this as a deliberate answer, not a failure:
|
||||
|
||||
```python
|
||||
if report.outcome == "nothing-solid":
|
||||
lines.extend([
|
||||
"**Nothing solid this window.** No topic cleared the confidence "
|
||||
"floor - not enough cross-source confirmation or engagement to "
|
||||
"call anything a trend, and ranked noise would be worse than an "
|
||||
"honest empty result.",
|
||||
"",
|
||||
])
|
||||
if report.weak_signal:
|
||||
lines.extend([
|
||||
f"Closest weak signal: {report.weak_signal} (sub-floor; "
|
||||
"single-source or too little engagement).",
|
||||
"",
|
||||
])
|
||||
```
|
||||
|
||||
Naming the weak signal matters: it tells the user the sweep actually ran and looked at real data, and it gives them a thread to pull ("closest weak signal: X" often suggests the narrower query that would work). There is also a soft middle state - if some topics clear but fewer than five, `run_discover()` emits a warning ("Fewer than five topic clusters cleared the confidence floor this window") rather than padding the list to the minimum.
|
||||
|
||||
### 4. Pin the failing corpus as a regression test
|
||||
|
||||
The exact junk corpus that produced the bad output is frozen in `test_junk_corpus_returns_nothing_solid_not_ranked_noise` in `tests/test_discover_floor.py`: five single-source 1-like tweets on the "sports" domain, asserting `report.topics == []`, `report.outcome == "nothing-solid"`, a non-None `weak_signal`, and a "confidence floor" warning. Sibling tests pin the other side so the floor cannot silently become a wall: a 1,084-point HN thread ranks (`test_strong_single_source_spike_clears_floor`), a mixed corpus keeps the strong story and drops the junk (`test_mixed_corpus_emits_only_floor_clearing_topics`), and enriched topics are judged on their enriched evidence, not their thin seed (`test_enriched_evidence_is_judged_not_seed_evidence`).
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Trust in a ranked surface is asymmetric. Users cannot see the corpus behind the ranking; they can only judge the output. One junk trend list - five 1-like tweets dressed up with ranks, velocity scores, and momentum labels - teaches a user that the feature is garbage, and they generalize instantly: if it confidently ranked noise once, every future list is suspect. The presentation makes it worse, because ranking machinery (rank numbers, scores, "why spiking" prose) signals confidence the evidence never had.
|
||||
|
||||
Honest emptiness does the opposite. "Nothing solid this window" plus a named weak signal tells the user three things at once: the sweep ran, the bar is real, and here is roughly where the signal petered out. That preserves trust in every future non-empty list (topics that do appear are known to have cleared an absolute bar - the renderer even badges cross-source topics with "confirmed across N sources") and it invites a narrower, more productive follow-up query instead of a shrug. The empty state is a feature, not an apology.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Any top-N surface over variable-quality inputs, where the input pool can be thin, noisy, or empty and the ranker will still dutifully order whatever it gets:
|
||||
|
||||
- Search and retrieval results ("no good matches" beats ten irrelevant hits)
|
||||
- Trending / discovery feeds (this case)
|
||||
- Recommendation lists ("nothing new worth recommending" beats recycled filler)
|
||||
- Leaderboards and "top contributors" style rankings over sparse activity
|
||||
- LLM-generated shortlists, digests, and "best of" summaries, where the model will fill N slots on request regardless of evidence quality
|
||||
|
||||
The tell that you need this pattern: the code computes `top N by score` with no branch that can produce an empty result from a non-empty input. If the only way to get an empty list is an empty corpus, the ranker cannot say "nothing here is good enough" - and someday the corpus will be five 1-like tweets.
|
||||
|
||||
Design notes when applying it:
|
||||
|
||||
- The floor must be absolute (engagement counts, source counts, item counts), not relative (percentile of the current pool). A relative floor degrades with the pool, which is exactly the failure being prevented.
|
||||
- Prefer composite clearing criteria over a single threshold: independent corroboration OR a strong single-signal spike. Tune the constants to the domain and keep them named and commented as deliberately tunable (see the comment block above the constants in `skills/last30days/scripts/lib/rerank.py`).
|
||||
- The empty state must name the nearest miss. A bare "no results" reads as breakage; "nothing cleared the bar, closest was X" reads as judgment.
|
||||
|
||||
## Examples
|
||||
|
||||
Before (v3.13.x behavior, reconstructed from the pinned regression corpus): `--discover "sports"` on a quiet window returned a ranked list built from this corpus -
|
||||
|
||||
```
|
||||
x: "Wii Sports nostalgia thread about sports" 1 like, single source
|
||||
x: "kids travel sports burnout post" 1 like, single source
|
||||
x: "motorsports vs stick and ball sports" 1 like, single source
|
||||
x: "midjourney skateboarder sports prompt" 1 like, single source
|
||||
x: "manga review mentioning sports matches" 1 like, single source
|
||||
```
|
||||
|
||||
- rendered as topics 1-5 with velocity scores, because `topic_limit = max(5, min(10, limit))` took the top N unconditionally.
|
||||
|
||||
After (v3.14.0, PR #816): the same corpus produces `outcome="nothing-solid"`, an empty `topics` list, and the renderer's explicit empty state ("**Nothing solid this window.** No topic cleared the confidence floor ... Closest weak signal: ... (sub-floor; single-source or too little engagement)."). Verified live in the implementing session: `--discover "sports"` returned nothing-solid, while global trending (no domain) returned six real cross-source topics with community quotes - the floor removed the junk without starving the healthy path.
|
||||
|
||||
The strong-corpus side, from `tests/test_discover_floor.py`: a single 1,084-point, 577-comment HN thread clears the floor alone via the single-source-spike branch (`engagement_total >= FLOOR_SINGLE_SOURCE_ENGAGEMENT`) and ranks as a real topic; a 25-upvote single-source Reddit post stays buried. The decision logic, in full, is small enough to quote:
|
||||
|
||||
```python
|
||||
if item_count <= 0 or engagement_total < FLOOR_MIN_ENGAGEMENT:
|
||||
return False
|
||||
if junk_shape:
|
||||
corroboration = seed_source_count if seed_source_count is not None else source_count
|
||||
return corroboration >= FLOOR_MIN_SOURCES
|
||||
if source_count >= FLOOR_MIN_SOURCES:
|
||||
return True
|
||||
return engagement_total >= FLOOR_SINGLE_SOURCE_ENGAGEMENT
|
||||
```
|
||||
|
||||
A handful of lines of gate, placed before the ranker, are the difference between a feature that fills five slots no matter what and one whose non-empty answers can be believed.
|
||||
|
||||
## Related
|
||||
|
||||
- [Entity grounding: full-phrase false demotion](../logic-errors/entity-grounding-full-phrase-false-demotion.md) - sibling ranking-quality fix in the same rerank module, opposite failure direction (false demotion of good signal vs. junk promotion). Together they bracket the two ways a ranker fails.
|
||||
- [Search-quality eval: manual by default](../architecture/search-quality-eval-manual-by-default-2026-05-10.md) - how to validate a ranking-threshold change like this floor: manual eval run plus deterministic regression tests, not CI-gated quality scoring.
|
||||
- [Non-daemon executor threads defeat wall-clock budgets](../logic-errors/non-daemon-executor-threads-defeat-wall-clock-budget.md) - sibling learning from the same PR #816 rebuild: the process-lifetime half (enrichment budget enforcement) vs this doc's ranking-quality half.
|
||||
- [argparse optional-value flag dispatch](../conventions/argparse-optional-value-flag-dispatch-truthiness.md) - third lesson from the same PR #816: the CLI flag semantics that route into this feature.
|
||||
- [PR #816](https://github.com/mvanhorn/last30days-skill/pull/816) - the discovery rebuild that introduced `passes_discovery_floor()` and the nothing-solid empty state (released v3.14.0).
|
||||
- [PR #852](https://github.com/mvanhorn/last30days-skill/pull/852) - the discovery content pipeline that added the junk-shape branch and seed-source corroboration (section 2b).
|
||||
@@ -0,0 +1,80 @@
|
||||
---
|
||||
title: Digg NUX must match printing-press-library install paths and agent subprocess PATH
|
||||
date: 2026-06-17
|
||||
category: docs/solutions/integration-issues
|
||||
module: lib/setup_wizard
|
||||
problem_type: integration_issue
|
||||
component: development_workflow
|
||||
severity: medium
|
||||
symptoms:
|
||||
- Digg source silently off after first-run setup reports success on Hermes or OpenClaw
|
||||
- Users who already installed pp-digg via printing-press-library still see Digg missing from --diagnose available_sources
|
||||
- Setup wizard probed ~/go/bin while the catalog installer writes to ~/.local/bin (printing-press-library 0.1.16+)
|
||||
- OpenClaw setup --openclaw path skipped Digg install entirely
|
||||
root_cause: config_error
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- lib/pipeline
|
||||
- lib/digg
|
||||
- CONFIGURATION.md
|
||||
tags:
|
||||
- digg
|
||||
- setup-wizard
|
||||
- printing-press-library
|
||||
- agent-path
|
||||
- hermes
|
||||
- openclaw
|
||||
- nux
|
||||
- optional-cli-sources
|
||||
---
|
||||
|
||||
# Digg NUX must match printing-press-library install paths and agent subprocess PATH
|
||||
|
||||
## Problem
|
||||
|
||||
First-run setup auto-install for `digg-pp-cli` could report success while the engine still omitted Digg, especially on Hermes and OpenClaw where the agent subprocess PATH often excludes `$HOME/.local/bin`. The initial PR also used the deprecated `@mvanhorn/printing-press` package and probed legacy `~/go/bin` fallbacks instead of the current Printing Press default install dir.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- `--diagnose` `available_sources` lacks `digg` even though pp-digg or setup "installed" the CLI.
|
||||
- Hermes/OpenClaw users with a prior `npx @mvanhorn/printing-press-library install digg --cli-only` run hit false failures or false "now active" messages depending on probe logic.
|
||||
- OpenClaw `setup --openclaw` never attempted Digg install (desktop NUX only).
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
- **Treating "binary exists somewhere" as installed** — `pipeline.available_sources()` and `digg._is_available()` gate on `shutil.which("digg-pp-cli")` only. Probing `~/go/bin` without PATH visibility produced false positives.
|
||||
- **Assuming Hermes vs OpenClaw use different binary locations** — both harnesses use the same printing-press-library default (`$HOME/.local/bin`); only the focused pp-digg *skill* wiring differs.
|
||||
- **Using `@mvanhorn/printing-press`** — superseded by `@mvanhorn/printing-press-library`; install defaults moved from `$GOPATH/bin` to `$HOME/.local/bin` in npm 0.1.16.
|
||||
|
||||
## Solution
|
||||
|
||||
Align setup wizard with the catalog installer and the engine PATH gate:
|
||||
|
||||
1. **Pin installer:** `npx -y @mvanhorn/printing-press-library@0.1.16 install digg --cli-only` (`--cli-only` only — last30days embeds Digg as an engine source, not pp-digg skill).
|
||||
2. **Split outcomes:** `already_installed` / `installed` only when `shutil.which` resolves; `installed_off_path` when the binary exists under known dirs (`~/.local/bin`, legacy `~/go/bin`, Windows PrintingPress bin) but is not PATH-visible; surface `digg_path` and PATH-restart guidance in status text.
|
||||
3. **OpenClaw parity:** `run_openclaw_setup()` runs the same `_install_digg_cli()` and returns `digg_cli`, `digg_action`, optional `digg_path`.
|
||||
4. **Docs:** CONFIGURATION.md, SKILL.md Step 0, HERMES_SETUP.md, AGENTS.md rule for CLI-gated sources.
|
||||
|
||||
Key helper shape in `setup_wizard.py`:
|
||||
|
||||
```python
|
||||
def _digg_on_path() -> Optional[str]:
|
||||
return shutil.which(DIGG_CLI_BIN) # engine gate
|
||||
|
||||
def _digg_off_path_binary() -> Optional[str]:
|
||||
for candidate in _digg_bin_candidate_paths(): # ~/.local/bin first
|
||||
if candidate.is_file() and os.access(candidate, os.X_OK):
|
||||
return str(candidate)
|
||||
return None
|
||||
```
|
||||
|
||||
## Why This Works
|
||||
|
||||
The engine never reads "is pp-digg skill installed?" — every research run shells out to `digg-pp-cli` by name on PATH. Printing Press already installs to a managed user bin dir and warns when that dir is off PATH; last30days setup must mirror that contract instead of inventing a separate success definition. Detecting off-PATH binaries lets setup reuse prior pp-digg installs without lying about activation.
|
||||
|
||||
## Prevention
|
||||
|
||||
- When adding NUX auto-install for a CLI-gated source, match the upstream installer's default bin dir and pin the npm semver.
|
||||
- Success messaging must use the same probe as `available_sources()` (`shutil.which`), with a separate off-PATH outcome when the binary exists on disk.
|
||||
- Cover Hermes/OpenClaw in tests with redirected `HOME` and mocked PATH; add OpenClaw JSON fields when server setup should mirror desktop NUX.
|
||||
- Search `docs/solutions/` for `digg`, `setup-wizard`, and `agent-path` before changing optional-source onboarding.
|
||||
@@ -0,0 +1,117 @@
|
||||
---
|
||||
title: Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
|
||||
date: 2026-06-09
|
||||
category: docs/solutions/logic-errors
|
||||
module: lib/rerank
|
||||
problem_type: logic_error
|
||||
component: search_ranking
|
||||
severity: high
|
||||
symptoms:
|
||||
- on-entity, high-engagement items that name the brand but omit the trailing descriptor of a multi-word query are demoted in keyless/fallback rerank results
|
||||
- observed case is a 323-point HN thread about Stripe scoring 0 on a "Stripe payments" query
|
||||
- the entity-miss demotion lands twice (ENTITY_MISS_PENALTY on rerank_score plus a secondary final_score penalty), so a false miss guarantees burial regardless of engagement
|
||||
- reddit keyless comment-enrichment slot selection skips the same on-entity threads via an independently duplicated full-phrase check in _slot_priority
|
||||
root_cause: logic_error
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- reddit_keyless
|
||||
- comment_enrichment
|
||||
tags:
|
||||
- entity-grounding
|
||||
- rerank
|
||||
- keyless-fallback
|
||||
- multi-word-entity
|
||||
- substring-match
|
||||
- false-demotion
|
||||
- reddit-keyless
|
||||
- duplicated-logic
|
||||
---
|
||||
|
||||
# Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
|
||||
|
||||
## Problem
|
||||
|
||||
The keyless/fallback rerank path's entity-grounding demotion required the FULL multi-word primary-entity phrase as a contiguous substring of the candidate's text (`primary_entity.lower() not in haystack`), so on-entity items that omitted a trailing search descriptor were falsely flagged as entity misses and buried by a deliberately decisive double penalty.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- On a "Stripe payments" query, a 323-point HN thread titled "Stripe is friendly to 'friendly fraud'" was demoted to score 0 — purely because its text never contained the literal phrase "stripe payments" (the trailing word "payments" was missing).
|
||||
- The burial is guaranteed by design, not incidental: a flagged entity miss takes −25 `ENTITY_MISS_PENALTY` on `rerank_score` in `_fallback_tuple`, PLUS `ENTITY_MISS_FINAL_PENALTY` applied directly in `_final_score` (added 2026-04-19 after engagement + freshness drowned the diluted penalty). A false positive on the check means confirmed-good signal cannot recover.
|
||||
- The same over-strict check had been independently re-implemented in `reddit_keyless._slot_priority` (keyless Reddit comment-enrichment slot selection), so scarce comment slots were also steered away from head-token-only posts.
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
- **Naively relaxing the check** — the full-phrase check existed for a real reason: on 2026-04-19 an off-topic video with zero brand mentions ranked #2 on a Hermes query (documented in the `ENTITY_MISS_FINAL_PENALTY` comment in `skills/last30days/scripts/lib/rerank.py`). Any fix had to keep that demotion firing.
|
||||
- **Word-boundary matching** — rejected; it re-introduces over-demotion on plurals/possessives/compounds ("stripes", "Stripe's").
|
||||
- **Graded penalty** (full-phrase = 0, head-only = half, none = full) — rejected; it half-punishes items that are 100% about the entity. Lexical coverage is not topical degree.
|
||||
- **Any-token grounding** — rejected; "payments" alone would ground completely generic posts.
|
||||
- **Distinctiveness gate for generic heads** — rejected as complexity to patch a failure mode that is already a safe no-op (see Why This Works).
|
||||
- **Trusting the docstring** — `reddit_keyless._slot_priority`'s docstring claimed to "mirror rerank's demotion signal," but its inline reimplementation (`entity in _post_text(post).lower()`) had silently drifted from being a mirror into being a second copy of the bug. It was found only by a code-reuse review, not by tests.
|
||||
|
||||
## Solution
|
||||
|
||||
Ground on the **head token** of the primary entity instead of the full phrase, via one shared helper used by both paths.
|
||||
|
||||
**Site 1 — new helper in `skills/last30days/scripts/lib/rerank.py`:**
|
||||
|
||||
```python
|
||||
def _entity_grounded(haystack: str, primary_entity: str) -> bool:
|
||||
tokens = primary_entity.lower().split()
|
||||
if not tokens:
|
||||
return True
|
||||
return tokens[0] in haystack
|
||||
```
|
||||
|
||||
`_fallback_tuple` switches from the inline phrase check to the helper:
|
||||
|
||||
```python
|
||||
# before
|
||||
if haystack.strip() and primary_entity.lower() not in haystack:
|
||||
# after
|
||||
if haystack.strip() and not _entity_grounded(haystack, primary_entity):
|
||||
```
|
||||
|
||||
**Site 2 — secondary penalty in `_final_score`: no code change needed.** It keys off the explanation string set by site 1, so it inherits the fix automatically:
|
||||
|
||||
```python
|
||||
if candidate.explanation and "entity-miss" in candidate.explanation:
|
||||
base = max(0.0, base - ENTITY_MISS_FINAL_PENALTY)
|
||||
```
|
||||
|
||||
**Site 3 — `skills/last30days/scripts/lib/reddit_keyless.py` `_slot_priority`:** replace the drifted reimplementation with a call to the shared helper:
|
||||
|
||||
```python
|
||||
# before
|
||||
return entity in _post_text(post).lower()
|
||||
# after
|
||||
return rerank._entity_grounded(_post_text(post).lower(), entity)
|
||||
```
|
||||
|
||||
Tests: `tests/test_rerank_v3.py` gained `test_fallback_grounds_on_head_token_not_full_phrase` (the Stripe regression) and `test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` (guards the 2026-04-19 behavior). `tests/test_reddit_keyless.py`'s two old-contract tests were rewritten as `test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match`.
|
||||
|
||||
## Why This Works
|
||||
|
||||
- **Root cause:** trailing tokens of a multi-word query ("payments" in "Stripe payments") are usually category descriptors the user/planner appended for search, not part of the entity name. Requiring the whole phrase conflates "doesn't repeat my search phrasing" with "isn't about my entity." The brand head token alone is sufficient grounding; items that never name the brand at all still miss the head token and stay demoted — so the original 2026-04-19 fix keeps firing.
|
||||
- **Asymmetry argument:** the demotion is engineered to be decisive (double penalty across `rerank_score` and `final_score`), so a false entity-miss is fatal-by-design, while a false grounding merely defers the item to normal relevance/freshness/quality ranking. When the punishment is capital, the conviction standard should be conservative.
|
||||
- **Substring (not word-boundary) is deliberate:** it catches plurals/possessives/compounds ("stripes", "Stripe's"). Degenerate short heads ("X", "Go", "C") make the check vacuously true, which merely **disables** the penalty — reverting to the pre-grounding baseline — rather than burying good items. Every failure mode of this rule degrades toward "no penalty," never toward "bury good signal."
|
||||
- **Accepted, bounded limitation:** head-collision with a different famous entity ("Hermes Agent" → a "Hermes Birkin" thread now escapes demotion). This is lexically unfixable — any token rule strong enough to kill the collision re-kills the Stripe case; the discriminator is semantic. The LLM rerank path (which receives the full phrase as prompt guidance and judges semantically) covers this when API keys exist; the keyless path accepts the bounded risk.
|
||||
|
||||
## Prevention
|
||||
|
||||
- **Shared helper as single source of truth:** when one module's behavior must "mirror" another's signal, it must *call* the same function, not re-implement the check. The `reddit_keyless._slot_priority` drift happened precisely because the mirror was a copy. The fix wires it to `rerank._entity_grounded`, and the docstring now states this explicitly: "keying on the same head token keeps the two paths from diverging."
|
||||
- **Docstrings record deliberate trade-offs:** `_entity_grounded`'s docstring documents WHY head-token (not phrase), why substring (not word-boundary), and the safe-failure direction. Future readers see the rejected alternatives were considered, not overlooked — and won't "tighten" the check into a regression.
|
||||
- **Both directions pinned by named tests:**
|
||||
- `tests/test_rerank_v3.py::test_fallback_grounds_on_head_token_not_full_phrase` — false-demotion regression (the Stripe HN thread must not be flagged).
|
||||
- `tests/test_rerank_v3.py::test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` — the fix must not neuter the demotion (guards the 2026-04-19 off-topic-video incident).
|
||||
- `tests/test_reddit_keyless.py::test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match` — the mirrored path asserts the same contract.
|
||||
- **Audit tests when changing a contract:** tests that encode the old behavior as correct must be rewritten to the new contract, not worked around — the two old `test_reddit_keyless.py` tests would have silently re-blessed the bug.
|
||||
- **For decisive penalties, route through one flag:** the `_final_score` backstop keys off `"entity-miss" in candidate.explanation` rather than re-running the check — so there was exactly one site to fix and the second penalty inherited it for free. Prefer this signal-propagation pattern over duplicating predicate logic at each penalty site.
|
||||
|
||||
## Related Issues
|
||||
|
||||
- [PR #484](https://github.com/mvanhorn/last30days-skill/pull/484) — "fix(reddit): relevance-aware comment-enrichment slot selection in keyless path" — introduced the `_slot_priority` mirror this fix reroutes through the shared helper.
|
||||
- [PR #457](https://github.com/mvanhorn/last30days-skill/pull/457) — "fix(reddit): restore free path via keyless RSS + shreddit scrape" — established the keyless Reddit path.
|
||||
- [PR #488](https://github.com/mvanhorn/last30days-skill/pull/488) (open) — "fix(reddit): relevance floor + relevance-first ranking" — external PR touching the same ranking surface; coordinate before merging both.
|
||||
- [Issue #468](https://github.com/mvanhorn/last30days-skill/issues/468) (open) — relevance scoring over-pruning on-topic YouTube items; same symptom family in a different source.
|
||||
- [../architecture/search-quality-eval-manual-by-default-2026-05-10.md](../architecture/search-quality-eval-manual-by-default-2026-05-10.md) — how to validate ranking/grounding changes like this one (manual eval, not CI-gated).
|
||||
- [../workflow-issues/release-consistency-test-cascade-2026-05-16.md](../workflow-issues/release-consistency-test-cascade-2026-05-16.md) — sibling prevention pattern: lockstep artifacts drift unless mechanically unified.
|
||||
@@ -0,0 +1,137 @@
|
||||
---
|
||||
title: "Wall-clock budget over ThreadPoolExecutor is illusory - non-daemon workers are joined at interpreter shutdown"
|
||||
category: logic-errors
|
||||
module: discover-enrichment
|
||||
date: 2026-07-12
|
||||
problem_type: logic_error
|
||||
component: background_job
|
||||
severity: high
|
||||
symptoms:
|
||||
- "CLI process stayed alive after the enrichment budget expired, even though the hung topic had already been dropped"
|
||||
- "as_completed(futures, timeout=budget) returned control on time, but one still-running sub-run kept the interpreter from exiting"
|
||||
- "shutdown(wait=False, cancel_futures=True) cancelled unstarted futures but could not stop an already-running worker thread"
|
||||
root_cause: thread_violation
|
||||
resolution_type: code_fix
|
||||
tags:
|
||||
- threadpoolexecutor
|
||||
- daemon-threads
|
||||
- wall-clock-budget
|
||||
- interpreter-shutdown
|
||||
- semaphore
|
||||
- concurrency
|
||||
- timeout
|
||||
- python
|
||||
related_components:
|
||||
- testing_framework
|
||||
---
|
||||
|
||||
# Wall-clock budget over ThreadPoolExecutor is illusory - non-daemon workers are joined at interpreter shutdown
|
||||
|
||||
## Problem
|
||||
|
||||
Discovery enrichment fans out one research sub-run per nominated topic under a wall-clock batch budget (`ENRICH_BUDGET_SECONDS`, `skills/last30days/scripts/lib/pipeline.py`), but the first implementation built the budget on `ThreadPoolExecutor` - whose worker threads are non-daemon and joined at interpreter shutdown - so the budget bounded the answer without bounding the process. One hung sub-run could keep the whole CLI alive indefinitely after its topic had already been downgraded to nomination-only.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- The CLI process stays alive past the enrichment budget - potentially indefinitely - whenever one sub-run hangs (e.g. a network fetch that stalls without tripping a timeout). The batch "finishes", results are assembled, but the process will not exit because Python joins the executor's non-daemon threads at interpreter shutdown.
|
||||
- Only visible under a genuinely hung worker. All unit tests passed: the slow-topic test observed exactly what `as_completed(timeout=...)` guarantees - the fast topic returned, the slow one was dropped from results - and the test process exited fine because the "slow" worker was merely sleeping a few seconds, not hung. The defect lives entirely in process-lifetime behavior that a result-oriented test never touches.
|
||||
- Caught by code review on PR #816 (Greptile P1 "Enrichment Budget Keeps Running") before release; shipped fixed in v3.14.0.
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
The first version of `enrich_nominations()`:
|
||||
|
||||
```python
|
||||
executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
futures = {executor.submit(_run_one, n): n for n in nominations}
|
||||
try:
|
||||
for future in as_completed(futures, timeout=budget_seconds):
|
||||
...collect result...
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
```
|
||||
|
||||
This looks like budget enforcement, and every knob in it does something real - just not the thing needed. Precisely why each one fails to stop a thread that is already running:
|
||||
|
||||
- `as_completed(futures, timeout=budget)` bounds only the consumer. When the timeout expires it raises `TimeoutError` in the collecting loop; it sends nothing to the workers. Python threads cannot be killed from outside, so a running sub-run keeps running.
|
||||
- `cancel_futures=True` cancels only futures still sitting in the executor's queue - futures whose callable has not started. A future in the RUNNING state is not cancellable (`Future.cancel()` returns False for it); the worker executing it is untouched.
|
||||
- `shutdown(wait=False)` merely skips joining the threads at shutdown-call time. It does not detach them. CPython's `ThreadPoolExecutor` creates its worker threads non-daemon and (since Python 3.9) registers a `threading._register_atexit` hook that joins every worker at interpreter shutdown. So even after `shutdown(wait=False)`, interpreter exit blocks until the hung worker returns - which for a stalled fetch may be never.
|
||||
|
||||
Net effect: the budget expired, the slow topic was correctly reported as nomination-only, and the process then sat there anyway, held open by the very thread the budget had supposedly abandoned.
|
||||
|
||||
## Solution
|
||||
|
||||
Replace the executor with plain daemon threads, a semaphore for the concurrency cap, a queue for results, and a monotonic deadline in the consumer. From `skills/last30days/scripts/lib/pipeline.py`, `enrich_nominations()`:
|
||||
|
||||
```python
|
||||
# Daemon threads + a semaphore instead of ThreadPoolExecutor: executor
|
||||
# threads are non-daemon and joined at interpreter shutdown, so one hung
|
||||
# sub-run could keep the whole process alive long after its topic was
|
||||
# downgraded to nomination-only. Daemon workers make the wall-clock budget
|
||||
# real - stragglers cannot delay process exit. Abandonment is safe because
|
||||
# internal_subrun passes write nothing to disk (no save, no library sync,
|
||||
# no store), and every fetch layer inside run() carries its own timeout.
|
||||
enriched: dict[str, EnrichedTopic] = {}
|
||||
results_queue: queue.Queue[tuple[Nomination, schema.Report | None, Exception | None]] = queue.Queue()
|
||||
slots = threading.Semaphore(max(1, max_workers))
|
||||
|
||||
def _worker(nomination: Nomination) -> None:
|
||||
with slots:
|
||||
try:
|
||||
results_queue.put((nomination, _run_one(nomination), None))
|
||||
except Exception as exc: # noqa: BLE001 - containment is the contract
|
||||
results_queue.put((nomination, None, exc))
|
||||
|
||||
for nomination in nominations:
|
||||
threading.Thread(
|
||||
target=_worker,
|
||||
args=(nomination,),
|
||||
name=f"discover-enrich-{nomination.name[:32]}",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
deadline = time.monotonic() + max(1.0, budget_seconds)
|
||||
pending = len(nominations)
|
||||
while pending and (remaining := deadline - time.monotonic()) > 0:
|
||||
try:
|
||||
nomination, report, exc = results_queue.get(timeout=min(remaining, 0.5))
|
||||
except queue.Empty:
|
||||
continue
|
||||
pending -= 1
|
||||
...record EnrichedTopic success or error...
|
||||
# Budget expired (or all done): unfinished topics fall through below as
|
||||
# nomination-only; their daemon workers are abandoned and cannot block exit.
|
||||
```
|
||||
|
||||
Topics still unfinished when the loop exits fall through with `error="enrichment budget exhausted"` and survive as nomination-only entries; the batch preserves nomination order and never raises. Defaults live beside the function: `ENRICH_MAX_WORKERS = 3`, `ENRICH_BUDGET_SECONDS = 240.0`.
|
||||
|
||||
Three tests in `tests/test_discover_enrich.py` pin the contract:
|
||||
|
||||
- `test_enrich_workers_are_daemon_threads` asserts `threading.current_thread().daemon` from inside every worker - the daemon property is tested directly, not inferred from process behavior.
|
||||
- `test_enrich_concurrency_capped_by_semaphore` runs 6 nominations with `max_workers=2` and asserts peak in-flight workers never exceeds 2.
|
||||
- `test_enrich_budget_expiry_drops_slow_topic_to_nomination_only` runs a fast and a 5-second topic under `budget_seconds=1.0` and asserts the fast one returns enriched while the slow one drops to nomination-only with a budget error.
|
||||
|
||||
## Why This Works
|
||||
|
||||
- Daemon threads are not joined at interpreter exit. CPython's shutdown sequence waits only for non-daemon threads; a daemon worker mid-fetch simply dies with the process. That is what makes the wall-clock budget real: expiry means the process can exit now, not "after the straggler finishes".
|
||||
- The monotonic deadline bounds the consumer independently of worker behavior. `results_queue.get(timeout=min(remaining, 0.5))` wakes at least twice a second to re-check the deadline, so the collecting loop exits within ~0.5s of budget expiry no matter what any worker is doing.
|
||||
- The semaphore preserves the executor's one useful property. `threading.Semaphore(max(1, max_workers))` acquired inside each worker caps in-flight sub-runs at `max_workers`, so upstream APIs see the same low parallelism as before; threads beyond the cap exist but block on the semaphore, costing almost nothing.
|
||||
- The write-free precondition is what makes abandonment safe, and it is documented in the code comment where the next editor will see it: enrichment sub-runs are `internal_subrun=True` passes that write nothing to disk - no save, no library sync, no store - and every fetch layer inside `run()` carries its own timeout. Killing such a worker at process exit can corrupt nothing. A worker that mutates shared state (files, databases, caches) must not be abandoned this way; it needs cooperative cancellation instead.
|
||||
|
||||
The general lesson: `as_completed(timeout=...)` plus `shutdown(wait=False, cancel_futures=True)` is answer-bounding, not process-bounding. If the requirement is "this batch may not extend the life of the process", non-daemon executor threads cannot deliver it (CPython behavior since 3.9, when executor threads moved from atexit-daemon handling to `threading._register_atexit` joining), and no combination of executor knobs changes that.
|
||||
|
||||
## Prevention
|
||||
|
||||
- Any "budget" or "timeout" over threaded work must state what happens to a RUNNING straggler. If the design doc or comment only says what happens to the result, the process-lifetime question is unanswered - and the default answer (non-daemon threads joined at exit) is usually wrong for a CLI.
|
||||
- Prefer explicit daemon threads for abandonable work. When stragglers are safe to drop, `threading.Thread(daemon=True)` + semaphore + queue + monotonic deadline is barely more code than an executor and actually enforces the budget. Reserve `ThreadPoolExecutor` for work you intend to wait for.
|
||||
- Require the write-free precondition in a comment next to the daemon flag. Daemon abandonment is only safe for workers that mutate no shared state and hold no resources needing cleanup. State the precondition where the code is (as `enrich_nominations()` does), so a future change that adds a disk write inside the worker trips over the warning.
|
||||
- Test daemon-ness explicitly. Process-hang bugs are invisible to result-oriented unit tests - the passing slow-topic test proved the wrong thing. Assert `threading.current_thread().daemon` inside the worker (see `test_enrich_workers_are_daemon_threads` in `tests/test_discover_enrich.py`); it is a one-line assertion that pins the property the budget depends on.
|
||||
- Per-request timeouts inside workers remain the first line of defense. Daemon abandonment is the backstop for the pathological case; every network call inside a worker should still carry its own timeout so hung workers are rare, not routine.
|
||||
- In review, treat `shutdown(wait=False, cancel_futures=True)` in a `finally` as a signal to ask the straggler question. It is the idiom people reach for when they want abandonment, and it does not provide it.
|
||||
|
||||
## Related Issues
|
||||
|
||||
- [Ranked-output confidence floor + honest empty state](../design-patterns/ranked-output-confidence-floor-honest-empty-state.md) - sibling learning from the same PR #816 discovery rebuild: the ranking-quality half vs this doc's process-lifetime half. Both live in `skills/last30days/scripts/lib/pipeline.py`.
|
||||
- [argparse optional-value flag dispatch](../conventions/argparse-optional-value-flag-dispatch-truthiness.md) - third lesson from the same PR #816: bare-flag vs flag-absent conflation, another defect class invisible to result-oriented tests.
|
||||
- [PR #816](https://github.com/mvanhorn/last30days-skill/pull/816) - the discovery rebuild that replaced the executor with daemon threads + semaphore + result queue + monotonic deadline in `enrich_nominations()` (released v3.14.0).
|
||||
- Note: `skills/last30days/scripts/lib/pipeline.py` still uses `ThreadPoolExecutor` at other call sites where work is genuinely waited for; the daemon-thread pattern was applied only to `enrich_nominations()`, whose stragglers are abandonable. Apply the straggler question, not the pattern, when touching those.
|
||||
@@ -0,0 +1,219 @@
|
||||
---
|
||||
title: Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
date: 2026-05-16
|
||||
category: docs/solutions/workflow-issues
|
||||
module: ci-release-engineering
|
||||
problem_type: workflow_issue
|
||||
component: testing_framework
|
||||
severity: high
|
||||
applies_when:
|
||||
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
|
||||
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
|
||||
- multiple long-lived PRs are open simultaneously against the same base branch
|
||||
symptoms:
|
||||
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
|
||||
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
|
||||
- PR authors must rebase and manually fix an artifact they did not touch
|
||||
root_cause: missing_workflow_step
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- development_workflow
|
||||
- documentation
|
||||
tags:
|
||||
- ci
|
||||
- release-engineering
|
||||
- consistency-test
|
||||
- version-pin
|
||||
- cascade-failure
|
||||
- test-design
|
||||
- workflow
|
||||
---
|
||||
|
||||
# Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
|
||||
## Context
|
||||
|
||||
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
|
||||
|
||||
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
|
||||
|
||||
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
|
||||
2. The release PR merges to `main`.
|
||||
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
|
||||
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
|
||||
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
|
||||
|
||||
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
|
||||
|
||||
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
|
||||
|
||||
This pattern looks safe but is not:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # reads SKILL.md
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # asserts sync.sh contains that string
|
||||
)
|
||||
```
|
||||
|
||||
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
|
||||
|
||||
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
|
||||
|
||||
Remove the hardcoded pin from `sync.sh` and compute it:
|
||||
|
||||
```bash
|
||||
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
|
||||
```
|
||||
|
||||
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
|
||||
|
||||
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
|
||||
|
||||
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
|
||||
if not sync_sh.exists():
|
||||
self.skipTest("sync.sh not present; skipping pin consistency check")
|
||||
sync_text = sync_sh.read_text(encoding="utf-8")
|
||||
version = _skill_version()
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text,
|
||||
)
|
||||
```
|
||||
|
||||
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
|
||||
|
||||
### 4. Run consistency tests against the merge-base diff, not main
|
||||
|
||||
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
|
||||
|
||||
```yaml
|
||||
- name: Check sync.sh version pin consistency
|
||||
run: |
|
||||
BASE=$(git merge-base HEAD origin/main)
|
||||
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
|
||||
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
|
||||
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
|
||||
echo "SKILL.md version bumped but sync.sh pin was not updated"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
|
||||
|
||||
### 5. Ask whether you actually need this test
|
||||
|
||||
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The damage from a stale-pin consistency test is asymmetric. It:
|
||||
|
||||
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
|
||||
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
|
||||
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
|
||||
- Blocks work that has already been reviewed and approved.
|
||||
|
||||
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
|
||||
|
||||
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Apply this guidance whenever you find yourself:
|
||||
|
||||
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
|
||||
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
|
||||
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
|
||||
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
|
||||
|
||||
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
|
||||
|
||||
## Examples
|
||||
|
||||
### Before — the pattern that caused the cascade
|
||||
|
||||
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
|
||||
|
||||
```python
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
||||
if not match:
|
||||
raise AssertionError("SKILL.md version frontmatter not found")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
class TestVersionConsistency(unittest.TestCase):
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # source 1: SKILL.md frontmatter
|
||||
self.assertIn( # assertion: sync.sh must contain
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # source 2: hardcoded string in sync.sh
|
||||
)
|
||||
```
|
||||
|
||||
`sync.sh` contained a line like:
|
||||
|
||||
```bash
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
|
||||
```
|
||||
|
||||
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
|
||||
|
||||
### After — what we did: delete both
|
||||
|
||||
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
|
||||
|
||||
### After — what we could have done instead: derive at runtime
|
||||
|
||||
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
|
||||
if [ -z "$SKILL_VERSION" ]; then
|
||||
echo "error: could not parse version from SKILL.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
|
||||
# ... rest of sync logic
|
||||
```
|
||||
|
||||
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
|
||||
|
||||
## Related
|
||||
|
||||
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
|
||||
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
|
||||
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
|
||||
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
|
||||
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
|
||||
@@ -1,388 +0,0 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -1,310 +0,0 @@
|
||||
# V1 vs V2 Comparison Analysis
|
||||
|
||||
**Date:** 2026-02-06
|
||||
**Queries tested:** 4 (1 head-to-head, 3 V1-only)
|
||||
**Scope:** Quick smoke test, not full 17-query matrix
|
||||
|
||||
---
|
||||
|
||||
## Part 1: Head-to-Head -- "kanye west" (NEWS Query)
|
||||
|
||||
### Dimension-by-Dimension Scoring
|
||||
|
||||
#### 1. Query Parsing Display
|
||||
|
||||
Does it show the `🔍 **{TOPIC}** · {QUERY_TYPE}` line before running tools?
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 1 | No parsing display at all. Output starts with "## What I learned:" -- jumps straight into synthesis. No acknowledgment of topic or query type before research. |
|
||||
| V2 | 1 | No parsing display either. Output starts with "Here's what I found:" then "## What I learned:" -- same problem as V1. |
|
||||
|
||||
**Analysis:** Neither version actually rendered the query parsing display. V2 SKILL.md explicitly requires `🔍 **kanye west** · News` before any tools run, but the agent did not produce it. This is a V2 instruction that failed to land. Both score 1/5.
|
||||
|
||||
Possible cause: The parsing display is supposed to appear *before* tools are called -- it may have been shown during execution but not captured in the final output text. If so, both outputs represent only the post-research synthesis, not the full session. Regardless, based on what is in the output files, neither shows it.
|
||||
|
||||
---
|
||||
|
||||
#### 2. Source Coverage (Reddit/X/Web counts)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | `Reddit: 0 relevant threads` / `X: 30 posts │ ~10 likes` / `Web: 20+ pages`. Two of three sources returned results. Reddit was zero. |
|
||||
| V2 | 3 | `Reddit: 0 threads (no results this cycle)` / `X: 29 posts │ 33 likes │ 14 reposts` / `Web: 30+ pages`. Same pattern: two of three returned results. |
|
||||
|
||||
**Analysis:** Nearly identical coverage. Both got zero Reddit results (likely a script/API issue for this topic, not a SKILL.md problem). V2 has slightly more precise X metrics (33 likes, 14 reposts vs. V1's vague "~10 likes"). V2 has more web pages (30+ vs 20+). Both miss the 10+ Reddit threshold for a score of 4+.
|
||||
|
||||
---
|
||||
|
||||
#### 3. Citation Quality (sparse vs every-sentence)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 2 | No inline citations at all. The body text makes claims ("full-page Wall Street Journal apology," "Hellwatt Festival in Italy") but never attributes them to a specific source. The stats box lists "Washington Post, Billboard, AllHipHop" but the body has zero `per @handle` or `per Rolling Stone` attributions. |
|
||||
| V2 | 5 | Every bold section ends with a sparse, clean citation. Examples: `"per Rolling Stone"`, `"per The Washington Post"`, `"per Billboard"`, `"per AllHipHop"`, `"per The News International"`. One citation per topic, never chained. Exactly what V2 SKILL.md specifies. |
|
||||
|
||||
**Analysis:** This is the single biggest quality gap between V1 and V2. V1's output reads like a Wikipedia summary -- informative but ungrounded. V2 reads like a researched briefing where every claim has a named source. V2 nails the "sparse citation" rule from its SKILL.md: `"cite 1 source per pattern, short format: 'per @handle' or 'per r/sub'"`.
|
||||
|
||||
V1 quote (no citation): `"He'll headline the new Hellwatt Festival in Italy (July 4-18, 2026)."`
|
||||
V2 quote (cited): `"Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard."`
|
||||
|
||||
---
|
||||
|
||||
#### 4. Summary Structure (bold topic headers, organized sections)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | Has a coherent narrative structure with a paragraph of synthesis, then a `**KEY THEMES:**` numbered list. But the opening is a single dense paragraph, not broken into scannable sections with bold headers. |
|
||||
| V2 | 5 | Each storyline gets its own bold header: `**BULLY Album — March 20, 2026 via Gamma**`, `**Public Apology for Antisemitism**`, `**Hellwatt Festival in Italy**`, `**Health Concerns**`, `**Grammys Ban**`, `**Kim & Lewis Hamilton Buzz**`. Each is a standalone scannable unit with 1-3 sentences. |
|
||||
|
||||
**Analysis:** V2 follows the SKILL.md template exactly: `**{Topic 1}** — [1-2 sentences, per source]`. V1 uses a blob + list approach which is readable but less scannable. V2 is notably better for a user who wants to skim and find the story they care about.
|
||||
|
||||
V1 structure: 1 dense paragraph -> 5-item `KEY THEMES` list
|
||||
V2 structure: 6 bold topic cards, each self-contained -> no KEY THEMES list (but doesn't need one because the structure itself is the organization)
|
||||
|
||||
---
|
||||
|
||||
#### 5. Stats Box Format (emoji tree vs plain text)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 4 | Uses `├─` tree format with emoji: `├─ 🟠 Reddit: 0 relevant threads` / `├─ 🔵 X: 30 posts` / `├─ 🌐 Web: 20+ pages` / `└─ Top voices:`. Minor deviation: says "0 relevant threads (filtered out noise)" instead of the V1 SKILL.md template "0 threads (no results this cycle)". Also omits the `🗣️` emoji on the Top voices line. |
|
||||
| V2 | 5 | Perfect match to V2 SKILL.md template: `├─ 🟠 Reddit: 0 threads (no results this cycle)` / `├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)` / `├─ 🌐 Web: 30+ pages │ rollingstone.com, ...` / `└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex`. Includes `(via xAI)` notation, `🗣️` emoji, @handles with engagement counts. |
|
||||
|
||||
**Analysis:** V2 is tighter and matches its template exactly. V1 is close but has minor deviations (custom "filtered out noise" text, missing `🗣️` emoji, no @handles or engagement counts on Top voices). V2's inclusion of actual @handles with like counts (`@honest30bgfan_ (33 likes)`) adds credibility.
|
||||
|
||||
---
|
||||
|
||||
#### 6. Research Grounding (actual research vs generic knowledge)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 4 | Clearly grounded: mentions specific details like "Wall Street Journal apology (Jan 26, 2026)," "four-month-long manic episode," "frontal-lobe brain injury," "North West collaborated on 'Piercings on My Hand,'" "Monumental Plaza de Toros." These are specific enough to be from research, not pre-training. Minor generic leakage: the "KEY THEMES" list uses editorial framing ("Accountability arc," "Mental health transparency") that feels more like analysis than research extraction. |
|
||||
| V2 | 5 | Every fact is specific and attributed: "12th studio album," "13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign," "earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded," "103,000-capacity RCF Arena." The AI-deepfaked vocals detail is a standout -- it is clearly from research, not something a model would know from pre-training. The Kim/Lewis Hamilton item (`"X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton"`) is explicitly sourced from X data, not general knowledge. |
|
||||
|
||||
**Analysis:** Both are well-grounded, but V2 has more "could only come from research" details. The deepfaked vocals story, the exact venue capacity, and the explicit X chatter observation are details that prove the synthesis is from the research output, not hallucinated.
|
||||
|
||||
---
|
||||
|
||||
#### 7. Prompt Quality (invitation to share vision, not dumping prompts)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | Ends with: `"Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in."` This is a follow-up invitation, but it is NOT the SKILL.md-specified invitation. It is topic-specific and conversational, which is nice, but it does not ask the user to "share your vision for what you want to create." It misses the prompt-generation angle entirely. |
|
||||
| V2 | 5 | Ends with exactly: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice."` This matches the V2 SKILL.md template verbatim. It positions the skill correctly: not a news summarizer but a research-to-prompt pipeline. |
|
||||
|
||||
**Analysis:** V1's closing is friendly but off-brand. It treats the skill as a research tool, not a research-to-prompt tool. V2 correctly frames the next step as "tell me what to create and I'll write the prompt." This is a meaningful difference -- V1 would leave a user thinking they just got a summary, while V2 primes them to get a usable output.
|
||||
|
||||
---
|
||||
|
||||
### Head-to-Head Scorecard
|
||||
|
||||
| Dimension | V1 | V2 | Winner |
|
||||
|-----------|----|----|--------|
|
||||
| 1. Query Parsing Display | 1 | 1 | Tie (both failed) |
|
||||
| 2. Source Coverage | 3 | 3 | Tie |
|
||||
| 3. Citation Quality | 2 | 5 | **V2 (+3)** |
|
||||
| 4. Summary Structure | 3 | 5 | **V2 (+2)** |
|
||||
| 5. Stats Box Format | 4 | 5 | **V2 (+1)** |
|
||||
| 6. Research Grounding | 4 | 5 | **V2 (+1)** |
|
||||
| 7. Prompt Quality (invitation) | 3 | 5 | **V2 (+2)** |
|
||||
| **TOTAL** | **20/35** | **29/35** | **V2 wins by 9 points** |
|
||||
|
||||
**V2 is clearly better.** The biggest gaps are citation quality (+3) and summary structure (+2). V2's output reads like a professional research briefing; V1's reads like a decent but unstructured summary.
|
||||
|
||||
---
|
||||
|
||||
## Part 2: V1-Only Outputs Analysis
|
||||
|
||||
### Output 1: "open claw" (GENERAL query)
|
||||
|
||||
**What V1 does well:**
|
||||
- Strong research grounding. Mentions exact numbers: "145,000+ GitHub stars," "20,000+ forks," "700+ skills," "341 malicious skills." These are clearly from research.
|
||||
- The KEY PATTERNS section is excellent: 5 well-organized patterns with community quotes (`"I give it sudo and let it configure everything"` vs `"prompt injection is terrifying when you give the bot access to your actual bank account"`).
|
||||
- Good synthesis of the security vs. enthusiasm tension -- captures the community split accurately.
|
||||
- Stats box uses the emoji tree format correctly with `├──` (though note: uses double-dash `──` instead of single `─`, minor inconsistency).
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display (`🔍 **open claw** · General`).
|
||||
- No inline citations in the body text. The 5 KEY PATTERNS have no `per @handle` or `per r/sub` attribution. Which Reddit thread said "I give it sudo"? Which X post raised the security concern? We do not know.
|
||||
- The stats box says `├── 🟠 Reddit: 25 threads │ ~750+ upvotes` -- the tilde and plus are imprecise. V2 SKILL.md wants exact parsed numbers.
|
||||
- Top voices line lists subreddits and handles but no engagement counts: `@grok, @Starlink` -- are these the highest-engagement handles? No like counts shown.
|
||||
- No bold topic headers in the body -- it is a single paragraph followed by a numbered list, not the `**{Topic}** — sentence, per source` format V2 requires.
|
||||
|
||||
**V1 Score (estimated):** 22/35
|
||||
|
||||
---
|
||||
|
||||
### Output 2: "nano banana pro prompting" (PROMPTING query)
|
||||
|
||||
**What V1 does well:**
|
||||
- Correctly identifies two prompting styles (JSON structured vs. natural language "Creative Director") and explains when each works best. This is excellent PROMPTING-type synthesis.
|
||||
- KEY PATTERNS are specific and actionable: "85mm lens at f/1.8," "three-point lighting with key at 45 degrees," "text rendering works -- keep text under 3 words for best results (75% success rate)." These are concrete tips a user can apply immediately.
|
||||
- Research grounding is strong: cites specific upvote counts ("149-259 upvotes"), subreddit names (`r/nanobanana2pro`), and the Google AI blog.
|
||||
- The invitation correctly targets Nano Banana Pro: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro."`
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display.
|
||||
- Stats box uses plain text dashes: `- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments` instead of the tree format `├─ 🟠 Reddit:`. Uses `|` pipe instead of `│` box-drawing character. V2 SKILL.md explicitly says: "NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji."
|
||||
- No inline body citations. KEY PATTERNS mention Reddit upvote ranges but no specific `per @handle` attributions.
|
||||
- Missing `✅ All agents reported back!` header -- just says "All agents reported back!" without the checkmark.
|
||||
- Body structure is paragraph + numbered list, not bold topic headers.
|
||||
|
||||
**V1 Score (estimated):** 23/35 (slightly higher than open claw due to better actionability)
|
||||
|
||||
---
|
||||
|
||||
### Output 3: "how to best setup clawdbot" (HOW-TO query)
|
||||
|
||||
**What V1 does well:**
|
||||
- This is the best V1 output of the batch. It goes beyond synthesis and actually delivers a **Quick-Start guide** with numbered steps, a **Security Hardening** checklist, and a **Budget Option** -- all grounded in research.
|
||||
- Excellent research grounding: `"per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0"` -- this is an actual citation with an @handle!
|
||||
- Specific, actionable recommendations: exact commands (`curl -fsSL https://clawd.bot/install.sh | bash`), specific model recommendations (Claude Opus 4.5 for best results, GLM 4.7 Flash for local), specific channel advice (Telegram first, WhatsApp QR code fails).
|
||||
- Stats box is correct emoji tree format with engagement counts: `@aashatwt (452 likes), @recap_david (329 likes)`.
|
||||
- Captures the naming confusion accurately: "Clawdbot -> Moltbot -> OpenClaw."
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display.
|
||||
- Body text has no inline citations except the Budget Option section. The 5 KEY PATTERNS have no `per @handle` attribution.
|
||||
- Bold topic headers are used only in the Quick-Start and Security sections, not in the KEY PATTERNS or intro.
|
||||
- The output delivers the "answer" directly (setup guide) rather than waiting for the user's vision and offering to write a prompt. For a HOW-TO query this might be the right call, but it skips the SKILL.md flow of "show research -> invite vision -> write prompt."
|
||||
|
||||
**V1 Score (estimated):** 26/35 (best of the V1 outputs)
|
||||
|
||||
---
|
||||
|
||||
### Patterns Across All V1 Outputs
|
||||
|
||||
**Consistent strengths:**
|
||||
1. Research grounding is solid across all three. V1 does not hallucinate -- the facts are clearly from the research output, not pre-training.
|
||||
2. KEY PATTERNS lists are consistently useful and actionable.
|
||||
3. Stats boxes are present in all outputs (though formatting varies).
|
||||
4. The invitation/closing line is present in all outputs.
|
||||
|
||||
**Consistent weaknesses:**
|
||||
1. **No query parsing display** in any output (0 for 4, including Kanye West).
|
||||
2. **No inline citations** in the body text (except one @handle in the clawdbot output). The research feels real but is unattributed.
|
||||
3. **Stats box formatting is inconsistent.** Open claw uses `├──` (double dash), nano banana pro uses `- 🟠` (plain dash + pipe), clawdbot uses `├─` (correct). Three different formats in three outputs.
|
||||
4. **Body structure defaults to paragraph + numbered list** instead of bold topic headers. Only clawdbot partially uses bold headers (in the guide section, not the research section).
|
||||
5. **No `(via Bird/xAI)` notation** on X stats in any output.
|
||||
|
||||
---
|
||||
|
||||
## Part 3: SKILL.md Feature Diff
|
||||
|
||||
### Features in V2 but NOT V1
|
||||
|
||||
| Feature | V2 Lines | Impact |
|
||||
|---------|----------|--------|
|
||||
| **Query parsing display** (`🔍 **{TOPIC}** · {QUERY_TYPE}`) | 40-53 | HIGH -- confirms to user the skill understood their request before spending time on research. |
|
||||
| **Sparse citation rules** with BAD/GOOD examples | 186-193 | HIGH -- this is the #1 quality differentiator in the Kanye head-to-head. `"per @handle"` format, never chain multiple citations. |
|
||||
| **Bold topic headers** template (`**{Topic 1}** — [1-2 sentences, per source]`) | 195-208 | HIGH -- makes output scannable. |
|
||||
| **Strict stats template** with "NEVER use plain text dashes" instruction | 217-230 | MEDIUM -- prevents the formatting inconsistency seen across V1 outputs. |
|
||||
| **RECOMMENDATIONS source attribution** (each item MUST have Sources: line with @handles) | 178-182 | MEDIUM -- only affects RECOMMENDATIONS queries. |
|
||||
| **Reddit 0 results handling** (explicit instruction for what to write) | 229 | LOW -- edge case, but prevents ad-hoc text like V1's "filtered out noise." |
|
||||
| **Bird CLI / xAI notation** in stats | 223 | LOW -- cosmetic transparency about data source. |
|
||||
| **Step 2 phrasing: "DO WEBSEARCH WHILE SCRIPT RUNS"** | 71-73 | LOW -- execution optimization, no output impact. |
|
||||
|
||||
### Features in V1 but NOT V2
|
||||
|
||||
| Feature | V1 Lines | Impact | Should Restore? |
|
||||
|---------|----------|--------|-----------------|
|
||||
| **Use cases block** (4 examples in intro) | 12-17 | LOW | No |
|
||||
| **Setup Check section** (3 modes, bash script, "keys are OPTIONAL") | 50-78 | MEDIUM for new users | Yes, for public release |
|
||||
| **BAD/GOOD synthesis anti-pattern examples** | 172-191 | MEDIUM-HIGH | YES |
|
||||
| **Self-check instruction** ("Re-read your 'What I learned' section...") | 269 | MEDIUM | YES |
|
||||
| **Quality Checklist** (5-point checklist before delivering prompt) | 306-324 | HIGH | YES |
|
||||
| **Prompt format anti-pattern** ("Research says JSON but you write prose") | 302 | MEDIUM | YES |
|
||||
| **"IF USER ASKS FOR MORE OPTIONS"** section | 327-329 | LOW-MEDIUM | YES |
|
||||
| **Web-only mode stats template + promo** | 248-259 | MEDIUM for no-key users | For public release |
|
||||
| **TARGET_TOOL question template** (4 options) | 272-280 | LOW | No |
|
||||
| **Context Memory: explicit "don't re-search" instructions** | 342-358 | MEDIUM | YES |
|
||||
| **Output footer emoji + engagement counts** | 366-380 | LOW | YES |
|
||||
|
||||
### Features in BOTH (Shared)
|
||||
|
||||
| Feature | Notes |
|
||||
|---------|-------|
|
||||
| Parse User Intent (TOPIC, TARGET_TOOL, QUERY_TYPE) | Same 4 query types, same detection logic |
|
||||
| "Don't ask about tool before research" rule | Identical |
|
||||
| Research script execution command | Same `python3` command |
|
||||
| WebSearch queries by QUERY_TYPE | Same search strategies |
|
||||
| "Use user's exact terminology" instruction | V2 shorter but same intent |
|
||||
| Judge Agent synthesis logic | Same 5-step weighting process |
|
||||
| "Ground in actual research" instruction | Same core instruction, V1 has more examples |
|
||||
| RECOMMENDATIONS: extract specific names | Same logic |
|
||||
| Prompt format matching | Same instruction |
|
||||
| Wait for user's vision | Same |
|
||||
| Write ONE perfect prompt | Same structure |
|
||||
| Context Memory | V2 shorter version |
|
||||
| Output summary footer | Both have it, V1 has emoji |
|
||||
| Depth options (quick/default/deep) | Same |
|
||||
| "After each prompt: Stay in Expert Mode" | Same |
|
||||
|
||||
### Overall Assessment
|
||||
|
||||
**V2 is a clear upgrade in output formatting and citation quality.** The three features V2 adds (query parsing display, sparse citation rules, bold topic headers) directly address the three biggest weaknesses seen across all V1 outputs. The Kanye West head-to-head proves it: V2 scores 29/35 vs V1's 20/35.
|
||||
|
||||
**However, V2 dropped several quality guardrails from V1** that do not affect formatting but affect *correctness*: the self-check instruction, the anti-pattern examples, the quality checklist for prompts, and the "don't re-search" context memory rule. These are cheap to restore (under 25 lines total) and protect against subtle failure modes that may not show up in a 1-query test but will appear over dozens of uses.
|
||||
|
||||
---
|
||||
|
||||
## Part 4: Verdict
|
||||
|
||||
### Ship V2 or Not?
|
||||
|
||||
**Ship V2 -- but restore the guardrails first.**
|
||||
|
||||
V2 is unambiguously better on every formatting dimension. The citation quality improvement alone (V1: 2/5 -> V2: 5/5) makes it worth shipping. The bold topic headers and strict stats template fix the inconsistency problems visible across all V1 outputs.
|
||||
|
||||
But V2 dropped 6 guardrail features from V1 that cost almost nothing to include and protect against real failure modes. These should be restored before V2 goes public.
|
||||
|
||||
### Remaining Gaps
|
||||
|
||||
**Must fix before shipping (affects correctness):**
|
||||
|
||||
1. **Restore the quality checklist for prompts.** This is the test plan's #1 priority item. V1 had a 5-point checklist; V2 reduced it to one line. The checklist is what makes prompts feel polished -- it is the "that's a great prompt" mechanism. Add 8 lines.
|
||||
|
||||
2. **Restore BAD/GOOD anti-pattern examples.** V2 says "ground in actual research" but does not show what *bad* grounding looks like. V1's ClawdBot/Claude Code conflation example is exactly the kind of concrete negative example that prevents real failures. Add 5 lines.
|
||||
|
||||
3. **Restore self-check instruction.** One sentence: "Re-read your 'What I learned' section -- does it match what the research ACTUALLY says?" Zero cost, catches hallucination. Add 2 lines.
|
||||
|
||||
4. **Restore "don't re-search" context memory rule.** V2 only says "only do new research if user asks about a DIFFERENT topic." V1 explicitly bans re-searching and tells the agent to answer from existing research. Add 3 lines.
|
||||
|
||||
**Should fix (polish):**
|
||||
|
||||
5. Restore prompt format anti-pattern ("Research says JSON but you write prose"). Add 2 lines.
|
||||
6. Restore "IF USER ASKS FOR MORE OPTIONS" section. Add 2 lines.
|
||||
7. Add emoji + engagement counts back to the output summary footer. Edit 3 lines.
|
||||
|
||||
**Skip for now:**
|
||||
|
||||
8. Setup Check section -- add back for public release, not needed for execution.
|
||||
9. Web-only mode stats template -- lower priority, most testers have API keys.
|
||||
10. TARGET_TOOL question template -- agent handles this naturally.
|
||||
|
||||
### Query Parsing Display: Investigate
|
||||
|
||||
Both V1 and V2 scored 1/5 on query parsing display. V2 has the feature in its SKILL.md but the agent did not render it in the captured output. This could mean:
|
||||
- The display was shown during execution but not captured (likely -- it appears before tools run, and the output files may only contain post-research content).
|
||||
- The instruction is not strong enough and the agent skips it.
|
||||
|
||||
**Recommendation:** Verify in a live session whether the parsing display actually appears. If it does not, strengthen the instruction (e.g., "This line MUST be the first thing you output, before any tool calls").
|
||||
|
||||
### Total Effort
|
||||
|
||||
Restoring all 7 priority items: approximately 25 lines added to V2 SKILL.md. Under 15 minutes of work. The V2 formatting wins are substantial and proven; the V1 guardrails are small and proven. Combining both produces the best version.
|
||||
|
||||
### Final Score Summary
|
||||
|
||||
| | V1 (Kanye) | V2 (Kanye) | Delta |
|
||||
|--|-----------|-----------|-------|
|
||||
| Total | 20/35 | 29/35 | **V2 +9** |
|
||||
|
||||
| | V1 (Open Claw) | V1 (Nano Banana) | V1 (Clawdbot) | V1 Average |
|
||||
|--|---------------|-----------------|--------------|------------|
|
||||
| Estimated Total | 22/35 | 23/35 | 26/35 | **23.7/35** |
|
||||
|
||||
V2 at 29/35 beats every V1 output, including V1's best (clawdbot at 26/35).
|
||||
|
||||
**Decision: Ship V2 with guardrails restored.**
|
||||
@@ -1,388 +0,0 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -1,25 +0,0 @@
|
||||
## What I learned:
|
||||
|
||||
The last 30 days have been one of Kanye West's most significant periods in years. The biggest story is his **full-page Wall Street Journal apology** (Jan 26, 2026) titled "To Those I've Hurt" — a comprehensive open letter addressing his antisemitic remarks and erratic behavior. He attributed his actions to a "four-month-long manic episode" and revealed that a car accident 25 years ago caused a frontal-lobe brain injury that wasn't properly diagnosed until 2023, leading to his bipolar type-1 diagnosis. Multiple sources describe wife **Bianca Censori** as instrumental in getting him to seek treatment during a manic depressive episode.
|
||||
|
||||
On the music side, he's announced **two upcoming albums**: *BULLY* (now scheduled for March 20, 2026) and *IN A PERFECT WORLD*. Daughter **North West** collaborated with him on a track called "Piercings on My Hand." He performed a massive show in Mexico City (Jan 30) at the Monumental Plaza de Toros, and he'll headline the new **Hellwatt Festival** in Italy (July 4-18, 2026).
|
||||
|
||||
The **2026 Grammys** remain contentious — he's reportedly not invited after last year's red carpet drama. Health concerns also surfaced after a January LA outing where he appeared "noticeably bloated and moving with sluggish energy."
|
||||
|
||||
**KEY THEMES:**
|
||||
1. **Accountability arc** — The WSJ apology is being widely discussed as either genuine growth or calculated PR
|
||||
2. **Mental health transparency** — Brain injury + bipolar diagnosis framing his past behavior
|
||||
3. **Bianca Censori's role** — Increasingly seen as a stabilizing force (despite continued provocative public appearances)
|
||||
4. **Music comeback** — Two albums announced, active touring in Latin America and Europe
|
||||
5. **Grammy exile** — Still persona non grata at major award shows
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 0 relevant threads (filtered out noise)
|
||||
├─ 🔵 X: 30 posts │ ~10 likes │ casual fan chatter + Bianca Censori discourse
|
||||
├─ 🌐 Web: 20+ pages │ washingtonpost.com, billboard.com, allhiphop.com, euronews.com, rap-up.com
|
||||
└─ Top voices: Washington Post, Billboard, AllHipHop, Euronews, The Mirror
|
||||
|
||||
---
|
||||
|
||||
Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in.
|
||||
@@ -1,24 +0,0 @@
|
||||
## What I learned:
|
||||
|
||||
OpenClaw (formerly ClawdBot, formerly Moltbot) is an open-source autonomous AI agent created by Peter Steinberger that went massively viral in late January 2026 — hitting 145,000+ GitHub stars and 20,000+ forks. It runs locally on your machine, connects to messaging platforms (Telegram, WhatsApp, Discord, Signal, etc.), and uses LLMs like Claude, GPT, or Gemini to actually perform tasks on your computer — file management, scheduling, web browsing, purchases, and more. Think "JARVIS for everyone."
|
||||
|
||||
The community is simultaneously excited and alarmed. The "ClawHub" skill marketplace has 700+ skills but a VirusTotal/Koi Security audit found **341 malicious skills** doing data exfiltration and crypto theft. A high-severity RCE vulnerability was also disclosed. The #1 most downloaded skill on ClawHub was confirmed malware.
|
||||
|
||||
**KEY PATTERNS from the research:**
|
||||
|
||||
1. **Cost is the top complaint** — API usage burns money fast; many users recommend Gemini CLI (free 1,000 req/day) over Claude/GPT for OpenClaw to control spend
|
||||
2. **Security is a dumpster fire** — Supply chain attacks via ClawHub skills, prompt injection risks when browsing the web, RCE vulnerabilities, and orgs are banning it at the EDR level
|
||||
3. **Setup is nontrivial** — Even after install, most users get stuck on skills/tools/automation config; the `openclaw onboard` wizard is the recommended path
|
||||
4. **The skill ecosystem is powerful but dangerous** — 700+ skills available, but no code signing, no real moderation, and active malware campaigns
|
||||
5. **Community is split** — Enthusiasts love the autonomy ("I give it sudo and let it configure everything"), while security folks are terrified ("prompt injection is terrifying when you give the bot access to your actual bank account")
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├── 🟠 Reddit: 25 threads │ ~750+ upvotes │ ~600+ comments
|
||||
├── 🔵 X: 27 posts │ 1,750+ likes │ 325+ reposts
|
||||
├── 🌐 Web: 20+ pages │ Wikipedia, Bloomberg, The Register, Hacker News, Nature, VirusTotal, DigitalOcean, Codecademy, Tom's Hardware
|
||||
└── Top voices: r/openclaw, r/cybersecurity, r/clawdbot, r/ArtificialInteligence │ @grok, @Starlink │ Nature, Bloomberg, ACM
|
||||
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly.
|
||||
@@ -1,27 +0,0 @@
|
||||
Now I have a comprehensive picture. Let me synthesize everything.
|
||||
|
||||
**What I learned:**
|
||||
|
||||
Nano Banana Pro is fundamentally different from older image models — it's a "thinking" model that understands intent, physics, and composition. The community has converged on two distinct prompting styles, and **both work well** depending on the use case:
|
||||
|
||||
1. **JSON structured prompts** — Dominant on Reddit (r/nanobanana2pro) and X. Power users share elaborate JSON objects with nested fields for subject, appearance, lighting, composition, and style. These produce the most consistent, controlled results for portraits, editorials, and product shots.
|
||||
|
||||
2. **Natural language "Creative Director" prompts** — Recommended by Google's official guide. Full sentences describing the scene as if briefing a photographer. Better for cinematic shots, complex edits, and iterative refinement.
|
||||
|
||||
**KEY PATTERNS I'll use:**
|
||||
1. **JSON format for maximum control** — The top Reddit posts (149-259 upvotes) all use structured JSON with nested objects: `subject > appearance > hair/skin/expression`, `lighting > key/fill/rim`, `camera > lens/angle/depth_of_field`
|
||||
2. **Be a Creative Director, not a tag spammer** — Replace "4k, realistic, beautiful" with specific technical directions: "85mm lens at f/1.8", "three-point lighting with key at 45°", "shallow depth of field with bokeh"
|
||||
3. **Provide context for WHY** — Adding purpose ("for a luxury fashion editorial", "for a high-end gourmet cookbook") helps the model infer appropriate styling, plating, lighting
|
||||
4. **Edit, don't regenerate** — When 80% is right, ask for the specific change conversationally
|
||||
5. **Text rendering works** — Keep text under 3 words for best results (75% success rate), specify font style and placement
|
||||
|
||||
---
|
||||
All agents reported back!
|
||||
- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments
|
||||
- 🔵 X: 29 posts | 312 likes | 117 reposts
|
||||
- 🌐 Web: 10+ pages | blog.google, dev.to, eweek.com, imagine.art, github.com, leonardo.ai, medium.com
|
||||
- Top voices: r/nanobanana2pro | @Xmira_belle, @doctorwasif, @chatgptpaglu | Google AI blog, DEV Community
|
||||
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro.
|
||||
@@ -1,48 +0,0 @@
|
||||
**What I learned:**
|
||||
|
||||
Clawdbot (now officially renamed **OpenClaw**) is a self-hosted, open-source personal AI assistant that connects to messaging apps (Telegram, WhatsApp, Discord, Slack, Signal, iMessage) and uses LLMs to act as an autonomous agent on your behalf. It's roughly one week old as a public project and the community is moving fast — the name has already changed twice (Clawdbot → Moltbot → OpenClaw).
|
||||
|
||||
**KEY PATTERNS from the research:**
|
||||
|
||||
1. **Security is the #1 concern** — Multiple Reddit threads and guides warn that many setups are wide open to the internet with zero auth. Run it on dedicated/isolated hardware (VM, Mac Mini, NAS), never your main machine. Block dangerous commands via allowlist. Never add it to group chats.
|
||||
|
||||
2. **Use the onboarding wizard, not Docker** — Docker was reported as unreliable (shims, bridges). The single-line install script (`curl -fsSL https://clawd.bot/install.sh | bash`) followed by `openclaw onboard` is the smoothest path. Docker is a "no-go" per multiple users.
|
||||
|
||||
3. **Model choice matters a lot** — Claude Opus 4.5 is recommended for best results (trained to resist prompt injection, ~99% resistance). For budget setups, Claude Haiku works at $0. Local models (Ollama/LM Studio) work but cause hallucination and token issues — GLM 4.7 Flash (30B) was recommended as the best local option.
|
||||
|
||||
4. **Telegram is the easiest channel to start with** — Most guides and users start with a Telegram bot token. WhatsApp QR code pairing fails frequently. Get Telegram working first, then expand.
|
||||
|
||||
5. **Sandbox and isolate** — Set `agents.defaults.sandbox.mode` to `"non-main"` for Docker sandboxing. Use a Linux VM or UTM on Mac to segregate traffic. Tailscale for secure remote access.
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 19 threads │ 415+ upvotes │ 290+ comments
|
||||
├─ 🔵 X: 29 posts │ 815+ likes │ 65+ reposts
|
||||
├─ 🌐 Web: 10+ pages │ Medium, DataCamp, DEV, GitHub, QNAP, docs.openclaw.ai
|
||||
└─ Top voices: r/clawdbot, r/LocalLLM, r/AskVibecoders │ @aashatwt (452 likes), @recap_david (329 likes) │ Lukas Niessen on Medium
|
||||
|
||||
---
|
||||
|
||||
**Here's the consensus best setup path based on what the community actually recommends:**
|
||||
|
||||
### Quick-Start (30 min)
|
||||
1. **Get dedicated hardware** — Mac Mini, old laptop, NAS (QNAP has an official guide), or a VPS (DigitalOcean gist available). Don't use your daily driver.
|
||||
2. **Install via script**: `curl -fsSL https://clawd.bot/install.sh | bash`
|
||||
3. **Run the wizard**: `openclaw onboard` — it walks through gateway, workspace, channels, and skills
|
||||
4. **Connect Telegram first** — create a bot via @BotFather, paste the token
|
||||
5. **Set your API key** — Anthropic API key for Claude, or connect Ollama for local models
|
||||
6. **Customize SOUL.md** — this controls your bot's personality and behavior
|
||||
|
||||
### Security Hardening (do this immediately)
|
||||
- Bind gateway to localhost only, use Tailscale for remote access
|
||||
- Set sandbox mode to `"non-main"`
|
||||
- Create an explicit command allowlist (block `rm -rf`, `sudo`, `chmod`)
|
||||
- Rotate API keys every 90 days
|
||||
- Never add the bot to group chats
|
||||
|
||||
### Budget Option ($0)
|
||||
Per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0.
|
||||
|
||||
---
|
||||
|
||||
Want me to help you with a specific part of the setup, or do you have a particular use case in mind (home automation, CRM, coding assistant, etc.)?
|
||||
@@ -1,332 +0,0 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
**DISPLAY your parsing to the user.** Before running any tools, output a single line:
|
||||
|
||||
🔍 **{TOPIC}** · {QUERY_TYPE}
|
||||
Searching Reddit, X, and the web for {natural language description of what you'll look for}...
|
||||
|
||||
Example outputs:
|
||||
- 🔍 **kanye west** · News — Searching Reddit, X, and the web for the latest kanye west news and discussions...
|
||||
- 🔍 **best MCP servers** · Recommendations — Searching Reddit, X, and the web for the most recommended MCP servers...
|
||||
- 🔍 **nano banana pro prompting** · Prompting — Searching Reddit, X, and the web for nano banana pro prompting techniques and tips...
|
||||
- 🔍 **open claw** · General — Searching Reddit, X, and the web for what people are saying about open claw...
|
||||
|
||||
If TARGET_TOOL is known, mention it: "...for nano banana pro prompting techniques to use in ChatGPT..."
|
||||
|
||||
This text MUST appear before you call any tools. It confirms to the user that you understood their request.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
---
|
||||
|
||||
## STEP 2: DO WEBSEARCH WHILE SCRIPT RUNS
|
||||
|
||||
The script auto-detects sources (Bird CLI, API keys, etc). While waiting for it, do WebSearch.
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords?
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned with sources:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
|
||||
[Tool Name] - {n}x mentions
|
||||
Use Case: [what it does]
|
||||
Sources: @handle1, @handle2, r/sub, blog.com
|
||||
|
||||
[Tool Name] - {n}x mentions
|
||||
Use Case: [what it does]
|
||||
Sources: @handle3, r/sub2, Complex
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**CRITICAL for RECOMMENDATIONS:**
|
||||
- Each item MUST have a "Sources:" line with actual @handles from X posts (e.g., @LONGLIVE47, @ByDobson)
|
||||
- Include subreddit names (r/hiphopheads) and web sources (Complex, Variety)
|
||||
- Parse @handles from research output and include the highest-engagement ones
|
||||
- Format naturally - tables work well for wide terminals, stacked cards for narrow
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
|
||||
CITATION RULE: Cite sources sparingly to prove research is real.
|
||||
- In the "What I learned" intro: cite 1-2 top sources total, not every sentence
|
||||
- In KEY PATTERNS: cite 1 source per pattern, short format: "per @handle" or "per r/sub"
|
||||
- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box
|
||||
- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one.
|
||||
|
||||
**BAD:** "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)."
|
||||
**GOOD:** "His album BULLY is set for March 20 via Gamma, per Rolling Stone."
|
||||
|
||||
```
|
||||
What I learned:
|
||||
|
||||
**{Topic 1}** — [1-2 sentences about this storyline, per source]
|
||||
|
||||
**{Topic 2}** — [1-2 sentences, per source]
|
||||
|
||||
**{Topic 3}** — [1-2 sentences, per source]
|
||||
|
||||
KEY PATTERNS from the research:
|
||||
1. [Pattern] — per @handle
|
||||
2. [Pattern] — per r/sub
|
||||
3. [Pattern] — per source
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
**CRITICAL: Calculate actual totals from the research output.**
|
||||
- Count posts/threads from each section
|
||||
- Sum engagement: parse `[Xlikes, Yrt]` from each X post, `[Xpts, Ycmt]` from Reddit
|
||||
- Identify top voices: highest-engagement @handles from X, most active subreddits
|
||||
|
||||
**Copy this EXACTLY, replacing only the {placeholders}:**
|
||||
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments
|
||||
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI)
|
||||
├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3}
|
||||
└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2}
|
||||
---
|
||||
```
|
||||
|
||||
If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)"
|
||||
NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT.**
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Quality Checklist (run before delivering):
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
@@ -1,25 +0,0 @@
|
||||
Here's what I found:
|
||||
|
||||
## What I learned:
|
||||
|
||||
**BULLY Album — March 20, 2026 via Gamma** — After years of delays (first announced September 2024), Kanye's 12th studio album finally has a firm release date. The 13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign. Sonically it recalls *808s & Heartbreak* and *MBDTF* — Ye mostly sings rather than raps. Notably, earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded with his real voice for the official release, per Rolling Stone.
|
||||
|
||||
**Public Apology for Antisemitism** — On January 26, Ye took out a full-page Wall Street Journal ad titled "To Those I've Hurt," publicly apologizing for his antisemitic remarks. He disclosed a previously undiagnosed brain injury from his 2002 car accident and attributed his behavior to a four-month manic episode fueled by bipolar disorder, psychosis, and paranoia. "I lost touch with reality," he wrote. The apology also extended to the Black community, per The Washington Post.
|
||||
|
||||
**Hellwatt Festival in Italy** — Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard.
|
||||
|
||||
**Health Concerns** — A rare January 2026 outing in LA with Bianca Censori reignited concern about Ye's physical appearance. Insiders point to medication side effects, frequent travel, and inconsistent routines, per AllHipHop.
|
||||
|
||||
**Grammys Ban** — Ye is reportedly not welcome at the 2026 Grammy Awards after clashing with organizers last year over his invitation terms, per The News International.
|
||||
|
||||
**Kim & Lewis Hamilton Buzz** — X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton, with users contrasting her new relationship against her marriage to Ye.
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 0 threads (no results this cycle)
|
||||
├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)
|
||||
├─ 🌐 Web: 30+ pages │ rollingstone.com, washingtonpost.com, complex.com, billboard.com, npr.org
|
||||
└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice.
|
||||
@@ -0,0 +1,8 @@
|
||||
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/community-more-posts/top/?name=technology&t=week on 2026-05-29; trimmed to 5 post cards (start-tag attrs only). -->
|
||||
<div id="feed">
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tq0zk7/the_netherlands_just_blocked_a_us_company_from/" content-href="https://www.techspot.com/news/112552-netherlands-blocked-us-company-buying-app-dutch-citizens.html" view-context="SubredditFeed" comment-count="1743" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="0" award-count="23" award-id="award_obsessed_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/Obsessed_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-28T11:37:01.506000+0000" domain="techspot.com" id="t3_1tq0zk7" post-title="The Netherlands just blocked a US company from buying the app Dutch citizens use for everything" post-language="en" post-type="link" score="52692" upvote-ratio="0.9606269354736776" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1toe7m2/erin_brockovich_launches_map_of_over_4200_data/" content-href="https://www.newsweek.com/erin-brockovich-asks-americans-for-help-as-she-launches-data-center-map-11989813" view-context="SubredditFeed" comment-count="673" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="2" award-count="6" award-id="award_this_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/this_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T17:39:43.272000+0000" domain="newsweek.com" id="t3_1toe7m2" post-title="Erin Brockovich launches map of over 4,200 data centres in the US, appeals for local communities to report environmental impact and other costs" post-language="en" post-type="link" score="33567" upvote-ratio="0.973297166968053" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_fj9vsvfd" author="marketrent" icon="https://www.redditstatic.com/avatars/defaults/v2/avatar_default_1.png" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tollgz/majority_of_americans_support_ban_on_surveillance/" content-href="https://gizmodo.com/majority-of-americans-support-ban-on-surveillance-pricing-and-electronic-shelf-labels-2000762717" view-context="SubredditFeed" comment-count="1043" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="3" award-count="7" award-id="award_free_bravo" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/bravo_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T21:55:07.322000+0000" domain="gizmodo.com" id="t3_1tollgz" post-title="Majority of Americans Support Ban on Surveillance Pricing and Electronic Shelf Labels" post-language="en" post-type="link" score="29791" upvote-ratio="0.9815063671850003" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_98wao505" author="Plastic_Ninja_9014" icon="https://preview.redd.it/snoovatar/avatars/69af2b53-b0a1-4ab6-b119-d90f21c423fe-headshot.png?width=64&height=64&crop=smart&auto=webp&s=f3661eb511798004968f8b115a689dcee30f1428" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tp5qz2/tech_ceos_are_apparently_suffering_from_ai/" content-href="https://techcrunch.com/2026/05/27/tech-ceos-are-apparently-suffering-from-ai-psychosis/" view-context="SubredditFeed" comment-count="1653" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="4" award-count="6" award-id="award_free_regret_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/regret_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-27T13:33:49.280000+0000" domain="techcrunch.com" id="t3_1tp5qz2" post-title="Tech CEOs are apparently suffering from AI psychosis" post-language="en" post-type="link" score="26419" upvote-ratio="0.9605741880002646" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tn5g7s/pope_leo_issues_ai_encyclical_warning_that_opaque/" content-href="https://variety.com/2026/biz/global/pope-leo-ai-encyclical-algorithms-threaten-dehumanisation-1236758186/" view-context="SubredditFeed" comment-count="608" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="6" award-count="7" award-id="award_hooray_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/FTUE_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-25T10:45:04.093000+0000" domain="variety.com" id="t3_1tn5g7s" post-title="Pope Leo Issues AI Encyclical Warning That ‘Opaque Algorithms’ Controlled by a ‘Few’ Companies Can Bring ‘New Forms of Dehumanisation’" post-language="en" post-type="link" score="25835" upvote-ratio="0.9760626539506095" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_1i1zizibn9" author="yourfavchoom" icon="https://styles.redditmedia.com/t5_dgdrt8/styles/profileIcon_k9x929ihm8rg1.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=2e8a5042cccc4555167f98d28bc0de4e13fd3ca5" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
</div>
|
||||
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- FIXTURE: captured live from reddit.com/r/Rakuten/top.rss on 2026-05-29; trimmed to 5 entries. Atom shape identical to search.rss. --><feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/"><category term="Rakuten" label="r/Rakuten"/><updated>2026-05-29T14:14:32+00:00</updated><icon>https://www.redditstatic.com/icon.png/</icon><id>/r/Rakuten/top.rss?t=month</id><link rel="self" href="https://www.reddit.com/r/Rakuten/top.rss?t=month" type="application/atom+xml" /><link rel="alternate" href="https://www.reddit.com/r/Rakuten/top?t=month" type="text/html" /><subtitle>This is an unofficial subreddit for Rakuten Rewards, the cash back website. We are not affiliated with, endorsed by, or sponsored by Rakuten or any of its subsidiaries.</subtitle><title>top scoring links : Rakuten</title><entry><author><name>/u/InternetUser52</name><uri>https://www.reddit.com/user/InternetUser52</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>I&#39;m rich!!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/InternetUser52"> /u/InternetUser52 </a> <br/> <span><a href="https://i.redd.it/q8fgmxs29c2h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/">[comments]</a></span></content><id>t3_1tiv013</id><link href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/" /><updated>2026-05-20T18:48:31+00:00</updated><published>2026-05-20T18:48:31+00:00</published><title>LETS GOO! $0.02!!!</title></entry>
|
||||
<entry><author><name>/u/Immediate-Duck-6351</name><uri>https://www.reddit.com/user/Immediate-Duck-6351</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>I don’t travel and I’m buying a house in a few weeks so cash back is amazing 🙌 hoping to keep the pace in the next quarter so I can buy new kitchen appliances lol. </p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Immediate-Duck-6351"> /u/Immediate-Duck-6351 </a> <br/> <span><a href="https://i.redd.it/d2a4s0ipvb1h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/">[comments]</a></span></content><id>t3_1te1fp8</id><link href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/" /><updated>2026-05-15T16:29:28+00:00</updated><published>2026-05-15T16:29:28+00:00</published><title>So excited 🥳</title></entry>
|
||||
<entry><author><name>/u/gnibgnib</name><uri>https://www.reddit.com/user/gnibgnib</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>128k for the May transfer</p> <p>41k pending for August </p> <p>Got another 9k at Asics not showing but overall pretty happy with Rakuten</p> <p>P2 was able to secure 85k for May transfer</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/gnibgnib"> /u/gnibgnib </a> <br/> <span><a href="https://www.reddit.com/gallery/1tb8674">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/">[comments]</a></span></content><id>t3_1tb8674</id><link href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/" /><updated>2026-05-12T17:17:19+00:00</updated><published>2026-05-12T17:17:19+00:00</published><title>Had a great run so far this year thanks to this sub!</title></entry>
|
||||
<entry><author><name>/u/TravelVet93</name><uri>https://www.reddit.com/user/TravelVet93</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&#32; submitted by &#32; <a href="https://www.reddit.com/user/TravelVet93"> /u/TravelVet93 </a> <br/> <span><a href="https://i.redd.it/x6b9whvupb1h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/">[comments]</a></span></content><id>t3_1te0hom</id><link href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/" /><updated>2026-05-15T15:56:40+00:00</updated><published>2026-05-15T15:56:40+00:00</published><title>My best payout so far</title></entry>
|
||||
<entry><author><name>/u/Beautiful-Piece-4252</name><uri>https://www.reddit.com/user/Beautiful-Piece-4252</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>The amount of $$ available in sign up bonuses is amazing. It&#39;s kind of a part time job ensuring Rakuten captures everything, but my August and November payout should be sizeable. I&#39;m new to this and it always seemed like a lot of work for little reward. I know it&#39;s not sustainable, but wow!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Beautiful-Piece-4252"> /u/Beautiful-Piece-4252 </a> <br/> <span><a href="https://i.redd.it/1vqvajsci42h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/">[comments]</a></span></content><id>t3_1thsnm1</id><link href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/" /><updated>2026-05-19T16:46:17+00:00</updated><published>2026-05-19T16:46:17+00:00</published><title>How can this be real?</title></entry>
|
||||
</feed>
|
||||
@@ -0,0 +1,29 @@
|
||||
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0 on 2026-05-29;
|
||||
trimmed to 6 real comment elements (real attrs + real bodies) + 2 synthetic edge cases. -->
|
||||
<shreddit-comment-tree-stats total-comments="14"></shreddit-comment-tree-stats>
|
||||
<shreddit-comment-tree id="comment-tree" post-id="t3_1taeiw0">
|
||||
<shreddit-comment created="2026-05-11T20:16:57.590000+0000" author="Obvious_Painting_881" thingId="t1_ol8tp8n" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8tp8n/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_ol8tp8n-comment-rtjson-content" slot="comment"><div id="t1_ol8tp8n-post-rtjson-content" dir="auto"><p dir="auto">Where do you find $750? The highest available package for Total was $284.99 when I did the lifelock promotion. I did get the full 284.99 from Rakuten.</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-12T12:26:14.973000+0000" author="Stormtrooper149" thingId="t1_olcy1iv" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/olcy1iv/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_olcy1iv-comment-rtjson-content" slot="comment"><div id="t1_olcy1iv-post-rtjson-content" dir="auto"><p dir="auto">It went to pending ($712.49)</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-19T01:43:48.026000+0000" author="heythereyou01" thingId="t1_omlbiqg" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/omlbiqg/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_omlbiqg-comment-rtjson-content" slot="comment"><div id="t1_omlbiqg-post-rtjson-content" dir="auto"><p dir="auto">Hey I PM’d. can I get the screenshot ?</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-11T20:21:16.398000+0000" author="Stormtrooper149" thingId="t1_ol8undb" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8undb/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_ol8undb-comment-rtjson-content" slot="comment"><div id="t1_ol8undb-post-rtjson-content" dir="auto"><p dir="auto">Family plan</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-11T20:28:33.803000+0000" author="Obvious_Painting_881" thingId="t1_ol8w8w6" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8w8w6/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_ol8w8w6-comment-rtjson-content" slot="comment"><div id="t1_ol8w8w6-post-rtjson-content" dir="auto"><p dir="auto">Price seems to change every time I go to the page but I see only 249.99-369.99 for Total/Advanced. No where near your $750. Just saying the Total plan for 299.99 worked for me and I got 284.99 which is 95%.</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-12T02:33:48.200000+0000" author="jwegener" thingId="t1_olaqzjk" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/olaqzjk/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_olaqzjk-comment-rtjson-content" slot="comment"><div id="t1_olaqzjk-post-rtjson-content" dir="auto"><p dir="auto">I did that one. Let’s pray</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-13T10:00:00.000000+0000" author="[deleted]" thingId="t1_synthdel" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/synthdel/" score="5" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_synthdel-comment-rtjson-content" slot="comment"><div id="t1_synthdel-post-rtjson-content" dir="auto"><p dir="auto">[removed]</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-13T11:00:00.000000+0000" author="NegScoreUser" thingId="t1_synthneg" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/synthneg/" score="-7" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_synthneg-comment-rtjson-content" slot="comment"><div id="t1_synthneg-post-rtjson-content" dir="auto"><p dir="auto">A downvoted but real reply with negative score for edge-case coverage.</p></div></div>
|
||||
</shreddit-comment>
|
||||
</shreddit-comment-tree>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.0.5",
|
||||
"version": "3.17.0",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"triggerOnUpdates": true,
|
||||
"statusCheck": true
|
||||
}
|
||||
+1
-2
@@ -6,8 +6,7 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
||||
"timeout": 5
|
||||
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+182
-21
@@ -2,15 +2,33 @@
|
||||
set -euo pipefail
|
||||
|
||||
# Check last30days configuration status and show appropriate welcome message.
|
||||
# Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars
|
||||
# Priority for this status hook:
|
||||
# .claude/last30days.env > ~/.config/last30days/.env > env vars > Keychain presence
|
||||
|
||||
PROJECT_ENV=".claude/last30days.env"
|
||||
GLOBAL_ENV="$HOME/.config/last30days/.env"
|
||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||
GLOBAL_ENV="$LAST30DAYS_CONFIG_DIR/.env"
|
||||
else
|
||||
GLOBAL_ENV=""
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ensure LAST30DAYS_MEMORY_DIR exists for HTML-brief / raw-markdown saves.
|
||||
# SKILL.md and the engine default this via the same env-var fallback. Fresh
|
||||
# installs otherwise fail silently on first --emit=html run. See #395.
|
||||
mkdir -p "${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}" 2>/dev/null || true
|
||||
|
||||
# Helper: warn if file permissions are too open
|
||||
check_perms() {
|
||||
local file="$1"
|
||||
if [[ ! -f "$file" ]]; then return; fi
|
||||
# Git-for-Windows / MSYS / Cygwin run stat in noacl mode (always 644),
|
||||
# so this POSIX check is a false positive. Windows perms use ACLs.
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
MINGW*|MSYS*|CYGWIN*) return ;;
|
||||
esac
|
||||
local perms
|
||||
# Try GNU stat first (Linux), fall back to BSD stat (macOS).
|
||||
# On Linux, `stat -f` prints filesystem info (not permissions) and exits 0,
|
||||
@@ -18,11 +36,29 @@ check_perms() {
|
||||
# every Linux session start and printed a false WARNING.
|
||||
perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
|
||||
if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
|
||||
echo "/last30days: WARNING — $file has permissions $perms (should be 600)."
|
||||
echo " Fix: chmod 600 $file"
|
||||
chmod 600 "$file" && echo "/last30days: WARNING — $file had permissions $perms — auto-fixed with chmod 600" || echo "/last30days: WARNING — $file has permissions $perms (should be 600). Fix: chmod 600 $file"
|
||||
fi
|
||||
}
|
||||
|
||||
trim_ws() {
|
||||
local s="$1"
|
||||
s="${s#"${s%%[![:space:]]*}"}"
|
||||
s="${s%"${s##*[![:space:]]}"}"
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
strip_outer_quotes() {
|
||||
local s="$1"
|
||||
if [[ ${#s} -ge 2 ]]; then
|
||||
if [[ "${s:0:1}" == '"' && "${s: -1}" == '"' ]]; then
|
||||
s="${s:1:${#s}-2}"
|
||||
elif [[ "${s:0:1}" == "'" && "${s: -1}" == "'" ]]; then
|
||||
s="${s:1:${#s}-2}"
|
||||
fi
|
||||
fi
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
# Load env file into variables for inspection (without exporting)
|
||||
load_env_vars() {
|
||||
local file="$1"
|
||||
@@ -31,10 +67,15 @@ load_env_vars() {
|
||||
# Skip comments, empty lines
|
||||
[[ "$key" =~ ^[[:space:]]*# ]] && continue
|
||||
[[ -z "$key" ]] && continue
|
||||
key=$(echo "$key" | xargs)
|
||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||
key="$(trim_ws "$key")"
|
||||
value="$(strip_outer_quotes "$(trim_ws "$value")")"
|
||||
# Strip inline comments (# preceded by whitespace) to prevent
|
||||
# command substitution in backtick-containing comments
|
||||
value="${value%%[[:space:]]#*}"
|
||||
if [[ -n "$key" && -n "$value" ]]; then
|
||||
eval "ENV_${key}=\"${value}\""
|
||||
# printf -v writes via assignment semantics (global from inside a
|
||||
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
|
||||
printf -v "ENV_${key}" '%s' "$value"
|
||||
fi
|
||||
done < "$file"
|
||||
fi
|
||||
@@ -55,28 +96,123 @@ if [[ -n "$CONFIG_FILE" ]]; then
|
||||
load_env_vars "$CONFIG_FILE"
|
||||
fi
|
||||
|
||||
# Check SETUP_COMPLETE (from file or env)
|
||||
# Load Keychain item presence for status checks without reading secret values.
|
||||
# Runtime credential resolution still happens in lib/env.py; this hook only
|
||||
# needs to avoid stale first-run/source-count messages.
|
||||
load_keychain_presence() {
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
Darwin*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
command -v security >/dev/null 2>&1 || return 0
|
||||
|
||||
local user key env_var current
|
||||
user="${USER:-}"
|
||||
if [[ -z "$user" ]]; then
|
||||
user="$(id -un 2>/dev/null || true)"
|
||||
fi
|
||||
[[ -n "$user" ]] || return 0
|
||||
|
||||
for key in SETUP_COMPLETE OPENAI_API_KEY SCRAPECREATORS_API_KEY AUTH_TOKEN CT0 XAI_API_KEY BSKY_HANDLE EXA_API_KEY; do
|
||||
env_var="ENV_${key}"
|
||||
current="${!env_var:-}"
|
||||
if [[ -z "$current" ]]; then
|
||||
current="${!key:-}"
|
||||
fi
|
||||
[[ -n "$current" ]] && continue
|
||||
if security find-generic-password -a "$user" -s "last30days-${key}" >/dev/null 2>&1; then
|
||||
printf -v "ENV_${key}" '%s' "keychain"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
load_keychain_presence
|
||||
|
||||
# Check SETUP_COMPLETE (from file, env, or Keychain presence)
|
||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||
cat <<'EOF'
|
||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||
# Compute last-run summary line (if last-run.json exists)
|
||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
|
||||
else
|
||||
LAST_RUN_FILE=""
|
||||
fi
|
||||
else
|
||||
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
||||
fi
|
||||
LAST_RUN_LINE=""
|
||||
# python3 -c, NOT a heredoc: bash 5.3 feeds heredocs to the child through a
|
||||
# pipe and can deadlock in heredoc_write inside command substitution, hanging
|
||||
# this hook forever at session start (observed on Homebrew bash 5.3.15).
|
||||
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 -c '
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
|
||||
Reddit, Hacker News, and Polymarket work out of the box.
|
||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||
EOF
|
||||
path = os.environ["LAST_RUN_FILE"]
|
||||
try:
|
||||
with open(path) as fh:
|
||||
d = json.load(fh)
|
||||
topic = (d.get("topic") or "?")[:60]
|
||||
ts = d.get("timestamp", "")
|
||||
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
|
||||
if delta < 60: ago = f"{int(delta)}s ago"
|
||||
elif delta < 3600: ago = f"{int(delta//60)}m ago"
|
||||
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
|
||||
else: ago = f"{int(delta//86400)}d ago"
|
||||
total = d.get("total", 0)
|
||||
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
||||
except Exception:
|
||||
pass
|
||||
' 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
# Detect capability that doesn't need a config file: yt-dlp on PATH.
|
||||
# Done before the new-user early-exit so first-run users with yt-dlp
|
||||
# installed see YouTube is already available. See #394.
|
||||
HAS_YTDLP=""
|
||||
if command -v yt-dlp &>/dev/null; then
|
||||
HAS_YTDLP="yes"
|
||||
fi
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${ENV_OPENAI_API_KEY:-${OPENAI_API_KEY:-}}" && -z "${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}" && -z "${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}" && -z "${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}" ]]; then
|
||||
# printf, NOT cat-with-heredoc: see the bash 5.3 heredoc deadlock note above.
|
||||
if [[ -n "$HAS_YTDLP" ]]; then
|
||||
# YouTube is already working via the on-system yt-dlp binary — don't list
|
||||
# it as something the wizard needs to unlock. See #394.
|
||||
printf '%s\n' \
|
||||
'/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.' \
|
||||
' Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).' \
|
||||
'' \
|
||||
'Reddit, Hacker News, Polymarket, and YouTube (yt-dlp detected) work out of the box.' \
|
||||
'The setup wizard can unlock X/Twitter and more.' \
|
||||
' Detected: yt-dlp is installed (YouTube transcripts ready, no setup needed).'
|
||||
else
|
||||
printf '%s\n' \
|
||||
'/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.' \
|
||||
' Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).' \
|
||||
'' \
|
||||
'Reddit, Hacker News, and Polymarket work out of the box.' \
|
||||
'The setup wizard can unlock X/Twitter, YouTube, and more.'
|
||||
fi
|
||||
if [[ -n "$LAST_RUN_LINE" ]]; then
|
||||
echo "$LAST_RUN_LINE"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Setup done but check for ScrapeCreators
|
||||
HAS_SCRAPECREATORS="${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}"
|
||||
HAS_X="${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}"
|
||||
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
|
||||
HAS_YTDLP=""
|
||||
if command -v yt-dlp &>/dev/null; then
|
||||
HAS_YTDLP="yes"
|
||||
HAS_X=""
|
||||
if [[ -n "${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}" && -n "${ENV_CT0:-${CT0:-}}" ]]; then
|
||||
HAS_X="yes"
|
||||
fi
|
||||
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
|
||||
HAS_BSKY="${ENV_BSKY_HANDLE:-${BSKY_HANDLE:-}}"
|
||||
HAS_EXA="${ENV_EXA_API_KEY:-${EXA_API_KEY:-}}"
|
||||
|
||||
@@ -97,16 +233,41 @@ if [[ -n "$HAS_BSKY" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
||||
# Normalise EXCLUDED by removing whitespace; case-insensitive matches below
|
||||
# mirror pipeline.py's .strip().lower() parsing without requiring sed/tr.
|
||||
SC_ADD=3
|
||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
||||
EXCLUDED_NORM="${EXCLUDED//[[:space:]]/}"
|
||||
if [[ ",$EXCLUDED_NORM," == *",[Tt][Ii][Kk][Tt][Oo][Kk],"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
if [[ ",$EXCLUDED_NORM," == *",[Ii][Nn][Ss][Tt][Aa][Gg][Rr][Aa][Mm],"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
||||
fi
|
||||
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Fully configured — compact ready message
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
if [[ -n "$LAST_RUN_LINE" ]]; then
|
||||
echo "$LAST_RUN_LINE"
|
||||
fi
|
||||
else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
if [[ -n "$LAST_RUN_LINE" ]]; then
|
||||
echo "$LAST_RUN_LINE"
|
||||
fi
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
|
||||
# The branches above end with `[[ -n "$LAST_RUN_LINE" ]] && echo ...`. When
|
||||
# LAST_RUN_LINE is empty, that test returns 1 and is the script's last command,
|
||||
# leaking exit=1 to callers (e.g. SessionStart hook drivers) despite no error.
|
||||
exit 0
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ module github.com/mvanhorn/last30days-skill/mcp
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/mark3labs/mcp-go v0.54.0
|
||||
require github.com/mark3labs/mcp-go v0.55.0
|
||||
|
||||
require (
|
||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mark3labs/mcp-go v0.54.0 h1:PZhQvd+5xrT43cUoiaKn/hDcvLUhcLc1twSEKYPTcTA=
|
||||
github.com/mark3labs/mcp-go v0.54.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/mark3labs/mcp-go v0.55.0 h1:lJfz2aoctiwK+sI991+uIYwmKNIBciI+O7zsyDsa4U8=
|
||||
github.com/mark3labs/mcp-go v0.55.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -137,6 +138,10 @@ func resolveTimeout(explicit time.Duration) time.Duration {
|
||||
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
// Accept bare integer seconds (e.g. "300") as documented.
|
||||
if secs, err := strconv.Atoi(raw); err == nil && secs > 0 {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
}
|
||||
return DefaultTimeout
|
||||
}
|
||||
|
||||
@@ -279,3 +279,22 @@ func TestResolveTimeoutHonorsEnv(t *testing.T) {
|
||||
t.Fatalf("explicit value not honored: got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTimeoutBareIntegerSeconds(t *testing.T) {
|
||||
t.Setenv(TimeoutEnvOverride, "300")
|
||||
if got := resolveTimeout(0); got != 300*time.Second {
|
||||
t.Fatalf("bare integer 300: got %v, want 5m0s", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "1")
|
||||
if got := resolveTimeout(0); got != 1*time.Second {
|
||||
t.Fatalf("bare integer 1: got %v, want 1s", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "0")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("bare integer 0: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "-1")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("bare integer -1: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
func registerPreflightTool(s *server.MCPServer, cfg Config) {
|
||||
s.AddTool(
|
||||
mcplib.NewTool("preflight",
|
||||
mcplib.WithDescription(
|
||||
"Safely summarize what last30days would read, write, execute, and contact "+
|
||||
"without running research, saving files, or reading browser cookies.",
|
||||
),
|
||||
mcplib.WithString("format", mcplib.Description("Output shape: 'text' (default) for a concise summary or 'json' for structured details.")),
|
||||
mcplib.WithReadOnlyHintAnnotation(true),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(false),
|
||||
),
|
||||
makePreflightHandler(cfg),
|
||||
)
|
||||
}
|
||||
|
||||
func makePreflightHandler(cfg Config) server.ToolHandlerFunc {
|
||||
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
|
||||
format, err := preflightFormatArgument(req.GetArguments())
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
src, err := engine.EngineFS()
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
|
||||
}
|
||||
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf(
|
||||
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
|
||||
err, engine.CacheEnvOverride,
|
||||
)), nil
|
||||
}
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
Args: preflightRunArgs(format),
|
||||
})
|
||||
if runErr != nil {
|
||||
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
|
||||
}
|
||||
return mcplib.NewToolResultText(string(res.Stdout)), nil
|
||||
}
|
||||
}
|
||||
|
||||
func preflightRunArgs(format string) []string {
|
||||
runArgs := []string{"--preflight", "--preflight-report-on-save-dir", mcpSaveDir()}
|
||||
if format == "json" {
|
||||
runArgs = append(runArgs, "--emit=json")
|
||||
}
|
||||
return runArgs
|
||||
}
|
||||
|
||||
func preflightFormatArgument(args map[string]any) (string, error) {
|
||||
raw, ok := args["format"]
|
||||
if !ok {
|
||||
return "text", nil
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return "", errors.New("format must be a string")
|
||||
}
|
||||
switch value {
|
||||
case "", "text":
|
||||
return "text", nil
|
||||
case "json":
|
||||
return "json", nil
|
||||
default:
|
||||
return "", fmt.Errorf("format must be 'text' or 'json', got %q", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreflightRunArgsDefaultTextIsSafe(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
|
||||
args := preflightRunArgs("text")
|
||||
want := []string{
|
||||
"--preflight",
|
||||
"--preflight-report-on-save-dir",
|
||||
"~/Documents/Last30Days",
|
||||
}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightRunArgsJSONIsSafeAndStructured(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
|
||||
args := preflightRunArgs("json")
|
||||
want := []string{
|
||||
"--preflight",
|
||||
"--preflight-report-on-save-dir",
|
||||
"/tmp/last30days-reports",
|
||||
"--emit=json",
|
||||
}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightFormatArgumentDefaultsAndValidates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"missing defaults to text", map[string]any{}, "text", false},
|
||||
{"empty defaults to text", map[string]any{"format": ""}, "text", false},
|
||||
{"text passes", map[string]any{"format": "text"}, "text", false},
|
||||
{"json passes", map[string]any{"format": "json"}, "json", false},
|
||||
{"invalid rejected", map[string]any{"format": "xml"}, "", true},
|
||||
{"non-string rejected", map[string]any{"format": true}, "", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := preflightFormatArgument(tc.args)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,11 @@
|
||||
// Package tools owns the MCP tool surface for last30days. Today there is
|
||||
// exactly one tool, research, mirroring the /last30days <topic> slash
|
||||
// command available in Claude Code. Adding new tools means another file
|
||||
// here plus an additional s.AddTool call in Register.
|
||||
// Package tools owns the MCP tool surface for last30days.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
@@ -25,6 +23,7 @@ type Config struct {
|
||||
// Register adds every tool this server exposes to s. The caller supplies a
|
||||
// Config so test harnesses can pin a version without touching globals.
|
||||
func Register(s *server.MCPServer, cfg Config) {
|
||||
registerPreflightTool(s, cfg)
|
||||
s.AddTool(
|
||||
mcplib.NewTool("research",
|
||||
mcplib.WithDescription(
|
||||
@@ -36,7 +35,7 @@ func Register(s *server.MCPServer, cfg Config) {
|
||||
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
|
||||
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
|
||||
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
|
||||
mcplib.WithReadOnlyHintAnnotation(true),
|
||||
mcplib.WithReadOnlyHintAnnotation(false),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(true),
|
||||
),
|
||||
@@ -74,10 +73,7 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
)), nil
|
||||
}
|
||||
|
||||
runArgs := []string{topic, "--emit=" + emit}
|
||||
if save {
|
||||
runArgs = append(runArgs, "--save")
|
||||
}
|
||||
runArgs := researchRunArgs(topic, emit, save)
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
@@ -90,6 +86,22 @@ func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func researchRunArgs(topic, emit string, save bool) []string {
|
||||
runArgs := []string{topic, "--emit=" + emit, "--no-browser-cookies"}
|
||||
if save {
|
||||
runArgs = append(runArgs, "--save-dir", mcpSaveDir())
|
||||
}
|
||||
return runArgs
|
||||
}
|
||||
|
||||
func mcpSaveDir() string {
|
||||
saveDir := os.Getenv("LAST30DAYS_MEMORY_DIR")
|
||||
if saveDir == "" {
|
||||
return "~/Documents/Last30Days"
|
||||
}
|
||||
return saveDir
|
||||
}
|
||||
|
||||
func requireString(args map[string]any, name string) (string, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
|
||||
@@ -97,6 +97,36 @@ func TestBoolArgument(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsIncludesNoBrowserCookies(t *testing.T) {
|
||||
args := researchRunArgs("OpenAI", "compact", false)
|
||||
want := []string{"OpenAI", "--emit=compact", "--no-browser-cookies"}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsSaveUsesSupportedSaveDir(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "")
|
||||
args := researchRunArgs("OpenAI", "html", true)
|
||||
got := strings.Join(args, "\x00")
|
||||
if strings.Contains(got, "--save\x00") || strings.HasSuffix(got, "--save") {
|
||||
t.Fatalf("args still include unsupported --save: %#v", args)
|
||||
}
|
||||
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "~/Documents/Last30Days"}
|
||||
if got != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchRunArgsSaveUsesMemoryDirEnvOverride(t *testing.T) {
|
||||
t.Setenv("LAST30DAYS_MEMORY_DIR", "/tmp/last30days-reports")
|
||||
args := researchRunArgs("OpenAI", "html", true)
|
||||
want := []string{"OpenAI", "--emit=html", "--no-browser-cookies", "--save-dir", "/tmp/last30days-reports"}
|
||||
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
|
||||
t.Fatalf("args = %#v, want %#v", args, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
|
||||
// Validation failures are returned as MCP tool errors (not Go errors)
|
||||
// so Claude sees a structured failure with a readable message rather
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
"manifest_version": "0.3",
|
||||
"name": "last30days-pp-mcp",
|
||||
"display_name": "Last30Days",
|
||||
"version": "3.0.0",
|
||||
"version": "3.6.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 8.6 MiB |
+7
-2
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.2.3"
|
||||
version = "3.17.0"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -8,7 +8,7 @@ dependencies = []
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9,<10",
|
||||
"pytest>=9.1.0,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
@@ -31,6 +31,11 @@ omit = [
|
||||
[tool.coverage.report]
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
# Coverage gate (issue #254). Floor intended to rise over time, not a ceiling.
|
||||
# Baseline measured 2026-07-03 on main before feat/hosted-api-mode
|
||||
# (source = scripts + tests): TOTAL 84.06%. Gate pinned at that baseline.
|
||||
# Do not lower without documenting why in the PR (see AGENTS.md Rules).
|
||||
fail_under = 84
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
The AI world reinvents itself every month. This skill keeps you current.
|
||||
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
|
||||
## v3 is the intelligent search release
|
||||
|
||||
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
|
||||
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
||||
|
||||
## Headline features
|
||||
|
||||
### Intelligent pre-research
|
||||
|
||||
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
||||
|
||||
### Best Takes
|
||||
|
||||
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
||||
|
||||
### Cross-source cluster merging
|
||||
|
||||
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
|
||||
### Single-pass comparisons
|
||||
|
||||
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
||||
|
||||
### GitHub person-mode and project-mode
|
||||
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
||||
|
||||
### ELI5 mode
|
||||
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
||||
|
||||
### 13+ sources
|
||||
|
||||
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
||||
|
||||
### Per-author cap and entity disambiguation
|
||||
|
||||
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
||||
|
||||
## Install
|
||||
|
||||
Claude Code:
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
OpenClaw:
|
||||
|
||||
```
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
## v3 Community
|
||||
|
||||
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
||||
|
||||
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
||||
|
||||
Contributors who shaped the release itself:
|
||||
|
||||
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
||||
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
||||
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
||||
|
||||
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
||||
|
||||
## Earlier contributors
|
||||
|
||||
From the v1 and v2 lineage:
|
||||
|
||||
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
||||
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
||||
|
||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Hermes scans from this skill directory, not the repository root.
|
||||
# Keep non-runtime packaging/dev/eval artifacts out of install-time security scans.
|
||||
assets/
|
||||
agents/
|
||||
scripts/build-skill.sh
|
||||
scripts/compare.sh
|
||||
scripts/evaluate_search_quality.py
|
||||
scripts/test_device_auth.py
|
||||
scripts/test-v1-vs-v2.sh
|
||||
scripts/verify_v3.py
|
||||
|
||||
# Vendored third-party X-search client (node_modules analog); excluded from scan, still installed.
|
||||
scripts/lib/vendor/
|
||||
+677
-160
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,16 @@
|
||||
# Save shareable HTML brief
|
||||
|
||||
This reference file is loaded by the main `SKILL.md` when the user asked for an HTML brief (either explicitly via `--emit=html` / `--emit:html` / `--html`, or in natural language - "give me a shareable HTML brief", "for Slack", "for Notion", "export as HTML", etc.). The detection happens in `SKILL.md` so that the common no-HTML path stays short; the implementation lives here.
|
||||
This reference file is loaded by the main `SKILL.md` when the user asked for an HTML brief (either through an HTML-looking prompt argument like `--emit=html` / `--emit:html` / `--html`, or in natural language - "give me a shareable HTML brief", "give it to me in HTML", "for Slack", "for Notion", "export as HTML", etc.). The detection happens in `SKILL.md` so that the common no-HTML path stays short; the implementation lives here. Those prompt arguments are user intent signals for the skill; they are not the full Python CLI contract.
|
||||
|
||||
The contract: the synthesis still appears in chat as the primary output. The HTML is an additional artifact saved to disk for sharing. Both happen in the same turn.
|
||||
The contract has two modes:
|
||||
|
||||
- **HTML as the requested deliverable** (`--emit=html`, `--emit:html`, `--html`, or prose like "give it to me in HTML"): the HTML artifact is the primary output. Write the synthesis to the temp file, render the HTML, then give a concise artifact handoff in chat instead of pasting the full Markdown report again.
|
||||
- **Normal report plus HTML copy** (the user asks for the normal report and also wants an HTML copy): the synthesis still appears in chat as the primary output. The HTML is an additional artifact saved to disk for sharing. Both happen in the same turn.
|
||||
|
||||
## When to fire this flow
|
||||
|
||||
- After you have already emitted the full chat response: badge, "What I learned:" (or comparison title), bold-lead-in paragraphs with citations, KEY PATTERNS list, engine footer pass-through, invitation block.
|
||||
- For normal-report-plus-HTML mode: after you have already emitted the full chat response: badge, "What I learned:" (or comparison title), bold-lead-in paragraphs with citations, KEY PATTERNS list, engine footer pass-through, invitation block.
|
||||
- For HTML-as-deliverable mode: after you have drafted the synthesis that will go into the HTML, before emitting the final chat response.
|
||||
- BEFORE the WAIT FOR USER'S RESPONSE pause.
|
||||
- ONLY if the user asked. Do NOT save HTML when the user didn't ask for it.
|
||||
|
||||
@@ -15,14 +19,19 @@ The contract: the synthesis still appears in chat as the primary output. The HTM
|
||||
```bash
|
||||
# 1. Write your synthesis prose VERBATIM to a temp file. The synthesis is the
|
||||
# "What I learned:" prose label, the bold-lead-in paragraphs with their
|
||||
# inline citations as you wrote them in chat, and the "KEY PATTERNS from
|
||||
# the research:" numbered list. Do NOT include the badge or the engine
|
||||
# footer in the temp file - the engine adds those when it renders the HTML.
|
||||
# Use the EXACT text you just wrote in chat. Do not paraphrase, do not
|
||||
# summarize, do not reorder. The HTML must read identically to the chat
|
||||
# response in voice and citations.
|
||||
# inline citations, and the "KEY PATTERNS from the research:" numbered list.
|
||||
# Do NOT include the badge or the engine footer in the temp file - the engine
|
||||
# adds those when it renders the HTML.
|
||||
# - HTML-as-deliverable mode: use the exact synthesis draft you prepared for
|
||||
# the artifact. Do not paste it to chat first.
|
||||
# - Normal-report-plus-HTML mode: use the exact synthesis text you already
|
||||
# wrote in chat.
|
||||
# In both modes, do not paraphrase, summarize, or reorder. The HTML must read
|
||||
# identically to the intended report in voice and citations.
|
||||
SYNTHESIS_FILE="/tmp/last30days-synthesis-${CLAUDE_SESSION_ID}.md"
|
||||
cat > "$SYNTHESIS_FILE" <<'SYNTHESIS_EOF'
|
||||
# >| not >: fixed path may already exist on a same-session re-run; a plain >
|
||||
# is refused under `set -o noclobber`.
|
||||
cat >| "$SYNTHESIS_FILE" <<'SYNTHESIS_EOF'
|
||||
What I learned:
|
||||
|
||||
**{First headline}** - {body with [name](url) inline citations}
|
||||
@@ -38,21 +47,118 @@ KEY PATTERNS from the research:
|
||||
SYNTHESIS_EOF
|
||||
|
||||
# 2. Convert the synthesis to a self-contained HTML file via the engine.
|
||||
# The engine reuses the cache from your earlier engine run (same topic
|
||||
# + plan), so this second invocation is typically <1s on cache hit.
|
||||
# REPLAY THE SAME SCOPE FLAGS as your original run (--plan, --hiring-signals,
|
||||
# resolved --x-handle/--subreddits/etc). On a same-topic follow-up, the
|
||||
# engine reuses the structured last-report cache at
|
||||
# ~/.config/last30days/last-report.json to build badge metadata and footer
|
||||
# without re-running source fetchers. That cache is intentionally short-lived
|
||||
# (default: one hour; tune with LAST30DAYS_REPORT_CACHE_TTL_SECONDS, or set
|
||||
# it to 0 to disable reuse). If the cache is stale, missing, or for a
|
||||
# different topic, stderr says "No matching cached report data" and the
|
||||
# engine falls back to a fresh run; the same scope flags keep that fallback
|
||||
# aligned with the synthesis body.
|
||||
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/^-//;s/-$//')
|
||||
HTML_PATH="${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html"
|
||||
# Collision guard: the `> "$HTML_PATH"` redirect below OVERWRITES - the engine
|
||||
# does NOT auto-date the brief (its date-suffix logic applies only to --save-dir
|
||||
# raw files, not to this redirected --emit=html stream). So if the clean name
|
||||
# already exists, date-suffix it here to avoid clobbering a prior brief.
|
||||
if [ -f "$HTML_PATH" ]; then
|
||||
HTML_PATH="${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief-$(date +%F).html"
|
||||
fi
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "${TOPIC}" \
|
||||
--emit=html \
|
||||
--synthesis-file "$SYNTHESIS_FILE" \
|
||||
> "$HTML_PATH"
|
||||
"${SCOPE_FLAGS[@]}" \
|
||||
>| "$HTML_PATH" # >| not >: noclobber-safe write to the collision-guarded path
|
||||
# where SCOPE_FLAGS is the same array you passed the first time, e.g.
|
||||
# SCOPE_FLAGS=(--hiring-signals --plan "$QUERY_PLAN_FILE" --x-handle=acme).
|
||||
# For a scoped --hiring-signals brief, --hiring-signals MUST be here too so
|
||||
# the footer reflects the jobs-scoped board, not a generic crawl.
|
||||
|
||||
# 3. Append ONE line to your already-emitted chat response, after the
|
||||
# invitation block. Use a paperclip emoji as a visible signal that an
|
||||
# artifact was produced:
|
||||
echo "📎 Shareable brief saved to $HTML_PATH"
|
||||
# 3. Finish with the artifact handoff described below. Do not print the saved
|
||||
# path from the shell block; the chat handoff is the single user-visible
|
||||
# completion message.
|
||||
```
|
||||
|
||||
## Optional hosted publishing
|
||||
|
||||
Only publish after the local HTML file has already been saved and the user chooses a publish option. The local HTML save is always first, and its absolute path is always shown before any publish/upload step.
|
||||
|
||||
Respect any existing user, project, or host preference for HTML publishing first. If the user already has a preferred publisher or internal sharing workflow, include that option. If multiple publishing options are available, show each as its own choice and include `ht-ml.app` as one option; label `ht-ml.app` as supporting optional password protection. If no preference exists, use `ht-ml.app` as the fallback publishing option.
|
||||
|
||||
Use this decision flow:
|
||||
|
||||
- Save the local HTML file.
|
||||
- Show the absolute saved path.
|
||||
- Then proactively present next-step choices:
|
||||
1. Open HTML file
|
||||
2. Publish to `<preferred/configured service>`; if `ht-ml.app` is shown, say password protection is available
|
||||
3. Done for now
|
||||
- Do not upload until the user chooses a publishing option.
|
||||
|
||||
When publishing to `ht-ml.app`, ask a second question:
|
||||
|
||||
- **Public link** - publish without a password.
|
||||
- **Password-protected link** - ask the user to type the shared password in free form, then publish with that password.
|
||||
|
||||
Before the `ht-ml.app` choice, tell the user that public pages may be crawled or indexed, and that password protection is available. If the user chooses password protection, use a unique shared password they provide for this report; do not use their own account password.
|
||||
|
||||
Agents should discover the current publishing mechanics for the selected service when needed, including by visiting the service site, rather than hard-coding detailed service-specific instructions in chat. For the built-in `ht-ml.app` path, the engine supports `--publish-html`; on the password-protected branch, pass the shared password through `LAST30DAYS_PUBLISH_PASSWORD` rather than command-line arguments.
|
||||
|
||||
When the user chooses the built-in `ht-ml.app` path, add `--publish-html` to the same `--emit=html` command. Use `--output "$HTML_PATH"` rather than shell redirection so the engine can write the `.publish.json` companion metadata next to the local HTML file. On the password-protected branch, set `LAST30DAYS_PUBLISH_PASSWORD` in the subprocess environment instead of passing `--publish-password` in the shell command.
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PUBLISH_PASSWORD="${PUBLISH_PASSWORD:-}" \
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "${TOPIC}" \
|
||||
--emit=html \
|
||||
--synthesis-file "$SYNTHESIS_FILE" \
|
||||
--output "$HTML_PATH" \
|
||||
--publish-html \
|
||||
"${SCOPE_FLAGS[@]}" \
|
||||
>/dev/null
|
||||
```
|
||||
|
||||
The hosted URL appears on stderr as `[last30days] Published HTML to https://...`. Confirm the result with the hosted URL. If the user chose password protection, also repeat the shared password they selected so they can send the URL and password together. The engine writes URL metadata to `<HTML_PATH>.publish.json`. The provider may return an `update_key`; treat it as secret. The engine deliberately does not write the update key to stdout, the HTML artifact, or `.publish.json` companion metadata.
|
||||
|
||||
## Chat handoff after saving
|
||||
|
||||
Use the mode that matches the request.
|
||||
|
||||
### HTML as the requested deliverable
|
||||
|
||||
When HTML is the requested deliverable - whether by `--emit=html`, `--emit:html`, `--html`, or natural-language phrasing - do **not** paste the full Markdown report back into chat after saving the artifact. The user asked for an HTML deliverable; repeating the Markdown makes the run feel like a normal report with an attachment bolted on.
|
||||
|
||||
Respond with a concise handoff that includes the next-step choices:
|
||||
|
||||
```text
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
|
||||
📎 Shareable brief saved to <absolute HTML path>
|
||||
|
||||
What do you want to do next?
|
||||
1. Open HTML file
|
||||
2. Publish to <available HTML publishing service> (<service-specific note, e.g. ht-ml.app supports optional password protection>)
|
||||
3. Done for now
|
||||
```
|
||||
|
||||
If the user chooses open, open the HTML file when the host can safely open local files, leave the saved-path line in chat, and add `Opened locally.` Let the host choose the correct OS-specific mechanism; do not print a menu of shell commands. If opening fails or the host is headless, do not treat that as a failed report; show the path and say the file is ready to open in a browser.
|
||||
|
||||
### Normal report plus HTML copy
|
||||
|
||||
When the user asked for a normal `/last30days` report and also asked for an HTML copy, keep the full chat synthesis and append this artifact block after the invitation:
|
||||
|
||||
```text
|
||||
📎 Shareable brief saved to <absolute HTML path>
|
||||
|
||||
What do you want to do next?
|
||||
1. Open HTML file
|
||||
2. Publish to <available HTML publishing service> (<service-specific note, e.g. ht-ml.app supports optional password protection>)
|
||||
3. Done for now
|
||||
```
|
||||
|
||||
If the user chooses open, open it when the host can safely open local files; otherwise the saved-path line is enough. Do not upload in this flow unless the user chooses a publishing option.
|
||||
|
||||
## What ends up in the HTML file
|
||||
|
||||
The engine's `--emit=html` renderer combines:
|
||||
@@ -72,15 +178,23 @@ Same flow when the topic is `X vs Y` (or `X vs Y vs Z`). The engine routes throu
|
||||
|
||||
## Follow-up turn
|
||||
|
||||
If the user runs `/last30days OpenClaw` normally, sees the synthesis in chat, and THEN says "save that as HTML" or "give me a shareable version" in a follow-up turn, do the same save flow on the synthesis you wrote in the previous turn. Do not re-research; the synthesis is already in the conversation history. Just write it to the temp file and call the engine with `--emit=html --synthesis-file`.
|
||||
If the user runs `/last30days OpenClaw` normally, sees the synthesis in chat, and THEN explicitly refers back to that visible synthesis ("save that as HTML", "make this shareable", "turn the above into HTML"), do the same save flow on the synthesis you wrote in the previous turn. Do not re-research; the synthesis is already in the conversation history. Just write it to the temp file and call the engine with `--emit=html --synthesis-file`, then use the normal-report-plus-HTML artifact block.
|
||||
|
||||
If the follow-up instead asks for a new HTML deliverable ("give it to me in HTML", `--emit=html`, `--html`) rather than referring back to an already-visible report, treat it as HTML-as-deliverable mode.
|
||||
|
||||
The engine will try to reuse `~/.config/last30days/last-report.json` for that second invocation when it is still within `LAST30DAYS_REPORT_CACHE_TTL_SECONDS` (default: one hour). If stderr says it is reusing cached report data, continue normally. If stderr says no matching cache exists, the cache may be stale; let the command finish only if you supplied the same scope flags as the original run. Otherwise stop and re-run with the original flags so the HTML footer does not describe a different dataset.
|
||||
|
||||
## What NOT to do
|
||||
|
||||
- Do NOT save HTML if the user didn't ask. The sparse mode (no synthesis) produces a thin file; not useful as a shareable.
|
||||
- Do NOT add content to the temp file beyond your synthesis prose. The badge / footer / colophon come from the engine.
|
||||
- Do NOT change the file path convention. `${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html` is the canonical location.
|
||||
- Do NOT silently overwrite an existing file without telling the user. If `$HTML_PATH` already exists from a prior run, the engine will pick a date-suffixed name (`{slug}-brief-YYYY-MM-DD.html`) automatically; just print whichever path the redirect produced.
|
||||
- Do NOT silently overwrite an existing file. The `--emit=html` output is written via a shell redirect (`>| "$HTML_PATH"`), which OVERWRITES the collision-guarded path — use `>|` not `>` because `set -o noclobber` refuses plain `>` when the file already exists. The collision guard in step 2 handles same-topic re-runs: if `{slug}-brief.html` already exists it date-suffixes to `{slug}-brief-YYYY-MM-DD.html`. Always report whichever path the redirect actually used in the chat handoff.
|
||||
- Do NOT include the data quality warning text in the temp file or in your final chat line. Warnings are an engine-stderr concern, not an artifact concern.
|
||||
- Do NOT publish, upload, or send the HTML to a third-party service as part of the local save flow.
|
||||
- Do NOT publish to any service merely because HTML was requested. Show the saved path and next-step choices first; publishing requires the user to choose a publish option.
|
||||
- Do NOT block a local HTML export on a hosting decision unless the user explicitly asked for a hosted URL.
|
||||
- Do NOT paste or store the `update_key` in chat, Markdown, HTML, raw output, or companion metadata.
|
||||
|
||||
## Edge cases
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ def generate_daily(since: str = None) -> dict:
|
||||
|
||||
# Extract top finding by engagement
|
||||
if findings:
|
||||
top = max(findings, key=lambda f: f.get("engagement_score", 0))
|
||||
top = max(findings, key=lambda f: f.get("engagement_score") or 0)
|
||||
topic_data["top_finding"] = {
|
||||
"title": top.get("source_title", ""),
|
||||
"source": top.get("source", ""),
|
||||
@@ -110,7 +110,7 @@ def generate_daily(since: str = None) -> dict:
|
||||
|
||||
top_overall = None
|
||||
if all_findings:
|
||||
top_overall = max(all_findings, key=lambda f: f.get("engagement_score", 0))
|
||||
top_overall = max(all_findings, key=lambda f: f.get("engagement_score") or 0)
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
@@ -172,8 +172,8 @@ def generate_weekly() -> dict:
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
this_engagement = sum(f.get("engagement_score", 0) for f in this_week)
|
||||
last_engagement = sum(f.get("engagement_score", 0) for f in last_week)
|
||||
this_engagement = sum(f.get("engagement_score") or 0 for f in this_week)
|
||||
last_engagement = sum(f.get("engagement_score") or 0 for f in last_week)
|
||||
|
||||
# Trend calculation
|
||||
if last_engagement > 0:
|
||||
@@ -188,7 +188,15 @@ def generate_weekly() -> dict:
|
||||
"this_week_engagement": this_engagement,
|
||||
"last_week_engagement": last_engagement,
|
||||
"engagement_change_pct": round(engagement_change, 1),
|
||||
"top_findings": this_week[:5], # Top 5 by engagement (already sorted)
|
||||
# get_new_findings returns first_seen DESC, so sort by engagement
|
||||
# before slicing — otherwise the digest headlines the most recent
|
||||
# items, not the highest-engagement ones (the daily path keys on
|
||||
# engagement too).
|
||||
"top_findings": sorted(
|
||||
this_week,
|
||||
key=lambda f: f.get("engagement_score") or 0,
|
||||
reverse=True,
|
||||
)[:5],
|
||||
})
|
||||
|
||||
result = {
|
||||
|
||||
@@ -28,7 +28,7 @@ echo ""
|
||||
# Run 1: public release
|
||||
echo "[1/2] Running /last30days (public release)..."
|
||||
echo " This takes 2-4 minutes..."
|
||||
claude -p --dangerously-skip-permissions "/last30days $TOPIC" > /dev/null 2>&1 || true
|
||||
claude -p "/last30days $TOPIC" > /dev/null 2>&1 || true
|
||||
RELEASE_FILE="$DIR/${SLUG}-raw.md"
|
||||
[ -f "$RELEASE_FILE" ] && echo " Done: $RELEASE_FILE" || echo " FAILED: no output file"
|
||||
echo ""
|
||||
@@ -39,7 +39,7 @@ sleep 30
|
||||
# Run 2: private beta
|
||||
echo "[2/2] Running /last30days-beta (private beta)..."
|
||||
echo " This takes 2-4 minutes..."
|
||||
claude -p --dangerously-skip-permissions "/last30days-beta $TOPIC" > /dev/null 2>&1 || true
|
||||
claude -p "/last30days-beta $TOPIC" > /dev/null 2>&1 || true
|
||||
BETA_FILE="$DIR/${SLUG}-raw-beta.md"
|
||||
[ -f "$BETA_FILE" ] && echo " Done: $BETA_FILE" || echo " FAILED: no output file"
|
||||
echo ""
|
||||
|
||||
@@ -20,6 +20,7 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from lib import env as envlib
|
||||
from lib import schema
|
||||
from lib.providers import GEMINI_FLASH_LITE
|
||||
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -43,8 +44,21 @@ def _load_default_topics() -> list[tuple[str, str]]:
|
||||
|
||||
DEFAULT_TOPICS = _load_default_topics()
|
||||
DEFAULT_SEARCH = ""
|
||||
DEFAULT_JUDGE_MODEL = "gemini-3.1-flash-lite-preview"
|
||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
EVAL_CREDENTIAL_ENV_KEYS = (
|
||||
"GOOGLE_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"SCRAPECREATORS_API_KEY",
|
||||
"BSKY_HANDLE",
|
||||
"BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN",
|
||||
"AUTH_TOKEN",
|
||||
"CT0",
|
||||
)
|
||||
|
||||
|
||||
def stable_item_key(item: dict[str, Any]) -> str:
|
||||
@@ -268,12 +282,30 @@ def get_judgments(
|
||||
) -> dict[str, int]:
|
||||
cache_file = output_dir / "judgments" / f"{slug}.json"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale_cache = False
|
||||
if cache_file.exists():
|
||||
payload = json.loads(cache_file.read_text())
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
# The cache key is the topic slug alone, but judgments are model-
|
||||
# specific. Only reuse the cache when it was produced by the same judge
|
||||
# model; otherwise re-judge, so a --judge-model change cannot return
|
||||
# stale grades that silently skew precision@k / nDCG. Caches written
|
||||
# before judge_model was recorded miss here and get refreshed once.
|
||||
if payload.get("judge_model") == judge_model:
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
stale_cache = True
|
||||
if not gemini_api_key or not items:
|
||||
if stale_cache:
|
||||
# Discarded a different-model cache but can't re-judge. Returning {}
|
||||
# scores every item as ungraded (zero precision@k / nDCG); say so
|
||||
# rather than letting the run report silently wrong numbers.
|
||||
sys.stderr.write(
|
||||
f"[Eval] Cached judgments for {slug!r} were graded by a different "
|
||||
f"judge model and no Gemini API key is set to re-judge; returning "
|
||||
f"no grades (metrics for this topic will be zero).\n"
|
||||
)
|
||||
return {}
|
||||
payload = call_gemini_judge(gemini_api_key, judge_model, build_judge_prompt(topic, query_type, items))
|
||||
payload["judge_model"] = judge_model
|
||||
cache_file.write_text(json.dumps(payload, indent=2))
|
||||
return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []}
|
||||
|
||||
@@ -288,19 +320,7 @@ def create_eval_env() -> dict[str, str]:
|
||||
"PYTHONUTF8": "1",
|
||||
"LAST30DAYS_CONFIG_DIR": "",
|
||||
}
|
||||
for key in (
|
||||
"GOOGLE_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"SCRAPECREATORS_API_KEY",
|
||||
"BSKY_HANDLE",
|
||||
"BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN",
|
||||
"AUTH_TOKEN",
|
||||
"CT0",
|
||||
):
|
||||
for key in EVAL_CREDENTIAL_ENV_KEYS:
|
||||
value = os.environ.get(key) or config.get(key)
|
||||
if value:
|
||||
passthrough[key] = value
|
||||
@@ -312,6 +332,12 @@ def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds:
|
||||
if not engine.exists():
|
||||
engine = repo_dir / "scripts" / "last30days.py"
|
||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
||||
# Current engines default to the stable agent export, while older revisions
|
||||
# used by the evaluator implicitly emit the raw report and do not recognize
|
||||
# --json-profile. Request raw explicitly whenever the checked-out engine
|
||||
# supports the selector.
|
||||
if not engine.exists() or "--json-profile" in engine.read_text(encoding="utf-8"):
|
||||
cmd.append("--json-profile=raw")
|
||||
if search:
|
||||
cmd.extend(["--search", search])
|
||||
if quick:
|
||||
@@ -329,7 +355,16 @@ def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds:
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"{repo_dir.name} failed for '{topic}' with exit {result.returncode}\n{result.stderr.strip()}")
|
||||
return json.loads(result.stdout)
|
||||
payload = json.loads(result.stdout)
|
||||
# Shape guard: the evaluator compares raw Report fields. If the engine
|
||||
# emitted the agent profile anyway (flag detection missed a future
|
||||
# spelling), fail loudly instead of scoring empty ranked_candidates.
|
||||
if "schema_version" in payload and "ranked_candidates" not in payload:
|
||||
raise RuntimeError(
|
||||
f"{repo_dir.name} emitted the agent JSON profile; the evaluator "
|
||||
"requires the raw Report (--json-profile=raw)."
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def create_worktree(rev: str) -> Path:
|
||||
|
||||
+2583
-129
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,290 @@
|
||||
"""arXiv research-paper source for last30days.
|
||||
|
||||
Shells out to ``arxiv-pp-cli`` (open Atom API, no auth) to surface recent
|
||||
research papers relevant to a topic. arXiv carries no engagement signal, so
|
||||
ranking leans on relevance (the CLI's own relevance sort plus token overlap)
|
||||
and recency.
|
||||
|
||||
Activation gate: this source is only available when ``arxiv-pp-cli`` is on
|
||||
PATH. ``pipeline.available_sources`` checks ``shutil.which`` before including
|
||||
``arxiv``. The functions below also detect the missing-binary case defensively.
|
||||
|
||||
Default-on safety (two gates, both required):
|
||||
1. Query construction. arXiv is queried with a *quoted* phrase and
|
||||
``--sort-by relevance``. Sorting by submitted-date instead returns the
|
||||
newest cs.* papers regardless of topic -- topic-blind noise.
|
||||
2. Recency cutoff. Entries older than ``RECENCY_DAYS`` are dropped. Research
|
||||
does not trend on a 30-day clock, so this window is wider than the social
|
||||
sources' 30 days; it keeps arXiv current while dropping stale keyword
|
||||
matches (e.g. a 2017 sports-statistics paper that an off-topic query like
|
||||
"Golden State Warriors" would otherwise surface).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import log, subproc
|
||||
from .relevance import token_overlap_relevance
|
||||
|
||||
|
||||
CLI_BIN = "arxiv-pp-cli"
|
||||
|
||||
# Per-depth result counts.
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 5,
|
||||
"default": 10,
|
||||
"deep": 20,
|
||||
}
|
||||
|
||||
# Recency window for arXiv specifically. Papers do not trend daily; a year keeps
|
||||
# the source current (the off-topic 2017 paper still drops) without discarding
|
||||
# the genuinely-relevant work from the last few months.
|
||||
RECENCY_DAYS = 365
|
||||
|
||||
SEARCH_TIMEOUT = 30
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("arXiv", msg, tty_only=False)
|
||||
|
||||
|
||||
def _is_available() -> bool:
|
||||
"""True when the arxiv-pp-cli binary is on PATH."""
|
||||
return shutil.which(CLI_BIN) is not None
|
||||
|
||||
|
||||
def _today() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _build_search_query(topic: str) -> str:
|
||||
"""Quote the topic so arXiv treats it as a phrase across all fields.
|
||||
|
||||
Inner double-quotes are stripped (arXiv has no phrase-escaping); the outer
|
||||
quotes plus ``all:`` give a phrase-scoped relevance search.
|
||||
"""
|
||||
return f'all:"{_clean_phrase(topic)}"'
|
||||
|
||||
|
||||
def _clean_phrase(topic: str) -> str:
|
||||
"""Strip quotes and collapse whitespace into a phrase for the query."""
|
||||
return " ".join(topic.replace('"', " ").split())
|
||||
|
||||
|
||||
def _build_search_args(topic: str, limit: int) -> List[str]:
|
||||
return [
|
||||
CLI_BIN,
|
||||
"query",
|
||||
"--search-query",
|
||||
_build_search_query(topic),
|
||||
"--sort-by",
|
||||
"relevance",
|
||||
"--max-results",
|
||||
str(limit),
|
||||
"--agent",
|
||||
]
|
||||
|
||||
|
||||
def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
|
||||
"""Invoke arxiv-pp-cli and parse the JSON envelope.
|
||||
|
||||
arXiv returns ``{"meta": ..., "results": {"entries": [...]}}``. This
|
||||
normalizes to ``{"results": [...entries...]}`` so the parse step sees a
|
||||
flat list, matching the other sources' shape. Never raises.
|
||||
"""
|
||||
if not _is_available():
|
||||
return {"results": [], "error": f"{CLI_BIN} not on PATH"}
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=timeout)
|
||||
except subproc.SubprocTimeout as exc:
|
||||
_log(f"Timeout: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
except FileNotFoundError as exc:
|
||||
_log(f"Binary missing: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
except OSError as exc:
|
||||
_log(f"Spawn failed: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
|
||||
if result.returncode != 0:
|
||||
snippet = (result.stderr or "").strip().splitlines()[:1]
|
||||
first = snippet[0] if snippet else f"exit {result.returncode}"
|
||||
_log(f"CLI exit {result.returncode}: {first}")
|
||||
return {"results": [], "error": first}
|
||||
|
||||
stdout = result.stdout or ""
|
||||
if not stdout.strip():
|
||||
return {"results": []}
|
||||
try:
|
||||
data = json.loads(stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
_log(f"JSON decode failed: {exc}")
|
||||
return {"results": [], "error": f"json decode: {exc}"}
|
||||
|
||||
return {"results": _extract_entries(data)}
|
||||
|
||||
|
||||
def _extract_entries(data: Any) -> List[Dict[str, Any]]:
|
||||
"""Pull the entries list out of arXiv's nested envelope.
|
||||
|
||||
Tolerates ``{"results": {"entries": [...]}}`` (current shape),
|
||||
``{"entries": [...]}``, and a bare list.
|
||||
"""
|
||||
if isinstance(data, list):
|
||||
return [e for e in data if isinstance(e, dict)]
|
||||
if isinstance(data, dict):
|
||||
results = data.get("results")
|
||||
if isinstance(results, dict):
|
||||
entries = results.get("entries")
|
||||
if isinstance(entries, list):
|
||||
return [e for e in entries if isinstance(e, dict)]
|
||||
if isinstance(results, list):
|
||||
return [e for e in results if isinstance(e, dict)]
|
||||
entries = data.get("entries")
|
||||
if isinstance(entries, list):
|
||||
return [e for e in entries if isinstance(e, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def search_arxiv(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search arXiv via arxiv-pp-cli using a quoted, relevance-sorted query.
|
||||
|
||||
Returns a dict with a flat ``results`` list of entry dicts. On failure,
|
||||
``results`` is empty and an ``error`` key carries a one-line description.
|
||||
"""
|
||||
if not topic or not topic.strip():
|
||||
return {"results": []}
|
||||
# A topic of only quote characters cleans to an empty phrase (all:""),
|
||||
# which is a topic-blind query; bail rather than search for nothing.
|
||||
if not _clean_phrase(topic):
|
||||
return {"results": []}
|
||||
limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
cmd = _build_search_args(topic, limit)
|
||||
_log(f"query '{topic}' (relevance, max={limit})")
|
||||
response = _run_cli(cmd, timeout=SEARCH_TIMEOUT)
|
||||
_log(f"found {len(response.get('results') or [])} entries")
|
||||
return response
|
||||
|
||||
|
||||
def _parse_published(published: Optional[str]) -> Optional[datetime]:
|
||||
"""Parse an arXiv ``published`` timestamp (ISO 8601, e.g.
|
||||
'2026-06-25T17:59:48Z') into an aware datetime. Returns None on failure."""
|
||||
if not published or not isinstance(published, str):
|
||||
return None
|
||||
text = published.strip().replace("Z", "+00:00")
|
||||
try:
|
||||
dt = datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def _alternate_url(entry: Dict[str, Any]) -> str:
|
||||
"""Return the human-facing abstract URL (rel=alternate), not the PDF."""
|
||||
links = entry.get("links")
|
||||
if isinstance(links, list):
|
||||
for link in links:
|
||||
if isinstance(link, dict) and link.get("rel") == "alternate":
|
||||
href = str(link.get("href") or "").strip()
|
||||
if href:
|
||||
return href
|
||||
# Fall back to the abstract URL derived from the entry id.
|
||||
entry_id = str(entry.get("id") or "").strip()
|
||||
if entry_id.startswith("http"):
|
||||
return entry_id
|
||||
return ""
|
||||
|
||||
|
||||
def _author_names(entry: Dict[str, Any]) -> List[str]:
|
||||
authors = entry.get("authors")
|
||||
out: List[str] = []
|
||||
if isinstance(authors, list):
|
||||
for a in authors:
|
||||
if isinstance(a, dict):
|
||||
name = str(a.get("name") or "").strip()
|
||||
if name:
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def parse_arxiv_response(
|
||||
response: Dict[str, Any],
|
||||
query: str = "",
|
||||
today: Optional[datetime] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Parse an arXiv envelope into normalized item dicts.
|
||||
|
||||
Applies the recency cutoff (drops entries older than ``RECENCY_DAYS`` and
|
||||
entries with an unparseable date) and computes a token-overlap relevance
|
||||
hint. Returns dicts ready for ``normalize._normalize_arxiv``.
|
||||
"""
|
||||
raw = response.get("results") if isinstance(response, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
|
||||
now = today or _today()
|
||||
items: List[Dict[str, Any]] = []
|
||||
for i, entry in enumerate(raw):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
title = " ".join(str(entry.get("title") or "").split()).strip()
|
||||
if not title:
|
||||
continue
|
||||
published = _parse_published(entry.get("published") or entry.get("updated"))
|
||||
if published is None:
|
||||
# No usable date -> cannot honor the recency contract; drop.
|
||||
continue
|
||||
age_days = (now - published).days
|
||||
# Allow a one-day grace on the future side: a paper announced later in
|
||||
# the same UTC day yields age_days == -1 (timedelta.days floors toward
|
||||
# negative); dropping it as "future" would discard the freshest work.
|
||||
if age_days > RECENCY_DAYS or age_days < -1:
|
||||
continue
|
||||
|
||||
summary = " ".join(str(entry.get("summary") or "").split()).strip()
|
||||
authors = _author_names(entry)
|
||||
url = _alternate_url(entry)
|
||||
|
||||
rank_decay = max(0.3, 1.0 - (i * 0.03))
|
||||
if query:
|
||||
content_score = token_overlap_relevance(query, f"{title} {summary}".strip())
|
||||
else:
|
||||
content_score = 0.5
|
||||
relevance = min(1.0, 0.6 * rank_decay + 0.4 * content_score)
|
||||
|
||||
primary_author = authors[0] if authors else ""
|
||||
author_label = primary_author
|
||||
if len(authors) > 1:
|
||||
author_label = f"{primary_author} et al."
|
||||
|
||||
items.append(
|
||||
{
|
||||
"id": str(entry.get("id") or url or f"AX{i + 1}"),
|
||||
"title": title,
|
||||
"url": url,
|
||||
"summary": summary,
|
||||
"author": author_label,
|
||||
"authors": authors,
|
||||
"date": published.date().isoformat(),
|
||||
"engagement": {},
|
||||
"relevance": round(relevance, 2),
|
||||
"why_relevant": (
|
||||
f"arXiv paper ({primary_author}, {published.date().isoformat()})"
|
||||
if primary_author
|
||||
else f"arXiv paper ({published.date().isoformat()})"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return items
|
||||
@@ -0,0 +1,607 @@
|
||||
"""Backend-chain descriptors with predicted selection (doctor, R4).
|
||||
|
||||
Chained sources declare their routing here ONCE — imported from the
|
||||
definitions ``lib/env.py`` already owns (chain order, pin var names) — and
|
||||
``resolve()`` turns side-effect-free probes into a truthful prediction of
|
||||
what the next run will do.
|
||||
|
||||
Two resolution modes:
|
||||
|
||||
- ``alternative`` (X, YouTube, web search): the pipeline tries genuinely
|
||||
interchangeable backends in a declared order. Resolution probes ALL
|
||||
candidates first, then picks (collect-then-pick): the first fully-usable
|
||||
backend wins the "will use" prediction; otherwise the best degraded
|
||||
candidate resolves with a warn tier; otherwise the source is an error
|
||||
carrying the highest-priority backend's prescription. Collecting before
|
||||
picking prevents an installed-but-unauthenticated preferred backend from
|
||||
shadowing a fully working fallback.
|
||||
|
||||
- ``conditional`` (Reddit): routing is per-query and outcome-dependent —
|
||||
public keyless composite by default, ScrapeCreators backfill only when
|
||||
results fall below the configured thinness floor (see the gating in
|
||||
``lib/pipeline.py``). No probe can pick one winner, so resolution renders
|
||||
honest conditional wording instead of an ``active_backend``. Reddit's
|
||||
internal keyless lanes (rss/listing/arctic/shreddit) are sub-probe detail
|
||||
inside the public composite, never chain entries.
|
||||
|
||||
``active_backend`` semantics: a PREDICTION — "the first backend the probes
|
||||
say the next run will try" — rendered as "will use". It is not an
|
||||
observation of what served a past run, and runtime failover can still
|
||||
diverge mid-run (a present-but-expired paid key passes a presence probe).
|
||||
|
||||
Paid lanes (xai, xquik, serper, and every other API-key backend, including
|
||||
ScrapeCreators) probe KEY PRESENCE ONLY: a dict lookup, never a network
|
||||
call or credential spend. Binary-backed lanes reuse the U1 dependency
|
||||
probe layer (``health.probe_dependency``) so a stale shim reads as BROKEN,
|
||||
not available (#692).
|
||||
|
||||
This module observes and predicts only. It must never alter which backend
|
||||
the pipeline actually uses; parity with the pipeline's pre-failover
|
||||
selection is asserted in ``tests/test_backend_descriptors.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from shutil import which
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from . import env, health, prescriptions
|
||||
|
||||
# Resolution modes.
|
||||
MODE_ALTERNATIVE = "alternative" # probe-ordered chain, first-usable wins
|
||||
MODE_CONDITIONAL = "conditional" # per-query routing; wording, never a winner
|
||||
|
||||
# Rollup tiers for a resolved chain (doctor maps these into its R1 table).
|
||||
TIER_OK = "ok"
|
||||
TIER_WARN = "warn"
|
||||
TIER_ERROR = "error"
|
||||
|
||||
# Web search backend order. grounding.web_search's auto branch owns the
|
||||
# runtime behavior (brave -> exa -> serper -> parallel -> keyless floor);
|
||||
# there is no importable constant there, so this declaration is guarded by
|
||||
# the grounding-auto parity test rather than an import.
|
||||
WEB_BACKEND_ORDER: Tuple[str, ...] = ("brave", "exa", "serper", "parallel", "keyless")
|
||||
|
||||
# YouTube backend order (pipeline: yt-dlp first, ScrapeCreators search
|
||||
# fallback when yt-dlp is absent or fails — see lib/pipeline.py).
|
||||
YOUTUBE_BACKEND_ORDER: Tuple[str, ...] = ("yt-dlp", "scrapecreators")
|
||||
|
||||
# Chain-failure fixes embed the registry's CLI forms (KTD 7): the command a
|
||||
# backend finding prescribes and the one doctor/quality-nudge render for the
|
||||
# same failure mode come from one entry and cannot drift.
|
||||
_SC_PRESCRIPTION = (
|
||||
"set SCRAPECREATORS_API_KEY (free 10,000-call signup: "
|
||||
f"{prescriptions.get('scrapecreators', 'key_missing').fix_cli})"
|
||||
)
|
||||
_X_COOKIES_PRESCRIPTION = (
|
||||
"run setup with browser-cookie consent: "
|
||||
f"{prescriptions.get('x', 'cookies_missing').fix_cli}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackendFinding:
|
||||
"""Side-effect-free probe outcome for one backend of a chained source.
|
||||
|
||||
``status`` uses the ``lib.health`` vocabulary (OK/DEGRADED/MISSING/
|
||||
BROKEN/TIMEOUT/ERROR). ``prescription`` is the fix when non-OK.
|
||||
``requires`` is the backend's requirement note for report rendering.
|
||||
"""
|
||||
|
||||
name: str
|
||||
status: str
|
||||
detail: str = ""
|
||||
prescription: str = ""
|
||||
requires: str = ""
|
||||
|
||||
@property
|
||||
def usable(self) -> bool:
|
||||
"""Fully or partially usable (OK/DEGRADED) — eligible for selection."""
|
||||
return self.status in (health.OK, health.DEGRADED)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackendSpec:
|
||||
"""One backend in a chain: name, probe, requirement note, paid flag.
|
||||
|
||||
``probe`` must be side-effect-free. When ``paid`` is True the probe is
|
||||
key-presence only: no subprocess, no network, no credential spend.
|
||||
"""
|
||||
|
||||
name: str
|
||||
requires: str
|
||||
probe: Callable[[Dict[str, Any]], "BackendFinding"]
|
||||
paid: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChainDescriptor:
|
||||
"""A chained source's declared routing: backends, mode, and pin knob."""
|
||||
|
||||
source: str
|
||||
mode: str
|
||||
backends: Tuple[BackendSpec, ...]
|
||||
pin_var: Optional[str] = None # env var pin (X, Reddit)
|
||||
pin_flag: Optional[str] = None # CLI flag pin (web: --web-backend)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackendResolution:
|
||||
"""Resolved routing for one chained source.
|
||||
|
||||
``active_backend`` is the will-use PREDICTION for alternative chains
|
||||
and always None for conditional mode (Reddit never gets a computed
|
||||
winner — ``conditional`` carries the honest wording instead).
|
||||
"""
|
||||
|
||||
source: str
|
||||
mode: str
|
||||
chain: List[str]
|
||||
findings: List[BackendFinding]
|
||||
active_backend: Optional[str] = None
|
||||
tier: str = TIER_OK
|
||||
pinned: bool = False
|
||||
pin: Optional[str] = None
|
||||
prescription: str = ""
|
||||
conditional: str = ""
|
||||
|
||||
@property
|
||||
def summary(self) -> str:
|
||||
"""One-line rendering: will-use prediction or conditional wording."""
|
||||
if self.mode == MODE_CONDITIONAL:
|
||||
return self.conditional
|
||||
if self.active_backend is None:
|
||||
line = f"no usable backend (chain: {' -> '.join(self.chain)})"
|
||||
if self.prescription:
|
||||
line += f"; fix: {self.prescription}"
|
||||
return line
|
||||
line = f"will use: {self.active_backend}"
|
||||
if self.pinned:
|
||||
line += f" (pinned via {self._pin_origin()})"
|
||||
return line
|
||||
|
||||
def _pin_origin(self) -> str:
|
||||
d = DESCRIPTORS.get(self.source)
|
||||
if d is None:
|
||||
return "pin"
|
||||
return d.pin_var or d.pin_flag or "pin"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probes. All side-effect-free; paid lanes are pure dict lookups.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _key_probe(name: str, key_var: str, requires: str, note: str = "") -> Callable:
|
||||
"""Key-presence probe for a paid API lane. Never touches the network."""
|
||||
|
||||
def probe(config: Dict[str, Any]) -> BackendFinding:
|
||||
if config.get(key_var):
|
||||
return BackendFinding(
|
||||
name=name,
|
||||
status=health.OK,
|
||||
detail=f"{key_var} present",
|
||||
requires=requires,
|
||||
)
|
||||
prescription = note or f"set {key_var} in ~/.config/last30days/.env"
|
||||
return BackendFinding(
|
||||
name=name,
|
||||
status=health.MISSING,
|
||||
detail=f"{key_var} not set",
|
||||
prescription=prescription,
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
return probe
|
||||
|
||||
|
||||
def _probe_bird(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""Bird = vendored X GraphQL client (node script) + browser-cookie creds.
|
||||
|
||||
Cookie presence is checked FIRST, mirroring ``env._x_backend_available``'s
|
||||
gating (``has_bird_creds and is_bird_installed()``): without cookies bird
|
||||
is unconfigured regardless of node/script state, and the fix is the
|
||||
cookie-consent flow — a broken node runtime must not turn an unconfigured
|
||||
backend into an error carrying a node prescription.
|
||||
"""
|
||||
from . import bird_x
|
||||
|
||||
requires = "X browser cookies (AUTH_TOKEN/CT0) + node"
|
||||
if not (config.get("AUTH_TOKEN") and config.get("CT0")):
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=health.MISSING,
|
||||
detail="X browser cookies (AUTH_TOKEN/CT0) not configured",
|
||||
prescription=_X_COOKIES_PRESCRIPTION,
|
||||
requires=requires,
|
||||
)
|
||||
if not bird_x.is_bird_installed():
|
||||
# Distinguish a missing/broken node runtime from a missing script.
|
||||
node = health.probe_dependency("node")
|
||||
if node.status != health.OK:
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=node.status,
|
||||
detail=node.detail,
|
||||
prescription=node.prescription,
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=health.MISSING,
|
||||
detail="vendored bird-search client not found",
|
||||
prescription="reinstall the skill (npx skills add . -g -y) to restore lib/vendor/bird-search",
|
||||
requires=requires,
|
||||
)
|
||||
node = health.probe_dependency("node")
|
||||
if node.status != health.OK:
|
||||
# Resolvable-but-broken node (stale shim) must not read as usable.
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=node.status,
|
||||
detail=node.detail,
|
||||
prescription=node.prescription,
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="bird",
|
||||
status=health.OK,
|
||||
detail="browser-cookie auth (AUTH_TOKEN/CT0) configured",
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
|
||||
def _probe_xurl(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""xurl = official X API v2 CLI (OAuth2). Free lane; LOCAL-ONLY probe.
|
||||
|
||||
Doctor's no-network guarantee forbids the live ``xurl whoami`` check
|
||||
(``xurl_x.is_available()`` — an authenticated X API call, reserved for
|
||||
research time). This probe keys on local evidence instead: the binary
|
||||
on PATH plus xurl's on-disk token store (~/.xurl). Stored credentials
|
||||
read as OK with an explicit "not live-verified" caveat; an unreadable
|
||||
token store is a typed ERROR (broken, not unconfigured).
|
||||
"""
|
||||
from . import xurl_x
|
||||
|
||||
requires = "xurl CLI installed + OAuth2 login"
|
||||
if which("xurl") is None:
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.MISSING,
|
||||
detail="xurl CLI not found on PATH",
|
||||
prescription="npm install -g xurl && xurl auth oauth2 login",
|
||||
requires=requires,
|
||||
)
|
||||
store_status, store_detail = xurl_x.stored_auth_status()
|
||||
if store_status == xurl_x.AUTH_OK:
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.OK,
|
||||
detail=(
|
||||
"installed; stored OAuth2 credentials present; "
|
||||
"auth not live-verified (no network)"
|
||||
),
|
||||
requires=requires,
|
||||
)
|
||||
if store_status == xurl_x.AUTH_ERROR:
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.ERROR,
|
||||
detail=store_detail,
|
||||
prescription="xurl auth oauth2 login",
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="xurl",
|
||||
status=health.MISSING,
|
||||
detail="xurl installed but not authenticated",
|
||||
prescription="xurl auth oauth2 login",
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
|
||||
def _probe_ytdlp(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""yt-dlp via the U1 dependency-probe layer (missing/broken/timeout)."""
|
||||
dep = health.probe_dependency("yt-dlp")
|
||||
return BackendFinding(
|
||||
name="yt-dlp",
|
||||
status=dep.status,
|
||||
detail=dep.detail,
|
||||
prescription=dep.prescription,
|
||||
requires="yt-dlp on the agent-subprocess PATH",
|
||||
)
|
||||
|
||||
|
||||
def _probe_web_keyless(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""The keyless web-search floor: works keyless, but degraded quality."""
|
||||
requires = "no key; suppressed on native-search hosts"
|
||||
if env.keyless_web_allowed(config):
|
||||
return BackendFinding(
|
||||
name="keyless",
|
||||
status=health.DEGRADED,
|
||||
detail="keyless search floor (no paid key; lower quality)",
|
||||
requires=requires,
|
||||
)
|
||||
return BackendFinding(
|
||||
name="keyless",
|
||||
status=health.MISSING,
|
||||
detail="keyless floor suppressed: host has native web search",
|
||||
prescription="",
|
||||
requires=requires,
|
||||
)
|
||||
|
||||
|
||||
def _probe_reddit_public(config: Dict[str, Any]) -> BackendFinding:
|
||||
"""Public keyless Reddit composite; internal lanes are sub-probe detail."""
|
||||
return BackendFinding(
|
||||
name="public",
|
||||
status=health.OK,
|
||||
detail="public keyless composite (lanes: rss, listing, arctic, shreddit)",
|
||||
requires="none (public endpoints)",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry: routing declared once, from env.py's definitions where they exist.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_X_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = {
|
||||
"xai": _key_probe("xai", "XAI_API_KEY", "XAI_API_KEY (xAI/Grok live search)"),
|
||||
"bird": _probe_bird,
|
||||
"xurl": _probe_xurl,
|
||||
"xquik": _key_probe("xquik", "XQUIK_API_KEY", "XQUIK_API_KEY (xquik.com)"),
|
||||
}
|
||||
_X_PAID = {"xai", "xquik"}
|
||||
|
||||
_WEB_PROBES: Dict[str, Callable[[Dict[str, Any]], BackendFinding]] = {
|
||||
"brave": _key_probe("brave", "BRAVE_API_KEY", "BRAVE_API_KEY"),
|
||||
"exa": _key_probe("exa", "EXA_API_KEY", "EXA_API_KEY"),
|
||||
"serper": _key_probe("serper", "SERPER_API_KEY", "SERPER_API_KEY"),
|
||||
"parallel": _key_probe("parallel", "PARALLEL_API_KEY", "PARALLEL_API_KEY"),
|
||||
"keyless": _probe_web_keyless,
|
||||
}
|
||||
_WEB_KEYED = {"brave", "exa", "serper", "parallel"}
|
||||
|
||||
_SC_SPEC = BackendSpec(
|
||||
name="scrapecreators",
|
||||
requires="SCRAPECREATORS_API_KEY",
|
||||
probe=_key_probe(
|
||||
"scrapecreators", "SCRAPECREATORS_API_KEY", "SCRAPECREATORS_API_KEY",
|
||||
note=_SC_PRESCRIPTION,
|
||||
),
|
||||
paid=True,
|
||||
)
|
||||
|
||||
DESCRIPTORS: Dict[str, ChainDescriptor] = {
|
||||
# X: chain order and pin var imported from env.py (single source of truth).
|
||||
"x": ChainDescriptor(
|
||||
source="x",
|
||||
mode=MODE_ALTERNATIVE,
|
||||
backends=tuple(
|
||||
BackendSpec(
|
||||
name=name,
|
||||
requires={
|
||||
"xai": "XAI_API_KEY (xAI/Grok live search)",
|
||||
"bird": "X browser cookies (AUTH_TOKEN/CT0) + node",
|
||||
"xurl": "xurl CLI installed + OAuth2 login",
|
||||
"xquik": "XQUIK_API_KEY (xquik.com)",
|
||||
}[name],
|
||||
probe=_X_PROBES[name],
|
||||
paid=name in _X_PAID,
|
||||
)
|
||||
for name in env.X_BACKEND_ORDER
|
||||
),
|
||||
pin_var=env.X_BACKEND_PIN_VAR,
|
||||
),
|
||||
"youtube": ChainDescriptor(
|
||||
source="youtube",
|
||||
mode=MODE_ALTERNATIVE,
|
||||
backends=(
|
||||
BackendSpec(
|
||||
name="yt-dlp",
|
||||
requires="yt-dlp on the agent-subprocess PATH",
|
||||
probe=_probe_ytdlp,
|
||||
),
|
||||
_SC_SPEC,
|
||||
),
|
||||
pin_var=None, # no YouTube pin knob exists
|
||||
),
|
||||
"web": ChainDescriptor(
|
||||
source="web",
|
||||
mode=MODE_ALTERNATIVE,
|
||||
backends=tuple(
|
||||
BackendSpec(
|
||||
name=name,
|
||||
requires=(f"{name.upper()}_API_KEY" if name in _WEB_KEYED
|
||||
else "no key; suppressed on native-search hosts"),
|
||||
probe=_WEB_PROBES[name],
|
||||
paid=name in _WEB_KEYED,
|
||||
)
|
||||
for name in WEB_BACKEND_ORDER
|
||||
),
|
||||
pin_var=None, # pinned per-run via --web-backend, not an env var
|
||||
pin_flag="--web-backend",
|
||||
),
|
||||
"reddit": ChainDescriptor(
|
||||
source="reddit",
|
||||
mode=MODE_CONDITIONAL,
|
||||
backends=(
|
||||
BackendSpec(
|
||||
name="public",
|
||||
requires="none (public endpoints)",
|
||||
probe=_probe_reddit_public,
|
||||
),
|
||||
_SC_SPEC,
|
||||
),
|
||||
pin_var=env.REDDIT_BACKEND_PIN_VAR,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_descriptor(source: str) -> ChainDescriptor:
|
||||
"""Return the declared routing descriptor for ``source`` (KeyError if none)."""
|
||||
return DESCRIPTORS[source]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def resolve(
|
||||
source: str,
|
||||
config: Dict[str, Any],
|
||||
pin: Optional[str] = None,
|
||||
) -> BackendResolution:
|
||||
"""Resolve a chained source's routing into a truthful prediction.
|
||||
|
||||
``pin`` is an explicit per-run pin (the ``--web-backend`` flag); it
|
||||
takes precedence over the descriptor's env pin var. ``"auto"``/None
|
||||
mean unpinned. Probing is side-effect-free and collect-then-pick.
|
||||
|
||||
Time budget: backends are probed sequentially, so a chain's budget is
|
||||
ADDITIVE across its backends — each binary-backed probe is bounded by
|
||||
``health.PROBE_TIMEOUT`` and paid/key lanes are dict lookups that cost
|
||||
nothing, giving a worst case of roughly (binary probes in the chain) x
|
||||
``health.PROBE_TIMEOUT``. Deliberately no intra-chain concurrency:
|
||||
probes are memoized per process and the worst case only occurs when
|
||||
multiple binaries are simultaneously hung.
|
||||
"""
|
||||
descriptor = get_descriptor(source)
|
||||
findings = [
|
||||
_run_probe(spec, config) for spec in descriptor.backends
|
||||
]
|
||||
if descriptor.mode == MODE_CONDITIONAL:
|
||||
return _resolve_conditional(descriptor, config, findings)
|
||||
return _resolve_alternative(descriptor, config, findings, pin)
|
||||
|
||||
|
||||
def _run_probe(spec: BackendSpec, config: Dict[str, Any]) -> BackendFinding:
|
||||
"""Run one probe, isolating failures so one bad probe can't blank a chain."""
|
||||
try:
|
||||
finding = spec.probe(config)
|
||||
except Exception as exc: # a probe bug must not take the report down
|
||||
finding = BackendFinding(
|
||||
name=spec.name,
|
||||
status=health.ERROR,
|
||||
detail=f"probe failed: {type(exc).__name__}: {exc}",
|
||||
requires=spec.requires,
|
||||
)
|
||||
if not finding.requires:
|
||||
finding.requires = spec.requires
|
||||
return finding
|
||||
|
||||
|
||||
def _resolve_alternative(
|
||||
descriptor: ChainDescriptor,
|
||||
config: Dict[str, Any],
|
||||
findings: List[BackendFinding],
|
||||
pin: Optional[str],
|
||||
) -> BackendResolution:
|
||||
names = [spec.name for spec in descriptor.backends]
|
||||
by_name = {f.name: f for f in findings}
|
||||
res = BackendResolution(
|
||||
source=descriptor.source,
|
||||
mode=MODE_ALTERNATIVE,
|
||||
chain=list(names),
|
||||
findings=findings,
|
||||
)
|
||||
|
||||
pin_name: Optional[str] = None
|
||||
if pin and pin not in ("auto", "none") and pin in by_name:
|
||||
pin_name = pin
|
||||
elif descriptor.pin_var:
|
||||
raw = (config.get(descriptor.pin_var) or "").lower()
|
||||
if raw in by_name:
|
||||
pin_name = raw
|
||||
|
||||
if pin_name:
|
||||
# A pin forces a single backend (no failover) — mirror
|
||||
# env.x_backend_chain's pin semantics exactly.
|
||||
res.pinned = True
|
||||
res.pin = pin_name
|
||||
finding = by_name[pin_name]
|
||||
if finding.status == health.OK:
|
||||
res.active_backend = pin_name
|
||||
res.tier = TIER_OK
|
||||
elif finding.status == health.DEGRADED:
|
||||
res.active_backend = pin_name
|
||||
res.tier = TIER_WARN
|
||||
else:
|
||||
res.tier = TIER_ERROR
|
||||
res.prescription = finding.prescription or (
|
||||
f"unpin {descriptor.pin_var or descriptor.pin_flag} or fix {pin_name}"
|
||||
)
|
||||
return res
|
||||
|
||||
# Collect-then-pick: first fully-usable wins; else best degraded; else
|
||||
# error carrying the highest-priority backend's prescription.
|
||||
for finding in findings:
|
||||
if finding.status == health.OK:
|
||||
res.active_backend = finding.name
|
||||
res.tier = TIER_OK
|
||||
return res
|
||||
for finding in findings:
|
||||
if finding.status == health.DEGRADED:
|
||||
res.active_backend = finding.name
|
||||
res.tier = TIER_WARN
|
||||
return res
|
||||
res.tier = TIER_ERROR
|
||||
res.prescription = findings[0].prescription if findings else ""
|
||||
return res
|
||||
|
||||
|
||||
def _reddit_sc_min_items(config: Dict[str, Any]) -> int:
|
||||
"""The thinness floor, parsed exactly as the pipeline parses it
|
||||
(lib/pipeline.py reddit fetch: int(... or 0), malformed -> 0)."""
|
||||
try:
|
||||
return int(config.get(env.REDDIT_SC_MIN_ITEMS_VAR) or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _resolve_conditional(
|
||||
descriptor: ChainDescriptor,
|
||||
config: Dict[str, Any],
|
||||
findings: List[BackendFinding],
|
||||
) -> BackendResolution:
|
||||
"""Reddit: render the real per-query semantics, never a computed winner."""
|
||||
res = BackendResolution(
|
||||
source=descriptor.source,
|
||||
mode=MODE_CONDITIONAL,
|
||||
chain=[spec.name for spec in descriptor.backends],
|
||||
findings=findings,
|
||||
active_backend=None, # conditional mode never picks a winner
|
||||
tier=TIER_OK, # the public keyless composite is always reachable
|
||||
)
|
||||
has_key = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||
raw_pin = (config.get(descriptor.pin_var) or "").lower() if descriptor.pin_var else ""
|
||||
pinned_sc = has_key and raw_pin == "scrapecreators"
|
||||
floor = _reddit_sc_min_items(config)
|
||||
|
||||
if pinned_sc:
|
||||
res.pinned = True
|
||||
res.pin = "scrapecreators"
|
||||
res.conditional = (
|
||||
f"ScrapeCreators primary (pinned via {descriptor.pin_var}); "
|
||||
"public keyless composite fallback"
|
||||
)
|
||||
return res
|
||||
|
||||
if has_key:
|
||||
if floor > 0:
|
||||
backfill = (
|
||||
f"ScrapeCreators backfill when results fall below the "
|
||||
f"{floor}-item floor"
|
||||
)
|
||||
else:
|
||||
backfill = "ScrapeCreators backfill when the free path returns nothing"
|
||||
res.conditional = f"public keyless composite (default); {backfill}"
|
||||
return res
|
||||
|
||||
res.conditional = "public keyless composite (default); no ScrapeCreators key for backfill"
|
||||
if raw_pin == "scrapecreators":
|
||||
# The pipeline ignores the pin without a key; say so honestly.
|
||||
res.conditional += (
|
||||
f" ({descriptor.pin_var} pin ignored: SCRAPECREATORS_API_KEY not set)"
|
||||
)
|
||||
return res
|
||||
@@ -9,14 +9,30 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log, subproc
|
||||
from . import env, health, http, log, subproc
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
# How many times to retry the bird-search subprocess when stdout is non-JSON
|
||||
# (typically an HTML anti-bot interstitial from Twitter's edge).
|
||||
MAX_JSON_DECODE_RETRIES = 2
|
||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
||||
|
||||
|
||||
def _leading_mentions(text: str) -> list:
|
||||
"""Leading-run @mention parse, shared with other X-shaped sources (xquik).
|
||||
|
||||
Thin wrapper over ``query.leading_mentions`` so bird and xquik share one
|
||||
implementation; kept here for existing call sites and tests.
|
||||
"""
|
||||
from .query import leading_mentions
|
||||
return leading_mentions(text)
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
@@ -54,7 +70,7 @@ def _has_injected_credentials() -> bool:
|
||||
|
||||
def _has_process_credentials() -> bool:
|
||||
"""Return True when AUTH_TOKEN/CT0 are present in process env."""
|
||||
return bool(os.environ.get("AUTH_TOKEN") and os.environ.get("CT0"))
|
||||
return bool(env.read_secret_env("AUTH_TOKEN") and env.read_secret_env("CT0"))
|
||||
|
||||
|
||||
def _subprocess_env() -> Dict[str, str]:
|
||||
@@ -71,6 +87,19 @@ def _log(msg: str):
|
||||
log.source_log("Bird", msg, tty_only=False)
|
||||
|
||||
|
||||
def classify_run_failure(detail: str) -> str:
|
||||
"""Map Bird's subprocess-only failure shapes to run outcome states."""
|
||||
text = detail.lower()
|
||||
if any(marker in text for marker in ("interstitial", "non-json", "invalid json")):
|
||||
return health.SCHEMA_DRIFT
|
||||
if any(
|
||||
marker in text
|
||||
for marker in ("cookie expired", "expired cookie", "unauthorized", "forbidden", "login required")
|
||||
):
|
||||
return health.AUTH_FAILED
|
||||
return http.classify_failure(message=detail)
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for X search.
|
||||
|
||||
@@ -109,6 +138,40 @@ def is_bird_authenticated() -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
_probe_cache: Optional[Optional[bool]] = "unset" # "unset" | True | False | None
|
||||
|
||||
|
||||
def probe_works(timeout: int = 8) -> Optional[bool]:
|
||||
"""Cheap runtime check that X auth actually returns data.
|
||||
|
||||
Returns True when a 1-result probe comes back without an error, False on a
|
||||
clear failure (auth error / generic search failure), and None when the
|
||||
result is inconclusive (network timeout) so callers can fail open and keep
|
||||
the static credential-presence status rather than reporting a false-down.
|
||||
Cached per process so repeated diagnose calls don't re-probe.
|
||||
"""
|
||||
global _probe_cache
|
||||
if _probe_cache != "unset":
|
||||
return _probe_cache # type: ignore[return-value]
|
||||
if not (_has_injected_credentials() or _has_process_credentials()):
|
||||
_probe_cache = False
|
||||
return False
|
||||
from datetime import datetime, timedelta, timezone
|
||||
since = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
# @x (the platform's own account) posts frequently, so a no-error response
|
||||
# means auth works even if this particular window is quiet.
|
||||
resp = _run_bird_search(f"from:x since:{since}", count=1, timeout=timeout)
|
||||
if isinstance(resp, dict) and resp.get("error"):
|
||||
err = str(resp.get("error")).lower()
|
||||
if "timed out" in err or "timeout" in err:
|
||||
_probe_cache = None # inconclusive — don't downgrade on a transient timeout
|
||||
return None
|
||||
_probe_cache = False
|
||||
return False
|
||||
_probe_cache = True
|
||||
return True
|
||||
|
||||
|
||||
def check_npm_available() -> bool:
|
||||
"""Check if npm is available (kept for API compatibility).
|
||||
|
||||
@@ -148,16 +211,14 @@ def get_bird_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"""Invoke the vendored bird-search.mjs subprocess once.
|
||||
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
Returns (result, error_dict). If error_dict is non-None, treat it as the
|
||||
final result and do not retry — those errors are terminal (timeout,
|
||||
spawn failure). If error_dict is None, the subprocess ran to completion
|
||||
and `result` is the SubprocResult; the caller decides whether to retry
|
||||
based on the result.stdout content.
|
||||
"""
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -184,9 +245,9 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
on_pid=_register,
|
||||
)
|
||||
except subproc.SubprocTimeout:
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
return None, {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
return None, {"error": str(e), "items": []}
|
||||
finally:
|
||||
if pid_holder:
|
||||
try:
|
||||
@@ -195,22 +256,88 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
return result, None
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
Retries the subprocess on JSON-decode failure (typically a Twitter
|
||||
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
|
||||
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
|
||||
errors (subprocess timeout, non-zero return code) are returned
|
||||
immediately without retry.
|
||||
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds (per attempt)
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
last_decode_error: Optional[str] = None
|
||||
|
||||
for attempt in range(MAX_JSON_DECODE_RETRIES):
|
||||
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
|
||||
if terminal_error is not None:
|
||||
return terminal_error
|
||||
|
||||
output = result.stdout.strip()
|
||||
if result.returncode != 0:
|
||||
if not output:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
# Windows/Node 24: the vendored Bird CLI uses native fetch (undici),
|
||||
# and calling process.exit() while keep-alive sockets are still
|
||||
# closing trips a libuv assertion -> non-zero exit code AFTER it has
|
||||
# already written a complete, valid JSON result to stdout. Trust
|
||||
# stdout when it has content; only treat a non-zero exit as a real
|
||||
# failure when stdout is empty.
|
||||
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
# Twitter's edge sometimes serves an HTML anti-bot interstitial
|
||||
# in place of JSON. Tag the failure shape so it's distinguishable
|
||||
# from "no results" in logs, then retry the subprocess.
|
||||
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
|
||||
attempt_num = attempt + 1
|
||||
log_msg = (
|
||||
f"Bird search returned non-JSON stdout "
|
||||
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
|
||||
f"first 80 chars: {output[:80]!r})"
|
||||
)
|
||||
last_decode_error = str(e)
|
||||
if attempt_num < MAX_JSON_DECODE_RETRIES:
|
||||
log.source_log(
|
||||
"X/bird",
|
||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
||||
tty_only=False,
|
||||
)
|
||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
||||
continue
|
||||
log.source_log("X/bird", log_msg, tty_only=False)
|
||||
return {
|
||||
"error": (
|
||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
||||
f"(likely Twitter anti-bot interstitial): {e}"
|
||||
),
|
||||
"items": [],
|
||||
}
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
# Defensive fallthrough — loop should always return above.
|
||||
return {
|
||||
"error": f"Bird search exhausted retries: {last_decode_error}",
|
||||
"items": [],
|
||||
}
|
||||
|
||||
|
||||
def search_x(
|
||||
@@ -273,9 +400,17 @@ def search_x(
|
||||
}
|
||||
candidates = [w for w in core_words if w not in low_signal]
|
||||
if candidates:
|
||||
# Keep an entity anchor (the first distinctive topic token) in the
|
||||
# retry so it can't collapse to a bare generic token like "compound"
|
||||
# and flood the X pool with off-topic noise. Add the strongest
|
||||
# (longest) distinctive token when it differs from the anchor;
|
||||
# otherwise query the anchor alone. Better to return 0 than to
|
||||
# over-broaden to an unanchored generic term.
|
||||
anchor = candidates[0]
|
||||
strongest = max(candidates, key=len)
|
||||
_log(f"0 results for '{core_topic}', retrying with strongest token '{strongest}'")
|
||||
query = f"{strongest} since:{from_date}"
|
||||
retry_terms = anchor if strongest == anchor else f"{anchor} {strongest}"
|
||||
_log(f"0 results for '{core_topic}', retrying anchored on '{retry_terms}'")
|
||||
query = f"{retry_terms} since:{from_date}"
|
||||
response = _run_bird_search(query, count, timeout)
|
||||
|
||||
return response
|
||||
@@ -289,12 +424,15 @@ def search_handles(
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search specific X handles for topic-related content.
|
||||
|
||||
Runs targeted Bird searches using `from:handle topic` syntax.
|
||||
Used in Phase 2 supplemental search after entity extraction.
|
||||
Pulls each handle's actual timeline via `from:handle since:` — the FROM
|
||||
lane (tweets BY the person), engagement-weighted downstream. The topic is
|
||||
used for relevance RANKING, never AND'd into the query: X search is literal,
|
||||
so `from:handle <their name>` only matched tweets where they wrote their own
|
||||
name and returned ~0. Used in Phase 2 after entity extraction.
|
||||
|
||||
Args:
|
||||
handles: List of X handles to search (without @)
|
||||
topic: Search topic (core subject), or None for unfiltered search
|
||||
topic: Search topic — used for relevance ranking only, not the query
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
count_per: Results to request per handle
|
||||
|
||||
@@ -305,10 +443,8 @@ def search_handles(
|
||||
|
||||
def _search_one_handle(handle: str) -> List[Dict[str, Any]]:
|
||||
handle = handle.lstrip("@")
|
||||
if core_topic:
|
||||
query = f"from:{handle} {core_topic} since:{from_date}"
|
||||
else:
|
||||
query = f"from:{handle} since:{from_date}"
|
||||
# Always unfiltered: pull the timeline, rank by topic relevance below.
|
||||
query = f"from:{handle} since:{from_date}"
|
||||
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -326,11 +462,14 @@ def search_handles(
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
if result.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
|
||||
output = result.stdout.strip()
|
||||
if result.returncode != 0:
|
||||
if not output:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
# Windows/Node 24: benign libuv assertion can cause non-zero exit
|
||||
# AFTER valid JSON is written to stdout. Trust stdout content.
|
||||
|
||||
if not output:
|
||||
return []
|
||||
|
||||
@@ -339,7 +478,11 @@ def search_handles(
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
return []
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
items = parse_bird_response(response, query=core_topic)
|
||||
# Log on success/empty too (not only on failure): a silent handle search
|
||||
# made the from: query look like it never ran and caused wrong diagnoses.
|
||||
_log(f"Searching: {query} -> {len(items)} results")
|
||||
return items
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
@@ -352,6 +495,77 @@ def search_handles(
|
||||
return all_items
|
||||
|
||||
|
||||
def search_mentions(
|
||||
handles: List[str],
|
||||
from_date: str,
|
||||
count_per: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search for tweets ABOUT/TO each handle — the mention lane.
|
||||
|
||||
Queries `@handle since:` (tweets that mention the account) and excludes the
|
||||
handle's OWN tweets (those belong to the FROM lane via search_handles), so
|
||||
this surfaces what OTHERS are saying about the person. Engagement-weighted
|
||||
downstream; deduped against the FROM lane by URL at normalize time.
|
||||
|
||||
Args:
|
||||
handles: List of X handles (without @)
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
count_per: Results to request per handle
|
||||
|
||||
Returns:
|
||||
List of raw item dicts (same format as parse_bird_response output).
|
||||
"""
|
||||
def _search_one(handle: str) -> List[Dict[str, Any]]:
|
||||
handle = handle.lstrip("@")
|
||||
query = f"@{handle} since:{from_date}"
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
query,
|
||||
"--count", str(count_per),
|
||||
"--json",
|
||||
]
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
||||
except subproc.SubprocTimeout:
|
||||
_log(f"Mention search timed out for @{handle}")
|
||||
return []
|
||||
except OSError as e:
|
||||
_log(f"Mention search error for @{handle}: {e}")
|
||||
return []
|
||||
if result.returncode != 0:
|
||||
_log(f"Mention search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return []
|
||||
try:
|
||||
response = json.loads(output)
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from mention search for @{handle}")
|
||||
return []
|
||||
items = parse_bird_response(response, query=None)
|
||||
# ABOUT lane = OTHERS mentioning the handle. Drop the handle's own tweets
|
||||
# (the FROM lane already covers those); identify by the status URL author.
|
||||
hl = handle.lower()
|
||||
# The Bird API may return either x.com or twitter.com permalinks, so
|
||||
# match both when excluding the handle's own tweets.
|
||||
def _is_own(url):
|
||||
u = (url or "").lower()
|
||||
return f"x.com/{hl}/status" in u or f"twitter.com/{hl}/status" in u
|
||||
about = [it for it in items if not _is_own(it.get("url"))]
|
||||
_log(f"Searching: {query} -> {len(about)} mentions")
|
||||
return about
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
all_items: List[Dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=min(5, len(handles))) as executor:
|
||||
futures = {executor.submit(_search_one, h): h for h in handles}
|
||||
for future in as_completed(futures):
|
||||
all_items.extend(future.result())
|
||||
return all_items
|
||||
|
||||
|
||||
def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
"""Parse Bird response to match xai_x output format.
|
||||
|
||||
@@ -427,11 +641,16 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
|
||||
engagement[key] = None
|
||||
|
||||
# Build normalized item
|
||||
text = str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500]
|
||||
item = {
|
||||
"id": f"X{i+1}",
|
||||
"text": str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500],
|
||||
"text": text,
|
||||
"url": url,
|
||||
"author_handle": author_handle.lstrip("@"),
|
||||
# Leading @mentions parsed from the post text identify who a reply is
|
||||
# directed at (X replies open with the target handle(s)). Used by the
|
||||
# interaction-signal classifier in rerank.
|
||||
"mentioned_handles": _leading_mentions(text),
|
||||
"date": date,
|
||||
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
|
||||
"why_relevant": "", # Bird doesn't provide relevance explanations
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
"""Bluesky search via AT Protocol (requires app password).
|
||||
|
||||
Uses bsky.social for auth and public.api.bsky.app for post search.
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
|
||||
Uses bsky.social for auth and api.bsky.app for post search (the canonical
|
||||
authenticated AppView). The previous default `public.api.bsky.app` is the
|
||||
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
|
||||
regardless of auth header (verified 2026-05-04). Override the search host
|
||||
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
|
||||
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
|
||||
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
|
||||
The createSession endpoint accepts main-account passwords too, but they're
|
||||
bad hygiene (no scope, can't revoke individually).
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
@@ -14,7 +23,64 @@ from typing import Any, Dict, List, Optional
|
||||
from . import http, log
|
||||
|
||||
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
||||
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
||||
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
|
||||
|
||||
|
||||
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
|
||||
|
||||
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
|
||||
.env file. The project's env.py loads .env into config but not into
|
||||
os.environ, so check both — same hybrid pattern as last30days.py for
|
||||
LAST30DAYS_STORE.
|
||||
|
||||
Hardens user-supplied host values against three common mis-configurations:
|
||||
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
|
||||
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
|
||||
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
|
||||
we log a warning and fall back to the default rather than building an
|
||||
invalid URL with an opaque downstream error.
|
||||
"""
|
||||
config = config or {}
|
||||
raw = (
|
||||
os.environ.get("BSKY_SEARCH_HOST")
|
||||
or config.get("BSKY_SEARCH_HOST")
|
||||
or _DEFAULT_BSKY_SEARCH_HOST
|
||||
)
|
||||
host = raw.strip().rstrip("/")
|
||||
# Strip embedded scheme so users who paste full URLs do not break the f-string.
|
||||
for prefix in ("https://", "http://"):
|
||||
if host.lower().startswith(prefix):
|
||||
host = host[len(prefix):]
|
||||
break
|
||||
if not host or "/" in host or " " in host:
|
||||
# Embedded path or whitespace remains — don't trust it. Default + log.
|
||||
if raw != _DEFAULT_BSKY_SEARCH_HOST:
|
||||
_log(
|
||||
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
|
||||
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
|
||||
)
|
||||
host = _DEFAULT_BSKY_SEARCH_HOST
|
||||
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
|
||||
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
|
||||
# with three hyphens at fixed positions).
|
||||
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
|
||||
|
||||
|
||||
def _validate_app_password_format(value) -> bool:
|
||||
"""Return True if value matches Bluesky's 19-char app-password format.
|
||||
|
||||
False for non-strings (None, int, list) so callers passing config dict
|
||||
values directly don't crash. Detect-but-not-gate: the createSession
|
||||
endpoint also accepts main-account passwords, so failing this check is
|
||||
a hygiene smell, not a hard error.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return bool(_APP_PASSWORD_RE.fullmatch(value))
|
||||
|
||||
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 15,
|
||||
@@ -30,7 +96,7 @@ _TOKEN_MAX_AGE_SECONDS = 5400 # 90 minutes (conservative, tokens last ~2 hours)
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
log.source_log("Bluesky", msg)
|
||||
log.source_log("Bluesky", msg, tty_only=False)
|
||||
|
||||
|
||||
def _create_session(handle: str, app_password: str) -> Optional[str]:
|
||||
@@ -92,14 +158,8 @@ def _reset_session_cache() -> None:
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for Bluesky search."""
|
||||
from .query import extract_core_subject
|
||||
_BSKY_NOISE = frozenset({
|
||||
'best', 'top', 'good', 'great', 'awesome',
|
||||
'latest', 'new', 'news', 'update', 'updates',
|
||||
'trending', 'hottest', 'popular', 'viral',
|
||||
'practices', 'features', 'recommendations', 'advice',
|
||||
})
|
||||
return extract_core_subject(topic, noise=_BSKY_NOISE)
|
||||
from .query import SOCIAL_NOISE, extract_core_subject
|
||||
return extract_core_subject(topic, noise=SOCIAL_NOISE)
|
||||
|
||||
|
||||
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||
@@ -144,6 +204,20 @@ def search_bluesky(
|
||||
if not handle or not app_password:
|
||||
return {"posts": [], "error": "Bluesky credentials not configured"}
|
||||
|
||||
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
|
||||
# form. createSession accepts main-account passwords too — but main
|
||||
# passwords have no scope (full account access), can't be revoked
|
||||
# individually, and rotating them breaks every service that holds them.
|
||||
# We warn but do not gate, matching the project's detect-don't-block
|
||||
# philosophy elsewhere.
|
||||
if not _validate_app_password_format(app_password):
|
||||
_log(
|
||||
"BSKY_APP_PASSWORD does not look like an app password "
|
||||
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
|
||||
"account password — those work but are bad hygiene. Generate "
|
||||
"an app password at https://bsky.app/settings/app-passwords"
|
||||
)
|
||||
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
@@ -155,7 +229,7 @@ def search_bluesky(
|
||||
"limit": str(min(count, 100)),
|
||||
"sort": "top",
|
||||
}
|
||||
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
||||
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
|
||||
|
||||
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
token = _create_session(handle, app_password)
|
||||
|
||||
@@ -23,6 +23,7 @@ before `ai_chat_model` so "gpt image 2" matches the image-gen category.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List, Optional, TypedDict
|
||||
|
||||
|
||||
@@ -126,6 +127,8 @@ CATEGORY_PEERS: dict[str, _CategoryEntry] = {
|
||||
},
|
||||
"ai_agent_framework": {
|
||||
"patterns": [
|
||||
"ai agent",
|
||||
"ai agents",
|
||||
"agent framework",
|
||||
"agentic framework",
|
||||
"langchain",
|
||||
@@ -264,7 +267,10 @@ def detect_category(topic: Optional[str]) -> Optional[str]:
|
||||
lowered = topic.lower()
|
||||
for category_id, entry in CATEGORY_PEERS.items():
|
||||
for pattern in entry["patterns"]:
|
||||
if pattern in lowered:
|
||||
# Word-boundary match: "ai agent" must not fire on "Dubai agents"
|
||||
# or "Thai agents". Substring matching classified those as
|
||||
# ai_agent_framework and routed discovery to LangChain subreddits.
|
||||
if re.search(rf"(?<![a-z0-9]){re.escape(pattern)}(?![a-z0-9])", lowered):
|
||||
return category_id
|
||||
return None
|
||||
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
"""Chrome cookie extraction for macOS.
|
||||
"""Chromium-family cookie extraction for macOS.
|
||||
|
||||
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
||||
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
||||
Extracts cookies from Chromium-based browser SQLite databases using only
|
||||
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
||||
dependencies.
|
||||
|
||||
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
Every Chromium-based browser (Chrome, Brave, Edge, Vivaldi, Opera, Arc,
|
||||
Chromium) shares the same algorithm; only the profile directory and Keychain
|
||||
service name differ, so they all run through the same decryption core.
|
||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
@@ -18,10 +23,35 @@ from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Chrome cookie DB location on macOS
|
||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||
|
||||
# Chrome v10 encryption constants
|
||||
def _lock_temp_cookie_copy(path: str) -> None:
|
||||
"""Restrict copied cookie DB temp files to the current user on POSIX."""
|
||||
if os.name == "nt":
|
||||
return
|
||||
Path(path).chmod(0o600)
|
||||
|
||||
# Cookie DB locations on macOS
|
||||
_APP_SUPPORT = Path.home() / "Library" / "Application Support"
|
||||
CHROME_BASE_DIR = _APP_SUPPORT / "Google" / "Chrome"
|
||||
# Kept for backward compatibility; resolution now goes through the profile
|
||||
# finder (which also handles the modern Network/Cookies layout).
|
||||
CHROME_COOKIES_DB = CHROME_BASE_DIR / "Default" / "Cookies"
|
||||
BRAVE_BASE_DIR = _APP_SUPPORT / "BraveSoftware" / "Brave-Browser"
|
||||
|
||||
# Other Chromium-based browsers, keyed by FROM_BROWSER name. Each maps to
|
||||
# (profile base directory, macOS Keychain service name). Chrome and Brave keep
|
||||
# their dedicated helpers below for backward compatibility; everything here is
|
||||
# resolved generically by extract_chromium_browser_cookies_macos(). Keychain
|
||||
# service names follow Chromium's "<Browser> Safe Storage" convention.
|
||||
CHROMIUM_BROWSER_PROFILES: dict[str, tuple[Path, str]] = {
|
||||
"edge": (_APP_SUPPORT / "Microsoft Edge", "Microsoft Edge Safe Storage"),
|
||||
"vivaldi": (_APP_SUPPORT / "Vivaldi", "Vivaldi Safe Storage"),
|
||||
"opera": (_APP_SUPPORT / "com.operasoftware.Opera", "Opera Safe Storage"),
|
||||
"arc": (_APP_SUPPORT / "Arc" / "User Data", "Arc Safe Storage"),
|
||||
"chromium": (_APP_SUPPORT / "Chromium", "Chromium Safe Storage"),
|
||||
}
|
||||
|
||||
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
||||
CHROME_SALT = b"saltysalt"
|
||||
CHROME_PBKDF2_ITERATIONS = 1003
|
||||
CHROME_KEY_LENGTH = 16
|
||||
@@ -29,8 +59,8 @@ CHROME_KEY_LENGTH = 16
|
||||
CHROME_IV_HEX = "20" * 16
|
||||
|
||||
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
||||
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
|
||||
|
||||
Calls `security find-generic-password` which may trigger a system dialog
|
||||
on first access.
|
||||
@@ -39,30 +69,34 @@ def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
||||
["security", "find-generic-password", "-w", "-s", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
||||
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
|
||||
return None
|
||||
passphrase = result.stdout.strip()
|
||||
if not passphrase:
|
||||
logger.info("Chrome Keychain returned empty passphrase")
|
||||
logger.info("%s Keychain returned empty passphrase", service_name)
|
||||
return None
|
||||
return passphrase.encode("utf-8")
|
||||
except FileNotFoundError:
|
||||
logger.info("'security' command not found — not on macOS?")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.info("Chrome Keychain access timed out")
|
||||
logger.info("%s Keychain access timed out", service_name)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Failed to get Chrome encryption key: %s", e)
|
||||
logger.info("Failed to get %s encryption key: %s", service_name, e)
|
||||
return None
|
||||
|
||||
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
return _get_chromium_encryption_key("Chrome Safe Storage")
|
||||
|
||||
|
||||
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||
return hashlib.pbkdf2_hmac(
|
||||
@@ -165,36 +199,43 @@ def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS.
|
||||
def _extract_chromium_cookies_macos(
|
||||
db_path: Path,
|
||||
keychain_service: str,
|
||||
domain: str,
|
||||
cookie_names: list[str],
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from any Chromium-based browser on macOS.
|
||||
|
||||
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||
|
||||
Args:
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
||||
cookie_names: List of cookie names to extract
|
||||
db_path: Path to the browser's Cookies SQLite file.
|
||||
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
|
||||
Returns:
|
||||
Dict mapping cookie name to decrypted value, or None on failure.
|
||||
Only includes cookies that were successfully found and decrypted.
|
||||
"""
|
||||
if not CHROME_COOKIES_DB.exists():
|
||||
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
||||
if not db_path.exists():
|
||||
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
||||
return None
|
||||
|
||||
# Get encryption key from Keychain
|
||||
passphrase = _get_chrome_encryption_key()
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
|
||||
# Copy DB to temp file (Chrome locks the original)
|
||||
# Copy DB to temp file (browser locks the original while running)
|
||||
tmp_fd = None
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
||||
# mkstemp creates the file 0600. copy2 would copy the source DB's
|
||||
# permission bits onto the temp file before the chmod below runs,
|
||||
# briefly exposing live cookies when the source DB is looser.
|
||||
shutil.copyfile(str(db_path), tmp_path)
|
||||
_lock_temp_cookie_copy(tmp_path)
|
||||
except Exception as e:
|
||||
logger.info("Failed to copy Chrome cookies database: %s", e)
|
||||
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
||||
if tmp_path:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
@@ -211,27 +252,34 @@ def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Option
|
||||
cursor = conn.cursor()
|
||||
|
||||
db_version = _get_db_version(cursor)
|
||||
logger.debug("Chrome cookie DB version: %d", db_version)
|
||||
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
|
||||
|
||||
# Build query with placeholders for cookie names
|
||||
placeholders = ",".join("?" for _ in cookie_names)
|
||||
query = (
|
||||
f"SELECT name, value, encrypted_value FROM cookies "
|
||||
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||
)
|
||||
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
||||
params = [f"%{domain}"] + list(cookie_names)
|
||||
cursor.execute(query, params)
|
||||
|
||||
results: dict[str, str] = {}
|
||||
aes_key = None
|
||||
key_fetched = False
|
||||
for name, value, encrypted_value in cursor.fetchall():
|
||||
# Prefer unencrypted value if present
|
||||
if value:
|
||||
results[name] = value
|
||||
continue
|
||||
|
||||
# Handle encrypted value
|
||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||
if not key_fetched:
|
||||
# Fetch the Keychain key lazily — only once we actually have
|
||||
# an encrypted cookie to decrypt. This avoids a macOS
|
||||
# Keychain prompt for browsers that don't hold the requested
|
||||
# cookie, which matters for FROM_BROWSER=auto across several
|
||||
# installed Chromium browsers.
|
||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
key_fetched = True
|
||||
if aes_key is None:
|
||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||
continue
|
||||
@@ -241,25 +289,129 @@ def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Option
|
||||
else:
|
||||
logger.debug("Failed to decrypt cookie %s", name)
|
||||
elif encrypted_value:
|
||||
# Unknown encryption version
|
||||
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||
|
||||
conn.close()
|
||||
|
||||
if not results:
|
||||
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
||||
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
|
||||
return None
|
||||
|
||||
return results
|
||||
|
||||
except sqlite3.Error as e:
|
||||
logger.info("Failed to read Chrome cookies database: %s", e)
|
||||
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
||||
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS.
|
||||
|
||||
Resolves the cookie DB through the shared profile finder so Chrome gets the
|
||||
same modern ``Default/Network/Cookies`` (Chromium >= 96) and legacy
|
||||
``Default/Cookies`` probing as the rest of the Chromium family.
|
||||
"""
|
||||
db_path = _find_chromium_cookies_db(CHROME_BASE_DIR)
|
||||
if db_path is None:
|
||||
logger.info("Chrome cookies database not found under %s", CHROME_BASE_DIR)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(
|
||||
db_path, "Chrome Safe Storage", domain, cookie_names
|
||||
)
|
||||
|
||||
|
||||
def _profile_cookie_db(profile_dir: Path) -> Optional[Path]:
|
||||
"""Return the Cookies DB inside a profile dir, or None.
|
||||
|
||||
Prefers the modern ``Network/Cookies`` location (Chromium >= 96 moved the
|
||||
cookie store into a per-profile ``Network/`` subdirectory) and falls back
|
||||
to the legacy flat ``Cookies`` file. Different browsers and versions use
|
||||
different layouts, so both are probed.
|
||||
"""
|
||||
for rel in ("Network/Cookies", "Cookies"):
|
||||
candidate = profile_dir / rel
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _find_chromium_cookies_db(base_dir: Path) -> Optional[Path]:
|
||||
"""Find a Chromium-based browser's Cookies database under base_dir.
|
||||
|
||||
Checks the Default profile first, then the base dir itself (Opera's flat
|
||||
layout), then numbered "Profile N" directories by most-recently-modified.
|
||||
Each location is probed for both the modern ``Network/Cookies`` and legacy
|
||||
``Cookies`` paths (see _profile_cookie_db). Chromium browsers create extra
|
||||
profiles as "Profile 1", "Profile 2", etc. alongside Default; the most
|
||||
recently used one is the likeliest to hold current cookies. Lexicographic
|
||||
sort would visit "Profile 10" before "Profile 2", which can return the
|
||||
wrong profile, so we sort by mtime.
|
||||
"""
|
||||
found = _profile_cookie_db(base_dir / "Default")
|
||||
if found:
|
||||
return found
|
||||
|
||||
found = _profile_cookie_db(base_dir)
|
||||
if found:
|
||||
return found
|
||||
|
||||
try:
|
||||
candidates = [
|
||||
child for child in base_dir.iterdir()
|
||||
if child.is_dir() and child.name.startswith("Profile ")
|
||||
]
|
||||
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||
found = _profile_cookie_db(child)
|
||||
if found:
|
||||
return found
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _find_brave_cookies_db() -> Optional[Path]:
|
||||
"""Find Brave's Cookies database on macOS (Default, then Profile N)."""
|
||||
return _find_chromium_cookies_db(BRAVE_BASE_DIR)
|
||||
|
||||
|
||||
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Brave on macOS.
|
||||
|
||||
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
||||
path and Keychain service name differ.
|
||||
"""
|
||||
db_path = _find_brave_cookies_db()
|
||||
if db_path is None:
|
||||
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_chromium_browser_cookies_macos(
|
||||
browser: str, domain: str, cookie_names: list[str]
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from a registry-defined Chromium browser on macOS.
|
||||
|
||||
Covers every browser in CHROMIUM_BROWSER_PROFILES (Edge, Vivaldi, Opera,
|
||||
Arc, Chromium). They all reuse Chrome's v10 AES-128-CBC encryption; only
|
||||
the profile directory and Keychain service name differ.
|
||||
"""
|
||||
spec = CHROMIUM_BROWSER_PROFILES.get(browser)
|
||||
if spec is None:
|
||||
logger.debug("Unknown Chromium browser: %s", browser)
|
||||
return None
|
||||
base_dir, keychain_service = spec
|
||||
db_path = _find_chromium_cookies_db(base_dir)
|
||||
if db_path is None:
|
||||
logger.info("%s cookies database not found under %s", keychain_service, base_dir)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(db_path, keychain_service, domain, cookie_names)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""CJK-aware tokenization for relevance scoring and near-duplicate detection.
|
||||
|
||||
The skill ships with zero hard dependencies (pyproject ``dependencies = []``)
|
||||
so it installs across 50+ Agent Skills hosts as plain Python. Chinese text has
|
||||
no whitespace word boundaries, so the original ``str.split()`` tokenizers in
|
||||
relevance.py / dedupe.py collapse a whole sentence into a single token and
|
||||
break token-overlap scoring and Jaccard de-duplication for Chinese sources
|
||||
(Xiaohongshu, Bilibili).
|
||||
|
||||
``segment(text)`` fixes that. It splits text into maximal CJK and non-CJK runs:
|
||||
|
||||
- Non-CJK (ASCII / Latin) runs keep the original ``\\w+`` word behaviour.
|
||||
- CJK runs are routed through jieba when it is installed (best quality), and
|
||||
fall back to character bigrams when jieba is absent. Bigrams are a
|
||||
dictionary-free segmentation that still gives robust overlap signal — e.g.
|
||||
query "大模型" -> {大模, 模型} overlaps text "国产大模型评测" -> {..大模, 模型..}.
|
||||
|
||||
jieba stays OPTIONAL: present -> used; absent -> bigram fallback. We never add
|
||||
it to the hard dependency set, preserving the install-anywhere property.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
# CJK ideographs + Japanese kana + Korean hangul. The Chinese ideograph block
|
||||
# (一-鿿) and its extension-A (㐀-䶿) cover the cases we care
|
||||
# about; kana/hangul are included so mixed-language text degrades gracefully.
|
||||
_CJK_CHARS = r"㐀-䶿一-鿿豈--ヿ가-"
|
||||
_CJK_RE = re.compile(f"[{_CJK_CHARS}]")
|
||||
_CJK_RUN_RE = re.compile(f"[{_CJK_CHARS}]+")
|
||||
_LATIN_RE = re.compile(r"\w+")
|
||||
|
||||
# High-frequency Chinese function words that dilute overlap signal, mirroring
|
||||
# the role of the English STOPWORDS sets in relevance.py / dedupe.py.
|
||||
CHINESE_STOPWORDS = frozenset(
|
||||
{
|
||||
"的", "了", "和", "是", "在", "我", "有", "也", "就", "不", "人", "都",
|
||||
"一", "一个", "上", "很", "到", "说", "要", "去", "你", "会", "着",
|
||||
"没有", "看", "好", "自己", "这", "那", "这个", "那个", "什么", "怎么",
|
||||
"为什么", "以及", "或者", "但是", "因为", "所以", "如果", "可以",
|
||||
"这样", "那样", "他们", "我们", "你们", "它", "她", "他", "吗", "呢",
|
||||
"吧", "啊", "哦", "嗯", "与", "及", "等", "被", "把", "让", "给", "向",
|
||||
"还", "再", "又", "从", "对", "为", "以", "之", "其", "中",
|
||||
}
|
||||
)
|
||||
|
||||
# Optional jieba, resolved once at import time. Binding it here (rather than
|
||||
# lazily on first use) avoids a race: the pipeline scores relevance inside a
|
||||
# ThreadPoolExecutor, so a lazy initializer with mutable globals could have two
|
||||
# threads import concurrently and observe a half-initialized state. Doing it at
|
||||
# module load means the binding is settled before any worker thread runs.
|
||||
#
|
||||
# The BROAD `except Exception` is intentional: jieba is an optional enhancement,
|
||||
# so ANY failure to load it — package absent, corrupted install, missing data
|
||||
# files, or a setLogLevel signature change across versions — must degrade to the
|
||||
# bigram fallback, never crash the skill. jieba guards its own first-call
|
||||
# dictionary build with an internal lock, so concurrent `cut()` is safe once the
|
||||
# module object is bound.
|
||||
try:
|
||||
import jieba as _jieba # type: ignore
|
||||
|
||||
_jieba.setLogLevel(60) # silence dictionary-build chatter on stderr
|
||||
except Exception:
|
||||
_jieba = None
|
||||
|
||||
|
||||
def has_cjk(text: str) -> bool:
|
||||
"""True if the text contains any CJK / kana / hangul character."""
|
||||
return bool(text) and _CJK_RE.search(text) is not None
|
||||
|
||||
|
||||
def _cjk_tokens(run: str) -> List[str]:
|
||||
# Reads the module-global _jieba at call time, so tests can force the bigram
|
||||
# path deterministically by setting cjk._jieba = None regardless of whether
|
||||
# jieba is installed in the environment.
|
||||
if _jieba is not None:
|
||||
return [w for w in _jieba.cut(run) if w.strip() and _CJK_RE.search(w)]
|
||||
# Dictionary-free fallback: character bigrams (single char if run length 1).
|
||||
if len(run) <= 1:
|
||||
return [run] if run else []
|
||||
return [run[i:i + 2] for i in range(len(run) - 1)]
|
||||
|
||||
|
||||
def segment(text: str) -> List[str]:
|
||||
"""Tokenize mixed CJK / Latin text into a flat list of lowercased tokens.
|
||||
|
||||
CJK runs -> jieba words or character bigrams. Latin runs -> ``\\w+`` words.
|
||||
Order is preserved; callers that want a set can wrap the result.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
text = text.lower()
|
||||
if not has_cjk(text):
|
||||
return _LATIN_RE.findall(text)
|
||||
|
||||
out: List[str] = []
|
||||
pos = 0
|
||||
for match in _CJK_RUN_RE.finditer(text):
|
||||
if match.start() > pos:
|
||||
out.extend(_LATIN_RE.findall(text[pos:match.start()]))
|
||||
out.extend(_cjk_tokens(match.group()))
|
||||
pos = match.end()
|
||||
if pos < len(text):
|
||||
out.extend(_LATIN_RE.findall(text[pos:]))
|
||||
return out
|
||||
@@ -2,59 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from . import dedupe, schema
|
||||
from . import dedupe, entity_extract, schema
|
||||
|
||||
CLUSTERABLE_INTENTS = {"breaking_news", "opinion", "comparison", "prediction"}
|
||||
|
||||
# Words too common to signal shared topic between clusters.
|
||||
_ENTITY_STOPWORDS = frozenset({
|
||||
"the", "a", "an", "to", "for", "how", "is", "in", "of", "on", "and",
|
||||
"with", "from", "by", "at", "this", "that", "it", "what", "are", "do",
|
||||
"can", "his", "her", "he", "she", "its", "was", "has", "new", "just",
|
||||
"says", "said", "will", "about", "after", "now", "all", "been", "here",
|
||||
"not", "out", "up", "more", "also", "but", "who", "year", "first",
|
||||
"make", "being", "making", "over", "into", "than", "they", "their",
|
||||
"would", "could", "get", "got", "some", "like", "back", "going",
|
||||
"breaking", "https", "http", "www", "com",
|
||||
})
|
||||
|
||||
|
||||
def _candidate_text(candidate: schema.Candidate) -> str:
|
||||
return " ".join(part for part in [candidate.title, candidate.snippet] if part).strip()
|
||||
|
||||
|
||||
def _extract_entities(text: str) -> set[str]:
|
||||
"""Extract significant words (proper nouns, numbers, capitalized words) from text.
|
||||
|
||||
Used for cross-source cluster merging where phrasing differs but entities overlap.
|
||||
"""
|
||||
# Normalize but preserve word boundaries
|
||||
words = re.sub(r"[^\w\s]", " ", text).split()
|
||||
entities = set()
|
||||
for word in words:
|
||||
lower = word.lower()
|
||||
if lower in _ENTITY_STOPWORDS or len(word) <= 2:
|
||||
continue
|
||||
# Keep words that are: capitalized, ALL CAPS, contain digits, or 4+ chars
|
||||
if word[0].isupper() or word.isupper() or any(c.isdigit() for c in word) or len(word) >= 4:
|
||||
entities.add(lower)
|
||||
return entities
|
||||
|
||||
|
||||
def _entity_overlap(entities_a: set[str], entities_b: set[str]) -> float:
|
||||
"""Jaccard-style overlap on extracted entities."""
|
||||
if not entities_a or not entities_b:
|
||||
return 0.0
|
||||
intersection = entities_a & entities_b
|
||||
smaller = min(len(entities_a), len(entities_b))
|
||||
# Use overlap coefficient (intersection / min) instead of Jaccard,
|
||||
# because a short tweet about the same event as a long Reddit post
|
||||
# will have fewer total entities but high overlap with the larger set.
|
||||
return len(intersection) / smaller if smaller > 0 else 0.0
|
||||
|
||||
|
||||
def _mmr_representatives(
|
||||
candidates: list[schema.Candidate],
|
||||
text_cache: dict[str, dedupe._PreparedText],
|
||||
@@ -153,7 +107,11 @@ def cluster_candidates(
|
||||
)
|
||||
|
||||
# Second pass: merge small clusters that share entities across sources.
|
||||
clusters = _merge_entity_clusters(clusters, candidates)
|
||||
clusters = _merge_entity_clusters(
|
||||
clusters,
|
||||
candidates,
|
||||
min_shared_entities=2 if "discover-mode" in plan.notes else 1,
|
||||
)
|
||||
|
||||
return sorted(clusters, key=lambda cluster: cluster.score, reverse=True)
|
||||
|
||||
@@ -161,6 +119,8 @@ def cluster_candidates(
|
||||
def _merge_entity_clusters(
|
||||
clusters: list[schema.Cluster],
|
||||
all_candidates: list[schema.Candidate],
|
||||
*,
|
||||
min_shared_entities: int = 1,
|
||||
) -> list[schema.Cluster]:
|
||||
"""Merge small clusters that cover the same story across different sources.
|
||||
|
||||
@@ -182,7 +142,7 @@ def _merge_entity_clusters(
|
||||
for cid in cl.candidate_ids:
|
||||
cand = candidate_map.get(cid)
|
||||
if cand:
|
||||
entities |= _extract_entities(_candidate_text(cand))
|
||||
entities |= entity_extract.extract_text_entities(_candidate_text(cand))
|
||||
cluster_entities.append(entities)
|
||||
|
||||
# Only merge clusters with <= 3 items (don't merge already-large clusters)
|
||||
@@ -207,8 +167,9 @@ def _merge_entity_clusters(
|
||||
if poly_i != poly_j:
|
||||
continue
|
||||
|
||||
overlap = _entity_overlap(cluster_entities[i], cluster_entities[j])
|
||||
if overlap >= 0.45:
|
||||
shared_entities = cluster_entities[i] & cluster_entities[j]
|
||||
overlap = entity_extract.entity_overlap(cluster_entities[i], cluster_entities[j])
|
||||
if len(shared_entities) >= min_shared_entities and overlap >= 0.45:
|
||||
merged_into[j] = i
|
||||
|
||||
if not merged_into:
|
||||
|
||||
@@ -13,11 +13,10 @@ the caller's requested count.
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from . import dates, grounding
|
||||
from . import dates, grounding, log
|
||||
from .resolve import _has_backend
|
||||
|
||||
# A "brand-shaped" token starts with uppercase OR is camelCase with an
|
||||
@@ -62,7 +61,7 @@ _STOPWORD_TOKENS: frozenset[str] = frozenset(
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
print(f"[Competitors] {msg}", file=sys.stderr)
|
||||
log.source_log("Competitors", msg, tty_only=False)
|
||||
|
||||
|
||||
def _topic_tokens(topic: str) -> set[str]:
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Browser cookie extraction for last30days.
|
||||
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Brave, Safari)
|
||||
to enable zero-config authentication for services like X/Twitter.
|
||||
Note: Chrome/Brave extraction is macOS-only; Windows Chrome/Edge use
|
||||
DPAPI-encrypted stores that are not yet supported.
|
||||
|
||||
Only uses Python stdlib — no external dependencies.
|
||||
"""
|
||||
@@ -9,6 +11,7 @@ Only uses Python stdlib — no external dependencies.
|
||||
import configparser
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import sqlite3
|
||||
@@ -19,6 +22,13 @@ from typing import Dict, List, Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _lock_temp_cookie_copy(path: str) -> None:
|
||||
"""Restrict copied cookie DB temp files to the current user on POSIX."""
|
||||
if os.name == "nt":
|
||||
return
|
||||
Path(path).chmod(0o600)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _is_wsl() -> bool:
|
||||
"""Detect if running under Windows Subsystem for Linux.
|
||||
@@ -59,7 +69,16 @@ def _get_firefox_profiles_dir() -> Optional[Path]:
|
||||
if system == "Darwin":
|
||||
path = Path.home() / "Library" / "Application Support" / "Firefox"
|
||||
elif system == "Linux":
|
||||
# Default location for most distros
|
||||
path = Path.home() / ".mozilla" / "firefox"
|
||||
if path.is_dir():
|
||||
return path
|
||||
# Some distros (e.g. Fedora) honour $XDG_CONFIG_HOME
|
||||
xdg_config = os.environ.get("XDG_CONFIG_HOME")
|
||||
if xdg_config and os.path.isabs(xdg_config):
|
||||
path = Path(xdg_config) / "mozilla" / "firefox"
|
||||
else:
|
||||
path = Path.home() / ".config" / "mozilla" / "firefox"
|
||||
else:
|
||||
# Windows: %APPDATA%\Mozilla\Firefox — best-effort
|
||||
appdata = Path.home() / "AppData" / "Roaming" / "Mozilla" / "Firefox"
|
||||
@@ -147,7 +166,13 @@ def _query_cookies_db(
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
# mkstemp creates the file 0600. copy2 would copy the source's mode
|
||||
# (Firefox cookies.sqlite is commonly 0644, looser on WSL /mnt/c) onto
|
||||
# the temp file, leaving live session secrets world-readable in shared
|
||||
# /tmp until the chmod below runs. copyfile writes content only and
|
||||
# leaves the 0600 perms intact, closing that window.
|
||||
shutil.copyfile(str(db_path), tmp_path)
|
||||
_lock_temp_cookie_copy(tmp_path)
|
||||
|
||||
conn = sqlite3.connect(tmp_path)
|
||||
try:
|
||||
@@ -189,12 +214,36 @@ def _query_cookies_db(
|
||||
|
||||
|
||||
def _try_firefox_dir(profiles_dir: Path, domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Try to extract cookies from a Firefox profiles directory."""
|
||||
profile_path = _find_default_profile(profiles_dir)
|
||||
if profile_path is None:
|
||||
logger.debug("No Firefox profile found in %s", profiles_dir)
|
||||
return None
|
||||
return _query_cookies_db(profile_path / "cookies.sqlite", domain, cookie_names)
|
||||
"""Try to extract cookies from a Firefox profiles directory.
|
||||
|
||||
Tries the default profile first, then falls back to scanning all
|
||||
profiles for matching cookies. This handles multi-profile setups
|
||||
where the user is logged into x.com on a non-default profile.
|
||||
"""
|
||||
default_profile = _find_default_profile(profiles_dir)
|
||||
profiles_tried = 0
|
||||
if default_profile is not None:
|
||||
result = _query_cookies_db(default_profile / "cookies.sqlite", domain, cookie_names)
|
||||
if result is not None:
|
||||
return result
|
||||
profiles_tried = 1
|
||||
# Fallback: scan every profile directory for matching cookies
|
||||
try:
|
||||
for child in sorted(profiles_dir.iterdir()):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
if default_profile is not None and child == default_profile:
|
||||
continue
|
||||
db = child / "cookies.sqlite"
|
||||
if db.is_file():
|
||||
result = _query_cookies_db(db, domain, cookie_names)
|
||||
if result is not None:
|
||||
return result
|
||||
profiles_tried += 1
|
||||
except OSError:
|
||||
pass
|
||||
logger.debug("No matching cookies found in %d Firefox profile(s)", profiles_tried)
|
||||
return None
|
||||
|
||||
|
||||
def extract_firefox_cookies(
|
||||
@@ -255,6 +304,74 @@ def extract_chrome_cookies(
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Brave for the given domain and cookie names.
|
||||
|
||||
macOS only — Brave uses the same v10 AES-128-CBC encryption as Chrome,
|
||||
with a different DB path and Keychain service name ("Brave Safe Storage").
|
||||
Tries the Default profile first, then scans numbered Profile directories.
|
||||
|
||||
Returns:
|
||||
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||
"""
|
||||
if platform.system() != "Darwin":
|
||||
logger.debug("Brave cookie extraction only supported on macOS")
|
||||
return None
|
||||
try:
|
||||
from .chrome_cookies import extract_brave_cookies_macos
|
||||
return extract_brave_cookies_macos(domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("Brave cookie extraction failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _extract_chromium_family_cookies(
|
||||
browser: str, domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from a non-Chrome/Brave Chromium browser on macOS.
|
||||
|
||||
macOS only — Edge, Vivaldi, Opera, Arc, and Chromium all reuse Chrome's
|
||||
v10 AES-128-CBC encryption, with their own profile path and Keychain
|
||||
service name (see chrome_cookies.CHROMIUM_BROWSER_PROFILES).
|
||||
"""
|
||||
if platform.system() != "Darwin":
|
||||
logger.debug("%s cookie extraction only supported on macOS", browser)
|
||||
return None
|
||||
try:
|
||||
from .chrome_cookies import extract_chromium_browser_cookies_macos
|
||||
return extract_chromium_browser_cookies_macos(browser, domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("%s cookie extraction failed: %s", browser, exc)
|
||||
return None
|
||||
|
||||
|
||||
def extract_edge_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Microsoft Edge for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("edge", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_vivaldi_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Vivaldi for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("vivaldi", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_opera_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Opera for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("opera", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_arc_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Arc for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("arc", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_chromium_cookies(domain: str, cookie_names: List[str]) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from open-source Chromium for the given domain (macOS only)."""
|
||||
return _extract_chromium_family_cookies("chromium", domain, cookie_names)
|
||||
|
||||
|
||||
def extract_safari_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
@@ -282,9 +399,10 @@ def extract_cookies(
|
||||
"""Extract cookies from the specified browser.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'edge', 'vivaldi',
|
||||
'opera', 'arc', 'chromium', 'safari', or 'auto'.
|
||||
'auto' tries browsers in platform-appropriate order:
|
||||
- macOS: Chrome -> Firefox -> Safari
|
||||
- macOS: Chrome -> Brave -> Edge -> Vivaldi -> Opera -> Arc -> Chromium -> Firefox -> Safari
|
||||
- Linux: Firefox only
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
@@ -333,7 +451,8 @@ def extract_cookies_with_source(
|
||||
so callers can track the source.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'edge', 'vivaldi',
|
||||
'opera', 'arc', 'chromium', 'safari', or 'auto'.
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
|
||||
@@ -344,6 +463,12 @@ def extract_cookies_with_source(
|
||||
extractors = {
|
||||
"firefox": extract_firefox_cookies,
|
||||
"chrome": extract_chrome_cookies,
|
||||
"brave": extract_brave_cookies,
|
||||
"edge": extract_edge_cookies,
|
||||
"vivaldi": extract_vivaldi_cookies,
|
||||
"opera": extract_opera_cookies,
|
||||
"arc": extract_arc_cookies,
|
||||
"chromium": extract_chromium_cookies,
|
||||
"safari": extract_safari_cookies,
|
||||
}
|
||||
|
||||
@@ -357,10 +482,14 @@ def extract_cookies_with_source(
|
||||
result = extractor(domain, cookie_names)
|
||||
return (result, browser) if result is not None else None
|
||||
|
||||
# Auto mode: try browsers in platform-appropriate order
|
||||
# Auto mode: try browsers in platform-appropriate order.
|
||||
# Note: the skill's own entry point (env.extract_browser_credentials) builds
|
||||
# its own list that tries the SILENT browsers (Firefox, Safari) first to
|
||||
# avoid macOS Keychain prompts. This standalone "auto" is Chromium-first; the
|
||||
# two orderings are intentional for their respective callers.
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
order = ["chrome", "firefox", "safari"]
|
||||
order = ["chrome", "brave", "edge", "vivaldi", "opera", "arc", "chromium", "firefox", "safari"]
|
||||
elif system == "Linux":
|
||||
order = ["firefox"]
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
"""Deterministic, local-only document corpus source.
|
||||
|
||||
The corpus adapter deliberately has no HTTP dependency. It scans explicitly
|
||||
registered directories, extracts small text documents (and PDFs only when the
|
||||
local ``pdftotext`` binary is available), and returns normalized ``SourceItem``
|
||||
objects for the shared relevance/fusion pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from shutil import which
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import entity_extract, log, relevance, schema
|
||||
|
||||
SOURCE = "corpus"
|
||||
SUPPORTED_SUFFIXES = {".md", ".txt", ".pdf"}
|
||||
IGNORED_DIRECTORIES = {".git", "node_modules"}
|
||||
MAX_FILES = 500
|
||||
MAX_TEXT_CHARS = 1_000_000
|
||||
MAX_CACHE_TEXT_CHARS = MAX_TEXT_CHARS
|
||||
MAX_CACHE_BYTES = 50 * 1024 * 1024
|
||||
MAX_CACHE_ENTRIES = 2_000
|
||||
CACHE_FILENAME = "corpus-cache.json"
|
||||
CACHE_SCHEMA_VERSION = "last30days-corpus-cache/v2"
|
||||
|
||||
_CACHE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CorpusScanResult:
|
||||
"""One bounded scan, including non-fatal extraction notes."""
|
||||
|
||||
items: list[schema.SourceItem]
|
||||
notes: list[str] = field(default_factory=list)
|
||||
files_scanned: int = 0
|
||||
cache_hits: int = 0
|
||||
|
||||
|
||||
def resolve_directories(
|
||||
cli_directories: Iterable[str] | None,
|
||||
configured: str | Iterable[str] | None,
|
||||
) -> list[Path]:
|
||||
"""Merge repeatable CLI paths with ``os.pathsep``-separated config paths."""
|
||||
raw: list[str] = [str(value) for value in (cli_directories or []) if str(value).strip()]
|
||||
if isinstance(configured, str):
|
||||
raw.extend(value for value in configured.split(os.pathsep) if value.strip())
|
||||
elif configured:
|
||||
raw.extend(str(value) for value in configured if str(value).strip())
|
||||
|
||||
resolved: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
for value in raw:
|
||||
path = Path(value.strip()).expanduser().resolve()
|
||||
key = os.path.normcase(str(path))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
resolved.append(path)
|
||||
return resolved
|
||||
|
||||
|
||||
def _safe_error(exc: BaseException) -> str:
|
||||
"""Describe an error without str(exc), which embeds absolute paths.
|
||||
|
||||
These notes travel into source_status detail and render in coverage
|
||||
diagnostics outside the private corpus block.
|
||||
"""
|
||||
reason = getattr(exc, "strerror", None)
|
||||
return str(reason) if reason else exc.__class__.__name__
|
||||
|
||||
|
||||
def search(
|
||||
topic: str,
|
||||
directories: Iterable[Path | str],
|
||||
*,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
all_time: bool = False,
|
||||
limit: int = 12,
|
||||
cache_dir: Path | None = None,
|
||||
) -> CorpusScanResult:
|
||||
"""Search registered directories without making any network calls."""
|
||||
roots = resolve_directories([str(path) for path in directories], None)
|
||||
notes: list[str] = []
|
||||
cache_path = cache_dir / CACHE_FILENAME if cache_dir is not None else None
|
||||
with _CACHE_LOCK:
|
||||
cache = _load_cache(cache_path)
|
||||
cache_entries = cache.setdefault("entries", {})
|
||||
cache_entry_sizes = {
|
||||
path: _cache_entry_fragment_size(path, value)
|
||||
for path, value in cache_entries.items()
|
||||
}
|
||||
|
||||
candidates: list[tuple[float, int, schema.SourceItem]] = []
|
||||
seen_files: set[str] = set()
|
||||
files_scanned = 0
|
||||
cache_hits = 0
|
||||
pdf_available = which("pdftotext")
|
||||
pdf_unavailable_noted = False
|
||||
|
||||
readable_roots: list[Path] = []
|
||||
for root in roots:
|
||||
if not root.is_dir():
|
||||
notes.append(f"Skipped corpus root '{Path(root).name}': not a readable directory")
|
||||
continue
|
||||
readable_roots.append(root)
|
||||
|
||||
per_root_limit, extra_slots = divmod(MAX_FILES, len(readable_roots) or 1)
|
||||
scan_limit_reached = False
|
||||
for root_index, root in enumerate(readable_roots):
|
||||
root_limit = per_root_limit + (1 if root_index < extra_slots else 0)
|
||||
root_files_scanned = 0
|
||||
for path in _iter_files(root, notes=notes):
|
||||
if root_files_scanned >= root_limit:
|
||||
scan_limit_reached = True
|
||||
break
|
||||
key = os.path.normcase(str(path))
|
||||
if key in seen_files:
|
||||
continue
|
||||
seen_files.add(key)
|
||||
root_files_scanned += 1
|
||||
files_scanned += 1
|
||||
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError as exc:
|
||||
notes.append(f"Skipped {_display_path(path, root)}: {_safe_error(exc)}")
|
||||
continue
|
||||
published_at = datetime.fromtimestamp(
|
||||
stat.st_mtime, tz=timezone.utc
|
||||
).date().isoformat()
|
||||
if not all_time and not (from_date <= published_at <= to_date):
|
||||
continue
|
||||
|
||||
cached = cache_entries.get(str(path))
|
||||
if (
|
||||
isinstance(cached, dict)
|
||||
and cached.get("mtime_ns") == stat.st_mtime_ns
|
||||
and cached.get("size") == stat.st_size
|
||||
and isinstance(cached.get("text"), str)
|
||||
):
|
||||
text = cached["text"]
|
||||
cache_hits += 1
|
||||
else:
|
||||
if path.suffix.lower() == ".pdf" and not pdf_available:
|
||||
if not pdf_unavailable_noted:
|
||||
notes.append("Skipped PDF files because pdftotext is not on PATH")
|
||||
pdf_unavailable_noted = True
|
||||
continue
|
||||
try:
|
||||
text = _extract_text(path, pdftotext=pdf_available)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
notes.append(f"Skipped {_display_path(path, root)}: {_safe_error(exc)}")
|
||||
continue
|
||||
_cache_entry_put(cache_entries, cache_entry_sizes, str(path), {
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"size": stat.st_size,
|
||||
"text": text[:MAX_CACHE_TEXT_CHARS],
|
||||
})
|
||||
|
||||
title = _path_title(path)
|
||||
score = _match_score(topic, f"{title}\n{text}")
|
||||
if score < 0.15:
|
||||
continue
|
||||
relative_path = str(path.relative_to(root))
|
||||
path_digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()
|
||||
item = schema.SourceItem(
|
||||
item_id=f"C{path_digest[:12]}",
|
||||
source=SOURCE,
|
||||
title=title,
|
||||
body=text,
|
||||
url=f"corpus://{path_digest}",
|
||||
container=str(path.parent),
|
||||
published_at=published_at,
|
||||
date_confidence="high",
|
||||
relevance_hint=score,
|
||||
why_relevant=f"Matched local file {relative_path}",
|
||||
# Leave empty so extract_best_snippet derives the matching
|
||||
# window; a file-prefix snippet is preserved verbatim and can
|
||||
# show unrelated intro text (and draw entity-miss demotion).
|
||||
snippet="",
|
||||
metadata={
|
||||
"path": str(path),
|
||||
"relative_path": relative_path,
|
||||
"extension": path.suffix.lower(),
|
||||
"local_only": True,
|
||||
},
|
||||
)
|
||||
candidates.append((score, stat.st_mtime_ns, item))
|
||||
if scan_limit_reached:
|
||||
notes.append(f"Stopped after the {MAX_FILES}-file corpus scan limit")
|
||||
|
||||
cache["schema_version"] = CACHE_SCHEMA_VERSION
|
||||
cache["entries"] = _bounded_entries(cache_entries)
|
||||
with _CACHE_LOCK:
|
||||
_write_cache(cache_path, cache, notes)
|
||||
|
||||
candidates.sort(key=lambda row: (-row[0], -row[1], row[2].title.casefold()))
|
||||
items = [item for _score, _mtime, item in candidates[: max(0, limit)]]
|
||||
log.source_log(
|
||||
"Corpus",
|
||||
f"scanned {files_scanned} file(s), {cache_hits} cache hit(s), {len(items)} match(es)",
|
||||
tty_only=False,
|
||||
)
|
||||
return CorpusScanResult(
|
||||
items=items,
|
||||
notes=notes,
|
||||
files_scanned=files_scanned,
|
||||
cache_hits=cache_hits,
|
||||
)
|
||||
|
||||
|
||||
def _display_path(path: Path | str, root: Path | None = None) -> str:
|
||||
"""Render a note-safe path: never the absolute local path.
|
||||
|
||||
Corpus notes flow into source_status detail and the Partial Coverage
|
||||
block, which render OUTSIDE the private corpus markers - an absolute
|
||||
path like /home/user/private/notes/foo.md must not escape there.
|
||||
"""
|
||||
candidate = Path(path)
|
||||
if root is not None:
|
||||
try:
|
||||
return str(Path(root).name / candidate.relative_to(root))
|
||||
except ValueError:
|
||||
pass
|
||||
return candidate.name
|
||||
|
||||
|
||||
def _iter_files(root: Path, notes: list[str] | None = None) -> Iterable[Path]:
|
||||
# Bounded newest-first selection: keep only the newest MAX_FILES paths in a
|
||||
# heap while walking, so registering a huge tree does not materialize every
|
||||
# path before the caller's extraction cap applies.
|
||||
import heapq
|
||||
|
||||
heap: list[tuple[int, str]] = []
|
||||
walk_errors = 0
|
||||
|
||||
def _on_walk_error(error: OSError) -> None:
|
||||
nonlocal walk_errors
|
||||
walk_errors += 1
|
||||
if notes is not None and walk_errors <= 3:
|
||||
unreadable = _display_path(error.filename, root) if error.filename else Path(root).name
|
||||
notes.append(f"corpus: could not read {unreadable}: {error.strerror}")
|
||||
|
||||
for current, directory_names, file_names in os.walk(
|
||||
root, followlinks=False, onerror=_on_walk_error
|
||||
):
|
||||
directory_names[:] = sorted(
|
||||
name
|
||||
for name in directory_names
|
||||
if name not in IGNORED_DIRECTORIES and not name.startswith(".")
|
||||
)
|
||||
current_path = Path(current)
|
||||
for name in sorted(file_names):
|
||||
if name.startswith("."):
|
||||
continue
|
||||
path = current_path / name
|
||||
if path.suffix.lower() in SUPPORTED_SUFFIXES and not path.is_symlink():
|
||||
entry = (_safe_mtime_ns(path), str(path))
|
||||
if len(heap) < MAX_FILES:
|
||||
heapq.heappush(heap, entry)
|
||||
else:
|
||||
heapq.heappushpop(heap, entry)
|
||||
if notes is not None and walk_errors > 3:
|
||||
notes.append(f"corpus: {walk_errors - 3} more unreadable directories suppressed")
|
||||
ordered = sorted(heap, key=lambda item: (-item[0], item[1].casefold()))
|
||||
for _mtime, raw_path in ordered:
|
||||
yield Path(raw_path)
|
||||
|
||||
|
||||
def _safe_mtime_ns(path: Path) -> int:
|
||||
try:
|
||||
return path.stat().st_mtime_ns
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_text(path: Path, *, pdftotext: str | None) -> str:
|
||||
if path.suffix.lower() == ".pdf":
|
||||
if not pdftotext:
|
||||
return ""
|
||||
completed = subprocess.run(
|
||||
[pdftotext, str(path), "-"],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
)
|
||||
return completed.stdout[:MAX_TEXT_CHARS]
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return handle.read(MAX_TEXT_CHARS)
|
||||
|
||||
|
||||
def _path_title(path: Path) -> str:
|
||||
title = path.stem.replace("_", " ").replace("-", " ")
|
||||
return " ".join(title.split()) or path.name
|
||||
|
||||
|
||||
def _match_score(topic: str, text: str) -> float:
|
||||
lexical = relevance.token_overlap_relevance(topic, text)
|
||||
topic_entities = entity_extract.extract_text_entities(topic)
|
||||
text_entities = entity_extract.extract_text_entities(text)
|
||||
entity_score = entity_extract.entity_overlap(topic_entities, text_entities)
|
||||
return round(max(lexical, entity_score * 0.9), 4)
|
||||
|
||||
|
||||
def _load_cache(path: Path | None) -> dict[str, Any]:
|
||||
if path is None:
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
try:
|
||||
if path.stat().st_size > MAX_CACHE_BYTES:
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
if not isinstance(payload, dict) or payload.get("schema_version") != CACHE_SCHEMA_VERSION:
|
||||
return {"schema_version": CACHE_SCHEMA_VERSION, "entries": {}}
|
||||
if not isinstance(payload.get("entries"), dict):
|
||||
payload["entries"] = {}
|
||||
payload["entries"] = _bounded_entries(payload["entries"])
|
||||
return payload
|
||||
|
||||
|
||||
def _bounded_entries(entries: Any) -> dict[str, Any]:
|
||||
if not isinstance(entries, dict):
|
||||
return {}
|
||||
ordered = sorted(
|
||||
(
|
||||
(path, value)
|
||||
for path, value in entries.items()
|
||||
if (
|
||||
isinstance(path, str)
|
||||
and isinstance(value, dict)
|
||||
and isinstance(value.get("text"), str)
|
||||
)
|
||||
),
|
||||
key=lambda row: int(row[1].get("mtime_ns") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
base_bytes = len(
|
||||
json.dumps(
|
||||
{"schema_version": CACHE_SCHEMA_VERSION, "entries": {}},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
)
|
||||
used_bytes = base_bytes
|
||||
bounded: dict[str, Any] = {}
|
||||
for path, value in ordered[:MAX_CACHE_ENTRIES]:
|
||||
normalized = {
|
||||
"mtime_ns": value.get("mtime_ns"),
|
||||
"size": value.get("size"),
|
||||
"text": value["text"][:MAX_CACHE_TEXT_CHARS],
|
||||
}
|
||||
fragment = json.dumps({path: normalized}, ensure_ascii=False).encode("utf-8")
|
||||
fragment_bytes = len(fragment) - 2 + (2 if bounded else 0)
|
||||
if used_bytes + fragment_bytes > MAX_CACHE_BYTES:
|
||||
continue
|
||||
bounded[path] = normalized
|
||||
used_bytes += fragment_bytes
|
||||
return bounded
|
||||
|
||||
|
||||
def _cache_entry_fragment_size(path: str, value: dict[str, Any]) -> int:
|
||||
return len(json.dumps({path: value}, ensure_ascii=False).encode("utf-8")) - 2
|
||||
|
||||
|
||||
def _cache_entry_put(
|
||||
entries: dict[str, Any],
|
||||
sizes: dict[str, int],
|
||||
path: str,
|
||||
value: dict[str, Any],
|
||||
) -> None:
|
||||
entries[path] = value
|
||||
sizes[path] = _cache_entry_fragment_size(path, value)
|
||||
while (
|
||||
len(entries) > MAX_CACHE_ENTRIES
|
||||
or _cache_payload_size(sizes) > MAX_CACHE_BYTES
|
||||
):
|
||||
oldest = min(
|
||||
entries,
|
||||
key=lambda candidate: (
|
||||
int(entries[candidate].get("mtime_ns") or 0),
|
||||
candidate,
|
||||
),
|
||||
)
|
||||
del entries[oldest]
|
||||
del sizes[oldest]
|
||||
|
||||
|
||||
def _cache_payload_size(sizes: dict[str, int]) -> int:
|
||||
base_bytes = len(
|
||||
json.dumps(
|
||||
{"schema_version": CACHE_SCHEMA_VERSION, "entries": {}},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
)
|
||||
separators = max(0, len(sizes) - 1) * 2
|
||||
return base_bytes + sum(sizes.values()) + separators
|
||||
|
||||
|
||||
def _write_cache(path: Path | None, payload: dict[str, Any], notes: list[str]) -> None:
|
||||
if path is None:
|
||||
return
|
||||
try:
|
||||
_ensure_private_directory(path.parent)
|
||||
payload["entries"] = _bounded_entries(payload.get("entries", {}))
|
||||
encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
try:
|
||||
fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
temporary.unlink()
|
||||
fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(encoded)
|
||||
temporary.replace(path)
|
||||
path.chmod(0o600)
|
||||
except OSError as exc:
|
||||
notes.append(f"Corpus cache unavailable: {_safe_error(exc)}")
|
||||
|
||||
|
||||
def _ensure_private_directory(path: Path) -> None:
|
||||
missing: list[Path] = []
|
||||
current = path
|
||||
while not current.exists():
|
||||
missing.append(current)
|
||||
current = current.parent
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
for directory in missing:
|
||||
directory.chmod(0o700)
|
||||
@@ -4,15 +4,55 @@ from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
def get_date_range(days: int = 30) -> Tuple[str, str]:
|
||||
"""Get the date range for the last N days.
|
||||
def parse_as_of_date(as_of_date: Optional[str]) -> Optional[str]:
|
||||
"""Validate and normalize an --as-of date.
|
||||
|
||||
Args:
|
||||
as_of_date: Date string in YYYY-MM-DD format.
|
||||
|
||||
Returns:
|
||||
Tuple of (from_date, to_date) as YYYY-MM-DD strings
|
||||
Normalized YYYY-MM-DD string, or None when no date was provided.
|
||||
|
||||
Raises:
|
||||
ValueError: If the date is not in YYYY-MM-DD format.
|
||||
"""
|
||||
today = datetime.now(timezone.utc).date()
|
||||
from_date = today - timedelta(days=days)
|
||||
return from_date.isoformat(), today.isoformat()
|
||||
if as_of_date is None:
|
||||
return None
|
||||
|
||||
if not as_of_date.strip():
|
||||
raise ValueError("--as-of must be in YYYY-MM-DD format.")
|
||||
|
||||
try:
|
||||
parsed = datetime.strptime(as_of_date, "%Y-%m-%d").date()
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"Invalid --as-of date: {as_of_date}. Expected YYYY-MM-DD."
|
||||
) from exc
|
||||
|
||||
return parsed.isoformat()
|
||||
|
||||
|
||||
def get_date_range(days: int = 30, as_of_date: Optional[str] = None) -> Tuple[str, str]:
|
||||
"""Get the date range for the last N days.
|
||||
|
||||
When as_of_date is provided, the range ends at that date instead of today.
|
||||
|
||||
Args:
|
||||
days: Number of days to look back.
|
||||
as_of_date: Optional end date in YYYY-MM-DD format.
|
||||
|
||||
Returns:
|
||||
Tuple of (from_date, to_date) as YYYY-MM-DD strings.
|
||||
"""
|
||||
normalized_as_of = parse_as_of_date(as_of_date)
|
||||
|
||||
if normalized_as_of:
|
||||
to_date = datetime.strptime(normalized_as_of, "%Y-%m-%d").date()
|
||||
else:
|
||||
to_date = datetime.now(timezone.utc).date()
|
||||
|
||||
from_date = to_date - timedelta(days=days)
|
||||
return from_date.isoformat(), to_date.isoformat()
|
||||
|
||||
|
||||
def parse_date(date_str: Optional[str]) -> Optional[datetime]:
|
||||
@@ -86,9 +126,10 @@ def get_date_confidence(date_str: Optional[str], from_date: str, to_date: str) -
|
||||
return 'low'
|
||||
|
||||
|
||||
def days_ago(date_str: Optional[str]) -> Optional[int]:
|
||||
"""Calculate how many days ago a date is.
|
||||
def days_ago(date_str: Optional[str], reference_date: Optional[str] = None) -> Optional[int]:
|
||||
"""Calculate how many days before the reference date a date is.
|
||||
|
||||
If reference_date is None, use real today for backward compatibility.
|
||||
Returns None if date is invalid or missing.
|
||||
"""
|
||||
if not date_str:
|
||||
@@ -96,24 +137,32 @@ def days_ago(date_str: Optional[str]) -> Optional[int]:
|
||||
|
||||
try:
|
||||
dt = datetime.strptime(date_str, "%Y-%m-%d").date()
|
||||
today = datetime.now(timezone.utc).date()
|
||||
if reference_date:
|
||||
today = datetime.strptime(reference_date, "%Y-%m-%d").date()
|
||||
else:
|
||||
today = datetime.now(timezone.utc).date()
|
||||
delta = today - dt
|
||||
return delta.days
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def recency_score(date_str: Optional[str], max_days: int = 30) -> int:
|
||||
def recency_score(
|
||||
date_str: Optional[str],
|
||||
max_days: int = 30,
|
||||
reference_date: Optional[str] = None,
|
||||
) -> int:
|
||||
"""Calculate recency score (0-100).
|
||||
|
||||
0 days ago = 100, max_days ago = 0, clamped.
|
||||
0 days before reference_date = 100, max_days before reference_date = 0.
|
||||
If reference_date is None, use real today for backward compatibility.
|
||||
"""
|
||||
age = days_ago(date_str)
|
||||
age = days_ago(date_str, reference_date=reference_date)
|
||||
if age is None:
|
||||
return 0 # Unknown date gets worst score
|
||||
return 0
|
||||
|
||||
if age < 0:
|
||||
return 100 # Future date (treat as today)
|
||||
return 100
|
||||
if age >= max_days:
|
||||
return 0
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from . import schema
|
||||
from . import cjk, schema
|
||||
|
||||
STOPWORDS = frozenset(
|
||||
{
|
||||
@@ -31,7 +31,7 @@ STOPWORDS = frozenset(
|
||||
"do",
|
||||
"can",
|
||||
}
|
||||
)
|
||||
) | cjk.CHINESE_STOPWORDS
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
@@ -61,12 +61,12 @@ def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||
def token_jaccard(text_a: str, text_b: str) -> float:
|
||||
tokens_a = {
|
||||
token
|
||||
for token in normalize_text(text_a).split()
|
||||
for token in cjk.segment(normalize_text(text_a))
|
||||
if len(token) > 1 and token not in STOPWORDS
|
||||
}
|
||||
tokens_b = {
|
||||
token
|
||||
for token in normalize_text(text_b).split()
|
||||
for token in cjk.segment(normalize_text(text_b))
|
||||
if len(token) > 1 and token not in STOPWORDS
|
||||
}
|
||||
return jaccard_similarity(tokens_a, tokens_b)
|
||||
@@ -81,7 +81,7 @@ def hybrid_similarity(text_a: str, text_b: str) -> float:
|
||||
|
||||
def _tokenize(normalized: str) -> frozenset[str]:
|
||||
return frozenset(
|
||||
tok for tok in normalized.split()
|
||||
tok for tok in cjk.segment(normalized)
|
||||
if len(tok) > 1 and tok not in STOPWORDS
|
||||
)
|
||||
|
||||
@@ -110,10 +110,25 @@ def item_text(item: schema.SourceItem) -> str:
|
||||
|
||||
|
||||
def dedupe_items(items: list[schema.SourceItem], threshold: float = 0.7) -> list[schema.SourceItem]:
|
||||
"""Remove near-duplicates while keeping earlier, better-scored items."""
|
||||
"""Remove near-duplicates while keeping earlier, better-scored items.
|
||||
|
||||
Jobs are deduped by exact URL only: distinct postings on the same careers
|
||||
board share heavy boilerplate (company intro, "TL;DR", benefits) that trips
|
||||
fuzzy text similarity and collapses unrelated roles (a 26-role board fell to
|
||||
7). A unique posting URL is an unambiguous identity, so use it instead.
|
||||
"""
|
||||
kept: list[schema.SourceItem] = []
|
||||
kept_prepared: list[_PreparedText] = []
|
||||
seen_job_urls: set[str] = set()
|
||||
for item in items:
|
||||
if item.source == "jobs":
|
||||
url = (item.url or "").strip()
|
||||
if url and url in seen_job_urls:
|
||||
continue
|
||||
if url:
|
||||
seen_job_urls.add(url)
|
||||
kept.append(item)
|
||||
continue
|
||||
text = item_text(item)
|
||||
if not text:
|
||||
kept.append(item)
|
||||
|
||||
@@ -22,6 +22,7 @@ import json
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import log, subproc
|
||||
from .relevance import token_overlap_relevance
|
||||
@@ -53,7 +54,7 @@ POSTS_TIMEOUT = 15
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("Digg", msg)
|
||||
log.source_log("Digg", msg, tty_only=False)
|
||||
|
||||
|
||||
def _is_available() -> bool:
|
||||
@@ -296,6 +297,20 @@ def parse_digg_response(
|
||||
return items
|
||||
|
||||
|
||||
def _is_safe_http_url(url: str) -> bool:
|
||||
"""True iff ``url`` parses with an http or https scheme.
|
||||
|
||||
Used to reject upstream-supplied post URLs whose scheme would be
|
||||
dangerous in a rendered ``<a href>`` (``javascript:``, ``data:``,
|
||||
``file:``, ``vbscript:``, ``about:``).
|
||||
"""
|
||||
try:
|
||||
scheme = urlparse(url).scheme.lower()
|
||||
except ValueError:
|
||||
return False
|
||||
return scheme in ("http", "https")
|
||||
|
||||
|
||||
def _parse_post(raw_post: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Reduce a digg post payload into the small dict render uses.
|
||||
|
||||
@@ -316,6 +331,17 @@ def _parse_post(raw_post: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
x_url = str(raw_post.get("xUrl") or "").strip()
|
||||
if not x_url:
|
||||
return None
|
||||
if not _is_safe_http_url(x_url):
|
||||
# Security-class drop: an upstream-supplied URL with a dangerous
|
||||
# scheme. Force tty_only=False so the rejection is visible in
|
||||
# non-interactive runs (Claude Code), which is the actual attack
|
||||
# surface — the default tty_only=True would suppress it there.
|
||||
log.source_log(
|
||||
"Digg",
|
||||
f"dropped post with unsafe xUrl scheme: {x_url!r}",
|
||||
tty_only=False,
|
||||
)
|
||||
return None
|
||||
return {
|
||||
"username": username,
|
||||
"display_name": str(author.get("display_name") or "").strip() or username,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user