spencer kelly
254 lines
11 KiB
JavaScript
254 lines
11 KiB
JavaScript
// generate machine-readable docs for LLMs and coding-agents, from source-of-truth.
|
|
//
|
|
// node ./scripts/docs.js
|
|
//
|
|
// emits:
|
|
// docs/tags.md - the full tagset graph (from the running library)
|
|
// docs/api.md - every method, signature + one-line description (from types/*.d.ts)
|
|
// llms-full.txt - all in-repo docs concatenated into one fetchable file
|
|
//
|
|
// the curated docs (AGENTS.md, docs/match-syntax.md, docs/recipes.md, docs/concepts.md)
|
|
// are hand-written - this script does not touch them, it only stitches them into llms-full.txt
|
|
import fs from 'fs'
|
|
import path from 'path'
|
|
import { fileURLToPath } from 'url'
|
|
import nlp from '../src/three.js'
|
|
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
|
|
const read = (f) => fs.readFileSync(path.join(root, f), 'utf-8')
|
|
const write = (f, str) => {
|
|
fs.writeFileSync(path.join(root, f), str)
|
|
console.log(` wrote ${f} (${(str.length / 1024).toFixed(1)}kb)`)
|
|
}
|
|
const stamp = `<!-- 🤖 generated by scripts/docs.js — do not edit by hand. -->\n`
|
|
|
|
// ------------------------------------------------------------------ tags.md
|
|
const colorName = { blue: 'Nouns', green: 'Verbs', red: 'Values & Dates', magenta: 'Adjectives', cyan: 'Closed-class' }
|
|
const generateTags = function () {
|
|
const tagSet = nlp.world().model.one.tagSet
|
|
const names = Object.keys(tagSet).sort()
|
|
// group by color, then 'other'
|
|
const groups = {}
|
|
for (const name of names) {
|
|
const t = tagSet[name]
|
|
const bucket = colorName[t.color] || 'Other'
|
|
groups[bucket] = groups[bucket] || []
|
|
groups[bucket].push(name)
|
|
}
|
|
let md = stamp + `# Compromise Tags (Part-of-Speech tagset)\n\n`
|
|
md += `Every term is assigned one or more of these **${names.length} tags**. `
|
|
md += `Use them in match-patterns with a \`#\` prefix, e.g. \`doc.match('#Person')\`.\n\n`
|
|
md += `Tags are a hierarchy: tagging a term \`#FirstName\` also makes it a \`#Person\` and a \`#Noun\`. `
|
|
md += `Tagging a term one thing may remove conflicting tags (a term can't be both \`#Singular\` and \`#Plural\`).\n\n`
|
|
md += `> ⚠️ Only the tags listed here are valid. \`#Name\`, \`#Subject\`, \`#Object\`, \`#Adj\` etc. are **not** real tags and silently match nothing.\n\n`
|
|
|
|
const order = ['Nouns', 'Verbs', 'Adjectives', 'Values & Dates', 'Closed-class', 'Other']
|
|
for (const bucket of order) {
|
|
if (!groups[bucket]) continue
|
|
md += `## ${bucket}\n\n`
|
|
md += `| Tag | Is also a | Example |\n|---|---|---|\n`
|
|
for (const name of groups[bucket]) {
|
|
const t = tagSet[name]
|
|
const parents = (t.parents || []).filter(p => p !== name)
|
|
const isAlso = parents.length ? parents.map(p => `\`#${p}\``).join(', ') : '—'
|
|
const eg = examples[name] || ''
|
|
md += `| \`#${name}\` | ${isAlso} | ${eg} |\n`
|
|
}
|
|
md += `\n`
|
|
}
|
|
return md
|
|
}
|
|
|
|
// representative example word(s) per tag — kept beside the generator so it's one file to update
|
|
const examples = {
|
|
Noun: 'cat', Singular: 'cat', Plural: 'cats', ProperNoun: 'Tesla', Uncountable: 'gravity',
|
|
Person: 'John Smith', FirstName: 'john', MaleName: 'john', FemaleName: 'mary', LastName: 'smith',
|
|
Honorific: 'dr.', Pronoun: 'he', Reflexive: 'yourself', Actor: 'swimmer', Activity: 'swimming',
|
|
Place: 'Paris', Country: 'Canada', City: 'Toronto', Region: 'California', Address: '4 main st.',
|
|
Organization: 'Google', Company: 'Google', School: 'UCLA', SportsTeam: 'the Leafs',
|
|
Unit: 'km', Demonym: 'Canadian', Possessive: "spencer's", Currency: '$', AtMention: '@nlp',
|
|
Verb: 'walk', PresentTense: 'walks', Infinitive: 'walk', Imperative: 'eat!', Gerund: 'walking',
|
|
PastTense: 'walked', FutureTense: 'will walk', Copula: 'is', Modal: 'could', Participle: 'awoken',
|
|
Auxiliary: 'will have', PhrasalVerb: 'walk out', Particle: 'out', Passive: 'was walked',
|
|
Adjective: 'quick', Comparable: 'quick', Comparative: 'quicker', Superlative: 'quickest', Adverb: 'quickly',
|
|
Value: '5', Cardinal: 'five', Ordinal: 'fifth', Fraction: '2/3', Multiple: 'million',
|
|
RomanNumeral: 'xviii', TextValue: 'five', NumericValue: '5', Money: '$5', Percent: '5%',
|
|
Date: 'monday', Month: 'march', WeekDay: 'monday', Year: '1992', FinancialQuarter: 'q2',
|
|
Holiday: 'easter', Season: 'summer', Timezone: 'EST', Time: '4:30pm', Duration: '2 weeks',
|
|
Determiner: 'the', Conjunction: 'and', Preposition: 'of', QuestionWord: 'who', Expression: 'hi',
|
|
Negative: 'not', Condition: 'if', There: 'there', Prefix: 'co-', Hyphenated: 'bone-headed',
|
|
Abbreviation: 'mrs.', Acronym: 'FBI', Url: 'compromise.cool', PhoneNumber: '(555) 123-4567',
|
|
HashTag: '#nlp', Email: 'hi@compromise.cool', Emoji: '💋', Emoticon: ':)', SlashedTerm: 'love/hate',
|
|
}
|
|
|
|
// ------------------------------------------------------------------ api.md
|
|
// parse the type-definitions into {name, signature, doc} grouped by section.
|
|
const parseDts = function (file) {
|
|
const lines = read(file).split('\n')
|
|
const out = []
|
|
let section = ''
|
|
let pendingDoc = ''
|
|
const memberRe = /^\s*([a-zA-Z][a-zA-Z0-9]*)(\??):\s*(.+?)\s*$/
|
|
for (let line of lines) {
|
|
// section headers, written as `// Match` or `// ### Pointers`
|
|
const sec = line.match(/^\s*\/\/\s*#*\s*([A-Z][A-Za-z ]+)\s*$/)
|
|
if (sec && !line.includes('alias') && !line.includes('support') && !line.includes('use ')) {
|
|
section = sec[1].trim()
|
|
continue
|
|
}
|
|
// one-line jsdoc
|
|
const doc = line.match(/\/\*\*\s*(.*?)\s*\*\//)
|
|
if (doc) {
|
|
pendingDoc = doc[1]
|
|
continue
|
|
}
|
|
const m = line.match(memberRe)
|
|
if (m && pendingDoc) {
|
|
const name = m[1]
|
|
let sig = m[3].replace(/,$/, '')
|
|
// turn `(a: T) => View` into `(a)` for readability
|
|
let args = ''
|
|
const arrow = sig.match(/^\((.*)\)\s*=>/)
|
|
if (arrow) {
|
|
args = arrow[1].split(',').map(s => s.split(':')[0].trim()).filter(Boolean).join(', ')
|
|
out.push({ section, name, call: `.${name}(${args})`, doc: pendingDoc })
|
|
} else {
|
|
// a getter/property
|
|
out.push({ section, name, call: `.${name}`, doc: pendingDoc, getter: true })
|
|
}
|
|
pendingDoc = ''
|
|
continue
|
|
}
|
|
if (line.trim() && !line.trim().startsWith('//')) pendingDoc = ''
|
|
}
|
|
return out
|
|
}
|
|
|
|
const generateApi = function () {
|
|
let md = stamp + `# Compromise API Reference\n\n`
|
|
md += `Every method returns a new **View** (a sub-selection of the document) unless noted, so calls chain: `
|
|
md += `\`nlp(text).match('#Verb').toPastTense().text()\`. Methods are grouped by what they do.\n\n`
|
|
md += `Method availability depends on the [build tier](concepts.md): \`compromise/one\` (tokenize), `
|
|
md += `\`compromise/two\` (+tags & contractions), and the default \`compromise\` / \`compromise/three\` (+ all selections below).\n\n`
|
|
|
|
// core View methods (one) + two additions
|
|
md += `## Core methods\n\n_(available on every View)_\n\n`
|
|
const core = [...parseDts('types/view/one.d.ts'), ...parseDts('types/view/two.d.ts')]
|
|
let lastSection = null
|
|
for (const m of core) {
|
|
if (m.section && m.section !== lastSection) {
|
|
md += `\n### ${m.section}\n\n`
|
|
lastSection = m.section
|
|
}
|
|
md += `- **\`${m.call}\`**${m.getter ? ' _[getter]_' : ''} — ${m.doc}\n`
|
|
}
|
|
|
|
// tier-three selection classes (Nouns, Verbs, Numbers, ...)
|
|
md += `\n## Selections (compromise/three)\n\n`
|
|
md += `These return specialised sub-views with extra methods. e.g. \`doc.verbs().toPastTense()\`.\n\n`
|
|
const three = read('types/view/three.d.ts')
|
|
// the main Three interface methods
|
|
const threeMain = three.match(/interface Three extends View \{([\s\S]*?)\n\}/)
|
|
if (threeMain) {
|
|
md += `### Selection methods on any View\n\n`
|
|
for (const m of parseBlock(threeMain[1])) {
|
|
md += `- **\`${m.call}\`** — ${m.doc}\n`
|
|
}
|
|
}
|
|
// the sub-classes
|
|
const classRe = /export interface (\w+) extends View \{([\s\S]*?)\n\}/g
|
|
let cm
|
|
while ((cm = classRe.exec(three))) {
|
|
const members = parseBlock(cm[2])
|
|
if (!members.length) continue
|
|
md += `\n### \`.${cm[1].toLowerCase()}()\` →\n\n`
|
|
for (const m of members) md += `- **\`${m.call}\`** — ${m.doc}\n`
|
|
}
|
|
|
|
// constructor / nlp.* methods
|
|
md += `\n## Constructor methods (\`nlp.*\`)\n\n_(called on the imported \`nlp\` object, not a View)_\n\n`
|
|
const ctor = read('types/three.d.ts')
|
|
for (const m of parseBlock(ctor, true)) {
|
|
md += `- **\`nlp${m.call.replace(/^\./, '.')}\`** — ${m.doc}\n`
|
|
}
|
|
return md
|
|
}
|
|
|
|
// parse a free block of jsdoc+member lines (no sections)
|
|
const parseBlock = function (str, isCtor) {
|
|
const lines = str.split('\n')
|
|
const out = []
|
|
let pendingDoc = ''
|
|
for (const line of lines) {
|
|
const doc = line.match(/\/\*\*\s*(.*?)\s*\*\//)
|
|
if (doc) { pendingDoc = doc[1]; continue }
|
|
let m = line.match(/^\s*([a-zA-Z][a-zA-Z0-9]*)(\??):\s*(.+?)\s*$/) // property/method
|
|
if (!m && isCtor) m = line.match(/^\s*export function ([a-zA-Z][a-zA-Z0-9]*)\s*(\(.*?\))/)
|
|
if (!m && isCtor) m = line.match(/^\s*export const ([a-zA-Z][a-zA-Z0-9]*)\s*(:)/) // e.g. version
|
|
if (m && pendingDoc) {
|
|
const name = m[1]
|
|
let rest = m[3] || m[2] || ''
|
|
const arrow = rest.match(/^\((.*?)\)\s*=>/) || rest.match(/^\((.*)\)/)
|
|
let call
|
|
if (arrow) {
|
|
const args = arrow[1].split(',').map(s => s.split(':')[0].trim()).filter(Boolean).join(', ')
|
|
call = `.${name}(${args})`
|
|
} else {
|
|
call = `.${name}`
|
|
}
|
|
out.push({ name, call, doc: pendingDoc })
|
|
pendingDoc = ''
|
|
} else if (line.trim() && !line.includes('/**')) {
|
|
pendingDoc = ''
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ------------------------------------------------------------------ llms.txt (curated index)
|
|
const generateLlms = function () {
|
|
return `# compromise
|
|
|
|
> A rule-based natural-language-processing library for English. Tokenizes text, tags parts-of-speech,
|
|
> and finds & transforms parts of the text. Runs offline in node and the browser, no dependencies.
|
|
> Not an LLM/neural model. Current version ${nlp.version}.
|
|
|
|
Docs below are plain markdown (the published observablehq.com notebooks do not render as readable text).
|
|
|
|
## Docs
|
|
- [AGENTS.md](AGENTS.md): start here — mental model, rules, and the mistakes to avoid
|
|
- [Concepts](docs/concepts.md): document/View/Term model, mutability, build tiers
|
|
- [Match syntax](docs/match-syntax.md): the .match() mini-language
|
|
- [Tags](docs/tags.md): the complete, valid part-of-speech tagset
|
|
- [API](docs/api.md): every method, signature, and description
|
|
- [Recipes](docs/recipes.md): copy-paste solutions to common tasks
|
|
|
|
## Optional
|
|
- [Full text](docs/llms-full.txt): every doc above concatenated into one file
|
|
- [README](README.md): the human-facing overview
|
|
`
|
|
}
|
|
|
|
// ------------------------------------------------------------------ llms-full.txt
|
|
const generateLlmsFull = function () {
|
|
const files = ['AGENTS.md', 'docs/concepts.md', 'docs/match-syntax.md', 'docs/tags.md', 'docs/api.md', 'docs/recipes.md']
|
|
let out = `# compromise ${nlp.version} — full documentation for LLMs\n`
|
|
out += `# https://github.com/spencermountain/compromise\n`
|
|
out += `# This file concatenates every in-repo doc into one fetchable text.\n\n`
|
|
for (const f of files) {
|
|
if (!fs.existsSync(path.join(root, f))) continue
|
|
out += `\n\n${'='.repeat(78)}\n# FILE: ${f}\n${'='.repeat(78)}\n\n`
|
|
out += read(f).replace(stamp, '')
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ------------------------------------------------------------------ run
|
|
console.log('generating docs...')
|
|
if (!fs.existsSync(path.join(root, 'docs'))) fs.mkdirSync(path.join(root, 'docs'))
|
|
write('docs/tags.md', generateTags())
|
|
write('docs/api.md', generateApi())
|
|
write('llms.txt', generateLlms())
|
|
write('docs/llms-full.txt', generateLlmsFull())
|
|
console.log('done.')
|