/** * Lightweight in-browser syntax highlighter for the docs. * Hand-rolled tokenizer for TypeScript / JavaScript / Svelte / JSON / shell. * No dependencies, no build step — colors come from CSS variables. */ export type TokenKind = | 'plain' | 'keyword' | 'literal' | 'string' | 'string-escape' | 'string-interp' | 'number' | 'comment' | 'type' | 'function' | 'property' | 'operator' | 'punctuation' | 'tag' | 'attr' | 'tag-close'; export interface Token { k: TokenKind; t: string; } const JS_KEYWORDS = new Set([ 'as', 'async', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', 'default', 'delete', 'do', 'else', 'export', 'extends', 'finally', 'for', 'from', 'function', 'get', 'if', 'implements', 'import', 'in', 'instanceof', 'interface', 'is', 'keyof', 'let', 'new', 'of', 'private', 'protected', 'public', 'readonly', 'return', 'satisfies', 'set', 'static', 'super', 'switch', 'throw', 'try', 'type', 'typeof', 'var', 'void', 'while', 'with', 'yield', 'enum', 'namespace', 'declare', 'abstract', 'infer', 'never' ]); const JS_LITERALS = new Set(['true', 'false', 'null', 'undefined', 'this', 'NaN', 'Infinity']); const JS_BUILTIN_TYPES = new Set([ 'string', 'number', 'boolean', 'object', 'symbol', 'bigint', 'never', 'unknown', 'any', 'void' ]); export function tokenize(source: string, lang: string): Token[] { switch (lang) { case 'ts': case 'tsx': case 'js': case 'jsx': case 'mjs': case 'cjs': return tokenizeJsTs(source); case 'svelte': case 'html': return tokenizeSvelte(source); case 'json': return tokenizeJson(source); case 'css': return tokenizeCss(source); case 'sh': case 'bash': case 'shell': return tokenizeShell(source); default: return [{ k: 'plain', t: source }]; } } function tokenizeJsTs(src: string): Token[] { const out: Token[] = []; const n = src.length; let i = 0; const push = (k: TokenKind, t: string) => { if (t.length === 0) return; const last = out[out.length - 1]; if (last && last.k === k) last.t += t; else out.push({ k, t }); }; while (i < n) { const ch = src[i]; const next = src[i + 1]; // Line comment if (ch === '/' && next === '/') { const end = src.indexOf('\n', i); const stop = end === -1 ? n : end; push('comment', src.slice(i, stop)); i = stop; continue; } // Block comment (incl. JSDoc) if (ch === '/' && next === '*') { const end = src.indexOf('*/', i + 2); const stop = end === -1 ? n : end + 2; push('comment', src.slice(i, stop)); i = stop; continue; } // Strings if (ch === '"' || ch === "'") { const quote = ch; let j = i + 1; while (j < n) { const c = src[j]; if (c === '\\') { j += 2; continue; } if (c === quote) { j++; break; } if (c === '\n') break; j++; } push('string', src.slice(i, j)); i = j; continue; } // Template literal if (ch === '`') { let j = i + 1; let depth = 0; while (j < n) { const c = src[j]; if (c === '\\') { j += 2; continue; } if (c === '$' && src[j + 1] === '{') { depth = 1; j += 2; while (j < n && depth > 0) { if (src[j] === '{') depth++; else if (src[j] === '}') depth--; j++; } continue; } if (c === '`') { j++; break; } j++; } push('string', src.slice(i, j)); i = j; continue; } // Number if (isDigit(ch) || (ch === '.' && isDigit(next))) { let j = i; while (j < n && /[0-9_xXoObBeE+\-.]/.test(src[j])) { if ((src[j] === '+' || src[j] === '-') && !/[eE]/.test(src[j - 1])) break; j++; } push('number', src.slice(i, j)); i = j; continue; } // Identifier if (isIdentStart(ch)) { let j = i; while (j < n && isIdentPart(src[j])) j++; const word = src.slice(i, j); const trailing = nextNonWs(src, j); if (JS_KEYWORDS.has(word)) push('keyword', word); else if (JS_LITERALS.has(word)) push('literal', word); else if (JS_BUILTIN_TYPES.has(word)) push('type', word); else if (/^[A-Z][A-Za-z0-9_]*$/.test(word)) push('type', word); else if (trailing === '(' || trailing === '<') push('function', word); else push('plain', word); i = j; continue; } // Property accessor: .name if (ch === '.' && isIdentStart(next)) { push('punctuation', '.'); i++; let j = i; while (j < n && isIdentPart(src[j])) j++; const word = src.slice(i, j); const trailing = nextNonWs(src, j); if (trailing === '(' || trailing === '<') push('function', word); else push('property', word); i = j; continue; } // Punctuation if ('{}()[];,'.includes(ch)) { push('punctuation', ch); i++; continue; } // Operator runs if ('+-*/%&|^~!?=<>:.'.includes(ch)) { let j = i; while (j < n && '+-*/%&|^~!?=<>:.'.includes(src[j])) j++; push('operator', src.slice(i, j)); i = j; continue; } // Whitespace and everything else push('plain', ch); i++; } return out; } function tokenizeJson(src: string): Token[] { const out: Token[] = []; const n = src.length; let i = 0; const push = (k: TokenKind, t: string) => { if (t.length === 0) return; const last = out[out.length - 1]; if (last && last.k === k) last.t += t; else out.push({ k, t }); }; while (i < n) { const ch = src[i]; // Strings (also keys — heuristic: followed by `:` becomes property) if (ch === '"') { let j = i + 1; while (j < n) { if (src[j] === '\\') { j += 2; continue; } if (src[j] === '"') { j++; break; } j++; } const trailing = nextNonWs(src, j); push(trailing === ':' ? 'property' : 'string', src.slice(i, j)); i = j; continue; } if (isDigit(ch) || (ch === '-' && isDigit(src[i + 1]))) { let j = i; while (j < n && /[0-9.eE+\-]/.test(src[j])) j++; push('number', src.slice(i, j)); i = j; continue; } if (isIdentStart(ch)) { let j = i; while (j < n && isIdentPart(src[j])) j++; const word = src.slice(i, j); if (word === 'true' || word === 'false' || word === 'null') push('literal', word); else push('plain', word); i = j; continue; } if ('{}[]'.includes(ch)) { push('punctuation', ch); i++; continue; } if (ch === ':' || ch === ',') { push('operator', ch); i++; continue; } push('plain', ch); i++; } return out; } function tokenizeShell(src: string): Token[] { const out: Token[] = []; for (const line of src.split(/(\n)/)) { if (line === '\n') { out.push({ k: 'plain', t: '\n' }); continue; } const trimmed = line.replace(/^\s+/, ''); const lead = line.slice(0, line.length - trimmed.length); if (lead) out.push({ k: 'plain', t: lead }); if (trimmed.startsWith('#')) { out.push({ k: 'comment', t: trimmed }); continue; } // First token = command (function-coloured) const m = trimmed.match(/^([\w./-]+)(.*)$/); if (m) { out.push({ k: 'function', t: m[1] }); if (m[2]) { // Color flags as type const rest = m[2]; let i = 0; while (i < rest.length) { const c = rest[i]; if (c === '"' || c === "'") { const q = c; let j = i + 1; while (j < rest.length && rest[j] !== q) { if (rest[j] === '\\') j++; j++; } out.push({ k: 'string', t: rest.slice(i, j + 1) }); i = j + 1; continue; } if (c === '-' && rest[i + 1] === '-') { let j = i + 2; while (j < rest.length && /[\w-]/.test(rest[j])) j++; out.push({ k: 'type', t: rest.slice(i, j) }); i = j; continue; } if (c === '-' && /\w/.test(rest[i + 1] ?? '')) { let j = i + 1; while (j < rest.length && /\w/.test(rest[j])) j++; out.push({ k: 'type', t: rest.slice(i, j) }); i = j; continue; } out.push({ k: 'plain', t: c }); i++; } } } else { out.push({ k: 'plain', t: trimmed }); } } return out; } function tokenizeCss(src: string): Token[] { const out: Token[] = []; const n = src.length; let i = 0; const push = (k: TokenKind, t: string) => { if (t.length === 0) return; const last = out[out.length - 1]; if (last && last.k === k) last.t += t; else out.push({ k, t }); }; while (i < n) { const ch = src[i]; if (ch === '/' && src[i + 1] === '*') { const end = src.indexOf('*/', i + 2); const stop = end === -1 ? n : end + 2; push('comment', src.slice(i, stop)); i = stop; continue; } if (ch === '"' || ch === "'") { const q = ch; let j = i + 1; while (j < n && src[j] !== q) { if (src[j] === '\\') j++; j++; } push('string', src.slice(i, j + 1)); i = j + 1; continue; } if (ch === '-' && src[i + 1] === '-') { let j = i; while (j < n && /[\w-]/.test(src[j])) j++; push('property', src.slice(i, j)); i = j; continue; } if (ch === '#' || ch === '.') { let j = i + 1; while (j < n && /[\w-]/.test(src[j])) j++; push('type', src.slice(i, j)); i = j; continue; } if (isDigit(ch)) { let j = i; while (j < n && /[0-9.%a-zA-Z]/.test(src[j])) j++; push('number', src.slice(i, j)); i = j; continue; } if (isIdentStart(ch)) { let j = i; while (j < n && (isIdentPart(src[j]) || src[j] === '-')) j++; const word = src.slice(i, j); const trail = nextNonWs(src, j); if (trail === ':') push('keyword', word); else if (trail === '(') push('function', word); else push('plain', word); i = j; continue; } push('plain', ch); i++; } return out; } function tokenizeSvelte(src: string): Token[] { const out: Token[] = []; const n = src.length; let i = 0; const push = (k: TokenKind, t: string) => { if (t.length === 0) return; const last = out[out.length - 1]; if (last && last.k === k) last.t += t; else out.push({ k, t }); }; while (i < n) { const ch = src[i]; // HTML comment if (src.startsWith('', i + 4); const stop = end === -1 ? n : end + 3; push('comment', src.slice(i, stop)); i = stop; continue; } // ' : ''; const close = src.indexOf(closeTag, i); const inner = src.slice(i, close === -1 ? n : close); const innerTokens = isScript ? tokenizeJsTs(inner) : tokenizeCss(inner); out.push(...innerTokens); i += inner.length; if (close !== -1) { push('tag', closeTag); i += closeTag.length; } continue; } // Tag if (ch === '<') { let j = i + 1; const isClose = src[j] === '/'; if (isClose) j++; while (j < n && /[\w:-]/.test(src[j])) j++; push(isClose ? 'tag-close' : 'tag', src.slice(i, j)); while (j < n && src[j] !== '>') { if (src[j] === '"' || src[j] === "'") { const q = src[j]; let k2 = j + 1; while (k2 < n && src[k2] !== q) k2++; push('string', src.slice(j, k2 + 1)); j = k2 + 1; continue; } if (/[\w-]/.test(src[j])) { let k2 = j; while (k2 < n && /[\w-]/.test(src[k2])) k2++; push('attr', src.slice(j, k2)); j = k2; continue; } push('plain', src[j]); j++; } if (j < n) { push('tag', src[j]); j++; } i = j; continue; } // Mustache expression if (ch === '{') { let depth = 1; let j = i + 1; while (j < n && depth > 0) { if (src[j] === '{') depth++; else if (src[j] === '}') depth--; if (depth > 0) j++; } const inner = src.slice(i + 1, j); push('punctuation', '{'); out.push(...tokenizeJsTs(inner)); push('punctuation', '}'); i = j + 1; continue; } push('plain', ch); i++; } return out; } function isDigit(ch: string | undefined): boolean { return ch !== undefined && ch >= '0' && ch <= '9'; } function isIdentStart(ch: string | undefined): boolean { return ch !== undefined && (/[A-Za-z_$]/.test(ch) || ch.charCodeAt(0) > 127); } function isIdentPart(ch: string): boolean { return /[A-Za-z0-9_$]/.test(ch) || ch.charCodeAt(0) > 127; } function nextNonWs(src: string, from: number): string { let j = from; while (j < src.length && /\s/.test(src[j])) j++; return src[j] ?? ''; }