You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
583 lines
12 KiB
583 lines
12 KiB
/**
|
|
* Lightweight in-browser syntax highlighter for the docs.
|
|
* Hand-rolled tokenizer for TypeScript / JavaScript / Svelte / JSON / shell.
|
|
* No dependencies, no build step — colors come from CSS variables.
|
|
*/
|
|
|
|
export type TokenKind =
|
|
| 'plain'
|
|
| 'keyword'
|
|
| 'literal'
|
|
| 'string'
|
|
| 'string-escape'
|
|
| 'string-interp'
|
|
| 'number'
|
|
| 'comment'
|
|
| 'type'
|
|
| 'function'
|
|
| 'property'
|
|
| 'operator'
|
|
| 'punctuation'
|
|
| 'tag'
|
|
| 'attr'
|
|
| 'tag-close';
|
|
|
|
export interface Token {
|
|
k: TokenKind;
|
|
t: string;
|
|
}
|
|
|
|
const JS_KEYWORDS = new Set([
|
|
'as',
|
|
'async',
|
|
'await',
|
|
'break',
|
|
'case',
|
|
'catch',
|
|
'class',
|
|
'const',
|
|
'continue',
|
|
'debugger',
|
|
'default',
|
|
'delete',
|
|
'do',
|
|
'else',
|
|
'export',
|
|
'extends',
|
|
'finally',
|
|
'for',
|
|
'from',
|
|
'function',
|
|
'get',
|
|
'if',
|
|
'implements',
|
|
'import',
|
|
'in',
|
|
'instanceof',
|
|
'interface',
|
|
'is',
|
|
'keyof',
|
|
'let',
|
|
'new',
|
|
'of',
|
|
'private',
|
|
'protected',
|
|
'public',
|
|
'readonly',
|
|
'return',
|
|
'satisfies',
|
|
'set',
|
|
'static',
|
|
'super',
|
|
'switch',
|
|
'throw',
|
|
'try',
|
|
'type',
|
|
'typeof',
|
|
'var',
|
|
'void',
|
|
'while',
|
|
'with',
|
|
'yield',
|
|
'enum',
|
|
'namespace',
|
|
'declare',
|
|
'abstract',
|
|
'infer',
|
|
'never'
|
|
]);
|
|
|
|
const JS_LITERALS = new Set(['true', 'false', 'null', 'undefined', 'this', 'NaN', 'Infinity']);
|
|
|
|
const JS_BUILTIN_TYPES = new Set([
|
|
'string',
|
|
'number',
|
|
'boolean',
|
|
'object',
|
|
'symbol',
|
|
'bigint',
|
|
'never',
|
|
'unknown',
|
|
'any',
|
|
'void'
|
|
]);
|
|
|
|
export function tokenize(source: string, lang: string): Token[] {
|
|
switch (lang) {
|
|
case 'ts':
|
|
case 'tsx':
|
|
case 'js':
|
|
case 'jsx':
|
|
case 'mjs':
|
|
case 'cjs':
|
|
return tokenizeJsTs(source);
|
|
case 'svelte':
|
|
case 'html':
|
|
return tokenizeSvelte(source);
|
|
case 'json':
|
|
return tokenizeJson(source);
|
|
case 'css':
|
|
return tokenizeCss(source);
|
|
case 'sh':
|
|
case 'bash':
|
|
case 'shell':
|
|
return tokenizeShell(source);
|
|
default:
|
|
return [{ k: 'plain', t: source }];
|
|
}
|
|
}
|
|
|
|
function tokenizeJsTs(src: string): Token[] {
|
|
const out: Token[] = [];
|
|
const n = src.length;
|
|
let i = 0;
|
|
|
|
const push = (k: TokenKind, t: string) => {
|
|
if (t.length === 0) return;
|
|
const last = out[out.length - 1];
|
|
if (last && last.k === k) last.t += t;
|
|
else out.push({ k, t });
|
|
};
|
|
|
|
while (i < n) {
|
|
const ch = src[i];
|
|
const next = src[i + 1];
|
|
|
|
// Line comment
|
|
if (ch === '/' && next === '/') {
|
|
const end = src.indexOf('\n', i);
|
|
const stop = end === -1 ? n : end;
|
|
push('comment', src.slice(i, stop));
|
|
i = stop;
|
|
continue;
|
|
}
|
|
// Block comment (incl. JSDoc)
|
|
if (ch === '/' && next === '*') {
|
|
const end = src.indexOf('*/', i + 2);
|
|
const stop = end === -1 ? n : end + 2;
|
|
push('comment', src.slice(i, stop));
|
|
i = stop;
|
|
continue;
|
|
}
|
|
// Strings
|
|
if (ch === '"' || ch === "'") {
|
|
const quote = ch;
|
|
let j = i + 1;
|
|
while (j < n) {
|
|
const c = src[j];
|
|
if (c === '\\') {
|
|
j += 2;
|
|
continue;
|
|
}
|
|
if (c === quote) {
|
|
j++;
|
|
break;
|
|
}
|
|
if (c === '\n') break;
|
|
j++;
|
|
}
|
|
push('string', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Template literal
|
|
if (ch === '`') {
|
|
let j = i + 1;
|
|
let depth = 0;
|
|
while (j < n) {
|
|
const c = src[j];
|
|
if (c === '\\') {
|
|
j += 2;
|
|
continue;
|
|
}
|
|
if (c === '$' && src[j + 1] === '{') {
|
|
depth = 1;
|
|
j += 2;
|
|
while (j < n && depth > 0) {
|
|
if (src[j] === '{') depth++;
|
|
else if (src[j] === '}') depth--;
|
|
j++;
|
|
}
|
|
continue;
|
|
}
|
|
if (c === '`') {
|
|
j++;
|
|
break;
|
|
}
|
|
j++;
|
|
}
|
|
push('string', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Number
|
|
if (isDigit(ch) || (ch === '.' && isDigit(next))) {
|
|
let j = i;
|
|
while (j < n && /[0-9_xXoObBeE+\-.]/.test(src[j])) {
|
|
if ((src[j] === '+' || src[j] === '-') && !/[eE]/.test(src[j - 1])) break;
|
|
j++;
|
|
}
|
|
push('number', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Identifier
|
|
if (isIdentStart(ch)) {
|
|
let j = i;
|
|
while (j < n && isIdentPart(src[j])) j++;
|
|
const word = src.slice(i, j);
|
|
const trailing = nextNonWs(src, j);
|
|
|
|
if (JS_KEYWORDS.has(word)) push('keyword', word);
|
|
else if (JS_LITERALS.has(word)) push('literal', word);
|
|
else if (JS_BUILTIN_TYPES.has(word)) push('type', word);
|
|
else if (/^[A-Z][A-Za-z0-9_]*$/.test(word)) push('type', word);
|
|
else if (trailing === '(' || trailing === '<') push('function', word);
|
|
else push('plain', word);
|
|
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Property accessor: .name
|
|
if (ch === '.' && isIdentStart(next)) {
|
|
push('punctuation', '.');
|
|
i++;
|
|
let j = i;
|
|
while (j < n && isIdentPart(src[j])) j++;
|
|
const word = src.slice(i, j);
|
|
const trailing = nextNonWs(src, j);
|
|
if (trailing === '(' || trailing === '<') push('function', word);
|
|
else push('property', word);
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Punctuation
|
|
if ('{}()[];,'.includes(ch)) {
|
|
push('punctuation', ch);
|
|
i++;
|
|
continue;
|
|
}
|
|
// Operator runs
|
|
if ('+-*/%&|^~!?=<>:.'.includes(ch)) {
|
|
let j = i;
|
|
while (j < n && '+-*/%&|^~!?=<>:.'.includes(src[j])) j++;
|
|
push('operator', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Whitespace and everything else
|
|
push('plain', ch);
|
|
i++;
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
function tokenizeJson(src: string): Token[] {
|
|
const out: Token[] = [];
|
|
const n = src.length;
|
|
let i = 0;
|
|
|
|
const push = (k: TokenKind, t: string) => {
|
|
if (t.length === 0) return;
|
|
const last = out[out.length - 1];
|
|
if (last && last.k === k) last.t += t;
|
|
else out.push({ k, t });
|
|
};
|
|
|
|
while (i < n) {
|
|
const ch = src[i];
|
|
// Strings (also keys — heuristic: followed by `:` becomes property)
|
|
if (ch === '"') {
|
|
let j = i + 1;
|
|
while (j < n) {
|
|
if (src[j] === '\\') {
|
|
j += 2;
|
|
continue;
|
|
}
|
|
if (src[j] === '"') {
|
|
j++;
|
|
break;
|
|
}
|
|
j++;
|
|
}
|
|
const trailing = nextNonWs(src, j);
|
|
push(trailing === ':' ? 'property' : 'string', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
if (isDigit(ch) || (ch === '-' && isDigit(src[i + 1]))) {
|
|
let j = i;
|
|
while (j < n && /[0-9.eE+\-]/.test(src[j])) j++;
|
|
push('number', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
if (isIdentStart(ch)) {
|
|
let j = i;
|
|
while (j < n && isIdentPart(src[j])) j++;
|
|
const word = src.slice(i, j);
|
|
if (word === 'true' || word === 'false' || word === 'null') push('literal', word);
|
|
else push('plain', word);
|
|
i = j;
|
|
continue;
|
|
}
|
|
if ('{}[]'.includes(ch)) {
|
|
push('punctuation', ch);
|
|
i++;
|
|
continue;
|
|
}
|
|
if (ch === ':' || ch === ',') {
|
|
push('operator', ch);
|
|
i++;
|
|
continue;
|
|
}
|
|
push('plain', ch);
|
|
i++;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function tokenizeShell(src: string): Token[] {
|
|
const out: Token[] = [];
|
|
for (const line of src.split(/(\n)/)) {
|
|
if (line === '\n') {
|
|
out.push({ k: 'plain', t: '\n' });
|
|
continue;
|
|
}
|
|
const trimmed = line.replace(/^\s+/, '');
|
|
const lead = line.slice(0, line.length - trimmed.length);
|
|
if (lead) out.push({ k: 'plain', t: lead });
|
|
if (trimmed.startsWith('#')) {
|
|
out.push({ k: 'comment', t: trimmed });
|
|
continue;
|
|
}
|
|
// First token = command (function-coloured)
|
|
const m = trimmed.match(/^([\w./-]+)(.*)$/);
|
|
if (m) {
|
|
out.push({ k: 'function', t: m[1] });
|
|
if (m[2]) {
|
|
// Color flags as type
|
|
const rest = m[2];
|
|
let i = 0;
|
|
while (i < rest.length) {
|
|
const c = rest[i];
|
|
if (c === '"' || c === "'") {
|
|
const q = c;
|
|
let j = i + 1;
|
|
while (j < rest.length && rest[j] !== q) {
|
|
if (rest[j] === '\\') j++;
|
|
j++;
|
|
}
|
|
out.push({ k: 'string', t: rest.slice(i, j + 1) });
|
|
i = j + 1;
|
|
continue;
|
|
}
|
|
if (c === '-' && rest[i + 1] === '-') {
|
|
let j = i + 2;
|
|
while (j < rest.length && /[\w-]/.test(rest[j])) j++;
|
|
out.push({ k: 'type', t: rest.slice(i, j) });
|
|
i = j;
|
|
continue;
|
|
}
|
|
if (c === '-' && /\w/.test(rest[i + 1] ?? '')) {
|
|
let j = i + 1;
|
|
while (j < rest.length && /\w/.test(rest[j])) j++;
|
|
out.push({ k: 'type', t: rest.slice(i, j) });
|
|
i = j;
|
|
continue;
|
|
}
|
|
out.push({ k: 'plain', t: c });
|
|
i++;
|
|
}
|
|
}
|
|
} else {
|
|
out.push({ k: 'plain', t: trimmed });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function tokenizeCss(src: string): Token[] {
|
|
const out: Token[] = [];
|
|
const n = src.length;
|
|
let i = 0;
|
|
const push = (k: TokenKind, t: string) => {
|
|
if (t.length === 0) return;
|
|
const last = out[out.length - 1];
|
|
if (last && last.k === k) last.t += t;
|
|
else out.push({ k, t });
|
|
};
|
|
|
|
while (i < n) {
|
|
const ch = src[i];
|
|
if (ch === '/' && src[i + 1] === '*') {
|
|
const end = src.indexOf('*/', i + 2);
|
|
const stop = end === -1 ? n : end + 2;
|
|
push('comment', src.slice(i, stop));
|
|
i = stop;
|
|
continue;
|
|
}
|
|
if (ch === '"' || ch === "'") {
|
|
const q = ch;
|
|
let j = i + 1;
|
|
while (j < n && src[j] !== q) {
|
|
if (src[j] === '\\') j++;
|
|
j++;
|
|
}
|
|
push('string', src.slice(i, j + 1));
|
|
i = j + 1;
|
|
continue;
|
|
}
|
|
if (ch === '-' && src[i + 1] === '-') {
|
|
let j = i;
|
|
while (j < n && /[\w-]/.test(src[j])) j++;
|
|
push('property', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
if (ch === '#' || ch === '.') {
|
|
let j = i + 1;
|
|
while (j < n && /[\w-]/.test(src[j])) j++;
|
|
push('type', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
if (isDigit(ch)) {
|
|
let j = i;
|
|
while (j < n && /[0-9.%a-zA-Z]/.test(src[j])) j++;
|
|
push('number', src.slice(i, j));
|
|
i = j;
|
|
continue;
|
|
}
|
|
if (isIdentStart(ch)) {
|
|
let j = i;
|
|
while (j < n && (isIdentPart(src[j]) || src[j] === '-')) j++;
|
|
const word = src.slice(i, j);
|
|
const trail = nextNonWs(src, j);
|
|
if (trail === ':') push('keyword', word);
|
|
else if (trail === '(') push('function', word);
|
|
else push('plain', word);
|
|
i = j;
|
|
continue;
|
|
}
|
|
push('plain', ch);
|
|
i++;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function tokenizeSvelte(src: string): Token[] {
|
|
const out: Token[] = [];
|
|
const n = src.length;
|
|
let i = 0;
|
|
const push = (k: TokenKind, t: string) => {
|
|
if (t.length === 0) return;
|
|
const last = out[out.length - 1];
|
|
if (last && last.k === k) last.t += t;
|
|
else out.push({ k, t });
|
|
};
|
|
|
|
while (i < n) {
|
|
const ch = src[i];
|
|
// HTML comment
|
|
if (src.startsWith('<!--', i)) {
|
|
const end = src.indexOf('-->', i + 4);
|
|
const stop = end === -1 ? n : end + 3;
|
|
push('comment', src.slice(i, stop));
|
|
i = stop;
|
|
continue;
|
|
}
|
|
// <script> / <style> blocks — delegate inner content
|
|
if (src.startsWith('<script', i) || src.startsWith('<style', i)) {
|
|
const tagEnd = src.indexOf('>', i);
|
|
if (tagEnd === -1) break;
|
|
const openTag = src.slice(i, tagEnd + 1);
|
|
push('tag', openTag);
|
|
i = tagEnd + 1;
|
|
const isScript = openTag.toLowerCase().startsWith('<script');
|
|
const closeTag = isScript ? '</script>' : '</style>';
|
|
const close = src.indexOf(closeTag, i);
|
|
const inner = src.slice(i, close === -1 ? n : close);
|
|
const innerTokens = isScript ? tokenizeJsTs(inner) : tokenizeCss(inner);
|
|
out.push(...innerTokens);
|
|
i += inner.length;
|
|
if (close !== -1) {
|
|
push('tag', closeTag);
|
|
i += closeTag.length;
|
|
}
|
|
continue;
|
|
}
|
|
// Tag
|
|
if (ch === '<') {
|
|
let j = i + 1;
|
|
const isClose = src[j] === '/';
|
|
if (isClose) j++;
|
|
while (j < n && /[\w:-]/.test(src[j])) j++;
|
|
push(isClose ? 'tag-close' : 'tag', src.slice(i, j));
|
|
while (j < n && src[j] !== '>') {
|
|
if (src[j] === '"' || src[j] === "'") {
|
|
const q = src[j];
|
|
let k2 = j + 1;
|
|
while (k2 < n && src[k2] !== q) k2++;
|
|
push('string', src.slice(j, k2 + 1));
|
|
j = k2 + 1;
|
|
continue;
|
|
}
|
|
if (/[\w-]/.test(src[j])) {
|
|
let k2 = j;
|
|
while (k2 < n && /[\w-]/.test(src[k2])) k2++;
|
|
push('attr', src.slice(j, k2));
|
|
j = k2;
|
|
continue;
|
|
}
|
|
push('plain', src[j]);
|
|
j++;
|
|
}
|
|
if (j < n) {
|
|
push('tag', src[j]);
|
|
j++;
|
|
}
|
|
i = j;
|
|
continue;
|
|
}
|
|
// Mustache expression
|
|
if (ch === '{') {
|
|
let depth = 1;
|
|
let j = i + 1;
|
|
while (j < n && depth > 0) {
|
|
if (src[j] === '{') depth++;
|
|
else if (src[j] === '}') depth--;
|
|
if (depth > 0) j++;
|
|
}
|
|
const inner = src.slice(i + 1, j);
|
|
push('punctuation', '{');
|
|
out.push(...tokenizeJsTs(inner));
|
|
push('punctuation', '}');
|
|
i = j + 1;
|
|
continue;
|
|
}
|
|
push('plain', ch);
|
|
i++;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function isDigit(ch: string | undefined): boolean {
|
|
return ch !== undefined && ch >= '0' && ch <= '9';
|
|
}
|
|
|
|
function isIdentStart(ch: string | undefined): boolean {
|
|
return ch !== undefined && (/[A-Za-z_$]/.test(ch) || ch.charCodeAt(0) > 127);
|
|
}
|
|
|
|
function isIdentPart(ch: string): boolean {
|
|
return /[A-Za-z0-9_$]/.test(ch) || ch.charCodeAt(0) > 127;
|
|
}
|
|
|
|
function nextNonWs(src: string, from: number): string {
|
|
let j = from;
|
|
while (j < src.length && /\s/.test(src[j])) j++;
|
|
return src[j] ?? '';
|
|
}
|