svelte-kit-vice/web/routes/active/_lib/tokenize.ts

/**
 * Lightweight in-browser syntax highlighter for the docs.
 * Hand-rolled tokenizer for TypeScript / JavaScript / Svelte / JSON / shell.
 * No dependencies, no build step — colors come from CSS variables.
 */

export type TokenKind =
	| 'plain'
	| 'keyword'
	| 'literal'
	| 'string'
	| 'string-escape'
	| 'string-interp'
	| 'number'
	| 'comment'
	| 'type'
	| 'function'
	| 'property'
	| 'operator'
	| 'punctuation'
	| 'tag'
	| 'attr'
	| 'tag-close';

export interface Token {
	k: TokenKind;
	t: string;
}

const JS_KEYWORDS = new Set([
	'as',
	'async',
	'await',
	'break',
	'case',
	'catch',
	'class',
	'const',
	'continue',
	'debugger',
	'default',
	'delete',
	'do',
	'else',
	'export',
	'extends',
	'finally',
	'for',
	'from',
	'function',
	'get',
	'if',
	'implements',
	'import',
	'in',
	'instanceof',
	'interface',
	'is',
	'keyof',
	'let',
	'new',
	'of',
	'private',
	'protected',
	'public',
	'readonly',
	'return',
	'satisfies',
	'set',
	'static',
	'super',
	'switch',
	'throw',
	'try',
	'type',
	'typeof',
	'var',
	'void',
	'while',
	'with',
	'yield',
	'enum',
	'namespace',
	'declare',
	'abstract',
	'infer',
	'never'
]);

const JS_LITERALS = new Set(['true', 'false', 'null', 'undefined', 'this', 'NaN', 'Infinity']);

const JS_BUILTIN_TYPES = new Set([
	'string',
	'number',
	'boolean',
	'object',
	'symbol',
	'bigint',
	'never',
	'unknown',
	'any',
	'void'
]);

export function tokenize(source: string, lang: string): Token[] {
	switch (lang) {
		case 'ts':
		case 'tsx':
		case 'js':
		case 'jsx':
		case 'mjs':
		case 'cjs':
			return tokenizeJsTs(source);
		case 'svelte':
		case 'html':
			return tokenizeSvelte(source);
		case 'json':
			return tokenizeJson(source);
		case 'css':
			return tokenizeCss(source);
		case 'sh':
		case 'bash':
		case 'shell':
			return tokenizeShell(source);
		default:
			return [{ k: 'plain', t: source }];
	}
}

function tokenizeJsTs(src: string): Token[] {
	const out: Token[] = [];
	const n = src.length;
	let i = 0;

	const push = (k: TokenKind, t: string) => {
		if (t.length === 0) return;
		const last = out[out.length - 1];
		if (last && last.k === k) last.t += t;
		else out.push({ k, t });
	};

	while (i < n) {
		const ch = src[i];
		const next = src[i + 1];

		// Line comment
		if (ch === '/' && next === '/') {
			const end = src.indexOf('\n', i);
			const stop = end === -1 ? n : end;
			push('comment', src.slice(i, stop));
			i = stop;
			continue;
		}
		// Block comment (incl. JSDoc)
		if (ch === '/' && next === '*') {
			const end = src.indexOf('*/', i + 2);
			const stop = end === -1 ? n : end + 2;
			push('comment', src.slice(i, stop));
			i = stop;
			continue;
		}
		// Strings
		if (ch === '"' || ch === "'") {
			const quote = ch;
			let j = i + 1;
			while (j < n) {
				const c = src[j];
				if (c === '\\') {
					j += 2;
					continue;
				}
				if (c === quote) {
					j++;
					break;
				}
				if (c === '\n') break;
				j++;
			}
			push('string', src.slice(i, j));
			i = j;
			continue;
		}
		// Template literal
		if (ch === '`') {
			let j = i + 1;
			let depth = 0;
			while (j < n) {
				const c = src[j];
				if (c === '\\') {
					j += 2;
					continue;
				}
				if (c === '$' && src[j + 1] === '{') {
					depth = 1;
					j += 2;
					while (j < n && depth > 0) {
						if (src[j] === '{') depth++;
						else if (src[j] === '}') depth--;
						j++;
					}
					continue;
				}
				if (c === '`') {
					j++;
					break;
				}
				j++;
			}
			push('string', src.slice(i, j));
			i = j;
			continue;
		}
		// Number
		if (isDigit(ch) || (ch === '.' && isDigit(next))) {
			let j = i;
			while (j < n && /[0-9_xXoObBeE+\-.]/.test(src[j])) {
				if ((src[j] === '+' || src[j] === '-') && !/[eE]/.test(src[j - 1])) break;
				j++;
			}
			push('number', src.slice(i, j));
			i = j;
			continue;
		}
		// Identifier
		if (isIdentStart(ch)) {
			let j = i;
			while (j < n && isIdentPart(src[j])) j++;
			const word = src.slice(i, j);
			const trailing = nextNonWs(src, j);

			if (JS_KEYWORDS.has(word)) push('keyword', word);
			else if (JS_LITERALS.has(word)) push('literal', word);
			else if (JS_BUILTIN_TYPES.has(word)) push('type', word);
			else if (/^[A-Z][A-Za-z0-9_]*$/.test(word)) push('type', word);
			else if (trailing === '(' || trailing === '<') push('function', word);
			else push('plain', word);

			i = j;
			continue;
		}
		// Property accessor: .name
		if (ch === '.' && isIdentStart(next)) {
			push('punctuation', '.');
			i++;
			let j = i;
			while (j < n && isIdentPart(src[j])) j++;
			const word = src.slice(i, j);
			const trailing = nextNonWs(src, j);
			if (trailing === '(' || trailing === '<') push('function', word);
			else push('property', word);
			i = j;
			continue;
		}
		// Punctuation
		if ('{}()[];,'.includes(ch)) {
			push('punctuation', ch);
			i++;
			continue;
		}
		// Operator runs
		if ('+-*/%&|^~!?=<>:.'.includes(ch)) {
			let j = i;
			while (j < n && '+-*/%&|^~!?=<>:.'.includes(src[j])) j++;
			push('operator', src.slice(i, j));
			i = j;
			continue;
		}
		// Whitespace and everything else
		push('plain', ch);
		i++;
	}

	return out;
}

function tokenizeJson(src: string): Token[] {
	const out: Token[] = [];
	const n = src.length;
	let i = 0;

	const push = (k: TokenKind, t: string) => {
		if (t.length === 0) return;
		const last = out[out.length - 1];
		if (last && last.k === k) last.t += t;
		else out.push({ k, t });
	};

	while (i < n) {
		const ch = src[i];
		// Strings (also keys — heuristic: followed by `:` becomes property)
		if (ch === '"') {
			let j = i + 1;
			while (j < n) {
				if (src[j] === '\\') {
					j += 2;
					continue;
				}
				if (src[j] === '"') {
					j++;
					break;
				}
				j++;
			}
			const trailing = nextNonWs(src, j);
			push(trailing === ':' ? 'property' : 'string', src.slice(i, j));
			i = j;
			continue;
		}
		if (isDigit(ch) || (ch === '-' && isDigit(src[i + 1]))) {
			let j = i;
			while (j < n && /[0-9.eE+\-]/.test(src[j])) j++;
			push('number', src.slice(i, j));
			i = j;
			continue;
		}
		if (isIdentStart(ch)) {
			let j = i;
			while (j < n && isIdentPart(src[j])) j++;
			const word = src.slice(i, j);
			if (word === 'true' || word === 'false' || word === 'null') push('literal', word);
			else push('plain', word);
			i = j;
			continue;
		}
		if ('{}[]'.includes(ch)) {
			push('punctuation', ch);
			i++;
			continue;
		}
		if (ch === ':' || ch === ',') {
			push('operator', ch);
			i++;
			continue;
		}
		push('plain', ch);
		i++;
	}
	return out;
}

function tokenizeShell(src: string): Token[] {
	const out: Token[] = [];
	for (const line of src.split(/(\n)/)) {
		if (line === '\n') {
			out.push({ k: 'plain', t: '\n' });
			continue;
		}
		const trimmed = line.replace(/^\s+/, '');
		const lead = line.slice(0, line.length - trimmed.length);
		if (lead) out.push({ k: 'plain', t: lead });
		if (trimmed.startsWith('#')) {
			out.push({ k: 'comment', t: trimmed });
			continue;
		}
		// First token = command (function-coloured)
		const m = trimmed.match(/^([\w./-]+)(.*)$/);
		if (m) {
			out.push({ k: 'function', t: m[1] });
			if (m[2]) {
				// Color flags as type
				const rest = m[2];
				let i = 0;
				while (i < rest.length) {
					const c = rest[i];
					if (c === '"' || c === "'") {
						const q = c;
						let j = i + 1;
						while (j < rest.length && rest[j] !== q) {
							if (rest[j] === '\\') j++;
							j++;
						}
						out.push({ k: 'string', t: rest.slice(i, j + 1) });
						i = j + 1;
						continue;
					}
					if (c === '-' && rest[i + 1] === '-') {
						let j = i + 2;
						while (j < rest.length && /[\w-]/.test(rest[j])) j++;
						out.push({ k: 'type', t: rest.slice(i, j) });
						i = j;
						continue;
					}
					if (c === '-' && /\w/.test(rest[i + 1] ?? '')) {
						let j = i + 1;
						while (j < rest.length && /\w/.test(rest[j])) j++;
						out.push({ k: 'type', t: rest.slice(i, j) });
						i = j;
						continue;
					}
					out.push({ k: 'plain', t: c });
					i++;
				}
			}
		} else {
			out.push({ k: 'plain', t: trimmed });
		}
	}
	return out;
}

function tokenizeCss(src: string): Token[] {
	const out: Token[] = [];
	const n = src.length;
	let i = 0;
	const push = (k: TokenKind, t: string) => {
		if (t.length === 0) return;
		const last = out[out.length - 1];
		if (last && last.k === k) last.t += t;
		else out.push({ k, t });
	};

	while (i < n) {
		const ch = src[i];
		if (ch === '/' && src[i + 1] === '*') {
			const end = src.indexOf('*/', i + 2);
			const stop = end === -1 ? n : end + 2;
			push('comment', src.slice(i, stop));
			i = stop;
			continue;
		}
		if (ch === '"' || ch === "'") {
			const q = ch;
			let j = i + 1;
			while (j < n && src[j] !== q) {
				if (src[j] === '\\') j++;
				j++;
			}
			push('string', src.slice(i, j + 1));
			i = j + 1;
			continue;
		}
		if (ch === '-' && src[i + 1] === '-') {
			let j = i;
			while (j < n && /[\w-]/.test(src[j])) j++;
			push('property', src.slice(i, j));
			i = j;
			continue;
		}
		if (ch === '#' || ch === '.') {
			let j = i + 1;
			while (j < n && /[\w-]/.test(src[j])) j++;
			push('type', src.slice(i, j));
			i = j;
			continue;
		}
		if (isDigit(ch)) {
			let j = i;
			while (j < n && /[0-9.%a-zA-Z]/.test(src[j])) j++;
			push('number', src.slice(i, j));
			i = j;
			continue;
		}
		if (isIdentStart(ch)) {
			let j = i;
			while (j < n && (isIdentPart(src[j]) || src[j] === '-')) j++;
			const word = src.slice(i, j);
			const trail = nextNonWs(src, j);
			if (trail === ':') push('keyword', word);
			else if (trail === '(') push('function', word);
			else push('plain', word);
			i = j;
			continue;
		}
		push('plain', ch);
		i++;
	}
	return out;
}

function tokenizeSvelte(src: string): Token[] {
	const out: Token[] = [];
	const n = src.length;
	let i = 0;
	const push = (k: TokenKind, t: string) => {
		if (t.length === 0) return;
		const last = out[out.length - 1];
		if (last && last.k === k) last.t += t;
		else out.push({ k, t });
	};

	while (i < n) {
		const ch = src[i];
		// HTML comment
		if (src.startsWith('<!--', i)) {
			const end = src.indexOf('-->', i + 4);
			const stop = end === -1 ? n : end + 3;
			push('comment', src.slice(i, stop));
			i = stop;
			continue;
		}
		// <script> / <style> blocks — delegate inner content
		if (src.startsWith('<script', i) || src.startsWith('<style', i)) {
			const tagEnd = src.indexOf('>', i);
			if (tagEnd === -1) break;
			const openTag = src.slice(i, tagEnd + 1);
			push('tag', openTag);
			i = tagEnd + 1;
			const isScript = openTag.toLowerCase().startsWith('<script');
			const closeTag = isScript ? '</script>' : '</style>';
			const close = src.indexOf(closeTag, i);
			const inner = src.slice(i, close === -1 ? n : close);
			const innerTokens = isScript ? tokenizeJsTs(inner) : tokenizeCss(inner);
			out.push(...innerTokens);
			i += inner.length;
			if (close !== -1) {
				push('tag', closeTag);
				i += closeTag.length;
			}
			continue;
		}
		// Tag
		if (ch === '<') {
			let j = i + 1;
			const isClose = src[j] === '/';
			if (isClose) j++;
			while (j < n && /[\w:-]/.test(src[j])) j++;
			push(isClose ? 'tag-close' : 'tag', src.slice(i, j));
			while (j < n && src[j] !== '>') {
				if (src[j] === '"' || src[j] === "'") {
					const q = src[j];
					let k2 = j + 1;
					while (k2 < n && src[k2] !== q) k2++;
					push('string', src.slice(j, k2 + 1));
					j = k2 + 1;
					continue;
				}
				if (/[\w-]/.test(src[j])) {
					let k2 = j;
					while (k2 < n && /[\w-]/.test(src[k2])) k2++;
					push('attr', src.slice(j, k2));
					j = k2;
					continue;
				}
				push('plain', src[j]);
				j++;
			}
			if (j < n) {
				push('tag', src[j]);
				j++;
			}
			i = j;
			continue;
		}
		// Mustache expression
		if (ch === '{') {
			let depth = 1;
			let j = i + 1;
			while (j < n && depth > 0) {
				if (src[j] === '{') depth++;
				else if (src[j] === '}') depth--;
				if (depth > 0) j++;
			}
			const inner = src.slice(i + 1, j);
			push('punctuation', '{');
			out.push(...tokenizeJsTs(inner));
			push('punctuation', '}');
			i = j + 1;
			continue;
		}
		push('plain', ch);
		i++;
	}
	return out;
}

function isDigit(ch: string | undefined): boolean {
	return ch !== undefined && ch >= '0' && ch <= '9';
}

function isIdentStart(ch: string | undefined): boolean {
	return ch !== undefined && (/[A-Za-z_$]/.test(ch) || ch.charCodeAt(0) > 127);
}

function isIdentPart(ch: string): boolean {
	return /[A-Za-z0-9_$]/.test(ch) || ch.charCodeAt(0) > 127;
}

function nextNonWs(src: string, from: number): string {
	let j = from;
	while (j < src.length && /\s/.test(src[j])) j++;
	return src[j] ?? '';
}