svelte-kit-vice/scripts/permutation-check.ts

/**
 * Permutation runner — cycles soma components through the state space declared
 * by their demo pages and validates morfo compliance after every transition.
 *
 * Closes the gap left by `morfo:check` (single state) and `smoke` (load-time
 * errors only): reactivity bugs like toolbar's A35 loop and form's A36
 * microtask loop manifest during state transitions, not on first paint.
 *
 * Opt-in per demo: tag interactive controls with `data-perm-step="N"`. The
 * runner discovers them, executes them in order, and re-validates morfo after
 * each settle. See `src/uix/morfo/PERMUTATION_RUNNER.md` for the full
 * authoring convention.
 *
 * Exit codes:
 *   0 — every demo with `data-perm-step` passes all permutations.
 *   1 — at least one permutation failed.
 *   2 — no dev server / no morfos found.
 */

import { chromium, type Page } from 'playwright';
import { readdirSync } from 'node:fs';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { dirname, join } from 'node:path';
import type { Morfo, MorfoPart, MorfoData } from '../src/uix/morfo/types';
import { validateMorfo } from '../src/uix/morfo/schema';

const __dirname = dirname(fileURLToPath(import.meta.url));
const MORFOS_DIR = join(__dirname, '..', 'src', 'uix', 'morfo', 'components');

// ── Shared with morfo-check.ts ──────────────────────────────────────────────

type Issue = { kind: 'missing' | 'bad-value' | 'undeclared'; message: string };

function flatParts(parts: readonly MorfoPart[]): MorfoPart[] {
	const out: MorfoPart[] = [];
	for (const p of parts) {
		out.push(p);
		if (p.parts && p.parts.length > 0) out.push(...flatParts(p.parts));
	}
	return out;
}

function dataAttrFor(kebab: string, part: MorfoPart): string {
	return part.kebab === 'provider' ? `data-${kebab}` : `data-${kebab}-${part.kebab}`;
}

async function validateMorfoAgainstDom(page: Page, morfo: Morfo): Promise<Issue[]> {
	const issues: Issue[] = [];
	const parts = flatParts(morfo.parts).filter((p) => p.kind === 'public');
	const allPartAttrs = new Set(parts.map((p) => dataAttrFor(morfo.kebab, p)));

	for (const part of parts) {
		const partAttr = dataAttrFor(morfo.kebab, part);
		const elementsAttrs = await page.$$eval(`[${partAttr}]`, (nodes) =>
			nodes.map((el) =>
				Array.from(el.attributes)
					.filter((a) => a.name.startsWith('data-'))
					.map((a) => ({ name: a.name, value: a.value }))
			)
		);

		if (elementsAttrs.length === 0) continue;

		const declared = new Set(part.data.map((d) => d.attr));
		declared.add(partAttr);
		for (const siblingAttr of allPartAttrs) declared.add(siblingAttr);
		const declaredByAttr = new Map<string, MorfoData>(part.data.map((d) => [d.attr, d]));

		for (const elAttrs of elementsAttrs) {
			for (const data of part.data) {
				if ((data.severity ?? 'required') !== 'required') continue;
				const found = elAttrs.find((a) => a.name === data.attr);
				if (!found) {
					issues.push({
						kind: 'missing',
						message: `${morfo.kebab}.${part.kebab}: required attr "${data.attr}" not emitted`
					});
				}
			}
			for (const { name, value } of elAttrs) {
				const decl = declaredByAttr.get(name);
				if (decl?.values && !decl.values.includes(value)) {
					issues.push({
						kind: 'bad-value',
						message: `${morfo.kebab}.${part.kebab}: "${name}" has value "${value}", morfo declares [${decl.values.join(', ')}]`
					});
				}
			}
			for (const { name } of elAttrs) {
				if (!name.startsWith(`data-${morfo.kebab}`) && !name.startsWith('data-')) continue;
				if (name.startsWith(`data-_`)) continue;
				if (declared.has(name)) continue;
				if (!name.startsWith(`data-${morfo.kebab}`)) continue;
				issues.push({
					kind: 'undeclared',
					message: `${morfo.kebab}.${part.kebab}: undeclared attr "${name}" (not in morfo)`
				});
			}
		}
	}

	return issues;
}

// ── Probe port + load morfos (copied from morfo-check) ──────────────────────

async function probePort(start: number, end: number): Promise<string | null> {
	for (let port = start; port <= end; port++) {
		try {
			const res = await fetch(`http://localhost:${port}/`, {
				signal: AbortSignal.timeout(500)
			});
			if (res.ok || res.status === 404 || res.status === 500) {
				return `http://localhost:${port}`;
			}
		} catch {
			// try next
		}
	}
	return null;
}

async function loadMorfos(): Promise<Morfo[]> {
	const files = readdirSync(MORFOS_DIR).filter(
		(f) => f.endsWith('.ts') && !f.endsWith('.test.ts')
	);
	const out: Morfo[] = [];
	for (const f of files) {
		const url = pathToFileURL(join(MORFOS_DIR, f)).href;
		const mod = (await import(url)) as Record<string, unknown>;
		for (const v of Object.values(mod)) {
			if (typeof v === 'object' && v !== null && 'kebab' in v && 'parts' in v) {
				out.push(validateMorfo(v));
			}
		}
	}
	return out;
}

// ── Permutation step schema ─────────────────────────────────────────────────

interface PermStep {
	index: number;
	label: string;
	mode: { kind: 'click' } | { kind: 'focus' } | { kind: 'type'; value: string } | { kind: 'key'; key: string };
	skipValidate: boolean;
	settle: number;
}

/** Parse a `data-perm-mode` attribute value. */
function parseMode(raw: string | null): PermStep['mode'] {
	if (!raw || raw === 'click') return { kind: 'click' };
	if (raw === 'focus') return { kind: 'focus' };
	const typeMatch = /^type="([^"]*)"$/.exec(raw);
	if (typeMatch) return { kind: 'type', value: typeMatch[1] };
	const keyMatch = /^key="([^"]+)"$/.exec(raw);
	if (keyMatch) return { kind: 'key', key: keyMatch[1] };
	return { kind: 'click' };
}

/**
 * Collect perm steps CURRENTLY present in the DOM, sorted by index. Called
 * before every step execution because new `data-perm-step` elements may
 * mount as earlier steps fire (e.g. a Dialog.Close inside a portal that
 * only exists after `open = true`).
 */
async function collectSteps(page: Page): Promise<PermStep[]> {
	const raw = await page.$$eval('[data-perm-step]', (els) =>
		els.map((el) => {
			const step = Number(el.getAttribute('data-perm-step') ?? '0');
			return {
				step,
				label:
					el.getAttribute('data-perm-label') ??
					(el.textContent ?? '').trim().slice(0, 40) ??
					`step ${step}`,
				mode: el.getAttribute('data-perm-mode'),
				skipValidate: el.hasAttribute('data-perm-skip-validate'),
				settle: Number(el.getAttribute('data-perm-settle') ?? '300')
			};
		})
	);
	// Deduplicate by step index — if multiple elements share an index, keep the
	// first (demos should not do this, but portals sometimes mirror content).
	const byIndex = new Map<number, (typeof raw)[number]>();
	for (const r of raw) if (!byIndex.has(r.step)) byIndex.set(r.step, r);
	return [...byIndex.values()]
		.sort((a, b) => a.step - b.step)
		.map((r) => ({
			index: r.step,
			label: r.label || `step ${r.step}`,
			mode: parseMode(r.mode),
			skipValidate: r.skipValidate,
			settle: Number.isFinite(r.settle) && r.settle > 0 ? r.settle : 300
		}));
}

/** Execute one step against the live page. Returns the label for logs. */
async function executeStep(page: Page, step: PermStep): Promise<void> {
	const selector = `[data-perm-step="${step.index}"]`;
	const locator = page.locator(selector).first();
	switch (step.mode.kind) {
		case 'click':
			await locator.click({ timeout: 3000 });
			break;
		case 'focus':
			await locator.focus({ timeout: 3000 });
			break;
		case 'type':
			await locator.focus({ timeout: 3000 });
			await page.keyboard.type(step.mode.value);
			break;
		case 'key':
			await locator.focus({ timeout: 3000 });
			await page.keyboard.press(step.mode.key);
			break;
	}
	await page.waitForTimeout(step.settle);
}

// ── URL variant matrix (v2 + v3 axis-at-a-time) ────────────────────────────

/**
 * Universal baseline: every instrumented demo runs at least under `ltr`
 * (plain URL) and `rtl` (`?perm.dir=rtl`). Additional per-component axes
 * are discovered at runtime from the demo's `<meta name="perm-axes">` tag;
 * each declared axis adds ONE variant that flips that axis from baseline
 * while keeping everything else at the default — linear, not cartesian,
 * so the matrix stays bounded.
 *
 * Supported axes (v3 initial set):
 *   dir          — ltr (baseline) + rtl
 *   orientation  — horizontal (baseline) + vertical
 *   disabled     — unset + true
 *   readonly     — unset + true
 *   invalid      — unset + true
 *   loading      — unset + true
 *
 * Demos opt in by listing the axes their `<script>` seed actually honours:
 *
 * ```svelte
 * <svelte:head>
 *   <meta name="perm-axes" content="dir,orientation,disabled" />
 * </svelte:head>
 * ```
 *
 * Without the meta tag, only the universal `dir` axis runs. With the tag,
 * the runner adds one variant per listed axis.
 */
type Variant = { suffix: string; label: string };

const AXIS_FLIP: Record<string, Variant> = {
	dir: { suffix: '?perm.dir=rtl', label: 'rtl' },
	orientation: { suffix: '?perm.orientation=vertical', label: 'orientation=vertical' },
	disabled: { suffix: '?perm.disabled=true', label: 'disabled' },
	readonly: { suffix: '?perm.readonly=true', label: 'readonly' },
	invalid: { suffix: '?perm.invalid=true', label: 'invalid' },
	loading: { suffix: '?perm.loading=true', label: 'loading' }
};

/** Read `<meta name="perm-axes" content="dir,orientation,...">` from the page. */
async function readDeclaredAxes(page: Page): Promise<string[]> {
	try {
		const content = await page.$eval(
			'meta[name="perm-axes"]',
			(el) => (el as HTMLMetaElement).content
		);
		return content
			.split(',')
			.map((s) => s.trim())
			.filter(Boolean);
	} catch {
		// Meta not present — demo opted out of per-axis coverage.
		return [];
	}
}

/**
 * Build the per-morfo variant list: baseline `ltr` + `rtl` (universal) +
 * one flipped variant per axis the demo declared.
 */
function buildVariants(declaredAxes: string[]): Variant[] {
	const base: Variant[] = [{ suffix: '', label: 'ltr' }, AXIS_FLIP.dir];
	const extra: Variant[] = [];
	for (const axis of declaredAxes) {
		if (axis === 'dir') continue; // already in base
		const v = AXIS_FLIP[axis];
		if (v) extra.push(v);
	}
	return [...base, ...extra];
}

// ── Main loop ───────────────────────────────────────────────────────────────

type StepResult =
	| { kind: 'pass'; label: string }
	| { kind: 'fail'; label: string; reason: string }
	| { kind: 'error'; label: string; err: string }
	| { kind: 'skipped'; label: string };

type VariantResult = {
	variant: string;
	url: string;
	stepsTotal: number;
	steps: StepResult[];
	pageErrors: string[];
};

type RouteResult = {
	morfo: string;
	variants: VariantResult[];
};

const BASE = process.argv[2] ?? (await probePort(5173, 5180));
if (!BASE) {
	console.error('Could not find a running dev server on 5173-5180.');
	console.error('Start it with `npm run dev` in another terminal.');
	process.exit(2);
}
console.error(`Using dev server at ${BASE}`);

const morfos = await loadMorfos();
if (morfos.length === 0) {
	console.error('No morfos found under src/uix/morfo/components/');
	process.exit(2);
}
console.error(`Loaded ${morfos.length} morfo${morfos.length === 1 ? '' : 's'}`);

const browser = await chromium.launch();
const ctx = await browser.newContext();
const results: RouteResult[] = [];
const skipped: string[] = [];

/** Run one URL variant: load, cycle click-driven steps, report. */
async function runVariant(
	base: string,
	url: string,
	variantLabel: string,
	morfo: Morfo
): Promise<VariantResult | 'no-steps'> {
	const page = await ctx.newPage();
	const pageErrors: string[] = [];
	page.on('pageerror', (e) => pageErrors.push(e.message));
	page.on('console', (m) => {
		if (m.type() === 'error') pageErrors.push(m.text());
	});

	const variantResult: VariantResult = {
		variant: variantLabel,
		url,
		stepsTotal: 0,
		steps: [],
		pageErrors: []
	};

	try {
		await page.goto(base + url, { waitUntil: 'networkidle', timeout: 20000 });
		await page.waitForTimeout(300);

		// Meta-only demos (no perm-steps) still get passive `[initial seed]`
		// validation at each URL variant — morfo compliance under RTL /
		// orientation flips is worth catching even without a click cycle.

		// Any errors from the URL-seeded initial state count against the variant.
		if (pageErrors.length > 0) {
			const captured = pageErrors.splice(0, pageErrors.length).join(' | ');
			variantResult.steps.push({
				kind: 'fail',
				label: '[initial seed]',
				reason: `pageerror: ${captured.slice(0, 200)}`
			});
		} else {
			// Validate morfo at the URL-seeded state BEFORE any clicks so v2
			// state-injection alone (no click sequence) is exercised.
			try {
				const seedIssues = await validateMorfoAgainstDom(page, morfo);
				if (seedIssues.length === 0) {
					variantResult.steps.push({ kind: 'pass', label: '[initial seed]' });
				} else {
					variantResult.steps.push({
						kind: 'fail',
						label: '[initial seed]',
						reason: seedIssues.map((i) => `[${i.kind}] ${i.message}`).join(' · ')
					});
				}
			} catch (e) {
				variantResult.steps.push({
					kind: 'error',
					label: '[initial seed]',
					err: `validation: ${(e as Error).message.slice(0, 120)}`
				});
			}
		}

		const executedIndices = new Set<number>();
		let safety = 50;
		while (safety-- > 0) {
			const current = await collectSteps(page);
			const next = current
				.filter((s) => !executedIndices.has(s.index))
				.sort((a, b) => a.index - b.index)[0];
			if (!next) break;

			const step = next;
			executedIndices.add(step.index);

			try {
				await executeStep(page, step);
			} catch (e) {
				variantResult.steps.push({
					kind: 'error',
					label: step.label,
					err: (e as Error).message.slice(0, 120)
				});
				continue;
			}

			if (pageErrors.length > 0) {
				const captured = pageErrors.splice(0, pageErrors.length).join(' | ');
				variantResult.steps.push({
					kind: 'fail',
					label: step.label,
					reason: `pageerror: ${captured.slice(0, 200)}`
				});
				continue;
			}

			if (step.skipValidate) {
				variantResult.steps.push({ kind: 'pass', label: step.label + ' (no-validate)' });
				continue;
			}

			try {
				const issues = await validateMorfoAgainstDom(page, morfo);
				if (issues.length === 0) {
					variantResult.steps.push({ kind: 'pass', label: step.label });
				} else {
					variantResult.steps.push({
						kind: 'fail',
						label: step.label,
						reason: issues.map((i) => `[${i.kind}] ${i.message}`).join(' · ')
					});
				}
			} catch (e) {
				variantResult.steps.push({
					kind: 'error',
					label: step.label,
					err: `validation: ${(e as Error).message.slice(0, 120)}`
				});
			}
		}

		variantResult.pageErrors = pageErrors;
		variantResult.stepsTotal = variantResult.steps.length;
	} catch (e) {
		variantResult.steps.push({
			kind: 'error',
			label: '[navigation]',
			err: (e as Error).message.slice(0, 120)
		});
	} finally {
		await page.close();
	}

	return variantResult;
}

/**
 * Discover opt-in signals from the demo page: the declared axes (`perm-axes`
 * meta) and whether any `data-perm-step` element exists. The demo is
 * considered "instrumented" if it declares at least one of these.
 */
async function probeDemo(
	base: string,
	morfoKebab: string
): Promise<{ axes: string[]; hasSteps: boolean } | 'no-demo'> {
	const page = await ctx.newPage();
	try {
		const res = await page.goto(`${base}/test/soma/${morfoKebab}`, {
			waitUntil: 'networkidle',
			timeout: 20000
		});
		if (!res || res.status() >= 400) return 'no-demo';
		await page.waitForTimeout(200);
		const axes = await readDeclaredAxes(page);
		const hasSteps = (await collectSteps(page)).length > 0;
		return { axes, hasSteps };
	} catch {
		return 'no-demo';
	} finally {
		await page.close();
	}
}

for (const morfo of morfos) {
	const variantResults: VariantResult[] = [];

	const probe = await probeDemo(BASE, morfo.kebab);
	if (probe === 'no-demo') {
		skipped.push(morfo.kebab);
		continue;
	}

	// Opt-in signal: either a `perm-axes` meta OR at least one
	// `data-perm-step` element. Meta-only demos get passive `[initial seed]`
	// validation at each URL variant; step-only demos get the v1 click cycle
	// at baseline; declaring both is the full v1+v2+v3 treatment.
	if (probe.axes.length === 0 && !probe.hasSteps) {
		skipped.push(morfo.kebab);
		continue;
	}

	const variants = buildVariants(probe.axes);

	for (const v of variants) {
		const url = `/test/soma/${morfo.kebab}${v.suffix}`;
		const r = await runVariant(BASE, url, v.label, morfo);
		if (r === 'no-steps') continue;
		variantResults.push(r);
	}

	if (variantResults.length === 0) {
		skipped.push(morfo.kebab);
		continue;
	}

	results.push({ morfo: morfo.kebab, variants: variantResults });
}

await browser.close();

// ── Report ──────────────────────────────────────────────────────────────────

console.log('');
let passedRoutes = 0;
let failedRoutes = 0;
let totalSteps = 0;
let totalFailed = 0;
for (const r of results) {
	const bad = r.variants
		.flatMap((v) => v.steps)
		.filter((s) => s.kind !== 'pass');
	totalSteps += r.variants.reduce((acc, v) => acc + v.steps.length, 0);
	totalFailed += bad.length;

	if (bad.length === 0) {
		const variantCount = r.variants.length;
		const stepCount = r.variants.reduce((acc, v) => acc + v.steps.length, 0);
		console.log(
			`PASS  ${r.morfo.padEnd(20)}  ${variantCount} variant${variantCount === 1 ? '' : 's'} · ${stepCount} step${stepCount === 1 ? '' : 's'}`
		);
		for (const v of r.variants) {
			console.log(`      [${v.variant}]  ${v.url}`);
			for (const s of v.steps) console.log(`         ·  ${s.label}`);
		}
		passedRoutes++;
	} else {
		console.log(`FAIL  ${r.morfo.padEnd(20)}  ${bad.length} step${bad.length === 1 ? '' : 's'} failed`);
		for (const v of r.variants) {
			console.log(`      [${v.variant}]  ${v.url}`);
			for (const s of v.steps) {
				if (s.kind === 'pass') console.log(`         ·  ${s.label}`);
				else if (s.kind === 'skipped') console.log(`         ·  ${s.label}  (skipped)`);
				else if (s.kind === 'fail') console.log(`         ✖  ${s.label}  ${s.reason}`);
				else console.log(`         ✖  ${s.label}  ERROR ${s.err}`);
			}
		}
		failedRoutes++;
	}
}

console.log('');
if (skipped.length > 0) {
	console.log(
		`SKIPPED (${skipped.length}): ${skipped.slice(0, 10).join(', ')}${skipped.length > 10 ? ', …' : ''}`
	);
	console.log('  (demo has no `data-perm-step` annotations yet — see src/uix/morfo/PERMUTATION_RUNNER.md)');
}
console.log('');
const totalVariants = results.reduce((acc, r) => acc + r.variants.length, 0);
const summary =
	failedRoutes === 0
		? `All ${passedRoutes} instrumented demo${passedRoutes === 1 ? '' : 's'} passed. (${totalSteps} step${totalSteps === 1 ? '' : 's'} across ${totalVariants} URL variant${totalVariants === 1 ? '' : 's'}.)`
		: `${failedRoutes} demo${failedRoutes === 1 ? '' : 's'} failed (${totalFailed}/${totalSteps} steps): ${results
				.filter((r) => r.variants.some((v) => v.steps.some((s) => s.kind !== 'pass')))
				.map((r) => r.morfo)
				.join(', ')}`;
console.log(summary);
process.exit(failedRoutes === 0 ? 0 : 1);