You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
574 lines
18 KiB
574 lines
18 KiB
/**
|
|
* Permutation runner — cycles soma components through the state space declared
|
|
* by their demo pages and validates morfo compliance after every transition.
|
|
*
|
|
* Closes the gap left by `morfo:check` (single state) and `smoke` (load-time
|
|
* errors only): reactivity bugs like toolbar's A35 loop and form's A36
|
|
* microtask loop manifest during state transitions, not on first paint.
|
|
*
|
|
* Opt-in per demo: tag interactive controls with `data-perm-step="N"`. The
|
|
* runner discovers them, executes them in order, and re-validates morfo after
|
|
* each settle. See `src/uix/morfo/PERMUTATION_RUNNER.md` for the full
|
|
* authoring convention.
|
|
*
|
|
* Exit codes:
|
|
* 0 — every demo with `data-perm-step` passes all permutations.
|
|
* 1 — at least one permutation failed.
|
|
* 2 — no dev server / no morfos found.
|
|
*/
|
|
|
|
import { chromium, type Page } from 'playwright';
|
|
import { readdirSync } from 'node:fs';
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
import { dirname, join } from 'node:path';
|
|
import type { Morfo, MorfoPart, MorfoData } from '../src/uix/morfo/types';
|
|
import { validateMorfo } from '../src/uix/morfo/schema';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const MORFOS_DIR = join(__dirname, '..', 'src', 'uix', 'morfo', 'components');
|
|
|
|
// ── Shared with morfo-check.ts ──────────────────────────────────────────────
|
|
|
|
type Issue = { kind: 'missing' | 'bad-value' | 'undeclared'; message: string };
|
|
|
|
function flatParts(parts: readonly MorfoPart[]): MorfoPart[] {
|
|
const out: MorfoPart[] = [];
|
|
for (const p of parts) {
|
|
out.push(p);
|
|
if (p.parts && p.parts.length > 0) out.push(...flatParts(p.parts));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function dataAttrFor(kebab: string, part: MorfoPart): string {
|
|
return part.kebab === 'provider' ? `data-${kebab}` : `data-${kebab}-${part.kebab}`;
|
|
}
|
|
|
|
async function validateMorfoAgainstDom(page: Page, morfo: Morfo): Promise<Issue[]> {
|
|
const issues: Issue[] = [];
|
|
const parts = flatParts(morfo.parts).filter((p) => p.kind === 'public');
|
|
const allPartAttrs = new Set(parts.map((p) => dataAttrFor(morfo.kebab, p)));
|
|
|
|
for (const part of parts) {
|
|
const partAttr = dataAttrFor(morfo.kebab, part);
|
|
const elementsAttrs = await page.$$eval(`[${partAttr}]`, (nodes) =>
|
|
nodes.map((el) =>
|
|
Array.from(el.attributes)
|
|
.filter((a) => a.name.startsWith('data-'))
|
|
.map((a) => ({ name: a.name, value: a.value }))
|
|
)
|
|
);
|
|
|
|
if (elementsAttrs.length === 0) continue;
|
|
|
|
const declared = new Set(part.data.map((d) => d.attr));
|
|
declared.add(partAttr);
|
|
for (const siblingAttr of allPartAttrs) declared.add(siblingAttr);
|
|
const declaredByAttr = new Map<string, MorfoData>(part.data.map((d) => [d.attr, d]));
|
|
|
|
for (const elAttrs of elementsAttrs) {
|
|
for (const data of part.data) {
|
|
if ((data.severity ?? 'required') !== 'required') continue;
|
|
const found = elAttrs.find((a) => a.name === data.attr);
|
|
if (!found) {
|
|
issues.push({
|
|
kind: 'missing',
|
|
message: `${morfo.kebab}.${part.kebab}: required attr "${data.attr}" not emitted`
|
|
});
|
|
}
|
|
}
|
|
for (const { name, value } of elAttrs) {
|
|
const decl = declaredByAttr.get(name);
|
|
if (decl?.values && !decl.values.includes(value)) {
|
|
issues.push({
|
|
kind: 'bad-value',
|
|
message: `${morfo.kebab}.${part.kebab}: "${name}" has value "${value}", morfo declares [${decl.values.join(', ')}]`
|
|
});
|
|
}
|
|
}
|
|
for (const { name } of elAttrs) {
|
|
if (!name.startsWith(`data-${morfo.kebab}`) && !name.startsWith('data-')) continue;
|
|
if (name.startsWith(`data-_`)) continue;
|
|
if (declared.has(name)) continue;
|
|
if (!name.startsWith(`data-${morfo.kebab}`)) continue;
|
|
issues.push({
|
|
kind: 'undeclared',
|
|
message: `${morfo.kebab}.${part.kebab}: undeclared attr "${name}" (not in morfo)`
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return issues;
|
|
}
|
|
|
|
// ── Probe port + load morfos (copied from morfo-check) ──────────────────────
|
|
|
|
async function probePort(start: number, end: number): Promise<string | null> {
|
|
for (let port = start; port <= end; port++) {
|
|
try {
|
|
const res = await fetch(`http://localhost:${port}/`, {
|
|
signal: AbortSignal.timeout(500)
|
|
});
|
|
if (res.ok || res.status === 404 || res.status === 500) {
|
|
return `http://localhost:${port}`;
|
|
}
|
|
} catch {
|
|
// try next
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function loadMorfos(): Promise<Morfo[]> {
|
|
const files = readdirSync(MORFOS_DIR).filter(
|
|
(f) => f.endsWith('.ts') && !f.endsWith('.test.ts')
|
|
);
|
|
const out: Morfo[] = [];
|
|
for (const f of files) {
|
|
const url = pathToFileURL(join(MORFOS_DIR, f)).href;
|
|
const mod = (await import(url)) as Record<string, unknown>;
|
|
for (const v of Object.values(mod)) {
|
|
if (typeof v === 'object' && v !== null && 'kebab' in v && 'parts' in v) {
|
|
out.push(validateMorfo(v));
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ── Permutation step schema ─────────────────────────────────────────────────
|
|
|
|
interface PermStep {
|
|
index: number;
|
|
label: string;
|
|
mode: { kind: 'click' } | { kind: 'focus' } | { kind: 'type'; value: string } | { kind: 'key'; key: string };
|
|
skipValidate: boolean;
|
|
settle: number;
|
|
}
|
|
|
|
/** Parse a `data-perm-mode` attribute value. */
|
|
function parseMode(raw: string | null): PermStep['mode'] {
|
|
if (!raw || raw === 'click') return { kind: 'click' };
|
|
if (raw === 'focus') return { kind: 'focus' };
|
|
const typeMatch = /^type="([^"]*)"$/.exec(raw);
|
|
if (typeMatch) return { kind: 'type', value: typeMatch[1] };
|
|
const keyMatch = /^key="([^"]+)"$/.exec(raw);
|
|
if (keyMatch) return { kind: 'key', key: keyMatch[1] };
|
|
return { kind: 'click' };
|
|
}
|
|
|
|
/**
|
|
* Collect perm steps CURRENTLY present in the DOM, sorted by index. Called
|
|
* before every step execution because new `data-perm-step` elements may
|
|
* mount as earlier steps fire (e.g. a Dialog.Close inside a portal that
|
|
* only exists after `open = true`).
|
|
*/
|
|
async function collectSteps(page: Page): Promise<PermStep[]> {
|
|
const raw = await page.$$eval('[data-perm-step]', (els) =>
|
|
els.map((el) => {
|
|
const step = Number(el.getAttribute('data-perm-step') ?? '0');
|
|
return {
|
|
step,
|
|
label:
|
|
el.getAttribute('data-perm-label') ??
|
|
(el.textContent ?? '').trim().slice(0, 40) ??
|
|
`step ${step}`,
|
|
mode: el.getAttribute('data-perm-mode'),
|
|
skipValidate: el.hasAttribute('data-perm-skip-validate'),
|
|
settle: Number(el.getAttribute('data-perm-settle') ?? '300')
|
|
};
|
|
})
|
|
);
|
|
// Deduplicate by step index — if multiple elements share an index, keep the
|
|
// first (demos should not do this, but portals sometimes mirror content).
|
|
const byIndex = new Map<number, (typeof raw)[number]>();
|
|
for (const r of raw) if (!byIndex.has(r.step)) byIndex.set(r.step, r);
|
|
return [...byIndex.values()]
|
|
.sort((a, b) => a.step - b.step)
|
|
.map((r) => ({
|
|
index: r.step,
|
|
label: r.label || `step ${r.step}`,
|
|
mode: parseMode(r.mode),
|
|
skipValidate: r.skipValidate,
|
|
settle: Number.isFinite(r.settle) && r.settle > 0 ? r.settle : 300
|
|
}));
|
|
}
|
|
|
|
/** Execute one step against the live page. Returns the label for logs. */
|
|
async function executeStep(page: Page, step: PermStep): Promise<void> {
|
|
const selector = `[data-perm-step="${step.index}"]`;
|
|
const locator = page.locator(selector).first();
|
|
switch (step.mode.kind) {
|
|
case 'click':
|
|
await locator.click({ timeout: 3000 });
|
|
break;
|
|
case 'focus':
|
|
await locator.focus({ timeout: 3000 });
|
|
break;
|
|
case 'type':
|
|
await locator.focus({ timeout: 3000 });
|
|
await page.keyboard.type(step.mode.value);
|
|
break;
|
|
case 'key':
|
|
await locator.focus({ timeout: 3000 });
|
|
await page.keyboard.press(step.mode.key);
|
|
break;
|
|
}
|
|
await page.waitForTimeout(step.settle);
|
|
}
|
|
|
|
// ── URL variant matrix (v2 + v3 axis-at-a-time) ────────────────────────────
|
|
|
|
/**
|
|
* Universal baseline: every instrumented demo runs at least under `ltr`
|
|
* (plain URL) and `rtl` (`?perm.dir=rtl`). Additional per-component axes
|
|
* are discovered at runtime from the demo's `<meta name="perm-axes">` tag;
|
|
* each declared axis adds ONE variant that flips that axis from baseline
|
|
* while keeping everything else at the default — linear, not cartesian,
|
|
* so the matrix stays bounded.
|
|
*
|
|
* Supported axes (v3 initial set):
|
|
* dir — ltr (baseline) + rtl
|
|
* orientation — horizontal (baseline) + vertical
|
|
* disabled — unset + true
|
|
* readonly — unset + true
|
|
* invalid — unset + true
|
|
* loading — unset + true
|
|
*
|
|
* Demos opt in by listing the axes their `<script>` seed actually honours:
|
|
*
|
|
* ```svelte
|
|
* <svelte:head>
|
|
* <meta name="perm-axes" content="dir,orientation,disabled" />
|
|
* </svelte:head>
|
|
* ```
|
|
*
|
|
* Without the meta tag, only the universal `dir` axis runs. With the tag,
|
|
* the runner adds one variant per listed axis.
|
|
*/
|
|
type Variant = { suffix: string; label: string };
|
|
|
|
const AXIS_FLIP: Record<string, Variant> = {
|
|
dir: { suffix: '?perm.dir=rtl', label: 'rtl' },
|
|
orientation: { suffix: '?perm.orientation=vertical', label: 'orientation=vertical' },
|
|
disabled: { suffix: '?perm.disabled=true', label: 'disabled' },
|
|
readonly: { suffix: '?perm.readonly=true', label: 'readonly' },
|
|
invalid: { suffix: '?perm.invalid=true', label: 'invalid' },
|
|
loading: { suffix: '?perm.loading=true', label: 'loading' }
|
|
};
|
|
|
|
/** Read `<meta name="perm-axes" content="dir,orientation,...">` from the page. */
|
|
async function readDeclaredAxes(page: Page): Promise<string[]> {
|
|
try {
|
|
const content = await page.$eval(
|
|
'meta[name="perm-axes"]',
|
|
(el) => (el as HTMLMetaElement).content
|
|
);
|
|
return content
|
|
.split(',')
|
|
.map((s) => s.trim())
|
|
.filter(Boolean);
|
|
} catch {
|
|
// Meta not present — demo opted out of per-axis coverage.
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build the per-morfo variant list: baseline `ltr` + `rtl` (universal) +
|
|
* one flipped variant per axis the demo declared.
|
|
*/
|
|
function buildVariants(declaredAxes: string[]): Variant[] {
|
|
const base: Variant[] = [{ suffix: '', label: 'ltr' }, AXIS_FLIP.dir];
|
|
const extra: Variant[] = [];
|
|
for (const axis of declaredAxes) {
|
|
if (axis === 'dir') continue; // already in base
|
|
const v = AXIS_FLIP[axis];
|
|
if (v) extra.push(v);
|
|
}
|
|
return [...base, ...extra];
|
|
}
|
|
|
|
// ── Main loop ───────────────────────────────────────────────────────────────
|
|
|
|
type StepResult =
|
|
| { kind: 'pass'; label: string }
|
|
| { kind: 'fail'; label: string; reason: string }
|
|
| { kind: 'error'; label: string; err: string }
|
|
| { kind: 'skipped'; label: string };
|
|
|
|
type VariantResult = {
|
|
variant: string;
|
|
url: string;
|
|
stepsTotal: number;
|
|
steps: StepResult[];
|
|
pageErrors: string[];
|
|
};
|
|
|
|
type RouteResult = {
|
|
morfo: string;
|
|
variants: VariantResult[];
|
|
};
|
|
|
|
const BASE = process.argv[2] ?? (await probePort(5173, 5180));
|
|
if (!BASE) {
|
|
console.error('Could not find a running dev server on 5173-5180.');
|
|
console.error('Start it with `npm run dev` in another terminal.');
|
|
process.exit(2);
|
|
}
|
|
console.error(`Using dev server at ${BASE}`);
|
|
|
|
const morfos = await loadMorfos();
|
|
if (morfos.length === 0) {
|
|
console.error('No morfos found under src/uix/morfo/components/');
|
|
process.exit(2);
|
|
}
|
|
console.error(`Loaded ${morfos.length} morfo${morfos.length === 1 ? '' : 's'}`);
|
|
|
|
const browser = await chromium.launch();
|
|
const ctx = await browser.newContext();
|
|
const results: RouteResult[] = [];
|
|
const skipped: string[] = [];
|
|
|
|
/** Run one URL variant: load, cycle click-driven steps, report. */
|
|
async function runVariant(
|
|
base: string,
|
|
url: string,
|
|
variantLabel: string,
|
|
morfo: Morfo
|
|
): Promise<VariantResult | 'no-steps'> {
|
|
const page = await ctx.newPage();
|
|
const pageErrors: string[] = [];
|
|
page.on('pageerror', (e) => pageErrors.push(e.message));
|
|
page.on('console', (m) => {
|
|
if (m.type() === 'error') pageErrors.push(m.text());
|
|
});
|
|
|
|
const variantResult: VariantResult = {
|
|
variant: variantLabel,
|
|
url,
|
|
stepsTotal: 0,
|
|
steps: [],
|
|
pageErrors: []
|
|
};
|
|
|
|
try {
|
|
await page.goto(base + url, { waitUntil: 'networkidle', timeout: 20000 });
|
|
await page.waitForTimeout(300);
|
|
|
|
const initialSteps = await collectSteps(page);
|
|
if (initialSteps.length === 0) {
|
|
await page.close();
|
|
return 'no-steps';
|
|
}
|
|
|
|
// Any errors from the URL-seeded initial state count against the variant.
|
|
if (pageErrors.length > 0) {
|
|
const captured = pageErrors.splice(0, pageErrors.length).join(' | ');
|
|
variantResult.steps.push({
|
|
kind: 'fail',
|
|
label: '[initial seed]',
|
|
reason: `pageerror: ${captured.slice(0, 200)}`
|
|
});
|
|
} else {
|
|
// Validate morfo at the URL-seeded state BEFORE any clicks so v2
|
|
// state-injection alone (no click sequence) is exercised.
|
|
try {
|
|
const seedIssues = await validateMorfoAgainstDom(page, morfo);
|
|
if (seedIssues.length === 0) {
|
|
variantResult.steps.push({ kind: 'pass', label: '[initial seed]' });
|
|
} else {
|
|
variantResult.steps.push({
|
|
kind: 'fail',
|
|
label: '[initial seed]',
|
|
reason: seedIssues.map((i) => `[${i.kind}] ${i.message}`).join(' · ')
|
|
});
|
|
}
|
|
} catch (e) {
|
|
variantResult.steps.push({
|
|
kind: 'error',
|
|
label: '[initial seed]',
|
|
err: `validation: ${(e as Error).message.slice(0, 120)}`
|
|
});
|
|
}
|
|
}
|
|
|
|
const executedIndices = new Set<number>();
|
|
let safety = 50;
|
|
while (safety-- > 0) {
|
|
const current = await collectSteps(page);
|
|
const next = current
|
|
.filter((s) => !executedIndices.has(s.index))
|
|
.sort((a, b) => a.index - b.index)[0];
|
|
if (!next) break;
|
|
|
|
const step = next;
|
|
executedIndices.add(step.index);
|
|
|
|
try {
|
|
await executeStep(page, step);
|
|
} catch (e) {
|
|
variantResult.steps.push({
|
|
kind: 'error',
|
|
label: step.label,
|
|
err: (e as Error).message.slice(0, 120)
|
|
});
|
|
continue;
|
|
}
|
|
|
|
if (pageErrors.length > 0) {
|
|
const captured = pageErrors.splice(0, pageErrors.length).join(' | ');
|
|
variantResult.steps.push({
|
|
kind: 'fail',
|
|
label: step.label,
|
|
reason: `pageerror: ${captured.slice(0, 200)}`
|
|
});
|
|
continue;
|
|
}
|
|
|
|
if (step.skipValidate) {
|
|
variantResult.steps.push({ kind: 'pass', label: step.label + ' (no-validate)' });
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
const issues = await validateMorfoAgainstDom(page, morfo);
|
|
if (issues.length === 0) {
|
|
variantResult.steps.push({ kind: 'pass', label: step.label });
|
|
} else {
|
|
variantResult.steps.push({
|
|
kind: 'fail',
|
|
label: step.label,
|
|
reason: issues.map((i) => `[${i.kind}] ${i.message}`).join(' · ')
|
|
});
|
|
}
|
|
} catch (e) {
|
|
variantResult.steps.push({
|
|
kind: 'error',
|
|
label: step.label,
|
|
err: `validation: ${(e as Error).message.slice(0, 120)}`
|
|
});
|
|
}
|
|
}
|
|
|
|
variantResult.pageErrors = pageErrors;
|
|
variantResult.stepsTotal = variantResult.steps.length;
|
|
} catch (e) {
|
|
variantResult.steps.push({
|
|
kind: 'error',
|
|
label: '[navigation]',
|
|
err: (e as Error).message.slice(0, 120)
|
|
});
|
|
} finally {
|
|
await page.close();
|
|
}
|
|
|
|
return variantResult;
|
|
}
|
|
|
|
/** Discover axes declared by the demo via `<meta name="perm-axes">`. */
|
|
async function discoverAxes(base: string, morfoKebab: string): Promise<string[] | 'no-demo'> {
|
|
const page = await ctx.newPage();
|
|
try {
|
|
const res = await page.goto(`${base}/test/soma/${morfoKebab}`, {
|
|
waitUntil: 'networkidle',
|
|
timeout: 20000
|
|
});
|
|
if (!res || res.status() >= 400) return 'no-demo';
|
|
return await readDeclaredAxes(page);
|
|
} catch {
|
|
return 'no-demo';
|
|
} finally {
|
|
await page.close();
|
|
}
|
|
}
|
|
|
|
for (const morfo of morfos) {
|
|
const variantResults: VariantResult[] = [];
|
|
let anyStepped = false;
|
|
|
|
const axes = await discoverAxes(BASE, morfo.kebab);
|
|
if (axes === 'no-demo') {
|
|
skipped.push(morfo.kebab);
|
|
continue;
|
|
}
|
|
const variants = buildVariants(axes);
|
|
|
|
for (const v of variants) {
|
|
const url = `/test/soma/${morfo.kebab}${v.suffix}`;
|
|
const r = await runVariant(BASE, url, v.label, morfo);
|
|
if (r === 'no-steps') continue;
|
|
anyStepped = true;
|
|
variantResults.push(r);
|
|
}
|
|
|
|
if (!anyStepped) {
|
|
skipped.push(morfo.kebab);
|
|
continue;
|
|
}
|
|
|
|
results.push({ morfo: morfo.kebab, variants: variantResults });
|
|
}
|
|
|
|
await browser.close();
|
|
|
|
// ── Report ──────────────────────────────────────────────────────────────────
|
|
|
|
console.log('');
|
|
let passedRoutes = 0;
|
|
let failedRoutes = 0;
|
|
let totalSteps = 0;
|
|
let totalFailed = 0;
|
|
for (const r of results) {
|
|
const bad = r.variants
|
|
.flatMap((v) => v.steps)
|
|
.filter((s) => s.kind !== 'pass');
|
|
totalSteps += r.variants.reduce((acc, v) => acc + v.steps.length, 0);
|
|
totalFailed += bad.length;
|
|
|
|
if (bad.length === 0) {
|
|
const variantCount = r.variants.length;
|
|
const stepCount = r.variants.reduce((acc, v) => acc + v.steps.length, 0);
|
|
console.log(
|
|
`PASS ${r.morfo.padEnd(20)} ${variantCount} variant${variantCount === 1 ? '' : 's'} · ${stepCount} step${stepCount === 1 ? '' : 's'}`
|
|
);
|
|
for (const v of r.variants) {
|
|
console.log(` [${v.variant}] ${v.url}`);
|
|
for (const s of v.steps) console.log(` · ${s.label}`);
|
|
}
|
|
passedRoutes++;
|
|
} else {
|
|
console.log(`FAIL ${r.morfo.padEnd(20)} ${bad.length} step${bad.length === 1 ? '' : 's'} failed`);
|
|
for (const v of r.variants) {
|
|
console.log(` [${v.variant}] ${v.url}`);
|
|
for (const s of v.steps) {
|
|
if (s.kind === 'pass') console.log(` · ${s.label}`);
|
|
else if (s.kind === 'skipped') console.log(` · ${s.label} (skipped)`);
|
|
else if (s.kind === 'fail') console.log(` ✖ ${s.label} ${s.reason}`);
|
|
else console.log(` ✖ ${s.label} ERROR ${s.err}`);
|
|
}
|
|
}
|
|
failedRoutes++;
|
|
}
|
|
}
|
|
|
|
console.log('');
|
|
if (skipped.length > 0) {
|
|
console.log(
|
|
`SKIPPED (${skipped.length}): ${skipped.slice(0, 10).join(', ')}${skipped.length > 10 ? ', …' : ''}`
|
|
);
|
|
console.log(' (demo has no `data-perm-step` annotations yet — see src/uix/morfo/PERMUTATION_RUNNER.md)');
|
|
}
|
|
console.log('');
|
|
const totalVariants = results.reduce((acc, r) => acc + r.variants.length, 0);
|
|
const summary =
|
|
failedRoutes === 0
|
|
? `All ${passedRoutes} instrumented demo${passedRoutes === 1 ? '' : 's'} passed. (${totalSteps} step${totalSteps === 1 ? '' : 's'} across ${totalVariants} URL variant${totalVariants === 1 ? '' : 's'}.)`
|
|
: `${failedRoutes} demo${failedRoutes === 1 ? '' : 's'} failed (${totalFailed}/${totalSteps} steps): ${results
|
|
.filter((r) => r.variants.some((v) => v.steps.some((s) => s.kind !== 'pass')))
|
|
.map((r) => r.morfo)
|
|
.join(', ')}`;
|
|
console.log(summary);
|
|
process.exit(failedRoutes === 0 ? 0 : 1);
|