DateKeys-App/scripts/sync-testdata.mjs

#!/usr/bin/env node
// Vendors testdata/ from the Go reference implementation (g.activething.com/go/DateKeys)
// at one pinned commit and verifies it. The Go fixtures and vectors are the source of
// truth: this project never generates its own. The word lists of the random words of a
// key of words (wordkey/lists of the Go repository) come from the same commit, into
// wordlists/, and so does the recovery annex that /create saves next to a capsule
// (annex/).
//
//   node scripts/sync-testdata.mjs sync  [--repo ../datekeys-go] [--commit HEAD]
//   node scripts/sync-testdata.mjs check [--against ../datekeys-go]
//
// sync copies every blob under testdata/, wordkey/lists/ and annex/ at the commit (read with
// git, so uncommitted changes in the Go checkout are never picked up) and writes the
// SOURCE.json of each copy with the full commit id and the SHA-256 of each file. check
// verifies that the files on disk match SOURCE.json exactly, with no missing or extra
// files; with --against it also re-reads every blob from the Go repository at the
// recorded commit. No dependencies: Node and git only.

import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';

const MODULE = 'g.activething.com/go/DateKeys';
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const DEFAULT_REPO = resolve(ROOT, '..', 'datekeys-go');

// The copies: the directory of the Go repository, and the local one with its SOURCE.json.
const TREES = [
  { name: 'testdata', from: 'testdata', dir: join(ROOT, 'testdata') },
  { name: 'wordlists', from: 'wordkey/lists', dir: join(ROOT, 'wordlists') },
  { name: 'annex', from: 'annex', dir: join(ROOT, 'annex') },
].map(tree => ({ ...tree, source: join(tree.dir, 'SOURCE.json') }));

function git(repo, args) {
  return execFileSync('git', ['-C', repo, ...args], { encoding: 'buffer', maxBuffer: 1 << 30 });
}

function sha256(buf) {
  return createHash('sha256').update(buf).digest('hex');
}

// Files under the directory `from` at commit, as paths relative to it with forward slashes.
function listBlobs(repo, commit, from) {
  const out = git(repo, ['ls-tree', '-r', '-z', '--full-tree', commit, '--', from]).toString('utf8');
  return out.split('\0').filter(Boolean).map(line => {
    const [meta, path] = line.split('\t');
    const [mode, type] = meta.split(' ');
    if (type !== 'blob' || mode === '120000') throw new Error(`unsupported entry ${path} (${mode} ${type})`);
    return path.slice(from.length + 1);
  }).sort();
}

function readBlob(repo, commit, from, path) {
  return git(repo, ['cat-file', 'blob', `${commit}:${from}/${path}`]);
}

// Files on disk under the directory of a copy, excluding its SOURCE.json.
function listLocal(tree, dir = tree.dir) {
  let files = [];
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const full = join(dir, entry.name);
    if (entry.isDirectory()) files = files.concat(listLocal(tree, full));
    else if (full !== tree.source) files.push(relative(tree.dir, full).split(sep).join('/'));
  }
  return files.sort();
}

function sync(repo, rev) {
  const commit = git(repo, ['rev-parse', '--verify', `${rev}^{commit}`]).toString('utf8').trim();
  // Read everything before touching the local copies, so a failed read leaves them intact.
  const read = TREES.map(tree => {
    const paths = listBlobs(repo, commit, tree.from);
    if (paths.length === 0) throw new Error(`no ${tree.from}/ at ${commit}`);
    return { tree, blobs: paths.map(path => [path, readBlob(repo, commit, tree.from, path)]) };
  });
  for (const { tree, blobs } of read) {
    rmSync(tree.dir, { recursive: true, force: true });
    const files = {};
    for (const [path, data] of blobs) {
      const target = join(tree.dir, ...path.split('/'));
      mkdirSync(dirname(target), { recursive: true });
      writeFileSync(target, data);
      files[path] = sha256(data);
    }
    writeFileSync(tree.source, JSON.stringify({ module: MODULE, commit, files }, null, 2) + '\n');
    console.log(`${tree.name}: ${blobs.length} files from ${MODULE} at ${commit}`);
  }
}

// The differences between a copy and its SOURCE.json, and with --against between
// SOURCE.json and the Go repository at its commit.
function checkTree(tree, against) {
  const source = JSON.parse(readFileSync(tree.source, 'utf8'));
  const expected = Object.keys(source.files).sort();
  const actual = listLocal(tree);
  const errors = [];
  for (const path of expected) {
    if (!actual.includes(path)) { errors.push(`missing ${path}`); continue; }
    if (sha256(readFileSync(join(tree.dir, ...path.split('/')))) !== source.files[path]) errors.push(`modified ${path}`);
  }
  for (const path of actual) if (!(path in source.files)) errors.push(`extra ${path}`);
  if (against) {
    const upstream = listBlobs(against, source.commit, tree.from);
    if (upstream.join('\n') !== expected.join('\n')) errors.push(`file list differs from ${source.commit}`);
    for (const path of upstream) {
      if (sha256(readBlob(against, source.commit, tree.from, path)) !== source.files[path]) errors.push(`differs from upstream ${path}`);
    }
  }
  return { source, files: expected.length, errors };
}

function check(against) {
  const results = TREES.map(tree => ({ tree, ...checkTree(tree, against) }));
  let failed = false;
  for (const { tree, errors } of results) {
    for (const e of errors) console.error(`${tree.name}: ${e}`);
    failed ||= errors.length > 0;
  }
  if (failed) process.exit(1);
  for (const { tree, source, files } of results) {
    console.log(`${tree.name}: ${files} files match ${source.module} at ${source.commit}${against ? ' (verified against the repository)' : ''}`);
  }
}

function option(args, name, fallback) {
  const i = args.indexOf(name);
  if (i === -1) return fallback;
  if (i + 1 >= args.length) throw new Error(`${name} needs a value`);
  return args[i + 1];
}

const [command, ...args] = process.argv.slice(2);
try {
  if (command === 'sync') sync(resolve(option(args, '--repo', DEFAULT_REPO)), option(args, '--commit', 'HEAD'));
  else if (command === 'check') {
    const against = option(args, '--against', null);
    check(against && resolve(against));
  } else {
    console.error('usage: sync-testdata.mjs sync [--repo PATH] [--commit REV] | check [--against PATH]');
    process.exit(2);
  }
} catch (err) {
  console.error(`testdata: ${err.message}`);
  process.exit(1);
}