dateKeys-dart/tool/sync_testdata.dart

// Vendors testdata/, the word lists of wordkey/lists/ and the recovery annex
// of annex/ from the Go reference implementation
// (g.activething.com/go/DateKeys) at one pinned commit and verifies them. The
// Go fixtures, vectors, lists and annex are the source of truth: this package
// never generates its own. A port of scripts/sync-testdata.mjs of
// datekeys-ts, with the same testdata/SOURCE.json.
//
//   dart run tool/sync_testdata.dart sync  [--repo ../datekeys-go] [--commit HEAD]
//   dart run tool/sync_testdata.dart check [--against ../datekeys-go]
//
// sync copies every blob under testdata/, wordkey/lists/ and annex/ at the
// commit (read with git, so uncommitted changes in the Go checkout are never
// picked up) into testdata/, wordlists/ and annex/, and writes the
// SOURCE.json of each, with the full commit id and the SHA-256 of each file.
// It reads every blob of every tree before it touches any copy, and a tree
// missing at the commit is an error. check verifies that the files on disk of
// each tree match its SOURCE.json exactly, with no missing or extra files;
// with --against it also re-reads every blob from the Go repository at the
// recorded commit. It prints one line per tree: testdata, wordlists, annex.
// test/testdata_test.dart runs check. No dependencies but package:crypto,
// Dart and git.

import 'dart:convert';
import 'dart:io';

import 'package:crypto/crypto.dart';

/// The module whose trees are vendored.
const module = 'g.activething.com/go/DateKeys';

/// A tree that sync vendors: [dir], a directory of this package, holds the
/// files of [upstream], a directory of the Go repository.
class Tree {
  const Tree(this.dir, this.upstream);

  final String dir;
  final String upstream;
}

/// testdata/, the fixtures and the vectors of the Go reference.
const testdataTree = Tree('testdata', 'testdata');

/// wordlists/, the word lists of wordkey.Generate, whose SHA-256
/// lib/src/wordlist.dart pins.
const wordlistsTree = Tree('wordlists', 'wordkey/lists');

/// annex/, the recovery annex of datekeys.RecoveryAnnex, which
/// tool/recovery_annex_copy.dart writes as the constant of
/// lib/src/recovery_annex.g.dart.
const annexTree = Tree('annex', 'annex');

/// The trees, in the order of the lines of check.
const trees = [testdataTree, wordlistsTree, annexTree];

/// What check found in a tree: the files, the commit and the module of its
/// SOURCE.json, and every difference, none when the copy is intact.
class CheckResult {
  CheckResult(this.tree, this.module, this.commit, this.files, this.errors);

  final Tree tree;
  final String module;
  final String commit;
  final int files;
  final List<String> errors;

  /// The line that check prints for an intact copy.
  String get line => '${tree.dir}: $files files match $module at $commit';
}

void main(List<String> args) {
  final root = File(Platform.script.toFilePath()).parent.parent;
  final command = args.isEmpty ? '' : args.first;
  final rest = args.skip(1).toList();
  try {
    switch (command) {
      case 'sync':
        final repo = _option(rest, '--repo') ?? '${root.path}/../datekeys-go';
        final commit = _option(rest, '--commit') ?? 'HEAD';
        sync(root, Directory(repo).absolute, commit);
      case 'check':
        final against = _option(rest, '--against');
        final verified = against == null
            ? ''
            : ' (verified against the repository)';
        for (final tree in trees) {
          try {
            final r = check(
              root,
              tree: tree,
              against: against == null ? null : Directory(against).absolute,
            );
            if (r.errors.isEmpty) {
              stdout.writeln('${r.line}$verified');
              continue;
            }
            for (final e in r.errors) {
              stderr.writeln('${tree.dir}: $e');
            }
          } catch (e) {
            stderr.writeln('${tree.dir}: $e');
          }
          exitCode = 1;
        }
      default:
        stderr.writeln(
          'usage: sync_testdata.dart sync [--repo PATH] [--commit REV] | check [--against PATH]',
        );
        exitCode = 2;
    }
  } catch (e) {
    stderr.writeln('$command: $e');
    exitCode = 1;
  }
}

/// Copies each tree of [trees] of the Go repository [repo] at [rev] into
/// [root], and writes its SOURCE.json.
void sync(Directory root, Directory repo, String rev) {
  final commit = utf8
      .decode(_git(repo, ['rev-parse', '--verify', '$rev^{commit}']))
      .trim();
  final listed = [
    for (final tree in trees) (tree, _listBlobs(repo, commit, tree)),
  ];
  for (final (tree, paths) in listed) {
    if (paths.isEmpty) throw StateError('no ${tree.upstream}/ at $commit');
  }
  // Read everything before touching the local copies, so a failed read
  // leaves them intact.
  final read = [
    for (final (tree, paths) in listed)
      (tree, [for (final p in paths) (p, _readBlob(repo, commit, tree, p))]),
  ];
  for (final (tree, blobs) in read) {
    final dir = Directory('${root.path}/${tree.dir}');
    if (dir.existsSync()) dir.deleteSync(recursive: true);
    final files = <String, String>{};
    for (final (path, data) in blobs) {
      File('${dir.path}/$path')
        ..createSync(recursive: true)
        ..writeAsBytesSync(data);
      files[path] = _sha256(data);
    }
    final source = {'module': module, 'commit': commit, 'files': files};
    File('${dir.path}/SOURCE.json').writeAsStringSync(
      '${const JsonEncoder.withIndent('  ').convert(source)}\n',
    );
    stdout.writeln(
      '${tree.dir}: ${blobs.length} files from $module at $commit',
    );
  }
}

/// Checks [tree] of [root], testdata/ by default, against its SOURCE.json
/// and, with [against], against the Go repository at the recorded commit.
CheckResult check(
  Directory root, {
  Tree tree = testdataTree,
  Directory? against,
}) {
  final dir = Directory('${root.path}/${tree.dir}');
  final source = jsonDecode(
    File('${dir.path}/SOURCE.json').readAsStringSync(),
  ) as Map<String, Object?>;
  final recorded = (source['files']! as Map<String, Object?>)
      .cast<String, String>();
  final commit = source['commit']! as String;
  final expected = recorded.keys.toList()..sort();
  final actual = _listLocal(dir);
  final errors = <String>[];
  for (final path in expected) {
    if (!actual.contains(path)) {
      errors.add('missing $path');
      continue;
    }
    final data = File('${dir.path}/$path').readAsBytesSync();
    if (_sha256(data) != recorded[path]) {
      errors.add('modified $path');
    }
  }
  for (final path in actual) {
    if (!recorded.containsKey(path)) errors.add('extra $path');
  }
  if (against != null) {
    final upstream = _listBlobs(against, commit, tree);
    if (upstream.join('\n') != expected.join('\n')) {
      errors.add('file list differs from $commit');
    }
    for (final path in upstream) {
      if (_sha256(_readBlob(against, commit, tree, path)) != recorded[path]) {
        errors.add('differs from upstream $path');
      }
    }
  }
  return CheckResult(
    tree,
    source['module']! as String,
    commit,
    expected.length,
    errors,
  );
}

List<int> _git(Directory repo, List<String> args) {
  final r = Process.runSync(
    'git',
    ['-C', repo.path, ...args],
    stdoutEncoding: null,
    stderrEncoding: utf8,
  );
  if (r.exitCode != 0) {
    final message = (r.stderr as String).trim();
    throw ProcessException('git', args, message, r.exitCode);
  }
  return r.stdout as List<int>;
}

String _sha256(List<int> data) => sha256.convert(data).toString();

// Files under the upstream directory of tree at commit, as paths relative to
// it with forward slashes.
List<String> _listBlobs(Directory repo, String commit, Tree tree) {
  final out = utf8.decode(
    _git(repo, [
      'ls-tree',
      '-r',
      '-z',
      '--full-tree',
      commit,
      '--',
      tree.upstream,
    ]),
  );
  final paths = <String>[];
  for (final line in out.split('\u0000').where((l) => l.isNotEmpty)) {
    final tab = line.indexOf('\t');
    final meta = line.substring(0, tab).split(' ');
    final path = line.substring(tab + 1);
    if (meta[1] != 'blob' || meta[0] == '120000') {
      throw StateError('unsupported entry $path (${meta[0]} ${meta[1]})');
    }
    paths.add(path.substring('${tree.upstream}/'.length));
  }
  return paths..sort();
}

List<int> _readBlob(Directory repo, String commit, Tree tree, String path) =>
    _git(repo, ['cat-file', 'blob', '$commit:${tree.upstream}/$path']);

// Files on disk under dir, excluding its SOURCE.json.
List<String> _listLocal(Directory dir) {
  final prefix = dir.absolute.path.length + 1;
  final files = <String>[];
  for (final f
      in dir.absolute
          .listSync(recursive: true, followLinks: false)
          .whereType<File>()) {
    final path = f.path
        .substring(prefix)
        .replaceAll(Platform.pathSeparator, '/');
    if (path != 'SOURCE.json') files.add(path);
  }
  return files..sort();
}

String? _option(List<String> args, String name) {
  final i = args.indexOf(name);
  if (i == -1) return null;
  if (i + 1 >= args.length) throw ArgumentError('$name needs a value');
  return args[i + 1];
}