dateKeys-dart/test/wordlist_test.dart

// The word lists of the key of words (spec §38.1) against
// test/vectors/wordlist_vectors.json, whose expected values the Go reference
// computed (tool/wordlist_go_vectors.go): the constants and the pinned
// lists; the text of a list read as wordkey.List reads it; wordkey.CheckList
// with its texts, also on each code point up to U+017F; the words that
// wordkey.Generate draws from the same bytes; and wordkey.Bits, bit for bit.
// And the cases of TestCheckList, TestGenerate and TestBits of Go, as Go
// writes them. The vectors come from a Dart constant, so that these tests
// also run compiled to JavaScript; the Spanish list, the CSPRNG of the
// platform and the alphabet of every code point are in wordlist_vm_test.dart.

import 'dart:convert';
import 'dart:math' as math;
import 'dart:typed_data';

import 'package:datekeys/src/bytes.dart';
import 'package:datekeys/src/pathrule.dart' show codePointName;
import 'package:datekeys/src/random.dart';
import 'package:datekeys/src/sha256.dart';
import 'package:datekeys/src/wordkey.dart';
import 'package:datekeys/src/wordlist.dart';
import 'package:test/test.dart';

import 'random_support.dart' show seeded;
import 'vectors/wordlist_vectors.g.dart';
import 'wordkey_support.dart' show fromWtf8;
import 'wordlist_support.dart';

final Json vectors = jsonDecode(wordlistVectorsJson) as Json;

List<Json> cases(String name) => (vectors[name]! as List).cast<Json>();

void main() {
  test('the constants and the pinned lists are those of Go', () {
    expect(vectors['default_count'], defaultWordCount);
    expect(vectors['min_list_size'], minListSize);
    expect(vectors['min_words'], minWords);
    expect(vectors['min_letters'], minLetters);
    // Languages of Go, and the SHA-256 of the file of each of its lists.
    final lists = cases('lists');
    expect(wordListLanguages(), [for (final l in lists) l['lang']]);
    expect(wordListSha256, {for (final l in lists) l['lang']: l['sha256']});
  });

  group('checkWordList', () {
    test('the cases of TestCheckList of Go', () {
      // pal followed by three letters: palaaa, palaab, …
      final base = [for (var i = 0; i < minListSize; i++) baseWord(i)];
      checkWordList('es', base);
      expect(
        outcome(() => checkWordList('xx', base)),
        'no alphabet for the language "xx"',
      );
      List<String> withWord(int i, String w) => [...base]..[i] = w;
      for (final (list, want) in [
        (base.sublist(0, minListSize - 1), '2047 words, fewer than 2048'),
        (
          withWord(5, 'dos palabras'),
          'line 6, "dos palabras", is not one word',
        ),
        (withWord(5, '  '), 'is not one word'),
        (withWord(5, 'mi'), '"mi", is not one word of 3 or more letters'),
        (withWord(5, 'casa\u200b'), 'invisible character U+200B'),
        // Only the letters of the alphabet of the language, as the list
        // writes them: no capitals, no Cyrillic U+0441 that looks like a
        // Latin c, no digits, no carriage return of a file with CRLF lines.
        (
          withWord(5, 'Palaaf'),
          'line 6, "Palaaf", holds U+0050, which is not in the alphabet of '
              '"es"',
        ),
        (
          withWord(5, '\u0441asa'),
          'holds U+0441, which is not in the alphabet',
        ),
        (withWord(5, 'pal1'), 'holds U+0031'),
        (withWord(5, 'palaaf\r'), r'line 6, "palaaf\r", holds U+000D'),
        (
          withWord(5, base[4]),
          'line 6, "palaae", is the same word as "palaae"',
        ),
        (
          withWord(5, 'pala\u00e1e'),
          'line 6, "pala\u00e1e", is the same word as "palaae"',
        ),
        (
          [...withWord(0, 'pap\u00e1'), 'papa'],
          '"papa", is the same word as "pap\u00e1"',
        ),
      ]) {
        expect(outcome(() => checkWordList('es', list)), contains(want));
      }
    });

    test('refuses lists as wordkey.CheckList, with its texts', () {
      final all = cases('check_list');
      var refused = 0;
      var strings = 0;
      for (final c in all) {
        final name = c['name']! as String;
        final lang = c['lang']! as String;
        final want = c['result']! as String;
        final words = caseWords(c);
        expect(
          toHex(sha256(joinBytes(words, const [0x0a]))),
          c['sha256'],
          reason: name,
        );
        expect(
          outcome(() => checkWordListUtf8(lang, words)),
          want,
          reason: name,
        );
        if (want != 'ok') refused++;
        // The same words as Strings, where they have one: the bytes of a
        // surrogate are a lone surrogate, as utf8Bytes writes it.
        final asStrings = [for (final w in words) fromWtf8(w)];
        if (asStrings.every((s) => s != null)) {
          strings++;
          expect(
            outcome(() => checkWordList(lang, [for (final s in asStrings) s!])),
            want,
            reason: name,
          );
        }
      }
      expect(all, hasLength(greaterThan(80)));
      expect(refused, greaterThan(70));
      // All but three, whose bytes no String has.
      expect(strings, all.length - 3);
    });

    test('the alphabet of each code point up to U+017F', () {
      final results = cases('alphabet_results');
      expect(
        [for (final c in results) c['rune']],
        [for (var r = 0; r < 0x180; r++) r],
      );
      // The first word is pala, the code point and zz.
      final words = caseWords(const {'size': minListSize});
      final accepted = <int>[];
      for (final c in results) {
        final r = c['rune']! as int;
        words[0] = utf8Bytes('pala${String.fromCharCode(r)}zz');
        final got = outcome(() => checkWordListUtf8('es', words));
        expect(got, c['result'], reason: codePointName(r));
        if (got == 'ok') accepted.add(r);
      }
      // a to z, á, é, í, ñ, ó, ú and ü: all that Go accepts in planes 0, 1
      // and 14.
      expect(
        String.fromCharCodes(accepted),
        'abcdefghijklmnopqrstuvwxyz'
        '\u00e1\u00e9\u00ed\u00f1\u00f3\u00fa\u00fc',
      );
      final ok = ((vectors['alphabet']! as Json)['ok']! as List).cast<String>();
      expect([for (final s in ok) s.runes.single], accepted);
    });
  });

  test('parseWordList reads the text of a list as wordkey.List', () {
    for (final c in cases('parse')) {
      final name = c['name']! as String;
      final lang = c['lang']! as String;
      final lines = caseWords(c);
      final text = concatBytes([
        fromHex(c['prefix']! as String),
        joinBytes(lines, fromHex(c['sep']! as String)),
        fromHex(c['suffix']! as String),
      ]);
      expect(toHex(sha256(text)), c['sha256'], reason: name);
      final want = c['result']! as String;
      expect(outcome(() => parseWordList(lang, text)), want, reason: name);
      if (want == 'ok') {
        expect(lines, hasLength(c['words']), reason: name);
        expect(parseWordList(lang, text), [
          for (final l in lines) decodeUtf8(l),
        ], reason: name);
      }
    }
  });

  test('readWordList refuses a language without a list and other bytes', () {
    expect(
      outcome(() => readWordList('xx', const [])),
      'wordkey: no word list for "xx"; the lists are es',
    );
    final text = utf8Bytes('palaaa\n');
    expect(
      outcome(() => readWordList('es', text)),
      'wordkey: the list "es" has the SHA-256 ${toHex(sha256(text))}, not '
      '${wordListSha256['es']}',
    );
    // The language first, as Go's List looks for its list first.
    expect(
      outcome(() => readWordList('ES', text)),
      'wordkey: no word list for "ES"; the lists are es',
    );
  });

  group('generateWords', () {
    test('draws the words of wordkey.Generate from the same bytes', () {
      final all = cases('generate');
      var drawn = 0;
      for (final c in all) {
        final name = c['name']! as String;
        final size = c['size']! as int;
        final n = c['n']! as int;
        final read = c['read']! as int;
        final error = c['error'] as String?;
        final source = CountingSource(streamOf(c['stream']! as Json));
        final list = IndexWords(size);
        if (error == null) {
          final words = generateWords(list, n, source);
          expect(
            [for (final w in words) int.parse(w)],
            c['indices'],
            reason: name,
          );
          expect(source.read, read, reason: name);
          final next = c['next'] as String?;
          if (next != null) {
            expect(toHex(randomBytes(source, 4)), next, reason: name);
          }
          drawn++;
        } else if (error.endsWith('EOF')) {
          // The bytes end: the source throws, and generateWords lets it
          // through, after the draws that Go made in full.
          expect(
            () => generateWords(list, n, source),
            throwsStateError,
            reason: name,
          );
          expect(source.read, read - read % drawSize(size), reason: name);
        } else {
          // The arguments, refused before anything is drawn.
          expect(
            outcome(() => generateWords(list, n, source)),
            error,
            reason: name,
          );
          expect(source.read, 0, reason: name);
        }
      }
      expect(all, hasLength(greaterThan(40)));
      expect(drawn, greaterThan(30));
    });

    test('the cases of TestGenerate of Go', () {
      final list = IndexWords(7776);
      // The same random bytes draw the same words: generateWords reads
      // nothing else.
      final a = generateWords(list, 6, seeded('TestGenerate'));
      expect(generateWords(list, 6, seeded('TestGenerate')), a);
      expect(a.toSet(), hasLength(6));
      // The seed of TestGenerate of Go draws two indices only, 1793 and
      // 2081, again and again until it runs out: Go's Generate then fails
      // with EOF, and its two results compare equal (see the vectors).
      expect(
        () => generateWords(
          list,
          6,
          ReplaySource(repeatBytes(const [7, 1, 200, 33], 64)),
        ),
        throwsStateError,
      );
      for (final (n, want) in [
        (5, 'at least 6 words, not 5'),
        (list.length, '7776 words of a list of 7776'),
      ]) {
        expect(
          outcome(() => generateWords(list, n, ReplaySource(const []))),
          contains(want),
        );
      }
      // Without random bytes: the source throws, where Go's Generate fails
      // with EOF.
      expect(
        () => generateWords(list, 6, ReplaySource(const [])),
        throwsStateError,
      );
    });
  });

  group('wordBits', () {
    double goBits(int size, int count) => hexDouble(
      cases(
            'bits',
          ).singleWhere((c) => c['size'] == size && c['count'] == count)['hex']!
          as String,
    );

    test('the cases of TestBits of Go', () {
      // Go: Bits(7776, 1) is math.Log2(7776).
      expect(wordBits(7776, 1), goBits(7776, 1));
      expect(wordBits(7776, 1), closeTo(math.log(7776) / math.ln2, 1e-12));
      // 7 words of 7776 are a little under 90.5 bits, and 6 of 2048, the
      // fewest that generateWords draws, a little under 66.
      for (final (size, count, low, high) in [
        (7776, 0, 0.0, 0.0),
        (7776, 7, 90.469, 90.470),
        (7776, 8, 103.393, 103.394),
        (2048, 6, 65.989, 65.990),
      ]) {
        expect(
          wordBits(size, count),
          inInclusiveRange(low, high),
          reason: '$size, $count',
        );
      }
    });

    test('is Bits of Go, bit for bit', () {
      for (final c in cases('bits')) {
        final size = c['size']! as int;
        final count = c['count']! as int;
        final want = c['hex']! as String;
        final got = wordBits(size, count);
        if (hexDouble(want).isNaN) {
          // The NaN of Go, 7ff8000000000001, is not that of Dart.
          expect(got.isNaN, isTrue, reason: '$size, $count');
        } else {
          expect(doubleHex(got), want, reason: '$size, $count: ${c['bits']}');
        }
      }
      for (final d in cases('bits_digests')) {
        final sizes = (d['sizes']! as List).cast<int>();
        final counts = (d['counts']! as List).cast<int>();
        final digest = Sha256();
        final b = ByteData(8);
        for (var size = sizes[0]; size <= sizes[1]; size++) {
          for (var count = counts[0]; count <= counts[1]; count++) {
            b.setFloat64(0, wordBits(size, count));
            digest.add(Uint8List.sublistView(b));
          }
        }
        expect(toHex(digest.finish()), d['sha256'], reason: '${d['name']}');
      }
    });
  });
}