DateKeys-App/src/lib/dkc/wordlist.test.ts

// Tests of wordlist.ts: the cases of generate_test.go and dice_test.go of Go
// wordkey, with their texts; the lists of datekeys-go, read with the SHA-256
// that their README records; draws that are uniform and never repeat a word;
// and the words of dice, with the lists numbered for them.

import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import { toHex, utf8Bytes } from './bytes.ts';
import type { RandomWords } from './random.ts';
import { checkWords, goFields, normalizeWords } from './wordkey.ts';
import {
  checkWordList,
  DEFAULT_WORD_COUNT,
  DICE_LIST_SIZE,
  diceList,
  diceNumber,
  diceWord,
  diceWords,
  generateWords,
  MIN_LIST_SIZE,
  readWordList,
  WORD_LIST_SHA256,
  wordBits,
} from './wordlist.ts';

const listFile = (name: string): Uint8Array => new Uint8Array(readFileSync(fileURLToPath(new URL(`../../../wordlists/${name}`, import.meta.url))));
const cp = (r: number): string => String.fromCodePoint(r);
const hash = async (b: Uint8Array): Promise<string> => toHex(new Uint8Array(await crypto.subtle.digest('SHA-256', b as Uint8Array<ArrayBuffer>)));

// pal followed by three letters: palaaa, palaab, …, as in Go.
const base = Array.from({ length: MIN_LIST_SIZE }, (_, i) => `pal${String.fromCharCode(97 + Math.floor(i / 676), 97 + (Math.floor(i / 26) % 26), 97 + (i % 26))}`);
const withWord = (i: number, w: string): string[] => base.map((x, j) => (j === i ? w : x));
const fileOf = (words: readonly string[]): Uint8Array => utf8Bytes(`${words.join('\n')}\n`);

// A source of 32-bit words that repeats `seed`.
function repeating(seed: readonly number[]): RandomWords {
  let i = 0;
  return () => seed[i++ % seed.length]!;
}

describe('readWordList', () => {
  it('reads the lists en and es of datekeys-go, with the SHA-256 that its README records', async () => {
    const readme = new TextDecoder().decode(listFile('README.md'));
    expect(Object.keys(WORD_LIST_SHA256).sort()).toEqual(['en', 'es']);
    for (const lang of ['en', 'es']) expect(readme).toContain(`| \`${lang}.txt\` | 7776 | \`${WORD_LIST_SHA256[lang]}\` |`);
    const es = await readWordList('es', listFile('es.txt'));
    expect(es).toHaveLength(7776);
    expect([es[0], es.at(-1)]).toEqual(['abad', 'útil']);
    // The list of the EFF in its order: the dice 11111 give its first word
    // and 66666 its last.
    const en = await readWordList('en', listFile('en.txt'));
    expect(en).toHaveLength(7776);
    expect([en[0], en[1], en.at(-1)]).toEqual(['abacus', 'abdomen', 'zoom']);
    expect(en.filter((w) => w.includes('-'))).toEqual(['drop-down', 'felt-tip', 't-shirt', 'yo-yo']);
  });

  it('refuses a list of another SHA-256 and a language without a list, with the texts of Go', async () => {
    const changed = listFile('es.txt');
    changed[0]! ^= 1;
    const refused = `wordkey: the list "es" has the SHA-256 ${await hash(changed)}, not ${WORD_LIST_SHA256['es']}`;
    await expect(readWordList('es', changed)).rejects.toThrow(refused);
    await expect(readWordList('xx', listFile('es.txt'))).rejects.toThrow(/^wordkey: no word list for "xx"; the lists are en, es$/);
    await expect(readWordList('es', fileOf(base), { fr: '00', de: '11' })).rejects.toThrow(/^wordkey: no word list for "es"; the lists are de, fr$/);
  });

  it('refuses a list with its pin that is not UTF-8, or that checkWordList refuses, as Go wordkey.List', async () => {
    const pin = async (lang: string, b: Uint8Array): Promise<Record<string, string>> => ({ [lang]: await hash(b) });
    const ok = fileOf(base);
    await expect(readWordList('es', ok, await pin('es', ok))).resolves.toEqual(base);
    // Without the end of its last line, too.
    const unended = ok.slice(0, -1);
    await expect(readWordList('es', unended, await pin('es', unended))).resolves.toEqual(base);
    const cyrillic = fileOf(withWord(5, `${cp(0x441)}asa`));
    const err = (await readWordList('es', cyrillic, await pin('es', cyrillic)).catch((e: unknown) => e)) as Error;
    expect(err.message).toBe(`wordkey: the list "es": line 6, "${cp(0x441)}asa", holds U+0441, which is not in the alphabet of "es"`);
    expect((err.cause as Error).message).toBe(`line 6, "${cp(0x441)}asa", holds U+0441, which is not in the alphabet of "es"`);
    // A BOM stays: it is an invisible character of the first word.
    const bom = utf8Bytes(`${cp(0xfeff)}${base.join('\n')}\n`);
    await expect(readWordList('es', bom, await pin('es', bom))).rejects.toThrow(/^wordkey: the list "es": line 1: wordkey: the words hold the invisible character U\+FEFF$/);
    const latin1 = fileOf(base);
    latin1[3] = 0xe1;
    await expect(readWordList('es', latin1, await pin('es', latin1))).rejects.toThrow(/^wordkey: the list "es" is not UTF-8$/);
    await expect(readWordList('xx', ok, await pin('xx', ok))).rejects.toThrow(/^wordkey: the list "xx": no alphabet for the language "xx"$/);
  });
});

describe('checkWordList', () => {
  it('accepts a list of 2048 different words of the alphabet, and refuses the cases of Go wordkey.TestCheckList', async () => {
    await expect(checkWordList('es', base)).resolves.toBeUndefined();
    await expect(checkWordList('xx', base)).rejects.toThrow(/^no alphabet for the language "xx"$/);
    const cases: [readonly string[], string][] = [
      [base.slice(0, MIN_LIST_SIZE - 1), '2047 words, fewer than 2048'],
      [withWord(5, 'dos palabras'), 'line 6, "dos palabras", is not one word'],
      [withWord(5, '  '), 'is not one word'],
      [withWord(5, 'mi'), '"mi", is not one word of 3 or more letters'],
      [withWord(5, `casa${cp(0x200b)}`), 'invisible character U+200B'],
      // Only the letters of the alphabet of the language, as the list writes
      // them: no capitals, no Cyrillic U+0441 that looks like a Latin c, no
      // digits, no carriage return of a file with CRLF lines.
      [withWord(5, 'Palaaf'), 'line 6, "Palaaf", holds U+0050, which is not in the alphabet of "es"'],
      [withWord(5, `${cp(0x441)}asa`), 'holds U+0441, which is not in the alphabet'],
      [withWord(5, 'pal1'), 'holds U+0031'],
      [withWord(5, `palaaf${cp(0x0d)}`), `line 6, "palaaf${cp(0x5c)}r", holds U+000D`],
      [withWord(5, base[4]!), 'line 6, "palaae", is the same word as "palaae"'],
      [withWord(5, 'palaáe'), 'line 6, "palaáe", is the same word as "palaae"'],
      [[...withWord(0, 'papá'), 'papa'], '"papa", is the same word as "papá"'],
    ];
    for (const [list, want] of cases) await expect(checkWordList('es', list)).rejects.toThrow(want);
  });

  it('takes the hyphen as a letter of en and not of es, and an accent as one of es and not of en, as Go', async () => {
    await expect(checkWordList('en', withWord(5, 't-shirt'))).resolves.toBeUndefined();
    await expect(checkWordList('es', withWord(5, 't-shirt'))).rejects.toThrow(/^line 6, "t-shirt", holds U\+002D, which is not in the alphabet of "es"$/);
    await expect(checkWordList('en', withWord(5, 'palaáf'))).rejects.toThrow(/^line 6, "palaáf", holds U\+00E1, which is not in the alphabet of "en"$/);
  });
});

describe('generateWords', () => {
  it('draws different words of the list that make a key, by default 7 with crypto.getRandomValues', async () => {
    const list = await readWordList('es', listFile('es.txt'));
    const words = generateWords(list);
    expect(words).toHaveLength(DEFAULT_WORD_COUNT);
    expect(new Set(words).size).toBe(DEFAULT_WORD_COUNT);
    for (const w of words) expect(list).toContain(w);
    await expect(checkWords(await normalizeWords(words.join(' ')))).resolves.toBeUndefined();
  });

  it('reads nothing but its random words, and draws a word only once', () => {
    const seed = [7, 1, 200, 33, 0x9e3779b9, 12345];
    expect(generateWords(base, 6, repeating(seed))).toEqual(generateWords(base, 6, repeating(seed)));
    // Index 0 comes twice: the second is drawn again.
    expect(generateWords(base, 6, repeating([0, 0, 1, 2, 3, 4, 5]))).toEqual(base.slice(0, 6));
    expect(() =>
      generateWords(base, 6, () => {
        throw new Error('no more randomness');
      }),
    ).toThrow('no more randomness');
  });

  it('refuses fewer than 6 words and more than half the list, with the texts of Go', () => {
    expect(() => generateWords(base, 5)).toThrow(/^wordkey: a key of words needs at least 6 words, not 5$/);
    expect(() => generateWords(base, 6.5)).toThrow(/^wordkey: a key of words needs at least 6 words, not 6\.5$/);
    expect(() => generateWords(base, 1025)).toThrow(/^wordkey: 1025 words of a list of 2048$/);
    expect(generateWords(base, 1024)).toHaveLength(1024);
  });

  // As TestGenerateUniform of Go: over 7776·40 draws of one word, each index
  // falls in its bucket of 64 between 0.8 and 1.2 times the mean.
  it('draws every word about as often', async () => {
    const list = await readWordList('es', listFile('es.txt'));
    const index = new Map(list.map((w, i) => [w, i]));
    const buckets = 64;
    const count = new Array<number>(buckets).fill(0);
    let draws = 0;
    while (draws < list.length * 40) {
      for (const w of generateWords(list, 6)) {
        count[Math.floor((index.get(w)! * buckets) / list.length)]!++;
        draws++;
      }
    }
    const mean = draws / buckets;
    for (const c of count) {
      expect(c).toBeGreaterThan(0.8 * mean);
      expect(c).toBeLessThan(1.2 * mean);
    }
  });
});

describe('wordBits', () => {
  it('is log2 of the draws in order, as Go wordkey.Bits', () => {
    expect(wordBits(7776, 1)).toBe(Math.log2(7776));
    expect(wordBits(7776, 0)).toBe(0);
    // 7 words of 7776 are a little under 90.5 bits, and 6 of 2048, the
    // fewest, a little under 66.
    expect(wordBits(7776, 7)).toBeCloseTo(90.4698, 4);
    expect(wordBits(7776, 8)).toBeCloseTo(103.3933, 4);
    expect(wordBits(2048, 6)).toBeCloseTo(65.9894, 4);
  });
});

describe('the dice', () => {
  it('number the positions of a list of 7776 words from 11111 to 66666, as Go wordkey.DiceNumber', () => {
    const cases: [number, string][] = [
      [0, '11111'],
      [1, '11112'],
      [5, '11116'],
      [6, '11121'],
      [35, '11166'],
      [36, '11211'],
      [1295, '16666'],
      [1296, '21111'],
      [7775, '66666'],
    ];
    for (const [i, want] of cases) expect(diceNumber(i)).toBe(want);
    for (const i of [-1, 7776, 1.5]) expect(() => diceNumber(i)).toThrow(new RegExp(`^wordkey: no dice give position ${i} of a list of 7776 words$`));
    expect(DICE_LIST_SIZE).toBe(7776);
  });

  it('give the word of their number, as Go wordkey.DiceWord, and refuse what is not five dice', async () => {
    const en = await readWordList('en', listFile('en.txt'));
    const es = await readWordList('es', listFile('es.txt'));
    for (let i = 0; i < en.length; i++) expect(diceWord(en, diceNumber(i))).toBe(en[i]);
    const cases: [readonly string[], string, string][] = [
      [en, '11111', 'abacus'],
      [en, '11112', 'abdomen'],
      [en, '35214', 'jovial'],
      [en, '66666', 'zoom'],
      [es, '11111', 'abad'],
      [es, '35214', 'glaciar'],
      [es, '66666', 'útil'],
    ];
    for (const [list, dice, want] of cases) expect(diceWord(list, dice)).toBe(want);
    for (const dice of ['', '1111', '111111', '11110', '11117', 'a1111', '1111 ', `${cp(0xff11)}1111`, `11111\n`]) {
      // Go's %q of these is their JSON.
      expect(() => diceWord(en, dice)).toThrow(`wordkey: ${JSON.stringify(dice)} is not five dice: five digits from 1 to 6`);
    }
    expect(() => diceWord(en.slice(0, 2048), '11111')).toThrow(/^wordkey: dice draw from a list of 7776 words, not 2048$/);
  });

  it('give the words of several numbers, as Go wordkey.DiceWords: at least 6, and no word twice', async () => {
    const en = await readWordList('en', listFile('en.txt'));
    const words = diceWords(en, ` 11111 11112\t11113\n11114 11115 11116 11121 `);
    expect(words).toEqual(['abacus', 'abdomen', 'abdominal', 'abide', 'abiding', 'ability', 'ablaze']);
    await expect(checkWords(await normalizeWords(words.join(' ')))).resolves.toBeUndefined();
    const cases: [string, string][] = [
      ['11111 11112 11113 11114 11115', 'wordkey: a key of words needs at least 6 words, not 5'],
      ['', 'wordkey: a key of words needs at least 6 words, not 0'],
      ['11111 11112 11113 11114 11115 1116', 'wordkey: "1116" is not five dice: five digits from 1 to 6'],
      ['11111 11112 11113 11114 11115 11111', 'wordkey: the dice 11111 give "abacus" a second time; roll them again'],
      ['11111,11112 11113 11114 11115 11116 11121', 'wordkey: "11111,11112" is not five dice: five digits from 1 to 6'],
    ];
    for (const [dice, want] of cases) expect(() => diceWords(en, dice)).toThrow(want);
    expect(() => diceWords(en.slice(1), '11111 11112 11113 11114 11115 11116')).toThrow(/^wordkey: dice draw from a list of 7776 words, not 7775$/);
  });

  it('number a whole list to print it, as Go wordkey.DiceList: for en, the file of the EFF', async () => {
    const want: Record<string, string> = {
      en: 'addd35536511597a02fa0a9ff1e5284677b8883b83e986e43f15a3db996b903e',
      es: '611f779a33df74587e9dfb486bb0185a9dfd4d1384e75993a21247ad31cefddb',
    };
    for (const lang of ['en', 'es']) {
      const list = await readWordList(lang, listFile(`${lang}.txt`));
      const text = diceList(list);
      expect(await hash(utf8Bytes(text))).toBe(want[lang]);
      expect(text.startsWith(`11111\t${list[0]}\n`) && text.endsWith(`66666\t${list[7775]}\n`)).toBe(true);
    }
    expect(() => diceList(new Array<string>(2048).fill('x'))).toThrow(/^wordkey: dice draw from a list of 7776 words, not 2048$/);
  });

  it('split numbers at white space as Go strings.Fields, without changing them', () => {
    expect(goFields(` 1\u0301 2\t3\u00a0 4\u3000`)).toEqual([`1${cp(0x301)}`, '2', '3', '4']);
    expect(goFields('')).toEqual([]);
  });
});