diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc0874..3a2b862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ Cambios notables de la librería TypeScript y de la página. El proyecto usa ver El formato 3 de la especificación 0.10, según `PLAN_formato3_ts.md` (en `../docs`). La versión que lo publique la decide el autor. +### Paso 5: el ZIP de la página + +- `zipsink.ts`: `ZipSink`, el sumidero de la página sobre el fichero temporal de OPFS. Un fichero de un segmento va tal cual, y los demás casos a un ZIP propio, con el CRC-32 de cada entrada parcheado con una escritura posicionada y el directorio central al hacer commit; nada se publica antes del paso 18. `zipOf` hace el mismo ZIP en memoria. +- `tempfile.ts`: el `writable` de un fichero temporal acepta trozos con posición (`TempChunk`), como `FileSystemWritableFileStream`. +- Interoperabilidad: `archive/zip` de Go lee las muestras de la página, 65 535 entradas con ZIP64 incluidas, con sus nombres, CRC-32, tamaños, fechas y contenidos (`testing/zip-vectors.json`, de `scripts/zip-ts-samples.mjs` y `scripts/zip-go-read.go`). + ### Paso 4: la escritura del formato 3 - `writer.ts` se parte como el writer de Go: `newSealer` comprueba las opciones que no dependen del contenido, y `seal` escribe la cápsula de un formato alrededor de un contenido dado en trozos. El formato 2 no cambia. diff --git a/README.md b/README.md index 438197b..557af13 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,9 @@ Rendimiento, informativo, en ese navegador con la ventana en segundo plano: una | `src/lib/inspector/release-input.ts` | Lee el release pegado (respuesta de drand o firma sola) y construye la URL de drand de la ronda | | `src/lib/inspector/opener.ts` | La apertura, cargada bajo demanda: identidades, `.dkk`, `open` con el release suministrado, SHA-256 del texto en claro | | `src/lib/inspector/opening.ts` | `buildOpenReport`: el modelo de la apertura (pasos 9 a 18, release, extensiones de CONTROL_CBOR, texto en claro), sin DOM ni reloj; nombre del fichero descifrado | -| `src/lib/inspector/tempfile.ts` | El fichero temporal de OPFS, con un directorio y un Web Lock por pestaña y por zona (la apertura y crear), la cuota libre y la limpieza de lo que quedó | +| `src/lib/inspector/tempfile.ts` | El fichero temporal de OPFS, con un directorio y un Web Lock por pestaña y por zona (la apertura y crear), la cuota libre y la limpieza de lo que quedó. Su `writable` acepta además trozos con posición (`TempChunk`), como `FileSystemWritableFileStream`, para parchear el CRC-32 del ZIP | +| `src/lib/inspector/crc32.ts`, `zip.ts` | El CRC-32 de ZIP, por trozos, y la disposición de un ZIP cuyas entradas se conocen de antemano: almacenadas, con nombres UTF-8 (bit 11), sin descriptores de datos ni entradas de carpeta; la hora DOS en UTC entre 1980 y 2107, el campo NTFS 0x000A, que manda, y 0x5455 cuando cabe en 32 bits con signo; ZIP64 por tamaño, posición o número de entradas | +| `src/lib/inspector/zipsink.ts` | El sumidero de la página para el formato 3 (apartado 5 del diseño): un fichero de un segmento va tal cual al fichero temporal, y los demás casos a un ZIP en ese fichero, con cada cabecera en su posición, el CRC-32 parcheado al cerrar cada fichero y el directorio central al hacer commit; cada fichero queda como un tramo continuo. `zipOf` hace el mismo ZIP en memoria, como un `Blob` de sus partes | | `src/lib/inspector/localtime.ts` | Una fecha y una hora locales de una zona como instante UTC, con `Intl`: las horas que no existen y las repetidas; la lista de zonas | | `src/lib/inspector/create-input.ts` | El formulario de `/create`, sin DOM, reloj ni writer: `planCapsule` comprueba los campos en su orden y da lo que se muestra antes de cifrar; los destinatarios y los nombres de los ficheros | | `src/lib/inspector/creator.ts` | La escritura, cargada bajo demanda: `encryptFiles` hacia el fichero temporal o la memoria, con la cancelación y la cuota, y los pasos 1 a 8 de lo escrito | @@ -240,6 +242,7 @@ Umbrales de cobertura (`vitest.config.ts`): `cbor.ts`, `ibe.ts`, `release.ts`, ` - `testdata/vectors/inspect_differential.json`: las 5 110 mutaciones de los catorce fixtures de base dan el mismo veredicto, código y paso que Go; los `bases` se comprueban por su SHA-256. - `testdata/vectors/paths.json` y `path_fold.json`, con las tablas cuyo digest nombran: cada ruta pasa por las reglas de una entrada con el texto exacto, cada árbol por la decodificación de un head de ficheros de 0 bytes, y cada segmento da su NFD y su clave de R7. - `testdata/vectors/head_schema.json` y `security.json`: cada head pasa por `decodeHead` con el código de la primera capa que falla y el texto exacto de `ERR_HEAD_INVALID`, y un head válido se reescribe a sus bytes; cada área de seguridad da sus veredictos. +- `src/lib/dkc/testing/zip-vectors.json`: cómo lee `archive/zip` de Go los ZIP de la página. `scripts/zip-ts-samples.mjs` escribe con `ZipSink` las muestras de `testing/zip.ts` en un directorio: ficheros en carpetas con nombres fuera de ASCII y todas las clases de fecha (1970, 2³¹ − 1, 2³¹, 9999 y ninguna, que toma la hora de la ronda), un fichero dentro de una carpeta, y 65 535 ficheros, que hacen el ZIP64 por número de entradas. `scripts/zip-go-read.go`, solo con la biblioteca estándar, registra el SHA-256 de cada ZIP y una línea por entrada: nombre, bit 11, método, CRC-32, tamaños, fecha leída de los campos extra y SHA-256 del contenido, leído con el CRC comprobado. `zipsink.test.ts` vuelve a escribir cada muestra, exige su SHA-256 y calcula las líneas que Go tiene que dar. Para regenerarlo: `node scripts/zip-ts-samples.mjs DIR > zip-samples.json` y `go run scripts/zip-go-read.go DIR zip-samples.json > zip-vectors.json`. - `src/lib/dkc/testing/mutation-texts.json`: el texto del error de `capsule.Open` para cada caso de `mutations.json`, u `ok`. Lo escribe `scripts/mutation-go-texts.go`, que reproduce los casos como `internal/testkit` de la referencia, que un módulo de fuera no puede importar: el perfil de Quicknet o ninguno, una fuente con el release del caso, la `.dkk` ya decodificada, las identidades, las extensiones del caso con los textos de `testkit.KnownExtensions` y un sumidero que descarta. Comprueba cada código contra el corpus. Para regenerarlo, desde un módulo Go temporal como el de abajo: `go run mutation-go-texts.go ../datekeys-ts/testdata > mutation-texts.json`. - `src/lib/dkc/testing/ibe-vectors.json`: los valores de referencia de `ibe.ts`. Los escribe `scripts/ibe-go-vectors.go` con kyber, tlock y `age`, las librerías de la referencia Go, y `ibe.test.ts` los comprueba todos: - el GT de e(G1, G2) y de su cuadrado, con H2 de 16 y 32 bytes; diff --git a/scripts/zip-go-read.go b/scripts/zip-go-read.go new file mode 100644 index 0000000..e1fe20a --- /dev/null +++ b/scripts/zip-go-read.go @@ -0,0 +1,132 @@ +//go:build ignore + +// Prints src/lib/dkc/testing/zip-vectors.json: how archive/zip of the Go +// standard library reads the ZIP samples that the page writes (plan of +// format 3 in datekeys-ts, step 5), which scripts/zip-ts-samples.mjs wrote +// into a directory. For each archive it records the SHA-256 of its bytes, +// the number of its entries and, for each entry, a line with its name, +// whether bit 11 marks the name as UTF-8, its method, its CRC-32, its sizes, +// the time archive/zip reads from its extra fields, in UTC, and the SHA-256 +// of its content, read with the CRC-32 checked. It keeps the lines of an +// archive of 20 entries at most, and the SHA-256 of all of them joined by +// newlines for any archive, so that the archive of 65535 entries, which is +// ZIP64 by its number of entries, weighs little. zipsink.test.ts writes each +// archive again, requires its SHA-256, and computes the lines it must give. +// +// It needs only the standard library: +// +// node scripts/zip-ts-samples.mjs DIR > zip-samples.json +// go run zip-go-read.go DIR zip-samples.json > zip-vectors.json +package main + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +type sampleIn struct { + Name string `json:"name"` + File string `json:"file"` + SHA256 string `json:"sha256"` +} + +type sampleOut struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + Count int `json:"count"` + LinesSHA256 string `json:"lines_sha256"` + Lines []string `json:"lines,omitempty"` + Error string `json:"error,omitempty"` +} + +type output struct { + Description string `json:"description"` + Generator string `json:"generator"` + SamplesGenerator string `json:"samples_generator"` + Go string `json:"go"` + Samples []sampleOut `json:"samples"` +} + +func must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} + +// read reads an archive with archive/zip, and gives one line for each entry. +func read(b []byte) ([]string, error) { + r, err := zip.NewReader(bytes.NewReader(b), int64(len(b))) + if err != nil { + return nil, err + } + var lines []string + for _, f := range r.File { + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + sum := sha256.Sum256(data) + lines = append(lines, fmt.Sprintf("%s\t%t\t%d\t%08x\t%d\t%d\t%s\t%s", + f.Name, f.Flags&0x800 != 0, f.Method, f.CRC32, f.UncompressedSize64, f.CompressedSize64, + f.Modified.UTC().Format(time.RFC3339), hex.EncodeToString(sum[:]))) + } + return lines, nil +} + +func main() { + dir := os.Args[1] + var in struct { + Generator string `json:"generator"` + Samples []sampleIn `json:"samples"` + } + if err := json.Unmarshal(must(os.ReadFile(os.Args[2])), &in); err != nil { + panic(err) + } + out := output{ + Description: "How archive/zip of Go reads the ZIP samples of the page; see the header of scripts/zip-go-read.go.", + Generator: "scripts/zip-go-read.go", + SamplesGenerator: in.Generator, + Go: runtime.Version(), + } + for _, s := range in.Samples { + b := must(os.ReadFile(filepath.Join(dir, s.File))) + sum := sha256.Sum256(b) + so := sampleOut{Name: s.Name, SHA256: hex.EncodeToString(sum[:])} + if so.SHA256 != s.SHA256 { + panic(fmt.Sprintf("%s: the file is not the sample written", s.Name)) + } + lines, err := read(b) + if err != nil { + so.Error = err.Error() + } + so.Count = len(lines) + all := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + so.LinesSHA256 = hex.EncodeToString(all[:]) + if len(lines) <= 20 { + so.Lines = lines + } + out.Samples = append(out.Samples, so) + } + e := json.NewEncoder(os.Stdout) + e.SetEscapeHTML(false) + e.SetIndent("", " ") + if err := e.Encode(out); err != nil { + panic(err) + } +} diff --git a/scripts/zip-ts-samples.mjs b/scripts/zip-ts-samples.mjs new file mode 100644 index 0000000..f28359a --- /dev/null +++ b/scripts/zip-ts-samples.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +// Writes the ZIP samples of the page (src/lib/dkc/testing/zip.ts), as the +// ZipSink of src/lib/inspector/zipsink.ts writes them, into a directory for +// scripts/zip-go-read.go, and prints their index as JSON: the name, the file +// and the SHA-256 of each. The samples are deterministic: zipsink.test.ts +// writes them again and requires the same SHA-256. Node runs the TypeScript +// sources directly (type stripping, Node 22.6+): +// +// node scripts/zip-ts-samples.mjs DIR > zip-samples.json + +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { sha256, toHex } from '../src/lib/dkc/bytes.ts'; +import { writeZip, zipSamples } from '../src/lib/dkc/testing/zip.ts'; + +const dir = process.argv[2]; +if (dir === undefined) throw new Error('usage: node scripts/zip-ts-samples.mjs DIR'); +mkdirSync(dir, { recursive: true }); +const samples = []; +for (const [i, s] of zipSamples().entries()) { + const { bytes } = await writeZip(s); + const file = `${i}.zip`; + writeFileSync(join(dir, file), bytes); + samples.push({ name: s.name, file, sha256: toHex(await sha256(bytes)) }); +} +process.stdout.write(`${JSON.stringify({ generator: 'scripts/zip-ts-samples.mjs', samples }, null, 1)}\n`); diff --git a/src/lib/dkc/testing/zip-vectors.json b/src/lib/dkc/testing/zip-vectors.json new file mode 100644 index 0000000..bf4d372 --- /dev/null +++ b/src/lib/dkc/testing/zip-vectors.json @@ -0,0 +1,37 @@ +{ + "description": "How archive/zip of Go reads the ZIP samples of the page; see the header of scripts/zip-go-read.go.", + "generator": "scripts/zip-go-read.go", + "samples_generator": "scripts/zip-ts-samples.mjs", + "go": "go1.26.8", + "samples": [ + { + "name": "files in folders, names out of ASCII and every kind of time", + "sha256": "75413198ecebafeb8c4ad03d176048170022b8e6131509e415ced919fe696c08", + "count": 6, + "lines_sha256": "8a877b3c02ac6f59d26416fcb05bb87897df36a334595111f60c8d8fd99cace7", + "lines": [ + "carta.txt\ttrue\t0\t151ad124\t37\t37\t2026-09-30T12:00:00Z\t0c84897e85eeb721badb3b345cd858784ad09ee350d87340937d274929161559", + "fotos/2025/playa.jpg\ttrue\t0\taf26b746\t70000\t70000\t2038-01-19T03:14:07Z\tc6618f49d56788ffe3af242905edd2ab826949fccbc6ae5b6fe23e76ef4b62ef", + "fotos/vacío.txt\ttrue\t0\t00000000\t0\t0\t2038-01-19T03:14:08Z\te3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "z/9999.bin\ttrue\t0\t65c2824e\t3\t3\t9999-12-31T23:59:59Z\tc90f0165b02cf99598f8cf7df688df50875844dc3cece28fc45fdb57608c8ed0", + "z/sin fecha.txt\ttrue\t0\td7e6d6c8\t4\t4\t2023-08-23T15:59:24Z\te61cdeaf027408b22b987fd3efeca4c800042aa0d9662b78a3e2380210236c9c", + "Ñandú/nota 🙂.txt\ttrue\t0\te246eb5c\t10\t10\t1970-01-01T00:00:01Z\td267b25e018d7546fc83f5c162840ab16d17780a9d02d4cfdea70cc0603c902d" + ] + }, + { + "name": "one file inside a folder", + "sha256": "4d69b37639cea9de05fe257ff65b2f9b790ca99d299074f7a4fd6e63a7c7dc70", + "count": 1, + "lines_sha256": "f10c60dbd513242427c5faafb5fefb10963fbedcee170c9b6c6f1fd68a896978", + "lines": [ + "informes/2026.pdf\ttrue\t0\tef7caf18\t1000\t1000\t2026-09-30T12:00:00Z\t9e148ea06d47eb7f031c1d20feab31f6251f6b54a778d123086a38c93ae21996" + ] + }, + { + "name": "65535 files: ZIP64 by the number of entries", + "sha256": "4c42a7155a6a09205da97c0f5cf99e932323ce72100c9d80b6247c24d9f30559", + "count": 65535, + "lines_sha256": "e650cafb0822a83ac524e637e73136fdfed406ec3de943471d15965762a86e20" + } + ] +} diff --git a/src/lib/dkc/testing/zip.ts b/src/lib/dkc/testing/zip.ts new file mode 100644 index 0000000..f775db4 --- /dev/null +++ b/src/lib/dkc/testing/zip.ts @@ -0,0 +1,129 @@ +// Tests only: a file in memory that takes the chunks of a writable of OPFS +// (TempChunk of inspector/tempfile.ts), and the ZIP samples that the page +// writes and the Go reference reads with archive/zip +// (scripts/zip-ts-samples.mjs, scripts/zip-go-read.go and +// inspector/zipsink.test.ts). The samples are deterministic: the same code +// writes the same bytes, whose SHA-256 is frozen with the reading of Go. + +import type { TempChunk } from '../../inspector/tempfile.ts'; +import { ZipSink } from '../../inspector/zipsink.ts'; +import type { Head, HeadFile } from '../head.ts'; + +/** A file in memory written through a WritableStream of TempChunk, and how its writing ended. */ +export interface MemoryFile { + readonly stream: WritableStream; + bytes(): Uint8Array; + readonly state: { closed: boolean; aborted: unknown; positioned: number }; +} + +/** + * A file in memory that takes TempChunk as FileSystemWritableFileStream + * does: bytes at its position, or at the position given, zeros filling any + * gap, each write moving the position to its end. + */ +export function memoryFile(): MemoryFile { + let buf = new Uint8Array(1 << 16); + let length = 0; + let position = 0; + const state = { closed: false, aborted: undefined as unknown, positioned: 0 }; + const put = (at: number, data: Uint8Array): void => { + const end = at + data.length; + if (end > buf.length) { + const bigger = new Uint8Array(Math.max(end, buf.length * 2)); + bigger.set(buf.subarray(0, length)); + buf = bigger; + } + buf.set(data, at); + length = Math.max(length, end); + position = end; + }; + const stream = new WritableStream({ + write(chunk) { + if (chunk instanceof Uint8Array) { + put(position, chunk); + } else { + state.positioned++; + put(chunk.position, chunk.data); + } + }, + close() { + state.closed = true; + }, + abort(reason) { + state.aborted = reason ?? 'aborted'; + }, + }); + return { stream, bytes: () => buf.slice(0, length), state }; +} + +/** A deterministic content of n bytes. */ +export function content(n: number, seed = 1): Uint8Array { + const b = new Uint8Array(n); + let x = seed; + for (let i = 0; i < n; i++) { + x = (x * 1103515245 + 12345) >>> 0; + b[i] = x >>> 24; + } + return b; +} + +/** A ZIP of the page: a head, the content of its files, and the time of its round, for the files without an mtime. */ +export interface ZipSample { + readonly name: string; + readonly head: Head; + readonly files: readonly Uint8Array[]; + readonly roundTime: number; +} + +// The time of round 1000 of Quicknet, 2023-08-23T15:59:24Z. +const ROUND_1000 = 1692806364; + +// A head of the files given in the byte order of their paths, which the +// samples give already sorted. +function headOf(files: readonly [string, Uint8Array, number | undefined][]): { head: Head; files: Uint8Array[] } { + let end = 0; + const entries = files.map(([path, bytes, mtime]): HeadFile => { + const f = { path, size: bytes.length, start: end, end: end + bytes.length, sha256: new Uint8Array(32), ...(mtime === undefined ? {} : { mtime }) }; + end += bytes.length; + return f; + }); + return { head: { salt: new Uint8Array(32), comment: '', author: '', files: entries, critical: [], noncritical: [] }, files: files.map(([, b]) => b) }; +} + +/** + * The samples: files in folders with names out of ASCII and every kind of + * time (in 1980 and before, up to 2^31 - 1 and after, up to 9999, and none), + * one of them over 64 KiB and one empty; one file inside a folder; and 65535 + * empty files, which make the archive ZIP64. + */ +export function zipSamples(): ZipSample[] { + const tree = headOf([ + ['carta.txt', content(37, 1), 1790769600], + ['fotos/2025/playa.jpg', content(70_000, 2), 2 ** 31 - 1], + ['fotos/vac\u00edo.txt', new Uint8Array(0), 2 ** 31], + ['z/9999.bin', content(3, 3), 253402300799], + ['z/sin fecha.txt', content(4, 4), undefined], + ['\u00d1and\u00fa/nota \u{1f642}.txt', content(10, 5), 1], + ]); + const inside = headOf([['informes/2026.pdf', content(1000, 6), 1790769600]]); + const many = headOf(Array.from({ length: 65535 }, (_, i): [string, Uint8Array, number] => [`d/${String(i).padStart(5, '0')}`, new Uint8Array(0), 1790769600])); + return [ + { name: 'files in folders, names out of ASCII and every kind of time', ...tree, roundTime: ROUND_1000 }, + { name: 'one file inside a folder', ...inside, roundTime: ROUND_1000 }, + { name: '65535 files: ZIP64 by the number of entries', ...many, roundTime: ROUND_1000 }, + ]; +} + +/** Writes a sample through a ZipSink into a file in memory, each file in pieces of `piece` bytes, as open would. */ +export async function writeZip(sample: ZipSample, piece = 16384): Promise<{ bytes: Uint8Array; sink: ZipSink; file: MemoryFile }> { + const file = memoryFile(); + const sink = new ZipSink(file.stream, sample.roundTime); + sink.begin(sample.head); + for (const [i, bytes] of sample.files.entries()) { + const w = sink.create(i).getWriter(); + for (let at = 0; at < bytes.length; at += piece) await w.write(bytes.subarray(at, at + piece)); + await w.close(); + } + await sink.commit(); + return { bytes: file.bytes(), sink, file }; +} diff --git a/src/lib/inspector/tempfile.ts b/src/lib/inspector/tempfile.ts index 23bccff..9f94089 100644 --- a/src/lib/inspector/tempfile.ts +++ b/src/lib/inspector/tempfile.ts @@ -37,9 +37,17 @@ export interface TempDirectory { keys(): AsyncIterable; } +/** + * A chunk of the writable of a temporary file, as FileSystemWritableFileStream + * takes it: bytes at the position of the file, or bytes at a given position, + * with which the ZIP of a format 3 capsule patches the CRC-32 of an entry in + * its local header (zipsink.ts). A write moves the position to its end. + */ +export type TempChunk = Uint8Array | { readonly type: 'write'; readonly position: number; readonly data: Uint8Array }; + /** The part of FileSystemFileHandle that this module uses. */ export interface TempFileHandle { - createWritable(): Promise>; + createWritable(): Promise>; getFile(): Promise; } @@ -84,8 +92,11 @@ export async function freeSpace(pf: TempPlatform): Promise { /** A temporary file of this tab, open for writing. */ export interface TempFile { - /** The output of open: closed after step 18, aborted on any failure. */ - readonly writable: WritableStream; + /** + * The output of open, or the stream of the sink of a format 3 capsule: + * closed after step 18, aborted on any failure. + */ + readonly writable: WritableStream; /** The committed content, once open has closed the output. */ file(): Promise; /** Deletes the file and its directory and releases the lock; never rejects. */ @@ -99,9 +110,9 @@ export interface TempFile { * an opening this way when the person moves on to another capsule. Closing * and aborting go to `w`. */ -export function cancellable(w: WritableStream, cancelled: () => boolean): WritableStream { +export function cancellable(w: WritableStream, cancelled: () => boolean): WritableStream { const inner = w.getWriter(); - return new WritableStream({ + return new WritableStream({ async write(chunk) { if (cancelled()) { const reason = new Error('opening cancelled'); @@ -121,7 +132,7 @@ export async function createTempFile(pf: TempPlatform, area: TempArea = OPEN_ARE const release = pf.locks === undefined ? () => undefined : await hold(pf.locks, `${area.root}/${id}`); let root: TempDirectory | undefined; let handle: TempFileHandle; - let writable: WritableStream; + let writable: WritableStream; try { root = await (await pf.getDirectory()).getDirectoryHandle(area.root, { create: true }); const dir = await root.getDirectoryHandle(id, { create: true }); diff --git a/src/lib/inspector/zipsink.test.ts b/src/lib/inspector/zipsink.test.ts new file mode 100644 index 0000000..7afb6da --- /dev/null +++ b/src/lib/inspector/zipsink.test.ts @@ -0,0 +1,154 @@ +// Tests of zipsink.ts: how the files of a capsule are laid out, the ZIP that +// the sink writes with positioned writes and zipOf in memory, byte for byte +// the same, each file a range of it, and the sink behind open on the format +// 3 fixtures, closed only at commit and aborted on a failure; and the ZIP +// samples as archive/zip of Go reads them (testing/zip-vectors.json). + +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { sha256, toHex } from '../dkc/bytes.ts'; +import { parseRFC3339 } from '../dkc/datekey.ts'; +import type { Head } from '../dkc/head.ts'; +import { open } from '../dkc/open.ts'; +import { suppliedRelease } from '../dkc/release.ts'; +import { h, hx, readBytes, readJSON } from '../dkc/testing/testdata.ts'; +import { memoryFile, writeZip, zipSamples } from '../dkc/testing/zip.ts'; +import { crc32 } from './crc32.ts'; +import { zipLayout } from './zip.ts'; +import { zipEntries, zipMode, ZipSink, zipOf } from './zipsink.ts'; + +interface Record { + release: { round: number; signature: string }; + unlock_at: string; + files?: { path: string; size: number; sha256: string; mtime?: number }[]; +} + +const file = (path: string, size = 1, mtime?: number) => ({ path, size, start: 0, end: size, sha256: new Uint8Array(32), ...(mtime === undefined ? {} : { mtime }) }); +const head = (...files: ReturnType[]): Head => ({ salt: new Uint8Array(32), comment: '', author: '', files, critical: [], noncritical: [] }); + +// Opens a fixture of format 3 into a ZipSink over a file in memory. +async function openInto(name: string, dkc = readBytes(`fixtures/${name}.dkc`)) { + const rec = readJSON(`fixtures/${name}.json`); + const unlock = parseRFC3339(rec.unlock_at); + const out = memoryFile(); + const sink = new ZipSink(out.stream, unlock.seconds); + const r = await open(dkc, { source: suppliedRelease({ round: rec.release.round, signature: h(rec.release.signature) }), now: () => unlock, sink }); + return { r, out, sink, rec, unlock }; +} + +describe('zipMode and zipEntries', () => { + it('lays out one file of one segment as itself, more files or one inside a folder as a ZIP, and no file as nothing', () => { + expect(zipMode(head())).toBe('none'); + expect(zipMode(head(file('nota.txt')))).toBe('file'); + expect(zipMode(head(file('docs/nota.txt')))).toBe('zip'); + expect(zipMode(head(file('a'), file('b')))).toBe('zip'); + }); + + it('gives a file without an mtime the time of the round', () => { + expect(zipEntries(head(file('a', 3, 7), file('b', 0)), 1692806364)).toEqual([ + { name: 'a', size: 3, mtime: 7 }, + { name: 'b', size: 0, mtime: 1692806364 }, + ]); + }); +}); + +describe('ZipSink and zipOf', () => { + it('write the same ZIP, each file a range of it, with its CRC-32 patched by a positioned write', async () => { + const sample = zipSamples()[0]!; + for (const piece of [1, 4096, 1 << 20]) { + const { bytes, sink, file: out } = await writeZip(sample, piece); + expect(hx(bytes)).toBe(hx(new Uint8Array(await zipOf(sample.head, sample.files, sample.roundTime).arrayBuffer()))); + expect([sink.mode, sink.length, out.state.closed, out.state.positioned]).toEqual(['zip', bytes.length, true, 2 * sample.files.length + 1]); + sink.ranges.forEach(([start, end], i) => expect(hx(bytes.subarray(start, end)), sample.head.files[i]!.path).toBe(hx(sample.files[i]!))); + // The CRC-32 of each entry, in its local header. + const v = new DataView(bytes.buffer); + zipLayout(zipEntries(sample.head, sample.roundTime)).entries.forEach((e, i) => expect(v.getUint32(e.crcOffset, true)).toBe(crc32(sample.files[i]!))); + } + }); + + it('writes the one file of one segment as it is, and nothing for a capsule without files', async () => { + const one = memoryFile(); + const sink = new ZipSink(one.stream, 0); + sink.begin(head(file('nota.txt', 4))); + const w = sink.create(0).getWriter(); + await w.write(new TextEncoder().encode('ho')); + await w.write(new TextEncoder().encode('la')); + await w.close(); + await sink.commit(); + expect([new TextDecoder().decode(one.bytes()), sink.mode, sink.ranges, sink.length, one.state.positioned]).toEqual(['hola', 'file', [[0, 4]], 4, 0]); + const none = memoryFile(); + const empty = new ZipSink(none.stream, 0); + empty.begin(head()); + await empty.commit(); + expect([none.bytes().length, none.state.closed, empty.mode, empty.ranges, empty.length]).toEqual([0, true, 'none', [], 0]); + }); + + it('aborts its file, whether it began or not', async () => { + const early = memoryFile(); + await new ZipSink(early.stream, 0).abort(new Error('failed at step 9')); + expect(early.state.aborted).toEqual(new Error('failed at step 9')); + const late = memoryFile(); + const sink = new ZipSink(late.stream, 0); + sink.begin(head(file('a/b', 2))); + await sink.abort(new Error('failed at step 17')); + expect([late.state.closed, late.state.aborted]).toEqual([false, new Error('failed at step 17')]); + // A file that fails to abort does not make the abort fail. + const stubborn = new ZipSink(new WritableStream({ abort: () => Promise.reject(new Error('cannot abort')) }), 0); + await expect(stubborn.abort(new Error('failed'))).resolves.toBeUndefined(); + }); +}); + +describe('ZipSink behind open', () => { + it('receives the files of format3_tree in a ZIP closed only after step 18', async () => { + const { r, out, sink, rec } = await openInto('format3_tree'); + expect([r.error, out.state.closed, sink.mode]).toEqual([undefined, true, 'zip']); + const bytes = out.bytes(); + expect(bytes.length).toBe(sink.length); + const files = rec.files!; + for (const [i, [start, end]] of sink.ranges.entries()) { + expect(toHex(await sha256(bytes.subarray(start, end))), files[i]!.path).toBe(files[i]!.sha256); + } + const contents = sink.ranges.map(([s, e]) => bytes.slice(s, e)); + expect(hx(bytes)).toBe(hx(new Uint8Array(await zipOf(r.head!, contents, parseRFC3339(readJSON('fixtures/format3_tree.json').unlock_at).seconds).arrayBuffer()))); + }); + + it('receives the one file of format3_single as it is, and aborts the file when the capsule is cut short', async () => { + const { r, out, sink, rec } = await openInto('format3_single'); + expect([r.error, sink.mode, toHex(await sha256(out.bytes()))]).toEqual([undefined, 'file', rec.files![0]!.sha256]); + const cut = readBytes('fixtures/format3_tree.dkc'); + const bad = await openInto('format3_tree', cut.subarray(0, -1)); + expect([bad.r.error?.code, bad.out.state.closed, bad.out.state.aborted === undefined]).toEqual(['ERR_INTEGRITY', false, false]); + }); +}); + +// scripts/zip-ts-samples.mjs wrote the samples of testing/zip.ts, and +// scripts/zip-go-read.go read them with archive/zip: the same bytes must give +// the entries they were written with. +describe('the ZIP samples, read by archive/zip of Go (zip-vectors.json)', () => { + const V = JSON.parse(readFileSync(new URL('../dkc/testing/zip-vectors.json', import.meta.url), 'utf8')) as { + samples: { name: string; sha256: string; count: number; lines_sha256: string; lines?: string[]; error?: string }[]; + }; + const samples = zipSamples(); + + it('holds every sample, in order', () => { + expect(V.samples.map((s) => s.name)).toEqual(samples.map((s) => s.name)); + }); + + it.each(samples.map((s, i) => [s.name, s, i] as const))('%s', async (_, sample, i) => { + const go = V.samples[i]!; + const { bytes } = await writeZip(sample); + expect(toHex(await sha256(bytes))).toBe(go.sha256); + // Each entry as Go gives it: name, UTF-8, stored, CRC-32, sizes, time and SHA-256. + const empty = toHex(await sha256(new Uint8Array(0))); + const lines: string[] = []; + for (const [k, e] of zipEntries(sample.head, sample.roundTime).entries()) { + const content = sample.files[k]!; + const time = new Date(e.mtime * 1000).toISOString().replace('.000Z', 'Z'); + const sum = content.length === 0 ? empty : toHex(await sha256(content)); + lines.push([e.name, 'true', '0', crc32(content).toString(16).padStart(8, '0'), e.size, e.size, time, sum].join('\t')); + } + expect([go.error, go.count]).toEqual([undefined, lines.length]); + expect(toHex(await sha256(new TextEncoder().encode(lines.join('\n'))))).toBe(go.lines_sha256); + if (go.lines !== undefined) expect(go.lines).toEqual(lines); + }); +}); diff --git a/src/lib/inspector/zipsink.ts b/src/lib/inspector/zipsink.ts new file mode 100644 index 0000000..1cd1c15 --- /dev/null +++ b/src/lib/inspector/zipsink.ts @@ -0,0 +1,135 @@ +// The sink of the page for the files of a format 3 capsule (design of format +// 3, section 5), and the same ZIP made in memory. The files go to one +// temporary file of OPFS: the file itself when the capsule holds one file of +// one segment, and otherwise a ZIP of stored entries that zip.ts lays out +// from the head before any byte arrives. Each file is written as open +// delivers it, after the local header of its entry; the CRC-32 of the entry +// is patched in that header with a positioned write once its bytes are +// known, and the central directory is written by commit, which closes the +// file. The writable of OPFS keeps the writes in its swap file until then, +// and abort discards them, so nothing is shown before step 18 (spec §56). +// Each file is then a contiguous range of the temporary file, which the page +// offers as a download of its own. + +import type { Head } from '../dkc/head.ts'; +import type { Sink } from '../dkc/sink.ts'; +import { Crc32, crc32 } from './crc32.ts'; +import type { TempChunk } from './tempfile.ts'; +import { type ZipEntry, zipLayout, type ZipLayout } from './zip.ts'; + +/** How the files of a capsule go into one file: none, the one file itself, or a ZIP. */ +export type ZipMode = 'none' | 'file' | 'zip'; + +/** The file itself when the head holds one file of one segment, a ZIP when it holds more or one inside a folder, and none without files. */ +export function zipMode(head: Head): ZipMode { + if (head.files.length === 0) return 'none'; + return head.files.length === 1 && !head.files[0]!.path.includes('/') ? 'file' : 'zip'; +} + +/** + * The entries of the ZIP of a head: each file with its path, its size and + * its mtime, or the time of the round, in seconds, when it has none. + */ +export function zipEntries(head: Head, roundTime: number): ZipEntry[] { + return head.files.map((f) => ({ name: f.path, size: f.size, mtime: f.mtime ?? roundTime })); +} + +/** + * A sink that writes the files of a capsule into one writable of OPFS, the + * file itself or a ZIP (see zipMode). It closes the writable at commit and + * aborts it at abort, which the caller may also call when the opening failed + * before begin. + */ +export class ZipSink implements Sink { + readonly #out: WritableStream; + readonly #roundTime: number; + #writer: WritableStreamDefaultWriter | undefined; + #mode: ZipMode = 'none'; + #layout: ZipLayout | undefined; + #crcs: number[] = []; + #ranges: (readonly [number, number])[] = []; + + /** `roundTime` is the time of the round of the capsule, in seconds, for the entries without an mtime. */ + constructor(out: WritableStream, roundTime: number) { + this.#out = out; + this.#roundTime = roundTime; + } + + begin(head: Head): void { + this.#writer = this.#out.getWriter(); + this.#mode = zipMode(head); + if (this.#mode === 'zip') { + this.#layout = zipLayout(zipEntries(head, this.#roundTime)); + this.#ranges = this.#layout.entries.map((e) => [e.dataOffset, e.dataOffset + e.size] as const); + } else if (this.#mode === 'file') { + this.#ranges = [[0, head.files[0]!.size]]; + } + } + + create(i: number): WritableStream { + const w = this.#writer!; + const entry = this.#layout?.entries[i]; + const sum = new Crc32(); + return new WritableStream({ + start: async () => { + // The local header of the entry, at its offset: the patch of the + // CRC-32 of the entry before moved the position of the file. + if (entry !== undefined) await w.write({ type: 'write', position: entry.headerOffset, data: entry.header }); + }, + write: async (chunk) => { + sum.update(chunk); + await w.write(chunk); + }, + close: async () => { + if (entry === undefined) return; + const crc = new Uint8Array(4); + new DataView(crc.buffer).setUint32(0, sum.value, true); + this.#crcs[i] = sum.value; + await w.write({ type: 'write', position: entry.crcOffset, data: crc }); + }, + }); + } + + async commit(): Promise { + const w = this.#writer!; + if (this.#layout !== undefined) await w.write({ type: 'write', position: this.#layout.centralOffset, data: this.#layout.central(this.#crcs) }); + await w.close(); + } + + async abort(reason: unknown): Promise { + await (this.#writer ?? this.#out).abort(reason).catch(() => undefined); + } + + /** How the files were laid out, once begun. */ + get mode(): ZipMode { + return this.#mode; + } + + /** The range of each file in the file written, once begun: [start, end), in the order of the head. */ + get ranges(): readonly (readonly [number, number])[] { + return this.#ranges; + } + + /** The length of the file written, once begun. */ + get length(): number { + return this.#layout?.length ?? this.#ranges[0]?.[1] ?? 0; + } +} + +/** + * The ZIP of the files of a head held in memory, the same bytes a ZipSink + * writes, as a Blob of its parts: the local headers with their CRC-32, the + * files themselves, not copied, and the central directory. + */ +export function zipOf(head: Head, files: readonly Uint8Array[], roundTime: number): Blob { + const layout = zipLayout(zipEntries(head, roundTime)); + const crcs = files.map(crc32); + const parts: Uint8Array[] = []; + layout.entries.forEach((e, i) => { + const header = e.header.slice(); + new DataView(header.buffer).setUint32(e.crcOffset - e.headerOffset, crcs[i]!, true); + parts.push(header, files[i]!); + }); + parts.push(layout.central(crcs)); + return new Blob(parts as Uint8Array[]); +} diff --git a/vitest.config.ts b/vitest.config.ts index e911647..88edcff 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -61,6 +61,7 @@ export default defineConfig({ 'src/lib/dkc/sink.ts': { 100: true }, 'src/lib/inspector/crc32.ts': { 100: true }, 'src/lib/inspector/zip.ts': { 100: true }, + 'src/lib/inspector/zipsink.ts': { 100: true }, 'src/lib/dkc/**/*.ts': { statements: 95, branches: 90, functions: 95, lines: 95 }, // The page model and helpers of the inspector (plan §8, phase 1). 'src/lib/inspector/**/*.ts': { statements: 95, branches: 90, functions: 95, lines: 95 },