zipsink.ts is the sink of the page for a format 3 capsule (design of format 3, section 5). Its files go to the temporary file of OPFS: the file itself when the capsule holds one file of one segment, and otherwise a ZIP of stored entries laid out from the head before any byte arrives. Each local header is written at its offset, the bytes of the file follow as open delivers them, the CRC-32 of the entry is patched in its header with a positioned write, and commit writes the central directory and closes the file; abort discards it, also when the opening failed before begin. Each file is a contiguous range of the file written (ranges), and zipOf makes the same ZIP in memory as a Blob of its parts. tempfile.ts: the writable of a temporary file takes TempChunk, bytes at the position of the file or at a given one, as FileSystemWritableFileStream does; cancellable is generic. Interoperability: scripts/zip-ts-samples.mjs writes the samples of testing/zip.ts with ZipSink, and scripts/zip-go-read.go reads them with archive/zip of the Go standard library: names out of ASCII with bit 11, stored entries, their CRC-32 and sizes, the times of the extra fields in 1970, at 2^31 - 1 and after it, in 9999 and the time of the round for a file without one, and 65535 entries, ZIP64 by their number. The reading is frozen in testing/zip-vectors.json, and zipsink.test.ts writes each sample again, requires its SHA-256 and computes the entries Go must have read. zipsink.ts is covered at 100 %. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>main
parent
2efc8bcda8
commit
ee82f4382f
@ -0,0 +1,132 @@
|
|||||||
|
//go:build ignore
|
||||||
|
|
||||||
|
// Prints src/lib/dkc/testing/zip-vectors.json: how archive/zip of the Go
|
||||||
|
// standard library reads the ZIP samples that the page writes (plan of
|
||||||
|
// format 3 in datekeys-ts, step 5), which scripts/zip-ts-samples.mjs wrote
|
||||||
|
// into a directory. For each archive it records the SHA-256 of its bytes,
|
||||||
|
// the number of its entries and, for each entry, a line with its name,
|
||||||
|
// whether bit 11 marks the name as UTF-8, its method, its CRC-32, its sizes,
|
||||||
|
// the time archive/zip reads from its extra fields, in UTC, and the SHA-256
|
||||||
|
// of its content, read with the CRC-32 checked. It keeps the lines of an
|
||||||
|
// archive of 20 entries at most, and the SHA-256 of all of them joined by
|
||||||
|
// newlines for any archive, so that the archive of 65535 entries, which is
|
||||||
|
// ZIP64 by its number of entries, weighs little. zipsink.test.ts writes each
|
||||||
|
// archive again, requires its SHA-256, and computes the lines it must give.
|
||||||
|
//
|
||||||
|
// It needs only the standard library:
|
||||||
|
//
|
||||||
|
// node scripts/zip-ts-samples.mjs DIR > zip-samples.json
|
||||||
|
// go run zip-go-read.go DIR zip-samples.json > zip-vectors.json
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sampleIn struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
File string `json:"file"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type sampleOut struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
LinesSHA256 string `json:"lines_sha256"`
|
||||||
|
Lines []string `json:"lines,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type output struct {
|
||||||
|
Description string `json:"description"`
|
||||||
|
Generator string `json:"generator"`
|
||||||
|
SamplesGenerator string `json:"samples_generator"`
|
||||||
|
Go string `json:"go"`
|
||||||
|
Samples []sampleOut `json:"samples"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func must[T any](v T, err error) T {
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// read reads an archive with archive/zip, and gives one line for each entry.
|
||||||
|
func read(b []byte) ([]string, error) {
|
||||||
|
r, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
for _, f := range r.File {
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", f.Name, err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(rc)
|
||||||
|
rc.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", f.Name, err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
lines = append(lines, fmt.Sprintf("%s\t%t\t%d\t%08x\t%d\t%d\t%s\t%s",
|
||||||
|
f.Name, f.Flags&0x800 != 0, f.Method, f.CRC32, f.UncompressedSize64, f.CompressedSize64,
|
||||||
|
f.Modified.UTC().Format(time.RFC3339), hex.EncodeToString(sum[:])))
|
||||||
|
}
|
||||||
|
return lines, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
dir := os.Args[1]
|
||||||
|
var in struct {
|
||||||
|
Generator string `json:"generator"`
|
||||||
|
Samples []sampleIn `json:"samples"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(must(os.ReadFile(os.Args[2])), &in); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
out := output{
|
||||||
|
Description: "How archive/zip of Go reads the ZIP samples of the page; see the header of scripts/zip-go-read.go.",
|
||||||
|
Generator: "scripts/zip-go-read.go",
|
||||||
|
SamplesGenerator: in.Generator,
|
||||||
|
Go: runtime.Version(),
|
||||||
|
}
|
||||||
|
for _, s := range in.Samples {
|
||||||
|
b := must(os.ReadFile(filepath.Join(dir, s.File)))
|
||||||
|
sum := sha256.Sum256(b)
|
||||||
|
so := sampleOut{Name: s.Name, SHA256: hex.EncodeToString(sum[:])}
|
||||||
|
if so.SHA256 != s.SHA256 {
|
||||||
|
panic(fmt.Sprintf("%s: the file is not the sample written", s.Name))
|
||||||
|
}
|
||||||
|
lines, err := read(b)
|
||||||
|
if err != nil {
|
||||||
|
so.Error = err.Error()
|
||||||
|
}
|
||||||
|
so.Count = len(lines)
|
||||||
|
all := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||||
|
so.LinesSHA256 = hex.EncodeToString(all[:])
|
||||||
|
if len(lines) <= 20 {
|
||||||
|
so.Lines = lines
|
||||||
|
}
|
||||||
|
out.Samples = append(out.Samples, so)
|
||||||
|
}
|
||||||
|
e := json.NewEncoder(os.Stdout)
|
||||||
|
e.SetEscapeHTML(false)
|
||||||
|
e.SetIndent("", " ")
|
||||||
|
if err := e.Encode(out); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
// Writes the ZIP samples of the page (src/lib/dkc/testing/zip.ts), as the
|
||||||
|
// ZipSink of src/lib/inspector/zipsink.ts writes them, into a directory for
|
||||||
|
// scripts/zip-go-read.go, and prints their index as JSON: the name, the file
|
||||||
|
// and the SHA-256 of each. The samples are deterministic: zipsink.test.ts
|
||||||
|
// writes them again and requires the same SHA-256. Node runs the TypeScript
|
||||||
|
// sources directly (type stripping, Node 22.6+):
|
||||||
|
//
|
||||||
|
// node scripts/zip-ts-samples.mjs DIR > zip-samples.json
|
||||||
|
|
||||||
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { sha256, toHex } from '../src/lib/dkc/bytes.ts';
|
||||||
|
import { writeZip, zipSamples } from '../src/lib/dkc/testing/zip.ts';
|
||||||
|
|
||||||
|
const dir = process.argv[2];
|
||||||
|
if (dir === undefined) throw new Error('usage: node scripts/zip-ts-samples.mjs DIR');
|
||||||
|
mkdirSync(dir, { recursive: true });
|
||||||
|
const samples = [];
|
||||||
|
for (const [i, s] of zipSamples().entries()) {
|
||||||
|
const { bytes } = await writeZip(s);
|
||||||
|
const file = `${i}.zip`;
|
||||||
|
writeFileSync(join(dir, file), bytes);
|
||||||
|
samples.push({ name: s.name, file, sha256: toHex(await sha256(bytes)) });
|
||||||
|
}
|
||||||
|
process.stdout.write(`${JSON.stringify({ generator: 'scripts/zip-ts-samples.mjs', samples }, null, 1)}\n`);
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"description": "How archive/zip of Go reads the ZIP samples of the page; see the header of scripts/zip-go-read.go.",
|
||||||
|
"generator": "scripts/zip-go-read.go",
|
||||||
|
"samples_generator": "scripts/zip-ts-samples.mjs",
|
||||||
|
"go": "go1.26.8",
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"name": "files in folders, names out of ASCII and every kind of time",
|
||||||
|
"sha256": "75413198ecebafeb8c4ad03d176048170022b8e6131509e415ced919fe696c08",
|
||||||
|
"count": 6,
|
||||||
|
"lines_sha256": "8a877b3c02ac6f59d26416fcb05bb87897df36a334595111f60c8d8fd99cace7",
|
||||||
|
"lines": [
|
||||||
|
"carta.txt\ttrue\t0\t151ad124\t37\t37\t2026-09-30T12:00:00Z\t0c84897e85eeb721badb3b345cd858784ad09ee350d87340937d274929161559",
|
||||||
|
"fotos/2025/playa.jpg\ttrue\t0\taf26b746\t70000\t70000\t2038-01-19T03:14:07Z\tc6618f49d56788ffe3af242905edd2ab826949fccbc6ae5b6fe23e76ef4b62ef",
|
||||||
|
"fotos/vacío.txt\ttrue\t0\t00000000\t0\t0\t2038-01-19T03:14:08Z\te3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||||
|
"z/9999.bin\ttrue\t0\t65c2824e\t3\t3\t9999-12-31T23:59:59Z\tc90f0165b02cf99598f8cf7df688df50875844dc3cece28fc45fdb57608c8ed0",
|
||||||
|
"z/sin fecha.txt\ttrue\t0\td7e6d6c8\t4\t4\t2023-08-23T15:59:24Z\te61cdeaf027408b22b987fd3efeca4c800042aa0d9662b78a3e2380210236c9c",
|
||||||
|
"Ñandú/nota 🙂.txt\ttrue\t0\te246eb5c\t10\t10\t1970-01-01T00:00:01Z\td267b25e018d7546fc83f5c162840ab16d17780a9d02d4cfdea70cc0603c902d"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "one file inside a folder",
|
||||||
|
"sha256": "4d69b37639cea9de05fe257ff65b2f9b790ca99d299074f7a4fd6e63a7c7dc70",
|
||||||
|
"count": 1,
|
||||||
|
"lines_sha256": "f10c60dbd513242427c5faafb5fefb10963fbedcee170c9b6c6f1fd68a896978",
|
||||||
|
"lines": [
|
||||||
|
"informes/2026.pdf\ttrue\t0\tef7caf18\t1000\t1000\t2026-09-30T12:00:00Z\t9e148ea06d47eb7f031c1d20feab31f6251f6b54a778d123086a38c93ae21996"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "65535 files: ZIP64 by the number of entries",
|
||||||
|
"sha256": "4c42a7155a6a09205da97c0f5cf99e932323ce72100c9d80b6247c24d9f30559",
|
||||||
|
"count": 65535,
|
||||||
|
"lines_sha256": "e650cafb0822a83ac524e637e73136fdfed406ec3de943471d15965762a86e20"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,129 @@
|
|||||||
|
// Tests only: a file in memory that takes the chunks of a writable of OPFS
|
||||||
|
// (TempChunk of inspector/tempfile.ts), and the ZIP samples that the page
|
||||||
|
// writes and the Go reference reads with archive/zip
|
||||||
|
// (scripts/zip-ts-samples.mjs, scripts/zip-go-read.go and
|
||||||
|
// inspector/zipsink.test.ts). The samples are deterministic: the same code
|
||||||
|
// writes the same bytes, whose SHA-256 is frozen with the reading of Go.
|
||||||
|
|
||||||
|
import type { TempChunk } from '../../inspector/tempfile.ts';
|
||||||
|
import { ZipSink } from '../../inspector/zipsink.ts';
|
||||||
|
import type { Head, HeadFile } from '../head.ts';
|
||||||
|
|
||||||
|
/** A file in memory written through a WritableStream of TempChunk, and how its writing ended. */
|
||||||
|
export interface MemoryFile {
|
||||||
|
readonly stream: WritableStream<TempChunk>;
|
||||||
|
bytes(): Uint8Array;
|
||||||
|
readonly state: { closed: boolean; aborted: unknown; positioned: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A file in memory that takes TempChunk as FileSystemWritableFileStream
|
||||||
|
* does: bytes at its position, or at the position given, zeros filling any
|
||||||
|
* gap, each write moving the position to its end.
|
||||||
|
*/
|
||||||
|
export function memoryFile(): MemoryFile {
|
||||||
|
let buf = new Uint8Array(1 << 16);
|
||||||
|
let length = 0;
|
||||||
|
let position = 0;
|
||||||
|
const state = { closed: false, aborted: undefined as unknown, positioned: 0 };
|
||||||
|
const put = (at: number, data: Uint8Array): void => {
|
||||||
|
const end = at + data.length;
|
||||||
|
if (end > buf.length) {
|
||||||
|
const bigger = new Uint8Array(Math.max(end, buf.length * 2));
|
||||||
|
bigger.set(buf.subarray(0, length));
|
||||||
|
buf = bigger;
|
||||||
|
}
|
||||||
|
buf.set(data, at);
|
||||||
|
length = Math.max(length, end);
|
||||||
|
position = end;
|
||||||
|
};
|
||||||
|
const stream = new WritableStream<TempChunk>({
|
||||||
|
write(chunk) {
|
||||||
|
if (chunk instanceof Uint8Array) {
|
||||||
|
put(position, chunk);
|
||||||
|
} else {
|
||||||
|
state.positioned++;
|
||||||
|
put(chunk.position, chunk.data);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
state.closed = true;
|
||||||
|
},
|
||||||
|
abort(reason) {
|
||||||
|
state.aborted = reason ?? 'aborted';
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return { stream, bytes: () => buf.slice(0, length), state };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A deterministic content of n bytes. */
|
||||||
|
export function content(n: number, seed = 1): Uint8Array {
|
||||||
|
const b = new Uint8Array(n);
|
||||||
|
let x = seed;
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
x = (x * 1103515245 + 12345) >>> 0;
|
||||||
|
b[i] = x >>> 24;
|
||||||
|
}
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A ZIP of the page: a head, the content of its files, and the time of its round, for the files without an mtime. */
|
||||||
|
export interface ZipSample {
|
||||||
|
readonly name: string;
|
||||||
|
readonly head: Head;
|
||||||
|
readonly files: readonly Uint8Array[];
|
||||||
|
readonly roundTime: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The time of round 1000 of Quicknet, 2023-08-23T15:59:24Z.
|
||||||
|
const ROUND_1000 = 1692806364;
|
||||||
|
|
||||||
|
// A head of the files given in the byte order of their paths, which the
|
||||||
|
// samples give already sorted.
|
||||||
|
function headOf(files: readonly [string, Uint8Array, number | undefined][]): { head: Head; files: Uint8Array[] } {
|
||||||
|
let end = 0;
|
||||||
|
const entries = files.map(([path, bytes, mtime]): HeadFile => {
|
||||||
|
const f = { path, size: bytes.length, start: end, end: end + bytes.length, sha256: new Uint8Array(32), ...(mtime === undefined ? {} : { mtime }) };
|
||||||
|
end += bytes.length;
|
||||||
|
return f;
|
||||||
|
});
|
||||||
|
return { head: { salt: new Uint8Array(32), comment: '', author: '', files: entries, critical: [], noncritical: [] }, files: files.map(([, b]) => b) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The samples: files in folders with names out of ASCII and every kind of
|
||||||
|
* time (in 1980 and before, up to 2^31 - 1 and after, up to 9999, and none),
|
||||||
|
* one of them over 64 KiB and one empty; one file inside a folder; and 65535
|
||||||
|
* empty files, which make the archive ZIP64.
|
||||||
|
*/
|
||||||
|
export function zipSamples(): ZipSample[] {
|
||||||
|
const tree = headOf([
|
||||||
|
['carta.txt', content(37, 1), 1790769600],
|
||||||
|
['fotos/2025/playa.jpg', content(70_000, 2), 2 ** 31 - 1],
|
||||||
|
['fotos/vac\u00edo.txt', new Uint8Array(0), 2 ** 31],
|
||||||
|
['z/9999.bin', content(3, 3), 253402300799],
|
||||||
|
['z/sin fecha.txt', content(4, 4), undefined],
|
||||||
|
['\u00d1and\u00fa/nota \u{1f642}.txt', content(10, 5), 1],
|
||||||
|
]);
|
||||||
|
const inside = headOf([['informes/2026.pdf', content(1000, 6), 1790769600]]);
|
||||||
|
const many = headOf(Array.from({ length: 65535 }, (_, i): [string, Uint8Array, number] => [`d/${String(i).padStart(5, '0')}`, new Uint8Array(0), 1790769600]));
|
||||||
|
return [
|
||||||
|
{ name: 'files in folders, names out of ASCII and every kind of time', ...tree, roundTime: ROUND_1000 },
|
||||||
|
{ name: 'one file inside a folder', ...inside, roundTime: ROUND_1000 },
|
||||||
|
{ name: '65535 files: ZIP64 by the number of entries', ...many, roundTime: ROUND_1000 },
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Writes a sample through a ZipSink into a file in memory, each file in pieces of `piece` bytes, as open would. */
|
||||||
|
export async function writeZip(sample: ZipSample, piece = 16384): Promise<{ bytes: Uint8Array; sink: ZipSink; file: MemoryFile }> {
|
||||||
|
const file = memoryFile();
|
||||||
|
const sink = new ZipSink(file.stream, sample.roundTime);
|
||||||
|
sink.begin(sample.head);
|
||||||
|
for (const [i, bytes] of sample.files.entries()) {
|
||||||
|
const w = sink.create(i).getWriter();
|
||||||
|
for (let at = 0; at < bytes.length; at += piece) await w.write(bytes.subarray(at, at + piece));
|
||||||
|
await w.close();
|
||||||
|
}
|
||||||
|
await sink.commit();
|
||||||
|
return { bytes: file.bytes(), sink, file };
|
||||||
|
}
|
||||||
@ -0,0 +1,154 @@
|
|||||||
|
// Tests of zipsink.ts: how the files of a capsule are laid out, the ZIP that
|
||||||
|
// the sink writes with positioned writes and zipOf in memory, byte for byte
|
||||||
|
// the same, each file a range of it, and the sink behind open on the format
|
||||||
|
// 3 fixtures, closed only at commit and aborted on a failure; and the ZIP
|
||||||
|
// samples as archive/zip of Go reads them (testing/zip-vectors.json).
|
||||||
|
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { sha256, toHex } from '../dkc/bytes.ts';
|
||||||
|
import { parseRFC3339 } from '../dkc/datekey.ts';
|
||||||
|
import type { Head } from '../dkc/head.ts';
|
||||||
|
import { open } from '../dkc/open.ts';
|
||||||
|
import { suppliedRelease } from '../dkc/release.ts';
|
||||||
|
import { h, hx, readBytes, readJSON } from '../dkc/testing/testdata.ts';
|
||||||
|
import { memoryFile, writeZip, zipSamples } from '../dkc/testing/zip.ts';
|
||||||
|
import { crc32 } from './crc32.ts';
|
||||||
|
import { zipLayout } from './zip.ts';
|
||||||
|
import { zipEntries, zipMode, ZipSink, zipOf } from './zipsink.ts';
|
||||||
|
|
||||||
|
interface Record {
|
||||||
|
release: { round: number; signature: string };
|
||||||
|
unlock_at: string;
|
||||||
|
files?: { path: string; size: number; sha256: string; mtime?: number }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = (path: string, size = 1, mtime?: number) => ({ path, size, start: 0, end: size, sha256: new Uint8Array(32), ...(mtime === undefined ? {} : { mtime }) });
|
||||||
|
const head = (...files: ReturnType<typeof file>[]): Head => ({ salt: new Uint8Array(32), comment: '', author: '', files, critical: [], noncritical: [] });
|
||||||
|
|
||||||
|
// Opens a fixture of format 3 into a ZipSink over a file in memory.
|
||||||
|
async function openInto(name: string, dkc = readBytes(`fixtures/${name}.dkc`)) {
|
||||||
|
const rec = readJSON<Record>(`fixtures/${name}.json`);
|
||||||
|
const unlock = parseRFC3339(rec.unlock_at);
|
||||||
|
const out = memoryFile();
|
||||||
|
const sink = new ZipSink(out.stream, unlock.seconds);
|
||||||
|
const r = await open(dkc, { source: suppliedRelease({ round: rec.release.round, signature: h(rec.release.signature) }), now: () => unlock, sink });
|
||||||
|
return { r, out, sink, rec, unlock };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('zipMode and zipEntries', () => {
|
||||||
|
it('lays out one file of one segment as itself, more files or one inside a folder as a ZIP, and no file as nothing', () => {
|
||||||
|
expect(zipMode(head())).toBe('none');
|
||||||
|
expect(zipMode(head(file('nota.txt')))).toBe('file');
|
||||||
|
expect(zipMode(head(file('docs/nota.txt')))).toBe('zip');
|
||||||
|
expect(zipMode(head(file('a'), file('b')))).toBe('zip');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('gives a file without an mtime the time of the round', () => {
|
||||||
|
expect(zipEntries(head(file('a', 3, 7), file('b', 0)), 1692806364)).toEqual([
|
||||||
|
{ name: 'a', size: 3, mtime: 7 },
|
||||||
|
{ name: 'b', size: 0, mtime: 1692806364 },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('ZipSink and zipOf', () => {
|
||||||
|
it('write the same ZIP, each file a range of it, with its CRC-32 patched by a positioned write', async () => {
|
||||||
|
const sample = zipSamples()[0]!;
|
||||||
|
for (const piece of [1, 4096, 1 << 20]) {
|
||||||
|
const { bytes, sink, file: out } = await writeZip(sample, piece);
|
||||||
|
expect(hx(bytes)).toBe(hx(new Uint8Array(await zipOf(sample.head, sample.files, sample.roundTime).arrayBuffer())));
|
||||||
|
expect([sink.mode, sink.length, out.state.closed, out.state.positioned]).toEqual(['zip', bytes.length, true, 2 * sample.files.length + 1]);
|
||||||
|
sink.ranges.forEach(([start, end], i) => expect(hx(bytes.subarray(start, end)), sample.head.files[i]!.path).toBe(hx(sample.files[i]!)));
|
||||||
|
// The CRC-32 of each entry, in its local header.
|
||||||
|
const v = new DataView(bytes.buffer);
|
||||||
|
zipLayout(zipEntries(sample.head, sample.roundTime)).entries.forEach((e, i) => expect(v.getUint32(e.crcOffset, true)).toBe(crc32(sample.files[i]!)));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('writes the one file of one segment as it is, and nothing for a capsule without files', async () => {
|
||||||
|
const one = memoryFile();
|
||||||
|
const sink = new ZipSink(one.stream, 0);
|
||||||
|
sink.begin(head(file('nota.txt', 4)));
|
||||||
|
const w = sink.create(0).getWriter();
|
||||||
|
await w.write(new TextEncoder().encode('ho'));
|
||||||
|
await w.write(new TextEncoder().encode('la'));
|
||||||
|
await w.close();
|
||||||
|
await sink.commit();
|
||||||
|
expect([new TextDecoder().decode(one.bytes()), sink.mode, sink.ranges, sink.length, one.state.positioned]).toEqual(['hola', 'file', [[0, 4]], 4, 0]);
|
||||||
|
const none = memoryFile();
|
||||||
|
const empty = new ZipSink(none.stream, 0);
|
||||||
|
empty.begin(head());
|
||||||
|
await empty.commit();
|
||||||
|
expect([none.bytes().length, none.state.closed, empty.mode, empty.ranges, empty.length]).toEqual([0, true, 'none', [], 0]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('aborts its file, whether it began or not', async () => {
|
||||||
|
const early = memoryFile();
|
||||||
|
await new ZipSink(early.stream, 0).abort(new Error('failed at step 9'));
|
||||||
|
expect(early.state.aborted).toEqual(new Error('failed at step 9'));
|
||||||
|
const late = memoryFile();
|
||||||
|
const sink = new ZipSink(late.stream, 0);
|
||||||
|
sink.begin(head(file('a/b', 2)));
|
||||||
|
await sink.abort(new Error('failed at step 17'));
|
||||||
|
expect([late.state.closed, late.state.aborted]).toEqual([false, new Error('failed at step 17')]);
|
||||||
|
// A file that fails to abort does not make the abort fail.
|
||||||
|
const stubborn = new ZipSink(new WritableStream({ abort: () => Promise.reject(new Error('cannot abort')) }), 0);
|
||||||
|
await expect(stubborn.abort(new Error('failed'))).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('ZipSink behind open', () => {
|
||||||
|
it('receives the files of format3_tree in a ZIP closed only after step 18', async () => {
|
||||||
|
const { r, out, sink, rec } = await openInto('format3_tree');
|
||||||
|
expect([r.error, out.state.closed, sink.mode]).toEqual([undefined, true, 'zip']);
|
||||||
|
const bytes = out.bytes();
|
||||||
|
expect(bytes.length).toBe(sink.length);
|
||||||
|
const files = rec.files!;
|
||||||
|
for (const [i, [start, end]] of sink.ranges.entries()) {
|
||||||
|
expect(toHex(await sha256(bytes.subarray(start, end))), files[i]!.path).toBe(files[i]!.sha256);
|
||||||
|
}
|
||||||
|
const contents = sink.ranges.map(([s, e]) => bytes.slice(s, e));
|
||||||
|
expect(hx(bytes)).toBe(hx(new Uint8Array(await zipOf(r.head!, contents, parseRFC3339(readJSON<Record>('fixtures/format3_tree.json').unlock_at).seconds).arrayBuffer())));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('receives the one file of format3_single as it is, and aborts the file when the capsule is cut short', async () => {
|
||||||
|
const { r, out, sink, rec } = await openInto('format3_single');
|
||||||
|
expect([r.error, sink.mode, toHex(await sha256(out.bytes()))]).toEqual([undefined, 'file', rec.files![0]!.sha256]);
|
||||||
|
const cut = readBytes('fixtures/format3_tree.dkc');
|
||||||
|
const bad = await openInto('format3_tree', cut.subarray(0, -1));
|
||||||
|
expect([bad.r.error?.code, bad.out.state.closed, bad.out.state.aborted === undefined]).toEqual(['ERR_INTEGRITY', false, false]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// scripts/zip-ts-samples.mjs wrote the samples of testing/zip.ts, and
|
||||||
|
// scripts/zip-go-read.go read them with archive/zip: the same bytes must give
|
||||||
|
// the entries they were written with.
|
||||||
|
describe('the ZIP samples, read by archive/zip of Go (zip-vectors.json)', () => {
|
||||||
|
const V = JSON.parse(readFileSync(new URL('../dkc/testing/zip-vectors.json', import.meta.url), 'utf8')) as {
|
||||||
|
samples: { name: string; sha256: string; count: number; lines_sha256: string; lines?: string[]; error?: string }[];
|
||||||
|
};
|
||||||
|
const samples = zipSamples();
|
||||||
|
|
||||||
|
it('holds every sample, in order', () => {
|
||||||
|
expect(V.samples.map((s) => s.name)).toEqual(samples.map((s) => s.name));
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(samples.map((s, i) => [s.name, s, i] as const))('%s', async (_, sample, i) => {
|
||||||
|
const go = V.samples[i]!;
|
||||||
|
const { bytes } = await writeZip(sample);
|
||||||
|
expect(toHex(await sha256(bytes))).toBe(go.sha256);
|
||||||
|
// Each entry as Go gives it: name, UTF-8, stored, CRC-32, sizes, time and SHA-256.
|
||||||
|
const empty = toHex(await sha256(new Uint8Array(0)));
|
||||||
|
const lines: string[] = [];
|
||||||
|
for (const [k, e] of zipEntries(sample.head, sample.roundTime).entries()) {
|
||||||
|
const content = sample.files[k]!;
|
||||||
|
const time = new Date(e.mtime * 1000).toISOString().replace('.000Z', 'Z');
|
||||||
|
const sum = content.length === 0 ? empty : toHex(await sha256(content));
|
||||||
|
lines.push([e.name, 'true', '0', crc32(content).toString(16).padStart(8, '0'), e.size, e.size, time, sum].join('\t'));
|
||||||
|
}
|
||||||
|
expect([go.error, go.count]).toEqual([undefined, lines.length]);
|
||||||
|
expect(toHex(await sha256(new TextEncoder().encode(lines.join('\n'))))).toBe(go.lines_sha256);
|
||||||
|
if (go.lines !== undefined) expect(go.lines).toEqual(lines);
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -0,0 +1,135 @@
|
|||||||
|
// The sink of the page for the files of a format 3 capsule (design of format
|
||||||
|
// 3, section 5), and the same ZIP made in memory. The files go to one
|
||||||
|
// temporary file of OPFS: the file itself when the capsule holds one file of
|
||||||
|
// one segment, and otherwise a ZIP of stored entries that zip.ts lays out
|
||||||
|
// from the head before any byte arrives. Each file is written as open
|
||||||
|
// delivers it, after the local header of its entry; the CRC-32 of the entry
|
||||||
|
// is patched in that header with a positioned write once its bytes are
|
||||||
|
// known, and the central directory is written by commit, which closes the
|
||||||
|
// file. The writable of OPFS keeps the writes in its swap file until then,
|
||||||
|
// and abort discards them, so nothing is shown before step 18 (spec §56).
|
||||||
|
// Each file is then a contiguous range of the temporary file, which the page
|
||||||
|
// offers as a download of its own.
|
||||||
|
|
||||||
|
import type { Head } from '../dkc/head.ts';
|
||||||
|
import type { Sink } from '../dkc/sink.ts';
|
||||||
|
import { Crc32, crc32 } from './crc32.ts';
|
||||||
|
import type { TempChunk } from './tempfile.ts';
|
||||||
|
import { type ZipEntry, zipLayout, type ZipLayout } from './zip.ts';
|
||||||
|
|
||||||
|
/** How the files of a capsule go into one file: none, the one file itself, or a ZIP. */
|
||||||
|
export type ZipMode = 'none' | 'file' | 'zip';
|
||||||
|
|
||||||
|
/** The file itself when the head holds one file of one segment, a ZIP when it holds more or one inside a folder, and none without files. */
|
||||||
|
export function zipMode(head: Head): ZipMode {
|
||||||
|
if (head.files.length === 0) return 'none';
|
||||||
|
return head.files.length === 1 && !head.files[0]!.path.includes('/') ? 'file' : 'zip';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The entries of the ZIP of a head: each file with its path, its size and
|
||||||
|
* its mtime, or the time of the round, in seconds, when it has none.
|
||||||
|
*/
|
||||||
|
export function zipEntries(head: Head, roundTime: number): ZipEntry[] {
|
||||||
|
return head.files.map((f) => ({ name: f.path, size: f.size, mtime: f.mtime ?? roundTime }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A sink that writes the files of a capsule into one writable of OPFS, the
|
||||||
|
* file itself or a ZIP (see zipMode). It closes the writable at commit and
|
||||||
|
* aborts it at abort, which the caller may also call when the opening failed
|
||||||
|
* before begin.
|
||||||
|
*/
|
||||||
|
export class ZipSink implements Sink {
|
||||||
|
readonly #out: WritableStream<TempChunk>;
|
||||||
|
readonly #roundTime: number;
|
||||||
|
#writer: WritableStreamDefaultWriter<TempChunk> | undefined;
|
||||||
|
#mode: ZipMode = 'none';
|
||||||
|
#layout: ZipLayout | undefined;
|
||||||
|
#crcs: number[] = [];
|
||||||
|
#ranges: (readonly [number, number])[] = [];
|
||||||
|
|
||||||
|
/** `roundTime` is the time of the round of the capsule, in seconds, for the entries without an mtime. */
|
||||||
|
constructor(out: WritableStream<TempChunk>, roundTime: number) {
|
||||||
|
this.#out = out;
|
||||||
|
this.#roundTime = roundTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
begin(head: Head): void {
|
||||||
|
this.#writer = this.#out.getWriter();
|
||||||
|
this.#mode = zipMode(head);
|
||||||
|
if (this.#mode === 'zip') {
|
||||||
|
this.#layout = zipLayout(zipEntries(head, this.#roundTime));
|
||||||
|
this.#ranges = this.#layout.entries.map((e) => [e.dataOffset, e.dataOffset + e.size] as const);
|
||||||
|
} else if (this.#mode === 'file') {
|
||||||
|
this.#ranges = [[0, head.files[0]!.size]];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
create(i: number): WritableStream<Uint8Array> {
|
||||||
|
const w = this.#writer!;
|
||||||
|
const entry = this.#layout?.entries[i];
|
||||||
|
const sum = new Crc32();
|
||||||
|
return new WritableStream<Uint8Array>({
|
||||||
|
start: async () => {
|
||||||
|
// The local header of the entry, at its offset: the patch of the
|
||||||
|
// CRC-32 of the entry before moved the position of the file.
|
||||||
|
if (entry !== undefined) await w.write({ type: 'write', position: entry.headerOffset, data: entry.header });
|
||||||
|
},
|
||||||
|
write: async (chunk) => {
|
||||||
|
sum.update(chunk);
|
||||||
|
await w.write(chunk);
|
||||||
|
},
|
||||||
|
close: async () => {
|
||||||
|
if (entry === undefined) return;
|
||||||
|
const crc = new Uint8Array(4);
|
||||||
|
new DataView(crc.buffer).setUint32(0, sum.value, true);
|
||||||
|
this.#crcs[i] = sum.value;
|
||||||
|
await w.write({ type: 'write', position: entry.crcOffset, data: crc });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async commit(): Promise<void> {
|
||||||
|
const w = this.#writer!;
|
||||||
|
if (this.#layout !== undefined) await w.write({ type: 'write', position: this.#layout.centralOffset, data: this.#layout.central(this.#crcs) });
|
||||||
|
await w.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
async abort(reason: unknown): Promise<void> {
|
||||||
|
await (this.#writer ?? this.#out).abort(reason).catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** How the files were laid out, once begun. */
|
||||||
|
get mode(): ZipMode {
|
||||||
|
return this.#mode;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The range of each file in the file written, once begun: [start, end), in the order of the head. */
|
||||||
|
get ranges(): readonly (readonly [number, number])[] {
|
||||||
|
return this.#ranges;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The length of the file written, once begun. */
|
||||||
|
get length(): number {
|
||||||
|
return this.#layout?.length ?? this.#ranges[0]?.[1] ?? 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The ZIP of the files of a head held in memory, the same bytes a ZipSink
|
||||||
|
* writes, as a Blob of its parts: the local headers with their CRC-32, the
|
||||||
|
* files themselves, not copied, and the central directory.
|
||||||
|
*/
|
||||||
|
export function zipOf(head: Head, files: readonly Uint8Array[], roundTime: number): Blob {
|
||||||
|
const layout = zipLayout(zipEntries(head, roundTime));
|
||||||
|
const crcs = files.map(crc32);
|
||||||
|
const parts: Uint8Array[] = [];
|
||||||
|
layout.entries.forEach((e, i) => {
|
||||||
|
const header = e.header.slice();
|
||||||
|
new DataView(header.buffer).setUint32(e.crcOffset - e.headerOffset, crcs[i]!, true);
|
||||||
|
parts.push(header, files[i]!);
|
||||||
|
});
|
||||||
|
parts.push(layout.central(crcs));
|
||||||
|
return new Blob(parts as Uint8Array<ArrayBuffer>[]);
|
||||||
|
}
|
||||||
Loading…
Reference in new issue