Format 3, step 8: fixes of the adversarial review of the library

Reader:
- PAYLOAD_AGE reaches age in pieces of one STREAM chunk, read only when
  asked (agefile.chunked): a whole payload was decrypted ahead of step
  17, with more memory, and a later STREAM failure hid the failure of
  the reader that Go reports. The text still differs from Go when the
  capsule is cut right after a full chunk: age-encryption holds that
  chunk until it sees one more byte.
- The unusable head extensions are found before the commit, so the
  sink is never aborted after it; a registry whose validateData throws
  rejects the data; getWriter of a created file is inside the failure
  of the sink; the sink gets copies, which it may keep or transfer.

Writer:
- Each piece of a file is copied as it is read: a source that reuses
  its buffer could make the head record a SHA-256 that is not that of
  the bytes encrypted, and the capsule would not open.
- A FileSource written as a class works: open is called on its object,
  and each property is read once. Extensions of the wrong types are a
  TypeError. In memory, the limit is checked before reading anything.
  A chunk that is not a Uint8Array, and a lone surrogate in a path, are
  named.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
main
dev 7 days ago
parent 7527c35208
commit 57e0a8a6e3

@ -0,0 +1,46 @@
// Tests of agefile.ts: chunked, which hands age an age file in pieces of at
// most one STREAM chunk, only as they are asked for.
import { describe, expect, it } from 'vitest';
import { chunked } from './agefile.ts';
const STORED = 65536 + 16;
async function pieces(s: ReadableStream<Uint8Array>): Promise<number[]> {
const out: number[] = [];
const reader = s.getReader();
for (let r = await reader.read(); !r.done; r = await reader.read()) out.push(r.value.length);
return out;
}
describe('chunked', () => {
it('cuts a Uint8Array and a stream in pieces of at most one stored chunk', async () => {
expect(await pieces(chunked(new Uint8Array(2 * STORED + 5)))).toEqual([STORED, STORED, 5]);
expect(await pieces(chunked(new Uint8Array(0)))).toEqual([]);
const whole = new Blob([new Uint8Array(STORED + 1)]).stream();
expect(await pieces(chunked(whole))).toEqual([STORED, 1]);
});
it('reads the source only when asked, and cancels it', async () => {
let pulls = 0;
let cancelled: unknown;
const source = new ReadableStream<Uint8Array>(
{
pull(c) {
pulls++;
c.enqueue(new Uint8Array(10));
},
cancel(reason) {
cancelled = reason;
},
},
{ highWaterMark: 0 },
);
const reader = chunked(source).getReader();
expect(pulls).toBe(0);
expect((await reader.read()).value?.length).toBe(10);
expect(pulls).toBe(1);
await reader.cancel('stop');
expect(cancelled).toBe('stop');
});
});

@ -23,6 +23,51 @@ export const streamOf = (b: Uint8Array): ReadableStream<Uint8Array> =>
},
});
// A chunk of an age STREAM as it is stored: 64 KiB of plaintext and its tag.
const STORED_CHUNK = 65536 + 16;
/**
* An age file, in memory or as a stream, in pieces of at most one STREAM
* chunk, each read only when the decryption asks for it. age-encryption
* decrypts every chunk of a piece it is given at once, so a larger piece, a
* whole Uint8Array or the one chunk that Blob.stream() gives in Node, would
* be decrypted ahead of the reader, all of it before step 17 checks
* anything: more memory, and a failure of a later chunk that hides the
* failure of the reader, which Go reports.
*/
export function chunked(file: Uint8Array | ReadableStream<Uint8Array>): ReadableStream<Uint8Array> {
let at = 0;
let pending: Uint8Array = new Uint8Array(0);
const reader = file instanceof Uint8Array ? undefined : file.getReader();
return new ReadableStream<Uint8Array>(
{
async pull(c) {
if (reader === undefined) {
const b = file as Uint8Array;
if (at >= b.length) c.close();
else {
c.enqueue(b.subarray(at, at + STORED_CHUNK));
at += STORED_CHUNK;
}
return;
}
while (pending.length === 0) {
const r = await reader.read();
if (r.done) {
c.close();
return;
}
pending = r.value;
}
c.enqueue(pending.subarray(0, STORED_CHUNK));
pending = pending.subarray(Math.min(STORED_CHUNK, pending.length));
},
cancel: (reason) => reader?.cancel(reason),
},
{ highWaterMark: 0 },
);
}
/**
* The error of `what` for a failure of age: the normative error an identity
* reported, prefixed, or ERR_INTEGRITY with `reason`.

@ -13,7 +13,7 @@ import { compareBytes, concatBytes, sha256, utf8Bytes } from './bytes.ts';
import { type Instant, parseRFC3339 } from './datekey.ts';
import { encryptFiles, type EncryptOptions, type FileSource, fileSource } from './encrypt.ts';
import { errorCode } from './errors.ts';
import { ExtensionSet } from './extension.ts';
import { type Extension, type ExtensionRegistry, ExtensionSet } from './extension.ts';
import { FORMAT_3 } from './framing.ts';
import { decodeHead, encodeHead, type Head, type HeadFile } from './head.ts';
import { TIME_AND_KEY, TIME_ONLY } from './header.ts';
@ -22,7 +22,7 @@ import { accessIdentity, open, type OpenOptions, payloadIdentity, timeIdentity }
import { BLOQUE256, MAX_PAYLOAD_LENGTH, type Padding, REFORZADO } from './padding.ts';
import { quicknet } from './profile.ts';
import { type Release, suppliedRelease } from './release.ts';
import { MemorySink } from './sink.ts';
import { MemorySink, type Sink } from './sink.ts';
import { split } from './testing/capsule.ts';
import { encryptFilesWith, wordsFor } from './testing/encrypt.ts';
import { h, hx, readBytes, readJSON } from './testing/testdata.ts';
@ -487,3 +487,162 @@ describe('encryptFiles, invalid inputs', () => {
expect((await failure(encryptFiles([a], options({ padding: REFORZADO, unlockAt: GENESIS })))).message).toMatch(/is not in the future$/);
});
});
describe('encryptFiles and open, after the adversarial review of format 3', () => {
const BS = String.fromCharCode(92);
const pause = (ms: number) => new Promise((r) => setTimeout(r, ms));
const sinkOf = (create: (i: number) => WritableStream<Uint8Array>, log: string[] = []): Sink => ({
begin: () => void log.push('begin'),
create,
commit: () => void log.push('commit'),
abort: () => void log.push('abort'),
});
it('hashes the bytes it encrypts, even from a source that reuses its buffer, into a slow output', async () => {
const content = Uint8Array.from({ length: 6 * 65536 }, (_, i) => (i * 13 + 5) & 0xff);
const reusing: FileSource = {
path: 'a.bin',
size: content.length,
open: () => {
const buf = new Uint8Array(65536);
let at = 0;
return new ReadableStream<Uint8Array>(
{
async pull(c) {
await pause(1);
if (at >= content.length) return c.close();
buf.set(content.subarray(at, at + 65536));
at += 65536;
c.enqueue(buf);
},
},
{ highWaterMark: 0 },
);
},
};
const chunks: Uint8Array[] = [];
const output = new WritableStream<Uint8Array>({
write: async (c) => {
chunks.push(c.slice());
await pause(5);
},
});
await encryptFiles([reusing], options({ output }));
expect((await openFiles(concatBytes(...chunks), R1000)).files.get('a.bin')).toBe(hx(content));
});
it('calls the open method of a FileSource written as a class on its object, and reads each property once', async () => {
class Held implements FileSource {
readonly #blob: Blob;
readonly path = 'held.txt';
reads = 0;
constructor(text: string) {
this.#blob = new Blob([te.encode(text)]);
}
get size(): number {
this.reads++;
return this.#blob.size;
}
open(): ReadableStream<Uint8Array> {
return this.#blob.stream();
}
}
const held = new Held('hola');
const res = await encryptFiles([held], options());
expect(held.reads).toBe(1);
expect((await openFiles(res.dkc!, R1000)).files.get('held.txt')).toBe(hx(te.encode('hola')));
});
it('refuses extensions of the wrong types before using them', async () => {
const bad: [unknown, string][] = [
['x', 'encrypt: a list of extensions is not an array'],
[[null], 'encrypt: an extension is not an Extension'],
[[{ id: 1, version: 1 }], 'encrypt: an extension_id is not a string'],
[[{ id: 'x.example', version: '1' }], 'encrypt: extension "x.example": extension_version is not a number'],
[[{ id: 'x.example', version: 1, data: [1, 300] }], 'encrypt: extension "x.example": data is not a Uint8Array'],
];
for (const [list, text] of bad) {
const err = await failure(encryptFiles([source('a', 'x')], options({ headNoncritical: list as Extension[] })));
expect([err instanceof TypeError, err.message]).toEqual([true, text]);
}
});
it('refuses in memory, before reading anything, a capsule over the limit', async () => {
let opened = 0;
const huge: FileSource = {
path: 'big.bin',
size: 2 ** 30,
open: () => {
opened++;
throw new Error('read');
},
};
const err = await failure(encryptFiles([huge], options()));
expect([err instanceof TypeError, opened]).toEqual([true, 0]);
expect(err.message).toMatch(/^encrypt: a capsule of \d+ bytes needs EncryptOptions\.output; in memory the limit is 1073741824$/);
});
it('names a chunk that is not a Uint8Array, and a lone surrogate of a path', async () => {
const odd: FileSource = {
path: 'a.txt',
size: 1,
open: () =>
new ReadableStream<Uint8Array>({
start(c) {
c.enqueue('x' as unknown as Uint8Array);
c.close();
},
}),
};
expect((await failure(encryptFiles([odd], options()))).message).toBe('capsule: file "a.txt": the stream gave a chunk that is not a Uint8Array');
const lone = `a${String.fromCharCode(0xd800)}b`;
expect((await failure(encryptFiles([source(lone, 'x')], options()))).message).toBe(`capsule: path "a${BS}ud800b": R1: not valid UTF-8`);
});
it('reports a sink that gives a locked stream as a failure of the sink, and aborts it', async () => {
const res = await encryptFiles([source('a.txt', 'hola')], options());
const locked = new WritableStream<Uint8Array>();
locked.getWriter();
const log: string[] = [];
const r = await open(res.dkc!, opening(R1000, { sink: sinkOf(() => locked, log) }));
expect(r.error?.message).toMatch(/^capsule: PAYLOAD_AGE: creating file 1: .*: ERR_INTEGRITY$/);
expect(log).toEqual(['begin', 'abort']);
});
it('gives the sink copies, which it may keep or transfer', async () => {
const res = await encryptFiles([source('a.bin', new Uint8Array(100_000).fill(7)), source('b.bin', 'dos')], options());
const sizes: number[] = [];
const sink = sinkOf(
() =>
new WritableStream<Uint8Array>({
write: (c) => {
sizes.push(c.length);
structuredClone(c.buffer, { transfer: [c.buffer as ArrayBuffer] });
},
}),
);
const r = await open(res.dkc!, opening(R1000, { sink }));
expect([r.error, sizes.reduce((a, b) => a + b, 0)]).toEqual([undefined, 100_003]);
});
it('treats a registry that throws as one that rejects the data, and never aborts the sink once it committed', async () => {
const throwing = (value: unknown): ExtensionRegistry => ({
known: (id) => id === 'x.example',
validateData: () => {
throw value;
},
});
const noted = await encryptFiles([source('a', 'x')], options({ headNoncritical: [{ id: 'x.example', version: 1, data: Uint8Array.of(1) }] }));
const log: string[] = [];
const sink = sinkOf(() => new WritableStream<Uint8Array>(), log);
const ok = await open(noted.dkc!, opening(R1000, { sink, extensions: throwing(new Error('malformed note')) }));
expect([ok.error, log, ok.unusableHeadExtensions.map((u) => u.error.message)]).toEqual([
undefined,
['begin', 'commit'],
['extension x.example v1: data: malformed note: ERR_EXTENSION_DATA_INVALID'],
]);
const critical = await encryptFiles([source('a', 'x')], options({ headCritical: [{ id: 'x.example', version: 1, data: Uint8Array.of(1) }] }));
const bad = await open(critical.dkc!, opening(R1000, { sink: sinkOf(() => new WritableStream<Uint8Array>()), extensions: throwing('bad') }));
expect([errorCode(bad.error), bad.error?.message]).toEqual(['ERR_EXTENSION_DATA_INVALID', expect.stringMatching(/data: bad: ERR_EXTENSION_DATA_INVALID$/)]);
});
});

@ -340,7 +340,13 @@ export function checkNoncritical(noncritical: readonly Extension[], reg?: Extens
}
function validateData(e: Extension, reg: ExtensionRegistry): DateKeysError | undefined {
const err = reg.validateData?.(e);
// A registry that throws rejects the data, as one that returns an Error.
let err: Error | undefined;
try {
err = reg.validateData?.(e);
} catch (thrown) {
err = thrown instanceof Error ? thrown : new Error(String(thrown));
}
if (err === undefined) return undefined;
return new DateKeysError('ERR_EXTENSION_DATA_INVALID', `extension ${e.id} v${e.version}: data: ${err.message}`);
}

@ -33,7 +33,7 @@
import { type Identity, type Stanza as AgeStanza } from 'age-encryption';
import { type AccessKey, checkAccessKeyMaterial, decodeAccessKey, wipeAccessKey } from './accesskey.ts';
import { ACCESS_SLOTS, ageStanzas, checkAccessStanzas, checkPayloadStanzas, checkTimeStanzas, type Stanza } from './age.ts';
import { classify, decrypt, decryptAll, STREAM_FAILURE, streamOf } from './agefile.ts';
import { chunked, classify, decrypt, decryptAll, STREAM_FAILURE } from './agefile.ts';
import { equalBytes, sha256, toHex } from './bytes.ts';
import { type Control, decodeControl } from './control.ts';
import { compareInstants, formatRFC3339, type Instant } from './datekey.ts';
@ -397,13 +397,14 @@ async function openCapsule(
}
// Step 17: PAYLOAD_AGE with I_PAYLOAD, streamed from the Blob or from
// memory, to the output or to memory. In format 2 the plaintext is
// memory in pieces of one STREAM chunk (chunked), to the output or to
// memory. In format 2 the plaintext is
// exactly P bytes and its bytes L to P-1 are zero; only the first L are
// delivered, and the padding is read and checked, never delivered. In
// format 3 the files of BODY go to the sink (open3.ts).
const offset = inspection.payloadOffset!;
const [payload, ciphertextLength] =
dkc instanceof Uint8Array ? [streamOf(sections.payload), sections.payload.length] : [dkc.slice(offset).stream(), dkc.size - offset];
dkc instanceof Uint8Array ? [chunked(sections.payload), sections.payload.length] : [chunked(dkc.slice(offset).stream()), dkc.size - offset];
let plaintext: Uint8Array | undefined;
let body: OpenedBody | undefined;
let n = 0;

@ -132,12 +132,14 @@ class Plaintext {
const piece = await this.next(left);
if (piece === undefined) throw this.short();
sum.update(piece);
left -= piece.length;
// The sink gets a copy: it may keep or transfer what it receives, and
// the rest of the chunk is still to be read.
try {
await w.write(piece);
await w.write(piece.slice());
} catch (err) {
throw new SinkError(err);
}
left -= piece.length;
}
return sum.digest();
}
@ -212,13 +214,12 @@ export async function openBody(
}
begun = true;
for (const [i, f] of head.files.entries()) {
let file: WritableStream<Uint8Array>;
let w: WritableStreamDefaultWriter<Uint8Array>;
try {
file = await sink.create(i);
w = (await sink.create(i)).getWriter();
} catch (err) {
throw sinkFailure(`creating file ${i + 1}`, err);
}
const w = file.getWriter();
let sum: Uint8Array;
try {
sum = await r.copy(w, f.size);
@ -237,13 +238,16 @@ export async function openBody(
// 17.8: the padding, up to P, and nothing after it.
await r.padding();
// Step 18.
// Step 18, once everything that could fail has run: the unusable
// noncritical extensions never fail the opening, and after the commit
// nothing may abort the sink.
const unusableHeadExtensions = checkNoncritical(head.noncritical, reg, 'head');
try {
await sink.commit();
} catch (err) {
throw sinkFailure('committing the files', err);
}
return { head, verdicts, areaLen: frame.areaLen, unusableHeadExtensions: checkNoncritical(head.noncritical, reg, 'head') };
return { head, verdicts, areaLen: frame.areaLen, unusableHeadExtensions };
} catch (err) {
await r.cancel(err);
if (begun) {

@ -29,7 +29,7 @@ import { DKC_PRELUDE_SIZE, FORMAT_2, FORMAT_3, headerBinding, MAX_SEALED_CONTROL
import { checkHeadEnd, decodeWrittenHead, encodeHead, type Head, type HeadFile, MAX_FILES, SALT_SIZE } from './head.ts';
import { decodeHeader, encodeHeader, type Policy, TIME_AND_KEY, TIME_ONLY } from './header.ts';
import { accessIdentity, payloadIdentity } from './open.ts';
import { headComment, mtimeSeconds, sealedControlLength } from './lengths.ts';
import { capsuleLength, headComment, mtimeSeconds, sealedControlLength } from './lengths.ts';
import { isPadding, MAX_PAYLOAD_LENGTH, paddedLength, type Padding, payloadAgeLength, REFORZADO } from './padding.ts';
import { checkAuthor, checkComment, checkPath, checkTree, MAX_AUTHOR_LEN, MAX_COMMENT_LEN, MAX_PATH_LEN, PathRuleError } from './pathrule.ts';
import { cloneProfile, type Profile, validateProfile } from './profile.ts';
@ -211,6 +211,21 @@ export async function writeFiles(files: readonly FileSource[], opts: EncryptOpti
const content = head.files.at(-1)?.end ?? 0;
const length = BODY_FRAME_SIZE + AREA_LEN + measured.length + content;
checkLength(length, s.code);
// In memory, the limit is known before reading anything.
if (opts.output === undefined) {
const total = capsuleLength({
profileId: s.dateKey.profileId,
round: s.dateKey.round,
policy: s.policy,
length,
padding: s.code,
critical: s.critical,
noncritical: s.noncritical,
controlCritical: s.controlCritical,
controlNoncritical: s.controlNoncritical,
});
if (total > MAX_MEMORY_DKC) throw new TypeError(`encrypt: a capsule of ${total} bytes needs EncryptOptions.output; in memory the limit is ${MAX_MEMORY_DKC}`);
}
// Step 3: the first reading, for the SHA-256 of each file.
const sums: Uint8Array[] = [];
@ -246,10 +261,13 @@ function copySources(files: readonly FileSource[]): FileSource[] {
if (!Array.isArray(files)) throw new TypeError('encrypt: the files are an array of FileSource');
return files.map((f: FileSource, i) => {
if (typeof f !== 'object' || f === null) throw new TypeError(`encrypt: file ${i} is not a FileSource`);
if (typeof f.path !== 'string') throw new TypeError(`encrypt: file ${i}: path is not a string`);
if (!Number.isSafeInteger(f.size)) throw new TypeError(`encrypt: file ${i}: size is not a safe integer`);
if (f.mtime !== undefined && !Number.isFinite(f.mtime)) throw new TypeError(`encrypt: file ${i}: mtime is not a finite number of milliseconds`);
return { path: f.path, size: f.size, ...(f.mtime === undefined ? {} : { mtime: f.mtime }), open: f.open };
// Each property read once, so that a getter cannot pass a check and
// then give another value; open called on its object, as a method.
const { path, size, mtime, open } = f;
if (typeof path !== 'string') throw new TypeError(`encrypt: file ${i}: path is not a string`);
if (!Number.isSafeInteger(size)) throw new TypeError(`encrypt: file ${i}: size is not a safe integer`);
if (mtime !== undefined && !Number.isFinite(mtime)) throw new TypeError(`encrypt: file ${i}: mtime is not a finite number of milliseconds`);
return { path, size, ...(mtime === undefined ? {} : { mtime }), open: typeof open === 'function' ? () => open.call(f) : open };
});
}
@ -285,8 +303,8 @@ function newHead(
for (const [i, j] of order.entries()) {
const src = sources[j]!;
const p = src.path;
if (i > 0 && p === paths[i - 1]) throw new Error(`capsule: path ${goQuote(p)} given twice`);
if (!p.isWellFormed()) throw new Error(`capsule: path ${goQuote(p)}: R1: not valid UTF-8`);
if (i > 0 && p === paths[i - 1]) throw new Error(`capsule: path ${quotePath(p)} given twice`);
if (!p.isWellFormed()) throw new Error(`capsule: path ${quotePath(p)}: R1: not valid UTF-8`);
const n = utf8Length(p);
if (n === 0 || n > MAX_PATH_LEN) throw new Error(`capsule: path ${goQuote(p)}: R1: ${n} bytes, not 1 to ${MAX_PATH_LEN}`);
if (src.size < 0) throw new Error(`capsule: file ${goQuote(p)}: negative size ${src.size}`);
@ -312,6 +330,24 @@ function newHead(
return { head: { salt: new Uint8Array(SALT_SIZE), comment, author, files, critical, noncritical }, order };
}
// A path quoted as goQuote does, with each lone surrogate, which UTF-8
// cannot hold and goQuote would show as U+FFFD, written as \uXXXX: the
// message names the character (spec §62.1 rule 15).
function quotePath(p: string): string {
if (p.isWellFormed()) return goQuote(p);
let out = '';
let run = '';
for (const ch of p) {
const r = ch.codePointAt(0)!;
if (r < 0xd800 || r > 0xdfff) run += ch;
else {
out += `${goQuote(run).slice(1, -1)}\\u${r.toString(16).padStart(4, '0')}`;
run = '';
}
}
return `"${out}${goQuote(run).slice(1, -1)}"`;
}
// A violation of a rule of the paths or the texts, after prefix; anything
// but a PathRuleError is a bug and propagates.
function pathFailure(err: unknown, prefix: string): Error {
@ -654,8 +690,19 @@ function sourceLength(src: EncryptSource, length: number | undefined): number {
throw new TypeError('encrypt: the source is a Uint8Array, a Blob or a ReadableStream');
}
const copyExtensions = (list: readonly Extension[] | undefined): Extension[] =>
(list ?? []).map((e) => ({ id: e.id, version: e.version, data: e.data === undefined ? undefined : copyBytes(e.data) }));
// Copies of extensions given by the caller, which must be of the types of
// Extension: new Uint8Array() would turn a number or a string into bytes.
function copyExtensions(list: readonly Extension[] | undefined): Extension[] {
if (list !== undefined && !Array.isArray(list)) throw new TypeError('encrypt: a list of extensions is not an array');
return (list ?? []).map((e: Extension) => {
if (typeof e !== 'object' || e === null) throw new TypeError('encrypt: an extension is not an Extension');
const { id, version, data } = e;
if (typeof id !== 'string') throw new TypeError('encrypt: an extension_id is not a string');
if (typeof version !== 'number') throw new TypeError(`encrypt: extension ${goQuote(id)}: extension_version is not a number`);
if (data !== undefined && !(data instanceof Uint8Array)) throw new TypeError(`encrypt: extension ${goQuote(id)}: data is not a Uint8Array`);
return { id, version, data: data === undefined ? undefined : copyBytes(data) };
});
}
// A random value of the length it must have.
function drawn(b: Uint8Array, n: number, what: string): Uint8Array {
@ -924,8 +971,11 @@ class FileReading {
if (this.#n !== this.#size) throw this.#mismatch(`${this.#n} bytes, not its size of ${this.#size}`);
return undefined;
}
if (!(r.value instanceof Uint8Array)) throw this.#failure(new TypeError('the stream gave a chunk that is not a Uint8Array'));
if (r.value.length > this.#size - this.#n) throw this.#mismatch(`more than its size of ${this.#size} bytes`);
this.#pending = r.value;
// A copy: a source may reuse its buffer once it has handed it over,
// and the bytes hashed must be the bytes that age encrypts.
this.#pending = r.value.slice();
}
const piece = this.#pending.subarray(0, CHUNK);
this.#pending = this.#pending.subarray(piece.length);

Loading…
Cancel
Save

Powered by TurnKey Linux.