From c531e936fa8188a4aab01575607cc85d4f5d4fa4 Mon Sep 17 00:00:00 2001 From: dev Date: Mon, 5 Oct 2026 18:08:12 +0200 Subject: [PATCH] The table generator writes a Dart library for datekeys-dart internal/pathrule/gen -dart renders the Unicode and best-fit tables as const lists of a Dart library, as -ts does for datekeys-ts: the same data and the same TablesDigest, with the formatter turned off for the file. tables.go and the TypeScript module come out unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++ docs/traceability.md | 2 +- internal/pathrule/gen/dart.go | 108 ++++++++++++++++++++++++++++++++++ internal/pathrule/gen/main.go | 12 ++++ 4 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 internal/pathrule/gen/dart.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fc4c355..5b9f332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,10 @@ reader of v0.11 opens these capsules, and this one opens those of v0.11. that fits, each row after the first behind ` ↳ `, so that the terminal never breaks them. `encrypt` reports L as the length of the payload, not of the content. +- **Tables for Dart.** `internal/pathrule/gen -dart` writes the Unicode and + best-fit tables as a Dart library for `datekeys-dart`, the third + implementation, as `-ts` does for `datekeys-ts`: the same lists and the same + `TablesDigest`. The tables and the TypeScript module do not change. - **Test data.** - `genfixtures -force` writes the fixtures of v0.10 again with their area of 512 bytes, through `EncryptOptions.TestAreaLen`, which needs diff --git a/docs/traceability.md b/docs/traceability.md index 3851e44..29a5dbf 100644 --- a/docs/traceability.md +++ b/docs/traceability.md @@ -49,7 +49,7 @@ the same. A case of §64 that is not in the repository yet is marked | 29.3 | `security`: the outer map of version 1, keys 2 and 3 byte strings that hold the author signature and the seal encoded apart; layers 2 and 3 without a code; it never decides the opening, and a failure of the signature or of the seal, a panic of a parser included, changes only its own verdict; `alg` 1 and 2 and `seal_type` 2 defined, `seal_type` 1 and 3 reserved, `alg` and `seal_type` 4294967295 reserved for tests; empty, 22 bytes, without a signature or a seal; no key 3 beside `alg` 2 | `capsule.EncodeSecurity`, `EncodeSecurityWith`, `EncodeAuthorSignature`, `EncodeSeal`, `DecodeAuthorSignature`, `SecurityKey2`, `SecurityKey3`; `EvaluateSecurityIn` (`recovered`, `setSeal`) in a `SecurityContext`, and `EvaluateSecurity`, without one, as a reader of v0.10; `AlgEd25519`, `AlgCMS`, `SealTypeRFC3161`, `AlgTest`, `SealTypeTest` | `capsule.TestSecurityVerdicts`, `TestEvaluateSecurityIn`, `FuzzEvaluateSecurity` (without a context); `testdata/vectors/security.json`, in the context of a capsule, with the lines of each case (`internal/testkit.SecurityVectors`, `SecurityResultIn`, `TestFormat3VectorFiles`, `TestVectorFilesAreCurrent`); fixtures `format3_security_v2`, `format3_signature_unsupported` and `format3_seal_unsupported` (`alg` and `seal_type` 4294967295); the four mutations of the list of format 3 of §64 that open with their verdicts; no test yet of a panic of a parser | | 29.4 | Head: version 1, a salt of 32 bytes, the comment and the declared author, 1 to 65535 files in the order of R8 with their layout, SHA-256 and mtime up to 9999, at most 16 MiB; layer 2, layer 3 with R1 and R8, then layer 4 in key order, `ERR_HEAD_INVALID`, and the critical extensions | `capsule.Head`, `File`, `EncodeHead`, `DecodeHead` (`decodeHead`, `checkHeadFields`), `CheckHeadEnd`; `extension.Head` | `capsule.TestHeadRoundTrip`, `TestDecodeHeadLayers`, `TestOpen3Substeps`; `testdata/vectors/head_schema.json` (`internal/testkit.HeadSchemaVectors`); the head mutations of the list of format 3 of §64 | | 29.5 | Paths: R1 and R8 in layer 3; R2 to R6c, R4b and R10 for each entry, then R7 and R9 over the tree, in layer 4; the key of R7; errors that name the rule and the character, never the text | `internal/pathrule` (`CheckPath`, `CheckTree`, `Key`, `NFD`, `Fold`, `Error`) | `pathrule.TestCheckPath`, `TestCheckTree`, `TestKey`, `TestNFD`; `testdata/vectors/paths.json` and `path_fold.json` (`internal/testkit.PathVectors`, `PathFoldVectors`, `TestFormat3VectorFiles`); the path mutations of §64 | -| 29.5.1 | Tables: Unicode 18.0.0 and the 15 WindowsBestFit tables, pinned by their SHA-256, never the Unicode functions of the platform | `internal/pathrule/gen`, which checks the 19 pinned files in `.cache` and writes `tables.go`; `pathrule.UnicodeVersion`, `TablesDigest` | `pathrule.TestTablesDigest`, `TestProperties`; NFD and folding compared with `golang.org/x/text` outside this module | +| 29.5.1 | Tables: Unicode 18.0.0 and the 15 WindowsBestFit tables, pinned by their SHA-256, never the Unicode functions of the platform | `internal/pathrule/gen`, which checks the 19 pinned files in `.cache` and writes `tables.go`, and with `-ts` and `-dart` the same tables for `datekeys-ts` and `datekeys-dart`; `pathrule.UnicodeVersion`, `TablesDigest` | `pathrule.TestTablesDigest`, `TestProperties`; NFD and folding compared with `golang.org/x/text` outside this module | | 29.6 | Text of the comment and of the declared author: no control but TAB and LF in the comment, no bidirectional control, separator, byte order mark or noncharacter, the invisibles rule with R4b for each line, no space at the ends of the author; the writer turns CR LF and a lone CR into LF | `pathrule.CheckComment`, `CheckAuthor`; `capsule.EncryptFiles` (`newHead`) | `pathrule.TestTexts`; `testdata/vectors/head_schema.json`; `capsule.TestEncryptFilesRoundTrip`, `TestEncryptFilesRejects` | | 29.7 | Verdicts X, F0 to F6 and S0 to S5, for each part the first row of the table that holds, with the texts of the table; the name of a certificate between « and », shown when it meets the rules of the declared author, has at most 64 code points and no two spaces in a row, and its SHA-256 otherwise; the holder by `givenName` and `surname` before `commonName` when both have text that is not empty, the issuer by `commonName` or, without one that has text, `organizationName`; after F6, a line for each required signer with the authority of its seal, and «DateKeys no comprueba quién emitió los sellos.» when one says before the date; a foreign signer apart, with its result in Spanish; before the date only by a valid seal with t + accuracy < round_time, with its warning; an mtime later than a valid seal shown as an inconsistency (SHOULD); the presentation: the verdicts first and last, the labels of the text of the creator, TABs expanded, pieces of at most W − 3 columns behind `│ ` with the width counted by excess, the lines of the verdicts in rows of at most W − 3 columns, broken at the last space that fits, each row after the first behind ` ↳ `, and warnings of risky names | `capsule.Verdict`, `Verdict.Text`, `Verdicts.Lines`, `Verdicts.SealedAt`, `Detail`, `SignerLine` (`quoted`, `instant`, `resultText`), `holderText`, `MaxNameLen`; `internal/cms` `Cert.Holder`, `Cert.IssuerName`; `capsule.Open` step 17.6 (`newSecurityContext`, `openBody`), `OpenOptions.AuthorKeys` (F3), `OpenOptions.Accept` (the verdicts before step 18); `cmd/datekeys.present` (`writeVerdicts`, `rows`, `contMark`, `writeCreator`, `pieces`, `expandTabs`, `outputWidth`, `termWidth`, `risks`) | `capsule.TestSecurityVerdicts`, `TestEvaluateSecurityIn` (the lines of F3 and F4), `TestEvaluateCMS` (the lines of F6 with the authority of each seal and the warning, a late seal without it, a foreign signer), `TestEvaluateSeal` (the line of S4), `TestIssuerTextFiltered`, `TestCMSVectors`; the lines of the records of the fixtures (`capsule.TestConformanceFixtures`) and of `testdata/vectors/security.json` (`internal/testkit.TestFormat3VectorFiles`); `cmd/datekeys.TestRows`, `TestPresent`, `TestMTimeAfterSeal`, `TestAuthorSignRoundTrip`, `TestEncryptDecryptRoundTrip`, `TestDecryptFormat3Fixtures`; the names of §64 of v0.12 (two spaces, more than 64 code points, ESC, U+202E, a byte order mark, `givenName` and `surname` with a NIF in `commonName`, an issuer without text): `security_cms.json` | | 29.8 | Author signature, what is signed: `payload_commit`; `CONTROL_SIG`, the control with `payload_commit` in place of `I_PAYLOAD` and L at zero; `control_commit`, `head_digest`, `signers_digest`, and `AUTHOR_MESSAGE`, ASCII of 99 bytes, with its code of 8 characters; never the area or the padding, so the area can grow after signing; every value recomputed from the opened capsule | `capsule.PayloadCommit`, `ControlCommit`, `HeadDigest`, `SignersDigest`, `AuthorMessage`, `AuthorMessagePrefix`, `AuthorMessageSize`, `AuthorCode`; `capsule.Open` step 17.6 (`newSecurityContext`); the writer signs once the control is final, in the `prepare` that `EncryptFiles` gives `sealer.write` (`sealer.security`) | `capsule.TestAuthorMessage` (99 bytes, the code, `control_commit` without L and with `I_PAYLOAD`), `TestSignedFixtureVerdicts` (another control or head: F2), `TestAreaChosenAfterSigning` (signed once); `TestConformanceFixtures` (`checkSignature3`: the commitments, `AUTHOR_MESSAGE` and its code in the records of `format3_signed`, `format3_signed_cms` and `format3_sealed`); mutations *the signature of alg 1 transplanted to another capsule …*, *the area widened to 64 KiB after signing …* | diff --git a/internal/pathrule/gen/dart.go b/internal/pathrule/gen/dart.go new file mode 100644 index 0000000..ef2a0f7 --- /dev/null +++ b/internal/pathrule/gen/dart.go @@ -0,0 +1,108 @@ +package main + +import ( + "bytes" + "fmt" +) + +// dartSource renders the tables as a Dart library for datekeys-dart, the +// third implementation (spec §29.5.1): the same data as tables.go and the +// module of tsSource, as const lists of ints, and the same digest, which its +// tests recompute from the lists with the canonical text of Canonical. The +// lists run to thousands of lines, so the library turns the formatter off: +// dart format keeps it as written here. +func (t *tables) dartSource() []byte { + var b bytes.Buffer + w := func(format string, args ...any) { fmt.Fprintf(&b, format, args...) } + ints := func(indent string, vs []int64) { + for i, v := range vs { + if i%16 == 0 { + w("\n%s", indent) + } else { + w(" ") + } + w("%d,", v) + } + } + list := func(name, doc string, vs []int64) { + w("/// %s\nconst List %s = [", doc, name) + ints(" ", vs) + w("\n];\n\n") + } + runes := func(rs []rune) []int64 { + out := make([]int64, len(rs)) + for i, r := range rs { + out[i] = int64(r) + } + return out + } + pairs := func(rs [][2]rune) []int64 { + out := make([]int64, 0, 2*len(rs)) + for _, r := range rs { + out = append(out, int64(r[0]), int64(r[1])) + } + return out + } + mapping := func(prefix, doc string, m map[rune][]rune) { + keys := sortedKeys(m) + var start, data []int64 + for _, k := range keys { + start = append(start, int64(len(data))) + data = append(data, runes(m[k])...) + } + start = append(start, int64(len(data))) + list(prefix+"Keys", doc+": the code points that have one, sorted.", runes(keys)) + list(prefix+"Start", doc+": where the value of each key starts in "+prefix+"Data, and its end.", start) + list(prefix+"Data", doc+": the values, one after another.", data) + } + + w("// Code generated by internal/pathrule/gen of the Go reference, from Unicode\n") + w("// %s and WindowsBestFit. DO NOT EDIT: regenerate it from datekeys-go.\n", UnicodeVersion) + w("//\n// The lists run to thousands of lines: the formatter leaves them as the\n") + w("// generator writes them.\n// dart format off\n\n") + w("/// The Unicode and best-fit tables of the path and text rules (spec\n") + w("/// §29.5.1), as internal/pathrule/tables.go of datekeys-go.\n") + w("///\n/// Internal: lib/datekeys.dart does not export it.\nlibrary;\n\n") + w("/// The version of Unicode of these tables, fixed with capsule format 3 (spec §29.5.1).\n") + w("const unicodeVersion = '%s';\n\n", UnicodeVersion) + w("/// The SHA-256 of the canonical text of these tables, as the Go reference\n") + w("/// computes it in pathrule.Canonical: the tests recompute it from the lists.\n") + w("const tablesDigest = '%s';\n\n", t.digest()) + w("/// The data files of the tables and their SHA-256 (spec §29.5.1).\n") + w("const List<({String path, String sha256})> sources = [\n") + for _, s := range t.sources { + w(" (path: '%s', sha256: '%s'),\n", s.path, s.sha256) + } + w("];\n\n") + + list("assignedRanges", "The assigned code points, as inclusive ranges: lo, hi, lo, hi, ...", pairs(t.assigned)) + list("ignorableRanges", "Default_Ignorable_Code_Point, as inclusive ranges.", pairs(t.ignorable)) + var ccc []int64 + for _, k := range sortedKeys(t.ccc) { + ccc = append(ccc, int64(k)<<8|int64(t.ccc[k])) + } + list("cccTable", "Each code point with a non-zero canonical combining class, as code point * 256 + class, sorted.", ccc) + mapping("decomposition", "The full canonical decomposition, Hangul syllables excepted", t.decomp) + mapping("folding", "The case folding of CaseFolding.txt, statuses C and F", t.fold) + mapping("lowercase", "The simple lowercase mapping of UnicodeData.txt, field 13", t.lower) + list("vs15Bases", "The characters of an emoji variation sequence with U+FE0E, sorted.", runes(t.vs15)) + list("vs16Bases", "The characters of an emoji variation sequence with U+FE0F, sorted.", runes(t.vs16)) + + w("/// For each code page of WindowsBestFit, the code points of U+0080 and above\n") + w("/// that it maps to a single ASCII byte, sorted, and those bytes.\n") + w("const List<({String name, List from, List to})> bestFitTables = [\n") + for _, bt := range t.bestFit { + keys := sortedKeys(bt.to) + to := make([]int64, len(keys)) + for i, k := range keys { + to[i] = int64(bt.to[k]) + } + w(" (\n name: '%s',\n from: [", bt.name) + ints(" ", runes(keys)) + w("\n ],\n to: [") + ints(" ", to) + w("\n ],\n ),\n") + } + w("];\n") + return b.Bytes() +} diff --git a/internal/pathrule/gen/main.go b/internal/pathrule/gen/main.go index 1d0735b..1d83391 100644 --- a/internal/pathrule/gen/main.go +++ b/internal/pathrule/gen/main.go @@ -18,6 +18,10 @@ // The tables are also described by a canonical text, whose SHA-256 is // TablesDigest: an implementation in another language generated from the // same files computes the same digest (see canonical in package pathrule). +// -ts and -dart write the same tables for the other two implementations: +// +// go run ./internal/pathrule/gen -data .cache -ts ../datekeys-ts/src/lib/dkc/pathrule-tables.ts +// go run ./internal/pathrule/gen -data .cache -dart ../datekeys-dart/lib/src/pathrule_tables.dart package main import ( @@ -73,6 +77,7 @@ func main() { data := flag.String("data", ".cache", "directory with the downloaded data files") out := flag.String("go", "internal/pathrule/tables.go", "Go file to write") tsOut := flag.String("ts", "", "TypeScript module to write for datekeys-ts; none when empty") + dartOut := flag.String("dart", "", "Dart library to write for datekeys-dart; none when empty") flag.Parse() t, err := build(*data) if err != nil { @@ -95,6 +100,13 @@ func main() { } fmt.Printf("wrote %s\n", *tsOut) } + if *dartOut != "" { + if err := os.WriteFile(*dartOut, t.dartSource(), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "gen:", err) + os.Exit(1) + } + fmt.Printf("wrote %s\n", *dartOut) + } fmt.Printf("wrote %s: %d assigned ranges, %d ignorable ranges, %d ccc, %d decompositions, %d foldings, %d+%d emoji bases, %d best-fit tables; digest %s\n", *out, len(t.assigned), len(t.ignorable), len(t.ccc), len(t.decomp), len(t.fold), len(t.vs15), len(t.vs16), len(t.bestFit), t.digest()) }