//go:build ignore // Writes src/lib/dkc/gounicode.ts: the three sets of code points that Go's // strings.ToLower, strings.ToUpper and strings.TrimSpace depend on, as the // package unicode of the Go that runs it defines them, for the key strings // and the key files of authorkey.ts: // // - the code points r with unicode.ToLower(r) != r, which strings.ToLower // changes, so that parsePublic tells "written in lower case" as Go does; // - those with unicode.ToUpper(r) != r, for parseSecret; // - those with unicode.IsSpace(r), which strings.TrimSpace trims from a // line of a key file. // // Each set is written as runs [first, last, stride], stride 1 or 2, in // increasing order. No value is written by hand: the generator scans every // code point and checks the runs it writes against the functions of Go, and // the premises of authorkey.ts against strings.ToLower, strings.ToUpper and // strings.TrimSpace. It also writes the SHA-256 of the three bit sets, which // gounicode.test.ts recomputes from the runs. // // Run it with the Go toolchain of the reference implementation (the same // tables as the Dart port's, Unicode 15.0.0 of Go 1.26): // // go run scripts/go-unicode-tables.go -out src/lib/dkc/gounicode.ts package main import ( "crypto/sha256" "flag" "fmt" "log" "os" "runtime" "strings" "unicode" "unicode/utf8" ) // runs returns the code points of in as runs [first, last, stride]: a run // of stride 2 is taken when it covers at least three points, so that // alternating cases stay short. func runs(in func(rune) bool) [][3]rune { var pts []rune for r := rune(0); r <= unicode.MaxRune; r++ { if r >= 0xd800 && r <= 0xdfff { continue // not in valid UTF-8: a decoder gives U+FFFD instead } if in(r) { pts = append(pts, r) } } var out [][3]rune for i := 0; i < len(pts); { j := i for j+1 < len(pts) && pts[j+1] == pts[j]+1 { j++ } k := i for k+1 < len(pts) && pts[k+1] == pts[k]+2 { k++ } switch { case j > i: out = append(out, [3]rune{pts[i], pts[j], 1}) i = j + 1 case k-i >= 2: out = append(out, [3]rune{pts[i], pts[k], 2}) i = k + 1 default: out = append(out, [3]rune{pts[i], pts[i], 1}) i++ } } member := map[rune]bool{} for _, r := range out { for c := r[0]; c <= r[1]; c += r[2] { if member[c] { log.Fatalf("U+%04X twice", c) } member[c] = true } } for r := rune(0); r <= unicode.MaxRune; r++ { if r >= 0xd800 && r <= 0xdfff { continue } if member[r] != in(r) { log.Fatalf("U+%04X: the runs disagree", r) } } return out } // bitSum is the SHA-256 of the set in: bit r at byte r>>3, mask 1<<(r&7). func bitSum(in func(rune) bool) string { bits := make([]byte, (unicode.MaxRune+1)/8) for r := rune(0); r <= unicode.MaxRune; r++ { if r >= 0xd800 && r <= 0xdfff { continue } if in(r) { bits[r>>3] |= 1 << (r & 7) } } return fmt.Sprintf("%x", sha256.Sum256(bits)) } func list(b *strings.Builder, name, doc string, rs [][3]rune) { fmt.Fprintf(b, "// %s\nconst %s: readonly number[] = [\n", doc, name) for _, r := range rs { fmt.Fprintf(b, " 0x%04x, 0x%04x, %d,\n", r[0], r[1], r[2]) } b.WriteString("];\n\n") } func main() { out := flag.String("out", "", "the TypeScript file to write") flag.Parse() if *out == "" { log.Fatal("usage: go run scripts/go-unicode-tables.go -out src/lib/dkc/gounicode.ts") } // strings.ToLower and ToUpper change a string exactly when one of its // runes changes, or when it is not valid UTF-8: check that premise on // every code point. for r := rune(0); r <= unicode.MaxRune; r++ { if r >= 0xd800 && r <= 0xdfff { continue } s := string(r) if (strings.ToLower(s) != s) != (unicode.ToLower(r) != r) { log.Fatalf("strings.ToLower and unicode.ToLower disagree on U+%04X", r) } if (strings.ToUpper(s) != s) != (unicode.ToUpper(r) != r) { log.Fatalf("strings.ToUpper and unicode.ToUpper disagree on U+%04X", r) } if (strings.TrimSpace(s) == "") != unicode.IsSpace(r) { log.Fatalf("strings.TrimSpace and unicode.IsSpace disagree on U+%04X", r) } } if strings.ToLower("\xff") == "\xff" || strings.ToUpper("\xff") == "\xff" { log.Fatal("an invalid byte no longer changes") } if strings.TrimSpace(" \xff ") != "\xff" || utf8.RuneError != 0xfffd { log.Fatal("TrimSpace no longer stops at an invalid byte") } lower := func(r rune) bool { return unicode.ToLower(r) != r } upper := func(r rune) bool { return unicode.ToUpper(r) != r } var b strings.Builder fmt.Fprintf(&b, `// Code generated by scripts/go-unicode-tables.go with Go %s, Unicode %s. // DO NOT EDIT: regenerate it. // // The code points that Go's strings.ToLower, strings.ToUpper and // strings.TrimSpace depend on, as the package unicode of Go defines them: // authorkey.ts reads the strings of a key and the lines of a key file with // them, so that its errors are those of Go whatever the bytes. They are not // the case mapping of the JavaScript engine (toLowerCase, \p{…}), whose // version of Unicode changes with the engine, nor the tables of the path // rules (Unicode 18.0.0, spec §29.5.1). Internal: index.ts does not // re-export it. /** The version of Unicode of the package unicode of Go %s. */ export const GO_UNICODE_VERSION = '%s'; /** The SHA-256 of the bit sets of the three tables (bit r at byte r >> 3), which the tests recompute. */ export const GO_UNICODE_SUMS = { lower: '%s', upper: '%s', space: '%s' } as const; `, runtime.Version(), unicode.Version, runtime.Version(), unicode.Version, bitSum(lower), bitSum(upper), bitSum(unicode.IsSpace)) list(&b, "LOWER_CHANGES", "The runs [first, last, stride] of r with unicode.ToLower(r) != r.", runs(lower)) list(&b, "UPPER_CHANGES", "The runs [first, last, stride] of r with unicode.ToUpper(r) != r.", runs(upper)) list(&b, "SPACES", "The runs [first, last, stride] of r with unicode.IsSpace(r).", runs(unicode.IsSpace)) b.WriteString(`// Whether r is in the runs of table, by binary search on the last points. function inRuns(table: readonly number[], r: number): boolean { let lo = 0; let hi = table.length / 3; while (lo < hi) { const m = (lo + hi) >>> 1; if (table[3 * m + 1]! < r) lo = m + 1; else hi = m; } if (lo === table.length / 3) return false; const first = table[3 * lo]!; return r >= first && (r - first) % table[3 * lo + 2]! === 0; } /** Whether Go's unicode.ToLower changes the code point r. */ export const goLowerChanges = (r: number): boolean => inRuns(LOWER_CHANGES, r); /** Whether Go's unicode.ToUpper changes the code point r. */ export const goUpperChanges = (r: number): boolean => inRuns(UPPER_CHANGES, r); /** Whether r is a space for Go's unicode.IsSpace. */ export const goIsSpace = (r: number): boolean => inRuns(SPACES, r); `) if err := os.WriteFile(*out, []byte(b.String()), 0o644); err != nil { log.Fatal(err) } fmt.Printf("wrote %s\n", *out) }