@ -1,37 +1,64 @@
/ / Vendors testdata / from the Go reference implementation
/ / Vendors testdata / and the word lists of wordkey / lists / from the Go
/ / ( g . activething . com / go / DateKeys ) at one pinned commit and verifies it . The Go
/ / reference implementation ( g . activething . com / go / DateKeys ) at one pinned
/ / fixtures and vectors are the source of truth: this package never generates
/ / commit and verifies them . The Go fixtures , vectors and lists are the source
/ / its own . A port of scripts / sync - testdata . mjs of datekeys - ts , with the same
/ / of truth: this package never generates its own . A port of
/ / testdata/ SOURCE . json .
/ / scripts/ sync - testdata . mjs of datekeys - ts , with the same testdata/ SOURCE . json .
/ /
/ /
/ / dart run tool / sync_testdata . dart sync [ - - repo . . / datekeys - go ] [ - - commit HEAD ]
/ / dart run tool / sync_testdata . dart sync [ - - repo . . / datekeys - go ] [ - - commit HEAD ]
/ / dart run tool / sync_testdata . dart check [ - - against . . / datekeys - go ]
/ / dart run tool / sync_testdata . dart check [ - - against . . / datekeys - go ]
/ /
/ /
/ / sync copies every blob under testdata / at the commit ( read with git , so
/ / sync copies every blob under testdata / and under wordkey / lists / at the
/ / uncommitted changes in the Go checkout are never picked up ) and writes
/ / commit ( read with git , so uncommitted changes in the Go checkout are never
/ / testdata / SOURCE . json with the full commit id and the SHA - 256 of each file .
/ / picked up ) into testdata / and wordlists / , and writes the SOURCE . json of
/ / check verifies that the files on disk match SOURCE . json exactly , with no
/ / each , with the full commit id and the SHA - 256 of each file . It reads every
/ / missing or extra files ; with - - against it also re - reads every blob from the
/ / blob of both before it touches either copy , and a tree missing at the
/ / Go repository at the recorded commit . test / testdata_test . dart runs check .
/ / commit is an error . check verifies that the files on disk of each tree
/ / No dependencies but package: crypto , Dart and git .
/ / match its SOURCE . json exactly , with no missing or extra files ; with
/ / - - against it also re - reads every blob from the Go repository at the
/ / recorded commit . It prints one line per tree , testdata first .
/ / test / testdata_test . dart runs check . No dependencies but package: crypto ,
/ / Dart and git .
import ' dart:convert ' ;
import ' dart:convert ' ;
import ' dart:io ' ;
import ' dart:io ' ;
import ' package:crypto/crypto.dart ' ;
import ' package:crypto/crypto.dart ' ;
/ / / The module whose t estdata/ is vendored .
/ / / The module whose t rees are vendored .
const module = ' g.activething.com/go/DateKeys ' ;
const module = ' g.activething.com/go/DateKeys ' ;
/ / / What check found: the files , the commit and the module of SOURCE . json , and
/ / / A tree that sync vendors: [ dir ] , a directory of this package , holds the
/ / / every difference , none when the copy is intact .
/ / / files of [ upstream ] , a directory of the Go repository .
class Tree {
const Tree ( this . dir , this . upstream ) ;
final String dir ;
final String upstream ;
}
/ / / testdata / , the fixtures and the vectors of the Go reference .
const testdataTree = Tree ( ' testdata ' , ' testdata ' ) ;
/ / / wordlists / , the word lists of wordkey . Generate , whose SHA - 256
/ / / lib / src / wordlist . dart pins .
const wordlistsTree = Tree ( ' wordlists ' , ' wordkey/lists ' ) ;
/ / / The trees , in the order of the lines of check .
const trees = [ testdataTree , wordlistsTree ] ;
/ / / What check found in a tree: the files , the commit and the module of its
/ / / SOURCE . json , and every difference , none when the copy is intact .
class CheckResult {
class CheckResult {
CheckResult ( this . module , this . commit , this . files , this . errors ) ;
CheckResult ( this . tree, this . module, this . commit , this . files , this . errors ) ;
final Tree tree ;
final String module ;
final String module ;
final String commit ;
final String commit ;
final int files ;
final int files ;
final List < String > errors ;
final List < String > errors ;
/ / / The line that check prints for an intact copy .
String get line = > ' ${ tree . dir } : $ files files match $ module at $ commit ' ;
}
}
void main ( List < String > args ) {
void main ( List < String > args ) {
@ -46,23 +73,28 @@ void main(List<String> args) {
sync ( root , Directory ( repo ) . absolute , commit ) ;
sync ( root , Directory ( repo ) . absolute , commit ) ;
case ' check ' :
case ' check ' :
final against = _option ( rest , ' --against ' ) ;
final against = _option ( rest , ' --against ' ) ;
final r = check (
root ,
against: against = = null ? null : Directory ( against ) . absolute ,
) ;
if ( r . errors . isNotEmpty ) {
for ( final e in r . errors ) {
stderr . writeln ( ' testdata: $ e ' ) ;
}
exitCode = 1 ;
return ;
}
final verified = against = = null
final verified = against = = null
? ' '
? ' '
: ' (verified against the repository) ' ;
: ' (verified against the repository) ' ;
stdout . writeln (
for ( final tree in trees ) {
' testdata: ${ r . files } files match ${ r . module } at ${ r . commit } $ verified ' ,
try {
) ;
final r = check (
root ,
tree: tree ,
against: against = = null ? null : Directory ( against ) . absolute ,
) ;
if ( r . errors . isEmpty ) {
stdout . writeln ( ' ${ r . line } $ verified ' ) ;
continue ;
}
for ( final e in r . errors ) {
stderr . writeln ( ' ${ tree . dir } : $ e ' ) ;
}
} catch ( e ) {
stderr . writeln ( ' ${ tree . dir } : $ e ' ) ;
}
exitCode = 1 ;
}
default :
default :
stderr . writeln (
stderr . writeln (
' usage: sync_testdata.dart sync [--repo PATH] [--commit REV] | check [--against PATH] ' ,
' usage: sync_testdata.dart sync [--repo PATH] [--commit REV] | check [--against PATH] ' ,
@ -70,56 +102,72 @@ void main(List<String> args) {
exitCode = 2 ;
exitCode = 2 ;
}
}
} catch ( e ) {
} catch ( e ) {
stderr . writeln ( ' testdata : $ e ' ) ;
stderr . writeln ( ' $ command : $ e ' ) ;
exitCode = 1 ;
exitCode = 1 ;
}
}
}
}
/ / / Copies testdata/ of the Go repository [ repo ] at [ rev ] into [ root ] , and
/ / / Copies each tree of [ trees ] of the Go repository [ repo ] at [ rev ] into
/ / / writes testdata / SOURCE . json .
/ / / [ root ] , and writes its SOURCE . json .
void sync ( Directory root , Directory repo , String rev ) {
void sync ( Directory root , Directory repo , String rev ) {
final commit = utf8
final commit = utf8
. decode ( _git ( repo , [ ' rev-parse ' , ' --verify ' , ' $ rev ^{commit} ' ] ) )
. decode ( _git ( repo , [ ' rev-parse ' , ' --verify ' , ' $ rev ^{commit} ' ] ) )
. trim ( ) ;
. trim ( ) ;
final paths = _listBlobs ( repo , commit ) ;
final listed = [
if ( paths . isEmpty ) throw StateError ( ' no testdata/ at $ commit ' ) ;
for ( final tree in trees ) ( tree , _listBlobs ( repo , commit , tree ) ) ,
/ / Read everything before touching the local copy , so a failed read leaves it intact .
] ;
final blobs = [ for ( final p in paths ) ( p , _readBlob ( repo , commit , p ) ) ] ;
for ( final ( tree , paths ) in listed ) {
final testdata = Directory ( ' ${ root . path } /testdata ' ) ;
if ( paths . isEmpty ) throw StateError ( ' no ${ tree . upstream } / at $ commit ' ) ;
if ( testdata . existsSync ( ) ) testdata . deleteSync ( recursive: true ) ;
}
final files = < String , String > { } ;
/ / Read everything before touching the local copies , so a failed read
for ( final ( path , data ) in blobs ) {
/ / leaves them intact .
File ( ' ${ testdata . path } / $ path ' )
final read = [
. . createSync ( recursive: true )
for ( final ( tree , paths ) in listed )
. . writeAsBytesSync ( data ) ;
( tree , [ for ( final p in paths ) ( p , _readBlob ( repo , commit , tree , p ) ) ] ) ,
files [ path ] = _sha256 ( data ) ;
] ;
for ( final ( tree , blobs ) in read ) {
final dir = Directory ( ' ${ root . path } / ${ tree . dir } ' ) ;
if ( dir . existsSync ( ) ) dir . deleteSync ( recursive: true ) ;
final files = < String , String > { } ;
for ( final ( path , data ) in blobs ) {
File ( ' ${ dir . path } / $ path ' )
. . createSync ( recursive: true )
. . writeAsBytesSync ( data ) ;
files [ path ] = _sha256 ( data ) ;
}
final source = { ' module ' : module , ' commit ' : commit , ' files ' : files } ;
File ( ' ${ dir . path } /SOURCE.json ' ) . writeAsStringSync (
' ${ const JsonEncoder . withIndent ( ' ' ) . convert ( source ) } \n ' ,
) ;
stdout . writeln (
' ${ tree . dir } : ${ blobs . length } files from $ module at $ commit ' ,
) ;
}
}
final source = { ' module ' : module , ' commit ' : commit , ' files ' : files } ;
File ( ' ${ testdata . path } /SOURCE.json ' ) . writeAsStringSync (
' ${ const JsonEncoder . withIndent ( ' ' ) . convert ( source ) } \n ' ,
) ;
stdout . writeln ( ' testdata: ${ paths . length } files from $ module at $ commit ' ) ;
}
}
/ / / Checks testdata / of [ root ] against its SOURCE . json and , with [ against ] ,
/ / / Checks [ tree ] of [ root ] , testdata / by default , against its SOURCE . json
/ / / against the Go repository at the recorded commit .
/ / / and , with [ against ] , against the Go repository at the recorded commit .
CheckResult check ( Directory root , { Directory ? against } ) {
CheckResult check (
final testdata = Directory ( ' ${ root . path } /testdata ' ) ;
Directory root , {
Tree tree = testdataTree ,
Directory ? against ,
} ) {
final dir = Directory ( ' ${ root . path } / ${ tree . dir } ' ) ;
final source = jsonDecode (
final source = jsonDecode (
File ( ' ${ testdata . path } /SOURCE.json ' ) . readAsStringSync ( ) ,
File ( ' ${ dir . path } /SOURCE.json ' ) . readAsStringSync ( ) ,
) as Map < String , Object ? > ;
) as Map < String , Object ? > ;
final recorded = ( source [ ' files ' ] ! as Map < String , Object ? > )
final recorded = ( source [ ' files ' ] ! as Map < String , Object ? > )
. cast < String , String > ( ) ;
. cast < String , String > ( ) ;
final commit = source [ ' commit ' ] ! as String ;
final commit = source [ ' commit ' ] ! as String ;
final expected = recorded . keys . toList ( ) . . sort ( ) ;
final expected = recorded . keys . toList ( ) . . sort ( ) ;
final actual = _listLocal ( testdata ) ;
final actual = _listLocal ( dir ) ;
final errors = < String > [ ] ;
final errors = < String > [ ] ;
for ( final path in expected ) {
for ( final path in expected ) {
if ( ! actual . contains ( path ) ) {
if ( ! actual . contains ( path ) ) {
errors . add ( ' missing $ path ' ) ;
errors . add ( ' missing $ path ' ) ;
continue ;
continue ;
}
}
final data = File ( ' ${ testdata . path } / $ path ' ) . readAsBytesSync ( ) ;
final data = File ( ' ${ dir . path } / $ path ' ) . readAsBytesSync ( ) ;
if ( _sha256 ( data ) ! = recorded [ path ] ) {
if ( _sha256 ( data ) ! = recorded [ path ] ) {
errors . add ( ' modified $ path ' ) ;
errors . add ( ' modified $ path ' ) ;
}
}
@ -128,17 +176,18 @@ CheckResult check(Directory root, {Directory? against}) {
if ( ! recorded . containsKey ( path ) ) errors . add ( ' extra $ path ' ) ;
if ( ! recorded . containsKey ( path ) ) errors . add ( ' extra $ path ' ) ;
}
}
if ( against ! = null ) {
if ( against ! = null ) {
final upstream = _listBlobs ( against , commit );
final upstream = _listBlobs ( against , commit , tree );
if ( upstream . join ( ' \n ' ) ! = expected . join ( ' \n ' ) ) {
if ( upstream . join ( ' \n ' ) ! = expected . join ( ' \n ' ) ) {
errors . add ( ' file list differs from $ commit ' ) ;
errors . add ( ' file list differs from $ commit ' ) ;
}
}
for ( final path in upstream ) {
for ( final path in upstream ) {
if ( _sha256 ( _readBlob ( against , commit , path) ) ! = recorded [ path ] ) {
if ( _sha256 ( _readBlob ( against , commit , tree, path) ) ! = recorded [ path ] ) {
errors . add ( ' differs from upstream $ path ' ) ;
errors . add ( ' differs from upstream $ path ' ) ;
}
}
}
}
}
}
return CheckResult (
return CheckResult (
tree ,
source [ ' module ' ] ! as String ,
source [ ' module ' ] ! as String ,
commit ,
commit ,
expected . length ,
expected . length ,
@ -162,8 +211,9 @@ List<int> _git(Directory repo, List<String> args) {
String _sha256 ( List < int > data ) = > sha256 . convert ( data ) . toString ( ) ;
String _sha256 ( List < int > data ) = > sha256 . convert ( data ) . toString ( ) ;
/ / Files under testdata / at commit , as paths relative to testdata / with forward slashes .
/ / Files under the upstream directory of tree at commit , as paths relative to
List < String > _listBlobs ( Directory repo , String commit ) {
/ / it with forward slashes .
List < String > _listBlobs ( Directory repo , String commit , Tree tree ) {
final out = utf8 . decode (
final out = utf8 . decode (
_git ( repo , [
_git ( repo , [
' ls-tree ' ,
' ls-tree ' ,
@ -172,7 +222,7 @@ List<String> _listBlobs(Directory repo, String commit) {
' --full-tree ' ,
' --full-tree ' ,
commit ,
commit ,
' -- ' ,
' -- ' ,
' testdata ' ,
tree . upstream ,
] ) ,
] ) ,
) ;
) ;
final paths = < String > [ ] ;
final paths = < String > [ ] ;
@ -183,20 +233,20 @@ List<String> _listBlobs(Directory repo, String commit) {
if ( meta [ 1 ] ! = ' blob ' | | meta [ 0 ] = = ' 120000 ' ) {
if ( meta [ 1 ] ! = ' blob ' | | meta [ 0 ] = = ' 120000 ' ) {
throw StateError ( ' unsupported entry $ path ( ${ meta [ 0 ] } ${ meta [ 1 ] } ) ' ) ;
throw StateError ( ' unsupported entry $ path ( ${ meta [ 0 ] } ${ meta [ 1 ] } ) ' ) ;
}
}
paths . add ( path . substring ( ' testdata /' . length ) ) ;
paths . add ( path . substring ( ' ${ tree . upstream } /' . length ) ) ;
}
}
return paths . . sort ( ) ;
return paths . . sort ( ) ;
}
}
List < int > _readBlob ( Directory repo , String commit , String path ) = >
List < int > _readBlob ( Directory repo , String commit , Tree tree , String path ) = >
_git ( repo , [ ' cat-file ' , ' blob ' , ' $ commit : testdata /$ path ' ] ) ;
_git ( repo , [ ' cat-file ' , ' blob ' , ' $ commit : ${ tree . upstream } /$ path ' ] ) ;
/ / Files on disk under testdata/ , excluding SOURCE . json .
/ / Files on disk under dir, excluding its SOURCE . json .
List < String > _listLocal ( Directory testdata ) {
List < String > _listLocal ( Directory dir ) {
final prefix = testdata . absolute . path . length + 1 ;
final prefix = dir . absolute . path . length + 1 ;
final files = < String > [ ] ;
final files = < String > [ ] ;
for ( final f
for ( final f
in testdata . absolute
in dir . absolute
. listSync ( recursive: true , followLinks: false )
. listSync ( recursive: true , followLinks: false )
. whereType < File > ( ) ) {
. whereType < File > ( ) ) {
final path = f . path
final path = f . path