func TestNew(t *testing.T) {
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
config.SetMIMEInfo("tika-mimetypes.xml")()
mi, err := newMIMEInfo(config.MIMEInfo())
if err != nil {
t.Error(err)
}
sigs, ids, err := mi.Signatures()
if err != nil {
t.Error(err)
}
for i, v := range sigs {
if len(v) == 0 {
t.Errorf("Empty signature: %s", ids[i])
}
}
id, _ := New()
str := id.String()
saver := persist.NewLoadSaver(nil)
id.Save(saver)
loader := persist.NewLoadSaver(saver.Bytes())
_ = loader.LoadByte()
id2 := Load(loader)
if str != id2.String() {
t.Errorf("Load identifier fail: got %s, expect %s", str, id2.String())
}
}
func TestNew(t *testing.T) {
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
_, err := NewPronom()
if err != nil {
t.Error(err)
}
}
// DROID parsing is tested by comparing it against Report parsing
func TestParseDroid(t *testing.T) {
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
d, err := newDroid(config.Droid())
if err != nil {
t.Fatal(err)
}
r, err := newReports(d.IDs(), d.idsPuids())
if err != nil {
t.Fatal(err)
}
dsigs, dpuids, err := d.Signatures()
if err != nil {
t.Fatal(err)
}
rsigs, rpuids, err := r.Signatures()
if err != nil {
t.Fatal(err)
}
if len(dpuids) != len(rpuids) {
t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8])
}
for i, v := range rpuids {
if v != dpuids[i] {
t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i])
}
}
if len(dsigs) != len(rsigs) {
t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs))
}
for i, v := range rsigs {
if !v.Equals(dsigs[i]) {
t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n Droid: %s", rpuids[i], v, dsigs[i])
}
}
}
func TestLOC(t *testing.T) {
var dump, dumpmagic bool // set to true to print out LOC sigs
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
config.SetLOC("")()
l, err := newLOC(config.LOC())
if l != nil {
if dump {
fdd := l.(fdds)
for _, v := range fdd.f {
fmt.Println(v)
fmt.Println("****************")
}
} else if dumpmagic {
fdd := l.(fdds)
for _, v := range fdd.f {
if len(v.Magics) > 0 {
fmt.Println("{")
for _, m := range v.Magics {
fmt.Println("`" + m + "`,")
}
fmt.Println("},")
}
}
}
if _, _, err = l.Signatures(); err != nil {
t.Fatal(err)
}
} else {
t.Fatalf("Expecting a LOC, got nothing! Error: %v", err)
}
}
func TestMakePronomTika(t *testing.T) {
s := siegfried.New()
config.SetHome(*testhome)
p, err := pronom.New()
if err != nil {
t.Fatal(err)
}
err = s.Add(p)
if err != nil {
t.Fatal(err)
}
m, err := mimeinfo.New(config.SetName("tika"), config.SetMIMEInfo("tika"))
if err != nil {
t.Fatal(err)
}
err = s.Add(m)
if err != nil {
t.Fatal(err)
}
l, err := loc.New(config.SetName("loc"), config.SetLOC(""))
if err != nil {
t.Fatal(err)
}
err = s.Add(l)
if err != nil {
t.Fatal(err)
}
sigs := filepath.Join("data", "pronom-tika-loc.sig")
err = s.Save(sigs)
if err != nil {
t.Fatal(err)
}
}
func blameSig(i int) error {
if *inspectHome != config.Home() {
config.SetHome(*inspectHome)
}
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Println(s.Blame(i, *inspectCType, *inspectCName))
return nil
}
func inspectSig(t core.MatcherType) error {
if *inspectHome != config.Home() {
config.SetHome(*inspectHome)
}
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Print(s.Inspect(t))
return nil
}
func TestUpdated(t *testing.T) {
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
l, err := newLOC(config.LOC())
if err != nil || l == nil {
t.Fatalf("couldn't parse LOC file: %v", err)
}
expect, _ := time.Parse(dateFmt, "2016-01-01")
f := l.(fdds)
if !f.Updated().After(expect) {
t.Fatalf("expected %v, got %v", expect, f.Updated())
}
}
func TestLoad(t *testing.T) {
s := New()
config.SetHome("./cmd/roy/data")
p, err := pronom.New()
if err != nil {
t.Fatal(err)
}
err = s.Add(p)
if err != nil {
t.Fatal(err)
}
}
func setup(opts ...config.Option) error {
if opts == nil && s != nil {
return nil
}
var err error
s = siegfried.New()
config.SetHome(*testhome)
opts = append(opts, config.SetDoubleUp())
p, err := pronom.New(opts...)
if err != nil {
return err
}
return s.Add(p)
}
func setHarvestOptions() {
if *harvestHome != config.Home() {
config.SetHome(*harvestHome)
}
if *harvestDroid != config.Droid() {
config.SetDroid(*harvestDroid)()
}
if *timeout != htimeout {
config.SetHarvestTimeout(*timeout)
}
if *throttlef > 0 {
config.SetHarvestThrottle(*throttlef)
}
}
func TestMakeDefault(t *testing.T) {
s := siegfried.New()
config.SetHome(*testhome)
p, err := pronom.New()
if err != nil {
t.Fatal(err)
}
err = s.Add(p)
if err != nil {
t.Fatal(err)
}
sigs := filepath.Join("data", config.SignatureBase())
err = s.Save(sigs)
if err != nil {
t.Fatal(err)
}
}
func TestSets(t *testing.T) {
config.SetHome(*testhome)
list := "fmt/1,fmt/2,@pdfa,x-fmt/19"
expect := "fmt/1,fmt/2,fmt/95,fmt/354,fmt/476,fmt/477,fmt/478,fmt/479,fmt/480,fmt/481,x-fmt/19"
res := strings.Join(expandSets(list), ",")
if res != expect {
t.Errorf("expecting %s, got %s", expect, res)
}
pdfs := strings.Join(expandSets("@pdf"), ",")
expect = "fmt/14,fmt/15,fmt/16,fmt/17,fmt/18,fmt/19,fmt/20,fmt/95,fmt/144,fmt/145,fmt/146,fmt/147,fmt/148,fmt/157,fmt/158,fmt/276,fmt/354,fmt/476,fmt/477,fmt/478,fmt/479,fmt/480,fmt/481,fmt/488,fmt/489,fmt/490,fmt/491,fmt/492,fmt/493"
if pdfs != expect {
t.Errorf("expecting %s, got %s", expect, pdfs)
}
compression := strings.Join(expandSets("@compression"), ",")
expect = "fmt/626,x-fmt/266,x-fmt/267,x-fmt/268"
if compression != expect {
t.Errorf("expecting %s, got %s", expect, compression)
}
}
func TestMakeArchivematica(t *testing.T) {
s := siegfried.New()
config.SetHome(*testhome)
p, err := pronom.New(
config.SetName("archivematica"),
config.SetExtend(expandSets("archivematica-fmt2.xml,archivematica-fmt3.xml,archivematica-fmt4.xml,archivematica-fmt5.xml")))
if err != nil {
t.Fatal(err)
}
err = s.Add(p)
if err != nil {
t.Fatal(err)
}
sigs := filepath.Join("data", "archivematica.sig")
err = s.Save(sigs)
if err != nil {
t.Fatal(err)
}
}
func main() {
flag.Parse()
/*//UNCOMMENT TO RUN PROFILER
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()*/
// configure home and signature if not default
if *home != config.Home() {
config.SetHome(*home)
}
if *sig != config.SignatureBase() {
config.SetSignature(*sig)
}
// handle -update
if *update {
msg, err := updateSigs()
if err != nil {
log.Fatalf("[FATAL] failed to update signature file, %v", err)
}
fmt.Println(msg)
return
}
// handle -hash error
hashT := getHash(*hashf)
if *hashf != "" && hashT < 0 {
log.Fatalf("[FATAL] invalid hash type; choose from %s", hashChoices)
}
// load and handle signature errors
s, err := siegfried.Load(config.Signature())
if err != nil {
log.Fatalf("[FATAL] error loading signature file, got: %v", err)
}
// handle -version
if *version {
version := config.Version()
fmt.Printf("siegfried %d.%d.%d\n%s", version[0], version[1], version[2], s)
return
}
// handle -fpr
if *fprflag {
log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
serveFpr(config.Fpr(), s)
return
}
// check -multi
if *multi > maxMulti || *multi < 1 || (*archive && *multi > 1) {
log.Println("[WARN] -multi must be > 0 and =< 1024. If -z, -multi must be 1. Resetting -multi to 1")
*multi = 1
}
// start logger
lg, err := newLogger(*logf)
if err != nil {
log.Fatalln(err)
}
if config.Slow() || config.Debug() {
if *serve != "" || *fprflag {
log.Fatalln("[FATAL] debug and slow logging cannot be run in server mode")
}
}
// start throttle
if *throttlef != 0 {
throttle = time.NewTicker(*throttlef)
defer throttle.Stop()
}
// start the printer
lenCtxts := *multi
if lenCtxts == 1 {
lenCtxts = 8
}
ctxts := make(chan *context, lenCtxts)
go printer(ctxts, lg)
// set default writer
var w writer
switch {
case *csvo:
w = newCSV(os.Stdout)
case *jsono:
w = newJSON(os.Stdout)
case *droido:
w = newDroid(os.Stdout)
if len(s.Fields()) != 1 || len(s.Fields()[0]) != 7 {
close(ctxts)
log.Fatalln("[FATAL] DROID output is limited to signature files with a single PRONOM identifier")
}
default:
w = newYAML(os.Stdout)
}
// overrite writer with nil writer if logging is to stdout
if lg != nil && lg.w == os.Stdout {
w = logWriter{}
}
// setup default waitgroup
wg := &sync.WaitGroup{}
// setup context pool
setCtxPool(s, w, wg, hashT, *archive)
// handle -serve
if *serve != "" {
log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
listen(*serve, s, ctxts)
return
}
// handle no file/directory argument
if flag.NArg() != 1 {
close(ctxts)
log.Fatalln("[FATAL] expecting a single file or directory argument")
}
w.writeHead(s, hashT)
// support reading list files from stdin
if flag.Arg(0) == "-" {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
info, err := os.Stat(scanner.Text())
if err != nil {
info, err = retryStat(scanner.Text(), err)
}
if err != nil || info.IsDir() {
ctx := getCtx(scanner.Text(), "", "", 0)
ctx.res <- results{fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil, nil}
ctx.wg.Add(1)
ctxts <- ctx
} else {
identifyFile(getCtx(scanner.Text(), "", info.ModTime().Format(time.RFC3339), info.Size()), ctxts, getCtx)
}
}
} else {
err = identify(ctxts, flag.Arg(0), "", *nr, getCtx)
}
wg.Wait()
close(ctxts)
w.writeTail()
// log time elapsed
if !lg.start.IsZero() {
fmt.Fprintf(lg.w, "%s %v\n", timeString, time.Since(lg.start))
}
if err != nil {
log.Fatal(err)
}
os.Exit(0)
}
func getOptions() []config.Option {
if *home != config.Home() {
config.SetHome(*home)
}
if *inspectHome != config.Home() {
config.SetHome(*inspectHome)
}
opts := []config.Option{}
// build options
if *droid != config.Droid() {
opts = append(opts, config.SetDroid(*droid))
}
if *container != config.Container() {
opts = append(opts, config.SetContainer(*container))
}
if *mi != "" {
opts = append(opts, config.SetMIMEInfo(*mi))
}
if *fdd != "" {
opts = append(opts, config.SetLOC(*fdd))
}
if *locfdd {
opts = append(opts, config.SetLOC(""))
}
if *nopronom {
opts = append(opts, config.SetNoPRONOM())
}
if *name != "" {
opts = append(opts, config.SetName(*name))
}
if *details != config.Details() {
opts = append(opts, config.SetDetails(*details))
}
if *extend != "" {
opts = append(opts, config.SetExtend(expandSets(*extend)))
}
if *extendc != "" {
if *extend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element describing the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*extendc)))
}
if *include != "" {
opts = append(opts, config.SetLimit(expandSets(*include)))
}
if *exclude != "" {
opts = append(opts, config.SetExclude(expandSets(*exclude)))
}
if *bof != 0 {
opts = append(opts, config.SetBOF(*bof))
}
if *eof != 0 {
opts = append(opts, config.SetEOF(*eof))
}
if *noeof {
opts = append(opts, config.SetNoEOF())
}
if *multi != "" {
opts = append(opts, config.SetMulti(strings.ToLower(*multi)))
}
if *nocontainer {
opts = append(opts, config.SetNoContainer())
}
if *notext {
opts = append(opts, config.SetNoText())
}
if *noname {
opts = append(opts, config.SetNoName())
}
if *nomime {
opts = append(opts, config.SetNoMIME())
}
if *noxml {
opts = append(opts, config.SetNoXML())
}
if *noriff {
opts = append(opts, config.SetNoRIFF())
}
if *noreports {
opts = append(opts, config.SetNoReports())
}
if *doubleup {
opts = append(opts, config.SetDoubleUp())
}
if *rng != config.Range() {
opts = append(opts, config.SetRange(*rng))
}
if *distance != config.Distance() {
opts = append(opts, config.SetDistance(*distance))
}
if *choices != config.Choices() {
opts = append(opts, config.SetChoices(*choices))
}
// inspect options
if *inspectMI != "" {
opts = append(opts, config.SetMIMEInfo(*inspectMI))
}
if *inspectFDD != "" {
opts = append(opts, config.SetLOC(*fdd))
}
if *inspectLOC {
opts = append(opts, config.SetLOC(""))
}
if *inspectInclude != "" {
opts = append(opts, config.SetLimit(expandSets(*inspectInclude)))
}
if *inspectExclude != "" {
opts = append(opts, config.SetExclude(expandSets(*inspectExclude)))
}
if *inspectExtend != "" {
opts = append(opts, config.SetExtend(expandSets(*inspectExtend)))
}
if *inspectExtendc != "" {
if *inspectExtend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element describing the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*inspectExtendc)))
}
return opts
}