func TestNew(t *testing.T) { config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data")) config.SetMIMEInfo("tika-mimetypes.xml")() mi, err := newMIMEInfo(config.MIMEInfo()) if err != nil { t.Error(err) } sigs, ids, err := mi.Signatures() if err != nil { t.Error(err) } for i, v := range sigs { if len(v) == 0 { t.Errorf("Empty signature: %s", ids[i]) } } id, _ := New() str := id.String() saver := persist.NewLoadSaver(nil) id.Save(saver) loader := persist.NewLoadSaver(saver.Bytes()) _ = loader.LoadByte() id2 := Load(loader) if str != id2.String() { t.Errorf("Load identifier fail: got %s, expect %s", str, id2.String()) } }
func TestNew(t *testing.T) { config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data")) _, err := NewPronom() if err != nil { t.Error(err) } }
// DROID parsing is tested by comparing it against Report parsing func TestParseDroid(t *testing.T) { config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data")) d, err := newDroid(config.Droid()) if err != nil { t.Fatal(err) } r, err := newReports(d.IDs(), d.idsPuids()) if err != nil { t.Fatal(err) } dsigs, dpuids, err := d.Signatures() if err != nil { t.Fatal(err) } rsigs, rpuids, err := r.Signatures() if err != nil { t.Fatal(err) } if len(dpuids) != len(rpuids) { t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8]) } for i, v := range rpuids { if v != dpuids[i] { t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i]) } } if len(dsigs) != len(rsigs) { t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs)) } for i, v := range rsigs { if !v.Equals(dsigs[i]) { t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n Droid: %s", rpuids[i], v, dsigs[i]) } } }
func TestLOC(t *testing.T) { var dump, dumpmagic bool // set to true to print out LOC sigs config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data")) config.SetLOC("")() l, err := newLOC(config.LOC()) if l != nil { if dump { fdd := l.(fdds) for _, v := range fdd.f { fmt.Println(v) fmt.Println("****************") } } else if dumpmagic { fdd := l.(fdds) for _, v := range fdd.f { if len(v.Magics) > 0 { fmt.Println("{") for _, m := range v.Magics { fmt.Println("`" + m + "`,") } fmt.Println("},") } } } if _, _, err = l.Signatures(); err != nil { t.Fatal(err) } } else { t.Fatalf("Expecting a LOC, got nothing! Error: %v", err) } }
func TestMakePronomTika(t *testing.T) { s := siegfried.New() config.SetHome(*testhome) p, err := pronom.New() if err != nil { t.Fatal(err) } err = s.Add(p) if err != nil { t.Fatal(err) } m, err := mimeinfo.New(config.SetName("tika"), config.SetMIMEInfo("tika")) if err != nil { t.Fatal(err) } err = s.Add(m) if err != nil { t.Fatal(err) } l, err := loc.New(config.SetName("loc"), config.SetLOC("")) if err != nil { t.Fatal(err) } err = s.Add(l) if err != nil { t.Fatal(err) } sigs := filepath.Join("data", "pronom-tika-loc.sig") err = s.Save(sigs) if err != nil { t.Fatal(err) } }
func blameSig(i int) error { if *inspectHome != config.Home() { config.SetHome(*inspectHome) } s, err := siegfried.Load(config.Signature()) if err != nil { return err } fmt.Println(s.Blame(i, *inspectCType, *inspectCName)) return nil }
func inspectSig(t core.MatcherType) error { if *inspectHome != config.Home() { config.SetHome(*inspectHome) } s, err := siegfried.Load(config.Signature()) if err != nil { return err } fmt.Print(s.Inspect(t)) return nil }
func TestUpdated(t *testing.T) { config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data")) l, err := newLOC(config.LOC()) if err != nil || l == nil { t.Fatalf("couldn't parse LOC file: %v", err) } expect, _ := time.Parse(dateFmt, "2016-01-01") f := l.(fdds) if !f.Updated().After(expect) { t.Fatalf("expected %v, got %v", expect, f.Updated()) } }
func TestLoad(t *testing.T) { s := New() config.SetHome("./cmd/roy/data") p, err := pronom.New() if err != nil { t.Fatal(err) } err = s.Add(p) if err != nil { t.Fatal(err) } }
func setup(opts ...config.Option) error { if opts == nil && s != nil { return nil } var err error s = siegfried.New() config.SetHome(*testhome) opts = append(opts, config.SetDoubleUp()) p, err := pronom.New(opts...) if err != nil { return err } return s.Add(p) }
func setHarvestOptions() { if *harvestHome != config.Home() { config.SetHome(*harvestHome) } if *harvestDroid != config.Droid() { config.SetDroid(*harvestDroid)() } if *timeout != htimeout { config.SetHarvestTimeout(*timeout) } if *throttlef > 0 { config.SetHarvestThrottle(*throttlef) } }
func TestMakeDefault(t *testing.T) { s := siegfried.New() config.SetHome(*testhome) p, err := pronom.New() if err != nil { t.Fatal(err) } err = s.Add(p) if err != nil { t.Fatal(err) } sigs := filepath.Join("data", config.SignatureBase()) err = s.Save(sigs) if err != nil { t.Fatal(err) } }
func TestSets(t *testing.T) { config.SetHome(*testhome) list := "fmt/1,fmt/2,@pdfa,x-fmt/19" expect := "fmt/1,fmt/2,fmt/95,fmt/354,fmt/476,fmt/477,fmt/478,fmt/479,fmt/480,fmt/481,x-fmt/19" res := strings.Join(expandSets(list), ",") if res != expect { t.Errorf("expecting %s, got %s", expect, res) } pdfs := strings.Join(expandSets("@pdf"), ",") expect = "fmt/14,fmt/15,fmt/16,fmt/17,fmt/18,fmt/19,fmt/20,fmt/95,fmt/144,fmt/145,fmt/146,fmt/147,fmt/148,fmt/157,fmt/158,fmt/276,fmt/354,fmt/476,fmt/477,fmt/478,fmt/479,fmt/480,fmt/481,fmt/488,fmt/489,fmt/490,fmt/491,fmt/492,fmt/493" if pdfs != expect { t.Errorf("expecting %s, got %s", expect, pdfs) } compression := strings.Join(expandSets("@compression"), ",") expect = "fmt/626,x-fmt/266,x-fmt/267,x-fmt/268" if compression != expect { t.Errorf("expecting %s, got %s", expect, compression) } }
func TestMakeArchivematica(t *testing.T) { s := siegfried.New() config.SetHome(*testhome) p, err := pronom.New( config.SetName("archivematica"), config.SetExtend(expandSets("archivematica-fmt2.xml,archivematica-fmt3.xml,archivematica-fmt4.xml,archivematica-fmt5.xml"))) if err != nil { t.Fatal(err) } err = s.Add(p) if err != nil { t.Fatal(err) } sigs := filepath.Join("data", "archivematica.sig") err = s.Save(sigs) if err != nil { t.Fatal(err) } }
func main() { flag.Parse() /*//UNCOMMENT TO RUN PROFILER go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }()*/ // configure home and signature if not default if *home != config.Home() { config.SetHome(*home) } if *sig != config.SignatureBase() { config.SetSignature(*sig) } // handle -update if *update { msg, err := updateSigs() if err != nil { log.Fatalf("[FATAL] failed to update signature file, %v", err) } fmt.Println(msg) return } // handle -hash error hashT := getHash(*hashf) if *hashf != "" && hashT < 0 { log.Fatalf("[FATAL] invalid hash type; choose from %s", hashChoices) } // load and handle signature errors s, err := siegfried.Load(config.Signature()) if err != nil { log.Fatalf("[FATAL] error loading signature file, got: %v", err) } // handle -version if *version { version := config.Version() fmt.Printf("siegfried %d.%d.%d\n%s", version[0], version[1], version[2], s) return } // handle -fpr if *fprflag { log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr()) serveFpr(config.Fpr(), s) return } // check -multi if *multi > maxMulti || *multi < 1 || (*archive && *multi > 1) { log.Println("[WARN] -multi must be > 0 and =< 1024. If -z, -multi must be 1. Resetting -multi to 1") *multi = 1 } // start logger lg, err := newLogger(*logf) if err != nil { log.Fatalln(err) } if config.Slow() || config.Debug() { if *serve != "" || *fprflag { log.Fatalln("[FATAL] debug and slow logging cannot be run in server mode") } } // start throttle if *throttlef != 0 { throttle = time.NewTicker(*throttlef) defer throttle.Stop() } // start the printer lenCtxts := *multi if lenCtxts == 1 { lenCtxts = 8 } ctxts := make(chan *context, lenCtxts) go printer(ctxts, lg) // set default writer var w writer switch { case *csvo: w = newCSV(os.Stdout) case *jsono: w = newJSON(os.Stdout) case *droido: w = newDroid(os.Stdout) if len(s.Fields()) != 1 || len(s.Fields()[0]) != 7 { close(ctxts) log.Fatalln("[FATAL] DROID output is limited to signature files with a single PRONOM identifier") } default: w = newYAML(os.Stdout) } // overrite writer with nil writer if logging is to stdout if lg != nil && lg.w == os.Stdout { w = logWriter{} } // setup default waitgroup wg := &sync.WaitGroup{} // setup context pool setCtxPool(s, w, wg, hashT, *archive) // handle -serve if *serve != "" { log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve) listen(*serve, s, ctxts) return } // handle no file/directory argument if flag.NArg() != 1 { close(ctxts) log.Fatalln("[FATAL] expecting a single file or directory argument") } w.writeHead(s, hashT) // support reading list files from stdin if flag.Arg(0) == "-" { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { info, err := os.Stat(scanner.Text()) if err != nil { info, err = retryStat(scanner.Text(), err) } if err != nil || info.IsDir() { ctx := getCtx(scanner.Text(), "", "", 0) ctx.res <- results{fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil, nil} ctx.wg.Add(1) ctxts <- ctx } else { identifyFile(getCtx(scanner.Text(), "", info.ModTime().Format(time.RFC3339), info.Size()), ctxts, getCtx) } } } else { err = identify(ctxts, flag.Arg(0), "", *nr, getCtx) } wg.Wait() close(ctxts) w.writeTail() // log time elapsed if !lg.start.IsZero() { fmt.Fprintf(lg.w, "%s %v\n", timeString, time.Since(lg.start)) } if err != nil { log.Fatal(err) } os.Exit(0) }
func getOptions() []config.Option { if *home != config.Home() { config.SetHome(*home) } if *inspectHome != config.Home() { config.SetHome(*inspectHome) } opts := []config.Option{} // build options if *droid != config.Droid() { opts = append(opts, config.SetDroid(*droid)) } if *container != config.Container() { opts = append(opts, config.SetContainer(*container)) } if *mi != "" { opts = append(opts, config.SetMIMEInfo(*mi)) } if *fdd != "" { opts = append(opts, config.SetLOC(*fdd)) } if *locfdd { opts = append(opts, config.SetLOC("")) } if *nopronom { opts = append(opts, config.SetNoPRONOM()) } if *name != "" { opts = append(opts, config.SetName(*name)) } if *details != config.Details() { opts = append(opts, config.SetDetails(*details)) } if *extend != "" { opts = append(opts, config.SetExtend(expandSets(*extend))) } if *extendc != "" { if *extend == "" { fmt.Println( `roy: warning! Unless the container extension only extends formats defined in the DROID signature file you should also include a regular signature extension (-extend) that includes a FileFormatCollection element describing the new formats.`) } opts = append(opts, config.SetExtendC(expandSets(*extendc))) } if *include != "" { opts = append(opts, config.SetLimit(expandSets(*include))) } if *exclude != "" { opts = append(opts, config.SetExclude(expandSets(*exclude))) } if *bof != 0 { opts = append(opts, config.SetBOF(*bof)) } if *eof != 0 { opts = append(opts, config.SetEOF(*eof)) } if *noeof { opts = append(opts, config.SetNoEOF()) } if *multi != "" { opts = append(opts, config.SetMulti(strings.ToLower(*multi))) } if *nocontainer { opts = append(opts, config.SetNoContainer()) } if *notext { opts = append(opts, config.SetNoText()) } if *noname { opts = append(opts, config.SetNoName()) } if *nomime { opts = append(opts, config.SetNoMIME()) } if *noxml { opts = append(opts, config.SetNoXML()) } if *noriff { opts = append(opts, config.SetNoRIFF()) } if *noreports { opts = append(opts, config.SetNoReports()) } if *doubleup { opts = append(opts, config.SetDoubleUp()) } if *rng != config.Range() { opts = append(opts, config.SetRange(*rng)) } if *distance != config.Distance() { opts = append(opts, config.SetDistance(*distance)) } if *choices != config.Choices() { opts = append(opts, config.SetChoices(*choices)) } // inspect options if *inspectMI != "" { opts = append(opts, config.SetMIMEInfo(*inspectMI)) } if *inspectFDD != "" { opts = append(opts, config.SetLOC(*fdd)) } if *inspectLOC { opts = append(opts, config.SetLOC("")) } if *inspectInclude != "" { opts = append(opts, config.SetLimit(expandSets(*inspectInclude))) } if *inspectExclude != "" { opts = append(opts, config.SetExclude(expandSets(*inspectExclude))) } if *inspectExtend != "" { opts = append(opts, config.SetExtend(expandSets(*inspectExtend))) } if *inspectExtendc != "" { if *inspectExtend == "" { fmt.Println( `roy: warning! Unless the container extension only extends formats defined in the DROID signature file you should also include a regular signature extension (-extend) that includes a FileFormatCollection element describing the new formats.`) } opts = append(opts, config.SetExtendC(expandSets(*inspectExtendc))) } return opts }