// DROID parsing is tested by comparing it against Report parsing func TestParseDroid(t *testing.T) { config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data")) d, err := newDroid(config.Droid()) if err != nil { t.Fatal(err) } r, err := newReports(d.IDs(), d.idsPuids()) if err != nil { t.Fatal(err) } dsigs, dpuids, err := d.Signatures() if err != nil { t.Fatal(err) } rsigs, rpuids, err := r.Signatures() if err != nil { t.Fatal(err) } if len(dpuids) != len(rpuids) { t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8]) } for i, v := range rpuids { if v != dpuids[i] { t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i]) } } if len(dsigs) != len(rsigs) { t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs)) } for i, v := range rsigs { if !v.Equals(dsigs[i]) { t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n Droid: %s", rpuids[i], v, dsigs[i]) } } }
// UTILS // Harvest fetches PRONOM reports listed in the DROID file func Harvest() []error { d, err := newDroid(config.Droid()) if err != nil { return []error{err} } apply := func(puid string) error { url, _, _, _ := config.HarvestOptions() return save(puid, url, config.Reports()) } return applyAll(5, d.IDs(), apply) }
func setHarvestOptions() { if *harvestHome != config.Home() { config.SetHome(*harvestHome) } if *harvestDroid != config.Droid() { config.SetDroid(*harvestDroid)() } if *timeout != htimeout { config.SetHarvestTimeout(*timeout) } if *throttlef > 0 { config.SetHarvestThrottle(*throttlef) } }
// set identifiers joins signatures in the DROID signature file with any extra reports and adds that to the pronom object func (p *pronom) setParseables() error { d, err := newDroid(config.Droid()) if err != nil { return fmt.Errorf("Pronom: error loading Droid file; got %s\nYou must have a Droid file to build a signature", err) } // if noreports set if config.Reports() == "" { p.Parseable = d } else { // otherwise build from reports // get list of puids that applies limit or exclude filters (actual filtering of Parseable delegated to core/identifier) puids := d.IDs() if config.HasLimit() { puids = config.Limit(puids) } else if config.HasExclude() { puids = config.Exclude(puids) } r, err := newReports(puids, d.idsPuids()) if err != nil { return fmt.Errorf("Pronom: error loading reports; got %s\nYou must download PRONOM reports to build a signature (unless you use the -noreports flag). You can use `roy harvest` to download reports", err) } p.Parseable = r } // add extensions for _, v := range config.Extend() { e, err := newDroid(v) if err != nil { return fmt.Errorf("Pronom: error loading extension file; got %s", err) } p.Parseable = identifier.Join(p.Parseable, e) } // exclude byte signatures where also have container signatures, unless doubleup set if !config.DoubleUp() { p.Parseable = doublesFilter{ config.ExcludeDoubles(p.IDs(), p.c.IDs()), p.Parseable, } } return nil }
func getOptions() []config.Option { if *home != config.Home() { config.SetHome(*home) } if *inspectHome != config.Home() { config.SetHome(*inspectHome) } opts := []config.Option{} // build options if *droid != config.Droid() { opts = append(opts, config.SetDroid(*droid)) } if *container != config.Container() { opts = append(opts, config.SetContainer(*container)) } if *mi != "" { opts = append(opts, config.SetMIMEInfo(*mi)) } if *fdd != "" { opts = append(opts, config.SetLOC(*fdd)) } if *locfdd { opts = append(opts, config.SetLOC("")) } if *nopronom { opts = append(opts, config.SetNoPRONOM()) } if *name != "" { opts = append(opts, config.SetName(*name)) } if *details != config.Details() { opts = append(opts, config.SetDetails(*details)) } if *extend != "" { opts = append(opts, config.SetExtend(expandSets(*extend))) } if *extendc != "" { if *extend == "" { fmt.Println( `roy: warning! Unless the container extension only extends formats defined in the DROID signature file you should also include a regular signature extension (-extend) that includes a FileFormatCollection element describing the new formats.`) } opts = append(opts, config.SetExtendC(expandSets(*extendc))) } if *include != "" { opts = append(opts, config.SetLimit(expandSets(*include))) } if *exclude != "" { opts = append(opts, config.SetExclude(expandSets(*exclude))) } if *bof != 0 { opts = append(opts, config.SetBOF(*bof)) } if *eof != 0 { opts = append(opts, config.SetEOF(*eof)) } if *noeof { opts = append(opts, config.SetNoEOF()) } if *multi != "" { opts = append(opts, config.SetMulti(strings.ToLower(*multi))) } if *nocontainer { opts = append(opts, config.SetNoContainer()) } if *notext { opts = append(opts, config.SetNoText()) } if *noname { opts = append(opts, config.SetNoName()) } if *nomime { opts = append(opts, config.SetNoMIME()) } if *noxml { opts = append(opts, config.SetNoXML()) } if *noriff { opts = append(opts, config.SetNoRIFF()) } if *noreports { opts = append(opts, config.SetNoReports()) } if *doubleup { opts = append(opts, config.SetDoubleUp()) } if *rng != config.Range() { opts = append(opts, config.SetRange(*rng)) } if *distance != config.Distance() { opts = append(opts, config.SetDistance(*distance)) } if *choices != config.Choices() { opts = append(opts, config.SetChoices(*choices)) } // inspect options if *inspectMI != "" { opts = append(opts, config.SetMIMEInfo(*inspectMI)) } if *inspectFDD != "" { opts = append(opts, config.SetLOC(*fdd)) } if *inspectLOC { opts = append(opts, config.SetLOC("")) } if *inspectInclude != "" { opts = append(opts, config.SetLimit(expandSets(*inspectInclude))) } if *inspectExclude != "" { opts = append(opts, config.SetExclude(expandSets(*inspectExclude))) } if *inspectExtend != "" { opts = append(opts, config.SetExtend(expandSets(*inspectExtend))) } if *inspectExtendc != "" { if *inspectExtend == "" { fmt.Println( `roy: warning! Unless the container extension only extends formats defined in the DROID signature file you should also include a regular signature extension (-extend) that includes a FileFormatCollection element describing the new formats.`) } opts = append(opts, config.SetExtendC(expandSets(*inspectExtendc))) } return opts }
-mi, -loc, -fdd Specify particular MIME-info or LOC FDD signature files for inspecting formats or viewing priorities. -reports Build from PRONOM reports files (rather than just using the DROID XML file as input). A bit slower but can be more accurate for a small set of formats like FLAC. -home Use a different siegfried home directory. ` var ( // BUILD, ADD flag sets build = flag.NewFlagSet("build | add", flag.ExitOnError) home = build.String("home", config.Home(), "override the default home directory") droid = build.String("droid", config.Droid(), "set name/path for DROID signature file") mi = build.String("mi", "", "set name/path for MIMEInfo signature file") fdd = build.String("fdd", "", "set name/path for LOC FDD signature file") locfdd = build.Bool("loc", false, "build a LOC FDD signature file") nopronom = build.Bool("nopronom", false, "don't include PRONOM sigs with LOC signature file") container = build.String("container", config.Container(), "set name/path for Droid Container signature file") name = build.String("name", "", "set identifier name") details = build.String("details", config.Details(), "set identifier details") extend = build.String("extend", "", "comma separated list of additional signatures") extendc = build.String("extendc", "", "comma separated list of additional container signatures") include = build.String("limit", "", "comma separated list of PRONOM signatures to include") exclude = build.String("exclude", "", "comma separated list of PRONOM signatures to exclude") bof = build.Int("bof", 0, "define a maximum BOF offset") eof = build.Int("eof", 0, "define a maximum EOF offset") noeof = build.Bool("noeof", false, "ignore EOF segments in signatures") multi = build.String("multi", "", "control how identifiers treat multiple results")