// DROID parsing is tested by comparing it against Report parsing
func TestParseDroid(t *testing.T) {
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
d, err := newDroid(config.Droid())
if err != nil {
t.Fatal(err)
}
r, err := newReports(d.IDs(), d.idsPuids())
if err != nil {
t.Fatal(err)
}
dsigs, dpuids, err := d.Signatures()
if err != nil {
t.Fatal(err)
}
rsigs, rpuids, err := r.Signatures()
if err != nil {
t.Fatal(err)
}
if len(dpuids) != len(rpuids) {
t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8])
}
for i, v := range rpuids {
if v != dpuids[i] {
t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i])
}
}
if len(dsigs) != len(rsigs) {
t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs))
}
for i, v := range rsigs {
if !v.Equals(dsigs[i]) {
t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n Droid: %s", rpuids[i], v, dsigs[i])
}
}
}
// UTILS
// Harvest fetches PRONOM reports listed in the DROID file
func Harvest() []error {
d, err := newDroid(config.Droid())
if err != nil {
return []error{err}
}
apply := func(puid string) error {
url, _, _, _ := config.HarvestOptions()
return save(puid, url, config.Reports())
}
return applyAll(5, d.IDs(), apply)
}
func setHarvestOptions() {
if *harvestHome != config.Home() {
config.SetHome(*harvestHome)
}
if *harvestDroid != config.Droid() {
config.SetDroid(*harvestDroid)()
}
if *timeout != htimeout {
config.SetHarvestTimeout(*timeout)
}
if *throttlef > 0 {
config.SetHarvestThrottle(*throttlef)
}
}
// set identifiers joins signatures in the DROID signature file with any extra reports and adds that to the pronom object
func (p *pronom) setParseables() error {
d, err := newDroid(config.Droid())
if err != nil {
return fmt.Errorf("Pronom: error loading Droid file; got %s\nYou must have a Droid file to build a signature", err)
}
// if noreports set
if config.Reports() == "" {
p.Parseable = d
} else { // otherwise build from reports
// get list of puids that applies limit or exclude filters (actual filtering of Parseable delegated to core/identifier)
puids := d.IDs()
if config.HasLimit() {
puids = config.Limit(puids)
} else if config.HasExclude() {
puids = config.Exclude(puids)
}
r, err := newReports(puids, d.idsPuids())
if err != nil {
return fmt.Errorf("Pronom: error loading reports; got %s\nYou must download PRONOM reports to build a signature (unless you use the -noreports flag). You can use `roy harvest` to download reports", err)
}
p.Parseable = r
}
// add extensions
for _, v := range config.Extend() {
e, err := newDroid(v)
if err != nil {
return fmt.Errorf("Pronom: error loading extension file; got %s", err)
}
p.Parseable = identifier.Join(p.Parseable, e)
}
// exclude byte signatures where also have container signatures, unless doubleup set
if !config.DoubleUp() {
p.Parseable = doublesFilter{
config.ExcludeDoubles(p.IDs(), p.c.IDs()),
p.Parseable,
}
}
return nil
}
func getOptions() []config.Option {
if *home != config.Home() {
config.SetHome(*home)
}
if *inspectHome != config.Home() {
config.SetHome(*inspectHome)
}
opts := []config.Option{}
// build options
if *droid != config.Droid() {
opts = append(opts, config.SetDroid(*droid))
}
if *container != config.Container() {
opts = append(opts, config.SetContainer(*container))
}
if *mi != "" {
opts = append(opts, config.SetMIMEInfo(*mi))
}
if *fdd != "" {
opts = append(opts, config.SetLOC(*fdd))
}
if *locfdd {
opts = append(opts, config.SetLOC(""))
}
if *nopronom {
opts = append(opts, config.SetNoPRONOM())
}
if *name != "" {
opts = append(opts, config.SetName(*name))
}
if *details != config.Details() {
opts = append(opts, config.SetDetails(*details))
}
if *extend != "" {
opts = append(opts, config.SetExtend(expandSets(*extend)))
}
if *extendc != "" {
if *extend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element describing the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*extendc)))
}
if *include != "" {
opts = append(opts, config.SetLimit(expandSets(*include)))
}
if *exclude != "" {
opts = append(opts, config.SetExclude(expandSets(*exclude)))
}
if *bof != 0 {
opts = append(opts, config.SetBOF(*bof))
}
if *eof != 0 {
opts = append(opts, config.SetEOF(*eof))
}
if *noeof {
opts = append(opts, config.SetNoEOF())
}
if *multi != "" {
opts = append(opts, config.SetMulti(strings.ToLower(*multi)))
}
if *nocontainer {
opts = append(opts, config.SetNoContainer())
}
if *notext {
opts = append(opts, config.SetNoText())
}
if *noname {
opts = append(opts, config.SetNoName())
}
if *nomime {
opts = append(opts, config.SetNoMIME())
}
if *noxml {
opts = append(opts, config.SetNoXML())
}
if *noriff {
opts = append(opts, config.SetNoRIFF())
}
if *noreports {
opts = append(opts, config.SetNoReports())
}
if *doubleup {
opts = append(opts, config.SetDoubleUp())
}
if *rng != config.Range() {
opts = append(opts, config.SetRange(*rng))
}
if *distance != config.Distance() {
opts = append(opts, config.SetDistance(*distance))
}
if *choices != config.Choices() {
opts = append(opts, config.SetChoices(*choices))
}
// inspect options
if *inspectMI != "" {
opts = append(opts, config.SetMIMEInfo(*inspectMI))
}
if *inspectFDD != "" {
opts = append(opts, config.SetLOC(*fdd))
}
if *inspectLOC {
opts = append(opts, config.SetLOC(""))
}
if *inspectInclude != "" {
opts = append(opts, config.SetLimit(expandSets(*inspectInclude)))
}
if *inspectExclude != "" {
opts = append(opts, config.SetExclude(expandSets(*inspectExclude)))
}
if *inspectExtend != "" {
opts = append(opts, config.SetExtend(expandSets(*inspectExtend)))
}
if *inspectExtendc != "" {
if *inspectExtend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element describing the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*inspectExtendc)))
}
return opts
}
-mi, -loc, -fdd
Specify particular MIME-info or LOC FDD signature files for inspecting
formats or viewing priorities.
-reports
Build from PRONOM reports files (rather than just using the DROID XML
file as input). A bit slower but can be more accurate for a small set
of formats like FLAC.
-home
Use a different siegfried home directory.
`
var (
// BUILD, ADD flag sets
build = flag.NewFlagSet("build | add", flag.ExitOnError)
home = build.String("home", config.Home(), "override the default home directory")
droid = build.String("droid", config.Droid(), "set name/path for DROID signature file")
mi = build.String("mi", "", "set name/path for MIMEInfo signature file")
fdd = build.String("fdd", "", "set name/path for LOC FDD signature file")
locfdd = build.Bool("loc", false, "build a LOC FDD signature file")
nopronom = build.Bool("nopronom", false, "don't include PRONOM sigs with LOC signature file")
container = build.String("container", config.Container(), "set name/path for Droid Container signature file")
name = build.String("name", "", "set identifier name")
details = build.String("details", config.Details(), "set identifier details")
extend = build.String("extend", "", "comma separated list of additional signatures")
extendc = build.String("extendc", "", "comma separated list of additional container signatures")
include = build.String("limit", "", "comma separated list of PRONOM signatures to include")
exclude = build.String("exclude", "", "comma separated list of PRONOM signatures to exclude")
bof = build.Int("bof", 0, "define a maximum BOF offset")
eof = build.Int("eof", 0, "define a maximum EOF offset")
noeof = build.Bool("noeof", false, "ignore EOF segments in signatures")
multi = build.String("multi", "", "control how identifiers treat multiple results")