// DROID parsing is tested by comparing it against Report parsing
func TestParseDroid(t *testing.T) {
config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
d, err := newDroid(config.Droid())
if err != nil {
t.Fatal(err)
}
r, err := newReports(d.puids(), d.idsPuids())
if err != nil {
t.Fatal(err)
}
dsigs, dpuids, err := d.signatures()
if err != nil {
t.Fatal(err)
}
rsigs, rpuids, err := r.signatures()
if err != nil {
t.Fatal(err)
}
if len(dpuids) != len(rpuids) {
t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8])
}
for i, v := range rpuids {
if v != dpuids[i] {
t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i])
}
}
if len(dsigs) != len(rsigs) {
t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs))
}
for i, v := range rsigs {
if !v.Equals(dsigs[i]) {
t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n Droid: %s", rpuids[i], v, dsigs[i])
}
}
}
// Harvest fetches PRONOM reports listed in the DROID file
func Harvest() []error {
d, err := newDroid(config.Droid())
if err != nil {
return []error{err}
}
apply := func(puid string) error {
url, _, _ := config.HarvestOptions()
return save(puid, url, config.Reports())
}
return applyAll(5, d.IDs(), apply)
}
func setHarvestOptions() {
if *harvestHome != config.Home() {
config.SetHome(*harvestHome)
}
if *harvestDroid != config.Droid() {
config.SetDroid(*harvestDroid)()
}
if *harvestReports != config.Reports() {
config.SetReports(*harvestReports)()
}
if *timeout != htimeout {
config.SetHarvestTimeout(*timeout)
}
}
// set parseables joins signatures in the DROID signature file with any extra reports and adds that to the pronom object
func (p *pronom) setParseables() error {
d, err := newDroid(config.Droid())
if err != nil {
return fmt.Errorf("Pronom: error loading Droid file; got %s\nYou must have a Droid file to build a signature", err)
}
// if we are just inspecting a single report file
if config.Inspect() {
r, err := newReports(config.Limit(d.IDs()), nil)
if err != nil {
return fmt.Errorf("Pronom: error loading reports; got %s\nYou must download PRONOM reports to build a signature (unless you use the -noreports flag). You can use `roy harvest` to download reports", err)
}
is := infos(r.Infos())
sigs, puids, err := r.Signatures()
if err != nil {
return fmt.Errorf("Pronom: parsing signatures; got %s", err)
}
var puid string
for i, sig := range sigs {
if puids[i] != puid {
puid = puids[i]
fmt.Printf("%s: \n", is[puid].name)
}
fmt.Println(sig)
}
p.j = r
return nil
}
// apply limit or exclude filters (only one can be applied)
puids := d.IDs()
if config.HasLimit() {
puids = config.Limit(puids)
} else if config.HasExclude() {
puids = config.Exclude(puids)
}
// if noreports set
if config.Reports() == "" {
p.j = d
// apply filter
if config.HasLimit() || config.HasExclude() {
p.j = parseable.Filter(puids, p.j)
}
} else { // otherwise build from reports
r, err := newReports(puids, d.idsPuids())
if err != nil {
return fmt.Errorf("Pronom: error loading reports; got %s\nYou must download PRONOM reports to build a signature (unless you use the -noreports flag). You can use `roy harvest` to download reports", err)
}
p.j = r
}
// exclude byte signatures where also have container signatures, unless doubleup set
if !config.DoubleUp() {
p.j = &doublesFilter{
config.ExcludeDoubles(puids, p.c.Puids()),
p.j,
}
}
// add extensions
for _, v := range config.Extend() {
e, err := newDroid(v)
if err != nil {
return fmt.Errorf("Pronom: error loading extension file; got %s", err)
}
p.j = parseable.Join(p.j, e)
}
// mirror PREV wild segments into EOF if maxBof and maxEOF set
if config.MaxBOF() > 0 && config.MaxEOF() > 0 {
p.j = &parseable.Mirror{p.j}
}
return nil
}
func buildOptions() []config.Option {
if *home != config.Home() {
config.SetHome(*home)
}
opts := []config.Option{}
if *droid != config.Droid() {
opts = append(opts, config.SetDroid(*droid))
}
if *container != config.Container() {
opts = append(opts, config.SetContainer(*container))
}
if *reports != config.Reports() {
opts = append(opts, config.SetReports(*reports))
}
if *name != config.Name() {
opts = append(opts, config.SetName(*name))
}
if *details != config.Details() {
opts = append(opts, config.SetDetails(*details))
}
if *extend != "" {
opts = append(opts, config.SetExtend(expandSets(*extend)))
}
if *extendc != "" {
if *extend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element defining the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*extendc)))
}
if *include != "" {
opts = append(opts, config.SetLimit(expandSets(*include)))
}
if *exclude != "" {
opts = append(opts, config.SetExclude(expandSets(*exclude)))
}
if *bof != 0 {
opts = append(opts, config.SetBOF(*bof))
}
if *eof != 0 {
opts = append(opts, config.SetEOF(*eof))
}
if *noeof {
opts = append(opts, config.SetNoEOF())
}
if *nopriority {
opts = append(opts, config.SetNoPriority())
}
if *nocontainer {
opts = append(opts, config.SetNoContainer())
}
if *notext {
opts = append(opts, config.SetNoText())
}
if *noext {
opts = append(opts, config.SetNoExt())
}
if *noreports {
opts = append(opts, config.SetNoReports())
}
if *doubleup {
opts = append(opts, config.SetDoubleUp())
}
if *rng != config.Range() {
opts = append(opts, config.SetRange(*rng))
}
if *distance != config.Distance() {
opts = append(opts, config.SetDistance(*distance))
}
if *choices != config.Choices() {
opts = append(opts, config.SetChoices(*choices))
}
return opts
}
"log"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/richardlehane/siegfried"
"github.com/richardlehane/siegfried/config"
"github.com/richardlehane/siegfried/pkg/pronom"
)
var (
// BUILD, ADD flag sets
build = flag.NewFlagSet("build | add", flag.ExitOnError)
home = build.String("home", config.Home(), "override the default home directory")
droid = build.String("droid", config.Droid(), "set name/path for DROID signature file")
container = build.String("container", config.Container(), "set name/path for Droid Container signature file")
reports = build.String("reports", config.Reports(), "set path for PRONOM reports directory")
name = build.String("name", config.Name(), "set identifier name")
details = build.String("details", config.Details(), "set identifier details")
extend = build.String("extend", "", "comma separated list of additional signatures")
extendc = build.String("extendc", "", "comma separated list of additional container signatures")
include = build.String("limit", "", "comma separated list of PRONOM signatures to include")
exclude = build.String("exclude", "", "comma separated list of PRONOM signatures to exclude")
bof = build.Int("bof", 0, "define a maximum BOF offset")
eof = build.Int("eof", 0, "define a maximum EOF offset")
noeof = build.Bool("noeof", false, "ignore EOF segments in signatures")
nopriority = build.Bool("nopriority", false, "ignore priority rules when recording results")
nocontainer = build.Bool("nocontainer", false, "skip container signatures")
notext = build.Bool("notext", false, "skip text matcher")
noext = build.Bool("noext", false, "skip extension matcher")
func buildOptions() []config.Option {
if *home != config.Home() {
config.SetHome(*home)
}
opts := []config.Option{}
if *droid != config.Droid() {
opts = append(opts, config.SetDroid(*droid))
}
if *container != config.Container() {
opts = append(opts, config.SetContainer(*container))
}
if *reports != config.Reports() {
opts = append(opts, config.SetReports(*reports))
}
if *name != config.Name() {
opts = append(opts, config.SetName(*name))
}
if *details != config.Details() {
opts = append(opts, config.SetDetails(*details))
}
if *extend != "" {
opts = append(opts, config.SetExtend(expandSets(*extend)))
}
if *extendc != "" {
opts = append(opts, config.SetExtendC(expandSets(*extendc)))
}
if *include != "" {
opts = append(opts, config.SetLimit(expandSets(*include)))
}
if *exclude != "" {
opts = append(opts, config.SetExclude(expandSets(*exclude)))
}
if *bof != 0 {
opts = append(opts, config.SetBOF(*bof))
}
if *eof != 0 {
opts = append(opts, config.SetEOF(*eof))
}
if *noeof {
opts = append(opts, config.SetNoEOF())
}
if *nopriority {
opts = append(opts, config.SetNoPriority())
}
if *nocontainer {
opts = append(opts, config.SetNoContainer())
}
if *notext {
opts = append(opts, config.SetNoText())
}
if *noreports {
opts = append(opts, config.SetNoReports())
}
if *rng != config.Range() {
opts = append(opts, config.SetRange(*rng))
}
if *distance != config.Distance() {
opts = append(opts, config.SetDistance(*distance))
}
if *choices != config.Choices() {
opts = append(opts, config.SetChoices(*choices))
}
return opts
}