Esempio n. 1
0
// DROID parsing is tested by comparing it against Report parsing
func TestParseDroid(t *testing.T) {
	config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
	d, err := newDroid(config.Droid())
	if err != nil {
		t.Fatal(err)
	}
	r, err := newReports(d.puids(), d.idsPuids())
	if err != nil {
		t.Fatal(err)
	}
	dsigs, dpuids, err := d.signatures()
	if err != nil {
		t.Fatal(err)
	}
	rsigs, rpuids, err := r.signatures()
	if err != nil {
		t.Fatal(err)
	}
	if len(dpuids) != len(rpuids) {
		t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8])
	}
	for i, v := range rpuids {
		if v != dpuids[i] {
			t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i])
		}
	}
	if len(dsigs) != len(rsigs) {
		t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs))
	}
	for i, v := range rsigs {
		if !v.Equals(dsigs[i]) {
			t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n  Droid: %s", rpuids[i], v, dsigs[i])
		}
	}
}
Esempio n. 2
0
// Harvest fetches PRONOM reports listed in the DROID file
func Harvest() []error {
	d, err := newDroid(config.Droid())
	if err != nil {
		return []error{err}
	}
	apply := func(puid string) error {
		url, _, _ := config.HarvestOptions()
		return save(puid, url, config.Reports())
	}
	return applyAll(5, d.IDs(), apply)
}
Esempio n. 3
0
func setHarvestOptions() {
	if *harvestHome != config.Home() {
		config.SetHome(*harvestHome)
	}
	if *harvestDroid != config.Droid() {
		config.SetDroid(*harvestDroid)()
	}
	if *harvestReports != config.Reports() {
		config.SetReports(*harvestReports)()
	}
	if *timeout != htimeout {
		config.SetHarvestTimeout(*timeout)
	}
}
Esempio n. 4
0
// set parseables joins signatures in the DROID signature file with any extra reports and adds that to the pronom object
func (p *pronom) setParseables() error {
	d, err := newDroid(config.Droid())
	if err != nil {
		return fmt.Errorf("Pronom: error loading Droid file; got %s\nYou must have a Droid file to build a signature", err)
	}
	// if we are just inspecting a single report file
	if config.Inspect() {
		r, err := newReports(config.Limit(d.IDs()), nil)
		if err != nil {
			return fmt.Errorf("Pronom: error loading reports; got %s\nYou must download PRONOM reports to build a signature (unless you use the -noreports flag). You can use `roy harvest` to download reports", err)
		}
		is := infos(r.Infos())
		sigs, puids, err := r.Signatures()
		if err != nil {
			return fmt.Errorf("Pronom: parsing signatures; got %s", err)
		}
		var puid string
		for i, sig := range sigs {
			if puids[i] != puid {
				puid = puids[i]
				fmt.Printf("%s: \n", is[puid].name)
			}
			fmt.Println(sig)
		}
		p.j = r
		return nil
	}
	// apply limit or exclude filters (only one can be applied)
	puids := d.IDs()
	if config.HasLimit() {
		puids = config.Limit(puids)
	} else if config.HasExclude() {
		puids = config.Exclude(puids)
	}
	// if noreports set
	if config.Reports() == "" {
		p.j = d
		// apply filter
		if config.HasLimit() || config.HasExclude() {
			p.j = parseable.Filter(puids, p.j)
		}
	} else { // otherwise build from reports
		r, err := newReports(puids, d.idsPuids())
		if err != nil {
			return fmt.Errorf("Pronom: error loading reports; got %s\nYou must download PRONOM reports to build a signature (unless you use the -noreports flag). You can use `roy harvest` to download reports", err)
		}
		p.j = r
	}
	// exclude byte signatures where also have container signatures, unless doubleup set
	if !config.DoubleUp() {
		p.j = &doublesFilter{
			config.ExcludeDoubles(puids, p.c.Puids()),
			p.j,
		}
	}
	// add extensions
	for _, v := range config.Extend() {
		e, err := newDroid(v)
		if err != nil {
			return fmt.Errorf("Pronom: error loading extension file; got %s", err)
		}
		p.j = parseable.Join(p.j, e)
	}
	// mirror PREV wild segments into EOF if maxBof and maxEOF set
	if config.MaxBOF() > 0 && config.MaxEOF() > 0 {
		p.j = &parseable.Mirror{p.j}
	}
	return nil
}
Esempio n. 5
0
func buildOptions() []config.Option {
	if *home != config.Home() {
		config.SetHome(*home)
	}
	opts := []config.Option{}
	if *droid != config.Droid() {
		opts = append(opts, config.SetDroid(*droid))
	}
	if *container != config.Container() {
		opts = append(opts, config.SetContainer(*container))
	}
	if *reports != config.Reports() {
		opts = append(opts, config.SetReports(*reports))
	}
	if *name != config.Name() {
		opts = append(opts, config.SetName(*name))
	}
	if *details != config.Details() {
		opts = append(opts, config.SetDetails(*details))
	}
	if *extend != "" {
		opts = append(opts, config.SetExtend(expandSets(*extend)))
	}
	if *extendc != "" {
		if *extend == "" {
			fmt.Println(
				`roy: warning! Unless the container extension only extends formats defined in 
the DROID signature file you should also include a regular signature extension 
(-extend) that includes a FileFormatCollection element defining the new formats.`)
		}
		opts = append(opts, config.SetExtendC(expandSets(*extendc)))
	}
	if *include != "" {
		opts = append(opts, config.SetLimit(expandSets(*include)))
	}
	if *exclude != "" {
		opts = append(opts, config.SetExclude(expandSets(*exclude)))
	}
	if *bof != 0 {
		opts = append(opts, config.SetBOF(*bof))
	}
	if *eof != 0 {
		opts = append(opts, config.SetEOF(*eof))
	}
	if *noeof {
		opts = append(opts, config.SetNoEOF())
	}
	if *nopriority {
		opts = append(opts, config.SetNoPriority())
	}
	if *nocontainer {
		opts = append(opts, config.SetNoContainer())
	}
	if *notext {
		opts = append(opts, config.SetNoText())
	}
	if *noext {
		opts = append(opts, config.SetNoExt())
	}
	if *noreports {
		opts = append(opts, config.SetNoReports())
	}
	if *doubleup {
		opts = append(opts, config.SetDoubleUp())
	}
	if *rng != config.Range() {
		opts = append(opts, config.SetRange(*rng))
	}
	if *distance != config.Distance() {
		opts = append(opts, config.SetDistance(*distance))
	}
	if *choices != config.Choices() {
		opts = append(opts, config.SetChoices(*choices))
	}
	return opts
}
Esempio n. 6
0
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/richardlehane/siegfried"
	"github.com/richardlehane/siegfried/config"
	"github.com/richardlehane/siegfried/pkg/pronom"
)

var (
	// BUILD, ADD flag sets
	build       = flag.NewFlagSet("build | add", flag.ExitOnError)
	home        = build.String("home", config.Home(), "override the default home directory")
	droid       = build.String("droid", config.Droid(), "set name/path for DROID signature file")
	container   = build.String("container", config.Container(), "set name/path for Droid Container signature file")
	reports     = build.String("reports", config.Reports(), "set path for PRONOM reports directory")
	name        = build.String("name", config.Name(), "set identifier name")
	details     = build.String("details", config.Details(), "set identifier details")
	extend      = build.String("extend", "", "comma separated list of additional signatures")
	extendc     = build.String("extendc", "", "comma separated list of additional container signatures")
	include     = build.String("limit", "", "comma separated list of PRONOM signatures to include")
	exclude     = build.String("exclude", "", "comma separated list of PRONOM signatures to exclude")
	bof         = build.Int("bof", 0, "define a maximum BOF offset")
	eof         = build.Int("eof", 0, "define a maximum EOF offset")
	noeof       = build.Bool("noeof", false, "ignore EOF segments in signatures")
	nopriority  = build.Bool("nopriority", false, "ignore priority rules when recording results")
	nocontainer = build.Bool("nocontainer", false, "skip container signatures")
	notext      = build.Bool("notext", false, "skip text matcher")
	noext       = build.Bool("noext", false, "skip extension matcher")
Esempio n. 7
0
func buildOptions() []config.Option {
	if *home != config.Home() {
		config.SetHome(*home)
	}
	opts := []config.Option{}
	if *droid != config.Droid() {
		opts = append(opts, config.SetDroid(*droid))
	}
	if *container != config.Container() {
		opts = append(opts, config.SetContainer(*container))
	}
	if *reports != config.Reports() {
		opts = append(opts, config.SetReports(*reports))
	}
	if *name != config.Name() {
		opts = append(opts, config.SetName(*name))
	}
	if *details != config.Details() {
		opts = append(opts, config.SetDetails(*details))
	}
	if *extend != "" {
		opts = append(opts, config.SetExtend(expandSets(*extend)))
	}
	if *extendc != "" {
		opts = append(opts, config.SetExtendC(expandSets(*extendc)))
	}
	if *include != "" {
		opts = append(opts, config.SetLimit(expandSets(*include)))
	}
	if *exclude != "" {
		opts = append(opts, config.SetExclude(expandSets(*exclude)))
	}
	if *bof != 0 {
		opts = append(opts, config.SetBOF(*bof))
	}
	if *eof != 0 {
		opts = append(opts, config.SetEOF(*eof))
	}
	if *noeof {
		opts = append(opts, config.SetNoEOF())
	}
	if *nopriority {
		opts = append(opts, config.SetNoPriority())
	}
	if *nocontainer {
		opts = append(opts, config.SetNoContainer())
	}
	if *notext {
		opts = append(opts, config.SetNoText())
	}
	if *noreports {
		opts = append(opts, config.SetNoReports())
	}
	if *rng != config.Range() {
		opts = append(opts, config.SetRange(*rng))
	}
	if *distance != config.Distance() {
		opts = append(opts, config.SetDistance(*distance))
	}
	if *choices != config.Choices() {
		opts = append(opts, config.SetChoices(*choices))
	}
	return opts
}