Example #1
0
func TestNew(t *testing.T) {
	config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
	mi, err := newMIMEInfo()
	if err != nil {
		t.Error(err)
	}
	tpmap := make(map[string]struct{})
	for _, v := range mi {
		//fmt.Println(v)
		//if len(v.Magic) > 1 {
		//	fmt.Printf("Multiple magics (%d): %s\n", len(v.Magic), v.MIME)
		//}
		for _, c := range v.Magic {
			for _, d := range c.Matches {
				tpmap[d.Typ] = struct{}{}
				if len(d.Mask) > 0 {
					if d.Typ == "string" {
						fmt.Println("MAGIC: " + d.Value)
					} else {
						fmt.Println("Type: " + d.Typ)
					}
					fmt.Println("MASK: " + d.Mask)
				}
			}
		}
	}
	/*
		for k, _ := range tpmap {
			fmt.Println(k)
		}
	*/
	if len(mi) != 1495 {
		t.Errorf("expecting %d MIMEInfos, got %d", 1495, len(mi))
	}
}
Example #2
0
func TestNew(t *testing.T) {
	config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
	_, err := newPronom()
	if err != nil {
		t.Error(err)
	}
}
Example #3
0
// DROID parsing is tested by comparing it against Report parsing
func TestParseDroid(t *testing.T) {
	config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
	d, err := newDroid(config.Droid())
	if err != nil {
		t.Fatal(err)
	}
	r, err := newReports(d.puids(), d.idsPuids())
	if err != nil {
		t.Fatal(err)
	}
	dsigs, dpuids, err := d.signatures()
	if err != nil {
		t.Fatal(err)
	}
	rsigs, rpuids, err := r.signatures()
	if err != nil {
		t.Fatal(err)
	}
	if len(dpuids) != len(rpuids) {
		t.Errorf("Parse Droid: Expecting length of reports and droid to be same, got %d, %d, %s", len(rpuids), len(dpuids), dpuids[len(dpuids)-8])
	}
	for i, v := range rpuids {
		if v != dpuids[i] {
			t.Errorf("Parse Droid: Expecting slices of puids to be identical but at index %d, got %s for reports and %s for droid", i, v, dpuids[i])
		}
	}
	if len(dsigs) != len(rsigs) {
		t.Errorf("Parse Droid: Expecting sig length of reports and droid to be same, got %d, %d", len(rsigs), len(dsigs))
	}
	for i, v := range rsigs {
		if !v.Equals(dsigs[i]) {
			t.Errorf("Parse Droid: signatures for %s are not equal:\nReports: %s\n  Droid: %s", rpuids[i], v, dsigs[i])
		}
	}
}
Example #4
0
func setInspectOptions() {
	if *inspectHome != config.Home() {
		config.SetHome(*inspectHome)
	}
	if *inspectReports != config.Reports() {
		config.SetReports(*inspectReports)()
	}
}
Example #5
0
func TestLoad(t *testing.T) {
	s := New()
	config.SetHome("./cmd/roy/data")
	p, err := pronom.New()
	if err != nil {
		t.Fatal(err)
	}
	err = s.Add(p)
	if err != nil {
		t.Fatal(err)
	}
}
Example #6
0
func TestNew(t *testing.T) {
	config.SetHome(filepath.Join("..", "..", "cmd", "roy", "data"))
	mi, err := newMIMEInfo()
	if err != nil {
		t.Error(err)
	}
	for _, v := range mi {
		fmt.Println(v)
	}
	if len(mi) != 1495 {
		t.Errorf("expecting %d MIMEInfos, got %d", 1495, len(mi))
	}
}
Example #7
0
func setHarvestOptions() {
	if *harvestHome != config.Home() {
		config.SetHome(*harvestHome)
	}
	if *harvestDroid != config.Droid() {
		config.SetDroid(*harvestDroid)()
	}
	if *harvestReports != config.Reports() {
		config.SetReports(*harvestReports)()
	}
	if *timeout != htimeout {
		config.SetHarvestTimeout(*timeout)
	}
}
Example #8
0
func setup(opts ...config.Option) error {
	if opts == nil && s != nil {
		return nil
	}
	var err error
	s = siegfried.New()
	config.SetHome(*testhome)
	opts = append(opts, config.SetDoubleUp())
	p, err := pronom.New(opts...)
	if err != nil {
		return err
	}
	return s.Add(p)
}
Example #9
0
func TestMakeGob(t *testing.T) {
	s := siegfried.New()
	config.SetHome(*testhome)
	p, err := pronom.New()
	if err != nil {
		t.Fatal(err)
	}
	err = s.Add(p)
	if err != nil {
		t.Fatal(err)
	}
	sigs := filepath.Join("data", config.SignatureBase())
	err = s.Save(sigs)
	if err != nil {
		t.Fatal(err)
	}
}
Example #10
0
func TestMakeArchivematica(t *testing.T) {
	s := siegfried.New()
	config.SetHome(*testhome)
	p, err := pronom.New(
		config.SetName("archivematica"),
		config.SetExtend(expandSets("archivematica-fmt2.xml,archivematica-fmt3.xml,archivematica-fmt4.xml,archivematica-fmt5.xml")))
	if err != nil {
		t.Fatal(err)
	}
	err = s.Add(p)
	if err != nil {
		t.Fatal(err)
	}
	sigs := filepath.Join("data", "archivematica.sig")
	err = s.Save(sigs)
	if err != nil {
		t.Fatal(err)
	}
}
Example #11
0
func TestSets(t *testing.T) {
	config.SetHome(*testhome)
	list := "fmt/1,fmt/2,@pdfa,x-fmt/19"
	expect := "fmt/1,fmt/2,fmt/95,fmt/354,fmt/476,fmt/477,fmt/478,fmt/479,fmt/480,fmt/481,x-fmt/19"
	res := strings.Join(expandSets(list), ",")
	if res != expect {
		t.Errorf("expecting %s, got %s", expect, res)
	}
	pdfs := strings.Join(expandSets("@pdf"), ",")
	expect = "fmt/14,fmt/15,fmt/16,fmt/17,fmt/18,fmt/19,fmt/20,fmt/95,fmt/144,fmt/145,fmt/146,fmt/147,fmt/148,fmt/157,fmt/158,fmt/276,fmt/354,fmt/476,fmt/477,fmt/478,fmt/479,fmt/480,fmt/481,fmt/488,fmt/489,fmt/490,fmt/491,fmt/492,fmt/493"
	if pdfs != expect {
		t.Errorf("expecting %s, got %s", expect, pdfs)
	}
	compression := strings.Join(expandSets("@compression"), ",")
	expect = "fmt/626,x-fmt/266,x-fmt/267,x-fmt/268"
	if compression != expect {
		t.Errorf("expecting %s, got %s", expect, compression)
	}
}
Example #12
0
func setup() error {
	var err error
	config.SetHome(*testhome)
	s, err = siegfried.Load(config.Signature())
	return err
}
Example #13
0
func buildOptions() []config.Option {
	if *home != config.Home() {
		config.SetHome(*home)
	}
	opts := []config.Option{}
	if *droid != config.Droid() {
		opts = append(opts, config.SetDroid(*droid))
	}
	if *container != config.Container() {
		opts = append(opts, config.SetContainer(*container))
	}
	if *reports != config.Reports() {
		opts = append(opts, config.SetReports(*reports))
	}
	if *name != config.Name() {
		opts = append(opts, config.SetName(*name))
	}
	if *details != config.Details() {
		opts = append(opts, config.SetDetails(*details))
	}
	if *extend != "" {
		opts = append(opts, config.SetExtend(expandSets(*extend)))
	}
	if *extendc != "" {
		if *extend == "" {
			fmt.Println(
				`roy: warning! Unless the container extension only extends formats defined in 
the DROID signature file you should also include a regular signature extension 
(-extend) that includes a FileFormatCollection element defining the new formats.`)
		}
		opts = append(opts, config.SetExtendC(expandSets(*extendc)))
	}
	if *include != "" {
		opts = append(opts, config.SetLimit(expandSets(*include)))
	}
	if *exclude != "" {
		opts = append(opts, config.SetExclude(expandSets(*exclude)))
	}
	if *bof != 0 {
		opts = append(opts, config.SetBOF(*bof))
	}
	if *eof != 0 {
		opts = append(opts, config.SetEOF(*eof))
	}
	if *noeof {
		opts = append(opts, config.SetNoEOF())
	}
	if *nopriority {
		opts = append(opts, config.SetNoPriority())
	}
	if *nocontainer {
		opts = append(opts, config.SetNoContainer())
	}
	if *notext {
		opts = append(opts, config.SetNoText())
	}
	if *noext {
		opts = append(opts, config.SetNoExt())
	}
	if *noreports {
		opts = append(opts, config.SetNoReports())
	}
	if *doubleup {
		opts = append(opts, config.SetDoubleUp())
	}
	if *rng != config.Range() {
		opts = append(opts, config.SetRange(*rng))
	}
	if *distance != config.Distance() {
		opts = append(opts, config.SetDistance(*distance))
	}
	if *choices != config.Choices() {
		opts = append(opts, config.SetChoices(*choices))
	}
	return opts
}
Example #14
0
func main() {

	flag.Parse()

	/*//UNCOMMENT TO RUN PROFILER
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()*/

	if *home != config.Home() {
		config.SetHome(*home)
	}

	if *sig != config.SignatureBase() {
		config.SetSignature(*sig)
	}

	if *version {
		version := config.Version()
		fmt.Printf("siegfried version: %d.%d.%d\n", version[0], version[1], version[2])
		return
	}

	if *update {
		msg, err := updateSigs()
		if err != nil {
			log.Fatalf("Error: failed to update signature file, %v", err)
		}
		fmt.Println(msg)
		return
	}

	// during parallel scanning or in server mode, unsafe to access the last read buffer - so can't unzip or hash
	if *multi > 1 || *serve != "" {
		if *archive {
			log.Fatalln("Error: cannot scan archive formats when running in parallel mode")
		}
		if *hashf != "" {
			log.Fatalln("Error: cannot calculate file checksum when running in parallel mode")
		}
	}

	if err := setHash(); err != nil {
		log.Fatal(err)
	}

	if *serve != "" || *fprflag {
		s, err := siegfried.Load(config.Signature())
		if err != nil {
			log.Fatalf("Error: error loading signature file, got: %v", err)

		}
		if *serve != "" {
			log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
			listen(*serve, s)
			return
		}
		log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
		serveFpr(config.Fpr(), s)
		return
	}

	if flag.NArg() != 1 {
		log.Fatalln("Error: expecting a single file or directory argument")
	}

	s, err := siegfried.Load(config.Signature())
	if err != nil {
		log.Fatalf("Error: error loading signature file, got: %v", err)

	}

	var w writer
	switch {
	case *debug:
		config.SetDebug()
		w = debugWriter{}
	case *slow:
		config.SetSlow()
		w = &slowWriter{os.Stdout}
	case *csvo:
		w = newCSV(os.Stdout)
	case *jsono:
		w = newJSON(os.Stdout)
	case *droido:
		w = newDroid(os.Stdout)
	case *knowno:
		w = &knownWriter{true, os.Stdout}
	case *unknowno:
		w = &knownWriter{false, os.Stdout}
	default:
		w = newYAML(os.Stdout)
	}

	// support reading list files from stdin
	if flag.Arg(0) == "-" {
		w.writeHead(s)
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			info, err := os.Stat(scanner.Text())
			if err != nil || info.IsDir() {
				w.writeFile(scanner.Text(), 0, "", nil, fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil)
			} else {
				identifyFile(w, s, scanner.Text(), info.Size(), info.ModTime().Format(time.RFC3339))
			}
		}
		w.writeTail()
		os.Exit(0)
	}

	info, err := os.Stat(flag.Arg(0))
	if err != nil {
		log.Fatalf("Error: error getting info for %v, got: %v", flag.Arg(0), err)
	}

	if info.IsDir() {
		if config.Debug() {
			log.Fatalln("Error: when scanning in debug mode, give a file rather than a directory argument")
		}
		w.writeHead(s)
		if *multi > 16 {
			*multi = 16
		}
		if *multi > 1 {
			multiIdentifyP(w, s, flag.Arg(0), *nr)
		} else {
			multiIdentifyS(w, s, flag.Arg(0), *nr)
		}
		w.writeTail()
		os.Exit(0)
	}

	w.writeHead(s)
	identifyFile(w, s, flag.Arg(0), info.Size(), info.ModTime().Format(time.RFC3339))
	w.writeTail()
	os.Exit(0)
}
Example #15
0
func buildOptions() []config.Option {
	if *home != config.Home() {
		config.SetHome(*home)
	}
	opts := []config.Option{}
	if *droid != config.Droid() {
		opts = append(opts, config.SetDroid(*droid))
	}
	if *container != config.Container() {
		opts = append(opts, config.SetContainer(*container))
	}
	if *reports != config.Reports() {
		opts = append(opts, config.SetReports(*reports))
	}
	if *name != config.Name() {
		opts = append(opts, config.SetName(*name))
	}
	if *details != config.Details() {
		opts = append(opts, config.SetDetails(*details))
	}
	if *extend != "" {
		opts = append(opts, config.SetExtend(expandSets(*extend)))
	}
	if *extendc != "" {
		opts = append(opts, config.SetExtendC(expandSets(*extendc)))
	}
	if *include != "" {
		opts = append(opts, config.SetLimit(expandSets(*include)))
	}
	if *exclude != "" {
		opts = append(opts, config.SetExclude(expandSets(*exclude)))
	}
	if *bof != 0 {
		opts = append(opts, config.SetBOF(*bof))
	}
	if *eof != 0 {
		opts = append(opts, config.SetEOF(*eof))
	}
	if *noeof {
		opts = append(opts, config.SetNoEOF())
	}
	if *nopriority {
		opts = append(opts, config.SetNoPriority())
	}
	if *nocontainer {
		opts = append(opts, config.SetNoContainer())
	}
	if *notext {
		opts = append(opts, config.SetNoText())
	}
	if *noreports {
		opts = append(opts, config.SetNoReports())
	}
	if *rng != config.Range() {
		opts = append(opts, config.SetRange(*rng))
	}
	if *distance != config.Distance() {
		opts = append(opts, config.SetDistance(*distance))
	}
	if *choices != config.Choices() {
		opts = append(opts, config.SetChoices(*choices))
	}
	return opts
}
Example #16
0
func main() {

	flag.Parse()

	/*//UNCOMMENT TO RUN PROFILER
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()*/

	if *version {
		version := config.Version()
		fmt.Printf("siegfried %d.%d.%d\n", version[0], version[1], version[2])
		s, err := siegfried.Load(config.Signature())
		if err != nil {
			fmt.Println(err)
			return
		}
		fmt.Print(s)
		return
	}

	if *home != config.Home() {
		config.SetHome(*home)
	}

	if *sig != config.SignatureBase() {
		config.SetSignature(*sig)
	}

	if *update {
		msg, err := updateSigs()
		if err != nil {
			log.Fatalf("[FATAL] failed to update signature file, %v", err)
		}
		fmt.Println(msg)
		return
	}

	// during parallel scanning or in server mode, unsafe to access the last read buffer - so can't unzip or hash
	if *multi > 1 || *serve != "" {
		if *archive {
			log.Fatalln("[FATAL] cannot scan archive formats when running in parallel or server mode")
		}
		if *hashf != "" {
			log.Fatalln("[FATAL] cannot calculate file checksum when running in parallel or server mode")
		}
	}

	if *logf != "" {
		if *multi > 1 && *logf != "error" {
			log.Fatalln("[FATAL] cannot log in parallel mode")
		}
		if err := newLogger(*logf); err != nil {
			log.Fatalln(err)
		}
	}

	if err := setHash(); err != nil {
		log.Fatal(err)
	}

	if *serve != "" || *fprflag {
		s, err := siegfried.Load(config.Signature())
		if err != nil {
			log.Fatalf("[FATAL] error loading signature file, got: %v", err)
		}
		if *serve != "" {
			log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
			listen(*serve, s)
			return
		}
		log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
		serveFpr(config.Fpr(), s)
		return
	}

	if flag.NArg() != 1 {
		log.Fatalln("[FATAL] expecting a single file or directory argument")
	}

	s, err := siegfried.Load(config.Signature())
	if err != nil {
		log.Fatalf("[FATAL] error loading signature file, got: %v", err)
	}

	var w writer
	switch {
	case *csvo:
		w = newCSV(os.Stdout)
	case *jsono:
		w = newJSON(os.Stdout)
	case *droido:
		w = newDroid(os.Stdout)
	default:
		w = newYAML(os.Stdout)
	}

	if lg != nil && lg.w == os.Stdout {
		w = logWriter{}
	}

	// support reading list files from stdin
	if flag.Arg(0) == "-" {
		w.writeHead(s)
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			info, err := os.Stat(scanner.Text())
			if err != nil {
				info, err = retryStat(scanner.Text(), err)
			}
			if err != nil || info.IsDir() {
				writeError(w, scanner.Text(), 0, "", fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err))
			} else {
				identifyFile(w, s, scanner.Text(), info.Size(), info.ModTime().Format(time.RFC3339))
			}
		}
		w.writeTail()
		lg.printElapsed()
		os.Exit(0)
	}

	info, err := os.Stat(flag.Arg(0))
	if err != nil {
		info, err = retryStat(flag.Arg(0), err)
		if err != nil {
			log.Fatalf("[FATAL] cannot get info for %v, got: %v", flag.Arg(0), err)
		}
	}

	if info.IsDir() {
		w.writeHead(s)
		if *multi > 16 {
			*multi = 16
		}
		if *multi > 1 {
			err = multiIdentifyP(w, s, flag.Arg(0), *nr)
		} else {
			if *throttlef != 0 {
				throttle = time.NewTicker(*throttlef)
				defer throttle.Stop()
			}
			err = multiIdentifyS(w, s, flag.Arg(0), "", *nr)
		}
		w.writeTail()
		if err != nil {
			log.Fatalf("[FATAL] %v\n", err)
		}
		lg.printElapsed()
		os.Exit(0)
	}
	w.writeHead(s)
	identifyFile(w, s, flag.Arg(0), info.Size(), info.ModTime().Format(time.RFC3339))
	w.writeTail()
	lg.printElapsed()
	os.Exit(0)
}