Example #1
0
func inspectGob() error {
	s, err := siegfried.Load(config.Signature())
	if err != nil {
		return err
	}
	fmt.Print(s)
	return nil
}
Example #2
0
func blameSig(i int) error {
	s, err := siegfried.Load(config.Signature())
	if err != nil {
		return err
	}
	fmt.Println(s.Blame(i, *inspectCType, *inspectCName))
	return nil
}
Example #3
0
func blameSig(i int) error {
	s, err := siegfried.Load(config.Signature())
	if err != nil {
		return err
	}
	fmt.Println(s.InspectTestTree(i))
	return nil
}
Example #4
0
func inspectSig(t core.MatcherType) error {
	s, err := siegfried.Load(config.Signature())
	if err != nil {
		return err
	}
	fmt.Print(s.Inspect(t))
	return nil
}
Example #5
0
func inspectSig(t core.MatcherType) error {
	if *inspectHome != config.Home() {
		config.SetHome(*inspectHome)
	}
	s, err := siegfried.Load(config.Signature())
	if err != nil {
		return err
	}
	fmt.Print(s.Inspect(t))
	return nil
}
Example #6
0
func updateSigs() (string, error) {
	url, _, _ := config.UpdateOptions()
	if url == "" {
		return "Update is not available for this distribution of siegfried", nil
	}
	response, err := getHttp(url)
	if err != nil {
		return "", err
	}
	var u Update
	if err := json.Unmarshal(response, &u); err != nil {
		return "", err
	}
	version := config.Version()
	if version[0] < u.Version[0] || (version[0] == u.Version[0] && version[1] < u.Version[1]) || // if the version is out of date
		u.Version == [3]int{0, 0, 0} || u.Created == "" || u.Size == 0 || u.Path == "" { // or if the unmarshalling hasn't worked and we have blank values
		return "Your version of siegfried is out of date; please install latest from http://www.itforarchivists.com/siegfried before continuing.", nil
	}
	s, err := siegfried.Load(config.Signature())
	if err == nil {
		if !s.Update(u.Created) {
			return "You are already up to date!", nil
		}
	} else {
		// this hairy bit of golang exception handling is thanks to Ross! :)
		if _, err = os.Stat(config.Home()); err != nil {
			if os.IsNotExist(err) {
				err = os.MkdirAll(config.Home(), os.ModePerm)
				if err != nil {
					return "", fmt.Errorf("Siegfried: cannot create home directory %s, %v", config.Home(), err)
				}
			} else {
				return "", fmt.Errorf("Siegfried: error opening directory %s, %v", config.Home(), err)
			}
		}
	}
	fmt.Println("... downloading latest signature file ...")
	response, err = getHttp(u.Path)
	if err != nil {
		return "", fmt.Errorf("Siegfried: error retrieving %s.\nThis may be a network or firewall issue. See https://github.com/richardlehane/siegfried/wiki/Getting-started for manual instructions.\nSystem error: %v", config.SignatureBase(), err)
	}
	if len(response) != u.Size {
		return "", fmt.Errorf("Siegfried: error retrieving %s; expecting %d bytes, got %d bytes", config.SignatureBase(), u.Size, len(response))
	}
	err = ioutil.WriteFile(config.Signature(), response, os.ModePerm)
	if err != nil {
		return "", fmt.Errorf("Siegfried: error writing to directory, %v", err)
	}
	fmt.Printf("... writing %s ...\n", config.Signature())
	return "Your signature file has been updated", nil
}
Example #7
0
func setup() error {
	var err error
	config.SetHome(*testhome)
	s, err = siegfried.Load(config.Signature())
	return err
}
Example #8
0
func main() {
	var err error
	if len(os.Args) < 2 {
		log.Fatal(usage)
	}

	switch os.Args[1] {
	case "build":
		err = build.Parse(os.Args[2:])
		if err == nil {
			if build.Arg(0) != "" {
				config.SetSignature(build.Arg(0))
			}
			s := siegfried.New()
			err = makegob(s, buildOptions())
		}
	case "add":
		err = build.Parse(os.Args[2:])
		if err == nil {
			if build.Arg(0) != "" {
				config.SetSignature(build.Arg(0))
			}
			var s *siegfried.Siegfried
			s, err = siegfried.Load(config.Signature())
			if err == nil {
				err = makegob(s, buildOptions())
			}
		}
	case "harvest":
		err = harvest.Parse(os.Args[2:])
		if err == nil {
			setHarvestOptions()
			err = savereps()
		}
	case "inspect":
		err = inspect.Parse(os.Args[2:])
		if err == nil {
			setInspectOptions()
			input := inspect.Arg(0)
			switch {
			case input == "":
				err = inspectGob()
			case filepath.Ext(input) == ".sig":
				config.SetSignature(input)
				err = inspectGob()
			case strings.Contains(input, "fmt"):
				err = inspectSig(input)
			default:
				var i int
				i, err = strconv.Atoi(input)
				if err != nil {
					log.Fatal(err)
				}
				err = blameSig(i)
			}
		}
	default:
		log.Fatal(usage)
	}
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
Example #9
0
func parseRequest(w http.ResponseWriter, r *http.Request, s *siegfried.Siegfried, wg *sync.WaitGroup) (error, string, writer, bool, hashTyp, *siegfried.Siegfried, getFn) {
	// json, csv, droid or yaml
	paramsErr := func(field, expect string) (error, string, writer, bool, hashTyp, *siegfried.Siegfried, getFn) {
		return fmt.Errorf("bad request; in param %s got %s; valid values %s", field, r.FormValue(field), expect), "", nil, false, -1, nil, nil
	}
	var (
		mime string
		wr   writer
		frmt int
	)
	switch {
	case *jsono:
		frmt = 1
	case *csvo:
		frmt = 2
	case *droido:
		frmt = 3
	}
	if v := r.FormValue("format"); v != "" {
		switch v {
		case "yaml":
			frmt = 0
		case "json":
			frmt = 1
		case "csv":
			frmt = 2
		case "droid":
			frmt = 3
		default:
			return paramsErr("format", "yaml, json, csv or droid")
		}
	}
	if accept := r.Header.Get("Accept"); accept != "" {
		switch accept {
		case "application/x-yaml":
			frmt = 0
		case "application/json":
			frmt = 1
		case "text/csv", "application/csv":
			frmt = 2
		case "application/x-droid":
			frmt = 3
		}
	}
	switch frmt {
	case 0:
		wr = newYAML(w)
		mime = "application/x-yaml"
	case 1:
		wr = newJSON(w)
		mime = "application/json"
	case 2:
		wr = newCSV(w)
		mime = "text/csv"
	case 3:
		wr = newDroid(w)
		mime = "application/x-droid"
	}
	// no recurse
	norec := *nr
	if v := r.FormValue("nr"); v != "" {
		switch v {
		case "true":
			norec = true
		case "false":
			norec = false
		default:
			paramsErr("nr", "true or false")
		}
	}
	// archive
	z := *archive
	if v := r.FormValue("z"); v != "" {
		switch v {
		case "true":
			z = true
		case "false":
			z = false
		default:
			paramsErr("z", "true or false")
		}
	}
	// checksum
	h := *hashf
	if v := r.FormValue("hash"); v != "" {
		h = v
	}
	ht := getHash(h)
	// sig
	sf := s
	if v := r.FormValue("sig"); v != "" {
		if _, err := os.Stat(config.Local(v)); err != nil {
			return fmt.Errorf("bad request; sig param should be path to a signature file (absolute or relative to home); got %v", err), "", nil, false, -1, nil, nil
		}
		nsf, err := siegfried.Load(config.Local(v))
		if err == nil {
			sf = nsf
		}
	}
	gf := func(path, mime, mod string, sz int64) *context {
		c := ctxPool.Get().(*context)
		c.path, c.mime, c.mod, c.sz = path, mime, mod, sz
		c.s, c.w, c.wg, c.h, c.z = sf, wr, wg, makeHash(ht), z
		return c
	}
	return nil, mime, wr, norec, ht, sf, gf
}
Example #10
0
func main() {

	flag.Parse()

	/*//UNCOMMENT TO RUN PROFILER
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()*/

	if *home != config.Home() {
		config.SetHome(*home)
	}

	if *sig != config.SignatureBase() {
		config.SetSignature(*sig)
	}

	if *version {
		version := config.Version()
		fmt.Printf("siegfried version: %d.%d.%d\n", version[0], version[1], version[2])
		return
	}

	if *update {
		msg, err := updateSigs()
		if err != nil {
			log.Fatalf("Error: failed to update signature file, %v", err)
		}
		fmt.Println(msg)
		return
	}

	// during parallel scanning or in server mode, unsafe to access the last read buffer - so can't unzip or hash
	if *multi > 1 || *serve != "" {
		if *archive {
			log.Fatalln("Error: cannot scan archive formats when running in parallel mode")
		}
		if *hashf != "" {
			log.Fatalln("Error: cannot calculate file checksum when running in parallel mode")
		}
	}

	if err := setHash(); err != nil {
		log.Fatal(err)
	}

	if *serve != "" || *fprflag {
		s, err := siegfried.Load(config.Signature())
		if err != nil {
			log.Fatalf("Error: error loading signature file, got: %v", err)

		}
		if *serve != "" {
			log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
			listen(*serve, s)
			return
		}
		log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
		serveFpr(config.Fpr(), s)
		return
	}

	if flag.NArg() != 1 {
		log.Fatalln("Error: expecting a single file or directory argument")
	}

	s, err := siegfried.Load(config.Signature())
	if err != nil {
		log.Fatalf("Error: error loading signature file, got: %v", err)

	}

	var w writer
	switch {
	case *debug:
		config.SetDebug()
		w = debugWriter{}
	case *slow:
		config.SetSlow()
		w = &slowWriter{os.Stdout}
	case *csvo:
		w = newCSV(os.Stdout)
	case *jsono:
		w = newJSON(os.Stdout)
	case *droido:
		w = newDroid(os.Stdout)
	case *knowno:
		w = &knownWriter{true, os.Stdout}
	case *unknowno:
		w = &knownWriter{false, os.Stdout}
	default:
		w = newYAML(os.Stdout)
	}

	// support reading list files from stdin
	if flag.Arg(0) == "-" {
		w.writeHead(s)
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			info, err := os.Stat(scanner.Text())
			if err != nil || info.IsDir() {
				w.writeFile(scanner.Text(), 0, "", nil, fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil)
			} else {
				identifyFile(w, s, scanner.Text(), info.Size(), info.ModTime().Format(time.RFC3339))
			}
		}
		w.writeTail()
		os.Exit(0)
	}

	info, err := os.Stat(flag.Arg(0))
	if err != nil {
		log.Fatalf("Error: error getting info for %v, got: %v", flag.Arg(0), err)
	}

	if info.IsDir() {
		if config.Debug() {
			log.Fatalln("Error: when scanning in debug mode, give a file rather than a directory argument")
		}
		w.writeHead(s)
		if *multi > 16 {
			*multi = 16
		}
		if *multi > 1 {
			multiIdentifyP(w, s, flag.Arg(0), *nr)
		} else {
			multiIdentifyS(w, s, flag.Arg(0), *nr)
		}
		w.writeTail()
		os.Exit(0)
	}

	w.writeHead(s)
	identifyFile(w, s, flag.Arg(0), info.Size(), info.ModTime().Format(time.RFC3339))
	w.writeTail()
	os.Exit(0)
}
Example #11
0
func main() {
	flag.Parse()
	/*//UNCOMMENT TO RUN PROFILER
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()*/
	// configure home and signature if not default
	if *home != config.Home() {
		config.SetHome(*home)
	}
	if *sig != config.SignatureBase() {
		config.SetSignature(*sig)
	}
	// handle -update
	if *update {
		msg, err := updateSigs()
		if err != nil {
			log.Fatalf("[FATAL] failed to update signature file, %v", err)
		}
		fmt.Println(msg)
		return
	}
	// handle -hash error
	hashT := getHash(*hashf)
	if *hashf != "" && hashT < 0 {
		log.Fatalf("[FATAL] invalid hash type; choose from %s", hashChoices)
	}
	// load and handle signature errors
	s, err := siegfried.Load(config.Signature())
	if err != nil {
		log.Fatalf("[FATAL] error loading signature file, got: %v", err)
	}
	// handle -version
	if *version {
		version := config.Version()
		fmt.Printf("siegfried %d.%d.%d\n%s", version[0], version[1], version[2], s)
		return
	}
	// handle -fpr
	if *fprflag {
		log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
		serveFpr(config.Fpr(), s)
		return
	}
	// check -multi
	if *multi > maxMulti || *multi < 1 || (*archive && *multi > 1) {
		log.Println("[WARN] -multi must be > 0 and =< 1024. If -z, -multi must be 1. Resetting -multi to 1")
		*multi = 1
	}
	// start logger
	lg, err := newLogger(*logf)
	if err != nil {
		log.Fatalln(err)
	}
	if config.Slow() || config.Debug() {
		if *serve != "" || *fprflag {
			log.Fatalln("[FATAL] debug and slow logging cannot be run in server mode")
		}
	}
	// start throttle
	if *throttlef != 0 {
		throttle = time.NewTicker(*throttlef)
		defer throttle.Stop()
	}
	// start the printer
	lenCtxts := *multi
	if lenCtxts == 1 {
		lenCtxts = 8
	}
	ctxts := make(chan *context, lenCtxts)
	go printer(ctxts, lg)
	// set default writer
	var w writer
	switch {
	case *csvo:
		w = newCSV(os.Stdout)
	case *jsono:
		w = newJSON(os.Stdout)
	case *droido:
		w = newDroid(os.Stdout)
		if len(s.Fields()) != 1 || len(s.Fields()[0]) != 7 {
			close(ctxts)
			log.Fatalln("[FATAL] DROID output is limited to signature files with a single PRONOM identifier")
		}
	default:
		w = newYAML(os.Stdout)
	}
	// overrite writer with nil writer if logging is to stdout
	if lg != nil && lg.w == os.Stdout {
		w = logWriter{}
	}
	// setup default waitgroup
	wg := &sync.WaitGroup{}
	// setup context pool
	setCtxPool(s, w, wg, hashT, *archive)
	// handle -serve
	if *serve != "" {
		log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
		listen(*serve, s, ctxts)
		return
	}
	// handle no file/directory argument
	if flag.NArg() != 1 {
		close(ctxts)
		log.Fatalln("[FATAL] expecting a single file or directory argument")
	}

	w.writeHead(s, hashT)
	// support reading list files from stdin
	if flag.Arg(0) == "-" {
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			info, err := os.Stat(scanner.Text())
			if err != nil {
				info, err = retryStat(scanner.Text(), err)
			}
			if err != nil || info.IsDir() {
				ctx := getCtx(scanner.Text(), "", "", 0)
				ctx.res <- results{fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil, nil}
				ctx.wg.Add(1)
				ctxts <- ctx
			} else {
				identifyFile(getCtx(scanner.Text(), "", info.ModTime().Format(time.RFC3339), info.Size()), ctxts, getCtx)
			}
		}
	} else {
		err = identify(ctxts, flag.Arg(0), "", *nr, getCtx)
	}
	wg.Wait()
	close(ctxts)
	w.writeTail()
	// log time elapsed
	if !lg.start.IsZero() {
		fmt.Fprintf(lg.w, "%s %v\n", timeString, time.Since(lg.start))
	}
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
Example #12
0
func main() {
	var err error
	if len(os.Args) < 2 {
		log.Fatal(usage)
	}

	switch os.Args[1] {
	case "build":
		err = build.Parse(os.Args[2:])
		if err == nil {
			if build.Arg(0) != "" {
				config.SetSignature(build.Arg(0))
			}
			s := siegfried.New()
			err = makegob(s, getOptions())
		}
	case "add":
		err = build.Parse(os.Args[2:])
		if err == nil {
			if build.Arg(0) != "" {
				config.SetSignature(build.Arg(0))
			}
			var s *siegfried.Siegfried
			s, err = siegfried.Load(config.Signature())
			if err == nil {
				err = makegob(s, getOptions())
			}
		}
	case "harvest":
		err = harvest.Parse(os.Args[2:])
		if err == nil {
			setHarvestOptions()
			err = savereps()
		}
	case "inspect":
		inspect.Usage = func() { fmt.Print(inspectUsage) }
		err = inspect.Parse(os.Args[2:])
		if err == nil {
			input := inspect.Arg(0)
			switch {
			case input == "":
				err = inspectSig(-1)
			case input == "bytematcher", input == "bm":
				err = inspectSig(core.ByteMatcher)
			case input == "containermatcher", input == "cm":
				err = inspectSig(core.ContainerMatcher)
			case input == "namematcher", input == "nm":
				err = inspectSig(core.NameMatcher)
			case input == "mimematcher", input == "mm":
				err = inspectSig(core.MIMEMatcher)
			case input == "riffmatcher", input == "rm":
				err = inspectSig(core.RIFFMatcher)
			case input == "xmlmatcher", input == "xm":
				err = inspectSig(core.XMLMatcher)
			case input == "textmatcher", input == "tm":
				err = inspectSig(core.TextMatcher)
			case input == "priorities", input == "p":
				err = graphPriorities(0)
			case input == "missing-priorities", input == "mp":
				err = graphPriorities(1)
			case input == "implicit-priorities", input == "ip":
				err = graphPriorities(2)
			case filepath.Ext(input) == ".sig":
				config.SetSignature(input)
				err = inspectSig(-1)
			case strings.Contains(input, "fmt"), *inspectMI != "", strings.HasPrefix(input, "fdd"), strings.HasPrefix(input, "@"):
				err = inspectFmt(input)
			default:
				var i int
				i, err = strconv.Atoi(input)
				if err == nil {
					err = blameSig(i)
				} else {
					err = inspectFmt(input)
				}
			}
		}
	default:
		log.Fatal(usage)
	}
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
Example #13
0
func main() {

	flag.Parse()

	/*//UNCOMMENT TO RUN PROFILER
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()*/

	if *version {
		version := config.Version()
		fmt.Printf("siegfried %d.%d.%d\n", version[0], version[1], version[2])
		s, err := siegfried.Load(config.Signature())
		if err != nil {
			fmt.Println(err)
			return
		}
		fmt.Print(s)
		return
	}

	if *home != config.Home() {
		config.SetHome(*home)
	}

	if *sig != config.SignatureBase() {
		config.SetSignature(*sig)
	}

	if *update {
		msg, err := updateSigs()
		if err != nil {
			log.Fatalf("[FATAL] failed to update signature file, %v", err)
		}
		fmt.Println(msg)
		return
	}

	// during parallel scanning or in server mode, unsafe to access the last read buffer - so can't unzip or hash
	if *multi > 1 || *serve != "" {
		if *archive {
			log.Fatalln("[FATAL] cannot scan archive formats when running in parallel or server mode")
		}
		if *hashf != "" {
			log.Fatalln("[FATAL] cannot calculate file checksum when running in parallel or server mode")
		}
	}

	if *logf != "" {
		if *multi > 1 && *logf != "error" {
			log.Fatalln("[FATAL] cannot log in parallel mode")
		}
		if err := newLogger(*logf); err != nil {
			log.Fatalln(err)
		}
	}

	if err := setHash(); err != nil {
		log.Fatal(err)
	}

	if *serve != "" || *fprflag {
		s, err := siegfried.Load(config.Signature())
		if err != nil {
			log.Fatalf("[FATAL] error loading signature file, got: %v", err)
		}
		if *serve != "" {
			log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
			listen(*serve, s)
			return
		}
		log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
		serveFpr(config.Fpr(), s)
		return
	}

	if flag.NArg() != 1 {
		log.Fatalln("[FATAL] expecting a single file or directory argument")
	}

	s, err := siegfried.Load(config.Signature())
	if err != nil {
		log.Fatalf("[FATAL] error loading signature file, got: %v", err)
	}

	var w writer
	switch {
	case *csvo:
		w = newCSV(os.Stdout)
	case *jsono:
		w = newJSON(os.Stdout)
	case *droido:
		w = newDroid(os.Stdout)
	default:
		w = newYAML(os.Stdout)
	}

	if lg != nil && lg.w == os.Stdout {
		w = logWriter{}
	}

	// support reading list files from stdin
	if flag.Arg(0) == "-" {
		w.writeHead(s)
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			info, err := os.Stat(scanner.Text())
			if err != nil {
				info, err = retryStat(scanner.Text(), err)
			}
			if err != nil || info.IsDir() {
				writeError(w, scanner.Text(), 0, "", fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err))
			} else {
				identifyFile(w, s, scanner.Text(), info.Size(), info.ModTime().Format(time.RFC3339))
			}
		}
		w.writeTail()
		lg.printElapsed()
		os.Exit(0)
	}

	info, err := os.Stat(flag.Arg(0))
	if err != nil {
		info, err = retryStat(flag.Arg(0), err)
		if err != nil {
			log.Fatalf("[FATAL] cannot get info for %v, got: %v", flag.Arg(0), err)
		}
	}

	if info.IsDir() {
		w.writeHead(s)
		if *multi > 16 {
			*multi = 16
		}
		if *multi > 1 {
			err = multiIdentifyP(w, s, flag.Arg(0), *nr)
		} else {
			if *throttlef != 0 {
				throttle = time.NewTicker(*throttlef)
				defer throttle.Stop()
			}
			err = multiIdentifyS(w, s, flag.Arg(0), "", *nr)
		}
		w.writeTail()
		if err != nil {
			log.Fatalf("[FATAL] %v\n", err)
		}
		lg.printElapsed()
		os.Exit(0)
	}
	w.writeHead(s)
	identifyFile(w, s, flag.Arg(0), info.Size(), info.ModTime().Format(time.RFC3339))
	w.writeTail()
	lg.printElapsed()
	os.Exit(0)
}