Example #1
0
	extendc     = build.String("extendc", "", "comma separated list of additional container signatures")
	include     = build.String("limit", "", "comma separated list of PRONOM signatures to include")
	exclude     = build.String("exclude", "", "comma separated list of PRONOM signatures to exclude")
	bof         = build.Int("bof", 0, "define a maximum BOF offset")
	eof         = build.Int("eof", 0, "define a maximum EOF offset")
	noeof       = build.Bool("noeof", false, "ignore EOF segments in signatures")
	multi       = build.String("multi", "", "control how identifiers treat multiple results")
	nocontainer = build.Bool("nocontainer", false, "skip container signatures")
	notext      = build.Bool("notext", false, "skip text matcher")
	noname      = build.Bool("noname", false, "skip filename matcher")
	nomime      = build.Bool("nomime", false, "skip MIME matcher")
	noxml       = build.Bool("noxml", false, "skip XML matcher")
	noriff      = build.Bool("noriff", false, "skip RIFF matcher")
	noreports   = build.Bool("noreports", false, "build directly from DROID file rather than PRONOM reports")
	doubleup    = build.Bool("doubleup", false, "include byte signatures for formats that also have container signatures")
	rng         = build.Int("range", config.Range(), "define a maximum range for segmentation")
	distance    = build.Int("distance", config.Distance(), "define a maximum distance for segmentation")
	choices     = build.Int("choices", config.Choices(), "define a maximum number of choices for segmentation")

	// HARVEST
	harvest           = flag.NewFlagSet("harvest", flag.ExitOnError)
	harvestHome       = harvest.String("home", config.Home(), "override the default home directory")
	harvestDroid      = harvest.String("droid", config.Droid(), "set name/path for DROID signature file")
	_, htimeout, _, _ = config.HarvestOptions()
	timeout           = harvest.Duration("timeout", htimeout, "set duration before timing-out harvesting requests e.g. 120s")
	throttlef         = harvest.Duration("throttle", 0, "set a time to wait HTTP requests e.g. 50ms")

	// INSPECT (roy inspect | roy inspect fmt/121 | roy inspect usr/local/mysig.sig | roy inspect 10)
	inspect        = flag.NewFlagSet("inspect", flag.ExitOnError)
	inspectHome    = inspect.String("home", config.Home(), "override the default home directory")
	inspectReports = inspect.Bool("reports", false, "build signatures from PRONOM reports (rather than DROID xml)")
Example #2
0
func getOptions() []config.Option {
	if *home != config.Home() {
		config.SetHome(*home)
	}
	if *inspectHome != config.Home() {
		config.SetHome(*inspectHome)
	}
	opts := []config.Option{}
	// build options
	if *droid != config.Droid() {
		opts = append(opts, config.SetDroid(*droid))
	}
	if *container != config.Container() {
		opts = append(opts, config.SetContainer(*container))
	}
	if *mi != "" {
		opts = append(opts, config.SetMIMEInfo(*mi))
	}
	if *fdd != "" {
		opts = append(opts, config.SetLOC(*fdd))
	}
	if *locfdd {
		opts = append(opts, config.SetLOC(""))
	}
	if *nopronom {
		opts = append(opts, config.SetNoPRONOM())
	}
	if *name != "" {
		opts = append(opts, config.SetName(*name))
	}
	if *details != config.Details() {
		opts = append(opts, config.SetDetails(*details))
	}
	if *extend != "" {
		opts = append(opts, config.SetExtend(expandSets(*extend)))
	}
	if *extendc != "" {
		if *extend == "" {
			fmt.Println(
				`roy: warning! Unless the container extension only extends formats defined in 
the DROID signature file you should also include a regular signature extension 
(-extend) that includes a FileFormatCollection element describing the new formats.`)
		}
		opts = append(opts, config.SetExtendC(expandSets(*extendc)))
	}
	if *include != "" {
		opts = append(opts, config.SetLimit(expandSets(*include)))
	}
	if *exclude != "" {
		opts = append(opts, config.SetExclude(expandSets(*exclude)))
	}
	if *bof != 0 {
		opts = append(opts, config.SetBOF(*bof))
	}
	if *eof != 0 {
		opts = append(opts, config.SetEOF(*eof))
	}
	if *noeof {
		opts = append(opts, config.SetNoEOF())
	}
	if *multi != "" {
		opts = append(opts, config.SetMulti(strings.ToLower(*multi)))
	}
	if *nocontainer {
		opts = append(opts, config.SetNoContainer())
	}
	if *notext {
		opts = append(opts, config.SetNoText())
	}
	if *noname {
		opts = append(opts, config.SetNoName())
	}
	if *nomime {
		opts = append(opts, config.SetNoMIME())
	}
	if *noxml {
		opts = append(opts, config.SetNoXML())
	}
	if *noriff {
		opts = append(opts, config.SetNoRIFF())
	}
	if *noreports {
		opts = append(opts, config.SetNoReports())
	}
	if *doubleup {
		opts = append(opts, config.SetDoubleUp())
	}
	if *rng != config.Range() {
		opts = append(opts, config.SetRange(*rng))
	}
	if *distance != config.Distance() {
		opts = append(opts, config.SetDistance(*distance))
	}
	if *choices != config.Choices() {
		opts = append(opts, config.SetChoices(*choices))
	}
	// inspect options
	if *inspectMI != "" {
		opts = append(opts, config.SetMIMEInfo(*inspectMI))
	}
	if *inspectFDD != "" {
		opts = append(opts, config.SetLOC(*fdd))
	}
	if *inspectLOC {
		opts = append(opts, config.SetLOC(""))
	}
	if *inspectInclude != "" {
		opts = append(opts, config.SetLimit(expandSets(*inspectInclude)))
	}
	if *inspectExclude != "" {
		opts = append(opts, config.SetExclude(expandSets(*inspectExclude)))
	}
	if *inspectExtend != "" {
		opts = append(opts, config.SetExtend(expandSets(*inspectExtend)))
	}
	if *inspectExtendc != "" {
		if *inspectExtend == "" {
			fmt.Println(
				`roy: warning! Unless the container extension only extends formats defined in 
the DROID signature file you should also include a regular signature extension 
(-extend) that includes a FileFormatCollection element describing the new formats.`)
		}
		opts = append(opts, config.SetExtendC(expandSets(*inspectExtendc)))
	}
	return opts
}
Example #3
0
func (b *Matcher) addSignature(sig frames.Signature) error {
	segments := sig.Segment(config.Distance(), config.Range())
	// apply config no eof option
	if config.NoEOF() {
		var hasEof bool
		var x int
		for i, segment := range segments {
			c := characterise(segment)
			if c > prev {
				hasEof = true
				x = i
				break
			}
		}
		if hasEof {
			if x == 0 {
				b.keyFrames = append(b.keyFrames, []keyFrame{})
				return nil
			}
			segments = segments[:x] // Otherwise trim segments to the first SUCC/EOF segment
		}
	}
	kf := make([]keyFrame, len(segments))
	clstr := newCluster(b)
	for i, segment := range segments {
		var pos position
		c := characterise(segment)
		switch c {
		case unknown:
			return fmt.Errorf("Zero length segment: signature %d, %v, segment %d", len(b.keyFrames), sig, i)
		case bofZero:
			pos = bofLength(segment, config.Choices())
		case eofZero:
			pos = eofLength(segment, config.Choices())
		default:
			pos = varLength(segment, config.Choices())
		}
		if pos.length < 1 {
			switch c {
			case bofZero, bofWindow:
				kf[i] = b.addToFrameSet(segment, i, b.bofFrames, 0, 1)
			case eofZero, eofWindow:
				kf[i] = b.addToFrameSet(segment, i, b.eofFrames, len(segment)-1, len(segment))
			default:
				return fmt.Errorf("Variable offset segment encountered that can't be turned into a sequence: signature %d, segment %d", len(b.keyFrames), i)
			}
		} else {
			switch c {
			case bofZero, bofWild:
				clstr = clstr.commit()
				kf[i] = clstr.add(segment, i, pos)
			case bofWindow:
				if i > 0 {
					kfB, _, _ := toKeyFrame(segment, pos)
					if crossOver(kf[i-1], kfB) {
						clstr = clstr.commit()
					}
				} else {
					clstr = clstr.commit()
				}
				kf[i] = clstr.add(segment, i, pos)
			case prev:
				kf[i] = clstr.add(segment, i, pos)
			case succ:
				if !clstr.rev {
					clstr = clstr.commit()
					clstr.rev = true
				}
				kf[i] = clstr.add(segment, i, pos)
			case eofZero, eofWindow, eofWild:
				if !clstr.rev {
					clstr = clstr.commit()
					clstr.rev = true
				}
				kf[i] = clstr.add(segment, i, pos)
				clstr = clstr.commit()
				clstr.rev = true
			}
		}
	}
	clstr.commit()
	updatePositions(kf)
	b.knownBOF, b.knownEOF = firstBOFandEOF(b.knownBOF, b.knownEOF, kf)
	b.maxBOF = maxBOF(b.maxBOF, kf)
	b.maxEOF = maxEOF(b.maxEOF, kf)
	b.keyFrames = append(b.keyFrames, kf)
	return nil
}