include = build.String("limit", "", "comma separated list of PRONOM signatures to include")
exclude = build.String("exclude", "", "comma separated list of PRONOM signatures to exclude")
bof = build.Int("bof", 0, "define a maximum BOF offset")
eof = build.Int("eof", 0, "define a maximum EOF offset")
noeof = build.Bool("noeof", false, "ignore EOF segments in signatures")
multi = build.String("multi", "", "control how identifiers treat multiple results")
nocontainer = build.Bool("nocontainer", false, "skip container signatures")
notext = build.Bool("notext", false, "skip text matcher")
noname = build.Bool("noname", false, "skip filename matcher")
nomime = build.Bool("nomime", false, "skip MIME matcher")
noxml = build.Bool("noxml", false, "skip XML matcher")
noriff = build.Bool("noriff", false, "skip RIFF matcher")
noreports = build.Bool("noreports", false, "build directly from DROID file rather than PRONOM reports")
doubleup = build.Bool("doubleup", false, "include byte signatures for formats that also have container signatures")
rng = build.Int("range", config.Range(), "define a maximum range for segmentation")
distance = build.Int("distance", config.Distance(), "define a maximum distance for segmentation")
choices = build.Int("choices", config.Choices(), "define a maximum number of choices for segmentation")
// HARVEST
harvest = flag.NewFlagSet("harvest", flag.ExitOnError)
harvestHome = harvest.String("home", config.Home(), "override the default home directory")
harvestDroid = harvest.String("droid", config.Droid(), "set name/path for DROID signature file")
_, htimeout, _, _ = config.HarvestOptions()
timeout = harvest.Duration("timeout", htimeout, "set duration before timing-out harvesting requests e.g. 120s")
throttlef = harvest.Duration("throttle", 0, "set a time to wait HTTP requests e.g. 50ms")
// INSPECT (roy inspect | roy inspect fmt/121 | roy inspect usr/local/mysig.sig | roy inspect 10)
inspect = flag.NewFlagSet("inspect", flag.ExitOnError)
inspectHome = inspect.String("home", config.Home(), "override the default home directory")
inspectReports = inspect.Bool("reports", false, "build signatures from PRONOM reports (rather than DROID xml)")
inspectExtend = inspect.String("extend", "", "comma separated list of additional signatures")
func getOptions() []config.Option {
if *home != config.Home() {
config.SetHome(*home)
}
if *inspectHome != config.Home() {
config.SetHome(*inspectHome)
}
opts := []config.Option{}
// build options
if *droid != config.Droid() {
opts = append(opts, config.SetDroid(*droid))
}
if *container != config.Container() {
opts = append(opts, config.SetContainer(*container))
}
if *mi != "" {
opts = append(opts, config.SetMIMEInfo(*mi))
}
if *fdd != "" {
opts = append(opts, config.SetLOC(*fdd))
}
if *locfdd {
opts = append(opts, config.SetLOC(""))
}
if *nopronom {
opts = append(opts, config.SetNoPRONOM())
}
if *name != "" {
opts = append(opts, config.SetName(*name))
}
if *details != config.Details() {
opts = append(opts, config.SetDetails(*details))
}
if *extend != "" {
opts = append(opts, config.SetExtend(expandSets(*extend)))
}
if *extendc != "" {
if *extend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element describing the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*extendc)))
}
if *include != "" {
opts = append(opts, config.SetLimit(expandSets(*include)))
}
if *exclude != "" {
opts = append(opts, config.SetExclude(expandSets(*exclude)))
}
if *bof != 0 {
opts = append(opts, config.SetBOF(*bof))
}
if *eof != 0 {
opts = append(opts, config.SetEOF(*eof))
}
if *noeof {
opts = append(opts, config.SetNoEOF())
}
if *multi != "" {
opts = append(opts, config.SetMulti(strings.ToLower(*multi)))
}
if *nocontainer {
opts = append(opts, config.SetNoContainer())
}
if *notext {
opts = append(opts, config.SetNoText())
}
if *noname {
opts = append(opts, config.SetNoName())
}
if *nomime {
opts = append(opts, config.SetNoMIME())
}
if *noxml {
opts = append(opts, config.SetNoXML())
}
if *noriff {
opts = append(opts, config.SetNoRIFF())
}
if *noreports {
opts = append(opts, config.SetNoReports())
}
if *doubleup {
opts = append(opts, config.SetDoubleUp())
}
if *rng != config.Range() {
opts = append(opts, config.SetRange(*rng))
}
if *distance != config.Distance() {
opts = append(opts, config.SetDistance(*distance))
}
if *choices != config.Choices() {
opts = append(opts, config.SetChoices(*choices))
}
// inspect options
if *inspectMI != "" {
opts = append(opts, config.SetMIMEInfo(*inspectMI))
}
if *inspectFDD != "" {
opts = append(opts, config.SetLOC(*fdd))
}
if *inspectLOC {
opts = append(opts, config.SetLOC(""))
}
if *inspectInclude != "" {
opts = append(opts, config.SetLimit(expandSets(*inspectInclude)))
}
if *inspectExclude != "" {
opts = append(opts, config.SetExclude(expandSets(*inspectExclude)))
}
if *inspectExtend != "" {
opts = append(opts, config.SetExtend(expandSets(*inspectExtend)))
}
if *inspectExtendc != "" {
if *inspectExtend == "" {
fmt.Println(
`roy: warning! Unless the container extension only extends formats defined in
the DROID signature file you should also include a regular signature extension
(-extend) that includes a FileFormatCollection element describing the new formats.`)
}
opts = append(opts, config.SetExtendC(expandSets(*inspectExtendc)))
}
return opts
}
func (b *Matcher) addSignature(sig frames.Signature) error {
segments := sig.Segment(config.Distance(), config.Range())
// apply config no eof option
if config.NoEOF() {
var hasEof bool
var x int
for i, segment := range segments {
c := characterise(segment)
if c > prev {
hasEof = true
x = i
break
}
}
if hasEof {
if x == 0 {
b.keyFrames = append(b.keyFrames, []keyFrame{})
return nil
}
segments = segments[:x] // Otherwise trim segments to the first SUCC/EOF segment
}
}
kf := make([]keyFrame, len(segments))
clstr := newCluster(b)
for i, segment := range segments {
var pos position
c := characterise(segment)
switch c {
case unknown:
return fmt.Errorf("Zero length segment: signature %d, %v, segment %d", len(b.keyFrames), sig, i)
case bofZero:
pos = bofLength(segment, config.Choices())
case eofZero:
pos = eofLength(segment, config.Choices())
default:
pos = varLength(segment, config.Choices())
}
if pos.length < 1 {
switch c {
case bofZero, bofWindow:
kf[i] = b.addToFrameSet(segment, i, b.bofFrames, 0, 1)
case eofZero, eofWindow:
kf[i] = b.addToFrameSet(segment, i, b.eofFrames, len(segment)-1, len(segment))
default:
return fmt.Errorf("Variable offset segment encountered that can't be turned into a sequence: signature %d, segment %d", len(b.keyFrames), i)
}
} else {
switch c {
case bofZero, bofWild:
clstr = clstr.commit()
kf[i] = clstr.add(segment, i, pos)
case bofWindow:
if i > 0 {
kfB, _, _ := toKeyFrame(segment, pos)
if crossOver(kf[i-1], kfB) {
clstr = clstr.commit()
}
} else {
clstr = clstr.commit()
}
kf[i] = clstr.add(segment, i, pos)
case prev:
kf[i] = clstr.add(segment, i, pos)
case succ:
if !clstr.rev {
clstr = clstr.commit()
clstr.rev = true
}
kf[i] = clstr.add(segment, i, pos)
case eofZero, eofWindow, eofWild:
if !clstr.rev {
clstr = clstr.commit()
clstr.rev = true
}
kf[i] = clstr.add(segment, i, pos)
clstr = clstr.commit()
clstr.rev = true
}
}
}
clstr.commit()
updatePositions(kf)
b.knownBOF, b.knownEOF = firstBOFandEOF(b.knownBOF, b.knownEOF, kf)
b.maxBOF = maxBOF(b.maxBOF, kf)
b.maxEOF = maxEOF(b.maxEOF, kf)
b.keyFrames = append(b.keyFrames, kf)
return nil
}