func inspectGob() error {
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Print(s)
return nil
}
func blameSig(i int) error {
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Println(s.Blame(i, *inspectCType, *inspectCName))
return nil
}
func blameSig(i int) error {
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Println(s.InspectTestTree(i))
return nil
}
func inspectSig(t core.MatcherType) error {
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Print(s.Inspect(t))
return nil
}
func inspectSig(t core.MatcherType) error {
if *inspectHome != config.Home() {
config.SetHome(*inspectHome)
}
s, err := siegfried.Load(config.Signature())
if err != nil {
return err
}
fmt.Print(s.Inspect(t))
return nil
}
func updateSigs() (string, error) {
url, _, _ := config.UpdateOptions()
if url == "" {
return "Update is not available for this distribution of siegfried", nil
}
response, err := getHttp(url)
if err != nil {
return "", err
}
var u Update
if err := json.Unmarshal(response, &u); err != nil {
return "", err
}
version := config.Version()
if version[0] < u.Version[0] || (version[0] == u.Version[0] && version[1] < u.Version[1]) || // if the version is out of date
u.Version == [3]int{0, 0, 0} || u.Created == "" || u.Size == 0 || u.Path == "" { // or if the unmarshalling hasn't worked and we have blank values
return "Your version of siegfried is out of date; please install latest from http://www.itforarchivists.com/siegfried before continuing.", nil
}
s, err := siegfried.Load(config.Signature())
if err == nil {
if !s.Update(u.Created) {
return "You are already up to date!", nil
}
} else {
// this hairy bit of golang exception handling is thanks to Ross! :)
if _, err = os.Stat(config.Home()); err != nil {
if os.IsNotExist(err) {
err = os.MkdirAll(config.Home(), os.ModePerm)
if err != nil {
return "", fmt.Errorf("Siegfried: cannot create home directory %s, %v", config.Home(), err)
}
} else {
return "", fmt.Errorf("Siegfried: error opening directory %s, %v", config.Home(), err)
}
}
}
fmt.Println("... downloading latest signature file ...")
response, err = getHttp(u.Path)
if err != nil {
return "", fmt.Errorf("Siegfried: error retrieving %s.\nThis may be a network or firewall issue. See https://github.com/richardlehane/siegfried/wiki/Getting-started for manual instructions.\nSystem error: %v", config.SignatureBase(), err)
}
if len(response) != u.Size {
return "", fmt.Errorf("Siegfried: error retrieving %s; expecting %d bytes, got %d bytes", config.SignatureBase(), u.Size, len(response))
}
err = ioutil.WriteFile(config.Signature(), response, os.ModePerm)
if err != nil {
return "", fmt.Errorf("Siegfried: error writing to directory, %v", err)
}
fmt.Printf("... writing %s ...\n", config.Signature())
return "Your signature file has been updated", nil
}
func setup() error {
var err error
config.SetHome(*testhome)
s, err = siegfried.Load(config.Signature())
return err
}
func main() {
var err error
if len(os.Args) < 2 {
log.Fatal(usage)
}
switch os.Args[1] {
case "build":
err = build.Parse(os.Args[2:])
if err == nil {
if build.Arg(0) != "" {
config.SetSignature(build.Arg(0))
}
s := siegfried.New()
err = makegob(s, buildOptions())
}
case "add":
err = build.Parse(os.Args[2:])
if err == nil {
if build.Arg(0) != "" {
config.SetSignature(build.Arg(0))
}
var s *siegfried.Siegfried
s, err = siegfried.Load(config.Signature())
if err == nil {
err = makegob(s, buildOptions())
}
}
case "harvest":
err = harvest.Parse(os.Args[2:])
if err == nil {
setHarvestOptions()
err = savereps()
}
case "inspect":
err = inspect.Parse(os.Args[2:])
if err == nil {
setInspectOptions()
input := inspect.Arg(0)
switch {
case input == "":
err = inspectGob()
case filepath.Ext(input) == ".sig":
config.SetSignature(input)
err = inspectGob()
case strings.Contains(input, "fmt"):
err = inspectSig(input)
default:
var i int
i, err = strconv.Atoi(input)
if err != nil {
log.Fatal(err)
}
err = blameSig(i)
}
}
default:
log.Fatal(usage)
}
if err != nil {
log.Fatal(err)
}
os.Exit(0)
}
func parseRequest(w http.ResponseWriter, r *http.Request, s *siegfried.Siegfried, wg *sync.WaitGroup) (error, string, writer, bool, hashTyp, *siegfried.Siegfried, getFn) {
// json, csv, droid or yaml
paramsErr := func(field, expect string) (error, string, writer, bool, hashTyp, *siegfried.Siegfried, getFn) {
return fmt.Errorf("bad request; in param %s got %s; valid values %s", field, r.FormValue(field), expect), "", nil, false, -1, nil, nil
}
var (
mime string
wr writer
frmt int
)
switch {
case *jsono:
frmt = 1
case *csvo:
frmt = 2
case *droido:
frmt = 3
}
if v := r.FormValue("format"); v != "" {
switch v {
case "yaml":
frmt = 0
case "json":
frmt = 1
case "csv":
frmt = 2
case "droid":
frmt = 3
default:
return paramsErr("format", "yaml, json, csv or droid")
}
}
if accept := r.Header.Get("Accept"); accept != "" {
switch accept {
case "application/x-yaml":
frmt = 0
case "application/json":
frmt = 1
case "text/csv", "application/csv":
frmt = 2
case "application/x-droid":
frmt = 3
}
}
switch frmt {
case 0:
wr = newYAML(w)
mime = "application/x-yaml"
case 1:
wr = newJSON(w)
mime = "application/json"
case 2:
wr = newCSV(w)
mime = "text/csv"
case 3:
wr = newDroid(w)
mime = "application/x-droid"
}
// no recurse
norec := *nr
if v := r.FormValue("nr"); v != "" {
switch v {
case "true":
norec = true
case "false":
norec = false
default:
paramsErr("nr", "true or false")
}
}
// archive
z := *archive
if v := r.FormValue("z"); v != "" {
switch v {
case "true":
z = true
case "false":
z = false
default:
paramsErr("z", "true or false")
}
}
// checksum
h := *hashf
if v := r.FormValue("hash"); v != "" {
h = v
}
ht := getHash(h)
// sig
sf := s
if v := r.FormValue("sig"); v != "" {
if _, err := os.Stat(config.Local(v)); err != nil {
return fmt.Errorf("bad request; sig param should be path to a signature file (absolute or relative to home); got %v", err), "", nil, false, -1, nil, nil
}
nsf, err := siegfried.Load(config.Local(v))
if err == nil {
sf = nsf
}
}
gf := func(path, mime, mod string, sz int64) *context {
c := ctxPool.Get().(*context)
c.path, c.mime, c.mod, c.sz = path, mime, mod, sz
c.s, c.w, c.wg, c.h, c.z = sf, wr, wg, makeHash(ht), z
return c
}
return nil, mime, wr, norec, ht, sf, gf
}
func main() {
flag.Parse()
/*//UNCOMMENT TO RUN PROFILER
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()*/
if *home != config.Home() {
config.SetHome(*home)
}
if *sig != config.SignatureBase() {
config.SetSignature(*sig)
}
if *version {
version := config.Version()
fmt.Printf("siegfried version: %d.%d.%d\n", version[0], version[1], version[2])
return
}
if *update {
msg, err := updateSigs()
if err != nil {
log.Fatalf("Error: failed to update signature file, %v", err)
}
fmt.Println(msg)
return
}
// during parallel scanning or in server mode, unsafe to access the last read buffer - so can't unzip or hash
if *multi > 1 || *serve != "" {
if *archive {
log.Fatalln("Error: cannot scan archive formats when running in parallel mode")
}
if *hashf != "" {
log.Fatalln("Error: cannot calculate file checksum when running in parallel mode")
}
}
if err := setHash(); err != nil {
log.Fatal(err)
}
if *serve != "" || *fprflag {
s, err := siegfried.Load(config.Signature())
if err != nil {
log.Fatalf("Error: error loading signature file, got: %v", err)
}
if *serve != "" {
log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
listen(*serve, s)
return
}
log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
serveFpr(config.Fpr(), s)
return
}
if flag.NArg() != 1 {
log.Fatalln("Error: expecting a single file or directory argument")
}
s, err := siegfried.Load(config.Signature())
if err != nil {
log.Fatalf("Error: error loading signature file, got: %v", err)
}
var w writer
switch {
case *debug:
config.SetDebug()
w = debugWriter{}
case *slow:
config.SetSlow()
w = &slowWriter{os.Stdout}
case *csvo:
w = newCSV(os.Stdout)
case *jsono:
w = newJSON(os.Stdout)
case *droido:
w = newDroid(os.Stdout)
case *knowno:
w = &knownWriter{true, os.Stdout}
case *unknowno:
w = &knownWriter{false, os.Stdout}
default:
w = newYAML(os.Stdout)
}
// support reading list files from stdin
if flag.Arg(0) == "-" {
w.writeHead(s)
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
info, err := os.Stat(scanner.Text())
if err != nil || info.IsDir() {
w.writeFile(scanner.Text(), 0, "", nil, fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil)
} else {
identifyFile(w, s, scanner.Text(), info.Size(), info.ModTime().Format(time.RFC3339))
}
}
w.writeTail()
os.Exit(0)
}
info, err := os.Stat(flag.Arg(0))
if err != nil {
log.Fatalf("Error: error getting info for %v, got: %v", flag.Arg(0), err)
}
if info.IsDir() {
if config.Debug() {
log.Fatalln("Error: when scanning in debug mode, give a file rather than a directory argument")
}
w.writeHead(s)
if *multi > 16 {
*multi = 16
}
if *multi > 1 {
multiIdentifyP(w, s, flag.Arg(0), *nr)
} else {
multiIdentifyS(w, s, flag.Arg(0), *nr)
}
w.writeTail()
os.Exit(0)
}
w.writeHead(s)
identifyFile(w, s, flag.Arg(0), info.Size(), info.ModTime().Format(time.RFC3339))
w.writeTail()
os.Exit(0)
}
func main() {
flag.Parse()
/*//UNCOMMENT TO RUN PROFILER
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()*/
// configure home and signature if not default
if *home != config.Home() {
config.SetHome(*home)
}
if *sig != config.SignatureBase() {
config.SetSignature(*sig)
}
// handle -update
if *update {
msg, err := updateSigs()
if err != nil {
log.Fatalf("[FATAL] failed to update signature file, %v", err)
}
fmt.Println(msg)
return
}
// handle -hash error
hashT := getHash(*hashf)
if *hashf != "" && hashT < 0 {
log.Fatalf("[FATAL] invalid hash type; choose from %s", hashChoices)
}
// load and handle signature errors
s, err := siegfried.Load(config.Signature())
if err != nil {
log.Fatalf("[FATAL] error loading signature file, got: %v", err)
}
// handle -version
if *version {
version := config.Version()
fmt.Printf("siegfried %d.%d.%d\n%s", version[0], version[1], version[2], s)
return
}
// handle -fpr
if *fprflag {
log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
serveFpr(config.Fpr(), s)
return
}
// check -multi
if *multi > maxMulti || *multi < 1 || (*archive && *multi > 1) {
log.Println("[WARN] -multi must be > 0 and =< 1024. If -z, -multi must be 1. Resetting -multi to 1")
*multi = 1
}
// start logger
lg, err := newLogger(*logf)
if err != nil {
log.Fatalln(err)
}
if config.Slow() || config.Debug() {
if *serve != "" || *fprflag {
log.Fatalln("[FATAL] debug and slow logging cannot be run in server mode")
}
}
// start throttle
if *throttlef != 0 {
throttle = time.NewTicker(*throttlef)
defer throttle.Stop()
}
// start the printer
lenCtxts := *multi
if lenCtxts == 1 {
lenCtxts = 8
}
ctxts := make(chan *context, lenCtxts)
go printer(ctxts, lg)
// set default writer
var w writer
switch {
case *csvo:
w = newCSV(os.Stdout)
case *jsono:
w = newJSON(os.Stdout)
case *droido:
w = newDroid(os.Stdout)
if len(s.Fields()) != 1 || len(s.Fields()[0]) != 7 {
close(ctxts)
log.Fatalln("[FATAL] DROID output is limited to signature files with a single PRONOM identifier")
}
default:
w = newYAML(os.Stdout)
}
// overrite writer with nil writer if logging is to stdout
if lg != nil && lg.w == os.Stdout {
w = logWriter{}
}
// setup default waitgroup
wg := &sync.WaitGroup{}
// setup context pool
setCtxPool(s, w, wg, hashT, *archive)
// handle -serve
if *serve != "" {
log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
listen(*serve, s, ctxts)
return
}
// handle no file/directory argument
if flag.NArg() != 1 {
close(ctxts)
log.Fatalln("[FATAL] expecting a single file or directory argument")
}
w.writeHead(s, hashT)
// support reading list files from stdin
if flag.Arg(0) == "-" {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
info, err := os.Stat(scanner.Text())
if err != nil {
info, err = retryStat(scanner.Text(), err)
}
if err != nil || info.IsDir() {
ctx := getCtx(scanner.Text(), "", "", 0)
ctx.res <- results{fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err), nil, nil}
ctx.wg.Add(1)
ctxts <- ctx
} else {
identifyFile(getCtx(scanner.Text(), "", info.ModTime().Format(time.RFC3339), info.Size()), ctxts, getCtx)
}
}
} else {
err = identify(ctxts, flag.Arg(0), "", *nr, getCtx)
}
wg.Wait()
close(ctxts)
w.writeTail()
// log time elapsed
if !lg.start.IsZero() {
fmt.Fprintf(lg.w, "%s %v\n", timeString, time.Since(lg.start))
}
if err != nil {
log.Fatal(err)
}
os.Exit(0)
}
func main() {
var err error
if len(os.Args) < 2 {
log.Fatal(usage)
}
switch os.Args[1] {
case "build":
err = build.Parse(os.Args[2:])
if err == nil {
if build.Arg(0) != "" {
config.SetSignature(build.Arg(0))
}
s := siegfried.New()
err = makegob(s, getOptions())
}
case "add":
err = build.Parse(os.Args[2:])
if err == nil {
if build.Arg(0) != "" {
config.SetSignature(build.Arg(0))
}
var s *siegfried.Siegfried
s, err = siegfried.Load(config.Signature())
if err == nil {
err = makegob(s, getOptions())
}
}
case "harvest":
err = harvest.Parse(os.Args[2:])
if err == nil {
setHarvestOptions()
err = savereps()
}
case "inspect":
inspect.Usage = func() { fmt.Print(inspectUsage) }
err = inspect.Parse(os.Args[2:])
if err == nil {
input := inspect.Arg(0)
switch {
case input == "":
err = inspectSig(-1)
case input == "bytematcher", input == "bm":
err = inspectSig(core.ByteMatcher)
case input == "containermatcher", input == "cm":
err = inspectSig(core.ContainerMatcher)
case input == "namematcher", input == "nm":
err = inspectSig(core.NameMatcher)
case input == "mimematcher", input == "mm":
err = inspectSig(core.MIMEMatcher)
case input == "riffmatcher", input == "rm":
err = inspectSig(core.RIFFMatcher)
case input == "xmlmatcher", input == "xm":
err = inspectSig(core.XMLMatcher)
case input == "textmatcher", input == "tm":
err = inspectSig(core.TextMatcher)
case input == "priorities", input == "p":
err = graphPriorities(0)
case input == "missing-priorities", input == "mp":
err = graphPriorities(1)
case input == "implicit-priorities", input == "ip":
err = graphPriorities(2)
case filepath.Ext(input) == ".sig":
config.SetSignature(input)
err = inspectSig(-1)
case strings.Contains(input, "fmt"), *inspectMI != "", strings.HasPrefix(input, "fdd"), strings.HasPrefix(input, "@"):
err = inspectFmt(input)
default:
var i int
i, err = strconv.Atoi(input)
if err == nil {
err = blameSig(i)
} else {
err = inspectFmt(input)
}
}
}
default:
log.Fatal(usage)
}
if err != nil {
log.Fatal(err)
}
os.Exit(0)
}
func main() {
flag.Parse()
/*//UNCOMMENT TO RUN PROFILER
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()*/
if *version {
version := config.Version()
fmt.Printf("siegfried %d.%d.%d\n", version[0], version[1], version[2])
s, err := siegfried.Load(config.Signature())
if err != nil {
fmt.Println(err)
return
}
fmt.Print(s)
return
}
if *home != config.Home() {
config.SetHome(*home)
}
if *sig != config.SignatureBase() {
config.SetSignature(*sig)
}
if *update {
msg, err := updateSigs()
if err != nil {
log.Fatalf("[FATAL] failed to update signature file, %v", err)
}
fmt.Println(msg)
return
}
// during parallel scanning or in server mode, unsafe to access the last read buffer - so can't unzip or hash
if *multi > 1 || *serve != "" {
if *archive {
log.Fatalln("[FATAL] cannot scan archive formats when running in parallel or server mode")
}
if *hashf != "" {
log.Fatalln("[FATAL] cannot calculate file checksum when running in parallel or server mode")
}
}
if *logf != "" {
if *multi > 1 && *logf != "error" {
log.Fatalln("[FATAL] cannot log in parallel mode")
}
if err := newLogger(*logf); err != nil {
log.Fatalln(err)
}
}
if err := setHash(); err != nil {
log.Fatal(err)
}
if *serve != "" || *fprflag {
s, err := siegfried.Load(config.Signature())
if err != nil {
log.Fatalf("[FATAL] error loading signature file, got: %v", err)
}
if *serve != "" {
log.Printf("Starting server at %s. Use CTRL-C to quit.\n", *serve)
listen(*serve, s)
return
}
log.Printf("FPR server started at %s. Use CTRL-C to quit.\n", config.Fpr())
serveFpr(config.Fpr(), s)
return
}
if flag.NArg() != 1 {
log.Fatalln("[FATAL] expecting a single file or directory argument")
}
s, err := siegfried.Load(config.Signature())
if err != nil {
log.Fatalf("[FATAL] error loading signature file, got: %v", err)
}
var w writer
switch {
case *csvo:
w = newCSV(os.Stdout)
case *jsono:
w = newJSON(os.Stdout)
case *droido:
w = newDroid(os.Stdout)
default:
w = newYAML(os.Stdout)
}
if lg != nil && lg.w == os.Stdout {
w = logWriter{}
}
// support reading list files from stdin
if flag.Arg(0) == "-" {
w.writeHead(s)
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
info, err := os.Stat(scanner.Text())
if err != nil {
info, err = retryStat(scanner.Text(), err)
}
if err != nil || info.IsDir() {
writeError(w, scanner.Text(), 0, "", fmt.Errorf("failed to identify %s (in scanning mode, inputs must all be files and not directories), got: %v", scanner.Text(), err))
} else {
identifyFile(w, s, scanner.Text(), info.Size(), info.ModTime().Format(time.RFC3339))
}
}
w.writeTail()
lg.printElapsed()
os.Exit(0)
}
info, err := os.Stat(flag.Arg(0))
if err != nil {
info, err = retryStat(flag.Arg(0), err)
if err != nil {
log.Fatalf("[FATAL] cannot get info for %v, got: %v", flag.Arg(0), err)
}
}
if info.IsDir() {
w.writeHead(s)
if *multi > 16 {
*multi = 16
}
if *multi > 1 {
err = multiIdentifyP(w, s, flag.Arg(0), *nr)
} else {
if *throttlef != 0 {
throttle = time.NewTicker(*throttlef)
defer throttle.Stop()
}
err = multiIdentifyS(w, s, flag.Arg(0), "", *nr)
}
w.writeTail()
if err != nil {
log.Fatalf("[FATAL] %v\n", err)
}
lg.printElapsed()
os.Exit(0)
}
w.writeHead(s)
identifyFile(w, s, flag.Arg(0), info.Size(), info.ModTime().Format(time.RFC3339))
w.writeTail()
lg.printElapsed()
os.Exit(0)
}