func main() {
dbDir := flag.String("db_dir", "", "FSCK state database directory")
blobDir := flag.String("blob_dir", "", "Camlistore blob directory")
mimeType := flag.String("mime_type", "image/jpeg", "MIME type of files to scan")
print := flag.Bool("print", false, "Print ref and camera model")
workers := fsck.Parallel{Workers: 32}
flag.Var(workers, "workers", "parallel worker goroutines")
flag.Parse()
fdb, err := db.New(*dbDir)
if err != nil {
log.Fatal(err)
}
bs, err := dir.New(*blobDir)
if err != nil {
log.Fatal(err)
}
stats := fsck.NewStats()
defer stats.LogTopNEvery(10, 10*time.Second).Stop()
defer log.Print(stats)
files := fsck.NewFiles(bs)
go func() {
files.ReadRefs(fdb.ListMIME(*mimeType))
files.Close()
}()
go files.LogErrors()
workers.Go(func() {
for r := range files.Readers {
ex, err := exif.Decode(r)
if err != nil {
stats.Add("error")
continue
}
tag, err := ex.Get(exif.Model)
if err != nil {
stats.Add("missing")
continue
}
stats.Add(tag.String())
if *print {
id := "unknown"
if tag, err := ex.Get(exif.ImageUniqueID); err == nil {
id = tag.String()
stats.Add("unique-id-exif")
} else if thumb, err := ex.JpegThumbnail(); err == nil {
hash := sha1.Sum(thumb)
id = hex.EncodeToString(hash[:20])
stats.Add("unique-id-thumb")
} else if r.PartsSize() < 1e7 {
if _, err := r.Seek(0, 0); err == nil {
hash := sha1.New()
io.Copy(hash, r)
id = hex.EncodeToString(hash.Sum(nil))
stats.Add("unique-id-sha1")
} else {
id = "read-error"
stats.Add("unique-id-sha1-error")
}
} else {
stats.Add("unique-id-too-big")
}
fmt.Printf("%s %s %q %q\n", r.BlobRef(), id, r.FileName(), tag)
}
}
})
workers.Wait()
}
func main() {
bs := Flag{}
cat := &commander.Command{
UsageLine: "cat prints blob contents",
Run: func(cmd *commander.Command, args []string) error {
if bs.BS == nil {
return errors.New("require --blob_dir")
}
for _, ref := range args {
br, ok := blob.Parse(ref)
if !ok {
fmt.Fprintf(os.Stderr, "couldn't parse ref\n")
continue
}
blob, _, err := bs.BS.Fetch(br)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s\n", ref, err)
continue
}
io.Copy(os.Stdout, blob)
blob.Close()
}
return nil
},
}
tar := &commander.Command{
UsageLine: "tar exports files from the blobstore",
Run: func(cmd *commander.Command, args []string) error {
if bs.BS == nil {
return errors.New("require --blob_dir")
}
files := fsck.NewFiles(bs.BS)
go files.LogErrors()
// read blobrefs from stdin
refsCh := make(chan string, 20)
go func() {
in := bufio.NewScanner(os.Stdin)
for in.Scan() {
// TODO(dichro): validate ref?
refsCh <- in.Text()
}
close(refsCh)
}()
go func() {
files.ReadRefs(refsCh)
files.Close()
}()
out := tar.NewWriter(os.Stdout)
defer out.Flush()
for r := range files.Readers {
size := r.PartsSize()
if err := out.WriteHeader(&tar.Header{
Name: r.FileName(),
Mode: int64(r.FileMode()),
Uid: r.MapUid(),
Gid: r.MapGid(),
Size: size,
ModTime: r.ModTime(),
Typeflag: tar.TypeReg,
}); err != nil {
log.Fatal(err)
}
switch n, err := io.Copy(out, r); {
case err != nil:
log.Fatal(err)
case n != size:
log.Fatalf("wrote %d of %d", n, size)
}
}
return nil
},
}
top := &commander.Command{
UsageLine: os.Args[0],
Subcommands: []*commander.Command{
cat,
tar,
},
}
for _, cmd := range top.Subcommands {
cmd.Flag.Var(&bs, "blob_dir", "Camlistore blob directory")
}
if err := top.Dispatch(os.Args[1:]); err != nil {
log.Fatal(err)
}
}