Пример #1
0
func main() {
	dbDir := flag.String("db_dir", "", "FSCK state database directory")
	blobDir := flag.String("blob_dir", "", "Camlistore blob directory")
	mimeType := flag.String("mime_type", "image/jpeg", "MIME type of files to scan")
	print := flag.Bool("print", false, "Print ref and camera model")
	workers := fsck.Parallel{Workers: 32}
	flag.Var(workers, "workers", "parallel worker goroutines")
	flag.Parse()

	fdb, err := db.New(*dbDir)
	if err != nil {
		log.Fatal(err)
	}
	bs, err := dir.New(*blobDir)
	if err != nil {
		log.Fatal(err)
	}

	stats := fsck.NewStats()
	defer stats.LogTopNEvery(10, 10*time.Second).Stop()
	defer log.Print(stats)

	files := fsck.NewFiles(bs)
	go func() {
		files.ReadRefs(fdb.ListMIME(*mimeType))
		files.Close()
	}()
	go files.LogErrors()

	workers.Go(func() {
		for r := range files.Readers {
			ex, err := exif.Decode(r)
			if err != nil {
				stats.Add("error")
				continue
			}
			tag, err := ex.Get(exif.Model)
			if err != nil {
				stats.Add("missing")
				continue
			}
			stats.Add(tag.String())
			if *print {
				id := "unknown"
				if tag, err := ex.Get(exif.ImageUniqueID); err == nil {
					id = tag.String()
					stats.Add("unique-id-exif")
				} else if thumb, err := ex.JpegThumbnail(); err == nil {
					hash := sha1.Sum(thumb)
					id = hex.EncodeToString(hash[:20])
					stats.Add("unique-id-thumb")
				} else if r.PartsSize() < 1e7 {
					if _, err := r.Seek(0, 0); err == nil {
						hash := sha1.New()
						io.Copy(hash, r)
						id = hex.EncodeToString(hash.Sum(nil))
						stats.Add("unique-id-sha1")
					} else {
						id = "read-error"
						stats.Add("unique-id-sha1-error")
					}
				} else {
					stats.Add("unique-id-too-big")
				}
				fmt.Printf("%s %s %q %q\n", r.BlobRef(), id, r.FileName(), tag)
			}
		}
	})
	workers.Wait()
}
Пример #2
0
func main() {
	bs := Flag{}

	cat := &commander.Command{
		UsageLine: "cat prints blob contents",
		Run: func(cmd *commander.Command, args []string) error {
			if bs.BS == nil {
				return errors.New("require --blob_dir")
			}
			for _, ref := range args {
				br, ok := blob.Parse(ref)
				if !ok {
					fmt.Fprintf(os.Stderr, "couldn't parse ref\n")
					continue
				}
				blob, _, err := bs.BS.Fetch(br)
				if err != nil {
					fmt.Fprintf(os.Stderr, "%s: %s\n", ref, err)
					continue
				}
				io.Copy(os.Stdout, blob)
				blob.Close()
			}
			return nil
		},
	}

	tar := &commander.Command{
		UsageLine: "tar exports files from the blobstore",
		Run: func(cmd *commander.Command, args []string) error {
			if bs.BS == nil {
				return errors.New("require --blob_dir")
			}
			files := fsck.NewFiles(bs.BS)
			go files.LogErrors()

			// read blobrefs from stdin
			refsCh := make(chan string, 20)
			go func() {
				in := bufio.NewScanner(os.Stdin)
				for in.Scan() {
					// TODO(dichro): validate ref?
					refsCh <- in.Text()
				}
				close(refsCh)
			}()

			go func() {
				files.ReadRefs(refsCh)
				files.Close()
			}()

			out := tar.NewWriter(os.Stdout)
			defer out.Flush()

			for r := range files.Readers {
				size := r.PartsSize()
				if err := out.WriteHeader(&tar.Header{
					Name:     r.FileName(),
					Mode:     int64(r.FileMode()),
					Uid:      r.MapUid(),
					Gid:      r.MapGid(),
					Size:     size,
					ModTime:  r.ModTime(),
					Typeflag: tar.TypeReg,
				}); err != nil {
					log.Fatal(err)
				}
				switch n, err := io.Copy(out, r); {
				case err != nil:
					log.Fatal(err)
				case n != size:
					log.Fatalf("wrote %d of %d", n, size)
				}
			}
			return nil
		},
	}

	top := &commander.Command{
		UsageLine: os.Args[0],
		Subcommands: []*commander.Command{
			cat,
			tar,
		},
	}

	for _, cmd := range top.Subcommands {
		cmd.Flag.Var(&bs, "blob_dir", "Camlistore blob directory")
	}

	if err := top.Dispatch(os.Args[1:]); err != nil {
		log.Fatal(err)
	}
}