func newUploader() *Uploader { var cc *client.Client var httpStats *httputil.StatsTransport if d := *flagBlobDir; d != "" { ss, err := dir.New(d) if err != nil && d == "discard" { ss = discardStorage{} err = nil } if err != nil { log.Fatalf("Error using dir %s as storage: %v", d, err) } cc = client.NewStorageClient(ss) } else { var proxy func(*http.Request) (*url.URL, error) if flagProxyLocal { proxy = proxyFromEnvironment } cc = client.NewOrFail(client.OptionTransportConfig(&client.TransportConfig{ Proxy: proxy, Verbose: *flagHTTP, })) httpStats = cc.HTTPStats() } if *cmdmain.FlagVerbose { cc.SetLogger(log.New(cmdmain.Stderr, "", log.LstdFlags)) } else { cc.SetLogger(nil) } pwd, err := os.Getwd() if err != nil { log.Fatalf("os.Getwd: %v", err) } return &Uploader{ Client: cc, stats: httpStats, pwd: pwd, fdGate: syncutil.NewGate(100), // gate things that waste fds, assuming a low system limit } }
func main() { client.AddFlags() flag.Parse() if *cmdmain.FlagHelp { flag.PrintDefaults() } if *flagVersion { fmt.Fprintf(os.Stderr, "camget version: %s\n", buildinfo.Version()) return } if *cmdmain.FlagLegal { cmdmain.PrintLicenses() return } if *flagGraph && flag.NArg() != 1 { log.Fatalf("The --graph option requires exactly one parameter.") } var cl *client.Client var items []blob.Ref optTransportConfig := client.OptionTransportConfig(&client.TransportConfig{ Verbose: *flagHTTP, }) if *flagShared != "" { if client.ExplicitServer() != "" { log.Fatal("Can't use --shared with an explicit blobserver; blobserver is implicit from the --shared URL.") } if flag.NArg() != 0 { log.Fatal("No arguments permitted when using --shared") } cl1, target, err := client.NewFromShareRoot(*flagShared, client.OptionInsecure(*flagInsecureTLS), client.OptionTrustedCert(*flagTrustedCert), optTransportConfig, ) if err != nil { log.Fatal(err) } cl = cl1 items = append(items, target) } else { if *flagTrustedCert != "" { log.Fatal("Can't use --cert without --shared.") } cl = client.NewOrFail(client.OptionInsecure(*flagInsecureTLS), optTransportConfig) for n := 0; n < flag.NArg(); n++ { arg := flag.Arg(n) br, ok := blob.Parse(arg) if !ok { log.Fatalf("Failed to parse argument %q as a blobref.", arg) } items = append(items, br) } } httpStats := cl.HTTPStats() diskCacheFetcher, err := cacher.NewDiskCache(cl) if err != nil { log.Fatalf("Error setting up local disk cache: %v", err) } defer diskCacheFetcher.Clean() if *flagVerbose { log.Printf("Using temp blob cache directory %s", diskCacheFetcher.Root) } if *flagShared != "" { diskCacheFetcher.SetCacheHitHook(func(br blob.Ref, rc io.ReadCloser) (io.ReadCloser, error) { var buf bytes.Buffer if err := cl.UpdateShareChain(br, io.TeeReader(rc, &buf)); err != nil { rc.Close() return nil, err } return struct { io.Reader io.Closer }{io.MultiReader(&buf, rc), rc}, nil }) } for _, br := range items { if *flagGraph { printGraph(diskCacheFetcher, br) return } if *flagCheck { // TODO: do HEAD requests checking if the blobs exists. log.Fatal("not implemented") return } if *flagOutput == "-" { var rc io.ReadCloser var err error if *flagContents { rc, err = schema.NewFileReader(diskCacheFetcher, br) if err == nil { rc.(*schema.FileReader).LoadAllChunks() } } else { rc, err = fetch(diskCacheFetcher, br) } if err != nil { log.Fatal(err) } defer rc.Close() if _, err := io.Copy(os.Stdout, rc); err != nil { log.Fatalf("Failed reading %q: %v", br, err) } } else { if err := smartFetch(diskCacheFetcher, *flagOutput, br); err != nil { log.Fatal(err) } } } if *flagVerbose { log.Printf("HTTP requests: %d\n", httpStats.Requests()) h1, h2 := httpStats.ProtoVersions() log.Printf(" responses: %d (h1), %d (h2)\n", h1, h2) } }