// which is one of "src", "dest", or "thirdleg" func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) { if val == "" { switch which { case storageThird: return nil, nil case storageSource: discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err) } val = src } if val == "" { return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified") } } if which == storageDest && val == "stdout" { return nil, nil } if looksLikePath(val) { disk, err := localdisk.New(val) if err != nil { return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", val, err) } return disk, nil } cl := client.New(val) // TODO(mpl): probably needs the transport setup for trusted certs here. cl.SetupAuth() cl.SetLogger(c.logger) return noHub{cl}, nil }
// killReindex starts a reindexing in a new process, and kills that process // after killTime. It then (naively for now ?) verifies that the kv store file is // not corrupted by reinitializing an (possibly incomplete) index (with a corpus) // with it. If the indexing was completed before we could kill the process, it // returns true, false otherwise. func killReindex(b *testing.B, dbfile string, killTime time.Duration, sortedProvider func(dbfile string) (sorted.KeyValue, error)) bool { cmd := exec.Command(os.Args[0], "-test.run=TestChildIndexer") cmd.Env = append(cmd.Env, "TEST_BE_CHILD=1", "TEST_BE_CHILD_DBFILE="+dbfile) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Start(); err != nil { b.Fatal(err) } waitc := make(chan error) go func() { waitc <- cmd.Wait() }() fullIndex := false select { case err := <-waitc: if err == nil { // indexer finished before we killed it fullIndex = true b.Logf("Finished indexing before being killed at %v", killTime) break } // TODO(mpl): do better if err.Error() != "signal: killed" { b.Fatalf("unexpected (not killed) error from indexer process: %v %v %v", err, stdout.String(), stderr.String()) } case <-time.After(killTime): if err := cmd.Process.Kill(); err != nil { b.Fatal(err) } err := <-waitc // TODO(mpl): do better if err != nil && err.Error() != "signal: killed" { b.Fatalf("unexpected (not killed) error from indexer process: %v %v %v", err, stdout.String(), stderr.String()) } } kv, err := sortedProvider(dbfile) if err != nil { b.Fatal(err) } idx, err := index.New(kv) if err != nil { b.Fatal(err) } bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs")) if err != nil { b.Fatal(err) } idx.InitBlobSource(bs) if _, err := idx.KeepInMemory(); err != nil { b.Fatal(err) } if err := idx.Close(); err != nil { b.Fatal(err) } return fullIndex }
// TODO(rh): tame copy/paste code from cammount func main() { client.AddFlags() flag.Parse() cacheDir, err := ioutil.TempDir("", "camlicache") if err != nil { log.Fatalf("Error creating temp cache directory: %v", err) } defer os.RemoveAll(cacheDir) diskcache, err := localdisk.New(cacheDir) if err != nil { log.Fatalf("Error setting up local disk cache: %v", err) } if flag.NArg() != 1 { log.Fatal("usage: camwebdav <blobref>") } br := blobref.Parse(flag.Arg(0)) if br == nil { log.Fatalf("%s was not a valid blobref.", flag.Arg(0)) } client := client.NewOrFail() fetcher := cacher.NewCachingFetcher(diskcache, client) f = fs.NewCamliFileSystem(fetcher, br) http.HandleFunc("/", webdav) err = http.ListenAndServe(*davaddr, nil) if err != nil { log.Fatalf("Error starting WebDAV server: %v", err) } }
func reindex(b *testing.B, dbfile string, sortedProvider func(dbfile string) (sorted.KeyValue, error)) *index.Index { b.Logf("reindexing") if err := os.RemoveAll(dbfile); err != nil { b.Fatal(err) } kv, err := sortedProvider(dbfile) if err != nil { b.Fatal(err) } bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs")) if err != nil { b.Fatal(err) } idx, err := index.New(kv) if err != nil { b.Fatal(err) } idx.InitBlobSource(bs) b.ResetTimer() if err := idx.Reindex(); err != nil { b.Fatal(err) } return idx }
func enumerateMeta(b *testing.B, dbfile string, sortedProvider func(dbfile string) (sorted.KeyValue, error)) int { b.Logf("enumerating meta blobs") kv, err := sortedProvider(dbfile) if err != nil { b.Fatal(err) } bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs")) if err != nil { b.Fatal(err) } idx, err := index.New(kv) if err != nil { b.Fatal(err) } idx.InitBlobSource(bs) defer idx.Close() ch := make(chan camtypes.BlobMeta, 100) go func() { if err := idx.EnumerateBlobMeta(nil, ch); err != nil { b.Fatal(err) } }() n := 0 for range ch { n++ } b.Logf("Enumerated %d meta blobs", n) return n }
func main() { // Scans the arg list and sets up flags debug := flag.Bool("debug", false, "print debugging messages.") client.AddFlags() flag.Parse() errorf := func(msg string, args ...interface{}) { fmt.Fprintf(os.Stderr, msg, args...) os.Exit(2) } if n := flag.NArg(); n < 1 || n > 2 { errorf("usage: cammount <mountpoint> [<root-blobref>]\n") } mountPoint := flag.Arg(0) client := client.NewOrFail() // automatic from flags cacheDir, err := ioutil.TempDir("", "camlicache") if err != nil { errorf("Error creating temp cache directory: %v\n", err) } defer os.RemoveAll(cacheDir) diskcache, err := localdisk.New(cacheDir) if err != nil { errorf("Error setting up local disk cache: %v", err) } fetcher := cacher.NewCachingFetcher(diskcache, client) var camfs *fs.CamliFileSystem if flag.NArg() == 2 { root := blobref.Parse(flag.Arg(1)) if root == nil { errorf("Error parsing root blobref: %q\n", root) } var err error camfs, err = fs.NewRootedCamliFileSystem(fetcher, root) if err != nil { errorf("Error creating root with %v: %v", root, err) } } else { camfs = fs.NewCamliFileSystem(fetcher) log.Printf("starting with fs %#v", camfs) } if *debug { // TODO: set fs's logger } conn, err := fuse.Mount(mountPoint) if err != nil { log.Fatalf("Mount: %v", err) } err = conn.Serve(camfs) if err != nil { log.Fatalf("Serve: %v", err) } log.Printf("fuse process ending.") }
// which is one of "src", "dest", or "thirdleg" func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) { var httpClient *http.Client if val == "" { switch which { case storageThird: return nil, nil case storageSource: discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err) } val = src httpClient = discl.HTTPClient() } if val == "" { return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified") } } if which == storageDest && val == "stdout" { return nil, nil } if looksLikePath(val) { disk, err := localdisk.New(val) if err != nil { return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", which, val, err) } c.oneIsDisk = true return disk, nil } cl := client.New(val) cl.InsecureTLS = c.insecureTLS if httpClient == nil { httpClient = &http.Client{ Transport: cl.TransportForConfig(nil), } } cl.SetHTTPClient(httpClient) if err := cl.SetupAuth(); err != nil { return nil, fmt.Errorf("could not setup auth for connecting to %v: %v", val, err) } cl.SetLogger(c.logger) serverKeyID, err := cl.ServerKeyID() if err != nil && err != client.ErrNoSigning { fmt.Fprintf(cmdmain.Stderr, "Failed to discover keyId for server %v: %v", val, err) } else { if which == storageSource { c.srcKeyID = serverKeyID } else if which == storageDest { c.destKeyID = serverKeyID } } return cl, nil }
// New returns a new blobserver Storage implementation, storing blobs in the provided dir. // If dir has an index.kv file, a diskpacked implementation is returnd. func New(dir string) (blobserver.Storage, error) { if v, err := diskpacked.IsDir(dir); err != nil { return nil, err } else if v { return diskpacked.New(dir) } if v, err := localdisk.IsDir(dir); err != nil { return nil, err } else if v { return localdisk.New(dir) } return diskpacked.New(dir) }
// NewDiskCache returns a new DiskCache from a StreamingFetcher, which // is usually the pkg/client HTTP client (which typically has much // higher latency and lower bandwidth than local disk). func NewDiskCache(fetcher blob.StreamingFetcher) (*DiskCache, error) { // TODO: max disk size, keep LRU of access, smarter cleaning, // persistent directory per-user, etc. cacheDir, err := ioutil.TempDir("", "camlicache") if err != nil { return nil, err } diskcache, err := localdisk.New(cacheDir) if err != nil { return nil, err } dc := &DiskCache{ CachingFetcher: NewCachingFetcher(diskcache, fetcher), Root: cacheDir, } return dc, nil }
func TestChildIndexer(t *testing.T) { if os.Getenv("TEST_BE_CHILD") != "1" { t.Skip("not a real test; used as a child process by the benchmarks") } dbfile := os.Getenv("TEST_BE_CHILD_DBFILE") if dbfile == "" { log.Fatal("empty TEST_BE_CHILD_DBFILE") } if err := os.RemoveAll(dbfile); err != nil { log.Fatal(err) } var kv sorted.KeyValue var err error switch { case strings.HasSuffix(dbfile, "leveldb.db"): kv, err = leveldb.NewStorage(dbfile) case strings.HasSuffix(dbfile, "kvfile.db"): kv, err = kvfile.NewStorage(dbfile) case strings.HasSuffix(dbfile, "sqlite.db"): kv, err = sqlite.NewStorage(dbfile) default: log.Fatalf("unknown sorted provider for %v", dbfile) } if err != nil { log.Fatal(err) } bs, err := localdisk.New(filepath.Join(filepath.Dir(dbfile), "bs")) if err != nil { log.Fatal(err) } idx, err := index.New(kv) if err != nil { log.Fatal(err) } idx.InitBlobSource(bs) defer func() { if err := idx.Close(); err != nil { log.Fatal(err) } }() if err := idx.Reindex(); err != nil { log.Fatal(err) } }
// which is one of "src", "dest", or "thirdleg" func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) { var httpClient *http.Client if val == "" { switch which { case storageThird: return nil, nil case storageSource: discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err) } val = src httpClient = discl.HTTPClient() } if val == "" { return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified") } } if which == storageDest && val == "stdout" { return nil, nil } if looksLikePath(val) { disk, err := localdisk.New(val) if err != nil { return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", which, val, err) } return disk, nil } cl := client.New(val) cl.InsecureTLS = c.insecureTLS if httpClient == nil { httpClient = &http.Client{ Transport: cl.TransportForConfig(nil), } } cl.SetHTTPClient(httpClient) cl.SetupAuth() cl.SetLogger(c.logger) return cl, nil }
// dataStores returns the blobserver that stores the instances configurations, and the kv // store for the instances states. func dataStores() (blobserver.Storage, sorted.KeyValue, error) { dataDir := os.Getenv("CAMLI_GCE_DATA") if dataDir == "" { dataDir = "camli-gce-data" log.Printf("data dir not provided as env var CAMLI_GCE_DATA, so defaulting to %v", dataDir) } blobsDir := filepath.Join(dataDir, "instance-conf") if err := os.MkdirAll(blobsDir, 0700); err != nil { return nil, nil, err } instConf, err := localdisk.New(blobsDir) if err != nil { return nil, nil, err } instState, err := leveldb.NewStorage(filepath.Join(dataDir, "instance-state")) if err != nil { return nil, nil, err } return instConf, instState, nil }
// NewDiskCache returns a new DiskCache from a Fetcher, which // is usually the pkg/client HTTP client (which typically has much // higher latency and lower bandwidth than local disk). func NewDiskCache(fetcher blob.Fetcher) (*DiskCache, error) { cacheDir := filepath.Join(osutil.CacheDir(), "blobs") if !osutil.DirExists(cacheDir) { if err := os.Mkdir(cacheDir, 0700); err != nil { log.Printf("Warning: failed to make %s: %v; using tempdir instead", cacheDir, err) cacheDir, err = ioutil.TempDir("", "camlicache") if err != nil { return nil, err } } } // TODO: max disk size, keep LRU of access, smarter cleaning, etc // TODO: use diskpacked instead? harder to clean, though. diskcache, err := localdisk.New(cacheDir) if err != nil { return nil, err } dc := &DiskCache{ CachingFetcher: NewCachingFetcher(diskcache, fetcher), Root: cacheDir, } return dc, nil }
// Populates the bs, and the index at the same time through the sync handler func populate(b *testing.B, dbfile string, sortedProvider func(dbfile string) (sorted.KeyValue, error)) *index.Index { b.Logf("populating %v", dbfile) kv, err := sortedProvider(dbfile) if err != nil { b.Fatal(err) } bsRoot := filepath.Join(filepath.Dir(dbfile), "bs") if err := os.MkdirAll(bsRoot, 0700); err != nil { b.Fatal(err) } dataDir, err := os.Open("testdata") if err != nil { b.Fatal(err) } fis, err := dataDir.Readdir(-1) if err != nil { b.Fatal(err) } if len(fis) == 0 { b.Fatalf("no files in %s dir", "testdata") } ks := doKeyStuff(b) bs, err := localdisk.New(bsRoot) if err != nil { b.Fatal(err) } if _, err := blobserver.Receive(bs, ks.pubKeyRef, strings.NewReader(ks.pubKey)); err != nil { b.Fatal(err) } idx, err := index.New(kv) if err != nil { b.Fatal(err) } idx.InitBlobSource(bs) sh := server.NewSyncHandler("/bs/", "/index/", bs, idx, sorted.NewMemoryKeyValue()) b.ResetTimer() for _, v := range fis { f, err := os.Open(filepath.Join(dataDir.Name(), v.Name())) if err != nil { b.Fatal(err) } td := &trackDigestReader{r: f} fm := schema.NewFileMap(v.Name()) fm.SetModTime(v.ModTime()) fileRef, err := schema.WriteFileMap(bs, fm, td) if err != nil { b.Fatal(err) } f.Close() unsigned := schema.NewPlannedPermanode(td.Sum()) unsigned.SetSigner(ks.pubKeyRef) sr := &jsonsign.SignRequest{ UnsignedJSON: unsigned.Blob().JSON(), // TODO(mpl): if we make a bs that discards, replace this with a memory bs that has only the pubkey Fetcher: bs, EntityFetcher: ks.entityFetcher, SignatureTime: time.Unix(0, 0), } signed, err := sr.Sign() if err != nil { b.Fatal("problem signing: " + err.Error()) } pn := blob.SHA1FromString(signed) // N.B: use blobserver.Receive so that the blob hub gets notified, and the blob gets enqueued into the index if _, err := blobserver.Receive(bs, pn, strings.NewReader(signed)); err != nil { b.Fatal(err) } contentAttr := schema.NewSetAttributeClaim(pn, "camliContent", fileRef.String()) claimTime, ok := fm.ModTime() if !ok { b.Fatal(err) } contentAttr.SetClaimDate(claimTime) contentAttr.SetSigner(ks.pubKeyRef) sr = &jsonsign.SignRequest{ UnsignedJSON: contentAttr.Blob().JSON(), // TODO(mpl): if we make a bs that discards, replace this with a memory bs that has only the pubkey Fetcher: bs, EntityFetcher: ks.entityFetcher, SignatureTime: claimTime, } signed, err = sr.Sign() if err != nil { b.Fatal("problem signing: " + err.Error()) } cl := blob.SHA1FromString(signed) if _, err := blobserver.Receive(bs, cl, strings.NewReader(signed)); err != nil { b.Fatal(err) } } sh.IdleWait() return idx }
func newPublishHandler(conf *config) *publishHandler { cl, err := app.Client() if err != nil { logger.Fatalf("could not get a client for the publish handler %v", err) } if conf.RootName == "" { logger.Fatal("camliRoot not found in the app configuration") } maxResizeBytes := conf.MaxResizeBytes if maxResizeBytes == 0 { maxResizeBytes = constants.DefaultMaxResizeMem } var CSSFiles, JSDeps []string if conf.SourceRoot != "" { appRoot := filepath.Join(conf.SourceRoot, "app", "publisher") Files = &fileembed.Files{ DirFallback: appRoot, } // TODO(mpl): Can I readdir by listing with "/" on Files, even with DirFallBack? // Apparently not, but retry later. dir, err := os.Open(appRoot) if err != nil { logger.Fatal(err) } defer dir.Close() names, err := dir.Readdirnames(-1) if err != nil { logger.Fatal(err) } for _, v := range names { if strings.HasSuffix(v, ".css") { CSSFiles = append(CSSFiles, v) continue } // TODO(mpl): document or fix (use a map?) the ordering // problem: i.e. jquery.js must be sourced before // publisher.js. For now, just cheat by sorting the // slice. if strings.HasSuffix(v, ".js") { JSDeps = append(JSDeps, v) } } sort.Strings(JSDeps) } else { Files.Listable = true dir, err := Files.Open("/") if err != nil { logger.Fatal(err) } defer dir.Close() fis, err := dir.Readdir(-1) if err != nil { logger.Fatal(err) } for _, v := range fis { name := v.Name() if strings.HasSuffix(name, ".css") { CSSFiles = append(CSSFiles, name) continue } if strings.HasSuffix(name, ".js") { JSDeps = append(JSDeps, name) } } sort.Strings(JSDeps) } // TODO(mpl): add all htmls found in Files to the template if none specified? if conf.GoTemplate == "" { logger.Fatal("a go template is required in the app configuration") } goTemplate, err := goTemplate(Files, conf.GoTemplate) if err != nil { logger.Fatal(err) } var cache blobserver.Storage var thumbMeta *server.ThumbMeta if conf.CacheRoot != "" { cache, err = localdisk.New(conf.CacheRoot) if err != nil { logger.Fatalf("Could not create localdisk cache: %v", err) } thumbsCacheDir := filepath.Join(os.TempDir(), "camli-publisher-cache") if err := os.MkdirAll(thumbsCacheDir, 0700); err != nil { logger.Fatalf("Could not create cache dir %s for %v publisher: %v", thumbsCacheDir, conf.RootName, err) } kv, err := sorted.NewKeyValue(map[string]interface{}{ "type": "kv", "file": filepath.Join(thumbsCacheDir, conf.RootName+"-thumbnails.kv"), }) if err != nil { logger.Fatalf("Could not create kv for %v's thumbs cache: %v", conf.RootName, err) } thumbMeta = server.NewThumbMeta(kv) } return &publishHandler{ rootName: conf.RootName, cl: cl, resizeSem: syncutil.NewSem(maxResizeBytes), staticFiles: Files, goTemplate: goTemplate, CSSFiles: CSSFiles, JSDeps: JSDeps, describedCache: make(map[string]*search.DescribedBlob), cache: cache, thumbMeta: thumbMeta, } }
func main() { client.AddFlags() flag.Parse() if *flagGraph && flag.NArg() != 1 { log.Fatalf("The --graph option requires exactly one parameter.") } var cl *client.Client var items []*blobref.BlobRef if *flagShared != "" { if client.ExplicitServer() != "" { log.Fatal("Can't use --shared with an explicit blobserver; blobserver is implicit from the --shared URL.") } if flag.NArg() != 0 { log.Fatal("No arguments permitted when using --shared") } cl1, target, err := client.NewFromShareRoot(*flagShared) if err != nil { log.Fatal(err) } cl = cl1 items = append(items, target) } else { cl = client.NewOrFail() for n := 0; n < flag.NArg(); n++ { arg := flag.Arg(n) br := blobref.Parse(arg) if br == nil { log.Fatalf("Failed to parse argument %q as a blobref.", arg) } items = append(items, br) } } httpStats := &httputil.StatsTransport{ VerboseLog: *flagHTTP, } if *flagHTTP { httpStats.Transport = &http.Transport{ Dial: func(net_, addr string) (net.Conn, error) { log.Printf("Dialing %s", addr) return net.Dial(net_, addr) }, } } cl.SetHTTPClient(&http.Client{Transport: httpStats}) // Put a local disk cache in front of the HTTP client. // TODO: this could be better about proactively cleaning things. // Fetching 2 TB shouldn't write 2 TB to /tmp before it's done. // Maybe the cache needs an LRU/size cap. cacheDir, err := ioutil.TempDir("", "camlicache") if err != nil { log.Fatalf("Error creating temp cache directory: %v\n", err) } defer os.RemoveAll(cacheDir) diskcache, err := localdisk.New(cacheDir) if err != nil { log.Fatalf("Error setting up local disk cache: %v", err) } if *flagVerbose { log.Printf("Using temp blob cache directory %s", cacheDir) } fetcher := cacher.NewCachingFetcher(diskcache, cl) for _, br := range items { if *flagGraph { printGraph(fetcher, br) return } if *flagCheck { // TODO: do HEAD requests checking if the blobs exists. log.Fatal("not implemented") return } if *flagOutput == "-" { var rc io.ReadCloser var err error if *flagContents { seekFetcher := blobref.SeekerFromStreamingFetcher(fetcher) rc, err = schema.NewFileReader(seekFetcher, br) if err == nil { rc.(*schema.FileReader).LoadAllChunks() } } else { rc, err = fetch(fetcher, br) } if err != nil { log.Fatal(err) } defer rc.Close() if _, err := io.Copy(os.Stdout, rc); err != nil { log.Fatalf("Failed reading %q: %v", br, err) } } else { if err := smartFetch(fetcher, *flagOutput, br); err != nil { log.Fatal(err) } } } if *flagVerbose { log.Printf("HTTP requests: %d\n", httpStats.Requests()) } }