func (c *syncCmd) RunCommand(args []string) error { if c.loop && !c.removeSrc { return cmdmain.UsageError("Can't use --loop without --removesrc") } if c.verbose { c.logger = log.New(os.Stderr, "", 0) // else nil } if c.all { err := c.syncAll() if err != nil { return fmt.Errorf("sync all failed: %v", err) } return nil } if c.dest == "" { return cmdmain.UsageError("No --dest specified.") } discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return fmt.Errorf("Failed to get blob source: %v", err) } sc := client.New(src) sc.SetupAuth() sc.SetLogger(c.logger) dc := client.New(c.dest) dc.SetupAuth() dc.SetLogger(c.logger) var tc *client.Client if c.third != "" { tc = client.New(c.third) tc.SetupAuth() tc.SetLogger(c.logger) } passNum := 0 for { passNum++ stats, err := c.doPass(sc, dc, tc) if c.verbose { log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied) } if err != nil { return fmt.Errorf("sync failed: %v", err) } if !c.loop { break } } return nil }
func (c *syncCmd) syncAll() error { if c.loop { return cmdmain.UsageError("--all can't be used with --loop") } if c.third != "" { return cmdmain.UsageError("--all can't be used with --thirdleg") } if c.dest != "" { return cmdmain.UsageError("--all can't be used with --dest") } dc := c.discoClient() dc.SetLogger(c.logger) syncHandlers, err := dc.SyncHandlers() if err != nil { return fmt.Errorf("sync handlers discovery failed: %v", err) } if c.verbose { log.Printf("To be synced:\n") for _, sh := range syncHandlers { log.Printf("%v -> %v", sh.From, sh.To) } } for _, sh := range syncHandlers { from := client.New(sh.From) from.SetLogger(c.logger) from.InsecureTLS = c.insecureTLS from.SetHTTPClient(&http.Client{ Transport: from.TransportForConfig(nil), }) if err := from.SetupAuth(); err != nil { return fmt.Errorf("could not setup auth for connecting to %v: %v", sh.From, err) } to := client.New(sh.To) to.SetLogger(c.logger) to.InsecureTLS = c.insecureTLS to.SetHTTPClient(&http.Client{ Transport: to.TransportForConfig(nil), }) if err := to.SetupAuth(); err != nil { return fmt.Errorf("could not setup auth for connecting to %v: %v", sh.To, err) } if c.verbose { log.Printf("Now syncing: %v -> %v", sh.From, sh.To) } stats, err := c.doPass(from, to, nil) if c.verbose { log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied) } if err != nil { return err } } return nil }
func main() { flag.Parse() if *flagLoop && !*flagRemoveSource { usage("Can't use --loop without --removesrc") } if *flagVerbose { logger = log.New(os.Stderr, "", 0) } if *flagAll { err := syncAll() if err != nil { log.Fatalf("sync all failed: %v", err) } return } if *flagDest == "" { usage("No --dest specified.") } discl := discoClient() discl.SetLogger(logger) src, err := discl.BlobRoot() if err != nil { log.Fatalf("Failed to get blob source: %v", err) } sc := client.New(src) sc.SetupAuth() dc := client.New(*flagDest) dc.SetupAuth() sc.SetLogger(logger) dc.SetLogger(logger) passNum := 0 for { passNum++ stats, err := doPass(sc, dc) if *flagVerbose { log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied) } if err != nil { log.Fatalf("sync failed: %v", err) } if !*flagLoop { break } } }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { url := config.RequiredString("url") auth := config.RequiredString("auth") skipStartupCheck := config.OptionalBool("skipStartupCheck", false) if err := config.Validate(); err != nil { return nil, err } client := client.New(url) if err = client.SetupAuthFromString(auth); err != nil { return nil, err } client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags)) sto := &remoteStorage{ client: client, } if !skipStartupCheck { // Do a quick dummy operation to check that our credentials are // correct. // TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason. c := make(chan blob.SizedRef, 1) err = sto.EnumerateBlobs(context.TODO(), c, "", 1) if err != nil { return nil, err } } return sto, nil }
// which is one of "src", "dest", or "thirdleg" func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) { if val == "" { switch which { case storageThird: return nil, nil case storageSource: discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err) } val = src } if val == "" { return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified") } } if which == storageDest && val == "stdout" { return nil, nil } if looksLikePath(val) { disk, err := localdisk.New(val) if err != nil { return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", val, err) } return disk, nil } cl := client.New(val) // TODO(mpl): probably needs the transport setup for trusted certs here. cl.SetupAuth() cl.SetLogger(c.logger) return noHub{cl}, nil }
// which is one of "src", "dest", or "thirdleg" func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) { var httpClient *http.Client if val == "" { switch which { case storageThird: return nil, nil case storageSource: discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err) } val = src httpClient = discl.HTTPClient() } if val == "" { return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified") } } if which == storageDest && val == "stdout" { return nil, nil } if looksLikePath(val) { disk, err := localdisk.New(val) if err != nil { return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", which, val, err) } c.oneIsDisk = true return disk, nil } cl := client.New(val) cl.InsecureTLS = c.insecureTLS if httpClient == nil { httpClient = &http.Client{ Transport: cl.TransportForConfig(nil), } } cl.SetHTTPClient(httpClient) if err := cl.SetupAuth(); err != nil { return nil, fmt.Errorf("could not setup auth for connecting to %v: %v", val, err) } cl.SetLogger(c.logger) serverKeyID, err := cl.ServerKeyID() if err != nil && err != client.ErrNoSigning { fmt.Fprintf(cmdmain.Stderr, "Failed to discover keyId for server %v: %v", val, err) } else { if which == storageSource { c.srcKeyID = serverKeyID } else if which == storageDest { c.destKeyID = serverKeyID } } return cl, nil }
// newClient returns a Camlistore client for the server. // The server may be: // * blank, to use the default in the config file // * an alias, to use that named alias in the config file // * host:port // * https?://host[:port][/path] func newClient(server string, opts ...client.ClientOption) *client.Client { if server == "" { return client.NewOrFail(opts...) } cl := client.New(server, opts...) if err := cl.SetupAuth(); err != nil { log.Fatalf("Could not setup auth for connecting to %v: %v", server, err) } return cl }
func (c *syncCmd) syncAll() error { if c.loop { return cmdmain.UsageError("--all can't be used with --loop") } if c.third != "" { return cmdmain.UsageError("--all can't be used with --thirdleg") } if c.dest != "" { return cmdmain.UsageError("--all can't be used with --dest") } dc := c.discoClient() dc.SetLogger(c.logger) syncHandlers, err := dc.SyncHandlers() if err != nil { return fmt.Errorf("sync handlers discovery failed: %v", err) } if c.verbose { log.Printf("To be synced:\n") for _, sh := range syncHandlers { log.Printf("%v -> %v", sh.From, sh.To) } } for _, sh := range syncHandlers { from := client.New(sh.From) from.SetLogger(c.logger) from.SetupAuth() to := client.New(sh.To) to.SetLogger(c.logger) to.SetupAuth() if c.verbose { log.Printf("Now syncing: %v -> %v", sh.From, sh.To) } stats, err := c.doPass(noHub{from}, noHub{to}, nil) if c.verbose { log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied) } if err != nil { return err } } return nil }
// discoClient returns a client initialized with a server // based from --src or from the configuration file if --src // is blank. The returned client can then be used to discover // the blobRoot and syncHandlers. func discoClient() *client.Client { var cl *client.Client if *flagSrc == "" { cl = client.NewOrFail() } else { cl = client.New(*flagSrc) } cl.SetupAuth() return cl }
// discoClient returns a client initialized with a server // based from --src or from the configuration file if --src // is blank. The returned client can then be used to discover // the blobRoot and syncHandlers. func (c *syncCmd) discoClient() *client.Client { var cl *client.Client if c.src == "" { cl = client.NewOrFail() } else { cl = client.New(c.src) } cl.SetupAuth() return cl }
func TestPublishURLs(t *testing.T) { rootName := "foo" idxd := setupContent(rootName) sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef) corpus, err := idxd.Index.KeepInMemory() if err != nil { t.Fatalf("error slurping index to memory: %v", err) } sh.SetCorpus(corpus) cl := camliClient.New("http://whatever.fake") fcl := &fakeClient{cl, sh} ph := &publishHandler{ rootName: rootName, cl: fcl, } if err := ph.initRootNode(); err != nil { t.Fatalf("initRootNode: %v", err) } for ti, tt := range publishURLTests { rw := httptest.NewRecorder() if !strings.HasPrefix(tt.path, "/pics/") { panic("expected /pics/ prefix on " + tt.path) } req, _ := http.NewRequest("GET", "http://foo.com"+tt.path, nil) pfxh := &httputil.PrefixHandler{ Prefix: "/", Handler: http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { // Because the app handler strips the prefix before passing it on to the app req.URL.Path = strings.TrimPrefix(req.URL.Path, "/pics/") pr, err := ph.NewRequest(rw, req) if err != nil { t.Fatalf("test #%d, NewRequest: %v", ti, err) } err = pr.findSubject() if tt.subject != "" { if err != nil { t.Errorf("test #%d, findSubject: %v", ti, err) return } if pr.subject.String() != tt.subject { t.Errorf("test #%d, got subject %q, want %q", ti, pr.subject, tt.subject) } } if pr.subres != tt.subres { t.Errorf("test #%d, got subres %q, want %q", ti, pr.subres, tt.subres) } }), } pfxh.ServeHTTP(rw, req) } }
func main() { flag.Parse() if *flagSrc == "" { usage("No --src specified.") } if *flagDest == "" { usage("No --dest specified.") } if *flagLoop && !*flagRemoveSource { usage("Can't use --loop without --removesrc") } // TODO(mpl): adapt to the userpass scheme once it has settled sc := client.New(*flagSrc) sc.SetupAuth() dc := client.New(*flagDest) dc.SetupAuth() var logger *log.Logger = nil if *flagVerbose { logger = log.New(os.Stderr, "", 0) } sc.SetLogger(logger) dc.SetLogger(logger) passNum := 0 for { passNum++ stats, err := doPass(sc, dc, passNum) if err != nil { log.Fatalf("sync failed: %v", err) } if *flagVerbose { log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied) } if !*flagLoop { break } } }
func setupClient(w *test.World) (*client.Client, error) { // Do the silly env vars dance to avoid the "non-hermetic use of host config panic". if err := os.Setenv("CAMLI_KEYID", w.ClientIdentity()); err != nil { return nil, err } if err := os.Setenv("CAMLI_SECRET_RING", w.SecretRingFile()); err != nil { return nil, err } osutil.AddSecretRingFlag() cl := client.New(w.ServerBaseURL()) return cl, nil }
func (c *claimsCmd) client() *client.Client { var cl *client.Client if c.src == "" { cl = client.NewOrFail() } else { cl = client.New(c.src) } cl.SetLogger(c.logger) cl.SetHTTPClient(&http.Client{ Transport: cl.TransportForConfig(nil), }) cl.SetupAuth() return cl }
func syncAll() error { if *flagLoop { usage("--all can not be used with --loop") } dc := discoClient() dc.SetLogger(logger) syncHandlers, err := dc.SyncHandlers() if err != nil { log.Fatalf("sync handlers discovery failed: %v", err) } if *flagVerbose { log.Printf("To be synced:\n") for _, sh := range syncHandlers { log.Printf("%v -> %v", sh.From, sh.To) } } for _, sh := range syncHandlers { from := client.New(sh.From) from.SetLogger(logger) from.SetupAuth() to := client.New(sh.To) to.SetLogger(logger) to.SetupAuth() if *flagVerbose { log.Printf("Now syncing: %v -> %v", sh.From, sh.To) } stats, err := doPass(from, to) if *flagVerbose { log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied) } if err != nil { return err } } return nil }
// newClient returns a Camlistore client for the server. // The server may be: // * blank, to use the default in the config file // * an alias, to use that named alias in the config file // * host:port // * https?://host[:port][/path] func newClient(server string) *client.Client { var cl *client.Client if server == "" { cl = client.NewOrFail() } else { cl = client.New(server) if err := cl.SetupAuth(); err != nil { log.Fatalf("Could not setup auth for connecting to %v: %v", server, err) } } cl.SetHTTPClient(&http.Client{ Transport: cl.TransportForConfig(nil), }) return cl }
// discoClient returns a client initialized with a server // based from --src or from the configuration file if --src // is blank. The returned client can then be used to discover // the blobRoot and syncHandlers. func (c *syncCmd) discoClient() *client.Client { var cl *client.Client if c.src == "" { cl = client.NewOrFail() } else { cl = client.New(c.src) } cl.SetLogger(c.logger) cl.InsecureTLS = c.insecureTLS cl.SetHTTPClient(&http.Client{ Transport: cl.TransportForConfig(nil), }) cl.SetupAuth() return cl }
func setupClient(w *test.World) (*client.Client, error) { // Do the silly env vars dance to avoid the "non-hermetic use of host config panic". if err := os.Setenv("CAMLI_KEYID", w.ClientIdentity()); err != nil { return nil, err } if err := os.Setenv("CAMLI_SECRET_RING", w.SecretRingFile()); err != nil { return nil, err } osutil.AddSecretRingFlag() cl := client.New(w.ServerBaseURL()) // This permanode is not needed in itself, but that takes care of uploading // behind the scenes the public key to the blob server. A bit gross, but // it's just for a test anyway. if _, err := cl.UploadNewPermanode(); err != nil { return nil, err } return cl, nil }
// which is one of "src", "dest", or "thirdleg" func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) { var httpClient *http.Client if val == "" { switch which { case storageThird: return nil, nil case storageSource: discl := c.discoClient() discl.SetLogger(c.logger) src, err := discl.BlobRoot() if err != nil { return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err) } val = src httpClient = discl.HTTPClient() } if val == "" { return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified") } } if which == storageDest && val == "stdout" { return nil, nil } if looksLikePath(val) { disk, err := localdisk.New(val) if err != nil { return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", which, val, err) } return disk, nil } cl := client.New(val) cl.InsecureTLS = c.insecureTLS if httpClient == nil { httpClient = &http.Client{ Transport: cl.TransportForConfig(nil), } } cl.SetHTTPClient(httpClient) cl.SetupAuth() cl.SetLogger(c.logger) return cl, nil }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { url := config.RequiredString("url") skipStartupCheck := config.OptionalBool("skipStartupCheck", false) if err := config.Validate(); err != nil { return nil, err } client := client.New(url) err = client.SetupAuthFromConfig(config) if err != nil { return nil, err } sto := &remoteStorage{ client: client, } if !skipStartupCheck { // TODO: do a server stat or something to check password } return sto, nil }
func TestPublishMembers(t *testing.T) { rootName := "foo" idxd := setupContent(rootName) sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef) corpus, err := idxd.Index.KeepInMemory() if err != nil { t.Fatalf("error slurping index to memory: %v", err) } sh.SetCorpus(corpus) cl := camliClient.New("http://whatever.fake") fcl := &fakeClient{cl, sh} ph := &publishHandler{ rootName: rootName, cl: fcl, } rw := httptest.NewRecorder() req, _ := http.NewRequest("GET", "http://foo.com/pics", nil) pfxh := &httputil.PrefixHandler{ Prefix: "/pics/", Handler: http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { pr, err := ph.NewRequest(rw, req) if err != nil { t.Fatalf("NewRequest: %v", err) } res, err := pr.ph.deepDescribe(pr.subject) if err != nil { t.Fatalf("deepDescribe: %v", err) } members, err := pr.subjectMembers(res.Meta) if len(members.Members) != 2 { t.Errorf("Expected two members in publish root (one camlipath, one camlimember), got %d", len(members.Members)) } }), } pfxh.ServeHTTP(rw, req) }
func createObjStore() (*objStore, error) { db := camli.New("localhost:3179") return &objStore{newObjInventory(), db}, nil }
func (c *serverCmd) makeThings() error { const importerPrefix = "/importer/" // check that "/importer/" prefix is in config, just in case it ever changes. configFile := filepath.Join(camliSrcRoot, "config", "dev-server-config.json") config, err := ioutil.ReadFile(configFile) if err != nil { return fmt.Errorf("could not read config file %v: %v", configFile, err) } if !bytes.Contains(config, []byte(importerPrefix)) { return fmt.Errorf("%s prefix not found in dev config. Did it change?", importerPrefix) } if err := netutil.AwaitReachable("localhost:"+c.port, time.Minute); err != nil { return err } osutil.AddSecretRingFlag() setCamdevVars() baseURL := c.env.m["CAMLI_BASEURL"] if baseURL == "" { return errors.New("CAMLI_BASEURL is not set") } cl := client.New(baseURL) signer, err := cl.Signer() if err != nil { return err } ClientId := make(map[string]string) ClientSecret := make(map[string]string) for name := range importer.All() { ClientId[name] = "fakeStaticClientId" ClientSecret[name] = "fakeStaticClientSecret" } hc := importer.HostConfig{ BaseURL: baseURL, Prefix: importerPrefix, Target: cl, BlobSource: cl, Signer: signer, Search: cl, ClientId: ClientId, ClientSecret: ClientSecret, } for name, imp := range importer.All() { mk, ok := imp.(importer.TestDataMaker) if !ok { continue } tr := mk.MakeTestData() hc.HTTPClient = &http.Client{Transport: tr} host, err := importer.NewHost(hc) if err != nil { return fmt.Errorf("could not obtain Host: %v", err) } rc, err := importer.CreateAccount(host, name) if err != nil { return err } if err := mk.SetTestAccount(rc.AccountNode()); err != nil { return fmt.Errorf("could not set fake account node for importer %v: %v", name, err) } if err := imp.Run(rc); err != nil { return err } } return nil }