func (c *syncCmd) RunCommand(args []string) error {
if c.loop && !c.removeSrc {
return cmdmain.UsageError("Can't use --loop without --removesrc")
}
if c.verbose {
c.logger = log.New(os.Stderr, "", 0) // else nil
}
if c.all {
err := c.syncAll()
if err != nil {
return fmt.Errorf("sync all failed: %v", err)
}
return nil
}
if c.dest == "" {
return cmdmain.UsageError("No --dest specified.")
}
discl := c.discoClient()
discl.SetLogger(c.logger)
src, err := discl.BlobRoot()
if err != nil {
return fmt.Errorf("Failed to get blob source: %v", err)
}
sc := client.New(src)
sc.SetupAuth()
sc.SetLogger(c.logger)
dc := client.New(c.dest)
dc.SetupAuth()
dc.SetLogger(c.logger)
var tc *client.Client
if c.third != "" {
tc = client.New(c.third)
tc.SetupAuth()
tc.SetLogger(c.logger)
}
passNum := 0
for {
passNum++
stats, err := c.doPass(sc, dc, tc)
if c.verbose {
log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
return fmt.Errorf("sync failed: %v", err)
}
if !c.loop {
break
}
}
return nil
}
func (c *syncCmd) syncAll() error {
if c.loop {
return cmdmain.UsageError("--all can't be used with --loop")
}
if c.third != "" {
return cmdmain.UsageError("--all can't be used with --thirdleg")
}
if c.dest != "" {
return cmdmain.UsageError("--all can't be used with --dest")
}
dc := c.discoClient()
dc.SetLogger(c.logger)
syncHandlers, err := dc.SyncHandlers()
if err != nil {
return fmt.Errorf("sync handlers discovery failed: %v", err)
}
if c.verbose {
log.Printf("To be synced:\n")
for _, sh := range syncHandlers {
log.Printf("%v -> %v", sh.From, sh.To)
}
}
for _, sh := range syncHandlers {
from := client.New(sh.From)
from.SetLogger(c.logger)
from.InsecureTLS = c.insecureTLS
from.SetHTTPClient(&http.Client{
Transport: from.TransportForConfig(nil),
})
if err := from.SetupAuth(); err != nil {
return fmt.Errorf("could not setup auth for connecting to %v: %v", sh.From, err)
}
to := client.New(sh.To)
to.SetLogger(c.logger)
to.InsecureTLS = c.insecureTLS
to.SetHTTPClient(&http.Client{
Transport: to.TransportForConfig(nil),
})
if err := to.SetupAuth(); err != nil {
return fmt.Errorf("could not setup auth for connecting to %v: %v", sh.To, err)
}
if c.verbose {
log.Printf("Now syncing: %v -> %v", sh.From, sh.To)
}
stats, err := c.doPass(from, to, nil)
if c.verbose {
log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
return err
}
}
return nil
}
func main() {
flag.Parse()
if *flagLoop && !*flagRemoveSource {
usage("Can't use --loop without --removesrc")
}
if *flagVerbose {
logger = log.New(os.Stderr, "", 0)
}
if *flagAll {
err := syncAll()
if err != nil {
log.Fatalf("sync all failed: %v", err)
}
return
}
if *flagDest == "" {
usage("No --dest specified.")
}
discl := discoClient()
discl.SetLogger(logger)
src, err := discl.BlobRoot()
if err != nil {
log.Fatalf("Failed to get blob source: %v", err)
}
sc := client.New(src)
sc.SetupAuth()
dc := client.New(*flagDest)
dc.SetupAuth()
sc.SetLogger(logger)
dc.SetLogger(logger)
passNum := 0
for {
passNum++
stats, err := doPass(sc, dc)
if *flagVerbose {
log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
log.Fatalf("sync failed: %v", err)
}
if !*flagLoop {
break
}
}
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
url := config.RequiredString("url")
auth := config.RequiredString("auth")
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
client := client.New(url)
if err = client.SetupAuthFromString(auth); err != nil {
return nil, err
}
client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags))
sto := &remoteStorage{
client: client,
}
if !skipStartupCheck {
// Do a quick dummy operation to check that our credentials are
// correct.
// TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason.
c := make(chan blob.SizedRef, 1)
err = sto.EnumerateBlobs(context.TODO(), c, "", 1)
if err != nil {
return nil, err
}
}
return sto, nil
}
// which is one of "src", "dest", or "thirdleg"
func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) {
if val == "" {
switch which {
case storageThird:
return nil, nil
case storageSource:
discl := c.discoClient()
discl.SetLogger(c.logger)
src, err := discl.BlobRoot()
if err != nil {
return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err)
}
val = src
}
if val == "" {
return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified")
}
}
if which == storageDest && val == "stdout" {
return nil, nil
}
if looksLikePath(val) {
disk, err := localdisk.New(val)
if err != nil {
return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", val, err)
}
return disk, nil
}
cl := client.New(val)
// TODO(mpl): probably needs the transport setup for trusted certs here.
cl.SetupAuth()
cl.SetLogger(c.logger)
return noHub{cl}, nil
}
// which is one of "src", "dest", or "thirdleg"
func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) {
var httpClient *http.Client
if val == "" {
switch which {
case storageThird:
return nil, nil
case storageSource:
discl := c.discoClient()
discl.SetLogger(c.logger)
src, err := discl.BlobRoot()
if err != nil {
return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err)
}
val = src
httpClient = discl.HTTPClient()
}
if val == "" {
return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified")
}
}
if which == storageDest && val == "stdout" {
return nil, nil
}
if looksLikePath(val) {
disk, err := localdisk.New(val)
if err != nil {
return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", which, val, err)
}
c.oneIsDisk = true
return disk, nil
}
cl := client.New(val)
cl.InsecureTLS = c.insecureTLS
if httpClient == nil {
httpClient = &http.Client{
Transport: cl.TransportForConfig(nil),
}
}
cl.SetHTTPClient(httpClient)
if err := cl.SetupAuth(); err != nil {
return nil, fmt.Errorf("could not setup auth for connecting to %v: %v", val, err)
}
cl.SetLogger(c.logger)
serverKeyID, err := cl.ServerKeyID()
if err != nil && err != client.ErrNoSigning {
fmt.Fprintf(cmdmain.Stderr, "Failed to discover keyId for server %v: %v", val, err)
} else {
if which == storageSource {
c.srcKeyID = serverKeyID
} else if which == storageDest {
c.destKeyID = serverKeyID
}
}
return cl, nil
}
// newClient returns a Camlistore client for the server.
// The server may be:
// * blank, to use the default in the config file
// * an alias, to use that named alias in the config file
// * host:port
// * https?://host[:port][/path]
func newClient(server string, opts ...client.ClientOption) *client.Client {
if server == "" {
return client.NewOrFail(opts...)
}
cl := client.New(server, opts...)
if err := cl.SetupAuth(); err != nil {
log.Fatalf("Could not setup auth for connecting to %v: %v", server, err)
}
return cl
}
func (c *syncCmd) syncAll() error {
if c.loop {
return cmdmain.UsageError("--all can't be used with --loop")
}
if c.third != "" {
return cmdmain.UsageError("--all can't be used with --thirdleg")
}
if c.dest != "" {
return cmdmain.UsageError("--all can't be used with --dest")
}
dc := c.discoClient()
dc.SetLogger(c.logger)
syncHandlers, err := dc.SyncHandlers()
if err != nil {
return fmt.Errorf("sync handlers discovery failed: %v", err)
}
if c.verbose {
log.Printf("To be synced:\n")
for _, sh := range syncHandlers {
log.Printf("%v -> %v", sh.From, sh.To)
}
}
for _, sh := range syncHandlers {
from := client.New(sh.From)
from.SetLogger(c.logger)
from.SetupAuth()
to := client.New(sh.To)
to.SetLogger(c.logger)
to.SetupAuth()
if c.verbose {
log.Printf("Now syncing: %v -> %v", sh.From, sh.To)
}
stats, err := c.doPass(noHub{from}, noHub{to}, nil)
if c.verbose {
log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
return err
}
}
return nil
}
// discoClient returns a client initialized with a server
// based from --src or from the configuration file if --src
// is blank. The returned client can then be used to discover
// the blobRoot and syncHandlers.
func discoClient() *client.Client {
var cl *client.Client
if *flagSrc == "" {
cl = client.NewOrFail()
} else {
cl = client.New(*flagSrc)
}
cl.SetupAuth()
return cl
}
// discoClient returns a client initialized with a server
// based from --src or from the configuration file if --src
// is blank. The returned client can then be used to discover
// the blobRoot and syncHandlers.
func (c *syncCmd) discoClient() *client.Client {
var cl *client.Client
if c.src == "" {
cl = client.NewOrFail()
} else {
cl = client.New(c.src)
}
cl.SetupAuth()
return cl
}
func TestPublishURLs(t *testing.T) {
rootName := "foo"
idxd := setupContent(rootName)
sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef)
corpus, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
sh.SetCorpus(corpus)
cl := camliClient.New("http://whatever.fake")
fcl := &fakeClient{cl, sh}
ph := &publishHandler{
rootName: rootName,
cl: fcl,
}
if err := ph.initRootNode(); err != nil {
t.Fatalf("initRootNode: %v", err)
}
for ti, tt := range publishURLTests {
rw := httptest.NewRecorder()
if !strings.HasPrefix(tt.path, "/pics/") {
panic("expected /pics/ prefix on " + tt.path)
}
req, _ := http.NewRequest("GET", "http://foo.com"+tt.path, nil)
pfxh := &httputil.PrefixHandler{
Prefix: "/",
Handler: http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) {
// Because the app handler strips the prefix before passing it on to the app
req.URL.Path = strings.TrimPrefix(req.URL.Path, "/pics/")
pr, err := ph.NewRequest(rw, req)
if err != nil {
t.Fatalf("test #%d, NewRequest: %v", ti, err)
}
err = pr.findSubject()
if tt.subject != "" {
if err != nil {
t.Errorf("test #%d, findSubject: %v", ti, err)
return
}
if pr.subject.String() != tt.subject {
t.Errorf("test #%d, got subject %q, want %q", ti, pr.subject, tt.subject)
}
}
if pr.subres != tt.subres {
t.Errorf("test #%d, got subres %q, want %q", ti, pr.subres, tt.subres)
}
}),
}
pfxh.ServeHTTP(rw, req)
}
}
func main() {
flag.Parse()
if *flagSrc == "" {
usage("No --src specified.")
}
if *flagDest == "" {
usage("No --dest specified.")
}
if *flagLoop && !*flagRemoveSource {
usage("Can't use --loop without --removesrc")
}
// TODO(mpl): adapt to the userpass scheme once it has settled
sc := client.New(*flagSrc)
sc.SetupAuth()
dc := client.New(*flagDest)
dc.SetupAuth()
var logger *log.Logger = nil
if *flagVerbose {
logger = log.New(os.Stderr, "", 0)
}
sc.SetLogger(logger)
dc.SetLogger(logger)
passNum := 0
for {
passNum++
stats, err := doPass(sc, dc, passNum)
if err != nil {
log.Fatalf("sync failed: %v", err)
}
if *flagVerbose {
log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied)
}
if !*flagLoop {
break
}
}
}
func setupClient(w *test.World) (*client.Client, error) {
// Do the silly env vars dance to avoid the "non-hermetic use of host config panic".
if err := os.Setenv("CAMLI_KEYID", w.ClientIdentity()); err != nil {
return nil, err
}
if err := os.Setenv("CAMLI_SECRET_RING", w.SecretRingFile()); err != nil {
return nil, err
}
osutil.AddSecretRingFlag()
cl := client.New(w.ServerBaseURL())
return cl, nil
}
func (c *claimsCmd) client() *client.Client {
var cl *client.Client
if c.src == "" {
cl = client.NewOrFail()
} else {
cl = client.New(c.src)
}
cl.SetLogger(c.logger)
cl.SetHTTPClient(&http.Client{
Transport: cl.TransportForConfig(nil),
})
cl.SetupAuth()
return cl
}
func syncAll() error {
if *flagLoop {
usage("--all can not be used with --loop")
}
dc := discoClient()
dc.SetLogger(logger)
syncHandlers, err := dc.SyncHandlers()
if err != nil {
log.Fatalf("sync handlers discovery failed: %v", err)
}
if *flagVerbose {
log.Printf("To be synced:\n")
for _, sh := range syncHandlers {
log.Printf("%v -> %v", sh.From, sh.To)
}
}
for _, sh := range syncHandlers {
from := client.New(sh.From)
from.SetLogger(logger)
from.SetupAuth()
to := client.New(sh.To)
to.SetLogger(logger)
to.SetupAuth()
if *flagVerbose {
log.Printf("Now syncing: %v -> %v", sh.From, sh.To)
}
stats, err := doPass(from, to)
if *flagVerbose {
log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
return err
}
}
return nil
}
// newClient returns a Camlistore client for the server.
// The server may be:
// * blank, to use the default in the config file
// * an alias, to use that named alias in the config file
// * host:port
// * https?://host[:port][/path]
func newClient(server string) *client.Client {
var cl *client.Client
if server == "" {
cl = client.NewOrFail()
} else {
cl = client.New(server)
if err := cl.SetupAuth(); err != nil {
log.Fatalf("Could not setup auth for connecting to %v: %v", server, err)
}
}
cl.SetHTTPClient(&http.Client{
Transport: cl.TransportForConfig(nil),
})
return cl
}
// discoClient returns a client initialized with a server
// based from --src or from the configuration file if --src
// is blank. The returned client can then be used to discover
// the blobRoot and syncHandlers.
func (c *syncCmd) discoClient() *client.Client {
var cl *client.Client
if c.src == "" {
cl = client.NewOrFail()
} else {
cl = client.New(c.src)
}
cl.SetLogger(c.logger)
cl.InsecureTLS = c.insecureTLS
cl.SetHTTPClient(&http.Client{
Transport: cl.TransportForConfig(nil),
})
cl.SetupAuth()
return cl
}
func setupClient(w *test.World) (*client.Client, error) {
// Do the silly env vars dance to avoid the "non-hermetic use of host config panic".
if err := os.Setenv("CAMLI_KEYID", w.ClientIdentity()); err != nil {
return nil, err
}
if err := os.Setenv("CAMLI_SECRET_RING", w.SecretRingFile()); err != nil {
return nil, err
}
osutil.AddSecretRingFlag()
cl := client.New(w.ServerBaseURL())
// This permanode is not needed in itself, but that takes care of uploading
// behind the scenes the public key to the blob server. A bit gross, but
// it's just for a test anyway.
if _, err := cl.UploadNewPermanode(); err != nil {
return nil, err
}
return cl, nil
}
// which is one of "src", "dest", or "thirdleg"
func (c *syncCmd) storageFromParam(which storageType, val string) (blobserver.Storage, error) {
var httpClient *http.Client
if val == "" {
switch which {
case storageThird:
return nil, nil
case storageSource:
discl := c.discoClient()
discl.SetLogger(c.logger)
src, err := discl.BlobRoot()
if err != nil {
return nil, fmt.Errorf("Failed to discover source server's blob path: %v", err)
}
val = src
httpClient = discl.HTTPClient()
}
if val == "" {
return nil, cmdmain.UsageError("No --" + string(which) + " flag value specified")
}
}
if which == storageDest && val == "stdout" {
return nil, nil
}
if looksLikePath(val) {
disk, err := localdisk.New(val)
if err != nil {
return nil, fmt.Errorf("Interpreted --%v=%q as a local disk path, but got error: %v", which, val, err)
}
return disk, nil
}
cl := client.New(val)
cl.InsecureTLS = c.insecureTLS
if httpClient == nil {
httpClient = &http.Client{
Transport: cl.TransportForConfig(nil),
}
}
cl.SetHTTPClient(httpClient)
cl.SetupAuth()
cl.SetLogger(c.logger)
return cl, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
url := config.RequiredString("url")
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
client := client.New(url)
err = client.SetupAuthFromConfig(config)
if err != nil {
return nil, err
}
sto := &remoteStorage{
client: client,
}
if !skipStartupCheck {
// TODO: do a server stat or something to check password
}
return sto, nil
}
func TestPublishMembers(t *testing.T) {
rootName := "foo"
idxd := setupContent(rootName)
sh := search.NewHandler(idxd.Index, idxd.SignerBlobRef)
corpus, err := idxd.Index.KeepInMemory()
if err != nil {
t.Fatalf("error slurping index to memory: %v", err)
}
sh.SetCorpus(corpus)
cl := camliClient.New("http://whatever.fake")
fcl := &fakeClient{cl, sh}
ph := &publishHandler{
rootName: rootName,
cl: fcl,
}
rw := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "http://foo.com/pics", nil)
pfxh := &httputil.PrefixHandler{
Prefix: "/pics/",
Handler: http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) {
pr, err := ph.NewRequest(rw, req)
if err != nil {
t.Fatalf("NewRequest: %v", err)
}
res, err := pr.ph.deepDescribe(pr.subject)
if err != nil {
t.Fatalf("deepDescribe: %v", err)
}
members, err := pr.subjectMembers(res.Meta)
if len(members.Members) != 2 {
t.Errorf("Expected two members in publish root (one camlipath, one camlimember), got %d", len(members.Members))
}
}),
}
pfxh.ServeHTTP(rw, req)
}
func createObjStore() (*objStore, error) {
db := camli.New("localhost:3179")
return &objStore{newObjInventory(), db}, nil
}
func (c *serverCmd) makeThings() error {
const importerPrefix = "/importer/"
// check that "/importer/" prefix is in config, just in case it ever changes.
configFile := filepath.Join(camliSrcRoot, "config", "dev-server-config.json")
config, err := ioutil.ReadFile(configFile)
if err != nil {
return fmt.Errorf("could not read config file %v: %v", configFile, err)
}
if !bytes.Contains(config, []byte(importerPrefix)) {
return fmt.Errorf("%s prefix not found in dev config. Did it change?", importerPrefix)
}
if err := netutil.AwaitReachable("localhost:"+c.port, time.Minute); err != nil {
return err
}
osutil.AddSecretRingFlag()
setCamdevVars()
baseURL := c.env.m["CAMLI_BASEURL"]
if baseURL == "" {
return errors.New("CAMLI_BASEURL is not set")
}
cl := client.New(baseURL)
signer, err := cl.Signer()
if err != nil {
return err
}
ClientId := make(map[string]string)
ClientSecret := make(map[string]string)
for name := range importer.All() {
ClientId[name] = "fakeStaticClientId"
ClientSecret[name] = "fakeStaticClientSecret"
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: ClientId,
ClientSecret: ClientSecret,
}
for name, imp := range importer.All() {
mk, ok := imp.(importer.TestDataMaker)
if !ok {
continue
}
tr := mk.MakeTestData()
hc.HTTPClient = &http.Client{Transport: tr}
host, err := importer.NewHost(hc)
if err != nil {
return fmt.Errorf("could not obtain Host: %v", err)
}
rc, err := importer.CreateAccount(host, name)
if err != nil {
return err
}
if err := mk.SetTestAccount(rc.AccountNode()); err != nil {
return fmt.Errorf("could not set fake account node for importer %v: %v", name, err)
}
if err := imp.Run(rc); err != nil {
return err
}
}
return nil
}
