Exemplo n.º 1
0
func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error {
	if len(args) > 0 {
		return errors.New("Permanode command doesn't take any additional arguments")
	}

	var (
		permaNode *client.PutResult
		err       error
	)
	permaNode, err = up.UploadNewPermanode()
	if handleResult("permanode", permaNode, err) != nil {
		return err
	}

	if c.name != "" {
		put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.name))
		handleResult("claim-permanode-title", put, err)
	}
	if c.tag != "" {
		tags := strings.Split(c.tag, ",")
		m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
		for _, tag := range tags {
			m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
			put, err := up.UploadAndSignMap(m)
			handleResult("claim-permanode-tag", put, err)
		}
	}
	return nil
}
Exemplo n.º 2
0
func (c *attrCmd) RunCommand(args []string) error {
	if len(args) != 3 {
		return errors.New("Attr takes 3 args: <permanode> <attr> <value>")
	}
	permanode, attr, value := args[0], args[1], args[2]

	var err error

	pn := blobref.Parse(permanode)
	if pn == nil {
		return fmt.Errorf("Error parsing blobref %q", permanode)
	}
	bb := schema.NewSetAttributeClaim(pn, attr, value)
	if c.add {
		if c.del {
			return errors.New("Add and del options are exclusive")
		}
		bb = schema.NewAddAttributeClaim(pn, attr, value)
	} else {
		// TODO: del, which can make <value> be optional
		if c.del {
			return errors.New("del not yet implemented")
		}
	}
	put, err := getUploader().UploadAndSignBlob(bb)
	handleResult(bb.Type(), put, err)
	return nil
}
Exemplo n.º 3
0
// SetAttrValues sets multi-valued attribute.
func (o *Object) SetAttrValues(key string, attrs []string) error {
	exists := asSet(o.Attrs(key))
	actual := asSet(attrs)
	o.mu.Lock()
	defer o.mu.Unlock()
	// add new values
	for v := range actual {
		if exists[v] {
			delete(exists, v)
			continue
		}
		_, err := o.h.upload(schema.NewAddAttributeClaim(o.pn, key, v))
		if err != nil {
			return err
		}
	}
	// delete unneeded values
	for v := range exists {
		_, err := o.h.upload(schema.NewDelAttributeClaim(o.pn, key, v))
		if err != nil {
			return err
		}
	}
	if o.attr == nil {
		o.attr = make(map[string][]string)
	}
	o.attr[key] = attrs
	return nil
}
Exemplo n.º 4
0
// uploadFilePermanode creates and uploads the planned permanode (with sum as a
// fixed key) associated with the file blobref fileRef.
// It also sets the optional tags for this permanode.
func (up *Uploader) uploadFilePermanode(sum string, fileRef blob.Ref, claimTime time.Time) error {
	// Use a fixed time value for signing; not using modtime
	// so two identical files don't have different modtimes?
	// TODO(bradfitz): consider this more?
	permaNodeSigTime := time.Unix(0, 0)
	permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
	if err != nil {
		return fmt.Errorf("Error uploading planned permanode: %v", err)
	}
	handleResult("node-permanode", permaNode, nil)

	contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", fileRef.String())
	contentAttr.SetClaimDate(claimTime)
	signer, err := up.Signer()
	if err != nil {
		return err
	}
	signed, err := contentAttr.SignAt(signer, claimTime)
	if err != nil {
		return fmt.Errorf("Failed to sign content claim: %v", err)
	}
	put, err := up.uploadString(signed)
	if err != nil {
		return fmt.Errorf("Error uploading permanode's attribute: %v", err)
	}

	handleResult("node-permanode-contentattr", put, nil)
	if tags := up.fileOpts.tags(); len(tags) > 0 {
		errch := make(chan error)
		for _, tag := range tags {
			go func(tag string) {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				m.SetClaimDate(claimTime)
				signed, err := m.SignAt(signer, claimTime)
				if err != nil {
					errch <- fmt.Errorf("Failed to sign tag claim: %v", err)
					return
				}
				put, err := up.uploadString(signed)
				if err != nil {
					errch <- fmt.Errorf("Error uploading permanode's tag attribute %v: %v", tag, err)
					return
				}
				handleResult("node-permanode-tag", put, nil)
				errch <- nil
			}(tag)
		}

		for range tags {
			if e := <-errch; e != nil && err == nil {
				err = e
			}
		}
		if err != nil {
			return err
		}
	}
	return nil
}
Exemplo n.º 5
0
func (c *permanodeCmd) RunCommand(args []string) error {
	if len(args) > 0 {
		return errors.New("Permanode command doesn't take any additional arguments")
	}

	var (
		permaNode *client.PutResult
		err       error
		up        = getUploader()
	)
	if (c.key != "") != (c.sigTime != "") {
		return errors.New("Both --key and --sigtime must be used to produce deterministic permanodes.")
	}
	if c.key == "" {
		// Normal case, with a random permanode.
		permaNode, err = up.UploadNewPermanode()
	} else {
		const format = "2006-01-02 15:04:05"
		sigTime, err := time.Parse(format, c.sigTime)
		if err != nil {
			return fmt.Errorf("Error parsing time %q; expecting time of form %q", c.sigTime, format)
		}
		permaNode, err = up.UploadPlannedPermanode(c.key, sigTime)
	}
	if handleResult("permanode", permaNode, err) != nil {
		return err
	}

	if c.title != "" {
		put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.title))
		handleResult("claim-permanode-title", put, err)
	}
	if c.tag != "" {
		tags := strings.Split(c.tag, ",")
		m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
		for _, tag := range tags {
			m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
			put, err := up.UploadAndSignBlob(m)
			handleResult("claim-permanode-tag", put, err)
		}
	}
	return nil
}
Exemplo n.º 6
0
func storePhoto(p photo) (string, error) {
	srcFile := localPathOf(p)

	f, err := os.Open(srcFile)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fileRef, err := schema.WriteFileFromReader(camliClient, p.Id+"."+p.Extension, f)

	res, err := camliClient.UploadNewPermanode()
	if err != nil {
		return "", err
	}
	perma := res.BlobRef

	p.Description = cleanHTML(p.Description)

	claims := []*schema.Builder{}
	claims = append(claims, schema.NewSetAttributeClaim(perma, "camliContent", fileRef.String()))
	claims = append(claims, schema.NewSetAttributeClaim(perma, "title", mkTitle(p.Description)))
	claims = append(claims, schema.NewSetAttributeClaim(perma, "description", p.Description))
	for _, t := range p.Tags {
		claims = append(claims, schema.NewAddAttributeClaim(perma, "tag", t))
	}
	if p.Cat == "Public" {
		claims = append(claims, schema.NewSetAttributeClaim(perma, "camliAccess", "public"))
	}

	grp := syncutil.Group{}
	for _, claimBuilder := range claims {
		claim := claimBuilder.Blob()
		grp.Go(func() error {
			_, err := camliClient.UploadAndSignBlob(claim)
			return err
		})
	}

	return perma.String(), grp.Err()
}
Exemplo n.º 7
0
func (id *IndexDeps) AddAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef {
	m := schema.NewAddAttributeClaim(permaNode, attr, value)
	m.SetClaimDate(id.advanceTime())
	return id.uploadAndSign(m)
}
Exemplo n.º 8
0
func (c *fileCmd) RunCommand(args []string) error {
	if c.vivify {
		if c.makePermanode || c.filePermanodes || c.tag != "" || c.title != "" {
			return cmdmain.UsageError("--vivify excludes any other option")
		}
	}
	if c.title != "" && !c.makePermanode {
		return cmdmain.UsageError("Can't set title without using --permanode")
	}
	if c.tag != "" && !c.makePermanode && !c.filePermanodes {
		return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
	}
	if c.histo != "" && !c.memstats {
		return cmdmain.UsageError("Can't use histo without memstats")
	}
	if c.deleteAfterUpload && !c.filePermanodes {
		return cmdmain.UsageError("Can't set use --delete_after_upload without --filenodes")
	}
	if c.filePermanodes && c.contentsOnly {
		return cmdmain.UsageError("--contents_only and --filenodes are exclusive. Use --permanode instead.")
	}
	// TODO(mpl): do it for other modes too. Or even better, do it once for all modes.
	if *cmdmain.FlagVerbose {
		log.SetOutput(cmdmain.Stderr)
	} else {
		log.SetOutput(ioutil.Discard)
	}
	up := getUploader()
	if c.memstats {
		sr := new(statspkg.Receiver)
		up.altStatReceiver = sr
		defer func() { DumpStats(sr, c.histo) }()
	}
	c.initCaches(up)

	if c.makePermanode || c.filePermanodes {
		testSigBlobRef := up.Client.SignerPublicKeyBlobref()
		if !testSigBlobRef.Valid() {
			return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
		}
	}
	up.fileOpts = &fileOptions{
		permanode:    c.filePermanodes,
		tag:          c.tag,
		vivify:       c.vivify,
		exifTime:     c.exifTime,
		capCtime:     c.capCtime,
		contentsOnly: c.contentsOnly,
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}
	if c.argsFromInput {
		if len(args) > 0 {
			return errors.New("args not supported with -argsfrominput")
		}
		tu := up.NewRootlessTreeUpload()
		tu.Start()
		br := bufio.NewReader(os.Stdin)
		for {
			path, err := br.ReadString('\n')
			if path = strings.TrimSpace(path); path != "" {
				tu.Enqueue(path)
			}
			if err == io.EOF {
				android.PreExit()
				os.Exit(0)
			}
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	if len(args) == 0 {
		return cmdmain.UsageError("No files or directories given.")
	}
	if up.statCache != nil {
		defer up.statCache.Close()
	}
	for _, filename := range args {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		// Skip ignored files or base directories.  Failing to skip the
		// latter results in a panic.
		if up.Client.IsIgnoredFile(filename) {
			log.Printf("Client configured to ignore %s; skipping.", filename)
			continue
		}
		if fi.IsDir() {
			if up.fileOpts.wantVivify() {
				vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
				continue
			}
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
			if err == nil && c.deleteAfterUpload {
				if err := os.Remove(filename); err != nil {
					log.Printf("Error deleting %v: %v", filename, err)
				} else {
					log.Printf("Deleted %v", filename)
				}
			}
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil && lastPut != nil {
		put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.title != "" {
			put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.title))
			handleResult("claim-permanode-title", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			for _, tag := range tags {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignBlob(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
Exemplo n.º 9
0
func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
	// TODO(mpl): maybe break this func into more maintainable pieces?
	filebb := schema.NewCommonFileMap(n.fullPath, n.fi)
	filebb.SetType("file")
	file, err := up.open(n.fullPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	if up.fileOpts.exifTime {
		ra, ok := file.(io.ReaderAt)
		if !ok {
			return nil, errors.New("Error asserting local file to io.ReaderAt")
		}
		modtime, err := schema.FileTime(ra)
		if err != nil {
			log.Printf("warning: getting time from EXIF failed for %v: %v", n.fullPath, err)
		} else {
			filebb.SetModTime(modtime)
		}
	}

	var (
		size                           = n.fi.Size()
		fileContents io.Reader         = io.LimitReader(file, size)
		br           *blobref.BlobRef  // of file schemaref
		sum          string            // sha1 hashsum of the file to upload
		pr           *client.PutResult // of the final "file" schema blob
	)

	const dupCheckThreshold = 256 << 10
	if size > dupCheckThreshold {
		sumRef, err := up.wholeFileDigest(n.fullPath)
		if err == nil {
			sum = sumRef.String()
			ok := false
			pr, ok = up.fileMapFromDuplicate(up.statReceiver(n), filebb, sum)
			if ok {
				br = pr.BlobRef
				noteFileUploaded(n.fullPath, !pr.Skipped)
				if up.fileOpts.wantVivify() {
					// we can return early in that case, because the other options
					// are disallowed in the vivify case.
					return pr, nil
				}
			}
		}
	}

	if up.fileOpts.wantVivify() {
		// If vivify wasn't already done in fileMapFromDuplicate.
		err := schema.WriteFileChunks(up.statReceiver(n), filebb, fileContents)
		if err != nil {
			return nil, err
		}
		json, err := filebb.JSON()
		if err != nil {
			return nil, err
		}
		br = blobref.SHA1FromString(json)
		h := &client.UploadHandle{
			BlobRef:  br,
			Size:     int64(len(json)),
			Contents: strings.NewReader(json),
			Vivify:   true,
		}
		pr, err = up.Upload(h)
		if err != nil {
			return nil, err
		}
		noteFileUploaded(n.fullPath, true)
		return pr, nil
	}

	if br == nil {
		// br still nil means fileMapFromDuplicate did not find the file on the server,
		// and the file has not just been uploaded subsequently to a vivify request.
		// So we do the full file + file schema upload here.
		if sum == "" && up.fileOpts.wantFilePermanode() {
			fileContents = &trackDigestReader{r: fileContents}
		}
		br, err = schema.WriteFileMap(up.statReceiver(n), filebb, fileContents)
		if err != nil {
			return nil, err
		}
	}

	// TODO(mpl): test that none of these claims get uploaded if they've already been done
	if up.fileOpts.wantFilePermanode() {
		if td, ok := fileContents.(*trackDigestReader); ok {
			sum = td.Sum()
		}
		// Use a fixed time value for signing; not using modtime
		// so two identical files don't have different modtimes?
		// TODO(bradfitz): consider this more?
		permaNodeSigTime := time.Unix(0, 0)
		permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode for node %v: %v", n, err)
		}
		handleResult("node-permanode", permaNode, nil)

		// claimTime is both the time of the "claimDate" in the
		// JSON claim, as well as the date in the OpenPGP
		// header.
		// TODO(bradfitz): this is a little clumsy to do by hand.
		// There should probably be a method on *Uploader to do this
		// from an unsigned schema map. Maybe ditch the schema.Claimer
		// type and just have the Uploader override the claimDate.
		claimTime, ok := filebb.ModTime()
		if !ok {
			return nil, fmt.Errorf("couldn't get modtime back for file %v", n.fullPath)
		}
		contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", br.String())
		contentAttr.SetClaimDate(claimTime)
		signed, err := up.SignBlob(contentAttr, claimTime)
		if err != nil {
			return nil, fmt.Errorf("Failed to sign content claim for node %v: %v", n, err)
		}
		put, err := up.uploadString(signed)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode's attribute for node %v: %v", n, err)
		}
		handleResult("node-permanode-contentattr", put, nil)
		if tags := up.fileOpts.tags(); len(tags) > 0 {
			errch := make(chan error)
			for _, tag := range tags {
				go func(tag string) {
					m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
					m.SetClaimDate(claimTime)
					signed, err := up.SignBlob(m, claimTime)
					if err != nil {
						errch <- fmt.Errorf("Failed to sign tag claim for node %v: %v", n, err)
						return
					}
					put, err := up.uploadString(signed)
					if err != nil {
						errch <- fmt.Errorf("Error uploading permanode's tag attribute %v for node %v: %v", tag, n, err)
						return
					}
					handleResult("node-permanode-tag", put, nil)
					errch <- nil
				}(tag)
			}

			for _ = range tags {
				if e := <-errch; e != nil && err == nil {
					err = e
				}
			}
			if err != nil {
				return nil, err
			}
		}
	}

	// TODO(bradfitz): faking a PutResult here to return
	// is kinda gross.  should instead make a
	// blobserver.Storage wrapper type (wrapping
	// statReceiver) that can track some of this?  or make
	// schemaWriteFileMap return it?
	json, _ := filebb.JSON()
	pr = &client.PutResult{BlobRef: br, Size: int64(len(json)), Skipped: false}
	return pr, nil
}
Exemplo n.º 10
0
func (c *fileCmd) RunCommand(args []string) error {
	if c.vivify {
		if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
			return cmdmain.UsageError("--vivify excludes any other option")
		}
	}
	if c.name != "" && !c.makePermanode {
		return cmdmain.UsageError("Can't set name without using --permanode")
	}
	if c.tag != "" && !c.makePermanode && !c.filePermanodes {
		return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
	}
	if c.histo != "" && !c.memstats {
		return cmdmain.UsageError("Can't use histo without memstats")
	}
	up := getUploader()
	if c.memstats {
		sr := new(statsStatReceiver)
		up.altStatReceiver = sr
		defer func() { sr.DumpStats(c.histo) }()
	}
	c.initCaches(up)

	if c.makePermanode || c.filePermanodes {
		testSigBlobRef := up.Client.SignerPublicKeyBlobref()
		if testSigBlobRef == nil {
			return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
		}
	}
	up.fileOpts = &fileOptions{
		permanode: c.filePermanodes,
		tag:       c.tag,
		vivify:    c.vivify,
		exifTime:  c.exifTime,
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}
	if c.argsFromInput {
		if len(args) > 0 {
			return errors.New("args not supported with -argsfrominput")
		}
		tu := up.NewRootlessTreeUpload()
		tu.Start()
		br := bufio.NewReader(os.Stdin)
		for {
			path, err := br.ReadString('\n')
			if path = strings.TrimSpace(path); path != "" {
				tu.Enqueue(path)
			}
			if err == io.EOF {
				os.Exit(0)
			}
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	if len(args) == 0 {
		return cmdmain.UsageError("No files or directories given.")
	}
	for _, filename := range args {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			if up.fileOpts.wantVivify() {
				vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
				continue
			}
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil {
		put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.name != "" {
			put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
			handleResult("claim-permanode-name", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
			for _, tag := range tags {
				m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignBlob(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
Exemplo n.º 11
0
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
	if len(args) == 0 {
		return UsageError("No files or directories given.")
	}
	if c.name != "" && !c.makePermanode {
		return UsageError("Can't set name without using --permanode")
	}
	if c.tag != "" && !c.makePermanode {
		return UsageError("Can't set tag without using --permanode")
	}
	if c.histo != "" && !c.memstats {
		return UsageError("Can't use histo without memstats")
	}
	if c.memstats {
		sr := new(statsStatReceiver)
		if c.histo != "" {
			num := 100
			sr.histo = histo.NewHisto(num)
		}
		up.altStatReceiver = sr
		defer func() { sr.DumpStats(c.histo) }()
	}
	if c.statcache {
		cache := NewFlatStatCache()
		up.statCache = cache
	}
	if c.havecache {
		cache := NewFlatHaveCache()
		up.haveCache = cache
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}
	if c.rollSplits {
		up.rollSplits = true
	}

	for _, filename := range args {
		if fi, err := os.Stat(filename); err == nil && fi.IsDir() {
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil {
		put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.name != "" {
			put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
			handleResult("claim-permanode-name", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
			for _, tag := range tags {
				m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignMap(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
Exemplo n.º 12
0
func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
	m := schema.NewCommonFileMap(n.fullPath, n.fi)
	m["camliType"] = "file"
	file, err := up.open(n.fullPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	size := n.fi.Size()

	var fileContents io.Reader = io.LimitReader(file, size)

	if up.fileOpts.wantVivify() {
		err := schema.WriteFileChunks(up.statReceiver(), m, fileContents)
		if err != nil {
			return nil, err
		}
		json, err := m.JSON()
		if err != nil {
			return nil, err
		}
		bref := blobref.SHA1FromString(json)
		h := &client.UploadHandle{
			BlobRef:  bref,
			Size:     int64(len(json)),
			Contents: strings.NewReader(json),
			Vivify:   true,
		}
		return up.Upload(h)
	}

	var (
		blobref *blobref.BlobRef // of file schemaref
		sum     string           // "sha1-xxxxx"
	)

	const dupCheckThreshold = 256 << 10
	if size > dupCheckThreshold {
		sumRef, err := up.wholeFileDigest(n.fullPath)
		if err == nil {
			sum = sumRef.String()
			if ref, ok := up.fileMapFromDuplicate(up.statReceiver(), m, sum); ok {
				blobref = ref
			}
		}
	}

	if blobref == nil {
		if sum == "" && up.fileOpts.wantFilePermanode() {
			fileContents = &trackDigestReader{r: fileContents}
		}
		blobref, err = schema.WriteFileMap(up.statReceiver(), m, fileContents)
		if err != nil {
			return nil, err
		}
	}

	// TODO(mpl): test that none of these claims get uploaded if they've already been done
	if up.fileOpts.wantFilePermanode() {
		if td, ok := fileContents.(*trackDigestReader); ok {
			sum = td.Sum()
		}
		// Use a fixed time value for signing; not using modtime
		// so two identical files don't have different modtimes?
		// TODO(bradfitz): consider this more?
		permaNodeSigTime := time.Unix(0, 0)
		permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode for node %v: %v", n, err)
		}
		handleResult("node-permanode", permaNode, nil)

		// claimTime is both the time of the "claimDate" in the
		// JSON claim, as well as the date in the OpenPGP
		// header.
		// TODO(bradfitz): this is a little clumsy to do by hand.
		// There should probably be a method on *Uploader to do this
		// from an unsigned schema map. Maybe ditch the schema.Claimer
		// type and just have the Uploader override the claimDate.
		claimTime := n.fi.ModTime()

		contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", blobref.String())
		contentAttr.SetClaimDate(claimTime)
		signed, err := up.SignMap(contentAttr, claimTime)
		if err != nil {
			return nil, fmt.Errorf("Failed to sign content claim for node %v: %v", n, err)
		}
		put, err := up.uploadString(signed)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode's attribute for node %v: %v", n, err)
		}
		handleResult("node-permanode-contentattr", put, nil)
		if tags := up.fileOpts.tags(); len(tags) > 0 {
			// TODO(mpl): do these claims concurrently, not in series
			for _, tag := range tags {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				m.SetClaimDate(claimTime)
				// TODO(mpl): verify that SetClaimDate does modify the GPG signature date of the claim
				signed, err := up.SignMap(m, claimTime)
				if err != nil {
					return nil, fmt.Errorf("Failed to sign tag claim for node %v: %v", n, err)
				}
				put, err := up.uploadString(signed)
				if err != nil {
					return nil, fmt.Errorf("Error uploading permanode's tag attribute %v for node %v: %v", tag, n, err)
				}
				handleResult("node-permanode-tag", put, nil)
			}
		}
	}

	// TODO(bradfitz): faking a PutResult here to return
	// is kinda gross.  should instead make a
	// blobserver.Storage wrapper type (wrapping
	// statReceiver) that can track some of this?  or make
	// schemaWriteFileMap return it?
	json, _ := m.JSON()
	pr := &client.PutResult{BlobRef: blobref, Size: int64(len(json)), Skipped: false}
	return pr, nil
}
Exemplo n.º 13
0
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
	if len(args) == 0 {
		return UsageError("No files or directories given.")
	}
	if c.vivify {
		if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
			return UsageError("--vivify excludes any other option")
		}
	}
	if c.name != "" && !c.makePermanode {
		return UsageError("Can't set name without using --permanode")
	}
	if c.tag != "" && !c.makePermanode && !c.filePermanodes {
		return UsageError("Can't set tag without using --permanode or --filenodes")
	}
	if c.histo != "" && !c.memstats {
		return UsageError("Can't use histo without memstats")
	}
	if c.memstats {
		sr := new(statsStatReceiver)
		up.altStatReceiver = sr
		defer func() { sr.DumpStats(c.histo) }()
	}
	if c.statcache || c.havecache {
		gen, err := up.StorageGeneration()
		if err != nil {
			log.Printf("WARNING: not using local caches; failed to retrieve server's storage generation: %v", err)
		} else {
			if c.statcache {
				cache := NewFlatStatCache(gen)
				up.statCache = cache
			}
			if c.havecache {
				cache := NewFlatHaveCache(gen)
				up.haveCache = cache
				up.Client.SetHaveCache(cache)
			}
		}
	}
	if c.makePermanode || c.filePermanodes {
		testSigBlobRef := up.Client.SignerPublicKeyBlobref()
		if testSigBlobRef == nil {
			return UsageError("A gpg key is needed to create permanodes; configure one or use vivify mode.")
		}
	}
	up.fileOpts = &fileOptions{
		permanode: c.filePermanodes,
		tag:       c.tag,
		vivify:    c.vivify,
		exifTime:  c.exifTime,
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}

	for _, filename := range args {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			if up.fileOpts.wantVivify() {
				vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
				continue
			}
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil {
		put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.name != "" {
			put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
			handleResult("claim-permanode-name", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
			for _, tag := range tags {
				m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignMap(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}