示例#1
0
func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error {
	if len(args) > 0 {
		return errors.New("Permanode command doesn't take any additional arguments")
	}

	var (
		permaNode *client.PutResult
		err       error
	)
	permaNode, err = up.UploadNewPermanode()
	if handleResult("permanode", permaNode, err) != nil {
		return err
	}

	if c.name != "" {
		put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.name))
		handleResult("claim-permanode-title", put, err)
	}
	if c.tag != "" {
		tags := strings.Split(c.tag, ",")
		m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
		for _, tag := range tags {
			m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
			put, err := up.UploadAndSignMap(m)
			handleResult("claim-permanode-tag", put, err)
		}
	}
	return nil
}
示例#2
0
func (ph *PublishHandler) setRootNode(jsonSign *JSONSignHandler, pn *blobref.BlobRef) (err error) {
	_, err = ph.signUpload(jsonSign, "set-attr camliRoot", schema.NewSetAttributeClaim(pn, "camliRoot", ph.RootName))
	if err != nil {
		return err
	}
	_, err = ph.signUpload(jsonSign, "set-attr title", schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+ph.RootName))
	return err
}
示例#3
0
func (n *mutDir) creat(name string, typ nodeType) (fuse.Node, error) {
	// Create a Permanode for the file/directory.
	pr, err := n.fs.client.UploadNewPermanode()
	if err != nil {
		return nil, err
	}

	var grp syncutil.Group
	grp.Go(func() (err error) {
		// Add a camliPath:name attribute to the directory permanode.
		claim := schema.NewSetAttributeClaim(n.permanode, "camliPath:"+name, pr.BlobRef.String())
		_, err = n.fs.client.UploadAndSignBlob(claim)
		return
	})
	if stupidMacExtendedAttributeName(name) {
		grp.Go(func() (err error) {
			// Add a camliPath:name attribute to the directory permanode.
			claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliDefVis", "hide")
			_, err = n.fs.client.UploadAndSignBlob(claim)
			return
		})
	}
	if err := grp.Err(); err != nil {
		return nil, err
	}

	// Add a child node to this node.
	var child mutFileOrDir
	switch typ {
	case dirType:
		child = &mutDir{
			fs:        n.fs,
			permanode: pr.BlobRef,
			parent:    n,
			name:      name,
		}
	case fileType, symlinkType:
		child = &mutFile{
			fs:        n.fs,
			permanode: pr.BlobRef,
			parent:    n,
			name:      name,
		}
	default:
		panic("bogus creat type")
	}
	n.mu.Lock()
	if n.children == nil {
		n.children = make(map[string]mutFileOrDir)
	}
	n.children[name] = child
	n.mu.Unlock()

	return child, nil
}
示例#4
0
文件: attr.go 项目: robryk/camlistore
func (c *attrCmd) RunCommand(args []string) error {
	if len(args) != 3 {
		return errors.New("Attr takes 3 args: <permanode> <attr> <value>")
	}
	permanode, attr, value := args[0], args[1], args[2]

	var err error

	pn := blobref.Parse(permanode)
	if pn == nil {
		return fmt.Errorf("Error parsing blobref %q", permanode)
	}
	bb := schema.NewSetAttributeClaim(pn, attr, value)
	if c.add {
		if c.del {
			return errors.New("Add and del options are exclusive")
		}
		bb = schema.NewAddAttributeClaim(pn, attr, value)
	} else {
		// TODO: del, which can make <value> be optional
		if c.del {
			return errors.New("del not yet implemented")
		}
	}
	put, err := getUploader().UploadAndSignBlob(bb)
	handleResult(bb.Type(), put, err)
	return nil
}
示例#5
0
func (n *rootsDir) Mkdir(req *fuse.MkdirRequest, intr fuse.Intr) (fuse.Node, fuse.Error) {
	name := req.Name

	// Create a Permanode for the root.
	pr, err := n.fs.client.UploadNewPermanode()
	if err != nil {
		log.Printf("rootsDir.Create(%q): %v", name, err)
		return nil, fuse.EIO
	}

	// Add a camliRoot attribute to the root permanode.
	claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliRoot", name)
	_, err = n.fs.client.UploadAndSignBlob(claim)
	if err != nil {
		log.Printf("rootsDir.Create(%q): %v", name, err)
		return nil, fuse.EIO
	}

	nod := &mutDir{
		fs:        n.fs,
		permanode: pr.BlobRef,
		name:      name,
	}
	n.mu.Lock()
	n.m[name] = pr.BlobRef
	n.mu.Unlock()

	return nod, nil
}
示例#6
0
// &RenameRequest{Header:fuse.Header{Conn:(*fuse.Conn)(0xc210048180), ID:0x2, Node:0x8, Uid:0xf0d4, Gid:0x1388, Pid:0x5edb}, NewDir:0x8, OldName:"1", NewName:"2"}
func (n *mutDir) Rename(req *fuse.RenameRequest, newDir fuse.Node, intr fuse.Intr) fuse.Error {
	n2, ok := newDir.(*mutDir)
	if !ok {
		log.Printf("*mutDir newDir node isn't a *mutDir; is a %T; can't handle. returning EIO.", newDir)
		return fuse.EIO
	}

	var wg syncutil.Group
	wg.Go(n.populate)
	wg.Go(n2.populate)
	if err := wg.Err(); err != nil {
		log.Printf("*mutDir.Rename src dir populate = %v", err)
		return fuse.EIO
	}

	n.mu.Lock()
	target, ok := n.children[req.OldName]
	n.mu.Unlock()
	if !ok {
		log.Printf("*mutDir.Rename src name %q isn't known", req.OldName)
		return fuse.ENOENT
	}

	now := time.Now()

	// Add a camliPath:name attribute to the dest permanode before unlinking it from
	// the source.
	claim := schema.NewSetAttributeClaim(n2.permanode, "camliPath:"+req.NewName, target.permanodeString())
	claim.SetClaimDate(now)
	_, err := n.fs.client.UploadAndSignBlob(claim)
	if err != nil {
		log.Printf("Upload rename link error: %v", err)
		return fuse.EIO
	}

	delClaim := schema.NewDelAttributeClaim(n.permanode, "camliPath:"+req.OldName, "")
	delClaim.SetClaimDate(now)
	_, err = n.fs.client.UploadAndSignBlob(delClaim)
	if err != nil {
		log.Printf("Upload rename src unlink error: %v", err)
		return fuse.EIO
	}

	// TODO(bradfitz): this locking would be racy, if the kernel
	// doesn't do it properly. (It should) Let's just trust the
	// kernel for now. Later we can verify and remove this
	// comment.
	n.mu.Lock()
	if n.children[req.OldName] != target {
		panic("Race.")
	}
	delete(n.children, req.OldName)
	n.mu.Unlock()
	n2.mu.Lock()
	n2.children[req.NewName] = target
	n2.mu.Unlock()

	return nil
}
示例#7
0
// uploadFilePermanode creates and uploads the planned permanode (with sum as a
// fixed key) associated with the file blobref fileRef.
// It also sets the optional tags for this permanode.
func (up *Uploader) uploadFilePermanode(sum string, fileRef blob.Ref, claimTime time.Time) error {
	// Use a fixed time value for signing; not using modtime
	// so two identical files don't have different modtimes?
	// TODO(bradfitz): consider this more?
	permaNodeSigTime := time.Unix(0, 0)
	permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
	if err != nil {
		return fmt.Errorf("Error uploading planned permanode: %v", err)
	}
	handleResult("node-permanode", permaNode, nil)

	contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", fileRef.String())
	contentAttr.SetClaimDate(claimTime)
	signer, err := up.Signer()
	if err != nil {
		return err
	}
	signed, err := contentAttr.SignAt(signer, claimTime)
	if err != nil {
		return fmt.Errorf("Failed to sign content claim: %v", err)
	}
	put, err := up.uploadString(signed)
	if err != nil {
		return fmt.Errorf("Error uploading permanode's attribute: %v", err)
	}

	handleResult("node-permanode-contentattr", put, nil)
	if tags := up.fileOpts.tags(); len(tags) > 0 {
		errch := make(chan error)
		for _, tag := range tags {
			go func(tag string) {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				m.SetClaimDate(claimTime)
				signed, err := m.SignAt(signer, claimTime)
				if err != nil {
					errch <- fmt.Errorf("Failed to sign tag claim: %v", err)
					return
				}
				put, err := up.uploadString(signed)
				if err != nil {
					errch <- fmt.Errorf("Error uploading permanode's tag attribute %v: %v", tag, err)
					return
				}
				handleResult("node-permanode-tag", put, nil)
				errch <- nil
			}(tag)
		}

		for range tags {
			if e := <-errch; e != nil && err == nil {
				err = e
			}
		}
		if err != nil {
			return err
		}
	}
	return nil
}
示例#8
0
func (n *rootsDir) Rename(req *fuse.RenameRequest, newDir fuse.Node, intr fuse.Intr) fuse.Error {
	log.Printf("rootsDir.Rename %q -> %q", req.OldName, req.NewName)
	n.mu.Lock()
	target, exists := n.m[req.OldName]
	_, collision := n.m[req.NewName]
	n.mu.Unlock()
	if !exists {
		log.Printf("*rootsDir.Rename src name %q isn't known", req.OldName)
		return fuse.ENOENT
	}
	if collision {
		log.Printf("*rootsDir.Rename dest %q already exists", req.NewName)
		return fuse.EIO
	}

	// Don't allow renames if the root contains content.  Rename
	// is mostly implemented to make GUIs that create directories
	// before asking for the directory name.
	res, err := n.fs.client.Describe(&search.DescribeRequest{BlobRef: target})
	if err != nil {
		log.Println("rootsDir.Rename:", err)
		return fuse.EIO
	}
	db := res.Meta[target.String()]
	if db == nil {
		log.Printf("Failed to pull meta for target: %v", target)
		return fuse.EIO
	}

	for k := range db.Permanode.Attr {
		const p = "camliPath:"
		if strings.HasPrefix(k, p) {
			log.Printf("Found file in %q: %q, disallowing rename", req.OldName, k[len(p):])
			return fuse.EIO
		}
	}

	claim := schema.NewSetAttributeClaim(target, "camliRoot", req.NewName)
	_, err = n.fs.client.UploadAndSignBlob(claim)
	if err != nil {
		log.Printf("Upload rename link error: %v", err)
		return fuse.EIO
	}

	// Comment transplanted from mutDir.Rename
	// TODO(bradfitz): this locking would be racy, if the kernel
	// doesn't do it properly. (It should) Let's just trust the
	// kernel for now. Later we can verify and remove this
	// comment.
	n.mu.Lock()
	if n.m[req.OldName] != target {
		panic("Race.")
	}
	delete(n.m, req.OldName)
	n.m[req.NewName] = target
	n.mu.Unlock()

	return nil
}
示例#9
0
func (n *mutFile) setContent(br blob.Ref, size int64) error {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.content = br
	n.size = size
	claim := schema.NewSetAttributeClaim(n.permanode, "camliContent", br.String())
	_, err := n.fs.client.UploadAndSignBlob(claim)
	return err
}
示例#10
0
func (n *rootsDir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	if n.isRO() {
		return nil, fuse.EPERM
	}

	name := req.Name

	// Create a Permanode for the root.
	pr, err := n.fs.client.UploadNewPermanode()
	if err != nil {
		log.Printf("rootsDir.Create(%q): %v", name, err)
		return nil, fuse.EIO
	}

	var grp syncutil.Group
	// Add a camliRoot attribute to the root permanode.
	grp.Go(func() (err error) {
		claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliRoot", name)
		_, err = n.fs.client.UploadAndSignBlob(claim)
		return
	})
	// Set the title of the root permanode to the root name.
	grp.Go(func() (err error) {
		claim := schema.NewSetAttributeClaim(pr.BlobRef, "title", name)
		_, err = n.fs.client.UploadAndSignBlob(claim)
		return
	})
	if err := grp.Err(); err != nil {
		log.Printf("rootsDir.Create(%q): %v", name, err)
		return nil, fuse.EIO
	}

	nod := &mutDir{
		fs:        n.fs,
		permanode: pr.BlobRef,
		name:      name,
		xattrs:    map[string][]byte{},
	}
	n.mu.Lock()
	n.m[name] = pr.BlobRef
	n.mu.Unlock()

	return nod, nil
}
示例#11
0
func (c *permanodeCmd) RunCommand(args []string) error {
	if len(args) > 0 {
		return errors.New("Permanode command doesn't take any additional arguments")
	}

	var (
		permaNode *client.PutResult
		err       error
		up        = getUploader()
	)
	if (c.key != "") != (c.sigTime != "") {
		return errors.New("Both --key and --sigtime must be used to produce deterministic permanodes.")
	}
	if c.key == "" {
		// Normal case, with a random permanode.
		permaNode, err = up.UploadNewPermanode()
	} else {
		const format = "2006-01-02 15:04:05"
		sigTime, err := time.Parse(format, c.sigTime)
		if err != nil {
			return fmt.Errorf("Error parsing time %q; expecting time of form %q", c.sigTime, format)
		}
		permaNode, err = up.UploadPlannedPermanode(c.key, sigTime)
	}
	if handleResult("permanode", permaNode, err) != nil {
		return err
	}

	if c.title != "" {
		put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.title))
		handleResult("claim-permanode-title", put, err)
	}
	if c.tag != "" {
		tags := strings.Split(c.tag, ",")
		m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
		for _, tag := range tags {
			m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
			put, err := up.UploadAndSignBlob(m)
			handleResult("claim-permanode-tag", put, err)
		}
	}
	return nil
}
示例#12
0
func storePhoto(p photo) (string, error) {
	srcFile := localPathOf(p)

	f, err := os.Open(srcFile)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fileRef, err := schema.WriteFileFromReader(camliClient, p.Id+"."+p.Extension, f)

	res, err := camliClient.UploadNewPermanode()
	if err != nil {
		return "", err
	}
	perma := res.BlobRef

	p.Description = cleanHTML(p.Description)

	claims := []*schema.Builder{}
	claims = append(claims, schema.NewSetAttributeClaim(perma, "camliContent", fileRef.String()))
	claims = append(claims, schema.NewSetAttributeClaim(perma, "title", mkTitle(p.Description)))
	claims = append(claims, schema.NewSetAttributeClaim(perma, "description", p.Description))
	for _, t := range p.Tags {
		claims = append(claims, schema.NewAddAttributeClaim(perma, "tag", t))
	}
	if p.Cat == "Public" {
		claims = append(claims, schema.NewSetAttributeClaim(perma, "camliAccess", "public"))
	}

	grp := syncutil.Group{}
	for _, claimBuilder := range claims {
		claim := claimBuilder.Blob()
		grp.Go(func() error {
			_, err := camliClient.UploadAndSignBlob(claim)
			return err
		})
	}

	return perma.String(), grp.Err()
}
示例#13
0
func (o *Object) SetAttr(key, value string) error {
	_, err := o.h.upload(schema.NewSetAttributeClaim(o.pn, key, value))
	if err != nil {
		return err
	}
	o.mu.Lock()
	defer o.mu.Unlock()
	if o.attr == nil {
		o.attr = make(map[string][]string)
	}
	o.attr[key] = []string{value}
	return nil
}
示例#14
0
文件: xattr.go 项目: rn2dy/camlistore
func (x *xattr) set(req *fuse.SetxattrRequest) fuse.Error {
	log.Printf("%s.setxattr(%q) -> %q", x.typeName, req.Name, req.Xattr)

	claim := schema.NewSetAttributeClaim(x.permanode, xattrPrefix+req.Name,
		base64.StdEncoding.EncodeToString(req.Xattr))
	_, err := x.fs.client.UploadAndSignBlob(claim)
	if err != nil {
		log.Printf("Error setting xattr: %v", err)
		return fuse.EIO
	}

	x.mu.Lock()
	(*x.xattrs)[req.Name] = req.Xattr
	x.mu.Unlock()

	return nil
}
示例#15
0
// &fuse.SymlinkRequest{Header:fuse.Header{Conn:(*fuse.Conn)(0xc210047180), ID:0x4, Node:0x8, Uid:0xf0d4, Gid:0x1388, Pid:0x7e88}, NewName:"some-link", Target:"../../some-target"}
func (n *mutDir) Symlink(req *fuse.SymlinkRequest, intr fuse.Intr) (fuse.Node, fuse.Error) {
	node, err := n.creat(req.NewName, symlinkType)
	if err != nil {
		log.Printf("mutDir.Symlink(%q): %v", req.NewName, err)
		return nil, fuse.EIO
	}
	mf := node.(*mutFile)
	mf.symLink = true
	mf.target = req.Target

	claim := schema.NewSetAttributeClaim(mf.permanode, "camliSymlinkTarget", req.Target)
	_, err = n.fs.client.UploadAndSignBlob(claim)
	if err != nil {
		log.Printf("mutDir.Symlink(%q) upload error: %v", req.NewName, err)
		return nil, fuse.EIO
	}

	return node, nil
}
示例#16
0
文件: mut.go 项目: supr/camlistore
func (n *mutDir) creat(name string, isDir bool) (fuse.Node, error) {
	// Create a Permanode for the file/directory.
	pr, err := n.fs.client.UploadNewPermanode()
	if err != nil {
		return nil, err
	}

	// Add a camliPath:name attribute to the directory permanode.
	claim := schema.NewSetAttributeClaim(n.permanode, "camliPath:"+name, pr.BlobRef.String())
	_, err = n.fs.client.UploadAndSignBlob(claim)
	if err != nil {
		return nil, err
	}

	// Add a child node to this node.
	var child fuse.Node
	if isDir {
		child = &mutDir{
			fs:        n.fs,
			permanode: pr.BlobRef,
			parent:    n,
			name:      name,
		}
	} else {
		child = &mutFile{
			fs:        n.fs,
			permanode: pr.BlobRef,
			parent:    n,
			name:      name,
		}
	}
	n.mu.Lock()
	if n.children == nil {
		n.children = make(map[string]fuse.Node)
	}
	n.children[name] = child
	n.mu.Unlock()

	return child, nil
}
示例#17
0
// vivify verifies that all the chunks for the file described by fileblob are on the blobserver.
// It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim
// on that permanode for fileblob, signs it, and uploads it to the blobserver.
func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blob.SizedRef) error {
	sf, ok := blobReceiver.(blob.StreamingFetcher)
	if !ok {
		return fmt.Errorf("BlobReceiver is not a StreamingFetcher")
	}
	fetcher := blob.SeekerFromStreamingFetcher(sf)
	fr, err := schema.NewFileReader(fetcher, fileblob.Ref)
	if err != nil {
		return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.Ref.String(), err)
	}
	defer fr.Close()

	h := sha1.New()
	n, err := io.Copy(h, fr)
	if err != nil {
		return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.Ref.String(), err)
	}
	if n != fr.Size() {
		return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.Ref.String(), fr.Size(), n)
	}

	config := blobReceiver.Config()
	if config == nil {
		return errors.New("blobReceiver has no config")
	}
	hf := config.HandlerFinder
	if hf == nil {
		return errors.New("blobReceiver config has no HandlerFinder")
	}
	JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign")
	if err != nil || sh == nil {
		return errors.New("jsonsign handler not found")
	}
	sigHelper, ok := sh.(*signhandler.Handler)
	if !ok {
		return errors.New("handler is not a JSON signhandler")
	}
	discoMap := sigHelper.DiscoveryMap(JSONSignRoot)
	publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string)
	if !ok {
		return fmt.Errorf("Discovery: json decoding error: %v", err)
	}

	// The file schema must have a modtime to vivify, as the modtime is used for all three of:
	// 1) the permanode's signature
	// 2) the camliContent attribute claim's "claimDate"
	// 3) the signature time of 2)
	claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime)
	if err != nil {
		return fmt.Errorf("While parsing modtime for file %v: %v", fr.FileSchema().FileName, err)
	}

	permanodeBB := schema.NewHashPlannedPermanode(h)
	permanodeBB.SetSigner(blob.MustParse(publicKeyBlobRef))
	permanodeBB.SetClaimDate(claimDate)
	permanodeSigned, err := sigHelper.Sign(permanodeBB)
	if err != nil {
		return fmt.Errorf("Signing permanode %v: %v", permanodeSigned, err)
	}
	permanodeRef := blob.SHA1FromString(permanodeSigned)
	_, err = blobserver.ReceiveNoHash(blobReceiver, permanodeRef, strings.NewReader(permanodeSigned))
	if err != nil {
		return fmt.Errorf("While uploading signed permanode %v, %v: %v", permanodeRef, permanodeSigned, err)
	}

	contentClaimBB := schema.NewSetAttributeClaim(permanodeRef, "camliContent", fileblob.Ref.String())
	contentClaimBB.SetSigner(blob.MustParse(publicKeyBlobRef))
	contentClaimBB.SetClaimDate(claimDate)
	contentClaimSigned, err := sigHelper.Sign(contentClaimBB)
	if err != nil {
		return fmt.Errorf("Signing camliContent claim: %v", err)
	}
	contentClaimRef := blob.SHA1FromString(contentClaimSigned)
	_, err = blobserver.ReceiveNoHash(blobReceiver, contentClaimRef, strings.NewReader(contentClaimSigned))
	if err != nil {
		return fmt.Errorf("While uploading signed camliContent claim %v, %v: %v", contentClaimRef, contentClaimSigned, err)
	}
	return nil
}
示例#18
0
func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
	m := schema.NewCommonFileMap(n.fullPath, n.fi)
	m["camliType"] = "file"
	file, err := up.open(n.fullPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	size := n.fi.Size()

	var fileContents io.Reader = io.LimitReader(file, size)

	if up.fileOpts.wantVivify() {
		err := schema.WriteFileChunks(up.statReceiver(), m, fileContents)
		if err != nil {
			return nil, err
		}
		json, err := m.JSON()
		if err != nil {
			return nil, err
		}
		bref := blobref.SHA1FromString(json)
		h := &client.UploadHandle{
			BlobRef:  bref,
			Size:     int64(len(json)),
			Contents: strings.NewReader(json),
			Vivify:   true,
		}
		return up.Upload(h)
	}

	var (
		blobref *blobref.BlobRef // of file schemaref
		sum     string           // "sha1-xxxxx"
	)

	const dupCheckThreshold = 256 << 10
	if size > dupCheckThreshold {
		sumRef, err := up.wholeFileDigest(n.fullPath)
		if err == nil {
			sum = sumRef.String()
			if ref, ok := up.fileMapFromDuplicate(up.statReceiver(), m, sum); ok {
				blobref = ref
			}
		}
	}

	if blobref == nil {
		if sum == "" && up.fileOpts.wantFilePermanode() {
			fileContents = &trackDigestReader{r: fileContents}
		}
		blobref, err = schema.WriteFileMap(up.statReceiver(), m, fileContents)
		if err != nil {
			return nil, err
		}
	}

	// TODO(mpl): test that none of these claims get uploaded if they've already been done
	if up.fileOpts.wantFilePermanode() {
		if td, ok := fileContents.(*trackDigestReader); ok {
			sum = td.Sum()
		}
		// Use a fixed time value for signing; not using modtime
		// so two identical files don't have different modtimes?
		// TODO(bradfitz): consider this more?
		permaNodeSigTime := time.Unix(0, 0)
		permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode for node %v: %v", n, err)
		}
		handleResult("node-permanode", permaNode, nil)

		// claimTime is both the time of the "claimDate" in the
		// JSON claim, as well as the date in the OpenPGP
		// header.
		// TODO(bradfitz): this is a little clumsy to do by hand.
		// There should probably be a method on *Uploader to do this
		// from an unsigned schema map. Maybe ditch the schema.Claimer
		// type and just have the Uploader override the claimDate.
		claimTime := n.fi.ModTime()

		contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", blobref.String())
		contentAttr.SetClaimDate(claimTime)
		signed, err := up.SignMap(contentAttr, claimTime)
		if err != nil {
			return nil, fmt.Errorf("Failed to sign content claim for node %v: %v", n, err)
		}
		put, err := up.uploadString(signed)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode's attribute for node %v: %v", n, err)
		}
		handleResult("node-permanode-contentattr", put, nil)
		if tags := up.fileOpts.tags(); len(tags) > 0 {
			// TODO(mpl): do these claims concurrently, not in series
			for _, tag := range tags {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				m.SetClaimDate(claimTime)
				// TODO(mpl): verify that SetClaimDate does modify the GPG signature date of the claim
				signed, err := up.SignMap(m, claimTime)
				if err != nil {
					return nil, fmt.Errorf("Failed to sign tag claim for node %v: %v", n, err)
				}
				put, err := up.uploadString(signed)
				if err != nil {
					return nil, fmt.Errorf("Error uploading permanode's tag attribute %v for node %v: %v", tag, n, err)
				}
				handleResult("node-permanode-tag", put, nil)
			}
		}
	}

	// TODO(bradfitz): faking a PutResult here to return
	// is kinda gross.  should instead make a
	// blobserver.Storage wrapper type (wrapping
	// statReceiver) that can track some of this?  or make
	// schemaWriteFileMap return it?
	json, _ := m.JSON()
	pr := &client.PutResult{BlobRef: blobref, Size: int64(len(json)), Skipped: false}
	return pr, nil
}
示例#19
0
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
	if len(args) == 0 {
		return UsageError("No files or directories given.")
	}
	if c.name != "" && !c.makePermanode {
		return UsageError("Can't set name without using --permanode")
	}
	if c.tag != "" && !c.makePermanode {
		return UsageError("Can't set tag without using --permanode")
	}
	if c.histo != "" && !c.memstats {
		return UsageError("Can't use histo without memstats")
	}
	if c.memstats {
		sr := new(statsStatReceiver)
		if c.histo != "" {
			num := 100
			sr.histo = histo.NewHisto(num)
		}
		up.altStatReceiver = sr
		defer func() { sr.DumpStats(c.histo) }()
	}
	if c.statcache {
		cache := NewFlatStatCache()
		up.statCache = cache
	}
	if c.havecache {
		cache := NewFlatHaveCache()
		up.haveCache = cache
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}
	if c.rollSplits {
		up.rollSplits = true
	}

	for _, filename := range args {
		if fi, err := os.Stat(filename); err == nil && fi.IsDir() {
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil {
		put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.name != "" {
			put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
			handleResult("claim-permanode-name", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
			for _, tag := range tags {
				m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignMap(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
示例#20
0
// Populates the bs, and the index at the same time through the sync handler
func populate(b *testing.B, dbfile string,
	sortedProvider func(dbfile string) (sorted.KeyValue, error)) *index.Index {
	b.Logf("populating %v", dbfile)
	kv, err := sortedProvider(dbfile)
	if err != nil {
		b.Fatal(err)
	}
	bsRoot := filepath.Join(filepath.Dir(dbfile), "bs")
	if err := os.MkdirAll(bsRoot, 0700); err != nil {
		b.Fatal(err)
	}
	dataDir, err := os.Open("testdata")
	if err != nil {
		b.Fatal(err)
	}
	fis, err := dataDir.Readdir(-1)
	if err != nil {
		b.Fatal(err)
	}
	if len(fis) == 0 {
		b.Fatalf("no files in %s dir", "testdata")
	}

	ks := doKeyStuff(b)

	bs, err := localdisk.New(bsRoot)
	if err != nil {
		b.Fatal(err)
	}
	if _, err := blobserver.Receive(bs, ks.pubKeyRef, strings.NewReader(ks.pubKey)); err != nil {
		b.Fatal(err)
	}
	idx, err := index.New(kv)
	if err != nil {
		b.Fatal(err)
	}
	idx.InitBlobSource(bs)
	sh := server.NewSyncHandler("/bs/", "/index/", bs, idx, sorted.NewMemoryKeyValue())

	b.ResetTimer()
	for _, v := range fis {
		f, err := os.Open(filepath.Join(dataDir.Name(), v.Name()))
		if err != nil {
			b.Fatal(err)
		}
		td := &trackDigestReader{r: f}
		fm := schema.NewFileMap(v.Name())
		fm.SetModTime(v.ModTime())
		fileRef, err := schema.WriteFileMap(bs, fm, td)
		if err != nil {
			b.Fatal(err)
		}
		f.Close()

		unsigned := schema.NewPlannedPermanode(td.Sum())
		unsigned.SetSigner(ks.pubKeyRef)
		sr := &jsonsign.SignRequest{
			UnsignedJSON: unsigned.Blob().JSON(),
			// TODO(mpl): if we make a bs that discards, replace this with a memory bs that has only the pubkey
			Fetcher:       bs,
			EntityFetcher: ks.entityFetcher,
			SignatureTime: time.Unix(0, 0),
		}
		signed, err := sr.Sign()
		if err != nil {
			b.Fatal("problem signing: " + err.Error())
		}
		pn := blob.SHA1FromString(signed)
		// N.B: use blobserver.Receive so that the blob hub gets notified, and the blob gets enqueued into the index
		if _, err := blobserver.Receive(bs, pn, strings.NewReader(signed)); err != nil {
			b.Fatal(err)
		}

		contentAttr := schema.NewSetAttributeClaim(pn, "camliContent", fileRef.String())
		claimTime, ok := fm.ModTime()
		if !ok {
			b.Fatal(err)
		}
		contentAttr.SetClaimDate(claimTime)
		contentAttr.SetSigner(ks.pubKeyRef)
		sr = &jsonsign.SignRequest{
			UnsignedJSON: contentAttr.Blob().JSON(),
			// TODO(mpl): if we make a bs that discards, replace this with a memory bs that has only the pubkey
			Fetcher:       bs,
			EntityFetcher: ks.entityFetcher,
			SignatureTime: claimTime,
		}
		signed, err = sr.Sign()
		if err != nil {
			b.Fatal("problem signing: " + err.Error())
		}
		cl := blob.SHA1FromString(signed)
		if _, err := blobserver.Receive(bs, cl, strings.NewReader(signed)); err != nil {
			b.Fatal(err)
		}
	}
	sh.IdleWait()

	return idx
}
示例#21
0
func (n *mutDir) creat(name string, typ nodeType) (fuse.Node, error) {
	// Create a Permanode for the file/directory.
	pr, err := n.fs.client.UploadNewPermanode()
	if err != nil {
		return nil, err
	}

	var grp syncutil.Group
	grp.Go(func() (err error) {
		// Add a camliPath:name attribute to the directory permanode.
		claim := schema.NewSetAttributeClaim(n.permanode, "camliPath:"+name, pr.BlobRef.String())
		_, err = n.fs.client.UploadAndSignBlob(claim)
		return
	})

	// Hide OS X Finder .DS_Store junk.  This is distinct from
	// extended attributes.
	if name == ".DS_Store" {
		grp.Go(func() (err error) {
			claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliDefVis", "hide")
			_, err = n.fs.client.UploadAndSignBlob(claim)
			return
		})
	}

	if typ == dirType {
		grp.Go(func() (err error) {
			// Set a directory type on the permanode
			claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliNodeType", "directory")
			_, err = n.fs.client.UploadAndSignBlob(claim)
			return
		})
	}
	if err := grp.Err(); err != nil {
		return nil, err
	}

	// Add a child node to this node.
	var child mutFileOrDir
	switch typ {
	case dirType:
		child = &mutDir{
			fs:        n.fs,
			permanode: pr.BlobRef,
			parent:    n,
			name:      name,
			xattrs:    map[string][]byte{},
		}
	case fileType, symlinkType:
		child = &mutFile{
			fs:        n.fs,
			permanode: pr.BlobRef,
			parent:    n,
			name:      name,
			xattrs:    map[string][]byte{},
		}
	default:
		panic("bogus creat type")
	}
	n.mu.Lock()
	if n.children == nil {
		n.children = make(map[string]mutFileOrDir)
	}
	n.children[name] = child
	n.mu.Unlock()

	return child, nil
}
示例#22
0
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
	if len(args) == 0 {
		return UsageError("No files or directories given.")
	}
	if c.vivify {
		if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
			return UsageError("--vivify excludes any other option")
		}
	}
	if c.name != "" && !c.makePermanode {
		return UsageError("Can't set name without using --permanode")
	}
	if c.tag != "" && !c.makePermanode && !c.filePermanodes {
		return UsageError("Can't set tag without using --permanode or --filenodes")
	}
	if c.histo != "" && !c.memstats {
		return UsageError("Can't use histo without memstats")
	}
	if c.memstats {
		sr := new(statsStatReceiver)
		up.altStatReceiver = sr
		defer func() { sr.DumpStats(c.histo) }()
	}
	if c.statcache || c.havecache {
		gen, err := up.StorageGeneration()
		if err != nil {
			log.Printf("WARNING: not using local caches; failed to retrieve server's storage generation: %v", err)
		} else {
			if c.statcache {
				cache := NewFlatStatCache(gen)
				up.statCache = cache
			}
			if c.havecache {
				cache := NewFlatHaveCache(gen)
				up.haveCache = cache
				up.Client.SetHaveCache(cache)
			}
		}
	}
	if c.makePermanode || c.filePermanodes {
		testSigBlobRef := up.Client.SignerPublicKeyBlobref()
		if testSigBlobRef == nil {
			return UsageError("A gpg key is needed to create permanodes; configure one or use vivify mode.")
		}
	}
	up.fileOpts = &fileOptions{
		permanode: c.filePermanodes,
		tag:       c.tag,
		vivify:    c.vivify,
		exifTime:  c.exifTime,
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}

	for _, filename := range args {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			if up.fileOpts.wantVivify() {
				vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
				continue
			}
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil {
		put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.name != "" {
			put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
			handleResult("claim-permanode-name", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
			for _, tag := range tags {
				m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignMap(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
示例#23
0
func (id *IndexDeps) SetAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef {
	m := schema.NewSetAttributeClaim(permaNode, attr, value)
	m.SetClaimDate(id.advanceTime())
	return id.uploadAndSign(m)
}
示例#24
0
func (c *fileCmd) RunCommand(args []string) error {
	if c.vivify {
		if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
			return cmdmain.UsageError("--vivify excludes any other option")
		}
	}
	if c.name != "" && !c.makePermanode {
		return cmdmain.UsageError("Can't set name without using --permanode")
	}
	if c.tag != "" && !c.makePermanode && !c.filePermanodes {
		return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
	}
	if c.histo != "" && !c.memstats {
		return cmdmain.UsageError("Can't use histo without memstats")
	}
	up := getUploader()
	if c.memstats {
		sr := new(statsStatReceiver)
		up.altStatReceiver = sr
		defer func() { sr.DumpStats(c.histo) }()
	}
	c.initCaches(up)

	if c.makePermanode || c.filePermanodes {
		testSigBlobRef := up.Client.SignerPublicKeyBlobref()
		if testSigBlobRef == nil {
			return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
		}
	}
	up.fileOpts = &fileOptions{
		permanode: c.filePermanodes,
		tag:       c.tag,
		vivify:    c.vivify,
		exifTime:  c.exifTime,
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}
	if c.argsFromInput {
		if len(args) > 0 {
			return errors.New("args not supported with -argsfrominput")
		}
		tu := up.NewRootlessTreeUpload()
		tu.Start()
		br := bufio.NewReader(os.Stdin)
		for {
			path, err := br.ReadString('\n')
			if path = strings.TrimSpace(path); path != "" {
				tu.Enqueue(path)
			}
			if err == io.EOF {
				os.Exit(0)
			}
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	if len(args) == 0 {
		return cmdmain.UsageError("No files or directories given.")
	}
	for _, filename := range args {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			if up.fileOpts.wantVivify() {
				vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
				continue
			}
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil {
		put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.name != "" {
			put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
			handleResult("claim-permanode-name", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
			for _, tag := range tags {
				m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignBlob(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
示例#25
0
func (hl *handlerLoader) initPublisherRootNode(ah *app.Handler) error {
	if !env.IsDev() {
		return nil
	}

	h, err := hl.GetHandler("/my-search/")
	if err != nil {
		return err
	}
	sh := h.(*search.Handler)
	camliRootQuery := func(camliRoot string) (*search.SearchResult, error) {
		return sh.Query(&search.SearchQuery{
			Limit: 1,
			Constraint: &search.Constraint{
				Permanode: &search.PermanodeConstraint{
					Attr:  "camliRoot",
					Value: camliRoot,
				},
			},
		})
	}

	appConfig := ah.AppConfig()
	if appConfig == nil {
		return errors.New("publisher app handler has no AppConfig")
	}
	camliRoot, ok := appConfig["camliRoot"].(string)
	if !ok {
		return fmt.Errorf("camliRoot in publisher app handler appConfig is %T, want string", appConfig["camliRoot"])
	}
	result, err := camliRootQuery(camliRoot)
	if err == nil && len(result.Blobs) > 0 && result.Blobs[0].Blob.Valid() {
		// root node found, nothing more to do.
		log.Printf("Found %v camliRoot node for publisher: %v", camliRoot, result.Blobs[0].Blob.String())
		return nil
	}

	log.Printf("No %v camliRoot node found, creating one from scratch now.", camliRoot)

	bs, err := hl.GetStorage("/bs-recv/")
	if err != nil {
		return err
	}
	h, err = hl.GetHandler("/sighelper/")
	if err != nil {
		return err
	}
	sigh := h.(*signhandler.Handler)

	signUpload := func(bb *schema.Builder) (blob.Ref, error) {
		signed, err := sigh.Sign(bb)
		if err != nil {
			return blob.Ref{}, fmt.Errorf("could not sign blob: %v", err)
		}
		br := blob.SHA1FromString(signed)
		if _, err := blobserver.Receive(bs, br, strings.NewReader(signed)); err != nil {
			return blob.Ref{}, fmt.Errorf("could not upload %v: %v", br.String(), err)
		}
		return br, nil
	}

	pn, err := signUpload(schema.NewUnsignedPermanode())
	if err != nil {
		return fmt.Errorf("could not create new camliRoot node: %v", err)
	}
	if _, err := signUpload(schema.NewSetAttributeClaim(pn, "camliRoot", camliRoot)); err != nil {
		return fmt.Errorf("could not set camliRoot on new node %v: %v", pn, err)
	}
	if _, err := signUpload(schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+camliRoot)); err != nil {
		return fmt.Errorf("could not set camliRoot on new node %v: %v", pn, err)
	}
	return nil
}
示例#26
0
func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
	// TODO(mpl): maybe break this func into more maintainable pieces?
	filebb := schema.NewCommonFileMap(n.fullPath, n.fi)
	filebb.SetType("file")
	file, err := up.open(n.fullPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	if up.fileOpts.exifTime {
		ra, ok := file.(io.ReaderAt)
		if !ok {
			return nil, errors.New("Error asserting local file to io.ReaderAt")
		}
		modtime, err := schema.FileTime(ra)
		if err != nil {
			log.Printf("warning: getting time from EXIF failed for %v: %v", n.fullPath, err)
		} else {
			filebb.SetModTime(modtime)
		}
	}

	var (
		size                           = n.fi.Size()
		fileContents io.Reader         = io.LimitReader(file, size)
		br           *blobref.BlobRef  // of file schemaref
		sum          string            // sha1 hashsum of the file to upload
		pr           *client.PutResult // of the final "file" schema blob
	)

	const dupCheckThreshold = 256 << 10
	if size > dupCheckThreshold {
		sumRef, err := up.wholeFileDigest(n.fullPath)
		if err == nil {
			sum = sumRef.String()
			ok := false
			pr, ok = up.fileMapFromDuplicate(up.statReceiver(n), filebb, sum)
			if ok {
				br = pr.BlobRef
				noteFileUploaded(n.fullPath, !pr.Skipped)
				if up.fileOpts.wantVivify() {
					// we can return early in that case, because the other options
					// are disallowed in the vivify case.
					return pr, nil
				}
			}
		}
	}

	if up.fileOpts.wantVivify() {
		// If vivify wasn't already done in fileMapFromDuplicate.
		err := schema.WriteFileChunks(up.statReceiver(n), filebb, fileContents)
		if err != nil {
			return nil, err
		}
		json, err := filebb.JSON()
		if err != nil {
			return nil, err
		}
		br = blobref.SHA1FromString(json)
		h := &client.UploadHandle{
			BlobRef:  br,
			Size:     int64(len(json)),
			Contents: strings.NewReader(json),
			Vivify:   true,
		}
		pr, err = up.Upload(h)
		if err != nil {
			return nil, err
		}
		noteFileUploaded(n.fullPath, true)
		return pr, nil
	}

	if br == nil {
		// br still nil means fileMapFromDuplicate did not find the file on the server,
		// and the file has not just been uploaded subsequently to a vivify request.
		// So we do the full file + file schema upload here.
		if sum == "" && up.fileOpts.wantFilePermanode() {
			fileContents = &trackDigestReader{r: fileContents}
		}
		br, err = schema.WriteFileMap(up.statReceiver(n), filebb, fileContents)
		if err != nil {
			return nil, err
		}
	}

	// TODO(mpl): test that none of these claims get uploaded if they've already been done
	if up.fileOpts.wantFilePermanode() {
		if td, ok := fileContents.(*trackDigestReader); ok {
			sum = td.Sum()
		}
		// Use a fixed time value for signing; not using modtime
		// so two identical files don't have different modtimes?
		// TODO(bradfitz): consider this more?
		permaNodeSigTime := time.Unix(0, 0)
		permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode for node %v: %v", n, err)
		}
		handleResult("node-permanode", permaNode, nil)

		// claimTime is both the time of the "claimDate" in the
		// JSON claim, as well as the date in the OpenPGP
		// header.
		// TODO(bradfitz): this is a little clumsy to do by hand.
		// There should probably be a method on *Uploader to do this
		// from an unsigned schema map. Maybe ditch the schema.Claimer
		// type and just have the Uploader override the claimDate.
		claimTime, ok := filebb.ModTime()
		if !ok {
			return nil, fmt.Errorf("couldn't get modtime back for file %v", n.fullPath)
		}
		contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", br.String())
		contentAttr.SetClaimDate(claimTime)
		signed, err := up.SignBlob(contentAttr, claimTime)
		if err != nil {
			return nil, fmt.Errorf("Failed to sign content claim for node %v: %v", n, err)
		}
		put, err := up.uploadString(signed)
		if err != nil {
			return nil, fmt.Errorf("Error uploading permanode's attribute for node %v: %v", n, err)
		}
		handleResult("node-permanode-contentattr", put, nil)
		if tags := up.fileOpts.tags(); len(tags) > 0 {
			errch := make(chan error)
			for _, tag := range tags {
				go func(tag string) {
					m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
					m.SetClaimDate(claimTime)
					signed, err := up.SignBlob(m, claimTime)
					if err != nil {
						errch <- fmt.Errorf("Failed to sign tag claim for node %v: %v", n, err)
						return
					}
					put, err := up.uploadString(signed)
					if err != nil {
						errch <- fmt.Errorf("Error uploading permanode's tag attribute %v for node %v: %v", tag, n, err)
						return
					}
					handleResult("node-permanode-tag", put, nil)
					errch <- nil
				}(tag)
			}

			for _ = range tags {
				if e := <-errch; e != nil && err == nil {
					err = e
				}
			}
			if err != nil {
				return nil, err
			}
		}
	}

	// TODO(bradfitz): faking a PutResult here to return
	// is kinda gross.  should instead make a
	// blobserver.Storage wrapper type (wrapping
	// statReceiver) that can track some of this?  or make
	// schemaWriteFileMap return it?
	json, _ := filebb.JSON()
	pr = &client.PutResult{BlobRef: br, Size: int64(len(json)), Skipped: false}
	return pr, nil
}
示例#27
0
func (c *fileCmd) RunCommand(args []string) error {
	if c.vivify {
		if c.makePermanode || c.filePermanodes || c.tag != "" || c.title != "" {
			return cmdmain.UsageError("--vivify excludes any other option")
		}
	}
	if c.title != "" && !c.makePermanode {
		return cmdmain.UsageError("Can't set title without using --permanode")
	}
	if c.tag != "" && !c.makePermanode && !c.filePermanodes {
		return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
	}
	if c.histo != "" && !c.memstats {
		return cmdmain.UsageError("Can't use histo without memstats")
	}
	if c.deleteAfterUpload && !c.filePermanodes {
		return cmdmain.UsageError("Can't set use --delete_after_upload without --filenodes")
	}
	if c.filePermanodes && c.contentsOnly {
		return cmdmain.UsageError("--contents_only and --filenodes are exclusive. Use --permanode instead.")
	}
	// TODO(mpl): do it for other modes too. Or even better, do it once for all modes.
	if *cmdmain.FlagVerbose {
		log.SetOutput(cmdmain.Stderr)
	} else {
		log.SetOutput(ioutil.Discard)
	}
	up := getUploader()
	if c.memstats {
		sr := new(statspkg.Receiver)
		up.altStatReceiver = sr
		defer func() { DumpStats(sr, c.histo) }()
	}
	c.initCaches(up)

	if c.makePermanode || c.filePermanodes {
		testSigBlobRef := up.Client.SignerPublicKeyBlobref()
		if !testSigBlobRef.Valid() {
			return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
		}
	}
	up.fileOpts = &fileOptions{
		permanode:    c.filePermanodes,
		tag:          c.tag,
		vivify:       c.vivify,
		exifTime:     c.exifTime,
		capCtime:     c.capCtime,
		contentsOnly: c.contentsOnly,
	}

	var (
		permaNode *client.PutResult
		lastPut   *client.PutResult
		err       error
	)
	if c.makePermanode {
		if len(args) != 1 {
			return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
		}
		permaNode, err = up.UploadNewPermanode()
		if err != nil {
			return fmt.Errorf("Uploading permanode: %v", err)
		}
	}
	if c.diskUsage {
		if len(args) != 1 {
			return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
		}
		dir := args[0]
		fi, err := up.stat(dir)
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			return fmt.Errorf("%q is not a directory.", dir)
		}
		t := up.NewTreeUpload(dir)
		t.DiskUsageMode = true
		t.Start()
		pr, err := t.Wait()
		if err != nil {
			return err
		}
		handleResult("tree-upload", pr, err)
		return nil
	}
	if c.argsFromInput {
		if len(args) > 0 {
			return errors.New("args not supported with -argsfrominput")
		}
		tu := up.NewRootlessTreeUpload()
		tu.Start()
		br := bufio.NewReader(os.Stdin)
		for {
			path, err := br.ReadString('\n')
			if path = strings.TrimSpace(path); path != "" {
				tu.Enqueue(path)
			}
			if err == io.EOF {
				android.PreExit()
				os.Exit(0)
			}
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	if len(args) == 0 {
		return cmdmain.UsageError("No files or directories given.")
	}
	if up.statCache != nil {
		defer up.statCache.Close()
	}
	for _, filename := range args {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		// Skip ignored files or base directories.  Failing to skip the
		// latter results in a panic.
		if up.Client.IsIgnoredFile(filename) {
			log.Printf("Client configured to ignore %s; skipping.", filename)
			continue
		}
		if fi.IsDir() {
			if up.fileOpts.wantVivify() {
				vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
				continue
			}
			t := up.NewTreeUpload(filename)
			t.Start()
			lastPut, err = t.Wait()
		} else {
			lastPut, err = up.UploadFile(filename)
			if err == nil && c.deleteAfterUpload {
				if err := os.Remove(filename); err != nil {
					log.Printf("Error deleting %v: %v", filename, err)
				} else {
					log.Printf("Deleted %v", filename)
				}
			}
		}
		if handleResult("file", lastPut, err) != nil {
			return err
		}
	}

	if permaNode != nil && lastPut != nil {
		put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
		if handleResult("claim-permanode-content", put, err) != nil {
			return err
		}
		if c.title != "" {
			put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.title))
			handleResult("claim-permanode-title", put, err)
		}
		if c.tag != "" {
			tags := strings.Split(c.tag, ",")
			for _, tag := range tags {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				put, err := up.UploadAndSignBlob(m)
				handleResult("claim-permanode-tag", put, err)
			}
		}
		handleResult("permanode", permaNode, nil)
	}
	return nil
}
示例#28
0
func (id *IndexDeps) SetAttribute_NoTimeMove(permaNode blob.Ref, attr, value string) blob.Ref {
	m := schema.NewSetAttributeClaim(permaNode, attr, value)
	m.SetClaimDate(id.LastTime())
	return id.uploadAndSign(m)
}
示例#29
0
// vivify verifies that all the chunks for the file described by fileblob are on the blobserver.
// It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim
// on that permanode for fileblob, signs it, and uploads it to the blobserver.
func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blobref.SizedBlobRef) error {
	sf, ok := blobReceiver.(blobref.StreamingFetcher)
	if !ok {
		return fmt.Errorf("BlobReceiver is not a StreamingFetcher")
	}
	fetcher := blobref.SeekerFromStreamingFetcher(sf)
	fr, err := schema.NewFileReader(fetcher, fileblob.BlobRef)
	if err != nil {
		return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.BlobRef.String(), err)
	}
	defer fr.Close()

	h := sha1.New()
	n, err := io.Copy(h, fr)
	if err != nil {
		return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.BlobRef.String(), err)
	}
	if n != fr.Size() {
		return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.BlobRef.String(), fr.Size(), n)
	}

	config := blobReceiver.Config()
	if config == nil {
		return errors.New("blobReceiver has no config")
	}
	hf := config.HandlerFinder
	if hf == nil {
		return errors.New("blobReceiver config has no HandlerFinder")
	}
	JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign")
	// TODO(mpl): second check should not be necessary, and yet it happens. Figure it out.
	if err != nil || sh == nil {
		return errors.New("jsonsign handler not found")
	}
	sigHelper, ok := sh.(*signhandler.Handler)
	if !ok {
		return errors.New("handler is not a JSON signhandler")
	}
	discoMap := sigHelper.DiscoveryMap(JSONSignRoot)
	publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string)
	if !ok {
		return fmt.Errorf("Discovery: json decoding error: %v", err)
	}

	unsigned := schema.NewHashPlannedPermanode(h)
	unsigned["camliSigner"] = publicKeyBlobRef
	signed, err := sigHelper.SignMap(unsigned)
	if err != nil {
		return fmt.Errorf("Signing permanode %v: %v", signed, err)
	}
	signedPerm := blobref.SHA1FromString(signed)
	_, err = blobReceiver.ReceiveBlob(signedPerm, strings.NewReader(signed))
	if err != nil {
		return fmt.Errorf("While uploading signed permanode %v: %v", signed, err)
	}

	contentAttr := schema.NewSetAttributeClaim(signedPerm, "camliContent", fileblob.BlobRef.String())
	claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime)
	contentAttr.SetClaimDate(claimDate)
	contentAttr["camliSigner"] = publicKeyBlobRef
	signed, err = sigHelper.SignMap(contentAttr)
	if err != nil {
		return fmt.Errorf("Signing camliContent claim: %v", err)
	}
	signedClaim := blobref.SHA1FromString(signed)
	_, err = blobReceiver.ReceiveBlob(signedClaim, strings.NewReader(signed))
	if err != nil {
		return fmt.Errorf("While uploading signed camliContent claim %v: %v", signed, err)
	}
	return nil
}