func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error {
if len(args) > 0 {
return errors.New("Permanode command doesn't take any additional arguments")
}
var (
permaNode *client.PutResult
err error
)
permaNode, err = up.UploadNewPermanode()
if handleResult("permanode", permaNode, err) != nil {
return err
}
if c.name != "" {
put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.name))
handleResult("claim-permanode-title", put, err)
}
if c.tag != "" {
tags := strings.Split(c.tag, ",")
m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
for _, tag := range tags {
m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
put, err := up.UploadAndSignMap(m)
handleResult("claim-permanode-tag", put, err)
}
}
return nil
}
func (ph *PublishHandler) setRootNode(jsonSign *JSONSignHandler, pn *blobref.BlobRef) (err error) {
_, err = ph.signUpload(jsonSign, "set-attr camliRoot", schema.NewSetAttributeClaim(pn, "camliRoot", ph.RootName))
if err != nil {
return err
}
_, err = ph.signUpload(jsonSign, "set-attr title", schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+ph.RootName))
return err
}
func (n *mutDir) creat(name string, typ nodeType) (fuse.Node, error) {
// Create a Permanode for the file/directory.
pr, err := n.fs.client.UploadNewPermanode()
if err != nil {
return nil, err
}
var grp syncutil.Group
grp.Go(func() (err error) {
// Add a camliPath:name attribute to the directory permanode.
claim := schema.NewSetAttributeClaim(n.permanode, "camliPath:"+name, pr.BlobRef.String())
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
if stupidMacExtendedAttributeName(name) {
grp.Go(func() (err error) {
// Add a camliPath:name attribute to the directory permanode.
claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliDefVis", "hide")
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
}
if err := grp.Err(); err != nil {
return nil, err
}
// Add a child node to this node.
var child mutFileOrDir
switch typ {
case dirType:
child = &mutDir{
fs: n.fs,
permanode: pr.BlobRef,
parent: n,
name: name,
}
case fileType, symlinkType:
child = &mutFile{
fs: n.fs,
permanode: pr.BlobRef,
parent: n,
name: name,
}
default:
panic("bogus creat type")
}
n.mu.Lock()
if n.children == nil {
n.children = make(map[string]mutFileOrDir)
}
n.children[name] = child
n.mu.Unlock()
return child, nil
}
func (c *attrCmd) RunCommand(args []string) error {
if len(args) != 3 {
return errors.New("Attr takes 3 args: <permanode> <attr> <value>")
}
permanode, attr, value := args[0], args[1], args[2]
var err error
pn := blobref.Parse(permanode)
if pn == nil {
return fmt.Errorf("Error parsing blobref %q", permanode)
}
bb := schema.NewSetAttributeClaim(pn, attr, value)
if c.add {
if c.del {
return errors.New("Add and del options are exclusive")
}
bb = schema.NewAddAttributeClaim(pn, attr, value)
} else {
// TODO: del, which can make <value> be optional
if c.del {
return errors.New("del not yet implemented")
}
}
put, err := getUploader().UploadAndSignBlob(bb)
handleResult(bb.Type(), put, err)
return nil
}
func (n *rootsDir) Mkdir(req *fuse.MkdirRequest, intr fuse.Intr) (fuse.Node, fuse.Error) {
name := req.Name
// Create a Permanode for the root.
pr, err := n.fs.client.UploadNewPermanode()
if err != nil {
log.Printf("rootsDir.Create(%q): %v", name, err)
return nil, fuse.EIO
}
// Add a camliRoot attribute to the root permanode.
claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliRoot", name)
_, err = n.fs.client.UploadAndSignBlob(claim)
if err != nil {
log.Printf("rootsDir.Create(%q): %v", name, err)
return nil, fuse.EIO
}
nod := &mutDir{
fs: n.fs,
permanode: pr.BlobRef,
name: name,
}
n.mu.Lock()
n.m[name] = pr.BlobRef
n.mu.Unlock()
return nod, nil
}
// &RenameRequest{Header:fuse.Header{Conn:(*fuse.Conn)(0xc210048180), ID:0x2, Node:0x8, Uid:0xf0d4, Gid:0x1388, Pid:0x5edb}, NewDir:0x8, OldName:"1", NewName:"2"}
func (n *mutDir) Rename(req *fuse.RenameRequest, newDir fuse.Node, intr fuse.Intr) fuse.Error {
n2, ok := newDir.(*mutDir)
if !ok {
log.Printf("*mutDir newDir node isn't a *mutDir; is a %T; can't handle. returning EIO.", newDir)
return fuse.EIO
}
var wg syncutil.Group
wg.Go(n.populate)
wg.Go(n2.populate)
if err := wg.Err(); err != nil {
log.Printf("*mutDir.Rename src dir populate = %v", err)
return fuse.EIO
}
n.mu.Lock()
target, ok := n.children[req.OldName]
n.mu.Unlock()
if !ok {
log.Printf("*mutDir.Rename src name %q isn't known", req.OldName)
return fuse.ENOENT
}
now := time.Now()
// Add a camliPath:name attribute to the dest permanode before unlinking it from
// the source.
claim := schema.NewSetAttributeClaim(n2.permanode, "camliPath:"+req.NewName, target.permanodeString())
claim.SetClaimDate(now)
_, err := n.fs.client.UploadAndSignBlob(claim)
if err != nil {
log.Printf("Upload rename link error: %v", err)
return fuse.EIO
}
delClaim := schema.NewDelAttributeClaim(n.permanode, "camliPath:"+req.OldName, "")
delClaim.SetClaimDate(now)
_, err = n.fs.client.UploadAndSignBlob(delClaim)
if err != nil {
log.Printf("Upload rename src unlink error: %v", err)
return fuse.EIO
}
// TODO(bradfitz): this locking would be racy, if the kernel
// doesn't do it properly. (It should) Let's just trust the
// kernel for now. Later we can verify and remove this
// comment.
n.mu.Lock()
if n.children[req.OldName] != target {
panic("Race.")
}
delete(n.children, req.OldName)
n.mu.Unlock()
n2.mu.Lock()
n2.children[req.NewName] = target
n2.mu.Unlock()
return nil
}
// uploadFilePermanode creates and uploads the planned permanode (with sum as a
// fixed key) associated with the file blobref fileRef.
// It also sets the optional tags for this permanode.
func (up *Uploader) uploadFilePermanode(sum string, fileRef blob.Ref, claimTime time.Time) error {
// Use a fixed time value for signing; not using modtime
// so two identical files don't have different modtimes?
// TODO(bradfitz): consider this more?
permaNodeSigTime := time.Unix(0, 0)
permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
if err != nil {
return fmt.Errorf("Error uploading planned permanode: %v", err)
}
handleResult("node-permanode", permaNode, nil)
contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", fileRef.String())
contentAttr.SetClaimDate(claimTime)
signer, err := up.Signer()
if err != nil {
return err
}
signed, err := contentAttr.SignAt(signer, claimTime)
if err != nil {
return fmt.Errorf("Failed to sign content claim: %v", err)
}
put, err := up.uploadString(signed)
if err != nil {
return fmt.Errorf("Error uploading permanode's attribute: %v", err)
}
handleResult("node-permanode-contentattr", put, nil)
if tags := up.fileOpts.tags(); len(tags) > 0 {
errch := make(chan error)
for _, tag := range tags {
go func(tag string) {
m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
m.SetClaimDate(claimTime)
signed, err := m.SignAt(signer, claimTime)
if err != nil {
errch <- fmt.Errorf("Failed to sign tag claim: %v", err)
return
}
put, err := up.uploadString(signed)
if err != nil {
errch <- fmt.Errorf("Error uploading permanode's tag attribute %v: %v", tag, err)
return
}
handleResult("node-permanode-tag", put, nil)
errch <- nil
}(tag)
}
for range tags {
if e := <-errch; e != nil && err == nil {
err = e
}
}
if err != nil {
return err
}
}
return nil
}
func (n *rootsDir) Rename(req *fuse.RenameRequest, newDir fuse.Node, intr fuse.Intr) fuse.Error {
log.Printf("rootsDir.Rename %q -> %q", req.OldName, req.NewName)
n.mu.Lock()
target, exists := n.m[req.OldName]
_, collision := n.m[req.NewName]
n.mu.Unlock()
if !exists {
log.Printf("*rootsDir.Rename src name %q isn't known", req.OldName)
return fuse.ENOENT
}
if collision {
log.Printf("*rootsDir.Rename dest %q already exists", req.NewName)
return fuse.EIO
}
// Don't allow renames if the root contains content. Rename
// is mostly implemented to make GUIs that create directories
// before asking for the directory name.
res, err := n.fs.client.Describe(&search.DescribeRequest{BlobRef: target})
if err != nil {
log.Println("rootsDir.Rename:", err)
return fuse.EIO
}
db := res.Meta[target.String()]
if db == nil {
log.Printf("Failed to pull meta for target: %v", target)
return fuse.EIO
}
for k := range db.Permanode.Attr {
const p = "camliPath:"
if strings.HasPrefix(k, p) {
log.Printf("Found file in %q: %q, disallowing rename", req.OldName, k[len(p):])
return fuse.EIO
}
}
claim := schema.NewSetAttributeClaim(target, "camliRoot", req.NewName)
_, err = n.fs.client.UploadAndSignBlob(claim)
if err != nil {
log.Printf("Upload rename link error: %v", err)
return fuse.EIO
}
// Comment transplanted from mutDir.Rename
// TODO(bradfitz): this locking would be racy, if the kernel
// doesn't do it properly. (It should) Let's just trust the
// kernel for now. Later we can verify and remove this
// comment.
n.mu.Lock()
if n.m[req.OldName] != target {
panic("Race.")
}
delete(n.m, req.OldName)
n.m[req.NewName] = target
n.mu.Unlock()
return nil
}
func (n *mutFile) setContent(br blob.Ref, size int64) error {
n.mu.Lock()
defer n.mu.Unlock()
n.content = br
n.size = size
claim := schema.NewSetAttributeClaim(n.permanode, "camliContent", br.String())
_, err := n.fs.client.UploadAndSignBlob(claim)
return err
}
func (n *rootsDir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
if n.isRO() {
return nil, fuse.EPERM
}
name := req.Name
// Create a Permanode for the root.
pr, err := n.fs.client.UploadNewPermanode()
if err != nil {
log.Printf("rootsDir.Create(%q): %v", name, err)
return nil, fuse.EIO
}
var grp syncutil.Group
// Add a camliRoot attribute to the root permanode.
grp.Go(func() (err error) {
claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliRoot", name)
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
// Set the title of the root permanode to the root name.
grp.Go(func() (err error) {
claim := schema.NewSetAttributeClaim(pr.BlobRef, "title", name)
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
if err := grp.Err(); err != nil {
log.Printf("rootsDir.Create(%q): %v", name, err)
return nil, fuse.EIO
}
nod := &mutDir{
fs: n.fs,
permanode: pr.BlobRef,
name: name,
xattrs: map[string][]byte{},
}
n.mu.Lock()
n.m[name] = pr.BlobRef
n.mu.Unlock()
return nod, nil
}
func (c *permanodeCmd) RunCommand(args []string) error {
if len(args) > 0 {
return errors.New("Permanode command doesn't take any additional arguments")
}
var (
permaNode *client.PutResult
err error
up = getUploader()
)
if (c.key != "") != (c.sigTime != "") {
return errors.New("Both --key and --sigtime must be used to produce deterministic permanodes.")
}
if c.key == "" {
// Normal case, with a random permanode.
permaNode, err = up.UploadNewPermanode()
} else {
const format = "2006-01-02 15:04:05"
sigTime, err := time.Parse(format, c.sigTime)
if err != nil {
return fmt.Errorf("Error parsing time %q; expecting time of form %q", c.sigTime, format)
}
permaNode, err = up.UploadPlannedPermanode(c.key, sigTime)
}
if handleResult("permanode", permaNode, err) != nil {
return err
}
if c.title != "" {
put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.title))
handleResult("claim-permanode-title", put, err)
}
if c.tag != "" {
tags := strings.Split(c.tag, ",")
m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
for _, tag := range tags {
m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
put, err := up.UploadAndSignBlob(m)
handleResult("claim-permanode-tag", put, err)
}
}
return nil
}
func storePhoto(p photo) (string, error) {
srcFile := localPathOf(p)
f, err := os.Open(srcFile)
if err != nil {
return "", err
}
defer f.Close()
fileRef, err := schema.WriteFileFromReader(camliClient, p.Id+"."+p.Extension, f)
res, err := camliClient.UploadNewPermanode()
if err != nil {
return "", err
}
perma := res.BlobRef
p.Description = cleanHTML(p.Description)
claims := []*schema.Builder{}
claims = append(claims, schema.NewSetAttributeClaim(perma, "camliContent", fileRef.String()))
claims = append(claims, schema.NewSetAttributeClaim(perma, "title", mkTitle(p.Description)))
claims = append(claims, schema.NewSetAttributeClaim(perma, "description", p.Description))
for _, t := range p.Tags {
claims = append(claims, schema.NewAddAttributeClaim(perma, "tag", t))
}
if p.Cat == "Public" {
claims = append(claims, schema.NewSetAttributeClaim(perma, "camliAccess", "public"))
}
grp := syncutil.Group{}
for _, claimBuilder := range claims {
claim := claimBuilder.Blob()
grp.Go(func() error {
_, err := camliClient.UploadAndSignBlob(claim)
return err
})
}
return perma.String(), grp.Err()
}
func (o *Object) SetAttr(key, value string) error {
_, err := o.h.upload(schema.NewSetAttributeClaim(o.pn, key, value))
if err != nil {
return err
}
o.mu.Lock()
defer o.mu.Unlock()
if o.attr == nil {
o.attr = make(map[string][]string)
}
o.attr[key] = []string{value}
return nil
}
func (x *xattr) set(req *fuse.SetxattrRequest) fuse.Error {
log.Printf("%s.setxattr(%q) -> %q", x.typeName, req.Name, req.Xattr)
claim := schema.NewSetAttributeClaim(x.permanode, xattrPrefix+req.Name,
base64.StdEncoding.EncodeToString(req.Xattr))
_, err := x.fs.client.UploadAndSignBlob(claim)
if err != nil {
log.Printf("Error setting xattr: %v", err)
return fuse.EIO
}
x.mu.Lock()
(*x.xattrs)[req.Name] = req.Xattr
x.mu.Unlock()
return nil
}
// &fuse.SymlinkRequest{Header:fuse.Header{Conn:(*fuse.Conn)(0xc210047180), ID:0x4, Node:0x8, Uid:0xf0d4, Gid:0x1388, Pid:0x7e88}, NewName:"some-link", Target:"../../some-target"}
func (n *mutDir) Symlink(req *fuse.SymlinkRequest, intr fuse.Intr) (fuse.Node, fuse.Error) {
node, err := n.creat(req.NewName, symlinkType)
if err != nil {
log.Printf("mutDir.Symlink(%q): %v", req.NewName, err)
return nil, fuse.EIO
}
mf := node.(*mutFile)
mf.symLink = true
mf.target = req.Target
claim := schema.NewSetAttributeClaim(mf.permanode, "camliSymlinkTarget", req.Target)
_, err = n.fs.client.UploadAndSignBlob(claim)
if err != nil {
log.Printf("mutDir.Symlink(%q) upload error: %v", req.NewName, err)
return nil, fuse.EIO
}
return node, nil
}
func (n *mutDir) creat(name string, isDir bool) (fuse.Node, error) {
// Create a Permanode for the file/directory.
pr, err := n.fs.client.UploadNewPermanode()
if err != nil {
return nil, err
}
// Add a camliPath:name attribute to the directory permanode.
claim := schema.NewSetAttributeClaim(n.permanode, "camliPath:"+name, pr.BlobRef.String())
_, err = n.fs.client.UploadAndSignBlob(claim)
if err != nil {
return nil, err
}
// Add a child node to this node.
var child fuse.Node
if isDir {
child = &mutDir{
fs: n.fs,
permanode: pr.BlobRef,
parent: n,
name: name,
}
} else {
child = &mutFile{
fs: n.fs,
permanode: pr.BlobRef,
parent: n,
name: name,
}
}
n.mu.Lock()
if n.children == nil {
n.children = make(map[string]fuse.Node)
}
n.children[name] = child
n.mu.Unlock()
return child, nil
}
// vivify verifies that all the chunks for the file described by fileblob are on the blobserver.
// It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim
// on that permanode for fileblob, signs it, and uploads it to the blobserver.
func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blob.SizedRef) error {
sf, ok := blobReceiver.(blob.StreamingFetcher)
if !ok {
return fmt.Errorf("BlobReceiver is not a StreamingFetcher")
}
fetcher := blob.SeekerFromStreamingFetcher(sf)
fr, err := schema.NewFileReader(fetcher, fileblob.Ref)
if err != nil {
return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.Ref.String(), err)
}
defer fr.Close()
h := sha1.New()
n, err := io.Copy(h, fr)
if err != nil {
return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.Ref.String(), err)
}
if n != fr.Size() {
return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.Ref.String(), fr.Size(), n)
}
config := blobReceiver.Config()
if config == nil {
return errors.New("blobReceiver has no config")
}
hf := config.HandlerFinder
if hf == nil {
return errors.New("blobReceiver config has no HandlerFinder")
}
JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign")
if err != nil || sh == nil {
return errors.New("jsonsign handler not found")
}
sigHelper, ok := sh.(*signhandler.Handler)
if !ok {
return errors.New("handler is not a JSON signhandler")
}
discoMap := sigHelper.DiscoveryMap(JSONSignRoot)
publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string)
if !ok {
return fmt.Errorf("Discovery: json decoding error: %v", err)
}
// The file schema must have a modtime to vivify, as the modtime is used for all three of:
// 1) the permanode's signature
// 2) the camliContent attribute claim's "claimDate"
// 3) the signature time of 2)
claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime)
if err != nil {
return fmt.Errorf("While parsing modtime for file %v: %v", fr.FileSchema().FileName, err)
}
permanodeBB := schema.NewHashPlannedPermanode(h)
permanodeBB.SetSigner(blob.MustParse(publicKeyBlobRef))
permanodeBB.SetClaimDate(claimDate)
permanodeSigned, err := sigHelper.Sign(permanodeBB)
if err != nil {
return fmt.Errorf("Signing permanode %v: %v", permanodeSigned, err)
}
permanodeRef := blob.SHA1FromString(permanodeSigned)
_, err = blobserver.ReceiveNoHash(blobReceiver, permanodeRef, strings.NewReader(permanodeSigned))
if err != nil {
return fmt.Errorf("While uploading signed permanode %v, %v: %v", permanodeRef, permanodeSigned, err)
}
contentClaimBB := schema.NewSetAttributeClaim(permanodeRef, "camliContent", fileblob.Ref.String())
contentClaimBB.SetSigner(blob.MustParse(publicKeyBlobRef))
contentClaimBB.SetClaimDate(claimDate)
contentClaimSigned, err := sigHelper.Sign(contentClaimBB)
if err != nil {
return fmt.Errorf("Signing camliContent claim: %v", err)
}
contentClaimRef := blob.SHA1FromString(contentClaimSigned)
_, err = blobserver.ReceiveNoHash(blobReceiver, contentClaimRef, strings.NewReader(contentClaimSigned))
if err != nil {
return fmt.Errorf("While uploading signed camliContent claim %v, %v: %v", contentClaimRef, contentClaimSigned, err)
}
return nil
}
func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
m := schema.NewCommonFileMap(n.fullPath, n.fi)
m["camliType"] = "file"
file, err := up.open(n.fullPath)
if err != nil {
return nil, err
}
defer file.Close()
size := n.fi.Size()
var fileContents io.Reader = io.LimitReader(file, size)
if up.fileOpts.wantVivify() {
err := schema.WriteFileChunks(up.statReceiver(), m, fileContents)
if err != nil {
return nil, err
}
json, err := m.JSON()
if err != nil {
return nil, err
}
bref := blobref.SHA1FromString(json)
h := &client.UploadHandle{
BlobRef: bref,
Size: int64(len(json)),
Contents: strings.NewReader(json),
Vivify: true,
}
return up.Upload(h)
}
var (
blobref *blobref.BlobRef // of file schemaref
sum string // "sha1-xxxxx"
)
const dupCheckThreshold = 256 << 10
if size > dupCheckThreshold {
sumRef, err := up.wholeFileDigest(n.fullPath)
if err == nil {
sum = sumRef.String()
if ref, ok := up.fileMapFromDuplicate(up.statReceiver(), m, sum); ok {
blobref = ref
}
}
}
if blobref == nil {
if sum == "" && up.fileOpts.wantFilePermanode() {
fileContents = &trackDigestReader{r: fileContents}
}
blobref, err = schema.WriteFileMap(up.statReceiver(), m, fileContents)
if err != nil {
return nil, err
}
}
// TODO(mpl): test that none of these claims get uploaded if they've already been done
if up.fileOpts.wantFilePermanode() {
if td, ok := fileContents.(*trackDigestReader); ok {
sum = td.Sum()
}
// Use a fixed time value for signing; not using modtime
// so two identical files don't have different modtimes?
// TODO(bradfitz): consider this more?
permaNodeSigTime := time.Unix(0, 0)
permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
if err != nil {
return nil, fmt.Errorf("Error uploading permanode for node %v: %v", n, err)
}
handleResult("node-permanode", permaNode, nil)
// claimTime is both the time of the "claimDate" in the
// JSON claim, as well as the date in the OpenPGP
// header.
// TODO(bradfitz): this is a little clumsy to do by hand.
// There should probably be a method on *Uploader to do this
// from an unsigned schema map. Maybe ditch the schema.Claimer
// type and just have the Uploader override the claimDate.
claimTime := n.fi.ModTime()
contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", blobref.String())
contentAttr.SetClaimDate(claimTime)
signed, err := up.SignMap(contentAttr, claimTime)
if err != nil {
return nil, fmt.Errorf("Failed to sign content claim for node %v: %v", n, err)
}
put, err := up.uploadString(signed)
if err != nil {
return nil, fmt.Errorf("Error uploading permanode's attribute for node %v: %v", n, err)
}
handleResult("node-permanode-contentattr", put, nil)
if tags := up.fileOpts.tags(); len(tags) > 0 {
// TODO(mpl): do these claims concurrently, not in series
for _, tag := range tags {
m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
m.SetClaimDate(claimTime)
// TODO(mpl): verify that SetClaimDate does modify the GPG signature date of the claim
signed, err := up.SignMap(m, claimTime)
if err != nil {
return nil, fmt.Errorf("Failed to sign tag claim for node %v: %v", n, err)
}
put, err := up.uploadString(signed)
if err != nil {
return nil, fmt.Errorf("Error uploading permanode's tag attribute %v for node %v: %v", tag, n, err)
}
handleResult("node-permanode-tag", put, nil)
}
}
}
// TODO(bradfitz): faking a PutResult here to return
// is kinda gross. should instead make a
// blobserver.Storage wrapper type (wrapping
// statReceiver) that can track some of this? or make
// schemaWriteFileMap return it?
json, _ := m.JSON()
pr := &client.PutResult{BlobRef: blobref, Size: int64(len(json)), Skipped: false}
return pr, nil
}
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
if len(args) == 0 {
return UsageError("No files or directories given.")
}
if c.name != "" && !c.makePermanode {
return UsageError("Can't set name without using --permanode")
}
if c.tag != "" && !c.makePermanode {
return UsageError("Can't set tag without using --permanode")
}
if c.histo != "" && !c.memstats {
return UsageError("Can't use histo without memstats")
}
if c.memstats {
sr := new(statsStatReceiver)
if c.histo != "" {
num := 100
sr.histo = histo.NewHisto(num)
}
up.altStatReceiver = sr
defer func() { sr.DumpStats(c.histo) }()
}
if c.statcache {
cache := NewFlatStatCache()
up.statCache = cache
}
if c.havecache {
cache := NewFlatHaveCache()
up.haveCache = cache
}
var (
permaNode *client.PutResult
lastPut *client.PutResult
err error
)
if c.makePermanode {
if len(args) != 1 {
return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
}
permaNode, err = up.UploadNewPermanode()
if err != nil {
return fmt.Errorf("Uploading permanode: %v", err)
}
}
if c.diskUsage {
if len(args) != 1 {
return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
}
dir := args[0]
fi, err := up.stat(dir)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%q is not a directory.", dir)
}
t := up.NewTreeUpload(dir)
t.DiskUsageMode = true
t.Start()
pr, err := t.Wait()
if err != nil {
return err
}
handleResult("tree-upload", pr, err)
return nil
}
if c.rollSplits {
up.rollSplits = true
}
for _, filename := range args {
if fi, err := os.Stat(filename); err == nil && fi.IsDir() {
t := up.NewTreeUpload(filename)
t.Start()
lastPut, err = t.Wait()
} else {
lastPut, err = up.UploadFile(filename)
}
if handleResult("file", lastPut, err) != nil {
return err
}
}
if permaNode != nil {
put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
if handleResult("claim-permanode-content", put, err) != nil {
return err
}
if c.name != "" {
put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
handleResult("claim-permanode-name", put, err)
}
if c.tag != "" {
tags := strings.Split(c.tag, ",")
m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
for _, tag := range tags {
m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
put, err := up.UploadAndSignMap(m)
handleResult("claim-permanode-tag", put, err)
}
}
handleResult("permanode", permaNode, nil)
}
return nil
}
// Populates the bs, and the index at the same time through the sync handler
func populate(b *testing.B, dbfile string,
sortedProvider func(dbfile string) (sorted.KeyValue, error)) *index.Index {
b.Logf("populating %v", dbfile)
kv, err := sortedProvider(dbfile)
if err != nil {
b.Fatal(err)
}
bsRoot := filepath.Join(filepath.Dir(dbfile), "bs")
if err := os.MkdirAll(bsRoot, 0700); err != nil {
b.Fatal(err)
}
dataDir, err := os.Open("testdata")
if err != nil {
b.Fatal(err)
}
fis, err := dataDir.Readdir(-1)
if err != nil {
b.Fatal(err)
}
if len(fis) == 0 {
b.Fatalf("no files in %s dir", "testdata")
}
ks := doKeyStuff(b)
bs, err := localdisk.New(bsRoot)
if err != nil {
b.Fatal(err)
}
if _, err := blobserver.Receive(bs, ks.pubKeyRef, strings.NewReader(ks.pubKey)); err != nil {
b.Fatal(err)
}
idx, err := index.New(kv)
if err != nil {
b.Fatal(err)
}
idx.InitBlobSource(bs)
sh := server.NewSyncHandler("/bs/", "/index/", bs, idx, sorted.NewMemoryKeyValue())
b.ResetTimer()
for _, v := range fis {
f, err := os.Open(filepath.Join(dataDir.Name(), v.Name()))
if err != nil {
b.Fatal(err)
}
td := &trackDigestReader{r: f}
fm := schema.NewFileMap(v.Name())
fm.SetModTime(v.ModTime())
fileRef, err := schema.WriteFileMap(bs, fm, td)
if err != nil {
b.Fatal(err)
}
f.Close()
unsigned := schema.NewPlannedPermanode(td.Sum())
unsigned.SetSigner(ks.pubKeyRef)
sr := &jsonsign.SignRequest{
UnsignedJSON: unsigned.Blob().JSON(),
// TODO(mpl): if we make a bs that discards, replace this with a memory bs that has only the pubkey
Fetcher: bs,
EntityFetcher: ks.entityFetcher,
SignatureTime: time.Unix(0, 0),
}
signed, err := sr.Sign()
if err != nil {
b.Fatal("problem signing: " + err.Error())
}
pn := blob.SHA1FromString(signed)
// N.B: use blobserver.Receive so that the blob hub gets notified, and the blob gets enqueued into the index
if _, err := blobserver.Receive(bs, pn, strings.NewReader(signed)); err != nil {
b.Fatal(err)
}
contentAttr := schema.NewSetAttributeClaim(pn, "camliContent", fileRef.String())
claimTime, ok := fm.ModTime()
if !ok {
b.Fatal(err)
}
contentAttr.SetClaimDate(claimTime)
contentAttr.SetSigner(ks.pubKeyRef)
sr = &jsonsign.SignRequest{
UnsignedJSON: contentAttr.Blob().JSON(),
// TODO(mpl): if we make a bs that discards, replace this with a memory bs that has only the pubkey
Fetcher: bs,
EntityFetcher: ks.entityFetcher,
SignatureTime: claimTime,
}
signed, err = sr.Sign()
if err != nil {
b.Fatal("problem signing: " + err.Error())
}
cl := blob.SHA1FromString(signed)
if _, err := blobserver.Receive(bs, cl, strings.NewReader(signed)); err != nil {
b.Fatal(err)
}
}
sh.IdleWait()
return idx
}
func (n *mutDir) creat(name string, typ nodeType) (fuse.Node, error) {
// Create a Permanode for the file/directory.
pr, err := n.fs.client.UploadNewPermanode()
if err != nil {
return nil, err
}
var grp syncutil.Group
grp.Go(func() (err error) {
// Add a camliPath:name attribute to the directory permanode.
claim := schema.NewSetAttributeClaim(n.permanode, "camliPath:"+name, pr.BlobRef.String())
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
// Hide OS X Finder .DS_Store junk. This is distinct from
// extended attributes.
if name == ".DS_Store" {
grp.Go(func() (err error) {
claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliDefVis", "hide")
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
}
if typ == dirType {
grp.Go(func() (err error) {
// Set a directory type on the permanode
claim := schema.NewSetAttributeClaim(pr.BlobRef, "camliNodeType", "directory")
_, err = n.fs.client.UploadAndSignBlob(claim)
return
})
}
if err := grp.Err(); err != nil {
return nil, err
}
// Add a child node to this node.
var child mutFileOrDir
switch typ {
case dirType:
child = &mutDir{
fs: n.fs,
permanode: pr.BlobRef,
parent: n,
name: name,
xattrs: map[string][]byte{},
}
case fileType, symlinkType:
child = &mutFile{
fs: n.fs,
permanode: pr.BlobRef,
parent: n,
name: name,
xattrs: map[string][]byte{},
}
default:
panic("bogus creat type")
}
n.mu.Lock()
if n.children == nil {
n.children = make(map[string]mutFileOrDir)
}
n.children[name] = child
n.mu.Unlock()
return child, nil
}
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
if len(args) == 0 {
return UsageError("No files or directories given.")
}
if c.vivify {
if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
return UsageError("--vivify excludes any other option")
}
}
if c.name != "" && !c.makePermanode {
return UsageError("Can't set name without using --permanode")
}
if c.tag != "" && !c.makePermanode && !c.filePermanodes {
return UsageError("Can't set tag without using --permanode or --filenodes")
}
if c.histo != "" && !c.memstats {
return UsageError("Can't use histo without memstats")
}
if c.memstats {
sr := new(statsStatReceiver)
up.altStatReceiver = sr
defer func() { sr.DumpStats(c.histo) }()
}
if c.statcache || c.havecache {
gen, err := up.StorageGeneration()
if err != nil {
log.Printf("WARNING: not using local caches; failed to retrieve server's storage generation: %v", err)
} else {
if c.statcache {
cache := NewFlatStatCache(gen)
up.statCache = cache
}
if c.havecache {
cache := NewFlatHaveCache(gen)
up.haveCache = cache
up.Client.SetHaveCache(cache)
}
}
}
if c.makePermanode || c.filePermanodes {
testSigBlobRef := up.Client.SignerPublicKeyBlobref()
if testSigBlobRef == nil {
return UsageError("A gpg key is needed to create permanodes; configure one or use vivify mode.")
}
}
up.fileOpts = &fileOptions{
permanode: c.filePermanodes,
tag: c.tag,
vivify: c.vivify,
exifTime: c.exifTime,
}
var (
permaNode *client.PutResult
lastPut *client.PutResult
err error
)
if c.makePermanode {
if len(args) != 1 {
return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
}
permaNode, err = up.UploadNewPermanode()
if err != nil {
return fmt.Errorf("Uploading permanode: %v", err)
}
}
if c.diskUsage {
if len(args) != 1 {
return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
}
dir := args[0]
fi, err := up.stat(dir)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%q is not a directory.", dir)
}
t := up.NewTreeUpload(dir)
t.DiskUsageMode = true
t.Start()
pr, err := t.Wait()
if err != nil {
return err
}
handleResult("tree-upload", pr, err)
return nil
}
for _, filename := range args {
fi, err := os.Stat(filename)
if err != nil {
return err
}
if fi.IsDir() {
if up.fileOpts.wantVivify() {
vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
continue
}
t := up.NewTreeUpload(filename)
t.Start()
lastPut, err = t.Wait()
} else {
lastPut, err = up.UploadFile(filename)
}
if handleResult("file", lastPut, err) != nil {
return err
}
}
if permaNode != nil {
put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
if handleResult("claim-permanode-content", put, err) != nil {
return err
}
if c.name != "" {
put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
handleResult("claim-permanode-name", put, err)
}
if c.tag != "" {
tags := strings.Split(c.tag, ",")
m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
for _, tag := range tags {
m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
put, err := up.UploadAndSignMap(m)
handleResult("claim-permanode-tag", put, err)
}
}
handleResult("permanode", permaNode, nil)
}
return nil
}
func (id *IndexDeps) SetAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef {
m := schema.NewSetAttributeClaim(permaNode, attr, value)
m.SetClaimDate(id.advanceTime())
return id.uploadAndSign(m)
}
func (c *fileCmd) RunCommand(args []string) error {
if c.vivify {
if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
return cmdmain.UsageError("--vivify excludes any other option")
}
}
if c.name != "" && !c.makePermanode {
return cmdmain.UsageError("Can't set name without using --permanode")
}
if c.tag != "" && !c.makePermanode && !c.filePermanodes {
return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
}
if c.histo != "" && !c.memstats {
return cmdmain.UsageError("Can't use histo without memstats")
}
up := getUploader()
if c.memstats {
sr := new(statsStatReceiver)
up.altStatReceiver = sr
defer func() { sr.DumpStats(c.histo) }()
}
c.initCaches(up)
if c.makePermanode || c.filePermanodes {
testSigBlobRef := up.Client.SignerPublicKeyBlobref()
if testSigBlobRef == nil {
return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
}
}
up.fileOpts = &fileOptions{
permanode: c.filePermanodes,
tag: c.tag,
vivify: c.vivify,
exifTime: c.exifTime,
}
var (
permaNode *client.PutResult
lastPut *client.PutResult
err error
)
if c.makePermanode {
if len(args) != 1 {
return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
}
permaNode, err = up.UploadNewPermanode()
if err != nil {
return fmt.Errorf("Uploading permanode: %v", err)
}
}
if c.diskUsage {
if len(args) != 1 {
return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
}
dir := args[0]
fi, err := up.stat(dir)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%q is not a directory.", dir)
}
t := up.NewTreeUpload(dir)
t.DiskUsageMode = true
t.Start()
pr, err := t.Wait()
if err != nil {
return err
}
handleResult("tree-upload", pr, err)
return nil
}
if c.argsFromInput {
if len(args) > 0 {
return errors.New("args not supported with -argsfrominput")
}
tu := up.NewRootlessTreeUpload()
tu.Start()
br := bufio.NewReader(os.Stdin)
for {
path, err := br.ReadString('\n')
if path = strings.TrimSpace(path); path != "" {
tu.Enqueue(path)
}
if err == io.EOF {
os.Exit(0)
}
if err != nil {
log.Fatal(err)
}
}
}
if len(args) == 0 {
return cmdmain.UsageError("No files or directories given.")
}
for _, filename := range args {
fi, err := os.Stat(filename)
if err != nil {
return err
}
if fi.IsDir() {
if up.fileOpts.wantVivify() {
vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
continue
}
t := up.NewTreeUpload(filename)
t.Start()
lastPut, err = t.Wait()
} else {
lastPut, err = up.UploadFile(filename)
}
if handleResult("file", lastPut, err) != nil {
return err
}
}
if permaNode != nil {
put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
if handleResult("claim-permanode-content", put, err) != nil {
return err
}
if c.name != "" {
put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name))
handleResult("claim-permanode-name", put, err)
}
if c.tag != "" {
tags := strings.Split(c.tag, ",")
m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0])
for _, tag := range tags {
m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
put, err := up.UploadAndSignBlob(m)
handleResult("claim-permanode-tag", put, err)
}
}
handleResult("permanode", permaNode, nil)
}
return nil
}
func (hl *handlerLoader) initPublisherRootNode(ah *app.Handler) error {
if !env.IsDev() {
return nil
}
h, err := hl.GetHandler("/my-search/")
if err != nil {
return err
}
sh := h.(*search.Handler)
camliRootQuery := func(camliRoot string) (*search.SearchResult, error) {
return sh.Query(&search.SearchQuery{
Limit: 1,
Constraint: &search.Constraint{
Permanode: &search.PermanodeConstraint{
Attr: "camliRoot",
Value: camliRoot,
},
},
})
}
appConfig := ah.AppConfig()
if appConfig == nil {
return errors.New("publisher app handler has no AppConfig")
}
camliRoot, ok := appConfig["camliRoot"].(string)
if !ok {
return fmt.Errorf("camliRoot in publisher app handler appConfig is %T, want string", appConfig["camliRoot"])
}
result, err := camliRootQuery(camliRoot)
if err == nil && len(result.Blobs) > 0 && result.Blobs[0].Blob.Valid() {
// root node found, nothing more to do.
log.Printf("Found %v camliRoot node for publisher: %v", camliRoot, result.Blobs[0].Blob.String())
return nil
}
log.Printf("No %v camliRoot node found, creating one from scratch now.", camliRoot)
bs, err := hl.GetStorage("/bs-recv/")
if err != nil {
return err
}
h, err = hl.GetHandler("/sighelper/")
if err != nil {
return err
}
sigh := h.(*signhandler.Handler)
signUpload := func(bb *schema.Builder) (blob.Ref, error) {
signed, err := sigh.Sign(bb)
if err != nil {
return blob.Ref{}, fmt.Errorf("could not sign blob: %v", err)
}
br := blob.SHA1FromString(signed)
if _, err := blobserver.Receive(bs, br, strings.NewReader(signed)); err != nil {
return blob.Ref{}, fmt.Errorf("could not upload %v: %v", br.String(), err)
}
return br, nil
}
pn, err := signUpload(schema.NewUnsignedPermanode())
if err != nil {
return fmt.Errorf("could not create new camliRoot node: %v", err)
}
if _, err := signUpload(schema.NewSetAttributeClaim(pn, "camliRoot", camliRoot)); err != nil {
return fmt.Errorf("could not set camliRoot on new node %v: %v", pn, err)
}
if _, err := signUpload(schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+camliRoot)); err != nil {
return fmt.Errorf("could not set camliRoot on new node %v: %v", pn, err)
}
return nil
}
func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
// TODO(mpl): maybe break this func into more maintainable pieces?
filebb := schema.NewCommonFileMap(n.fullPath, n.fi)
filebb.SetType("file")
file, err := up.open(n.fullPath)
if err != nil {
return nil, err
}
defer file.Close()
if up.fileOpts.exifTime {
ra, ok := file.(io.ReaderAt)
if !ok {
return nil, errors.New("Error asserting local file to io.ReaderAt")
}
modtime, err := schema.FileTime(ra)
if err != nil {
log.Printf("warning: getting time from EXIF failed for %v: %v", n.fullPath, err)
} else {
filebb.SetModTime(modtime)
}
}
var (
size = n.fi.Size()
fileContents io.Reader = io.LimitReader(file, size)
br *blobref.BlobRef // of file schemaref
sum string // sha1 hashsum of the file to upload
pr *client.PutResult // of the final "file" schema blob
)
const dupCheckThreshold = 256 << 10
if size > dupCheckThreshold {
sumRef, err := up.wholeFileDigest(n.fullPath)
if err == nil {
sum = sumRef.String()
ok := false
pr, ok = up.fileMapFromDuplicate(up.statReceiver(n), filebb, sum)
if ok {
br = pr.BlobRef
noteFileUploaded(n.fullPath, !pr.Skipped)
if up.fileOpts.wantVivify() {
// we can return early in that case, because the other options
// are disallowed in the vivify case.
return pr, nil
}
}
}
}
if up.fileOpts.wantVivify() {
// If vivify wasn't already done in fileMapFromDuplicate.
err := schema.WriteFileChunks(up.statReceiver(n), filebb, fileContents)
if err != nil {
return nil, err
}
json, err := filebb.JSON()
if err != nil {
return nil, err
}
br = blobref.SHA1FromString(json)
h := &client.UploadHandle{
BlobRef: br,
Size: int64(len(json)),
Contents: strings.NewReader(json),
Vivify: true,
}
pr, err = up.Upload(h)
if err != nil {
return nil, err
}
noteFileUploaded(n.fullPath, true)
return pr, nil
}
if br == nil {
// br still nil means fileMapFromDuplicate did not find the file on the server,
// and the file has not just been uploaded subsequently to a vivify request.
// So we do the full file + file schema upload here.
if sum == "" && up.fileOpts.wantFilePermanode() {
fileContents = &trackDigestReader{r: fileContents}
}
br, err = schema.WriteFileMap(up.statReceiver(n), filebb, fileContents)
if err != nil {
return nil, err
}
}
// TODO(mpl): test that none of these claims get uploaded if they've already been done
if up.fileOpts.wantFilePermanode() {
if td, ok := fileContents.(*trackDigestReader); ok {
sum = td.Sum()
}
// Use a fixed time value for signing; not using modtime
// so two identical files don't have different modtimes?
// TODO(bradfitz): consider this more?
permaNodeSigTime := time.Unix(0, 0)
permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
if err != nil {
return nil, fmt.Errorf("Error uploading permanode for node %v: %v", n, err)
}
handleResult("node-permanode", permaNode, nil)
// claimTime is both the time of the "claimDate" in the
// JSON claim, as well as the date in the OpenPGP
// header.
// TODO(bradfitz): this is a little clumsy to do by hand.
// There should probably be a method on *Uploader to do this
// from an unsigned schema map. Maybe ditch the schema.Claimer
// type and just have the Uploader override the claimDate.
claimTime, ok := filebb.ModTime()
if !ok {
return nil, fmt.Errorf("couldn't get modtime back for file %v", n.fullPath)
}
contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", br.String())
contentAttr.SetClaimDate(claimTime)
signed, err := up.SignBlob(contentAttr, claimTime)
if err != nil {
return nil, fmt.Errorf("Failed to sign content claim for node %v: %v", n, err)
}
put, err := up.uploadString(signed)
if err != nil {
return nil, fmt.Errorf("Error uploading permanode's attribute for node %v: %v", n, err)
}
handleResult("node-permanode-contentattr", put, nil)
if tags := up.fileOpts.tags(); len(tags) > 0 {
errch := make(chan error)
for _, tag := range tags {
go func(tag string) {
m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
m.SetClaimDate(claimTime)
signed, err := up.SignBlob(m, claimTime)
if err != nil {
errch <- fmt.Errorf("Failed to sign tag claim for node %v: %v", n, err)
return
}
put, err := up.uploadString(signed)
if err != nil {
errch <- fmt.Errorf("Error uploading permanode's tag attribute %v for node %v: %v", tag, n, err)
return
}
handleResult("node-permanode-tag", put, nil)
errch <- nil
}(tag)
}
for _ = range tags {
if e := <-errch; e != nil && err == nil {
err = e
}
}
if err != nil {
return nil, err
}
}
}
// TODO(bradfitz): faking a PutResult here to return
// is kinda gross. should instead make a
// blobserver.Storage wrapper type (wrapping
// statReceiver) that can track some of this? or make
// schemaWriteFileMap return it?
json, _ := filebb.JSON()
pr = &client.PutResult{BlobRef: br, Size: int64(len(json)), Skipped: false}
return pr, nil
}
func (c *fileCmd) RunCommand(args []string) error {
if c.vivify {
if c.makePermanode || c.filePermanodes || c.tag != "" || c.title != "" {
return cmdmain.UsageError("--vivify excludes any other option")
}
}
if c.title != "" && !c.makePermanode {
return cmdmain.UsageError("Can't set title without using --permanode")
}
if c.tag != "" && !c.makePermanode && !c.filePermanodes {
return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
}
if c.histo != "" && !c.memstats {
return cmdmain.UsageError("Can't use histo without memstats")
}
if c.deleteAfterUpload && !c.filePermanodes {
return cmdmain.UsageError("Can't set use --delete_after_upload without --filenodes")
}
if c.filePermanodes && c.contentsOnly {
return cmdmain.UsageError("--contents_only and --filenodes are exclusive. Use --permanode instead.")
}
// TODO(mpl): do it for other modes too. Or even better, do it once for all modes.
if *cmdmain.FlagVerbose {
log.SetOutput(cmdmain.Stderr)
} else {
log.SetOutput(ioutil.Discard)
}
up := getUploader()
if c.memstats {
sr := new(statspkg.Receiver)
up.altStatReceiver = sr
defer func() { DumpStats(sr, c.histo) }()
}
c.initCaches(up)
if c.makePermanode || c.filePermanodes {
testSigBlobRef := up.Client.SignerPublicKeyBlobref()
if !testSigBlobRef.Valid() {
return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
}
}
up.fileOpts = &fileOptions{
permanode: c.filePermanodes,
tag: c.tag,
vivify: c.vivify,
exifTime: c.exifTime,
capCtime: c.capCtime,
contentsOnly: c.contentsOnly,
}
var (
permaNode *client.PutResult
lastPut *client.PutResult
err error
)
if c.makePermanode {
if len(args) != 1 {
return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument")
}
permaNode, err = up.UploadNewPermanode()
if err != nil {
return fmt.Errorf("Uploading permanode: %v", err)
}
}
if c.diskUsage {
if len(args) != 1 {
return fmt.Errorf("The --du flag can only be used with exactly one directory argument")
}
dir := args[0]
fi, err := up.stat(dir)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%q is not a directory.", dir)
}
t := up.NewTreeUpload(dir)
t.DiskUsageMode = true
t.Start()
pr, err := t.Wait()
if err != nil {
return err
}
handleResult("tree-upload", pr, err)
return nil
}
if c.argsFromInput {
if len(args) > 0 {
return errors.New("args not supported with -argsfrominput")
}
tu := up.NewRootlessTreeUpload()
tu.Start()
br := bufio.NewReader(os.Stdin)
for {
path, err := br.ReadString('\n')
if path = strings.TrimSpace(path); path != "" {
tu.Enqueue(path)
}
if err == io.EOF {
android.PreExit()
os.Exit(0)
}
if err != nil {
log.Fatal(err)
}
}
}
if len(args) == 0 {
return cmdmain.UsageError("No files or directories given.")
}
if up.statCache != nil {
defer up.statCache.Close()
}
for _, filename := range args {
fi, err := os.Stat(filename)
if err != nil {
return err
}
// Skip ignored files or base directories. Failing to skip the
// latter results in a panic.
if up.Client.IsIgnoredFile(filename) {
log.Printf("Client configured to ignore %s; skipping.", filename)
continue
}
if fi.IsDir() {
if up.fileOpts.wantVivify() {
vlog.Printf("Directories not supported in vivify mode; skipping %v\n", filename)
continue
}
t := up.NewTreeUpload(filename)
t.Start()
lastPut, err = t.Wait()
} else {
lastPut, err = up.UploadFile(filename)
if err == nil && c.deleteAfterUpload {
if err := os.Remove(filename); err != nil {
log.Printf("Error deleting %v: %v", filename, err)
} else {
log.Printf("Deleted %v", filename)
}
}
}
if handleResult("file", lastPut, err) != nil {
return err
}
}
if permaNode != nil && lastPut != nil {
put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String()))
if handleResult("claim-permanode-content", put, err) != nil {
return err
}
if c.title != "" {
put, err := up.UploadAndSignBlob(schema.NewSetAttributeClaim(permaNode.BlobRef, "title", c.title))
handleResult("claim-permanode-title", put, err)
}
if c.tag != "" {
tags := strings.Split(c.tag, ",")
for _, tag := range tags {
m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
put, err := up.UploadAndSignBlob(m)
handleResult("claim-permanode-tag", put, err)
}
}
handleResult("permanode", permaNode, nil)
}
return nil
}
func (id *IndexDeps) SetAttribute_NoTimeMove(permaNode blob.Ref, attr, value string) blob.Ref {
m := schema.NewSetAttributeClaim(permaNode, attr, value)
m.SetClaimDate(id.LastTime())
return id.uploadAndSign(m)
}
// vivify verifies that all the chunks for the file described by fileblob are on the blobserver.
// It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim
// on that permanode for fileblob, signs it, and uploads it to the blobserver.
func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blobref.SizedBlobRef) error {
sf, ok := blobReceiver.(blobref.StreamingFetcher)
if !ok {
return fmt.Errorf("BlobReceiver is not a StreamingFetcher")
}
fetcher := blobref.SeekerFromStreamingFetcher(sf)
fr, err := schema.NewFileReader(fetcher, fileblob.BlobRef)
if err != nil {
return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.BlobRef.String(), err)
}
defer fr.Close()
h := sha1.New()
n, err := io.Copy(h, fr)
if err != nil {
return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.BlobRef.String(), err)
}
if n != fr.Size() {
return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.BlobRef.String(), fr.Size(), n)
}
config := blobReceiver.Config()
if config == nil {
return errors.New("blobReceiver has no config")
}
hf := config.HandlerFinder
if hf == nil {
return errors.New("blobReceiver config has no HandlerFinder")
}
JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign")
// TODO(mpl): second check should not be necessary, and yet it happens. Figure it out.
if err != nil || sh == nil {
return errors.New("jsonsign handler not found")
}
sigHelper, ok := sh.(*signhandler.Handler)
if !ok {
return errors.New("handler is not a JSON signhandler")
}
discoMap := sigHelper.DiscoveryMap(JSONSignRoot)
publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string)
if !ok {
return fmt.Errorf("Discovery: json decoding error: %v", err)
}
unsigned := schema.NewHashPlannedPermanode(h)
unsigned["camliSigner"] = publicKeyBlobRef
signed, err := sigHelper.SignMap(unsigned)
if err != nil {
return fmt.Errorf("Signing permanode %v: %v", signed, err)
}
signedPerm := blobref.SHA1FromString(signed)
_, err = blobReceiver.ReceiveBlob(signedPerm, strings.NewReader(signed))
if err != nil {
return fmt.Errorf("While uploading signed permanode %v: %v", signed, err)
}
contentAttr := schema.NewSetAttributeClaim(signedPerm, "camliContent", fileblob.BlobRef.String())
claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime)
contentAttr.SetClaimDate(claimDate)
contentAttr["camliSigner"] = publicKeyBlobRef
signed, err = sigHelper.SignMap(contentAttr)
if err != nil {
return fmt.Errorf("Signing camliContent claim: %v", err)
}
signedClaim := blobref.SHA1FromString(signed)
_, err = blobReceiver.ReceiveBlob(signedClaim, strings.NewReader(signed))
if err != nil {
return fmt.Errorf("While uploading signed camliContent claim %v: %v", signed, err)
}
return nil
}