Example #1
0
// fromHTTP panics with an httputil value on failure
func (r *WithAttrRequest) fromHTTP(req *http.Request) {
	r.Signer = blob.ParseOrZero(req.FormValue("signer"))
	r.Value = req.FormValue("value")
	fuzzy := req.FormValue("fuzzy") // exact match if empty
	fuzzyMatch := false
	if fuzzy != "" {
		lowered := strings.ToLower(fuzzy)
		if lowered == "true" || lowered == "t" {
			fuzzyMatch = true
		}
	}
	r.Attr = req.FormValue("attr") // all attributes if empty
	if r.Attr == "" {              // and force fuzzy in that case.
		fuzzyMatch = true
	}
	r.Fuzzy = fuzzyMatch
	r.ThumbnailSize = thumbnailSize(req)
	max := req.FormValue("max")
	if max != "" {
		maxR, err := strconv.Atoi(max)
		if err != nil {
			panic(httputil.InvalidParameterError("max"))
		}
		r.N = maxR
	}
	r.N = r.n()
}
Example #2
0
func (sto *appengineStorage) EnumerateBlobs(dest chan<- blob.SizedRef, after string, limit int, wait time.Duration) error {
	defer close(dest)

	ctx := sto.ctx
	if ctx == nil {
		loan := ctxPool.Get()
		defer loan.Return()
		ctx = loan
	}

	prefix := sto.namespace + "|"
	keyBegin := datastore.NewKey(ctx, memKind, prefix+after, 0, nil)
	keyEnd := datastore.NewKey(ctx, memKind, sto.namespace+"~", 0, nil)

	q := datastore.NewQuery(memKind).Limit(int(limit)).Filter("__key__>", keyBegin).Filter("__key__<", keyEnd)
	it := q.Run(ctx)
	var row memEnt
	for {
		key, err := it.Next(&row)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return err
		}
		dest <- blob.SizedRef{blob.ParseOrZero(key.StringID()[len(prefix):]), row.Size}
	}
	return nil
}
Example #3
0
func (sto *appengineStorage) EnumerateBlobs(ctx context.Context, dest chan<- blob.SizedRef, after string, limit int) error {
	defer close(dest)

	loan := ctxPool.Get()
	defer loan.Return()
	actx := loan

	prefix := sto.namespace + "|"
	keyBegin := datastore.NewKey(actx, memKind, prefix+after, 0, nil)
	keyEnd := datastore.NewKey(actx, memKind, sto.namespace+"~", 0, nil)

	q := datastore.NewQuery(memKind).Limit(int(limit)).Filter("__key__>", keyBegin).Filter("__key__<", keyEnd)
	it := q.Run(actx)
	var row memEnt
	for {
		key, err := it.Next(&row)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return err
		}
		select {
		case dest <- blob.SizedRef{blob.ParseOrZero(key.StringID()[len(prefix):]), uint32(row.Size)}:
		case <-ctx.Done():
			return ctx.Err()
		}
	}
	return nil
}
Example #4
0
func blobFromURLPath(path string) blob.Ref {
	matches := kGetPattern.FindStringSubmatch(path)
	if len(matches) != 3 {
		return blob.Ref{}
	}
	return blob.ParseOrZero(strings.TrimPrefix(matches[0], "/camli/"))
}
Example #5
0
func NewFlatStatCache(gen string) *FlatStatCache {
	filename := filepath.Join(osutil.CacheDir(), "camput.statcache."+escapeGen(gen))
	fc := &FlatStatCache{
		filename: filename,
		m:        make(map[string]fileInfoPutRes),
	}

	f, err := os.Open(filename)
	if os.IsNotExist(err) {
		return fc
	}
	if err != nil {
		log.Fatalf("opening camput stat cache: %v", filename, err)
	}
	defer f.Close()
	br := bufio.NewReader(f)
	for {
		ln, err := br.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Printf("Warning: (ignoring) reading stat cache: %v", err)
			break
		}
		ln = strings.TrimSpace(ln)
		f := strings.Split(ln, "\t")
		if len(f) < 3 {
			continue
		}
		filename, fp, putres := f[0], statFingerprint(f[1]), f[2]
		f = strings.Split(putres, "/")
		if len(f) != 2 {
			continue
		}
		blobrefStr := f[0]
		blobSize, err := strconv.ParseInt(f[1], 10, 64)
		if err != nil {
			continue
		}

		fc.m[filename] = fileInfoPutRes{
			Fingerprint: fp,
			Result: client.PutResult{
				BlobRef: blob.ParseOrZero(blobrefStr),
				Size:    blobSize,
				Skipped: true, // is this used?
			},
		}
	}
	vlog.Printf("Flatcache read %d entries from %s", len(fc.m), filename)
	return fc
}
Example #6
0
// processEncryptedMetaBlob decrypts dat (the data for the br meta blob) and parses
// its meta lines, updating the index.
//
// processEncryptedMetaBlob is not thread-safe.
func (s *storage) processEncryptedMetaBlob(br blob.Ref, dat []byte) error {
	mi := &metaBlobInfo{
		br: br,
		n:  len(dat),
	}
	log.Printf("processing meta blob %v: %d bytes", br, len(dat))
	ivSize := s.block.BlockSize()
	if len(dat) < ivSize+sha1.Size {
		return errors.New("data size is smaller than IV + SHA-1")
	}
	var (
		iv       = dat[:ivSize]
		wantHash = dat[ivSize : ivSize+sha1.Size]
		enc      = dat[ivSize+sha1.Size:]
	)
	plain := bytes.NewBuffer(make([]byte, 0, len(dat)))
	io.Copy(plain, cipher.StreamReader{
		S: cipher.NewCTR(s.block, iv),
		R: bytes.NewReader(enc),
	})
	s1 := sha1.New()
	s1.Write(plain.Bytes())
	if !bytes.Equal(wantHash, s1.Sum(nil)) {
		return errors.New("hash of encrypted data doesn't match")
	}
	sc := bufio.NewScanner(plain)
	if !sc.Scan() {
		return errors.New("No first line")
	}
	if sc.Text() != "#camlistore/encmeta=1" {
		line := sc.Text()
		if len(line) > 80 {
			line = line[:80]
		}
		return fmt.Errorf("unsupported first line %q", line)
	}
	for sc.Scan() {
		line := sc.Text()
		slash := strings.Index(line, "/")
		if slash < 0 {
			return errors.New("no slash in metaline")
		}
		plainBR, meta := line[:slash], line[slash+1:]
		log.Printf("Adding meta: %q = %q", plainBR, meta)
		mi.plains = append(mi.plains, blob.ParseOrZero(plainBR))
		if err := s.index.Set(plainBR, meta); err != nil {
			return err
		}
	}
	return sc.Err()
}
Example #7
0
func PathsOfSignerTarget(t *testing.T, initIdx func() *index.Index) {
	id := NewIndexDeps(initIdx())
	id.Fataler = t
	pn := id.NewPermanode()
	t.Logf("uploaded permanode %q", pn)

	claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123")
	claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124")
	t.Logf("made path claims %q and %q", claim1, claim2)

	id.dumpIndex(t)

	type test struct {
		blobref string
		want    int
	}
	tests := []test{
		{"targ-123", 1},
		{"targ-124", 1},
		{"targ-125", 0},
	}
	for _, tt := range tests {
		signer := id.SignerBlobRef
		paths, err := id.Index.PathsOfSignerTarget(signer, blob.ParseOrZero(tt.blobref))
		if err != nil {
			t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
		if tt.blobref == "targ-123" {
			p := paths[0]
			want := fmt.Sprintf(
				"Path{Claim: %s, 2011-11-28T01:32:37.000123456Z; Base: %s + Suffix \"somedir\" => Target targ-123}",
				claim1, pn)
			if g := p.String(); g != want {
				t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
			}
		}
	}

	path, err := id.Index.PathLookup(id.SignerBlobRef, pn, "with|pipe", time.Now())
	if err != nil {
		t.Fatalf("PathLookup = %v", err)
	}
	if g, e := path.Target.String(), "targ-124"; g != e {
		t.Errorf("PathLookup = %q; want %q", g, e)
	}
}
Example #8
0
// parent returns the base path and the blobRef of pr.subject's parent.
// It returns an error if pr.subject or pr.subjectBasePath were not set
// properly (with findSubject), or if the parent was not found.
func (pr *publishRequest) parent() (parentPath string, parentBlobRef blob.Ref, err error) {
	if !pr.subject.Valid() {
		return "", blob.Ref{}, errors.New("subject not set")
	}
	if pr.subjectBasePath == "" {
		return "", blob.Ref{}, errors.New("subjectBasePath not set")
	}
	// TODO(mpl): this fails when the parent is the root. fix it.
	hops := publishedPath(pr.subjectBasePath).splitHops()
	if len(hops) == 0 {
		return "", blob.Ref{}, errors.New("No subresource digest in subjectBasePath")
	}
	subjectDigest := hops[len(hops)-1]
	if subjectDigest != pr.subject.DigestPrefix(digestLen) {
		return "", blob.Ref{}, errors.New("subject digest not in subjectBasePath")
	}
	parentPath = strings.TrimSuffix(pr.subjectBasePath, "/"+digestPrefix+subjectDigest)

	if len(hops) == 1 {
		// the parent is the suffix, not one of the subresource hops
		for br, _ := range pr.inSubjectChain {
			if br != pr.subject.String() {
				parentBlobRef = blob.ParseOrZero(br)
				break
			}
		}
	} else {
		// nested collection(s)
		parentDigest := hops[len(hops)-2]
		for br, _ := range pr.inSubjectChain {
			bref, ok := blob.Parse(br)
			if !ok {
				return "", blob.Ref{}, fmt.Errorf("Could not parse %q as blobRef", br)
			}
			if bref.DigestPrefix(10) == parentDigest {
				parentBlobRef = bref
				break
			}
		}
	}
	if !parentBlobRef.Valid() {
		return "", blob.Ref{}, fmt.Errorf("No parent found for %v", pr.subjectBasePath)
	}
	return parentPath, parentBlobRef, nil
}
Example #9
0
// NewFromShareRoot uses shareBlobURL to set up and return a client that
// will be used to fetch shared blobs.
func NewFromShareRoot(shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
	var root string
	m := shareURLRx.FindStringSubmatch(shareBlobURL)
	if m == nil {
		return nil, blob.Ref{}, fmt.Errorf("Unkown share URL base")
	}
	c = New(m[1])
	c.discoOnce.Do(func() { /* nothing */
	})
	c.prefixOnce.Do(func() { /* nothing */
	})
	c.prefixv = m[1]
	c.isSharePrefix = true
	c.authMode = auth.None{}
	c.via = make(map[string]string)
	root = m[2]

	for _, v := range opts {
		v.modifyClient(c)
	}
	c.SetHTTPClient(&http.Client{Transport: c.TransportForConfig(nil)})

	req := c.newRequest("GET", shareBlobURL, nil)
	res, err := c.doReqGated(req)
	if err != nil {
		return nil, blob.Ref{}, fmt.Errorf("Error fetching %s: %v", shareBlobURL, err)
	}
	defer res.Body.Close()
	b, err := schema.BlobFromReader(blob.ParseOrZero(root), res.Body)
	if err != nil {
		return nil, blob.Ref{}, fmt.Errorf("Error parsing JSON from %s: %v", shareBlobURL, err)
	}
	if b.ShareAuthType() != schema.ShareHaveRef {
		return nil, blob.Ref{}, fmt.Errorf("Unknown share authType of %q", b.ShareAuthType())
	}
	target = b.ShareTarget()
	if !target.Valid() {
		return nil, blob.Ref{}, fmt.Errorf("No target.")
	}
	c.via[target.String()] = root
	return c, target, nil
}
Example #10
0
func (c *Corpus) pnCamliContent(pn blob.Ref) (cc blob.Ref, t time.Time, ok bool) {
	// TODO(bradfitz): keep this property cached
	pm, ok := c.permanodes[pn]
	if !ok {
		return
	}
	for _, cl := range pm.Claims {
		if cl.Attr != "camliContent" {
			continue
		}
		// TODO: pass down the 'PermanodeConstraint.At' parameter, and then do: if cl.Date.After(at) { continue }
		switch cl.Type {
		case string(schema.DelAttributeClaim):
			cc = blob.Ref{}
			t = time.Time{}
		case string(schema.SetAttributeClaim):
			cc = blob.ParseOrZero(cl.Value)
			t = cl.Date
		}
	}
	return cc, t, cc.Valid()

}
Example #11
0
// NewFromShareRoot uses shareBlobURL to set up and return a client that
// will be used to fetch shared blobs.
func NewFromShareRoot(shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
	var root string
	m := shareURLRx.FindStringSubmatch(shareBlobURL)
	if m == nil {
		return nil, blob.Ref{}, fmt.Errorf("Unkown share URL base")
	}
	c = New(m[1], opts...)
	c.discoOnce.Do(noop)
	c.prefixOnce.Do(noop)
	c.prefixv = m[1]
	c.isSharePrefix = true
	c.authMode = auth.None{}
	c.via = make(map[string]string)
	root = m[2]

	req := c.newRequest("GET", shareBlobURL, nil)
	res, err := c.expect2XX(req)
	if err != nil {
		return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %v", shareBlobURL, err)
	}
	defer res.Body.Close()
	var buf bytes.Buffer
	b, err := schema.BlobFromReader(blob.ParseOrZero(root), io.TeeReader(res.Body, &buf))
	if err != nil {
		return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %v , with response: %q", shareBlobURL, err, buf.Bytes())
	}
	if b.ShareAuthType() != schema.ShareHaveRef {
		return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
	}
	target = b.ShareTarget()
	if !target.Valid() {
		return nil, blob.Ref{}, fmt.Errorf("no target.")
	}
	c.via[target.String()] = root
	return c, target, nil
}
Example #12
0
func Index(t *testing.T, initIdx func() *index.Index) {
	id := NewIndexDeps(initIdx())
	id.Fataler = t
	pn := id.NewPermanode()
	t.Logf("uploaded permanode %q", pn)
	br1 := id.SetAttribute(pn, "tag", "foo1")
	br1Time := id.lastTime()
	t.Logf("set attribute %q", br1)
	br2 := id.SetAttribute(pn, "tag", "foo2")
	br2Time := id.lastTime()
	t.Logf("set attribute %q", br2)
	rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
	rootClaimTime := id.lastTime()
	t.Logf("set attribute %q", rootClaim)

	pnChild := id.NewPermanode()
	br3 := id.SetAttribute(pnChild, "tag", "bar")
	br3Time := id.lastTime()
	t.Logf("set attribute %q", br3)
	memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
	t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
	memberRefTime := id.lastTime()

	// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
	if false {
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		for i := 1; i <= 8; i++ {
			fileBase := fmt.Sprintf("f%d-exif.jpg", i)
			fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			id.UploadFile(fileBase, string(contents), noTime)
		}
	}

	// Upload a basic image.
	var jpegFileRef blob.Ref
	var exifFileRef blob.Ref
	{
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		uploadFile := func(file string, modTime time.Time) blob.Ref {
			fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			br, _ := id.UploadFile(file, string(contents), modTime)
			return br
		}
		jpegFileRef = uploadFile("dude.jpg", noTime)
		exifFileRef = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0))
	}

	// Upload the dir containing the two previous images
	imagesDirRef := id.UploadDir(
		"testdata",
		[]blob.Ref{jpegFileRef, exifFileRef},
		time.Now(),
	)

	lastPermanodeMutation := id.lastTime()
	id.dumpIndex(t)

	key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
	if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
		t.Fatalf("%q = %q, want %q", key, g, e)
	}

	key = "imagesize|" + jpegFileRef.String()
	if g, e := id.Get(key), "50|100"; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}
	key = "filetimes|" + jpegFileRef.String()
	if g, e := id.Get(key), ""; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}

	key = "filetimes|" + exifFileRef.String()
	if g, e := id.Get(key), "2013-02-18T01%3A11%3A20Z%2C2013-02-19T04%3A39%3A56Z"; g != e {
		t.Errorf("EXIF dude-exif.jpg key %q = %q; want %q", key, g, e)
	}

	key = "have:" + pn.String()
	pnSizeStr := id.Get(key)
	if pnSizeStr == "" {
		t.Fatalf("missing key %q", key)
	}

	key = "meta:" + pn.String()
	if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
		t.Errorf("key %q = %q, want %q", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
	if g, e := id.Get(key), "permanode|"; g != e {
		t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
	}

	// PermanodeOfSignerAttrValue
	{
		gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
		if err != nil {
			t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
		}
		if gotPN.String() != pn.String() {
			t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
		}
		_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
		if err == nil {
			t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
		}
	}

	// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
	{
		ch := make(chan blob.Ref, 10)
		req := &search.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag",
			Query:     "foo1"}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{pn}
		if len(got) < 1 || got[0].String() != want[0].String() {
			t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
		}
	}

	// SearchPermanodesWithAttr - match all with attr type "tag"
	{
		ch := make(chan blob.Ref, 10)
		req := &search.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag"}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{pn, pnChild}
		if len(got) != len(want) {
			t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if g.String() == w.String() {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
			}
		}
	}

	// GetRecentPermanodes
	{
		ch := make(chan *search.Result, 10) // expect 2 results, but maybe more if buggy.
		err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50)
		if err != nil {
			t.Fatalf("GetRecentPermanodes = %v", err)
		}
		got := []*search.Result{}
		for r := range ch {
			got = append(got, r)
		}
		want := []*search.Result{
			&search.Result{
				BlobRef:     pn,
				Signer:      id.SignerBlobRef,
				LastModTime: lastPermanodeMutation.Unix(),
			},
			&search.Result{
				BlobRef:     pnChild,
				Signer:      id.SignerBlobRef,
				LastModTime: br3Time.Unix(),
			},
		}
		if len(got) != len(want) {
			t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v",
				search.Results(got), search.Results(want))
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if reflect.DeepEqual(g, w) {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
					w, search.Results(got), search.Results(want))
			}
		}
	}

	// GetDirMembers
	{
		ch := make(chan blob.Ref, 10) // expect 2 results
		err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
		if err != nil {
			t.Fatalf("GetDirMembers = %v", err)
		}
		got := []blob.Ref{}
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{jpegFileRef, exifFileRef}
		if len(got) != len(want) {
			t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if w.String() == g.String() {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("GetDirMembers: %v was not found.", w)
			}
		}
	}

	// GetBlobMIMEType
	{
		mime, size, err := id.Index.GetBlobMIMEType(pn)
		if err != nil {
			t.Errorf("GetBlobMIMEType(%q) = %v", pn, err)
		} else {
			if e := "application/json; camliType=permanode"; mime != e {
				t.Errorf("GetBlobMIMEType(%q) mime = %q, want %q", pn, mime, e)
			}
			if size == 0 {
				t.Errorf("GetBlobMIMEType(%q) size is zero", pn)
			}
		}
		_, _, err = id.Index.GetBlobMIMEType(blob.ParseOrZero("abc-123"))
		if err != os.ErrNotExist {
			t.Errorf("GetBlobMIMEType(dummy blobref) = %v; want os.ErrNotExist", err)
		}
	}

	// GetOwnerClaims
	{
		claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef)
		if err != nil {
			t.Errorf("GetOwnerClaims = %v", err)
		} else {
			want := search.ClaimList([]*search.Claim{
				&search.Claim{
					BlobRef:   br1,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br1Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo1",
				},
				&search.Claim{
					BlobRef:   br2,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br2Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo2",
				},
				&search.Claim{
					BlobRef:   rootClaim,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      rootClaimTime.UTC(),
					Type:      "set-attribute",
					Attr:      "camliRoot",
					Value:     "rootval",
				},
				&search.Claim{
					BlobRef:   memberRef,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      memberRefTime.UTC(),
					Type:      "add-attribute",
					Attr:      "camliMember",
					Value:     pnChild.String(),
				},
			})
			if !reflect.DeepEqual(claims, want) {
				t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v",
					claims, want)
			}
		}
	}
}
Example #13
0
// populate hits the blobstore to populate map of child nodes.
func (n *mutDir) populate() error {
	n.mu.Lock()
	defer n.mu.Unlock()

	// Only re-populate if we haven't done so recently.
	now := time.Now()
	if n.lastPop.Add(populateInterval).After(now) {
		return nil
	}
	n.lastPop = now

	res, err := n.fs.client.Describe(&search.DescribeRequest{
		BlobRef: n.permanode,
		Depth:   3,
	})
	if err != nil {
		log.Println("mutDir.paths:", err)
		return nil
	}
	db := res.Meta[n.permanode.String()]
	if db == nil {
		return errors.New("dir blobref not described")
	}

	// Find all child permanodes and stick them in n.children
	if n.children == nil {
		n.children = make(map[string]mutFileOrDir)
	}
	currentChildren := map[string]bool{}
	for k, v := range db.Permanode.Attr {
		const p = "camliPath:"
		if !strings.HasPrefix(k, p) || len(v) < 1 {
			continue
		}
		name := k[len(p):]
		childRef := v[0]
		child := res.Meta[childRef]
		if child == nil {
			log.Printf("child not described: %v", childRef)
			continue
		}
		if child.Permanode == nil {
			log.Printf("invalid child, not a permanode: %v", childRef)
			continue
		}
		if target := child.Permanode.Attr.Get("camliSymlinkTarget"); target != "" {
			// This is a symlink.
			n.maybeAddChild(name, child.Permanode, &mutFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				symLink:   true,
				target:    target,
			})
		} else if isDir(child.Permanode) {
			// This is a directory.
			n.maybeAddChild(name, child.Permanode, &mutDir{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
			})
		} else if contentRef := child.Permanode.Attr.Get("camliContent"); contentRef != "" {
			// This is a file.
			content := res.Meta[contentRef]
			if content == nil {
				log.Printf("child content not described: %v", childRef)
				continue
			}
			if content.CamliType != "file" {
				log.Printf("child not a file: %v", childRef)
				continue
			}
			if content.File == nil {
				log.Printf("camlitype \"file\" child %v has no described File member", childRef)
				continue
			}
			n.maybeAddChild(name, child.Permanode, &mutFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				content:   blob.ParseOrZero(contentRef),
				size:      content.File.Size,
			})
		} else {
			// unhandled type...
			continue
		}
		currentChildren[name] = true
	}
	// Remove unreferenced children
	for name, oldchild := range n.children {
		if _, ok := currentChildren[name]; !ok {
			if oldchild.eligibleToDelete() {
				delete(n.children, name)
			}
		}
	}
	return nil
}
Example #14
0
func Index(t *testing.T, initIdx func() *index.Index) {
	oldLocal := time.Local
	time.Local = time.UTC
	defer func() { time.Local = oldLocal }()

	id := NewIndexDeps(initIdx())
	id.Fataler = t
	defer id.DumpIndex(t)
	pn := id.NewPermanode()
	t.Logf("uploaded permanode %q", pn)
	br1 := id.SetAttribute(pn, "tag", "foo1")
	br1Time := id.LastTime()
	t.Logf("set attribute %q", br1)
	br2 := id.SetAttribute(pn, "tag", "foo2")
	br2Time := id.LastTime()
	t.Logf("set attribute %q", br2)
	rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
	rootClaimTime := id.LastTime()
	t.Logf("set attribute %q", rootClaim)

	pnChild := id.NewPermanode()
	br3 := id.SetAttribute(pnChild, "tag", "bar")
	br3Time := id.LastTime()
	t.Logf("set attribute %q", br3)
	memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
	t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
	memberRefTime := id.LastTime()

	// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
	if false {
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		for i := 1; i <= 8; i++ {
			fileBase := fmt.Sprintf("f%d-exif.jpg", i)
			fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			id.UploadFile(fileBase, string(contents), noTime)
		}
	}

	// Upload some files.
	var jpegFileRef, exifFileRef, mediaFileRef, mediaWholeRef blob.Ref
	{
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			t.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		uploadFile := func(file string, modTime time.Time) (fileRef, wholeRef blob.Ref) {
			fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				t.Fatal(err)
			}
			fileRef, wholeRef = id.UploadFile(file, string(contents), modTime)
			return
		}
		jpegFileRef, _ = uploadFile("dude.jpg", noTime)
		exifFileRef, _ = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0))
		mediaFileRef, mediaWholeRef = uploadFile("0s.mp3", noTime)
	}

	// Upload the dir containing the previous files.
	imagesDirRef := id.UploadDir(
		"testdata",
		[]blob.Ref{jpegFileRef, exifFileRef, mediaFileRef},
		time.Now(),
	)

	lastPermanodeMutation := id.LastTime()

	key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
	if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
		t.Fatalf("%q = %q, want %q", key, g, e)
	}

	key = "imagesize|" + jpegFileRef.String()
	if g, e := id.Get(key), "50|100"; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}
	key = "filetimes|" + jpegFileRef.String()
	if g, e := id.Get(key), ""; g != e {
		t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
	}

	key = "filetimes|" + exifFileRef.String()
	if g, e := id.Get(key), "2013-02-18T01%3A11%3A20Z%2C2013-02-19T04%3A39%3A56Z"; g != e {
		t.Errorf("EXIF dude-exif.jpg key %q = %q; want %q", key, g, e)
	}

	key = "have:" + pn.String()
	pnSizeStr := strings.TrimSuffix(id.Get(key), "|indexed")
	if pnSizeStr == "" {
		t.Fatalf("missing key %q", key)
	}

	key = "meta:" + pn.String()
	if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
		t.Errorf("key %q = %q, want %q", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
	if g, e := id.Get(key), pn.String(); g != e {
		t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
	}

	key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
	if g, e := id.Get(key), "permanode|"; g != e {
		t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
	}

	mediaTests := []struct {
		prop, exp string
	}{
		{"title", "Zero Seconds"},
		{"artist", "Test Artist"},
		{"album", "Test Album"},
		{"genre", "(20)Alternative"},
		{"musicbrainzalbumid", "00000000-0000-0000-0000-000000000000"},
		{"year", "1992"},
		{"track", "1"},
		{"disc", "2"},
		{"mediaref", "sha1-fefac74a1d5928316d7131747107c8a61b71ffe4"},
		{"durationms", "26"},
	}
	for _, tt := range mediaTests {
		key = fmt.Sprintf("mediatag|%s|%s", mediaWholeRef.String(), tt.prop)
		if g, _ := url.QueryUnescape(id.Get(key)); g != tt.exp {
			t.Errorf("0s.mp3 key %q = %q; want %q", key, g, tt.exp)
		}
	}

	// PermanodeOfSignerAttrValue
	{
		gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
		if err != nil {
			t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
		}
		if gotPN.String() != pn.String() {
			t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
		}
		_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
		if err == nil {
			t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
		}
	}

	// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
	{
		ch := make(chan blob.Ref, 10)
		req := &camtypes.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag",
			Query:     "foo1",
		}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{pn}
		if len(got) < 1 || got[0].String() != want[0].String() {
			t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
		}
	}

	// SearchPermanodesWithAttr - match all with attr type "tag"
	{
		ch := make(chan blob.Ref, 10)
		req := &camtypes.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "tag",
		}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{pn, pnChild}
		if len(got) != len(want) {
			t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if g.String() == w.String() {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
			}
		}
	}

	// Delete value "pony" of type "title" (which does not actually exist) for pn
	br4 := id.DelAttribute(pn, "title", "pony")
	br4Time := id.LastTime()
	// and verify it is not found when searching by attr
	{
		ch := make(chan blob.Ref, 10)
		req := &camtypes.PermanodeByAttrRequest{
			Signer:    id.SignerBlobRef,
			Attribute: "title",
			Query:     "pony",
		}
		err := id.Index.SearchPermanodesWithAttr(ch, req)
		if err != nil {
			t.Fatalf("SearchPermanodesWithAttr = %v", err)
		}
		var got []blob.Ref
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{}
		if len(got) != len(want) {
			t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
				got, want)
		}
	}

	// GetRecentPermanodes
	{
		verify := func(prefix string, want []camtypes.RecentPermanode, before time.Time) {
			ch := make(chan camtypes.RecentPermanode, 10) // expect 2 results, but maybe more if buggy.
			err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50, before)
			if err != nil {
				t.Fatalf("[%s] GetRecentPermanodes = %v", prefix, err)
			}
			got := []camtypes.RecentPermanode{}
			for r := range ch {
				got = append(got, r)
			}
			if len(got) != len(want) {
				t.Errorf("[%s] GetRecentPermanode results differ.\n got: %v\nwant: %v",
					prefix, searchResults(got), searchResults(want))
			}
			for _, w := range want {
				found := false
				for _, g := range got {
					if g.Equal(w) {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("[%s] GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
						prefix, w, searchResults(got), searchResults(want))
				}
			}
		}

		want := []camtypes.RecentPermanode{
			{
				Permanode:   pn,
				Signer:      id.SignerBlobRef,
				LastModTime: br4Time,
			},
			{
				Permanode:   pnChild,
				Signer:      id.SignerBlobRef,
				LastModTime: br3Time,
			},
		}

		before := time.Time{}
		verify("Zero before", want, before)

		before = lastPermanodeMutation
		t.Log("lastPermanodeMutation", lastPermanodeMutation,
			lastPermanodeMutation.Unix())
		verify("Non-zero before", want[1:], before)
	}
	// GetDirMembers
	{
		ch := make(chan blob.Ref, 10) // expect 2 results
		err := id.Index.GetDirMembers(imagesDirRef, ch, 50)
		if err != nil {
			t.Fatalf("GetDirMembers = %v", err)
		}
		got := []blob.Ref{}
		for r := range ch {
			got = append(got, r)
		}
		want := []blob.Ref{jpegFileRef, exifFileRef, mediaFileRef}
		if len(got) != len(want) {
			t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v",
				got, want)
		}
		for _, w := range want {
			found := false
			for _, g := range got {
				if w == g {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("GetDirMembers: %v was not found.", w)
			}
		}
	}

	// GetBlobMeta
	{
		meta, err := id.Index.GetBlobMeta(pn)
		if err != nil {
			t.Errorf("GetBlobMeta(%q) = %v", pn, err)
		} else {
			if e := "permanode"; meta.CamliType != e {
				t.Errorf("GetBlobMeta(%q) mime = %q, want %q", pn, meta.CamliType, e)
			}
			if meta.Size == 0 {
				t.Errorf("GetBlobMeta(%q) size is zero", pn)
			}
		}
		_, err = id.Index.GetBlobMeta(blob.ParseOrZero("abc-123"))
		if err != os.ErrNotExist {
			t.Errorf("GetBlobMeta(dummy blobref) = %v; want os.ErrNotExist", err)
		}
	}

	// AppendClaims
	{
		claims, err := id.Index.AppendClaims(nil, pn, id.SignerBlobRef, "")
		if err != nil {
			t.Errorf("AppendClaims = %v", err)
		} else {
			want := []camtypes.Claim{
				{
					BlobRef:   br1,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br1Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo1",
				},
				{
					BlobRef:   br2,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br2Time.UTC(),
					Type:      "set-attribute",
					Attr:      "tag",
					Value:     "foo2",
				},
				{
					BlobRef:   rootClaim,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      rootClaimTime.UTC(),
					Type:      "set-attribute",
					Attr:      "camliRoot",
					Value:     "rootval",
				},
				{
					BlobRef:   memberRef,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      memberRefTime.UTC(),
					Type:      "add-attribute",
					Attr:      "camliMember",
					Value:     pnChild.String(),
				},
				{
					BlobRef:   br4,
					Permanode: pn,
					Signer:    id.SignerBlobRef,
					Date:      br4Time.UTC(),
					Type:      "del-attribute",
					Attr:      "title",
					Value:     "pony",
				},
			}
			if !reflect.DeepEqual(claims, want) {
				t.Errorf("AppendClaims results differ.\n got: %v\nwant: %v",
					claims, want)
			}
		}
	}
}
Example #15
0
func PathsOfSignerTarget(t *testing.T, initIdx func() *index.Index) {
	id := NewIndexDeps(initIdx())
	id.Fataler = t
	defer id.DumpIndex(t)
	signer := id.SignerBlobRef
	pn := id.NewPermanode()
	t.Logf("uploaded permanode %q", pn)

	claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123")
	claim1Time := id.LastTime().UTC()
	claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124")
	claim2Time := id.LastTime().UTC()
	t.Logf("made path claims %q and %q", claim1, claim2)

	type test struct {
		blobref string
		want    int
	}
	tests := []test{
		{"targ-123", 1},
		{"targ-124", 1},
		{"targ-125", 0},
	}
	for _, tt := range tests {
		paths, err := id.Index.PathsOfSignerTarget(signer, blob.ParseOrZero(tt.blobref))
		if err != nil {
			t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
		if tt.blobref == "targ-123" {
			p := paths[0]
			want := fmt.Sprintf(
				"Path{Claim: %s, %v; Base: %s + Suffix \"somedir\" => Target targ-123}",
				claim1, claim1Time, pn)
			if g := p.String(); g != want {
				t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
			}
		}
	}
	tests = []test{
		{"somedir", 1},
		{"with|pipe", 1},
		{"void", 0},
	}
	for _, tt := range tests {
		paths, err := id.Index.PathsLookup(id.SignerBlobRef, pn, tt.blobref)
		if err != nil {
			t.Fatalf("PathsLookup(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsLookup(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
		if tt.blobref == "with|pipe" {
			p := paths[0]
			want := fmt.Sprintf(
				"Path{Claim: %s, %s; Base: %s + Suffix \"with|pipe\" => Target targ-124}",
				claim2, claim2Time, pn)
			if g := p.String(); g != want {
				t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
			}
		}
	}

	// now test deletions
	// Delete an existing value
	claim3 := id.Delete(claim2)
	t.Logf("claim %q deletes path claim %q", claim3, claim2)
	tests = []test{
		{"targ-123", 1},
		{"targ-124", 0},
		{"targ-125", 0},
	}
	for _, tt := range tests {
		signer := id.SignerBlobRef
		paths, err := id.Index.PathsOfSignerTarget(signer, blob.ParseOrZero(tt.blobref))
		if err != nil {
			t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
	}
	tests = []test{
		{"somedir", 1},
		{"with|pipe", 0},
		{"void", 0},
	}
	for _, tt := range tests {
		paths, err := id.Index.PathsLookup(id.SignerBlobRef, pn, tt.blobref)
		if err != nil {
			t.Fatalf("PathsLookup(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsLookup(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
	}

	// recreate second path, and test if the previous deletion of it
	// is indeed ignored.
	claim4 := id.Delete(claim3)
	t.Logf("delete claim %q deletes claim %q, which should undelete %q", claim4, claim3, claim2)
	tests = []test{
		{"targ-123", 1},
		{"targ-124", 1},
		{"targ-125", 0},
	}
	for _, tt := range tests {
		signer := id.SignerBlobRef
		paths, err := id.Index.PathsOfSignerTarget(signer, blob.ParseOrZero(tt.blobref))
		if err != nil {
			t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
		// and check the modtime too
		if tt.blobref == "targ-124" {
			p := paths[0]
			want := fmt.Sprintf(
				"Path{Claim: %s, %v; Base: %s + Suffix \"with|pipe\" => Target targ-124}",
				claim2, claim2Time, pn)
			if g := p.String(); g != want {
				t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
			}
		}
	}
	tests = []test{
		{"somedir", 1},
		{"with|pipe", 1},
		{"void", 0},
	}
	for _, tt := range tests {
		paths, err := id.Index.PathsLookup(id.SignerBlobRef, pn, tt.blobref)
		if err != nil {
			t.Fatalf("PathsLookup(%q): %v", tt.blobref, err)
		}
		if len(paths) != tt.want {
			t.Fatalf("PathsLookup(%q) got %d results; want %d",
				tt.blobref, len(paths), tt.want)
		}
		// and check that modtime is now claim4Time
		if tt.blobref == "with|pipe" {
			p := paths[0]
			want := fmt.Sprintf(
				"Path{Claim: %s, %s; Base: %s + Suffix \"with|pipe\" => Target targ-124}",
				claim2, claim2Time, pn)
			if g := p.String(); g != want {
				t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
			}
		}
	}
}
Example #16
0
// populate hits the blobstore to populate map of child nodes.
func (n *mutDir) populate() error {
	n.mu.Lock()
	defer n.mu.Unlock()

	// Only re-populate if we haven't done so recently.
	now := time.Now()
	if n.lastPop.Add(populateInterval).After(now) {
		return nil
	}
	n.lastPop = now

	res, err := n.fs.client.Describe(&search.DescribeRequest{
		BlobRef: n.permanode,
		Depth:   3,
	})
	if err != nil {
		log.Println("mutDir.paths:", err)
		return nil
	}
	db := res.Meta[n.permanode.String()]
	if db == nil {
		return errors.New("dir blobref not described")
	}

	// Find all child permanodes and stick them in n.children
	if n.children == nil {
		n.children = make(map[string]mutFileOrDir)
	}
	for k, v := range db.Permanode.Attr {
		const p = "camliPath:"
		if !strings.HasPrefix(k, p) || len(v) < 1 {
			continue
		}
		name := k[len(p):]
		childRef := v[0]
		child := res.Meta[childRef]
		if child == nil {
			log.Printf("child not described: %v", childRef)
			continue
		}
		if target := child.Permanode.Attr.Get("camliSymlinkTarget"); target != "" {
			// This is a symlink.
			n.children[name] = &mutFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				symLink:   true,
				target:    target,
			}
			continue
		}
		if contentRef := child.Permanode.Attr.Get("camliContent"); contentRef != "" {
			// This is a file.
			content := res.Meta[contentRef]
			if content == nil {
				log.Printf("child content not described: %v", childRef)
				continue
			}
			if content.CamliType != "file" {
				log.Printf("child not a file: %v", childRef)
				continue
			}
			n.children[name] = &mutFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				content:   blob.ParseOrZero(contentRef),
				size:      content.File.Size,
			}
			continue
		}
		// This is a directory.
		n.children[name] = &mutDir{
			fs:        n.fs,
			permanode: blob.ParseOrZero(childRef),
			parent:    n,
			name:      name,
		}
	}
	return nil
}
Example #17
0
// populate hits the blobstore to populate map of child nodes.
func (n *roDir) populate() error {
	n.mu.Lock()
	defer n.mu.Unlock()
	ctx := context.TODO()
	// Things never change here, so if we've ever populated, we're
	// populated.
	if n.children != nil {
		return nil
	}

	log.Printf("roDir.populate(%q) - Sending request At %v", n.fullPath(), n.at)

	res, err := n.fs.client.Describe(ctx, &search.DescribeRequest{
		BlobRef: n.permanode,
		Depth:   3,
		At:      types.Time3339(n.at),
	})
	if err != nil {
		log.Println("roDir.paths:", err)
		return nil
	}
	db := res.Meta[n.permanode.String()]
	if db == nil {
		return errors.New("dir blobref not described")
	}

	// Find all child permanodes and stick them in n.children
	n.children = make(map[string]roFileOrDir)
	for k, v := range db.Permanode.Attr {
		const p = "camliPath:"
		if !strings.HasPrefix(k, p) || len(v) < 1 {
			continue
		}
		name := k[len(p):]
		childRef := v[0]
		child := res.Meta[childRef]
		if child == nil {
			log.Printf("child not described: %v", childRef)
			continue
		}
		if target := child.Permanode.Attr.Get("camliSymlinkTarget"); target != "" {
			// This is a symlink.
			n.children[name] = &roFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				symLink:   true,
				target:    target,
			}
		} else if isDir(child.Permanode) {
			// This is a directory.
			n.children[name] = &roDir{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				at:        n.at,
			}
		} else if contentRef := child.Permanode.Attr.Get("camliContent"); contentRef != "" {
			// This is a file.
			content := res.Meta[contentRef]
			if content == nil {
				log.Printf("child content not described: %v", childRef)
				continue
			}
			if content.CamliType != "file" {
				log.Printf("child not a file: %v", childRef)
				continue
			}
			n.children[name] = &roFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				content:   blob.ParseOrZero(contentRef),
				size:      content.File.Size,
			}
		} else {
			// unknown type
			continue
		}
		n.children[name].xattr().load(child.Permanode)
	}
	return nil
}