Exemple #1
0
func TestQueryPermanodeModtime(t *testing.T) {
	testQuery(t, func(qt *queryTest) {
		id := qt.id

		// indextest advances time one second per operation:
		p1 := id.NewPlannedPermanode("1")
		p2 := id.NewPlannedPermanode("2")
		p3 := id.NewPlannedPermanode("3")
		id.SetAttribute(p1, "someAttr", "value1") // 2011-11-28 01:32:37.000123456 +0000 UTC 1322443957
		id.SetAttribute(p2, "someAttr", "value2") // 2011-11-28 01:32:38.000123456 +0000 UTC 1322443958
		id.SetAttribute(p3, "someAttr", "value3") // 2011-11-28 01:32:39.000123456 +0000 UTC 1322443959

		sq := &SearchQuery{
			Constraint: &Constraint{
				Permanode: &PermanodeConstraint{
					ModTime: &TimeConstraint{
						After:  types.Time3339(time.Unix(1322443957, 456789)),
						Before: types.Time3339(time.Unix(1322443959, 0)),
					},
				},
			},
		}
		qt.wantRes(sq, p2)
	})
}
Exemple #2
0
func (c *desCmd) RunCommand(args []string) error {
	if len(args) == 0 {
		return cmdmain.UsageError("requires blobref")
	}
	var blobs []blob.Ref
	for _, arg := range args {
		br, ok := blob.Parse(arg)
		if !ok {
			return cmdmain.UsageError(fmt.Sprintf("invalid blobref %q", arg))
		}
		blobs = append(blobs, br)
	}
	var at time.Time // TODO: implement. from "2 days ago" "-2d", "-2h", "2013-02-05", etc

	cl := c.client()
	res, err := cl.Describe(&search.DescribeRequest{
		BlobRefs: blobs,
		Depth:    c.depth,
		At:       types.Time3339(at),
	})
	if err != nil {
		return err
	}
	resj, err := json.MarshalIndent(res, "", "  ")
	if err != nil {
		return err
	}
	resj = append(resj, '\n')
	_, err = os.Stdout.Write(resj)
	return err
}
Exemple #3
0
// GetClaims returns the claims on req.Permanode signed by sh.owner.
func (sh *Handler) GetClaims(req *ClaimsRequest) (*ClaimsResponse, error) {
	// TODO: rename GetOwnerClaims to GetClaims?
	if req.Permanode == nil {
		return nil, errors.New("Error getting claims: nil permanode.")
	}
	claims, err := sh.index.GetOwnerClaims(req.Permanode, sh.owner)
	if err != nil {
		return nil, fmt.Errorf("Error getting claims of %s: %v", req.Permanode.String(), err)
	}
	sort.Sort(claims)
	var jclaims []*ClaimsItem
	for _, claim := range claims {
		jclaim := &ClaimsItem{
			BlobRef:   claim.BlobRef,
			Signer:    claim.Signer,
			Permanode: claim.Permanode,
			Date:      types.Time3339(claim.Date),
			Type:      claim.Type,
			Attr:      claim.Attr,
			Value:     claim.Value,
		}
		jclaims = append(jclaims, jclaim)
	}

	res := &ClaimsResponse{
		Claims: jclaims,
	}
	return res, nil
}
Exemple #4
0
// GetRecentPermanodes returns recently-modified permanodes.
func (sh *Handler) GetRecentPermanodes(req *RecentRequest) (*RecentResponse, error) {
	ch := make(chan *Result)
	errch := make(chan error)
	go func() {
		errch <- sh.index.GetRecentPermanodes(ch, sh.owner, req.n())
	}()

	dr := sh.NewDescribeRequest()

	var recent []*RecentItem
	for res := range ch {
		dr.Describe(res.BlobRef, 2)
		recent = append(recent, &RecentItem{
			BlobRef: res.BlobRef,
			Owner:   res.Signer,
			ModTime: types.Time3339(time.Unix(res.LastModTime, 0)),
		})
		testHookBug121() // http://camlistore.org/issue/121
	}

	if err := <-errch; err != nil {
		return nil, err
	}

	metaMap, err := dr.metaMapThumbs(req.thumbnailSize())
	if err != nil {
		return nil, err
	}

	res := &RecentResponse{
		Recent: recent,
		Meta:   metaMap,
	}
	return res, nil
}
Exemple #5
0
// GetClaims returns the claims on req.Permanode signed by sh.owner.
func (sh *Handler) GetClaims(req *ClaimsRequest) (*ClaimsResponse, error) {
	if !req.Permanode.Valid() {
		return nil, errors.New("Error getting claims: nil permanode.")
	}
	var claims []camtypes.Claim
	claims, err := sh.index.AppendClaims(claims, req.Permanode, sh.owner, req.AttrFilter)
	if err != nil {
		return nil, fmt.Errorf("Error getting claims of %s: %v", req.Permanode.String(), err)
	}
	sort.Sort(camtypes.ClaimsByDate(claims))
	var jclaims []*ClaimsItem
	for _, claim := range claims {
		jclaim := &ClaimsItem{
			BlobRef:   claim.BlobRef,
			Signer:    claim.Signer,
			Permanode: claim.Permanode,
			Date:      types.Time3339(claim.Date),
			Type:      claim.Type,
			Attr:      claim.Attr,
			Value:     claim.Value,
		}
		jclaims = append(jclaims, jclaim)
	}

	res := &ClaimsResponse{
		Claims: jclaims,
	}
	return res, nil
}
Exemple #6
0
func (r *RecentRequest) URLSuffix() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "camli/search/recent?n=%d&thumbnails=%d", r.n(), r.thumbnailSize())
	if !r.Before.IsZero() {
		fmt.Fprintf(&buf, "&before=%s", types.Time3339(r.Before))
	}
	return buf.String()
}
Exemple #7
0
func (b before) Predicate(ctx context.Context, args []string) (*Constraint, error) {
	t, err := parseTimePrefix(args[0])
	if err != nil {
		return nil, err
	}
	tc := &TimeConstraint{}
	tc.Before = types.Time3339(t)
	c := &Constraint{
		Permanode: &PermanodeConstraint{
			Time: tc,
		},
	}
	return c, nil
}
Exemple #8
0
// GetRecentPermanodes returns recently-modified permanodes.
func (sh *Handler) GetRecentPermanodes(req *RecentRequest) (*RecentResponse, error) {
	ch := make(chan camtypes.RecentPermanode)
	errch := make(chan error, 1)
	before := time.Now()
	if !req.Before.IsZero() {
		before = req.Before
	}
	go func() {
		errch <- sh.index.GetRecentPermanodes(ch, sh.owner, req.n(), before)
	}()

	dr := sh.NewDescribeRequest()

	var recent []*RecentItem
	for res := range ch {
		dr.Describe(res.Permanode, 2)
		recent = append(recent, &RecentItem{
			BlobRef: res.Permanode,
			Owner:   res.Signer,
			ModTime: types.Time3339(res.LastModTime),
		})
		testHookBug121() // http://camlistore.org/issue/121
	}

	if err := <-errch; err != nil {
		return nil, err
	}

	metaMap, err := dr.metaMap()
	if err != nil {
		return nil, err
	}

	res := &RecentResponse{
		Recent: recent,
		Meta:   metaMap,
	}
	return res, nil
}
Exemple #9
0
func (rh *RootHandler) serveDiscovery(rw http.ResponseWriter, req *http.Request) {
	d := &camtypes.Discovery{
		BlobRoot:     rh.BlobRoot,
		JSONSignRoot: rh.JSONSignRoot,
		HelpRoot:     rh.helpRoot,
		ImporterRoot: rh.importerRoot,
		SearchRoot:   rh.SearchRoot,
		StatusRoot:   rh.statusRoot,
		OwnerName:    rh.OwnerName,
		UserName:     rh.Username,
		WSAuthToken:  auth.ProcessRandom(),
		ThumbVersion: images.ThumbnailVersion(),
	}
	if gener, ok := rh.Storage.(blobserver.Generationer); ok {
		initTime, gen, err := gener.StorageGeneration()
		if err != nil {
			d.StorageGenerationError = err.Error()
		} else {
			d.StorageInitTime = types.Time3339(initTime)
			d.StorageGeneration = gen
		}
	} else {
		log.Printf("Storage type %T is not a blobserver.Generationer; not sending storageGeneration", rh.Storage)
	}
	if rh.ui != nil {
		d.UIDiscovery = rh.ui.discovery()
	}
	if rh.sigh != nil {
		d.Signing = rh.sigh.Discovery(rh.JSONSignRoot)
	}
	if len(rh.sync) > 0 {
		syncHandlers := make([]camtypes.SyncHandlerDiscovery, 0, len(rh.sync))
		for _, sh := range rh.sync {
			syncHandlers = append(syncHandlers, sh.discovery())
		}
		d.SyncHandlers = syncHandlers
	}
	discoveryHelper(rw, req, d)
}
Exemple #10
0
// b: the parsed file schema blob
// mm: keys to populate
func (ix *Index) populateFile(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) (err error) {
	var times []time.Time // all creation or mod times seen; may be zero
	times = append(times, b.ModTime())

	blobRef := b.BlobRef()
	fr, err := b.NewFileReader(fetcher)
	if err != nil {
		return err
	}
	defer fr.Close()
	mime, reader := magic.MIMETypeFromReader(fr)

	sha1 := sha1.New()
	var copyDest io.Writer = sha1
	var imageBuf *keepFirstN // or nil
	if strings.HasPrefix(mime, "image/") {
		// Emperically derived 1MiB assuming CR2 images require more than any
		// other filetype we support:
		//   https://gist.github.com/wathiede/7982372
		imageBuf = &keepFirstN{N: 1 << 20}
		copyDest = io.MultiWriter(copyDest, imageBuf)
	}
	size, err := io.Copy(copyDest, reader)
	if err != nil {
		return err
	}
	wholeRef := blob.RefFromHash(sha1)

	if imageBuf != nil {
		if conf, err := images.DecodeConfig(bytes.NewReader(imageBuf.Bytes)); err == nil {
			mm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height)))
		}
		if ft, err := schema.FileTime(bytes.NewReader(imageBuf.Bytes)); err == nil {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
			times = append(times, ft)
		} else {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
		}

		indexEXIF(wholeRef, imageBuf.Bytes, mm)
	}

	var sortTimes []time.Time
	for _, t := range times {
		if !t.IsZero() {
			sortTimes = append(sortTimes, t)
		}
	}
	sort.Sort(types.ByTime(sortTimes))
	var time3339s string
	switch {
	case len(sortTimes) == 1:
		time3339s = types.Time3339(sortTimes[0]).String()
	case len(sortTimes) >= 2:
		oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1]
		time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String()
	}

	mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime))
	mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))

	if strings.HasPrefix(mime, "audio/") {
		indexMusic(io.NewSectionReader(fr, 0, fr.Size()), wholeRef, mm)
	}

	return nil
}
Exemple #11
0
func parseCoreAtom(ctx *context.Context, word string) (*Constraint, error) {
	if m := tagExpr.FindStringSubmatch(word); m != nil {
		c := &Constraint{
			Permanode: &PermanodeConstraint{
				Attr:       "tag",
				SkipHidden: true,
				Value:      m[1],
			},
		}
		return c, nil
	}
	if m := titleExpr.FindStringSubmatch(word); m != nil {
		c := &Constraint{
			Permanode: &PermanodeConstraint{
				Attr:       "title",
				SkipHidden: true,
				ValueMatches: &StringConstraint{
					Contains:        m[1],
					CaseInsensitive: true,
				},
			},
		}
		return c, nil
	}
	if m := attrExpr.FindStringSubmatch(word); m != nil {
		c := &Constraint{
			Permanode: &PermanodeConstraint{
				Attr:       m[1],
				SkipHidden: true,
				Value:      m[2],
			},
		}
		return c, nil
	}
	if m := childrenOfExpr.FindStringSubmatch(word); m != nil {
		c := &Constraint{
			Permanode: &PermanodeConstraint{
				Relation: &RelationConstraint{
					Relation: "parent",
					Any: &Constraint{
						BlobRefPrefix: m[1],
					},
				},
			},
		}
		return c, nil
	}
	if strings.HasPrefix(word, "before:") || strings.HasPrefix(word, "after:") {
		before := false
		when := ""
		if strings.HasPrefix(word, "before:") {
			before = true
			when = strings.TrimPrefix(word, "before:")
		} else {
			when = strings.TrimPrefix(word, "after:")
		}
		base := "0000-01-01T00:00:00Z"
		if len(when) < len(base) {
			when += base[len(when):]
		}
		t, err := time.Parse(time.RFC3339, when)
		if err != nil {
			return nil, err
		}
		tc := &TimeConstraint{}
		if before {
			tc.Before = types.Time3339(t)
		} else {
			tc.After = types.Time3339(t)
		}
		c := &Constraint{
			Permanode: &PermanodeConstraint{
				Time: tc,
			},
		}
		return c, nil
	}
	if strings.HasPrefix(word, "format:") {
		c := permOfFile(&FileConstraint{
			MIMEType: &StringConstraint{
				Equals: mimeFromFormat(strings.TrimPrefix(word, "format:")),
			},
		})
		return c, nil
	}
	return nil, errors.New(fmt.Sprintf("Not an core-atom: %v", word))
}
Exemple #12
0
// blobref: of the file or schema blob
//      blob: the parsed file schema blob
//      bm: keys to populate
func (ix *Index) populateFile(b *schema.Blob, bm BatchMutation) error {
	var times []time.Time // all creation or mod times seen; may be zero
	times = append(times, b.ModTime())

	blobRef := b.BlobRef()
	seekFetcher := blob.SeekerFromStreamingFetcher(ix.BlobSource)
	fr, err := b.NewFileReader(seekFetcher)
	if err != nil {
		// TODO(bradfitz): propagate up a transient failure
		// error type, so we can retry indexing files in the
		// future if blobs are only temporarily unavailable.
		// Basically the same as the TODO just below.
		log.Printf("index: error indexing file, creating NewFileReader %s: %v", blobRef, err)
		return nil
	}
	defer fr.Close()
	mime, reader := magic.MIMETypeFromReader(fr)

	sha1 := sha1.New()
	var copyDest io.Writer = sha1
	var imageBuf *keepFirstN // or nil
	if strings.HasPrefix(mime, "image/") {
		imageBuf = &keepFirstN{N: 256 << 10}
		copyDest = io.MultiWriter(copyDest, imageBuf)
	}
	size, err := io.Copy(copyDest, reader)
	if err != nil {
		// TODO: job scheduling system to retry this spaced
		// out max n times.  Right now our options are
		// ignoring this error (forever) or returning the
		// error and making the indexing try again (likely
		// forever failing).  Both options suck.  For now just
		// log and act like all's okay.
		log.Printf("index: error indexing file %s: %v", blobRef, err)
		return nil
	}

	if imageBuf != nil {
		if conf, err := images.DecodeConfig(bytes.NewReader(imageBuf.Bytes)); err == nil {
			bm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height)))
		}
		if ft, err := schema.FileTime(bytes.NewReader(imageBuf.Bytes)); err == nil {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
			times = append(times, ft)
		} else {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
		}
	}

	var sortTimes []time.Time
	for _, t := range times {
		if !t.IsZero() {
			sortTimes = append(sortTimes, t)
		}
	}
	sort.Sort(types.ByTime(sortTimes))
	var time3339s string
	switch {
	case len(sortTimes) == 1:
		time3339s = types.Time3339(sortTimes[0]).String()
	case len(sortTimes) >= 2:
		oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1]
		time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String()
	}

	wholeRef := blob.RefFromHash(sha1)
	bm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
	bm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime))
	bm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))

	if strings.HasPrefix(mime, "audio/") {
		tag, err := taglib.Decode(fr, fr.Size())
		if err == nil {
			indexMusic(tag, wholeRef, bm)
		} else {
			log.Print("index: error parsing tag: ", err)
		}
	}

	return nil
}
Exemple #13
0
var keywordTests = []keywordTestcase{
	// Core predicates
	{
		object:      newAfter(),
		args:        []string{"faulty"},
		errContains: "faulty",
	},

	{
		object: newAfter(),
		args:   []string{"2013-02-03"},
		want: &Constraint{
			Permanode: &PermanodeConstraint{
				Time: &TimeConstraint{
					After: types.Time3339(testtime),
				},
			},
		},
	},

	{
		object:      newBefore(),
		args:        []string{"faulty"},
		errContains: "faulty",
	},

	{
		object: newBefore(),
		args:   []string{"2013-02-03"},
		want: &Constraint{
Exemple #14
0
// parseExpression parses a search expression (e.g. "tag:funny
// near:portland") and returns a SearchQuery for that search text. The
// Constraint field will always be set. The Limit and Sort may also be
// set.
func parseExpression(ctx *context.Context, exp string) (*SearchQuery, error) {
	base := &Constraint{
		Permanode: &PermanodeConstraint{
			SkipHidden: true,
		},
	}
	sq := &SearchQuery{
		Constraint: base,
	}

	exp = strings.TrimSpace(exp)
	if exp == "" {
		return sq, nil
	}

	andNot := false // whether the next and(x) is really a and(!x)
	and := func(c *Constraint) {
		old := sq.Constraint
		if andNot {
			c = &Constraint{
				Logical: &LogicalConstraint{
					Op: "not",
					A:  c,
				},
			}
		}
		sq.Constraint = &Constraint{
			Logical: &LogicalConstraint{
				Op: "and",
				A:  old,
				B:  c,
			},
		}
	}
	permOfFile := func(fc *FileConstraint) *Constraint {
		return &Constraint{
			Permanode: &PermanodeConstraint{
				Attr:       "camliContent",
				ValueInSet: &Constraint{File: fc},
			},
		}
	}
	orConst := func(a, b *Constraint) *Constraint {
		return &Constraint{
			Logical: &LogicalConstraint{
				Op: "or",
				A:  a,
				B:  b,
			},
		}
	}
	andFile := func(fc *FileConstraint) {
		and(permOfFile(fc))
	}
	andWHRatio := func(fc *FloatConstraint) {
		andFile(&FileConstraint{
			IsImage: true,
			WHRatio: fc,
		})
	}

	words := strings.Fields(exp)
	for _, word := range words {
		andNot = false
		if strings.HasPrefix(word, "-") {
			andNot = true
			word = word[1:]
		}
		if m := tagExpr.FindStringSubmatch(word); m != nil {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr:       "tag",
					SkipHidden: true,
					Value:      m[1],
				},
			})
			continue
		}
		if m := titleExpr.FindStringSubmatch(word); m != nil {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr:       "title",
					SkipHidden: true,
					ValueMatches: &StringConstraint{
						Contains:        m[1],
						CaseInsensitive: true,
					},
				},
			})
			continue
		}
		if word == "is:image" {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr: "camliContent",
					ValueInSet: &Constraint{
						File: &FileConstraint{
							IsImage: true,
						},
					},
				},
			})
			continue
		}
		if word == "is:landscape" {
			andWHRatio(&FloatConstraint{Min: 1.0})
			continue
		}
		if word == "is:portrait" {
			andWHRatio(&FloatConstraint{Max: 1.0})
			continue
		}
		if word == "is:pano" {
			andWHRatio(&FloatConstraint{Min: 1.6})
			continue
		}
		if word == "has:location" {
			andFile(&FileConstraint{
				IsImage: true,
				Location: &LocationConstraint{
					Any: true,
				},
			})
			continue
		}
		if strings.HasPrefix(word, "format:") {
			andFile(&FileConstraint{
				MIMEType: &StringConstraint{
					Equals: mimeFromFormat(strings.TrimPrefix(word, "format:")),
				},
			})
			continue
		}
		if strings.HasPrefix(word, "width:") {
			m := whRangeExpr.FindStringSubmatch(strings.TrimPrefix(word, "width:"))
			if m == nil {
				return nil, errors.New("bogus width range")
			}
			andFile(&FileConstraint{
				IsImage: true,
				Width:   whIntConstraint(m[1], m[2]),
			})
			continue
		}
		if strings.HasPrefix(word, "height:") {
			m := whRangeExpr.FindStringSubmatch(strings.TrimPrefix(word, "height:"))
			if m == nil {
				return nil, errors.New("bogus height range")
			}
			andFile(&FileConstraint{
				IsImage: true,
				Height:  whIntConstraint(m[1], m[2]),
			})
			continue
		}
		if strings.HasPrefix(word, "before:") || strings.HasPrefix(word, "after:") {
			before := false
			when := ""
			if strings.HasPrefix(word, "before:") {
				before = true
				when = strings.TrimPrefix(word, "before:")
			} else {
				when = strings.TrimPrefix(word, "after:")
			}
			base := "0000-01-01T00:00:00Z"
			if len(when) < len(base) {
				when += base[len(when):]
			}
			t, err := time.Parse(time.RFC3339, when)
			if err != nil {
				return nil, err
			}
			tc := &TimeConstraint{}
			if before {
				tc.Before = types.Time3339(t)
			} else {
				tc.After = types.Time3339(t)
			}
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Time: tc,
				},
			})
			continue
		}
		if strings.HasPrefix(word, "loc:") {
			where := strings.TrimPrefix(word, "loc:")
			rects, err := geocode.Lookup(ctx, where)
			if err != nil {
				return nil, err
			}
			if len(rects) == 0 {
				return nil, fmt.Errorf("No location found for %q", where)
			}
			var locConstraint *Constraint
			for i, rect := range rects {
				rectConstraint := permOfFile(&FileConstraint{
					IsImage: true,
					Location: &LocationConstraint{
						West:  rect.SouthWest.Long,
						East:  rect.NorthEast.Long,
						North: rect.NorthEast.Lat,
						South: rect.SouthWest.Lat,
					},
				})
				if i == 0 {
					locConstraint = rectConstraint
				} else {
					locConstraint = orConst(locConstraint, rectConstraint)
				}
			}
			and(locConstraint)
			continue
		}
		log.Printf("Unknown search expression word %q", word)
		// TODO: finish. better tokenization. non-operator tokens
		// are text searches, etc.
	}

	return sq, nil
}
Exemple #15
0
// b: the parsed file schema blob
// mm: keys to populate
func (ix *Index) populateFile(b *schema.Blob, mm *mutationMap) (err error) {
	var times []time.Time // all creation or mod times seen; may be zero
	times = append(times, b.ModTime())

	blobRef := b.BlobRef()
	fetcher := &seekFetcherMissTracker{
		// TODO(bradfitz): cache this SeekFetcher on ix so it
		// it's have to be re-made each time? Probably small.
		src: blob.SeekerFromStreamingFetcher(ix.BlobSource),
	}
	defer func() {
		if err == nil {
			return
		}
		fetcher.mu.Lock()
		defer fetcher.mu.Unlock()
		if len(fetcher.missing) == 0 {
			return
		}
		// TODO(bradfitz): there was an error indexing this file, and
		// we failed to load the blobs in f.missing.  Add those as dependencies
		// somewhere so when we get one of those missing blobs, we kick off
		// a re-index of this file for whenever the indexer is idle.
	}()
	fr, err := b.NewFileReader(fetcher)
	if err != nil {
		// TODO(bradfitz): propagate up a transient failure
		// error type, so we can retry indexing files in the
		// future if blobs are only temporarily unavailable.
		// Basically the same as the TODO just below.
		//
		// We'll also want to bump the schemaVersion after this,
		// to fix anybody's index which is only partial due to
		// this old bug where it would return nil instead of doing
		// the necessary work.
		log.Printf("index: error indexing file, creating NewFileReader %s: %v", blobRef, err)
		return nil
	}
	defer fr.Close()
	mime, reader := magic.MIMETypeFromReader(fr)

	sha1 := sha1.New()
	var copyDest io.Writer = sha1
	var imageBuf *keepFirstN // or nil
	if strings.HasPrefix(mime, "image/") {
		// Emperically derived 1MiB assuming CR2 images require more than any
		// other filetype we support:
		//   https://gist.github.com/wathiede/7982372
		imageBuf = &keepFirstN{N: 1 << 20}
		copyDest = io.MultiWriter(copyDest, imageBuf)
	}
	size, err := io.Copy(copyDest, reader)
	if err != nil {
		// TODO: job scheduling system to retry this spaced
		// out max n times.  Right now our options are
		// ignoring this error (forever) or returning the
		// error and making the indexing try again (likely
		// forever failing).  Both options suck.  For now just
		// log and act like all's okay.
		//
		// See TODOs above, and the fetcher.missing stuff.
		log.Printf("index: error indexing file %s: %v", blobRef, err)
		return nil
	}
	wholeRef := blob.RefFromHash(sha1)

	if imageBuf != nil {
		if conf, err := images.DecodeConfig(bytes.NewReader(imageBuf.Bytes)); err == nil {
			mm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height)))
		}
		if ft, err := schema.FileTime(bytes.NewReader(imageBuf.Bytes)); err == nil {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
			times = append(times, ft)
		} else {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
		}

		indexEXIF(wholeRef, imageBuf.Bytes, mm)
	}

	var sortTimes []time.Time
	for _, t := range times {
		if !t.IsZero() {
			sortTimes = append(sortTimes, t)
		}
	}
	sort.Sort(types.ByTime(sortTimes))
	var time3339s string
	switch {
	case len(sortTimes) == 1:
		time3339s = types.Time3339(sortTimes[0]).String()
	case len(sortTimes) >= 2:
		oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1]
		time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String()
	}

	mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime))
	mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))

	if strings.HasPrefix(mime, "audio/") {
		indexMusic(io.NewSectionReader(fr, 0, fr.Size()), wholeRef, mm)
	}

	return nil
}
Exemple #16
0
// b: the parsed file schema blob
// mm: keys to populate
func (ix *Index) populateFile(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) (err error) {
	var times []time.Time // all creation or mod times seen; may be zero
	times = append(times, b.ModTime())

	blobRef := b.BlobRef()
	fr, err := b.NewFileReader(fetcher)
	if err != nil {
		return err
	}
	defer fr.Close()
	mime, mr := magic.MIMETypeFromReader(fr)

	sha1 := sha1.New()
	var copyDest io.Writer = sha1
	var imageBuf *keepFirstN // or nil
	if strings.HasPrefix(mime, "image/") {
		imageBuf = &keepFirstN{N: 512 << 10}
		copyDest = io.MultiWriter(copyDest, imageBuf)
	}
	size, err := io.Copy(copyDest, mr)
	if err != nil {
		return err
	}
	wholeRef := blob.RefFromHash(sha1)

	if imageBuf != nil {
		var conf images.Config
		decodeConfig := func(r filePrefixReader) error {
			conf, err = images.DecodeConfig(r)
			return err
		}
		if err := readPrefixOrFile(imageBuf.Bytes, fetcher, b, decodeConfig); err == nil {
			mm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height)))
		}

		var ft time.Time
		fileTime := func(r filePrefixReader) error {
			ft, err = schema.FileTime(r)
			return err
		}
		if err = readPrefixOrFile(imageBuf.Bytes, fetcher, b, fileTime); err == nil {
			times = append(times, ft)
		}
		if exifDebug {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
		}

		// TODO(mpl): find (generate?) more broken EXIF images to experiment with.
		indexEXIFData := func(r filePrefixReader) error {
			return indexEXIF(wholeRef, r, mm)
		}
		if err = readPrefixOrFile(imageBuf.Bytes, fetcher, b, indexEXIFData); err != nil {
			if exifDebug {
				log.Printf("error parsing EXIF: %v", err)
			}
		}
	}

	var sortTimes []time.Time
	for _, t := range times {
		if !t.IsZero() {
			sortTimes = append(sortTimes, t)
		}
	}
	sort.Sort(types.ByTime(sortTimes))
	var time3339s string
	switch {
	case len(sortTimes) == 1:
		time3339s = types.Time3339(sortTimes[0]).String()
	case len(sortTimes) >= 2:
		oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1]
		time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String()
	}

	mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime, wholeRef))
	mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))

	if strings.HasPrefix(mime, "audio/") {
		indexMusic(io.NewSectionReader(fr, 0, fr.Size()), wholeRef, mm)
	}

	return nil
}
Exemple #17
0
// populate hits the blobstore to populate map of child nodes.
func (n *roDir) populate() error {
	n.mu.Lock()
	defer n.mu.Unlock()

	// Things never change here, so if we've ever populated, we're
	// populated.
	if n.children != nil {
		return nil
	}

	log.Printf("roDir.populate(%q) - Sending request At %v", n.fullPath(), n.at)

	res, err := n.fs.client.Describe(&search.DescribeRequest{
		BlobRef: n.permanode,
		Depth:   3,
		At:      types.Time3339(n.at),
	})
	if err != nil {
		log.Println("roDir.paths:", err)
		return nil
	}
	db := res.Meta[n.permanode.String()]
	if db == nil {
		return errors.New("dir blobref not described")
	}

	// Find all child permanodes and stick them in n.children
	n.children = make(map[string]roFileOrDir)
	for k, v := range db.Permanode.Attr {
		const p = "camliPath:"
		if !strings.HasPrefix(k, p) || len(v) < 1 {
			continue
		}
		name := k[len(p):]
		childRef := v[0]
		child := res.Meta[childRef]
		if child == nil {
			log.Printf("child not described: %v", childRef)
			continue
		}
		if target := child.Permanode.Attr.Get("camliSymlinkTarget"); target != "" {
			// This is a symlink.
			n.children[name] = &roFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				symLink:   true,
				target:    target,
			}
		} else if isDir(child.Permanode) {
			// This is a directory.
			n.children[name] = &roDir{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
			}
		} else if contentRef := child.Permanode.Attr.Get("camliContent"); contentRef != "" {
			// This is a file.
			content := res.Meta[contentRef]
			if content == nil {
				log.Printf("child content not described: %v", childRef)
				continue
			}
			if content.CamliType != "file" {
				log.Printf("child not a file: %v", childRef)
				continue
			}
			n.children[name] = &roFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				content:   blob.ParseOrZero(contentRef),
				size:      content.File.Size,
			}
		} else {
			// unknown type
			continue
		}
		n.children[name].xattr().load(child.Permanode)
	}
	return nil
}
Exemple #18
0
// b: the parsed file schema blob
// mm: keys to populate
func (ix *Index) populateFile(fetcher blob.Fetcher, b *schema.Blob, mm *mutationMap) (err error) {
	var times []time.Time // all creation or mod times seen; may be zero
	times = append(times, b.ModTime())

	blobRef := b.BlobRef()
	fr, err := b.NewFileReader(fetcher)
	if err != nil {
		return err
	}
	defer fr.Close()
	mime, reader := magic.MIMETypeFromReader(fr)

	sha1 := sha1.New()
	var copyDest io.Writer = sha1
	var imageBuf *keepFirstN // or nil
	if strings.HasPrefix(mime, "image/") {
		imageBuf = &keepFirstN{N: 512 << 10}
		copyDest = io.MultiWriter(copyDest, imageBuf)
	}
	size, err := io.Copy(copyDest, reader)
	if err != nil {
		return err
	}
	wholeRef := blob.RefFromHash(sha1)

	if imageBuf != nil {
		conf, err := images.DecodeConfig(bytes.NewReader(imageBuf.Bytes))
		// If our optimistic 512KB in-memory prefix from above was too short to get the dimensions, pass the whole thing instead and try again.
		if err == io.ErrUnexpectedEOF {
			var fr *schema.FileReader
			fr, err = b.NewFileReader(fetcher)
			if err == nil {
				conf, err = images.DecodeConfig(fr)
				fr.Close()
			}
		}
		if err == nil {
			mm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height)))
		}
		if ft, err := schema.FileTime(bytes.NewReader(imageBuf.Bytes)); err == nil {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
			times = append(times, ft)
		} else {
			log.Printf("filename %q exif = %v, %v", b.FileName(), ft, err)
		}

		// TODO(mpl): find (generate?) more broken EXIF images to experiment with.
		err = indexEXIF(wholeRef, bytes.NewReader(imageBuf.Bytes), mm)
		if err == io.EOF {
			var fr *schema.FileReader
			fr, err = b.NewFileReader(fetcher)
			if err == nil {
				err = indexEXIF(wholeRef, fr, mm)
				fr.Close()
			}
		}
		if err != nil {
			log.Printf("error parsing EXIF: %v", err)
		}
	}

	var sortTimes []time.Time
	for _, t := range times {
		if !t.IsZero() {
			sortTimes = append(sortTimes, t)
		}
	}
	sort.Sort(types.ByTime(sortTimes))
	var time3339s string
	switch {
	case len(sortTimes) == 1:
		time3339s = types.Time3339(sortTimes[0]).String()
	case len(sortTimes) >= 2:
		oldest, newest := sortTimes[0], sortTimes[len(sortTimes)-1]
		time3339s = types.Time3339(oldest).String() + "," + types.Time3339(newest).String()
	}

	mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime, wholeRef))
	mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))

	if strings.HasPrefix(mime, "audio/") {
		indexMusic(io.NewSectionReader(fr, 0, fr.Size()), wholeRef, mm)
	}

	return nil
}
Exemple #19
0
func TestTimeConstraint(t *testing.T) {
	tests := []struct {
		c    *TimeConstraint
		t    time.Time
		want bool
	}{
		{
			&TimeConstraint{
				Before: types.Time3339(time.Unix(124, 0)),
			},
			time.Unix(123, 0),
			true,
		},
		{
			&TimeConstraint{
				Before: types.Time3339(time.Unix(123, 0)),
			},
			time.Unix(123, 1),
			false,
		},
		{
			&TimeConstraint{
				After: types.Time3339(time.Unix(123, 0)),
			},
			time.Unix(123, 0),
			true,
		},
		{
			&TimeConstraint{
				After: types.Time3339(time.Unix(123, 0)),
			},
			time.Unix(123, 1),
			true,
		},
		{
			&TimeConstraint{
				After: types.Time3339(time.Unix(123, 0)),
			},
			time.Unix(122, 0),
			false,
		},
		{
			// This test will pass for 20 years at least.
			&TimeConstraint{
				InLast: 20 * year,
			},
			time.Unix(1384034605, 0),
			true,
		},
		{
			&TimeConstraint{
				InLast: 1 * year,
			},
			time.Unix(123, 0),
			false,
		},
	}
	for i, tt := range tests {
		got := tt.c.timeMatches(tt.t)
		if got != tt.want {
			t.Errorf("%d. matches(tc=%+v, t=%v) = %v; want %v", i, tt.c, tt.t, got, tt.want)
		}
	}
}