Ejemplo n.º 1
0
// contToken is of forms:
//    ""                : start from beginning of zip files
//    "sha1-xxxxx:n"    : start at == (sha1-xxxx, file n), else next zip
func (st largeBlobStreamer) StreamBlobs(ctx context.Context, dest chan<- blobserver.BlobAndToken, contToken string) (err error) {
	defer close(dest)
	s := st.sto
	large := s.large

	var after string // for enumerateAll
	var skipFiles int
	var firstRef blob.Ref // first we care about

	if contToken != "" {
		f := strings.SplitN(contToken, ":", 2)
		if len(f) != 2 {
			return errContToken
		}
		firstRef, _ = blob.Parse(f[0])
		skipFiles, err = strconv.Atoi(f[1])
		if !firstRef.Valid() || err != nil {
			return errContToken
		}
		// EnumerateAllFrom takes a cursor that's greater, but
		// we want to start _at_ firstRef. So start
		// enumerating right before our target.
		after = firstRef.StringMinusOne()
	}
	return blobserver.EnumerateAllFrom(ctx, large, after, func(sb blob.SizedRef) error {
		if firstRef.Valid() {
			if sb.Ref.Less(firstRef) {
				// Skip.
				return nil
			}
			if firstRef.Less(sb.Ref) {
				skipFiles = 0 // reset it.
			}
		}
		fileN := 0
		return s.foreachZipBlob(sb.Ref, func(bap BlobAndPos) error {
			if skipFiles > 0 {
				skipFiles--
				fileN++
				return nil
			}
			select {
			case dest <- blobserver.BlobAndToken{
				Blob: blob.NewBlob(bap.Ref, bap.Size, func() types.ReadSeekCloser {
					return blob.NewLazyReadSeekCloser(s, bap.Ref)
				}),
				Token: fmt.Sprintf("%s:%d", sb.Ref, fileN),
			}:
				fileN++
				return nil
			case <-ctx.Done():
				return ctx.Err()
			}
		})
	})
}
Ejemplo n.º 2
0
func (c *PermanodeConstraint) blobMatches(s *search, br blob.Ref, bm camtypes.BlobMeta) (ok bool, err error) {
	if bm.CamliType != "permanode" {
		return false, nil
	}
	corpus := s.h.corpus

	var dp *DescribedPermanode
	if corpus == nil {
		dr, err := s.h.Describe(&DescribeRequest{BlobRef: br})
		if err != nil {
			return false, err
		}
		db := dr.Meta[br.String()]
		if db == nil || db.Permanode == nil {
			return false, nil
		}
		dp = db.Permanode
	}

	if c.Attr != "" {
		if !c.At.IsZero() && corpus == nil {
			panic("PermanodeConstraint.At not supported without an in-memory corpus")
		}
		var vals []string
		if corpus == nil {
			vals = dp.Attr[c.Attr]
		} else {
			s.ss = corpus.AppendPermanodeAttrValuesLocked(
				s.ss[:0], br, c.Attr, c.At, s.h.owner)
			vals = s.ss
		}
		ok, err := c.permanodeMatchesAttrVals(s, vals)
		if !ok || err != nil {
			return false, err
		}
	}

	if c.SkipHidden && corpus != nil {
		vals := corpus.AppendPermanodeAttrValuesLocked(s.ss[:0], br, "camliDefVis", time.Time{}, s.h.owner)
		for _, v := range vals {
			if v == "hide" {
				return false, nil
			}
		}
	}

	if c.ModTime != nil {
		if corpus != nil {
			mt, ok := corpus.PermanodeModtimeLocked(br)
			if !ok || !c.ModTime.timeMatches(mt) {
				return false, nil
			}
		} else if !c.ModTime.timeMatches(dp.ModTime) {
			return false, nil
		}
	}

	if c.Time != nil {
		if corpus != nil {
			t, ok := corpus.PermanodeTimeLocked(br)
			if !ok || !c.Time.timeMatches(t) {
				return false, nil
			}
		} else {
			panic("TODO: not yet supported")
		}
	}

	if cc := c.Continue; cc != nil {
		if corpus == nil {
			// Requires an in-memory index for infinite
			// scroll. At least for now.
			return false, nil
		}
		if !cc.LastMod.IsZero() {
			mt, ok := corpus.PermanodeModtimeLocked(br)
			if !ok || mt.After(cc.LastMod) {
				return false, nil
			}
			// Blobs are sorted by modtime, and then by
			// blobref, and then reversed overall.  From
			// top of page, imagining this scenario, where
			// the user requested a page size Limit of 4:
			//     mod5, sha1-25
			//     mod4, sha1-72
			//     mod3, sha1-cc
			//     mod3, sha1-bb <--- last seen ite, continue = "pn:mod3:sha1-bb"
			//     mod3, sha1-aa  <-- and we want this one next.
			// In the case above, we'll see all of cc, bb, and cc for mod3.
			if mt.Equal(cc.LastMod) && !br.Less(cc.Last) {
				return false, nil
			}
		}
	}
	return true, nil
}
Ejemplo n.º 3
0
func (c *PermanodeConstraint) blobMatches(s *search, br blob.Ref, bm camtypes.BlobMeta) (ok bool, err error) {
	if bm.CamliType != "permanode" {
		return false, nil
	}
	corpus := s.h.corpus

	var dp *DescribedPermanode
	if corpus == nil {
		dr, err := s.h.Describe(&DescribeRequest{BlobRef: br})
		if err != nil {
			return false, err
		}
		db := dr.Meta[br.String()]
		if db == nil || db.Permanode == nil {
			return false, nil
		}
		dp = db.Permanode
	}

	if c.Attr != "" {
		if !c.At.IsZero() && corpus == nil {
			panic("PermanodeConstraint.At not supported without an in-memory corpus")
		}
		var vals []string
		if corpus == nil {
			vals = dp.Attr[c.Attr]
		} else {
			s.ss = corpus.AppendPermanodeAttrValuesLocked(
				s.ss[:0], br, c.Attr, c.At, s.h.owner)
			vals = s.ss
		}
		ok, err := c.permanodeMatchesAttrVals(s, vals)
		if !ok || err != nil {
			return false, err
		}
	}

	if c.SkipHidden && corpus != nil {
		defVis := corpus.PermanodeAttrValueLocked(br, "camliDefVis", c.At, s.h.owner)
		if defVis == "hide" {
			return false, nil
		}
		nodeType := corpus.PermanodeAttrValueLocked(br, "camliNodeType", c.At, s.h.owner)
		if nodeType == "foursquare.com:venue" {
			// TODO: temporary. remove this, or change
			// when/where (time) we show these.  But these
			// are flooding my results and I'm about to
			// demo this.
			return false, nil
		}
	}

	if c.ModTime != nil {
		if corpus != nil {
			mt, ok := corpus.PermanodeModtimeLocked(br)
			if !ok || !c.ModTime.timeMatches(mt) {
				return false, nil
			}
		} else if !c.ModTime.timeMatches(dp.ModTime) {
			return false, nil
		}
	}

	if c.Time != nil {
		if corpus != nil {
			t, ok := corpus.PermanodeAnyTimeLocked(br)
			if !ok || !c.Time.timeMatches(t) {
				return false, nil
			}
		} else {
			panic("TODO: not yet supported")
		}
	}

	if rc := c.Relation; rc != nil {
		ok, err := rc.match(s, br, c.At)
		if !ok || err != nil {
			return ok, err
		}
	}

	if c.Location != nil {
		if corpus == nil {
			return false, nil
		}
		lat, long, ok := corpus.PermanodeLatLongLocked(br, c.At)
		if !ok || !c.Location.matchesLatLong(lat, long) {
			return false, nil
		}
	}

	if cc := c.Continue; cc != nil {
		if corpus == nil {
			// Requires an in-memory index for infinite
			// scroll. At least for now.
			return false, nil
		}
		var pnTime time.Time
		var ok bool
		switch {
		case !cc.LastMod.IsZero():
			pnTime, ok = corpus.PermanodeModtimeLocked(br)
			if !ok || pnTime.After(cc.LastMod) {
				return false, nil
			}
		case !cc.LastCreated.IsZero():
			pnTime, ok = corpus.PermanodeAnyTimeLocked(br)
			if !ok || pnTime.After(cc.LastCreated) {
				return false, nil
			}
		default:
			panic("Continue constraint without a LastMod or a LastCreated")
		}
		// Blobs are sorted by modtime, and then by
		// blobref, and then reversed overall.  From
		// top of page, imagining this scenario, where
		// the user requested a page size Limit of 4:
		//     mod5, sha1-25
		//     mod4, sha1-72
		//     mod3, sha1-cc
		//     mod3, sha1-bb <--- last seen item, continue = "pn:mod3:sha1-bb"
		//     mod3, sha1-aa  <-- and we want this one next.
		// In the case above, we'll see all of cc, bb, and cc for mod3.
		if (pnTime.Equal(cc.LastMod) || pnTime.Equal(cc.LastCreated)) && !br.Less(cc.Last) {
			return false, nil
		}
	}
	return true, nil
}