// contToken is of forms:
// "" : start from beginning of zip files
// "sha1-xxxxx:n" : start at == (sha1-xxxx, file n), else next zip
func (st largeBlobStreamer) StreamBlobs(ctx context.Context, dest chan<- blobserver.BlobAndToken, contToken string) (err error) {
defer close(dest)
s := st.sto
large := s.large
var after string // for enumerateAll
var skipFiles int
var firstRef blob.Ref // first we care about
if contToken != "" {
f := strings.SplitN(contToken, ":", 2)
if len(f) != 2 {
return errContToken
}
firstRef, _ = blob.Parse(f[0])
skipFiles, err = strconv.Atoi(f[1])
if !firstRef.Valid() || err != nil {
return errContToken
}
// EnumerateAllFrom takes a cursor that's greater, but
// we want to start _at_ firstRef. So start
// enumerating right before our target.
after = firstRef.StringMinusOne()
}
return blobserver.EnumerateAllFrom(ctx, large, after, func(sb blob.SizedRef) error {
if firstRef.Valid() {
if sb.Ref.Less(firstRef) {
// Skip.
return nil
}
if firstRef.Less(sb.Ref) {
skipFiles = 0 // reset it.
}
}
fileN := 0
return s.foreachZipBlob(sb.Ref, func(bap BlobAndPos) error {
if skipFiles > 0 {
skipFiles--
fileN++
return nil
}
select {
case dest <- blobserver.BlobAndToken{
Blob: blob.NewBlob(bap.Ref, bap.Size, func() types.ReadSeekCloser {
return blob.NewLazyReadSeekCloser(s, bap.Ref)
}),
Token: fmt.Sprintf("%s:%d", sb.Ref, fileN),
}:
fileN++
return nil
case <-ctx.Done():
return ctx.Err()
}
})
})
}
func (c *PermanodeConstraint) blobMatches(s *search, br blob.Ref, bm camtypes.BlobMeta) (ok bool, err error) {
if bm.CamliType != "permanode" {
return false, nil
}
corpus := s.h.corpus
var dp *DescribedPermanode
if corpus == nil {
dr, err := s.h.Describe(&DescribeRequest{BlobRef: br})
if err != nil {
return false, err
}
db := dr.Meta[br.String()]
if db == nil || db.Permanode == nil {
return false, nil
}
dp = db.Permanode
}
if c.Attr != "" {
if !c.At.IsZero() && corpus == nil {
panic("PermanodeConstraint.At not supported without an in-memory corpus")
}
var vals []string
if corpus == nil {
vals = dp.Attr[c.Attr]
} else {
s.ss = corpus.AppendPermanodeAttrValuesLocked(
s.ss[:0], br, c.Attr, c.At, s.h.owner)
vals = s.ss
}
ok, err := c.permanodeMatchesAttrVals(s, vals)
if !ok || err != nil {
return false, err
}
}
if c.SkipHidden && corpus != nil {
vals := corpus.AppendPermanodeAttrValuesLocked(s.ss[:0], br, "camliDefVis", time.Time{}, s.h.owner)
for _, v := range vals {
if v == "hide" {
return false, nil
}
}
}
if c.ModTime != nil {
if corpus != nil {
mt, ok := corpus.PermanodeModtimeLocked(br)
if !ok || !c.ModTime.timeMatches(mt) {
return false, nil
}
} else if !c.ModTime.timeMatches(dp.ModTime) {
return false, nil
}
}
if c.Time != nil {
if corpus != nil {
t, ok := corpus.PermanodeTimeLocked(br)
if !ok || !c.Time.timeMatches(t) {
return false, nil
}
} else {
panic("TODO: not yet supported")
}
}
if cc := c.Continue; cc != nil {
if corpus == nil {
// Requires an in-memory index for infinite
// scroll. At least for now.
return false, nil
}
if !cc.LastMod.IsZero() {
mt, ok := corpus.PermanodeModtimeLocked(br)
if !ok || mt.After(cc.LastMod) {
return false, nil
}
// Blobs are sorted by modtime, and then by
// blobref, and then reversed overall. From
// top of page, imagining this scenario, where
// the user requested a page size Limit of 4:
// mod5, sha1-25
// mod4, sha1-72
// mod3, sha1-cc
// mod3, sha1-bb <--- last seen ite, continue = "pn:mod3:sha1-bb"
// mod3, sha1-aa <-- and we want this one next.
// In the case above, we'll see all of cc, bb, and cc for mod3.
if mt.Equal(cc.LastMod) && !br.Less(cc.Last) {
return false, nil
}
}
}
return true, nil
}
func (c *PermanodeConstraint) blobMatches(s *search, br blob.Ref, bm camtypes.BlobMeta) (ok bool, err error) {
if bm.CamliType != "permanode" {
return false, nil
}
corpus := s.h.corpus
var dp *DescribedPermanode
if corpus == nil {
dr, err := s.h.Describe(&DescribeRequest{BlobRef: br})
if err != nil {
return false, err
}
db := dr.Meta[br.String()]
if db == nil || db.Permanode == nil {
return false, nil
}
dp = db.Permanode
}
if c.Attr != "" {
if !c.At.IsZero() && corpus == nil {
panic("PermanodeConstraint.At not supported without an in-memory corpus")
}
var vals []string
if corpus == nil {
vals = dp.Attr[c.Attr]
} else {
s.ss = corpus.AppendPermanodeAttrValuesLocked(
s.ss[:0], br, c.Attr, c.At, s.h.owner)
vals = s.ss
}
ok, err := c.permanodeMatchesAttrVals(s, vals)
if !ok || err != nil {
return false, err
}
}
if c.SkipHidden && corpus != nil {
defVis := corpus.PermanodeAttrValueLocked(br, "camliDefVis", c.At, s.h.owner)
if defVis == "hide" {
return false, nil
}
nodeType := corpus.PermanodeAttrValueLocked(br, "camliNodeType", c.At, s.h.owner)
if nodeType == "foursquare.com:venue" {
// TODO: temporary. remove this, or change
// when/where (time) we show these. But these
// are flooding my results and I'm about to
// demo this.
return false, nil
}
}
if c.ModTime != nil {
if corpus != nil {
mt, ok := corpus.PermanodeModtimeLocked(br)
if !ok || !c.ModTime.timeMatches(mt) {
return false, nil
}
} else if !c.ModTime.timeMatches(dp.ModTime) {
return false, nil
}
}
if c.Time != nil {
if corpus != nil {
t, ok := corpus.PermanodeAnyTimeLocked(br)
if !ok || !c.Time.timeMatches(t) {
return false, nil
}
} else {
panic("TODO: not yet supported")
}
}
if rc := c.Relation; rc != nil {
ok, err := rc.match(s, br, c.At)
if !ok || err != nil {
return ok, err
}
}
if c.Location != nil {
if corpus == nil {
return false, nil
}
lat, long, ok := corpus.PermanodeLatLongLocked(br, c.At)
if !ok || !c.Location.matchesLatLong(lat, long) {
return false, nil
}
}
if cc := c.Continue; cc != nil {
if corpus == nil {
// Requires an in-memory index for infinite
// scroll. At least for now.
return false, nil
}
var pnTime time.Time
var ok bool
switch {
case !cc.LastMod.IsZero():
pnTime, ok = corpus.PermanodeModtimeLocked(br)
if !ok || pnTime.After(cc.LastMod) {
return false, nil
}
case !cc.LastCreated.IsZero():
pnTime, ok = corpus.PermanodeAnyTimeLocked(br)
if !ok || pnTime.After(cc.LastCreated) {
return false, nil
}
default:
panic("Continue constraint without a LastMod or a LastCreated")
}
// Blobs are sorted by modtime, and then by
// blobref, and then reversed overall. From
// top of page, imagining this scenario, where
// the user requested a page size Limit of 4:
// mod5, sha1-25
// mod4, sha1-72
// mod3, sha1-cc
// mod3, sha1-bb <--- last seen item, continue = "pn:mod3:sha1-bb"
// mod3, sha1-aa <-- and we want this one next.
// In the case above, we'll see all of cc, bb, and cc for mod3.
if (pnTime.Equal(cc.LastMod) || pnTime.Equal(cc.LastCreated)) && !br.Less(cc.Last) {
return false, nil
}
}
return true, nil
}