func generateGroup(groupId uint32) (*task.GroupKeys, error) {
it := pstore.NewIterator()
defer it.Close()
g := &task.GroupKeys{
GroupId: groupId,
}
for it.SeekToFirst(); it.Valid(); it.Next() {
k, v := it.Key(), it.Value()
pk := x.Parse(k.Data())
if pk == nil {
continue
}
if group.BelongsTo(pk.Attr) != g.GroupId {
it.Seek(pk.SkipPredicate())
it.Prev() // To tackle it.Next() called by default.
continue
}
var pl types.PostingList
x.Check(pl.Unmarshal(v.Data()))
kdup := make([]byte, len(k.Data()))
copy(kdup, k.Data())
key := &task.KC{
Key: kdup,
Checksum: pl.Checksum,
}
g.Keys = append(g.Keys, key)
}
return g, it.Err()
}
// PredicateData can be used to return data corresponding to a predicate over
// a stream.
func (w *grpcWorker) PredicateData(gkeys *task.GroupKeys, stream Worker_PredicateDataServer) error {
if !groups().ServesGroup(gkeys.GroupId) {
return x.Errorf("Group %d not served.", gkeys.GroupId)
}
n := groups().Node(gkeys.GroupId)
if !n.AmLeader() {
return x.Errorf("Not leader of group: %d", gkeys.GroupId)
}
// TODO(pawan) - Shift to CheckPoints once we figure out how to add them to the
// RocksDB library we are using.
// http://rocksdb.org/blog/2609/use-checkpoints-for-efficient-snapshots/
it := pstore.NewIterator()
defer it.Close()
for it.SeekToFirst(); it.Valid(); it.Next() {
k, v := it.Key(), it.Value()
pk := x.Parse(k.Data())
if pk == nil {
continue
}
if group.BelongsTo(pk.Attr) != gkeys.GroupId {
it.Seek(pk.SkipPredicate())
it.Prev() // To tackle it.Next() called by default.
continue
}
var pl types.PostingList
x.Check(pl.Unmarshal(v.Data()))
idx := sort.Search(len(gkeys.Keys), func(i int) bool {
t := gkeys.Keys[i]
return bytes.Compare(k.Data(), t.Key) <= 0
})
if idx < len(gkeys.Keys) {
// Found a match.
t := gkeys.Keys[idx]
// Different keys would have the same prefix. So, check Checksum first,
// it would be cheaper when there's no match.
if bytes.Equal(pl.Checksum, t.Checksum) && bytes.Equal(k.Data(), t.Key) {
// No need to send this.
continue
}
}
// We just need to stream this kv. So, we can directly use the key
// and val without any copying.
kv := &task.KV{
Key: k.Data(),
Val: v.Data(),
}
if err := stream.Send(kv); err != nil {
return err
}
} // end of iterator
if err := it.Err(); err != nil {
return err
}
return nil
}
