func (b *BitFlagStorage) GetChanges(fromSeq base.SequenceClock, toSeq base.SequenceClock) ([]*LogEntry, error) {
// Determine which blocks have changed, and load those blocks
blocksByKey, blocksByVb, err := b.calculateChangedBlocks(fromSeq, toSeq)
if err != nil {
return nil, err
}
b.bulkLoadBlocks(blocksByKey)
// For each vbucket, create the entries from the blocks. Create in reverse sequence order, for
// deduplication once we've retrieved the full entry
entries := make([]*LogEntry, 0)
entryKeys := make([]string, 0)
for blockSetIndex := len(blocksByVb) - 1; blockSetIndex >= 0; blockSetIndex-- {
blockSet := blocksByVb[blockSetIndex]
vbNo := blockSet.vbNo
blocks := blockSet.blocks
fromVbSeq := fromSeq.GetSequence(vbNo) + 1
toVbSeq := toSeq.GetSequence(vbNo)
for blockIndex := len(blocks) - 1; blockIndex >= 0; blockIndex-- {
blockEntries, keys := blocks[blockIndex].GetEntries(vbNo, fromVbSeq, toVbSeq, true)
entries = append(entries, blockEntries...)
entryKeys = append(entryKeys, keys...)
}
}
// Bulk retrieval of individual entries. Performs deduplication, and reordering into ascending vb and sequence order
results := b.bulkLoadEntries(entryKeys, entries)
return results, nil
}
// Calculate the set of index blocks that need to be loaded.
// blocksByVb stores which blocks need to be processed for each vbucket, in ascending vbucket order. Multiple vb map
// values can point to the same IndexBlock (i.e. when those vbs share a partition).
// blocksByKey stores all IndexBlocks to be retrieved, indexed by block key - no duplicates
func (b *BitFlagStorage) calculateChangedBlocks(fromSeq base.SequenceClock, channelClock base.SequenceClock) (blocksByKey map[string]IndexBlock, blocksByVb []vbBlockSet, err error) {
blocksByKey = make(map[string]IndexBlock, 1)
blocksByVb = make([]vbBlockSet, 0)
for vbNo, clockVbSeq := range channelClock.Value() {
fromVbSeq := fromSeq.GetSequence(uint16(vbNo))
// Verify that the requested from value is less than the channel clock sequence (there are
// new entries for this vbucket in the channel)
if fromVbSeq >= clockVbSeq {
continue
}
blockSet := vbBlockSet{vbNo: uint16(vbNo)}
partition := b.partitions.VbMap[uint16(vbNo)]
for _, blockIndex := range generateBitFlagBlockIndexes(b.channelName, fromVbSeq, clockVbSeq, partition) {
blockKey := getIndexBlockKey(b.channelName, blockIndex, partition)
block, found := blocksByKey[blockKey]
if !found {
block = newBitFlagBufferBlockForKey(blockKey, b.channelName, blockIndex, partition, b.partitions.VbPositionMaps[partition])
blocksByKey[blockKey] = block
}
blockSet.blocks = append(blockSet.blocks, block)
}
blocksByVb = append(blocksByVb, blockSet)
}
return blocksByKey, blocksByVb, nil
}
// calculateChangedPartitions identifies which vbuckets have changed after sinceClock until toClock, groups these
// by partition, and returns as a array of PartitionRanges, indexed by partition number.
func (ds *DenseStorageReader) calculateChanged(sinceClock, toClock base.SequenceClock) (changedVbs []uint16, changedPartitions []*PartitionRange) {
changedVbs = make([]uint16, 0)
changedPartitions = make([]*PartitionRange, ds.partitions.PartitionCount())
for vbNoInt, toSeq := range toClock.Value() {
vbNo := uint16(vbNoInt)
sinceSeq := sinceClock.GetSequence(vbNo)
if sinceSeq < toSeq {
changedVbs = append(changedVbs, vbNo)
partitionNo := ds.partitions.PartitionForVb(vbNo)
if changedPartitions[partitionNo] == nil {
partitionRange := NewPartitionRange()
changedPartitions[partitionNo] = &partitionRange
}
changedPartitions[partitionNo].SetRange(vbNo, sinceSeq, toSeq)
}
}
return changedVbs, changedPartitions
}