// Determines whether the clock hash should be calculated for the entry. For non-continuous changes feeds, hash is only calculated for
// the last entry sent (for use in last_seq), and is done in the defer for the main VectorMultiChangesFeed.
// For continuous changes feeds, we want to calculate the hash for every nth entry, where n=kContinuousHashFrequency. To ensure that
// clients can restart a new changes feed based on any sequence in the continuous feed, we set the last hash calculated as the LowHash
// value on the sequence.
func (db *Database) calculateHashWhenNeeded(options ChangesOptions, entry *ChangeEntry, cumulativeClock base.SequenceClock, hashedEntryCount *int, continuousLastHash string) string {
if options.Continuous != true {
// For non-continuous, only need to calculate hash for lastSent entry. Initialize empty clock and return
entry.Seq.Clock = base.NewSyncSequenceClock()
} else {
// When hashedEntryCount == 0, recalculate hash
if *hashedEntryCount == 0 {
clockHash, err := db.SequenceHasher.GetHash(cumulativeClock)
if err != nil {
base.Warn("Error calculating hash for clock:%v", base.PrintClock(cumulativeClock))
return continuousLastHash
} else {
entry.Seq.Clock = base.NewSyncSequenceClock()
entry.Seq.Clock.SetHashedValue(clockHash)
continuousLastHash = clockHash
}
*hashedEntryCount = kContinuousHashFrequency
} else {
entry.Seq.LowHash = continuousLastHash
*hashedEntryCount--
}
}
return continuousLastHash
}
func (k *kvChannelIndex) loadChannelClock() (base.SequenceClock, error) {
chanClock := base.NewSyncSequenceClock()
key := getChannelClockKey(k.channelName)
value, _, err := k.indexBucket.GetRaw(key)
if err != nil {
base.LogTo("DIndex+", "Error loading channel clock for key %s:%v", key, err)
return chanClock, err
}
err = chanClock.Unmarshal(value)
if err != nil {
base.Warn("Error unmarshalling channel clock for channel %s, clock value %v", k.channelName, value)
}
return chanClock, err
}
// Determines whether the clock hash should be calculated for the entry. For non-continuous changes feeds, hash is only calculated for
// the last entry sent (for use in last_seq), and is done in the defer for the main VectorMultiChangesFeed.
// For continuous changes feeds, we want to calculate the hash for every nth entry, where n=kChangesHashFrequency. To ensure that
// clients can restart a new changes feed based on any sequence in the continuous feed, we set the last hash calculated as the LowHash
// value on the sequence.
func (db *Database) calculateHashWhenNeeded(options ChangesOptions, entry *ChangeEntry, cumulativeClock base.SequenceClock, hashedEntryCount *int, lastHashedValue string, forceHash bool) string {
// When hashedEntryCount == 0 or forceHash==true recalculate hash
if *hashedEntryCount == 0 || forceHash {
clockHash, err := db.SequenceHasher.GetHash(cumulativeClock)
if err != nil {
base.Warn("Error calculating hash for clock:%v", base.PrintClock(cumulativeClock))
return lastHashedValue
} else {
entry.Seq.Clock = base.NewSyncSequenceClock()
entry.Seq.Clock.SetHashedValue(clockHash)
lastHashedValue = clockHash
}
*hashedEntryCount = db.SequenceHasher.getHashFrequency()
} else {
entry.Seq.LowHash = lastHashedValue
*hashedEntryCount--
}
return lastHashedValue
}
// Returns the (ordered) union of all of the changes made to multiple channels.
func (db *Database) VectorMultiChangesFeed(chans base.Set, options ChangesOptions) (<-chan *ChangeEntry, error) {
to := ""
var userVbNo uint16
if db.user != nil && db.user.Name() != "" {
to = fmt.Sprintf(" (to %s)", db.user.Name())
userVbNo = uint16(db.Bucket.VBHash(db.user.DocID()))
}
base.LogTo("Changes+", "Vector MultiChangesFeed(%s, %+v) ... %s", chans, options, to)
output := make(chan *ChangeEntry, 50)
go func() {
var cumulativeClock *base.SyncSequenceClock
var lastHashedValue string
hashedEntryCount := 0
defer func() {
base.LogTo("Changes+", "MultiChangesFeed done %s", to)
close(output)
}()
var changeWaiter *changeWaiter
var userChangeCount uint64
var addedChannels base.Set // Tracks channels added to the user during changes processing.
if options.Wait {
// Note (Adam): I don't think there's a reason to set this to false here. We're outside the
// main iteration loop (so the if check above should only happen once), and I don't believe
// options.Wait is referenced elsewhere once MultiChangesFeed is called. Leaving it as-is
// makes it possible for channels to identify whether a getChanges call has options.Wait set to true,
// which is useful to identify active change listeners. However, it's possible there's a subtlety of
// longpoll or continuous processing I'm missing here - leaving this note instead of just deleting for now.
//options.Wait = false
changeWaiter = db.startChangeWaiter(chans)
userChangeCount = changeWaiter.CurrentUserCount()
}
cumulativeClock = base.NewSyncSequenceClock()
cumulativeClock.SetTo(getChangesClock(options.Since))
// This loop is used to re-run the fetch after every database change, in Wait mode
outer:
for {
// Get the last polled stable sequence. We don't return anything later than stable sequence in each iteration
stableClock, err := db.changeCache.GetStableClock(true)
if err != nil {
base.Warn("MultiChangesFeed got error reading stable sequence: %v", err)
return
}
// Restrict to available channels, expand wild-card, and find since when these channels
// have been available to the user:
var channelsSince channels.TimedSet
if db.user != nil {
channelsSince = db.user.FilterToAvailableChannels(chans)
} else {
channelsSince = channels.AtSequence(chans, 0)
}
// Updates the changeWaiter to the current set of available channels.
if changeWaiter != nil {
changeWaiter.UpdateChannels(channelsSince)
}
base.LogTo("Changes+", "MultiChangesFeed: channels expand to %#v ... %s", channelsSince, to)
// Build the channel feeds.
feeds, err := db.initializeChannelFeeds(channelsSince, options, addedChannels, userVbNo)
if err != nil {
return
}
// This loop reads the available entries from all the feeds in parallel, merges them,
// and writes them to the output channel:
current := make([]*ChangeEntry, len(feeds))
var sentSomething bool
nextEntry := getNextSequenceFromFeeds(current, feeds)
for {
minEntry := nextEntry
if minEntry == nil {
break // Exit the loop when there are no more entries
}
// Calculate next entry here, to help identify whether minEntry is the last entry we're sending,
// to guarantee hashing
nextEntry = getNextSequenceFromFeeds(current, feeds)
// Don't send any entries later than the stable sequence
if stableClock.GetSequence(minEntry.Seq.vbNo) < minEntry.Seq.Seq {
continue
}
// Add the doc body or the conflicting rev IDs, if those options are set:
if options.IncludeDocs || options.Conflicts {
db.addDocToChangeEntry(minEntry, options)
}
// Clock handling
if minEntry.Seq.TriggeredBy == 0 {
// Update the cumulative clock, and stick it on the entry.
cumulativeClock.SetMaxSequence(minEntry.Seq.vbNo, minEntry.Seq.Seq)
// Force new hash generation for non-continuous changes feeds if this is the last entry to be sent - either
// because there are no more entries in the channel feeds, or we're going to hit the limit.
forceHash := false
if options.Continuous == false && (nextEntry == nil || options.Limit == 1) {
forceHash = true
}
lastHashedValue = db.calculateHashWhenNeeded(
options,
minEntry,
cumulativeClock,
&hashedEntryCount,
lastHashedValue,
forceHash,
)
} else {
// For backfill (triggered by), we don't want to update the cumulative clock. All entries triggered by the
// same sequence reference the same triggered by clock, so it should only need to get hashed once.
// If this is the first entry for this triggered by, initialize the triggered by clock's
// hash value.
if minEntry.Seq.TriggeredByClock.GetHashedValue() == "" {
cumulativeClock.SetMaxSequence(minEntry.Seq.TriggeredByVbNo, minEntry.Seq.TriggeredBy)
clockHash, err := db.SequenceHasher.GetHash(cumulativeClock)
if err != nil {
base.Warn("Error calculating hash for triggered by clock:%v", base.PrintClock(cumulativeClock))
} else {
minEntry.Seq.TriggeredByClock.SetHashedValue(clockHash)
}
}
}
// Send the entry, and repeat the loop:
select {
case <-options.Terminator:
return
case output <- minEntry:
}
sentSomething = true
// Stop when we hit the limit (if any):
if options.Limit > 0 {
options.Limit--
if options.Limit == 0 {
break outer
}
}
}
if !options.Continuous && (sentSomething || changeWaiter == nil) {
break
}
// Update options.Since for use in the next outer loop iteration.
options.Since.Clock = cumulativeClock
// If nothing found, and in wait mode: wait for the db to change, then run again.
// First notify the reader that we're waiting by sending a nil.
base.LogTo("Changes+", "MultiChangesFeed waiting... %s", to)
output <- nil
waitForChanges:
for {
waitResponse := changeWaiter.Wait()
if waitResponse == WaiterClosed {
break outer
} else if waitResponse == WaiterHasChanges {
select {
case <-options.Terminator:
return
default:
break waitForChanges
}
} else if waitResponse == WaiterCheckTerminated {
// Check whether I was terminated while waiting for a change. If not, resume wait.
select {
case <-options.Terminator:
return
default:
}
}
}
// Before checking again, update the User object in case its channel access has
// changed while waiting:
userChangeCount, addedChannels, err = db.checkForUserUpdates(userChangeCount, changeWaiter)
if err != nil {
change := makeErrorEntry("User not found during reload - terminating changes feed")
base.LogTo("Changes+", "User not found during reload - terminating changes feed with entry %+v", change)
output <- &change
return
}
}
}()
return output, nil
}
