func encodeChannelLog(log *channels.ChangeLog) []byte { if log == nil { return nil } raw := bytes.NewBuffer(make([]byte, 0, 50000)) log.Encode(raw) return raw.Bytes() }
// Saves a channel log, _if_ there isn't already one in the database. func (c *channelLogWriter) addChangeLog_(log *channels.ChangeLog) (added bool, err error) { added, err = c.bucket.AddRaw(channelLogDocID(c.channelName), 0, encodeChannelLog(log)) if added { base.LogTo("ChannelLog", "Added missing channel-log %q with %d entries", c.channelName, log.Len()) } else { base.LogTo("ChannelLog", "Didn't add channel-log %q with %d entries (err=%v)", c.channelName, log.Len()) } return }
// Adds a new change to a channel log. func (db *Database) AddToChangeLog(channelName string, entry channels.LogEntry, parentRevID string) error { if channelName == "*" && !EnableStarChannelLog { return nil } var fullUpdate bool var removedCount int fullUpdateAttempts := 0 logDocID := channelLogDocID(channelName) err := db.Bucket.WriteUpdate(logDocID, 0, func(currentValue []byte) ([]byte, walrus.WriteOptions, error) { // (Be careful: this block can be invoked multiple times if there are races!) // Should I do a full update of the change log, removing older entries to limit its size? // This has to be done occasionaly, but it's slower than simply appending to it. This // test is a heuristic that seems to strike a good balance in practice: fullUpdate = AlwaysCompactChangeLog || (len(currentValue) > 20000 && (rand.Intn(100) < len(currentValue)/5000)) removedCount = 0 if len(currentValue) == 0 { channelLog := channels.ChangeLog{} channelLog.Add(entry) return encodeChannelLog(&channelLog), walrus.Raw, nil } if fullUpdate { fullUpdateAttempts++ var newValue bytes.Buffer removedCount = channels.TruncateEncodedChangeLog(bytes.NewReader(currentValue), MaxChangeLogLength-1, &newValue) if removedCount > 0 { entry.Encode(&newValue, parentRevID) return newValue.Bytes(), walrus.Raw, nil } } w := bytes.NewBuffer(make([]byte, 0, 50000)) entry.Encode(w, parentRevID) currentValue = append(currentValue, w.Bytes()...) return currentValue, walrus.Raw, nil }) /*if fullUpdate { base.Log("Removed %d entries from %q", removedCount, channelName) } else if fullUpdateAttempts > 0 { base.Log("Attempted to remove entries %d times but failed", fullUpdateAttempts) }*/ return err }
// Returns a list of all the changes made on a channel. // Does NOT handle the Wait option. Does NOT check authorization. func (db *Database) changesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) { since := options.Since[channel] channelLog, err := db.changesWriter.getChangeLog(channel, since) if err != nil { base.Warn("Error reading channel-log %q (using view instead): %v", channel, err) channelLog = nil } rebuildLog := channelLog == nil && err == nil && (EnableStarChannelLog || channel != "*") var log []*channels.LogEntry if channelLog != nil { log = channelLog.Entries } var viewFeed <-chan *ChangeEntry if channelLog == nil || channelLog.Since > since { var upToSeq uint64 if channelLog != nil { upToSeq = channelLog.Since } // Channel log may not go back far enough, so also fetch view-based change feed: viewFeed, err = db.changesFeedFromView(channel, options, upToSeq) if err != nil { return nil, err } } feed := make(chan *ChangeEntry, 5) go func() { defer close(feed) // First, if we need to backfill from the view, write its early entries to the channel: if viewFeed != nil { newLog := channels.ChangeLog{Since: since} for change := range viewFeed { if channelLog != nil && change.seqNo > channelLog.Since { // TODO: Close the view-based feed somehow break } select { case <-options.Terminator: base.LogTo("Changes+", "Aborting changesFeed (reading from view)") return case feed <- change: } if rebuildLog { // If there wasn't any channel log, build up a new one from the view: entry := channels.LogEntry{ Sequence: change.seqNo, DocID: change.ID, RevID: change.Changes[0]["rev"], } if change.Deleted { entry.Flags |= channels.Deleted } if change.Removed != nil { entry.Flags |= channels.Removed } newLog.Add(entry) newLog.TruncateTo(MaxChangeLogLength) } } if rebuildLog { // Save the missing channel log we just rebuilt: base.LogTo("Changes", "Saving rebuilt channel log %q with %d sequences", channel, len(newLog.Entries)) db.changesWriter.addChangeLog(channel, &newLog) } } // Now write each log entry to the 'feed' channel in turn: for _, logEntry := range log { if !options.Conflicts && (logEntry.Flags&channels.Hidden) != 0 { //continue // FIX: had to comment this out. // This entry is shadowed by a conflicting one. We would like to skip it. // The problem is that if this is the newest revision of this doc, then the // doc will appear under this sequence # in the changes view, which means // we won't emit the doc at all because we already stopped emitting entries // from the view before this point. } change := ChangeEntry{ seqNo: logEntry.Sequence, ID: logEntry.DocID, Deleted: (logEntry.Flags & channels.Deleted) != 0, Changes: []ChangeRev{{"rev": logEntry.RevID}}, } if logEntry.Flags&channels.Removed != 0 { change.Removed = channels.SetOf(channel) } else if options.IncludeDocs || options.Conflicts { doc, _ := db.GetDoc(logEntry.DocID) db.addDocToChangeEntry(doc, &change, options.IncludeDocs, false) } select { case <-options.Terminator: base.LogTo("Changes+", "Aborting changesFeed") return case feed <- &change: } if options.Limit > 0 { options.Limit-- if options.Limit == 0 { break } } } }() return feed, nil }
// Writes new changes to my channel log document. func (c *channelLogWriter) addToChangeLog_(entries []*changeEntry) { var err error dbExpvars.Add("channelLogAdds", 1) logDocID := channelLogDocID(c.channelName) // A fraction of the time we will do a full update and clean stuff out. fullUpdate := AlwaysCompactChangeLog || len(entries) > MaxChangeLogLength/2 || rand.Intn(MaxChangeLogLength/len(entries)) == 0 if !fullUpdate { // Non-full update; just append the new entries: w := bytes.NewBuffer(make([]byte, 0, 100*len(entries))) for _, entry := range entries { entry.logEntry.Encode(w, entry.parentRevID) } data := w.Bytes() err = c.bucket.Append(logDocID, data) if err == nil { base.LogTo("ChannelLog", "Appended %d sequence(s) to %q", len(entries), c.channelName) dbExpvars.Add("channelLogAppends", 1) } else if base.IsDocNotFoundError(err) { // Append failed due to doc not existing, so fall back to full update err = nil fullUpdate = true } else { base.Warn("Error appending to %q -- %v", len(entries), c.channelName, err) } } if fullUpdate { // Full update: do a CAS-based read+write: fullUpdateAttempts := 0 var oldChangeLogCount, newChangeLogCount int err = c.bucket.WriteUpdate(logDocID, 0, func(currentValue []byte) ([]byte, walrus.WriteOptions, error) { fullUpdateAttempts++ numToKeep := MaxChangeLogLength - len(entries) if len(currentValue) == 0 || numToKeep <= 0 { // If log was missing or empty, or will be entirely overwritten, create a new one: entriesToWrite := entries if numToKeep < 0 { entriesToWrite = entries[-numToKeep:] } channelLog := channels.ChangeLog{} for _, entry := range entriesToWrite { channelLog.Add(*entry.logEntry) } newChangeLogCount = len(entriesToWrite) oldChangeLogCount = newChangeLogCount return encodeChannelLog(&channelLog), walrus.Raw, nil } else { // Append to an already existing change log: var newValue bytes.Buffer var nRemoved int nRemoved, newChangeLogCount = channels.TruncateEncodedChangeLog( bytes.NewReader(currentValue), numToKeep, numToKeep/2, &newValue) for _, entry := range entries { entry.logEntry.Encode(&newValue, entry.parentRevID) } oldChangeLogCount = nRemoved + newChangeLogCount newChangeLogCount += len(entries) return newValue.Bytes(), walrus.Raw, nil } }) if err == nil { dbExpvars.Add("channelLogRewrites", 1) dbExpvars.Add("channelLogRewriteCollisions", int64(fullUpdateAttempts-1)) base.LogTo("ChannelLog", "Wrote %d sequences (was %d now %d) to %q in %d attempts", len(entries), oldChangeLogCount, newChangeLogCount, c.channelName, fullUpdateAttempts) } else { base.Warn("Error writing %d sequence(s) to %q -- %v", len(entries), c.channelName, err) } } }
// Returns a list of all the changes made on a channel. // Does NOT handle the Wait option. Does NOT check authorization. func (db *Database) changesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) { since := options.Since[channel] channelLog, err := db.GetChangeLog(channel, since) if err != nil { base.Warn("Error reading channel-log %q (using view instead) %v", channel, err) channelLog = nil } rebuildLog := channelLog == nil && err == nil && (EnableStarChannelLog || channel != "*") var log []*channels.LogEntry if channelLog != nil { log = channelLog.Entries } var viewFeed <-chan *ChangeEntry if channelLog == nil || channelLog.Since > since { // Channel log may not go back far enough, so also fetch view-based change feed: viewFeed, err = db.changesFeedFromView(channel, options) if err != nil { return nil, err } } feed := make(chan *ChangeEntry, 5) go func() { defer close(feed) // First, if we need to backfill from the view, write its early entries to the channel: if viewFeed != nil { newLog := channels.ChangeLog{Since: since} for change := range viewFeed { if len(log) > 0 && change.seqNo >= log[0].Sequence { // TODO: Close the view-based feed somehow break } feed <- change if rebuildLog { // If there wasn't any channel log, build up a new one from the view: entry := channels.LogEntry{ Sequence: change.seqNo, DocID: change.ID, RevID: change.Changes[0]["rev"], } if change.Deleted { entry.Flags |= channels.Deleted } if change.Removed != nil { entry.Flags |= channels.Removed } newLog.Add(entry) newLog.TruncateTo(MaxChangeLogLength) } } if rebuildLog { // Save the missing channel log we just rebuilt: base.LogTo("Changes", "Saving rebuilt channel log %q with %d sequences", channel, len(newLog.Entries)) if _, err := db.AddChangeLog(channel, &newLog); err != nil { base.Warn("ChangesFeed: AddChangeLog failed, %v", err) } } } // Now write each log entry to the 'feed' channel in turn: for _, logEntry := range log { hidden := (logEntry.Flags & channels.Hidden) != 0 if logEntry.RevID == "" || (hidden && !options.Conflicts) { continue } change := ChangeEntry{ seqNo: logEntry.Sequence, ID: logEntry.DocID, Deleted: (logEntry.Flags & channels.Deleted) != 0, Changes: []ChangeRev{{"rev": logEntry.RevID}}, } if logEntry.Flags&channels.Removed != 0 { change.Removed = channels.SetOf(channel) } else if options.IncludeDocs || options.Conflicts { doc, _ := db.getDoc(logEntry.DocID) db.addDocToChangeEntry(doc, &change, options.IncludeDocs, false) } feed <- &change if options.Limit > 0 { options.Limit-- if options.Limit == 0 { break } } } }() return feed, nil }