func (db *Database) AddToChangeLogs(changedChannels base.Set, channelMap ChannelMap, entry channels.LogEntry, parentRevID string) error {
var err error
base.LogTo("Changes", "Updating #%d %q/%q in channels %s", entry.Sequence, entry.DocID, entry.RevID, changedChannels)
for channel, removal := range channelMap {
if removal != nil && removal.Seq != entry.Sequence {
continue
}
// Set Removed flag appropriately for this channel:
if removal != nil {
entry.Flags |= channels.Removed
} else {
entry.Flags = entry.Flags &^ channels.Removed
}
if err1 := db.AddToChangeLog(channel, entry, parentRevID); err != nil {
err = err1
}
}
// Finally, add to the universal "*" channel.
if EnableStarChannelLog {
entry.Flags = entry.Flags &^ channels.Removed
if err1 := db.AddToChangeLog("*", entry, parentRevID); err != nil {
err = err1
}
}
return err
}
// Adds a new change to a channel log.
func (db *Database) AddToChangeLog(channelName string, entry channels.LogEntry, parentRevID string) error {
if channelName == "*" && !EnableStarChannelLog {
return nil
}
var fullUpdate bool
var removedCount int
fullUpdateAttempts := 0
logDocID := channelLogDocID(channelName)
err := db.Bucket.WriteUpdate(logDocID, 0, func(currentValue []byte) ([]byte, walrus.WriteOptions, error) {
// (Be careful: this block can be invoked multiple times if there are races!)
// Should I do a full update of the change log, removing older entries to limit its size?
// This has to be done occasionaly, but it's slower than simply appending to it. This
// test is a heuristic that seems to strike a good balance in practice:
fullUpdate = AlwaysCompactChangeLog ||
(len(currentValue) > 20000 && (rand.Intn(100) < len(currentValue)/5000))
removedCount = 0
if len(currentValue) == 0 {
channelLog := channels.ChangeLog{}
channelLog.Add(entry)
return encodeChannelLog(&channelLog), walrus.Raw, nil
}
if fullUpdate {
fullUpdateAttempts++
var newValue bytes.Buffer
removedCount = channels.TruncateEncodedChangeLog(bytes.NewReader(currentValue),
MaxChangeLogLength-1, &newValue)
if removedCount > 0 {
entry.Encode(&newValue, parentRevID)
return newValue.Bytes(), walrus.Raw, nil
}
}
w := bytes.NewBuffer(make([]byte, 0, 50000))
entry.Encode(w, parentRevID)
currentValue = append(currentValue, w.Bytes()...)
return currentValue, walrus.Raw, nil
})
/*if fullUpdate {
base.Log("Removed %d entries from %q", removedCount, channelName)
} else if fullUpdateAttempts > 0 {
base.Log("Attempted to remove entries %d times but failed", fullUpdateAttempts)
}*/
return err
}
// Returns a list of all the changes made on a channel.
// Does NOT handle the Wait option. Does NOT check authorization.
func (db *Database) changesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) {
since := options.Since[channel]
channelLog, err := db.changesWriter.getChangeLog(channel, since)
if err != nil {
base.Warn("Error reading channel-log %q (using view instead): %v", channel, err)
channelLog = nil
}
rebuildLog := channelLog == nil && err == nil && (EnableStarChannelLog || channel != "*")
var log []*channels.LogEntry
if channelLog != nil {
log = channelLog.Entries
}
var viewFeed <-chan *ChangeEntry
if channelLog == nil || channelLog.Since > since {
var upToSeq uint64
if channelLog != nil {
upToSeq = channelLog.Since
}
// Channel log may not go back far enough, so also fetch view-based change feed:
viewFeed, err = db.changesFeedFromView(channel, options, upToSeq)
if err != nil {
return nil, err
}
}
feed := make(chan *ChangeEntry, 5)
go func() {
defer close(feed)
// First, if we need to backfill from the view, write its early entries to the channel:
if viewFeed != nil {
newLog := channels.ChangeLog{Since: since}
for change := range viewFeed {
if channelLog != nil && change.seqNo > channelLog.Since {
// TODO: Close the view-based feed somehow
break
}
select {
case <-options.Terminator:
base.LogTo("Changes+", "Aborting changesFeed (reading from view)")
return
case feed <- change:
}
if rebuildLog {
// If there wasn't any channel log, build up a new one from the view:
entry := channels.LogEntry{
Sequence: change.seqNo,
DocID: change.ID,
RevID: change.Changes[0]["rev"],
}
if change.Deleted {
entry.Flags |= channels.Deleted
}
if change.Removed != nil {
entry.Flags |= channels.Removed
}
newLog.Add(entry)
newLog.TruncateTo(MaxChangeLogLength)
}
}
if rebuildLog {
// Save the missing channel log we just rebuilt:
base.LogTo("Changes", "Saving rebuilt channel log %q with %d sequences",
channel, len(newLog.Entries))
db.changesWriter.addChangeLog(channel, &newLog)
}
}
// Now write each log entry to the 'feed' channel in turn:
for _, logEntry := range log {
if !options.Conflicts && (logEntry.Flags&channels.Hidden) != 0 {
//continue // FIX: had to comment this out.
// This entry is shadowed by a conflicting one. We would like to skip it.
// The problem is that if this is the newest revision of this doc, then the
// doc will appear under this sequence # in the changes view, which means
// we won't emit the doc at all because we already stopped emitting entries
// from the view before this point.
}
change := ChangeEntry{
seqNo: logEntry.Sequence,
ID: logEntry.DocID,
Deleted: (logEntry.Flags & channels.Deleted) != 0,
Changes: []ChangeRev{{"rev": logEntry.RevID}},
}
if logEntry.Flags&channels.Removed != 0 {
change.Removed = channels.SetOf(channel)
} else if options.IncludeDocs || options.Conflicts {
doc, _ := db.GetDoc(logEntry.DocID)
db.addDocToChangeEntry(doc, &change, options.IncludeDocs, false)
}
select {
case <-options.Terminator:
base.LogTo("Changes+", "Aborting changesFeed")
return
case feed <- &change:
}
if options.Limit > 0 {
options.Limit--
if options.Limit == 0 {
break
}
}
}
}()
return feed, nil
}
// Common subroutine of Put and PutExistingRev: a shell that loads the document, lets the caller
// make changes to it in a callback and supply a new body, then saves the body and document.
func (db *Database) updateDoc(docid string, allowImport bool, callback func(*document) (Body, error)) (string, error) {
// As a special case, it's illegal to put a design document except in admin mode:
if strings.HasPrefix(docid, "_design/") && db.user != nil {
return "", base.HTTPErrorf(403, "Forbidden to update design doc")
}
key := realDocID(docid)
if key == "" {
return "", base.HTTPErrorf(400, "Invalid doc ID")
}
var newRevID, parentRevID string
var doc *document
var body Body
var changedChannels base.Set
var changedPrincipals, changedRoleUsers []string
var docSequence uint64
var inConflict = false
err := db.Bucket.WriteUpdate(key, 0, func(currentValue []byte) (raw []byte, writeOpts walrus.WriteOptions, err error) {
// Be careful: this block can be invoked multiple times if there are races!
if doc, err = unmarshalDocument(docid, currentValue); err != nil {
return
} else if !allowImport && currentValue != nil && !doc.hasValidSyncData() {
err = base.HTTPErrorf(409, "Not imported")
return
}
// Invoke the callback to update the document and return a new revision body:
body, err = callback(doc)
if err != nil {
return
}
// Determine which is the current "winning" revision (it's not necessarily the new one):
newRevID = body["_rev"].(string)
parentRevID = doc.History[newRevID].Parent
prevCurrentRev := doc.CurrentRev
doc.CurrentRev, inConflict = doc.History.winningRevision()
doc.Deleted = doc.History[doc.CurrentRev].Deleted
if doc.CurrentRev != prevCurrentRev && prevCurrentRev != "" && doc.body != nil {
// Store the doc's previous body into the revision tree:
bodyJSON, _ := json.Marshal(doc.body)
doc.History.setRevisionBody(prevCurrentRev, bodyJSON)
}
// Store the new revision body into the doc:
doc.setRevision(newRevID, body)
if doc.CurrentRev == newRevID {
doc.NewestRev = ""
} else {
doc.NewestRev = newRevID
if doc.CurrentRev != prevCurrentRev {
// If the new revision is not current, transfer the current revision's
// body to the top level doc.body:
doc.body = doc.History.getParsedRevisionBody(doc.CurrentRev)
doc.History.setRevisionBody(doc.CurrentRev, nil)
}
}
// Run the sync function, to validate the update and compute its channels/access:
body["_id"] = doc.ID
channels, access, roles, err := db.getChannelsAndAccess(doc, body, parentRevID)
if err != nil {
return
}
if len(channels) > 0 {
doc.History[newRevID].Channels = channels
}
// Move the body of the replaced revision out of the document so it can be compacted later.
db.backupAncestorRevs(doc, newRevID)
// Now that we know doc is valid, assign it the next sequence number, for _changes feed.
// But be careful not to request a second sequence # on a retry if we don't need one.
if docSequence <= doc.Sequence {
if docSequence, err = db.sequences.nextSequence(); err != nil {
return
}
}
doc.Sequence = docSequence
if doc.CurrentRev != prevCurrentRev {
// Most of the time this update will change the doc's current rev. (The exception is
// if the new rev is a conflict that doesn't win the revid comparison.) If so, we
// need to update the doc's top-level Channels and Access properties to correspond
// to the current rev's state.
if newRevID != doc.CurrentRev {
// In some cases an older revision might become the current one. If so, get its
// channels & access, for purposes of updating the doc:
var curBody Body
if curBody, err = db.getAvailableRev(doc, doc.CurrentRev); curBody != nil {
base.LogTo("CRUD+", "updateDoc(%q): Rev %q causes %q to become current again",
docid, newRevID, doc.CurrentRev)
curParent := doc.History[doc.CurrentRev].Parent
channels, access, roles, err = db.getChannelsAndAccess(doc, curBody, curParent)
if err != nil {
return
}
} else {
// Shouldn't be possible (CurrentRev is a leaf so won't have been compacted)
base.Warn("updateDoc(%q): Rev %q missing, can't call getChannelsAndAccess "+
"on it (err=%v)", docid, doc.CurrentRev, err)
channels = nil
access = nil
roles = nil
}
}
// Update the document struct's channel assignment and user access.
// (This uses the new sequence # so has to be done after updating doc.Sequence)
changedChannels = doc.updateChannels(channels) //FIX: Incorrect if new rev is not current!
changedPrincipals = doc.Access.updateAccess(doc, access)
changedRoleUsers = doc.RoleAccess.updateAccess(doc, roles)
if len(changedPrincipals) > 0 || len(changedRoleUsers) > 0 {
// If this update affects user/role access privileges, make sure the write blocks till
// the new value is indexable, otherwise when a User/Role updates (using a view) it
// might not incorporate the effects of this change.
writeOpts |= walrus.Indexable
}
} else {
base.LogTo("CRUD+", "updateDoc(%q): Rev %q leaves %q still current",
docid, newRevID, prevCurrentRev)
}
// Prune old revision history to limit the number of revisions:
if pruned := doc.History.pruneRevisions(db.RevsLimit); pruned > 0 {
base.LogTo("CRUD+", "updateDoc(%q): Pruned %d old revisions", docid, pruned)
}
// Return the new raw document value for the bucket to store.
raw, err = json.Marshal(doc)
return
})
if err == couchbase.UpdateCancel {
return "", nil
} else if err == couchbase.ErrOverwritten {
// ErrOverwritten is ok; if a later revision got persisted, that's fine too
base.LogTo("CRUD+", "Note: Rev %q/%q was overwritten in RAM before becoming indexable",
docid, newRevID)
} else if err != nil {
return "", err
}
dbExpvars.Add("revs_added", 1)
// Store the new revision in the cache
history := doc.History.getHistory(newRevID)
if doc.History[newRevID].Deleted {
body["_deleted"] = true
}
revChannels := doc.History[newRevID].Channels
db.revisionCache.Put(body, encodeRevisions(history), revChannels)
// Now that the document has successfully been stored, we can make other db changes:
base.LogTo("CRUD", "Stored doc %q / %q", docid, newRevID)
// Mark affected users/roles as needing to recompute their channel access:
for _, name := range changedPrincipals {
db.invalUserOrRoleChannels(name)
}
for _, name := range changedRoleUsers {
db.invalUserRoles(name)
}
// Add the new revision to the change logs of all affected channels:
newEntry := channels.LogEntry{
Sequence: doc.Sequence,
DocID: docid,
RevID: newRevID,
}
if doc.History[newRevID].Deleted {
newEntry.Flags |= channels.Deleted
}
if newRevID != doc.CurrentRev {
newEntry.Flags |= channels.Hidden
}
if inConflict {
newEntry.Flags |= channels.Conflict
}
db.changesWriter.addToChangeLogs(changedChannels, doc.Channels, newEntry, parentRevID)
return newRevID, nil
}
// Returns a list of all the changes made on a channel.
// Does NOT handle the Wait option. Does NOT check authorization.
func (db *Database) changesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) {
since := options.Since[channel]
channelLog, err := db.GetChangeLog(channel, since)
if err != nil {
base.Warn("Error reading channel-log %q (using view instead) %v", channel, err)
channelLog = nil
}
rebuildLog := channelLog == nil && err == nil && (EnableStarChannelLog || channel != "*")
var log []*channels.LogEntry
if channelLog != nil {
log = channelLog.Entries
}
var viewFeed <-chan *ChangeEntry
if channelLog == nil || channelLog.Since > since {
// Channel log may not go back far enough, so also fetch view-based change feed:
viewFeed, err = db.changesFeedFromView(channel, options)
if err != nil {
return nil, err
}
}
feed := make(chan *ChangeEntry, 5)
go func() {
defer close(feed)
// First, if we need to backfill from the view, write its early entries to the channel:
if viewFeed != nil {
newLog := channels.ChangeLog{Since: since}
for change := range viewFeed {
if len(log) > 0 && change.seqNo >= log[0].Sequence {
// TODO: Close the view-based feed somehow
break
}
feed <- change
if rebuildLog {
// If there wasn't any channel log, build up a new one from the view:
entry := channels.LogEntry{
Sequence: change.seqNo,
DocID: change.ID,
RevID: change.Changes[0]["rev"],
}
if change.Deleted {
entry.Flags |= channels.Deleted
}
if change.Removed != nil {
entry.Flags |= channels.Removed
}
newLog.Add(entry)
newLog.TruncateTo(MaxChangeLogLength)
}
}
if rebuildLog {
// Save the missing channel log we just rebuilt:
base.LogTo("Changes", "Saving rebuilt channel log %q with %d sequences",
channel, len(newLog.Entries))
if _, err := db.AddChangeLog(channel, &newLog); err != nil {
base.Warn("ChangesFeed: AddChangeLog failed, %v", err)
}
}
}
// Now write each log entry to the 'feed' channel in turn:
for _, logEntry := range log {
hidden := (logEntry.Flags & channels.Hidden) != 0
if logEntry.RevID == "" || (hidden && !options.Conflicts) {
continue
}
change := ChangeEntry{
seqNo: logEntry.Sequence,
ID: logEntry.DocID,
Deleted: (logEntry.Flags & channels.Deleted) != 0,
Changes: []ChangeRev{{"rev": logEntry.RevID}},
}
if logEntry.Flags&channels.Removed != 0 {
change.Removed = channels.SetOf(channel)
} else if options.IncludeDocs || options.Conflicts {
doc, _ := db.getDoc(logEntry.DocID)
db.addDocToChangeEntry(doc, &change, options.IncludeDocs, false)
}
feed <- &change
if options.Limit > 0 {
options.Limit--
if options.Limit == 0 {
break
}
}
}
}()
return feed, nil
}
