/*
Append logs the given message and returns the index of the next slot or an error if the message could not be logged.
If the end of the log doesn't match the previousIndex and the previousTerm the append must fail and previousMatch should be false.
If there is an existing entry at msg.Index with a different msg.Term then this entry and all subsequent entries must be deleted prior to the append.
NOTE: The lock is not held in this method as we do not touch commitIndex or waitingReaders
*/
func (clog *CommitLog) Append(msgs model.Messages, previousIndex int64, previousTerm int64, leaderFirstIndex int64) (nextIndex int64, previousMatch bool, err error) {
overlapMessageCount := 0
//clog.node_log("Recieved messages to append %v\n", msgs)
lastIndex, err := clog.log.GetLastIndex()
nextIndex = lastIndex + 1
if err != nil {
clog.node_log("Error getting last message details during append: %v\n", err)
return 0, false, err
}
//clog.node_log("LastIndex is %v, previousIndex passed in is %v\n", lastIndex, previousIndex)
if lastIndex > 0 {
// Get our copy of the message at the peviousIndex
previousMessageList, err := clog.log.GetMessages(previousIndex, 1)
if err != nil {
return nextIndex, false, err
}
if previousMessageList.GetCount() < 1 {
clog.node_log("Previous index given by leader (%v) is greater than or smaller than our last index (%v), returning no match\n", previousIndex, lastIndex)
return nextIndex, false, nil
}
previousMessageTerm, err := previousMessageList.GetMessageTerm(0)
if err != nil {
clog.node_log("Error getting previous term: %v\n", err)
return nextIndex, false, err
}
if previousMessageTerm != previousTerm {
// Term of the message at this index doesn't match.
clog.node_log("Previous term of %v doesn't match previousMessageTerm of %v\n", previousTerm, previousMessageTerm)
return nextIndex, false, nil
}
//clog.node_log("Our last index is %v. Leader thinks our last index is %v and we are trying to append %v messages\n", lastIndex, previousIndex, msgs.GetCount())
// How many messages overlap? If len(msgs) is shorter than the gap between lastIndex and previousIndex, use that.
// How many messages overlap? If we have 10 messages and previousIndex was 8 then we have two left to check (9 & 10)
// How many messages overlap? If we have a lastIndex of 10 and previousIndex of 8 then we have two left to check (9 & 10)
msgsToCheck := msgs.GetCount()
// We have already checked the first message, does the rest push us beyond the lastIndex?
// E.g. len (msgs) = 2, previousIndex = 5, lastIndex = 7.
if (int64(msgsToCheck) + previousIndex) >= lastIndex {
msgsToCheck = int(lastIndex - previousIndex)
}
// Next question - do all messages beyond the previousMessage match the contents of the new messages?
checkMessagesIndex := previousIndex
if msgsToCheck > 0 {
overlappingMessages, err := clog.getAtLeastMessages(checkMessagesIndex+1, int64(msgsToCheck))
clog.node_log("Retrieved %v messages from index %v to check for overlaps (needed at least %v)\n", overlappingMessages.GetCount(), checkMessagesIndex+1, msgsToCheck)
if err != nil {
return nextIndex, false, err
}
for i := 0; i < overlappingMessages.GetCount(); i++ {
// Check terms match for the given index
// TODO: Check payload and CRC as well?
checkMessagesIndex++
//clog.node_log("Loop index %v, overlapMessageCount: %v\n", i, overlapMessageCount)
overlappingMessagesTerm, err := overlappingMessages.GetMessageTerm(i)
if err != nil {
return nextIndex, false, err
}
msgsTerm, err := msgs.GetMessageTerm(overlapMessageCount)
if err != nil {
return nextIndex, false, err
}
if overlappingMessagesTerm != msgsTerm {
// We need to truncate from here to the end of the log.
clog.node_log("Truncating log due to mismatching terms to index %v.\n", checkMessagesIndex)
clog.log.TruncateMessages(int64(checkMessagesIndex))
break
}
// If we get this far then we have an overlapping message
overlapMessageCount++
}
}
} else {
if previousIndex != 0 {
clog.node_log("Empty log, but previousIndex of %v given by leader\n", previousIndex)
return nextIndex, false, nil
}
// We have our first messages - use AppendFirstMessages to set the index correctly (first message isn't always 1)
if msgs.GetCount() > 0 {
//clog.node_log("Attempting to append %v messages\n", len(msgsToAppend))
lastID, err := clog.log.AppendFirstMessages(msgs, leaderFirstIndex)
if err != nil {
clog.node_log("Error attempting to Append messages to the log: %v\n", err)
return nextIndex, false, nil
} else if clog.syncPolicy == WRITE_SYNC {
err = clog.log.Sync()
if err != nil {
clog.node_log("Error attempting to sync Append messages to the log: %v\n", err)
return nextIndex, false, nil
}
}
nextIndex = lastID + 1
return nextIndex, true, nil
}
}
// All messages beyond overlapMessageCount should now be appended.
msgsToAppend, err := msgs.Slice(overlapMessageCount, msgs.GetCount())
if err != nil {
clog.node_log("Error slicing overlapping messages: %v\n", err)
return nextIndex, false, nil
}
if msgsToAppend.GetCount() > 0 {
//clog.node_log("Attempting to append %v messages\n", len(msgsToAppend))
lastID, err := clog.log.AppendMessages(msgsToAppend)
if err != nil {
clog.node_log("Error attempting to Append messages to the log: %v\n", err)
return nextIndex, false, nil
} else if clog.syncPolicy == WRITE_SYNC {
err = clog.log.Sync()
if err != nil {
clog.node_log("Error attempting to sync Append messages to the log: %v\n", err)
return nextIndex, false, nil
}
}
nextIndex = lastID + 1
} else {
//clog.node_log("Call to append, but not message to add.\n")
}
return nextIndex, true, nil
}
