func updateRollbackTsFromResponse(bucket string,
rollbackTs *protobuf.TsVbuuid, res *protobuf.TopicResponse) *protobuf.TsVbuuid {
rollbackTsList := res.GetRollbackTimestamps()
for _, ts := range rollbackTsList {
if ts != nil && !ts.IsEmpty() && ts.GetBucket() == bucket {
if rollbackTs == nil {
rollbackTs = ts.Clone()
} else {
rollbackTs = rollbackTs.Union(ts)
}
}
}
return rollbackTs
}
//
// Handle error for adding instance. The following error can be returned from projector:
// 1) Unconditional Recoverable error by worker
// * generic http error
// * ErrorStreamRequest
// * ErrorResposneTimeout
// * ErrorFeeder
// 2) Non Recoverable error
// * ErrorInconsistentFeed
// 3) Recoverable error by other worker
// * ErrorInvalidVbucketBranch
// * ErrorNotMyVbucket
// * ErrorInvalidKVaddrs
// 4) Error that may not need retry
// * ErrorTopicExist
//
func (worker *adminWorker) shouldRetryAddInstances(requestTs []*protobuf.TsVbuuid,
response *protobuf.TopicResponse,
err error) ([]*protobuf.TsVbuuid, error) {
logging.Debugf("adminWorker::shouldRetryAddInstances(): start")
// First of all, let's check for any non-recoverable error.
errStr := err.Error()
logging.Debugf("adminWorker::shouldRetryAddInstances(): Error encountered when calling MutationTopicRequest. Error=%v", errStr)
if strings.Contains(errStr, projectorC.ErrorTopicExist.Error()) {
// TODO: Need pratap to define the semantic of ErrorTopExist. Right now return as an non-recoverable error.
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInconsistentFeed.Error()) {
// This is fatal error. Should only happen due to coding error. Need to return this error.
// For those projectors that have already been opened, let's leave it open. Eventually those
// projectors will fill up the buffer and terminate the connection by itself.
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorNotMyVbucket.Error()) {
return nil, NewError(ERROR_STREAM_WRONG_VBUCKET, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidVbucketBranch.Error()) {
return nil, NewError(ERROR_STREAM_INVALID_TIMESTAMP, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidKVaddrs.Error()) {
return nil, NewError(ERROR_STREAM_INVALID_KVADDRS, NORMAL, STREAM, err, "")
}
// There is no non-recoverable error, so we can retry. For retry, recompute the new set of timestamps based on the response.
rollbackTimestamps := response.GetRollbackTimestamps()
var newRequestTs []*protobuf.TsVbuuid = nil
for _, ts := range requestTs {
ts = recomputeRequestTimestamp(ts, rollbackTimestamps)
newRequestTs = append(newRequestTs, ts)
}
return newRequestTs, nil
}
//
// Handle error for restart vbuckets. The following error can be returned from projector:
// 1) Unconditional Recoverable error by worker
// * generic http error
// * ErrorStreamRequest
// * ErrorResposneTimeout
// 2) Non Recoverable error
// * ErrorTopicMissing
// * ErrorInvalidBucket
// 3) Recoverable error by other worker
// * ErrorInvalidVbucketBranch
// * ErrorNotMyVbucket
// * ErrorFeeder
// * ErrorStreamEnd
//
func (worker *adminWorker) shouldRetryRestartVbuckets(requestTs []*protobuf.TsVbuuid,
response *protobuf.TopicResponse,
err error) ([]*protobuf.TsVbuuid, error) {
logging.Debugf("adminWorker::shouldRetryRestartVbuckets(): start")
// First of all, let's check for any non-recoverable error.
errStr := err.Error()
logging.Debugf("adminWorker::shouldRetryRestartVbuckets(): Error encountered when calling RestartVbuckets. Error=%v", errStr)
if strings.Contains(errStr, projectorC.ErrorTopicMissing.Error()) {
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidBucket.Error()) {
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorFeeder.Error()) {
return nil, NewError(ERROR_STREAM_FEEDER, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorNotMyVbucket.Error()) {
return nil, NewError(ERROR_STREAM_WRONG_VBUCKET, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidVbucketBranch.Error()) {
return nil, NewError(ERROR_STREAM_INVALID_TIMESTAMP, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorStreamEnd.Error()) {
return nil, NewError(ERROR_STREAM_STREAM_END, NORMAL, STREAM, err, "")
}
// There is no non-recoverable error, so we can retry. For retry, recompute the new set of timestamps based on the response.
rollbackTimestamps := response.GetRollbackTimestamps()
var newRequestTs []*protobuf.TsVbuuid = nil
for _, ts := range requestTs {
ts = recomputeRequestTimestamp(ts, rollbackTimestamps)
newRequestTs = append(newRequestTs, ts)
}
return newRequestTs, nil
}
