func (c *monitorTestProjectorClient) MutationTopicRequest(topic, endpointType string,
reqTimestamps []*protobuf.TsVbuuid, instances []*protobuf.Instance) (*protobuf.TopicResponse, error) {
logging.Infof("monitorTestProjectorClient.MutationTopicRequest(): start")
if len(reqTimestamps) == 0 {
util.TT.Fatal("testProjectorClient.MutationTopicRequest(): reqTimestamps is nil")
}
response := new(protobuf.TopicResponse)
response.Topic = &topic
response.InstanceIds = make([]uint64, len(instances))
for i, inst := range instances {
response.InstanceIds[i] = inst.GetIndexInstance().GetInstId()
}
response.ActiveTimestamps = reqTimestamps
if reqTimestamps[0].GetSeqnos()[10] != 406 {
response.RollbackTimestamps = make([]*protobuf.TsVbuuid, 1)
response.RollbackTimestamps[0] = protobuf.NewTsVbuuid(manager.DEFAULT_POOL_NAME, reqTimestamps[0].GetBucket(), manager.NUM_VB)
response.RollbackTimestamps[0].Append(uint16(10), uint64(406), reqTimestamps[0].Vbuuids[10], 0, 0)
response.Err = protobuf.NewError(projectorC.ErrorStreamRequest)
return response, projectorC.ErrorStreamRequest
} else {
response.RollbackTimestamps = nil
response.Err = nil
return response, nil
}
}
func updateActiveTsFromResponse(bucket string,
activeTs *protobuf.TsVbuuid, res *protobuf.TopicResponse) *protobuf.TsVbuuid {
activeTsList := res.GetActiveTimestamps()
for _, ts := range activeTsList {
if ts != nil && !ts.IsEmpty() && ts.GetBucket() == bucket {
if activeTs == nil {
activeTs = ts.Clone()
} else {
activeTs = activeTs.Union(ts)
}
}
}
return activeTs
}
func updateRollbackTsFromResponse(bucket string,
rollbackTs *protobuf.TsVbuuid, res *protobuf.TopicResponse) *protobuf.TsVbuuid {
rollbackTsList := res.GetRollbackTimestamps()
for _, ts := range rollbackTsList {
if ts != nil && !ts.IsEmpty() && ts.GetBucket() == bucket {
if rollbackTs == nil {
rollbackTs = ts.Clone()
} else {
rollbackTs = rollbackTs.Union(ts)
}
}
}
return rollbackTs
}
//
// Handle error for adding instance. The following error can be returned from projector:
// 1) Unconditional Recoverable error by worker
// * generic http error
// * ErrorStreamRequest
// * ErrorResposneTimeout
// * ErrorFeeder
// 2) Non Recoverable error
// * ErrorInconsistentFeed
// 3) Recoverable error by other worker
// * ErrorInvalidVbucketBranch
// * ErrorNotMyVbucket
// * ErrorInvalidKVaddrs
// 4) Error that may not need retry
// * ErrorTopicExist
//
func (worker *adminWorker) shouldRetryAddInstances(requestTs []*protobuf.TsVbuuid,
response *protobuf.TopicResponse,
err error) ([]*protobuf.TsVbuuid, error) {
logging.Debugf("adminWorker::shouldRetryAddInstances(): start")
// First of all, let's check for any non-recoverable error.
errStr := err.Error()
logging.Debugf("adminWorker::shouldRetryAddInstances(): Error encountered when calling MutationTopicRequest. Error=%v", errStr)
if strings.Contains(errStr, projectorC.ErrorTopicExist.Error()) {
// TODO: Need pratap to define the semantic of ErrorTopExist. Right now return as an non-recoverable error.
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInconsistentFeed.Error()) {
// This is fatal error. Should only happen due to coding error. Need to return this error.
// For those projectors that have already been opened, let's leave it open. Eventually those
// projectors will fill up the buffer and terminate the connection by itself.
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorNotMyVbucket.Error()) {
return nil, NewError(ERROR_STREAM_WRONG_VBUCKET, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidVbucketBranch.Error()) {
return nil, NewError(ERROR_STREAM_INVALID_TIMESTAMP, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidKVaddrs.Error()) {
return nil, NewError(ERROR_STREAM_INVALID_KVADDRS, NORMAL, STREAM, err, "")
}
// There is no non-recoverable error, so we can retry. For retry, recompute the new set of timestamps based on the response.
rollbackTimestamps := response.GetRollbackTimestamps()
var newRequestTs []*protobuf.TsVbuuid = nil
for _, ts := range requestTs {
ts = recomputeRequestTimestamp(ts, rollbackTimestamps)
newRequestTs = append(newRequestTs, ts)
}
return newRequestTs, nil
}
//
// Handle error for restart vbuckets. The following error can be returned from projector:
// 1) Unconditional Recoverable error by worker
// * generic http error
// * ErrorStreamRequest
// * ErrorResposneTimeout
// 2) Non Recoverable error
// * ErrorTopicMissing
// * ErrorInvalidBucket
// 3) Recoverable error by other worker
// * ErrorInvalidVbucketBranch
// * ErrorNotMyVbucket
// * ErrorFeeder
// * ErrorStreamEnd
//
func (worker *adminWorker) shouldRetryRestartVbuckets(requestTs []*protobuf.TsVbuuid,
response *protobuf.TopicResponse,
err error) ([]*protobuf.TsVbuuid, error) {
logging.Debugf("adminWorker::shouldRetryRestartVbuckets(): start")
// First of all, let's check for any non-recoverable error.
errStr := err.Error()
logging.Debugf("adminWorker::shouldRetryRestartVbuckets(): Error encountered when calling RestartVbuckets. Error=%v", errStr)
if strings.Contains(errStr, projectorC.ErrorTopicMissing.Error()) {
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidBucket.Error()) {
return nil, NewError(ERROR_STREAM_REQUEST_ERROR, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorFeeder.Error()) {
return nil, NewError(ERROR_STREAM_FEEDER, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorNotMyVbucket.Error()) {
return nil, NewError(ERROR_STREAM_WRONG_VBUCKET, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorInvalidVbucketBranch.Error()) {
return nil, NewError(ERROR_STREAM_INVALID_TIMESTAMP, NORMAL, STREAM, err, "")
} else if strings.Contains(errStr, projectorC.ErrorStreamEnd.Error()) {
return nil, NewError(ERROR_STREAM_STREAM_END, NORMAL, STREAM, err, "")
}
// There is no non-recoverable error, so we can retry. For retry, recompute the new set of timestamps based on the response.
rollbackTimestamps := response.GetRollbackTimestamps()
var newRequestTs []*protobuf.TsVbuuid = nil
for _, ts := range requestTs {
ts = recomputeRequestTimestamp(ts, rollbackTimestamps)
newRequestTs = append(newRequestTs, ts)
}
return newRequestTs, nil
}
func (c *streamEndTestProjectorClient) RestartVbuckets(topic string,
restartTimestamps []*protobuf.TsVbuuid) (*protobuf.TopicResponse, error) {
c.sendSync(restartTimestamps)
response := new(protobuf.TopicResponse)
response.Topic = &topic
response.InstanceIds = nil
response.ActiveTimestamps = make([]*protobuf.TsVbuuid, 1)
response.ActiveTimestamps[0] = restartTimestamps[0]
response.RollbackTimestamps = nil
response.Err = nil
return response, nil
}
func (c *deleteTestProjectorClient) MutationTopicRequest(topic, endpointType string,
reqTimestamps []*protobuf.TsVbuuid, instances []*protobuf.Instance) (*protobuf.TopicResponse, error) {
if len(reqTimestamps) == 0 {
util.TT.Fatal("deleteTestProjectorClient.MutationTopicRequest(): reqTimestamps is nil")
}
for _, inst := range instances {
delete_test_status[inst.GetIndexInstance().GetInstId()] = inst
}
response := new(protobuf.TopicResponse)
response.Topic = &topic
response.InstanceIds = make([]uint64, len(instances))
for i, inst := range instances {
response.InstanceIds[i] = inst.GetIndexInstance().GetInstId()
}
response.ActiveTimestamps = nil
for _, ts := range reqTimestamps {
newTs := protobuf.NewTsVbuuid("default", ts.GetBucket(), manager.NUM_VB)
if c.server == "127.0.0.1" {
for i := 0; i < manager.NUM_VB/2; i++ {
newTs.Append(uint16(i), uint64(i), uint64(1234), uint64(0), uint64(0))
}
}
if c.server == "127.0.0.2" {
for i := manager.NUM_VB / 2; i < manager.NUM_VB; i++ {
newTs.Append(uint16(i), uint64(i), uint64(1234), uint64(0), uint64(0))
}
}
response.ActiveTimestamps = append(response.ActiveTimestamps, newTs)
}
response.RollbackTimestamps = nil
response.Err = nil
return response, nil
}
func (c *timerTestProjectorClient) MutationTopicRequest(topic, endpointType string,
reqTimestamps []*protobuf.TsVbuuid, instances []*protobuf.Instance) (*protobuf.TopicResponse, error) {
if len(reqTimestamps) == 0 {
util.TT.Fatal("timerTestProjectorClient.MutationTopicRequest(): reqTimestamps is nil")
}
c.sendSync(instances)
response := new(protobuf.TopicResponse)
response.Topic = &topic
response.InstanceIds = make([]uint64, len(instances))
for i, inst := range instances {
response.InstanceIds[i] = inst.GetIndexInstance().GetInstId()
}
response.ActiveTimestamps = reqTimestamps
response.RollbackTimestamps = nil
response.Err = nil
return response, nil
}