func (k *kvSender) addIndexForExistingBucket(streamId c.StreamId, bucket string, indexInstList []c.IndexInst, respCh MsgChannel, stopCh StopChannel) { addrs, err := k.getAllProjectorAddrs() if err != nil { logging.Errorf("KVSender::addIndexForExistingBucket %v %v Error in fetching cluster info %v", streamId, bucket, err) respCh <- &MsgError{ err: Error{code: ERROR_KVSENDER_STREAM_REQUEST_ERROR, severity: FATAL, cause: err}} return } var currentTs *protobuf.TsVbuuid protoInstList := convertIndexListToProto(k.config, k.cInfoCache, indexInstList, streamId) topic := getTopicForStreamId(streamId) fn := func(r int, err error) error { //clear the error before every retry err = nil for _, addr := range addrs { execWithStopCh(func() { ap := newProjClient(addr) if res, ret := sendAddInstancesRequest(ap, topic, protoInstList); ret != nil { logging.Errorf("KVSender::addIndexForExistingBucket %v %v Error Received %v from %v", streamId, bucket, ret, addr) err = ret } else { currentTs = updateCurrentTsFromResponse(bucket, currentTs, res) } }, stopCh) } //check if we have received currentTs for all vbuckets numVbuckets := k.config["numVbuckets"].Int() if currentTs == nil || currentTs.Len() != numVbuckets { return errors.New("ErrPartialVbStart") } else { return err } } rh := c.NewRetryHelper(MAX_KV_REQUEST_RETRY, time.Second, BACKOFF_FACTOR, fn) err = rh.Run() if err != nil { logging.Errorf("KVSender::addIndexForExistingBucket %v %v Error Received %v", streamId, bucket, err) respCh <- &MsgError{ err: Error{code: ERROR_KVSENDER_STREAM_REQUEST_ERROR, severity: FATAL, cause: err}} return } numVbuckets := k.config["numVbuckets"].Int() nativeTs := currentTs.ToTsVbuuid(numVbuckets) respCh <- &MsgStreamUpdate{mType: MSG_SUCCESS, streamId: streamId, bucket: bucket, restartTs: nativeTs} }
func (k *kvSender) openMutationStream(streamId c.StreamId, indexInstList []c.IndexInst, restartTs *c.TsVbuuid, respCh MsgChannel, stopCh StopChannel) { if len(indexInstList) == 0 { logging.Warnf("KVSender::openMutationStream Empty IndexList. Nothing to do.") respCh <- &MsgSuccess{} return } protoInstList := convertIndexListToProto(k.config, k.cInfoCache, indexInstList, streamId) bucket := indexInstList[0].Defn.Bucket //use any bucket as list of vbs remain the same for all buckets vbnos, err := k.getAllVbucketsInCluster(bucket) if err != nil { logging.Errorf("KVSender::openMutationStream %v %v Error in fetching vbuckets info %v", streamId, restartTs.Bucket, err) respCh <- &MsgError{ err: Error{code: ERROR_KVSENDER_STREAM_REQUEST_ERROR, severity: FATAL, cause: err}} return } restartTsList, err := k.makeRestartTsForVbs(bucket, restartTs, vbnos) if err != nil { logging.Errorf("KVSender::openMutationStream %v %v Error making restart ts %v", streamId, bucket, err) respCh <- &MsgError{ err: Error{code: ERROR_KVSENDER_STREAM_REQUEST_ERROR, severity: FATAL, cause: err}} return } addrs, err := k.getAllProjectorAddrs() if err != nil { logging.Errorf("KVSender::openMutationStream %v %v Error Fetching Projector Addrs %v", streamId, bucket, err) respCh <- &MsgError{ err: Error{code: ERROR_KVSENDER_STREAM_REQUEST_ERROR, severity: FATAL, cause: err}} return } var rollbackTs *protobuf.TsVbuuid var activeTs *protobuf.TsVbuuid topic := getTopicForStreamId(streamId) fn := func(r int, err error) error { //clear the error before every retry err = nil for _, addr := range addrs { execWithStopCh(func() { ap := newProjClient(addr) if res, ret := k.sendMutationTopicRequest(ap, topic, restartTsList, protoInstList); ret != nil { //for all errors, retry logging.Errorf("KVSender::openMutationStream %v %v Error Received %v from %v", streamId, bucket, ret, addr) err = ret } else { activeTs = updateActiveTsFromResponse(bucket, activeTs, res) rollbackTs = updateRollbackTsFromResponse(bucket, rollbackTs, res) } }, stopCh) } if rollbackTs != nil { //no retry required for rollback return nil } else if err != nil { //retry for any error return err } else { //check if we have received activeTs for all vbuckets retry := false if activeTs == nil || activeTs.Len() != len(vbnos) { retry = true } if retry { return errors.New("ErrPartialVbStart") } else { return nil } } } rh := c.NewRetryHelper(MAX_KV_REQUEST_RETRY, time.Second, BACKOFF_FACTOR, fn) err = rh.Run() if rollbackTs != nil { logging.Infof("KVSender::openMutationStream %v %v Rollback Received %v", streamId, bucket, rollbackTs) //convert from protobuf to native format numVbuckets := k.config["numVbuckets"].Int() nativeTs := rollbackTs.ToTsVbuuid(numVbuckets) respCh <- &MsgRollback{streamId: streamId, bucket: bucket, rollbackTs: nativeTs} } else if err != nil { logging.Errorf("KVSender::openMutationStream %v %v Error Received %v", streamId, bucket, err) respCh <- &MsgError{ err: Error{code: ERROR_KVSENDER_STREAM_REQUEST_ERROR, severity: FATAL, cause: err}} } else { numVbuckets := k.config["numVbuckets"].Int() respCh <- &MsgSuccessOpenStream{activeTs: activeTs.ToTsVbuuid(numVbuckets)} } }