Example #1
0
//CreateMutationStreamReader creates a new mutation stream and starts
//a reader to listen and process the mutations.
//In case returned MutationStreamReader is nil, Message will have the error msg.
func CreateMutationStreamReader(streamId common.StreamId, bucketQueueMap BucketQueueMap,
	bucketFilter map[string]*common.TsVbuuid, supvCmdch MsgChannel, supvRespch MsgChannel,
	numWorkers int, stats *IndexerStats) (MutationStreamReader, Message) {

	//start a new mutation stream
	streamMutch := make(chan interface{}, DATAPORT_MUTATION_BUFFER)
	config := common.SystemConfig.SectionConfig(
		"indexer.dataport.", true /*trim*/)
	stream, err := dataport.NewServer(
		string(StreamAddrMap[streamId]),
		common.SystemConfig["maxVbuckets"].Int(),
		config, streamMutch)
	if err != nil {
		//return stream init error
		logging.Errorf("MutationStreamReader: Error returned from NewServer."+
			"StreamId: %v, Err: %v", streamId, err)

		msgErr := &MsgError{
			err: Error{code: ERROR_STREAM_INIT,
				severity: FATAL,
				category: STREAM_READER,
				cause:    err}}
		return nil, msgErr
	}

	//init the reader
	r := &mutationStreamReader{streamId: streamId,
		stream:          stream,
		streamMutch:     streamMutch,
		supvCmdch:       supvCmdch,
		supvRespch:      supvRespch,
		numWorkers:      numWorkers,
		workerch:        make([]MutationChannel, numWorkers),
		workerStopCh:    make([]StopChannel, numWorkers),
		syncStopCh:      make(StopChannel),
		bucketQueueMap:  CopyBucketQueueMap(bucketQueueMap),
		bucketFilterMap: make(map[string]*common.TsVbuuid),
		bucketSyncDue:   make(map[string]bool),
	}

	r.stats.Set(stats)
	r.initBucketFilter(bucketFilter)

	//start the main reader loop
	go r.run()

	go r.syncWorker()

	//init worker buffers
	for w := 0; w < r.numWorkers; w++ {
		r.workerch[w] = make(MutationChannel, MAX_STREAM_READER_WORKER_BUFFER)
	}

	//start stream workers
	r.startWorkers()

	return r, &MsgSuccess{}
}
Example #2
0
func (s *Stream) start() (err error) {
	config := common.SystemConfig.SectionConfig("indexer.dataport.", true)
	maxvbs := common.SystemConfig["maxVbuckets"].Int()
	s.receiver, err = dataport.NewServer(s.hostStr, maxvbs, config, s.mutch)
	if err != nil {
		logging.Errorf("StreamManager: Error returned from dataport.NewServer = %s.", err.Error())
		close(s.stopch)
		return err
	}
	logging.Debugf("Stream.run(): dataport server started on addr %s", s.hostStr)

	// Start the listening go-routine.
	go s.run()

	// start dataport stream
	return nil
}