Example #1
0
// Read records from reader, pass them to transformer and read results from
// transformer and write them to writer. Do not exit until all records have been
// processed. Running transformers is a fundamental data processing operation,
// but you should almost never run this function directly. Instead, use
// RunPipeline to run a series of pipeline stages.
func RunTransformer(transformer Transformer, reader store.Reader, writer store.Writer) {
	inputChan := make(chan *store.Record)
	outputChan := make(chan *store.Record)

	if reader != nil {
		go func() {
			if err := reader.BeginReading(); err != nil {
				panic(err)
			}
			for {
				record, err := reader.ReadRecord()
				if err != nil {
					panic(err)
				}
				if record == nil {
					break
				}
				inputChan <- record
			}
			if err := reader.EndReading(); err != nil {
				panic(err)
			}
			close(inputChan)
		}()
	}

	transformerDone := make(chan bool)
	if transformer != nil {
		go func() {
			transformer.Do(inputChan, outputChan)
			close(outputChan)
			transformerDone <- true
		}()
	} else {
		outputChan = inputChan
		go func() {
			transformerDone <- true
		}()
	}

	if writer != nil {
		if err := writer.BeginWriting(); err != nil {
			panic(err)
		}
		for record := range outputChan {
			if err := writer.WriteRecord(record); err != nil {
				panic(err)
			}
		}
		if err := writer.EndWriting(); err != nil {
			panic(err)
		}
	}

	<-transformerDone
}
func runIncludeNodes(records store.Reader) {
	records.BeginReading()
	for {
		record, err := records.ReadRecord()
		if err != nil {
			panic(err)
		}
		if record == nil {
			break
		}
		var traceKey TraceKey
		lex.DecodeOrDie(record.Key, &traceKey)
		fmt.Printf("%s %d\n", traceKey.NodeId, traceKey.SequenceNumber)
	}
	records.EndReading()
}