func ExampleNewBranchingPipeline() {
logger.LogLevel = logger.LevelSilent
// This example is very contrived, but we'll first create
// DataProcessors that will spit out strings, do some basic
// transformation, and then filter out all the ones that don't
// match "HELLO".
hello := processors.NewIoReader(strings.NewReader("Hello world"))
hola := processors.NewIoReader(strings.NewReader("Hola mundo"))
bonjour := processors.NewIoReader(strings.NewReader("Bonjour monde"))
upperCaser := processors.NewFuncTransformer(func(d data.JSON) data.JSON {
return data.JSON(strings.ToUpper(string(d)))
})
lowerCaser := processors.NewFuncTransformer(func(d data.JSON) data.JSON {
return data.JSON(strings.ToLower(string(d)))
})
helloMatcher := processors.NewRegexpMatcher("HELLO")
stdout := processors.NewIoWriter(os.Stdout)
// Create the PipelineLayout that will run the DataProcessors
layout, err := ratchet.NewPipelineLayout(
// Stage 1 - spits out hello world in a few languages
ratchet.NewPipelineStage(
ratchet.Do(hello).Outputs(upperCaser, lowerCaser),
ratchet.Do(hola).Outputs(upperCaser),
ratchet.Do(bonjour).Outputs(lowerCaser),
),
// Stage 2 - transforms strings to upper and lower case
ratchet.NewPipelineStage(
ratchet.Do(upperCaser).Outputs(helloMatcher),
ratchet.Do(lowerCaser).Outputs(helloMatcher),
),
// Stage 3 - only lets through strings that match "hello"
ratchet.NewPipelineStage(
ratchet.Do(helloMatcher).Outputs(stdout),
),
// Stage 4 - prints to STDOUT
ratchet.NewPipelineStage(
ratchet.Do(stdout),
),
)
if err != nil {
panic(err.Error())
}
// Create and run the Pipeline
pipeline := ratchet.NewBranchingPipeline(layout)
err = <-pipeline.Run()
if err != nil {
fmt.Println("An error occurred in the ratchet pipeline:", err.Error())
}
// Output:
// HELLO WORLD
}
// Run finalizes the channel connections between PipelineStages
// and kicks off execution.
// Run will return a killChan that should be waited on so your calling function doesn't
// return prematurely. Any stage of the pipeline can send to the killChan to halt
// execution. Your calling function should check if the sent value is an error or nil to know if
// execution was a failure or a success (nil being the success value).
func (p *Pipeline) Run() (killChan chan error) {
p.timer = util.StartTimer()
killChan = make(chan error)
p.connectStages()
p.runStages(killChan)
for _, dp := range p.layout.stages[0].processors {
logger.Debug(p.Name, ": sending", StartSignal, "to", dp)
dp.inputChan <- data.JSON(StartSignal)
dp.Finish(dp.outputChan, killChan)
close(dp.inputChan)
}
// After all the stages are running, send the StartSignal
// to the initial stage processors to kick off execution, and
// then wait until all the processing goroutines are done to
// signal successful pipeline completion.
go func() {
p.wg.Wait()
p.timer.Stop()
killChan <- nil
}()
handleInterrupt(killChan)
return killChan
}
func (r *IoReader) scanLines(killChan chan error, forEach func(d data.JSON)) {
scanner := bufio.NewScanner(r.Reader)
for scanner.Scan() {
forEach(data.JSON(scanner.Text()))
}
err := scanner.Err()
util.KillPipelineIfErr(err, killChan)
}
func ExampleGetRequest() {
logger.LogLevel = logger.LevelSilent
getGoogle, err := processors.NewHTTPRequest("GET", "http://www.google.com", nil)
if err != nil {
panic(err)
}
// this is just a really basic checking function so we can have
// determinable example output.
checkHTML := processors.NewFuncTransformer(func(d data.JSON) data.JSON {
output := "Got HTML?\n"
if strings.Contains(strings.ToLower(string(d)), "html") {
output += "YES\n"
} else {
output += "NO\n"
}
output += "HTML contains Google Search?\n"
if strings.Contains(string(d), "Google Search") {
output += "YES\n"
} else {
output += "NO\n"
}
return data.JSON(output)
})
stdout := processors.NewIoWriter(os.Stdout)
pipeline := ratchet.NewPipeline(getGoogle, checkHTML, stdout)
err = <-pipeline.Run()
if err != nil {
fmt.Println("An error occurred in the ratchet pipeline:", err.Error())
}
// Output:
// Got HTML?
// YES
// HTML contains Google Search?
// YES
}