func ExampleNewBranchingPipeline() { logger.LogLevel = logger.LevelSilent // This example is very contrived, but we'll first create // DataProcessors that will spit out strings, do some basic // transformation, and then filter out all the ones that don't // match "HELLO". hello := processors.NewIoReader(strings.NewReader("Hello world")) hola := processors.NewIoReader(strings.NewReader("Hola mundo")) bonjour := processors.NewIoReader(strings.NewReader("Bonjour monde")) upperCaser := processors.NewFuncTransformer(func(d data.JSON) data.JSON { return data.JSON(strings.ToUpper(string(d))) }) lowerCaser := processors.NewFuncTransformer(func(d data.JSON) data.JSON { return data.JSON(strings.ToLower(string(d))) }) helloMatcher := processors.NewRegexpMatcher("HELLO") stdout := processors.NewIoWriter(os.Stdout) // Create the PipelineLayout that will run the DataProcessors layout, err := ratchet.NewPipelineLayout( // Stage 1 - spits out hello world in a few languages ratchet.NewPipelineStage( ratchet.Do(hello).Outputs(upperCaser, lowerCaser), ratchet.Do(hola).Outputs(upperCaser), ratchet.Do(bonjour).Outputs(lowerCaser), ), // Stage 2 - transforms strings to upper and lower case ratchet.NewPipelineStage( ratchet.Do(upperCaser).Outputs(helloMatcher), ratchet.Do(lowerCaser).Outputs(helloMatcher), ), // Stage 3 - only lets through strings that match "hello" ratchet.NewPipelineStage( ratchet.Do(helloMatcher).Outputs(stdout), ), // Stage 4 - prints to STDOUT ratchet.NewPipelineStage( ratchet.Do(stdout), ), ) if err != nil { panic(err.Error()) } // Create and run the Pipeline pipeline := ratchet.NewBranchingPipeline(layout) err = <-pipeline.Run() if err != nil { fmt.Println("An error occurred in the ratchet pipeline:", err.Error()) } // Output: // HELLO WORLD }
// Run finalizes the channel connections between PipelineStages // and kicks off execution. // Run will return a killChan that should be waited on so your calling function doesn't // return prematurely. Any stage of the pipeline can send to the killChan to halt // execution. Your calling function should check if the sent value is an error or nil to know if // execution was a failure or a success (nil being the success value). func (p *Pipeline) Run() (killChan chan error) { p.timer = util.StartTimer() killChan = make(chan error) p.connectStages() p.runStages(killChan) for _, dp := range p.layout.stages[0].processors { logger.Debug(p.Name, ": sending", StartSignal, "to", dp) dp.inputChan <- data.JSON(StartSignal) dp.Finish(dp.outputChan, killChan) close(dp.inputChan) } // After all the stages are running, send the StartSignal // to the initial stage processors to kick off execution, and // then wait until all the processing goroutines are done to // signal successful pipeline completion. go func() { p.wg.Wait() p.timer.Stop() killChan <- nil }() handleInterrupt(killChan) return killChan }
func (r *IoReader) scanLines(killChan chan error, forEach func(d data.JSON)) { scanner := bufio.NewScanner(r.Reader) for scanner.Scan() { forEach(data.JSON(scanner.Text())) } err := scanner.Err() util.KillPipelineIfErr(err, killChan) }
func ExampleGetRequest() { logger.LogLevel = logger.LevelSilent getGoogle, err := processors.NewHTTPRequest("GET", "http://www.google.com", nil) if err != nil { panic(err) } // this is just a really basic checking function so we can have // determinable example output. checkHTML := processors.NewFuncTransformer(func(d data.JSON) data.JSON { output := "Got HTML?\n" if strings.Contains(strings.ToLower(string(d)), "html") { output += "YES\n" } else { output += "NO\n" } output += "HTML contains Google Search?\n" if strings.Contains(string(d), "Google Search") { output += "YES\n" } else { output += "NO\n" } return data.JSON(output) }) stdout := processors.NewIoWriter(os.Stdout) pipeline := ratchet.NewPipeline(getGoogle, checkHTML, stdout) err = <-pipeline.Run() if err != nil { fmt.Println("An error occurred in the ratchet pipeline:", err.Error()) } // Output: // Got HTML? // YES // HTML contains Google Search? // YES }