func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
allTarballsPattern := filepath.Join(tarballsPath, "all", "health", "*", "*", "health_*.tar.gz")
dailyTarballsPattern := filepath.Join(tarballsPath, "by-date", "*", "health", "*", "health_*.tar.gz")
tarnamesStore := levelDbManager.ReadingWriter("tarnames")
tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
logsStore := levelDbManager.Writer("logs")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ScanLogTarballs",
Reader: store.NewGlobReader(allTarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "ScanDailyLogTarballs",
Reader: store.NewGlobReader(dailyTarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "ReadLogTarballs",
Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
Writer: store.NewMuxingWriter(logsStore, tarnamesIndexedStore),
},
}
}
func ExampleJoin_inner() {
left := store.SliceStore{}
left.BeginWriting()
left.WriteRecord(store.NewRecord("1", "news", 0))
left.WriteRecord(store.NewRecord("2", "book", 0))
left.EndWriting()
right := store.SliceStore{}
right.BeginWriting()
right.WriteRecord(store.NewRecord("1", "paper", 0))
right.WriteRecord(store.NewRecord("3", "brush", 0))
right.EndWriting()
output := store.SliceStore{}
joiner := Join(nil, nil)
transformer.RunTransformer(transformer.MakeGroupDoFunc(joiner), store.NewDemuxingReader(&left, &right), &output)
output.BeginReading()
for {
record, err := output.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
fmt.Printf("%s: %s\n", record.Key, record.Value)
}
output.EndReading()
// Output:
//
// 1: newspaper
}
func FilterSessionsPipeline(sessionStartTime, sessionEndTime int64, levelDbManager store.Manager, outputName string) transformer.Pipeline {
tracesStore := levelDbManager.Reader("traces")
traceKeyRangesStore := levelDbManager.Reader("availability-done")
filteredStore := levelDbManager.Writer(outputName)
parameters := filterSessions{
SessionStartTime: sessionStartTime * 1000000,
SessionEndTime: sessionEndTime * 1000000,
}
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "FilterSessions",
Reader: store.NewDemuxingReader(traceKeyRangesStore, tracesStore),
Transformer: parameters,
Writer: filteredStore,
},
}
}
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
tarballsPattern := filepath.Join(tarballsPath, "*", "*", "*.tar.gz")
tarnamesStore := levelDbManager.ReadingWriter("tarnames")
tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
tracesStore := levelDbManager.Writer("traces")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ScanTraceTarballs",
Reader: store.NewGlobReader(tarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "IndexTraces",
Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
Writer: store.NewMuxingWriter(tracesStore, tarnamesIndexedStore),
},
}
}
func ExampleJoin_multiple() {
left := store.SliceStore{}
left.BeginWriting()
left.WriteRecord(store.NewRecord("1", "what", 0))
left.WriteRecord(store.NewRecord("2", "news", 0))
left.EndWriting()
middle := store.SliceStore{}
middle.BeginWriting()
middle.WriteRecord(store.NewRecord("1", "so", 0))
middle.WriteRecord(store.NewRecord("3", "the", 0))
middle.EndWriting()
right := store.SliceStore{}
right.BeginWriting()
right.WriteRecord(store.NewRecord("1", "ever", 0))
right.WriteRecord(store.NewRecord("3", "less", 0))
right.EndWriting()
output := store.SliceStore{}
joiner := Join([]byte("never"), []byte("paper"), []byte("man"))
transformer.RunTransformer(transformer.MakeGroupDoFunc(joiner), store.NewDemuxingReader(&left, &middle, &right), &output)
output.BeginReading()
for {
record, err := output.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
fmt.Printf("%s: %s\n", record.Key, record.Value)
}
output.EndReading()
// Output:
//
// 1: whatsoever
// 2: newspaperman
// 3: nevertheless
}
