func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
allTarballsPattern := filepath.Join(tarballsPath, "all", "health", "*", "*", "health_*.tar.gz")
dailyTarballsPattern := filepath.Join(tarballsPath, "by-date", "*", "health", "*", "health_*.tar.gz")
tarnamesStore := levelDbManager.ReadingWriter("tarnames")
tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
logsStore := levelDbManager.Writer("logs")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ScanLogTarballs",
Reader: store.NewGlobReader(allTarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "ScanDailyLogTarballs",
Reader: store.NewGlobReader(dailyTarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "ReadLogTarballs",
Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
Writer: store.NewMuxingWriter(logsStore, tarnamesIndexedStore),
},
}
}
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
tarballsPattern := filepath.Join(tarballsPath, "*", "*", "*.tar.gz")
tarnamesStore := levelDbManager.ReadingWriter("tarnames")
tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
tracesStore := levelDbManager.Writer("traces")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ScanTraceTarballs",
Reader: store.NewGlobReader(tarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "IndexTraces",
Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
Writer: store.NewMuxingWriter(tracesStore, tarnamesIndexedStore),
},
}
}