func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline { allTarballsPattern := filepath.Join(tarballsPath, "all", "health", "*", "*", "health_*.tar.gz") dailyTarballsPattern := filepath.Join(tarballsPath, "by-date", "*", "health", "*", "health_*.tar.gz") tarnamesStore := levelDbManager.ReadingWriter("tarnames") tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed") logsStore := levelDbManager.Writer("logs") return []transformer.PipelineStage{ transformer.PipelineStage{ Name: "ScanLogTarballs", Reader: store.NewGlobReader(allTarballsPattern), Writer: tarnamesStore, }, transformer.PipelineStage{ Name: "ScanDailyLogTarballs", Reader: store.NewGlobReader(dailyTarballsPattern), Writer: tarnamesStore, }, transformer.PipelineStage{ Name: "ReadLogTarballs", Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore), Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2), Writer: store.NewMuxingWriter(logsStore, tarnamesIndexedStore), }, } }
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline { tarballsPattern := filepath.Join(tarballsPath, "*", "*", "*.tar.gz") tarnamesStore := levelDbManager.ReadingWriter("tarnames") tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed") tracesStore := levelDbManager.Writer("traces") return []transformer.PipelineStage{ transformer.PipelineStage{ Name: "ScanTraceTarballs", Reader: store.NewGlobReader(tarballsPattern), Writer: tarnamesStore, }, transformer.PipelineStage{ Name: "IndexTraces", Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2), Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore), Writer: store.NewMuxingWriter(tracesStore, tarnamesIndexedStore), }, } }