func SessionPipelineStage(inputStore store.Reader, sessionsStore store.Deleter) transformer.PipelineStage { return transformer.PipelineStage{ Name: "Sessions", Transformer: transformer.TransformFunc(sessions), Reader: inputStore, Writer: store.NewTruncatingWriter(sessionsStore), } }
func AvailabilityPipeline(levelDbManager store.Manager, jsonWriter io.Writer, timestamp int64) transformer.Pipeline { tracesStore := levelDbManager.Seeker("traces") intervalsStore := levelDbManager.ReadingWriter("availability-intervals") consolidatedStore := levelDbManager.ReadingDeleter("availability-consolidated") nodesStore := levelDbManager.ReadingDeleter("availability-nodes") excludeRangesStore := levelDbManager.ReadingDeleter("availability-done") consistentRangesStore := levelDbManager.ReadingDeleter("consistent-ranges") return []transformer.PipelineStage{ transformer.PipelineStage{ Name: "AvailabilityIntervals", Reader: store.NewRangeExcludingReader(tracesStore, excludeRangesStore), Transformer: transformer.TransformFunc(availabilityIntervals), Writer: intervalsStore, }, transformer.PipelineStage{ Name: "ConsolidateAvailabilityIntervals", Reader: intervalsStore, Transformer: transformer.TransformFunc(consolidateAvailabilityIntervals), Writer: store.NewTruncatingWriter(consolidatedStore), }, transformer.PipelineStage{ Name: "AvailabilityReducer", Reader: consolidatedStore, Transformer: transformer.TransformFunc(availabilityReducer), Writer: store.NewTruncatingWriter(nodesStore), }, transformer.PipelineStage{ Name: "AvailabilityJson", Reader: nodesStore, Writer: &availabilityJsonStore{writer: jsonWriter, timestamp: timestamp}, }, transformer.PipelineStage{ Name: "GenerateExcludedRanges", Reader: consolidatedStore, Transformer: transformer.MakeMapFunc(generateExcludedRanges), Writer: store.NewTruncatingWriter(excludeRangesStore), }, transformer.PipelineStage{ Name: "GenerateConsistentRanges", Reader: excludeRangesStore, Transformer: transformer.MakeDoFunc(generateConsistentRanges), Writer: store.NewTruncatingWriter(consistentRangesStore), }, } }
func TraceKeyRangesPipeline(newTraceKeysStore store.Reader, traceKeyRangesStore, consolidatedTraceKeyRangesStore store.ReadingDeleter) []transformer.PipelineStage { return []transformer.PipelineStage{ transformer.PipelineStage{ Name: "CalculateTraceKeyRanges", Transformer: transformer.TransformFunc(calculateTraceKeyRanges), Reader: newTraceKeysStore, Writer: traceKeyRangesStore, }, transformer.PipelineStage{ Name: "ConsolidateTraceKeyRanges", Transformer: transformer.TransformFunc(consolidateTraceKeyRanges), Reader: traceKeyRangesStore, Writer: store.NewTruncatingWriter(consolidatedTraceKeyRangesStore), }, transformer.PipelineStage{ Name: "CopyTraceKeyRanges", Reader: consolidatedTraceKeyRangesStore, Writer: store.NewTruncatingWriter(traceKeyRangesStore), }, } }
func writeAvailabilityLevelDb(allIntervals map[string][]availabilityInterval) error { rawStore := store.NewLevelDbStore(outputLevelDb, false) outputStore := store.NewTruncatingWriter(rawStore) if err := outputStore.BeginWriting(); err != nil { return err } for nodeId, intervals := range allIntervals { for _, interval := range intervals { record := store.Record{ Key: lex.EncodeOrDie(nodeId, interval.StartTime.Unix(), interval.EndTime.Unix()), } if err := outputStore.WriteRecord(&record); err != nil { return err } } } if err := outputStore.EndWriting(); err != nil { return err } return nil }
func SummarizePipeline(levelDbManager store.Manager, csvManager store.Manager) transformer.Pipeline { statsStore := levelDbManager.Reader("stats") statsWithHourStore := levelDbManager.ReadingDeleter("stats-with-hour") statsWithDayStore := levelDbManager.ReadingDeleter("stats-with-day") statsWithReceivedTimestampStore := levelDbManager.ReadingDeleter("stats-with-received-timestamp") interarrivalTimesStore := levelDbManager.ReadingDeleter("interarrival-times") sizeSummaryStore := levelDbManager.ReadingWriter("size-summary") sizeSummaryByHourStore := levelDbManager.ReadingWriter("size-summary-by-hour") sizeSummaryByDayStore := levelDbManager.ReadingWriter("size-summary-by-day") interarrivalTimesSummaryStore := levelDbManager.ReadingWriter("interarrival-times-summary") sizePerDayStore := levelDbManager.ReadingWriter("sizes-by-day") sizeSummaryWriter := makeSummaryCsvWriter(csvManager, "size-summary.csv") sizeSummaryByHourWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-hour.csv") sizeSummaryByDayWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-day.csv") interarrivalTimesSummaryWriter := makeSummaryCsvWriter(csvManager, "interarrival-times-summary.csv") sizesPerDayWriter := csvManager.Writer("sizes-per-day.csv", []string{"experiment", "node", "timestamp"}, []string{"count"}, new(string), new(string), new(int64), new(int64)) return []transformer.PipelineStage{ transformer.PipelineStage{ Name: "SummarizeSizes", Reader: statsStore, Transformer: transformer.TransformFunc(summarizeSizes), Writer: sizeSummaryStore, }, transformer.PipelineStage{ Name: "RekeyStatsByHour", Reader: statsStore, Transformer: transformer.MakeMapFunc(rekeyStatsByHour), Writer: store.NewTruncatingWriter(statsWithHourStore), }, transformer.PipelineStage{ Name: "SummarizeSizesByHour", Reader: statsWithHourStore, Transformer: transformer.TransformFunc(summarizeSizesByTimestamp), Writer: sizeSummaryByHourStore, }, transformer.PipelineStage{ Name: "RekeyStatsByDay", Reader: statsStore, Transformer: transformer.MakeMapFunc(rekeyStatsByDay), Writer: store.NewTruncatingWriter(statsWithDayStore), }, transformer.PipelineStage{ Name: "SummarizeSizesByDay", Reader: statsWithDayStore, Transformer: transformer.TransformFunc(summarizeSizesByTimestamp), Writer: sizeSummaryByDayStore, }, transformer.PipelineStage{ Name: "RekeyStatsByReceivedTimestamp", Reader: statsStore, Transformer: transformer.MakeMapFunc(rekeyStatsByReceviedTimestamp), Writer: store.NewTruncatingWriter(statsWithReceivedTimestampStore), }, transformer.PipelineStage{ Name: "ComputeInterarrivalTimes", Reader: statsWithReceivedTimestampStore, Transformer: transformer.TransformFunc(computeInterarrivalTimes), Writer: store.NewTruncatingWriter(interarrivalTimesStore), }, transformer.PipelineStage{ Name: "SummarizeInterarrival", Reader: interarrivalTimesStore, Transformer: transformer.TransformFunc(summarizeInterarrivalTimes), Writer: interarrivalTimesSummaryStore, }, transformer.PipelineStage{ Name: "SummarizeSizesPerDay", Reader: statsStore, Transformer: transformer.TransformFunc(summarizeSizesPerDay), Writer: sizePerDayStore, }, transformer.PipelineStage{ Name: "AggregateExperimentsPerDay", Reader: sizePerDayStore, Transformer: transformer.TransformFunc(aggregateSizesPerDay), Writer: sizePerDayStore, }, transformer.PipelineStage{ Name: "WriteSizesSummary", Reader: sizeSummaryStore, Writer: sizeSummaryWriter, }, transformer.PipelineStage{ Name: "WriteSizesSummaryByHour", Reader: sizeSummaryByHourStore, Writer: sizeSummaryByHourWriter, }, transformer.PipelineStage{ Name: "WriteSizesSummaryByDay", Reader: sizeSummaryByDayStore, Writer: sizeSummaryByDayWriter, }, transformer.PipelineStage{ Name: "WriteInterarrivalTimesSummary", Reader: interarrivalTimesSummaryStore, Writer: interarrivalTimesSummaryWriter, }, transformer.PipelineStage{ Name: "WriteSizePerDaySummary", Reader: sizePerDayStore, Writer: sizesPerDayWriter, }, } }