func SessionPipelineStage(inputStore store.Reader, sessionsStore store.Deleter) transformer.PipelineStage {
return transformer.PipelineStage{
Name: "Sessions",
Transformer: transformer.TransformFunc(sessions),
Reader: inputStore,
Writer: store.NewTruncatingWriter(sessionsStore),
}
}
func AvailabilityPipeline(levelDbManager store.Manager, jsonWriter io.Writer, timestamp int64) transformer.Pipeline {
tracesStore := levelDbManager.Seeker("traces")
intervalsStore := levelDbManager.ReadingWriter("availability-intervals")
consolidatedStore := levelDbManager.ReadingDeleter("availability-consolidated")
nodesStore := levelDbManager.ReadingDeleter("availability-nodes")
excludeRangesStore := levelDbManager.ReadingDeleter("availability-done")
consistentRangesStore := levelDbManager.ReadingDeleter("consistent-ranges")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "AvailabilityIntervals",
Reader: store.NewRangeExcludingReader(tracesStore, excludeRangesStore),
Transformer: transformer.TransformFunc(availabilityIntervals),
Writer: intervalsStore,
},
transformer.PipelineStage{
Name: "ConsolidateAvailabilityIntervals",
Reader: intervalsStore,
Transformer: transformer.TransformFunc(consolidateAvailabilityIntervals),
Writer: store.NewTruncatingWriter(consolidatedStore),
},
transformer.PipelineStage{
Name: "AvailabilityReducer",
Reader: consolidatedStore,
Transformer: transformer.TransformFunc(availabilityReducer),
Writer: store.NewTruncatingWriter(nodesStore),
},
transformer.PipelineStage{
Name: "AvailabilityJson",
Reader: nodesStore,
Writer: &availabilityJsonStore{writer: jsonWriter, timestamp: timestamp},
},
transformer.PipelineStage{
Name: "GenerateExcludedRanges",
Reader: consolidatedStore,
Transformer: transformer.MakeMapFunc(generateExcludedRanges),
Writer: store.NewTruncatingWriter(excludeRangesStore),
},
transformer.PipelineStage{
Name: "GenerateConsistentRanges",
Reader: excludeRangesStore,
Transformer: transformer.MakeDoFunc(generateConsistentRanges),
Writer: store.NewTruncatingWriter(consistentRangesStore),
},
}
}
func TraceKeyRangesPipeline(newTraceKeysStore store.Reader, traceKeyRangesStore, consolidatedTraceKeyRangesStore store.ReadingDeleter) []transformer.PipelineStage {
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "CalculateTraceKeyRanges",
Transformer: transformer.TransformFunc(calculateTraceKeyRanges),
Reader: newTraceKeysStore,
Writer: traceKeyRangesStore,
},
transformer.PipelineStage{
Name: "ConsolidateTraceKeyRanges",
Transformer: transformer.TransformFunc(consolidateTraceKeyRanges),
Reader: traceKeyRangesStore,
Writer: store.NewTruncatingWriter(consolidatedTraceKeyRangesStore),
},
transformer.PipelineStage{
Name: "CopyTraceKeyRanges",
Reader: consolidatedTraceKeyRangesStore,
Writer: store.NewTruncatingWriter(traceKeyRangesStore),
},
}
}
func writeAvailabilityLevelDb(allIntervals map[string][]availabilityInterval) error {
rawStore := store.NewLevelDbStore(outputLevelDb, false)
outputStore := store.NewTruncatingWriter(rawStore)
if err := outputStore.BeginWriting(); err != nil {
return err
}
for nodeId, intervals := range allIntervals {
for _, interval := range intervals {
record := store.Record{
Key: lex.EncodeOrDie(nodeId, interval.StartTime.Unix(), interval.EndTime.Unix()),
}
if err := outputStore.WriteRecord(&record); err != nil {
return err
}
}
}
if err := outputStore.EndWriting(); err != nil {
return err
}
return nil
}
func SummarizePipeline(levelDbManager store.Manager, csvManager store.Manager) transformer.Pipeline {
statsStore := levelDbManager.Reader("stats")
statsWithHourStore := levelDbManager.ReadingDeleter("stats-with-hour")
statsWithDayStore := levelDbManager.ReadingDeleter("stats-with-day")
statsWithReceivedTimestampStore := levelDbManager.ReadingDeleter("stats-with-received-timestamp")
interarrivalTimesStore := levelDbManager.ReadingDeleter("interarrival-times")
sizeSummaryStore := levelDbManager.ReadingWriter("size-summary")
sizeSummaryByHourStore := levelDbManager.ReadingWriter("size-summary-by-hour")
sizeSummaryByDayStore := levelDbManager.ReadingWriter("size-summary-by-day")
interarrivalTimesSummaryStore := levelDbManager.ReadingWriter("interarrival-times-summary")
sizePerDayStore := levelDbManager.ReadingWriter("sizes-by-day")
sizeSummaryWriter := makeSummaryCsvWriter(csvManager, "size-summary.csv")
sizeSummaryByHourWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-hour.csv")
sizeSummaryByDayWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-day.csv")
interarrivalTimesSummaryWriter := makeSummaryCsvWriter(csvManager, "interarrival-times-summary.csv")
sizesPerDayWriter := csvManager.Writer("sizes-per-day.csv", []string{"experiment", "node", "timestamp"}, []string{"count"}, new(string), new(string), new(int64), new(int64))
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "SummarizeSizes",
Reader: statsStore,
Transformer: transformer.TransformFunc(summarizeSizes),
Writer: sizeSummaryStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByHour",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByHour),
Writer: store.NewTruncatingWriter(statsWithHourStore),
},
transformer.PipelineStage{
Name: "SummarizeSizesByHour",
Reader: statsWithHourStore,
Transformer: transformer.TransformFunc(summarizeSizesByTimestamp),
Writer: sizeSummaryByHourStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByDay",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByDay),
Writer: store.NewTruncatingWriter(statsWithDayStore),
},
transformer.PipelineStage{
Name: "SummarizeSizesByDay",
Reader: statsWithDayStore,
Transformer: transformer.TransformFunc(summarizeSizesByTimestamp),
Writer: sizeSummaryByDayStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByReceivedTimestamp",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByReceviedTimestamp),
Writer: store.NewTruncatingWriter(statsWithReceivedTimestampStore),
},
transformer.PipelineStage{
Name: "ComputeInterarrivalTimes",
Reader: statsWithReceivedTimestampStore,
Transformer: transformer.TransformFunc(computeInterarrivalTimes),
Writer: store.NewTruncatingWriter(interarrivalTimesStore),
},
transformer.PipelineStage{
Name: "SummarizeInterarrival",
Reader: interarrivalTimesStore,
Transformer: transformer.TransformFunc(summarizeInterarrivalTimes),
Writer: interarrivalTimesSummaryStore,
},
transformer.PipelineStage{
Name: "SummarizeSizesPerDay",
Reader: statsStore,
Transformer: transformer.TransformFunc(summarizeSizesPerDay),
Writer: sizePerDayStore,
},
transformer.PipelineStage{
Name: "AggregateExperimentsPerDay",
Reader: sizePerDayStore,
Transformer: transformer.TransformFunc(aggregateSizesPerDay),
Writer: sizePerDayStore,
},
transformer.PipelineStage{
Name: "WriteSizesSummary",
Reader: sizeSummaryStore,
Writer: sizeSummaryWriter,
},
transformer.PipelineStage{
Name: "WriteSizesSummaryByHour",
Reader: sizeSummaryByHourStore,
Writer: sizeSummaryByHourWriter,
},
transformer.PipelineStage{
Name: "WriteSizesSummaryByDay",
Reader: sizeSummaryByDayStore,
Writer: sizeSummaryByDayWriter,
},
transformer.PipelineStage{
Name: "WriteInterarrivalTimesSummary",
Reader: interarrivalTimesSummaryStore,
Writer: interarrivalTimesSummaryWriter,
},
transformer.PipelineStage{
Name: "WriteSizePerDaySummary",
Reader: sizePerDayStore,
Writer: sizesPerDayWriter,
},
}
}