func SummarizeHealthPipeline(levelDbManager, csvManager store.Manager) transformer.Pipeline {
memoryStore := levelDbManager.Reader("memory")
memoryUsageByDayStore := levelDbManager.ReadingWriter("memory-usage-by-day")
memoryUsageByDaySummarizedStore := levelDbManager.ReadingWriter("memory-usage-by-day-summarized")
filesystemStore := levelDbManager.Reader("filesystem")
filesystemUsageByDayStore := levelDbManager.ReadingWriter("filesystem-usage-by-day")
filesystemUsageByDaySummarizedStore := levelDbManager.ReadingWriter("filesystem-usage-by-day-summarized")
var timestamp, usage int64
var filesystem, node string
memoryUsageSummaryCsv := csvManager.Writer("memory-usage-summary.csv", []string{"timestamp", "node"}, []string{"usage"}, ×tamp, &node, &usage)
filesystemUsageSummaryCsv := csvManager.Writer("filesystem-usage-summary.csv", []string{"filesystem", "timestamp", "node"}, []string{"usage"}, &filesystem, ×tamp, &node, &usage)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "OrderMemoryUsageByTimestamp",
Reader: memoryStore,
Transformer: transformer.MakeMapFunc(orderRecordsByDay),
Writer: memoryUsageByDayStore,
},
transformer.PipelineStage{
Name: "SummarizeMemoryUsage",
Reader: memoryUsageByDayStore,
Transformer: transformer.TransformFunc(summarizeMemoryUsage),
Writer: memoryUsageByDaySummarizedStore,
},
transformer.PipelineStage{
Name: "WriteMemoryUsageSummaryCsv",
Reader: memoryUsageByDaySummarizedStore,
Writer: memoryUsageSummaryCsv,
},
transformer.PipelineStage{
Name: "OrderFilesystemUsageByTimestamp",
Reader: filesystemStore,
Transformer: transformer.MakeMapFunc(orderFilesystemRecordsByDay),
Writer: filesystemUsageByDayStore,
},
transformer.PipelineStage{
Name: "SummarizeFilesystemUsage",
Reader: filesystemUsageByDayStore,
Transformer: transformer.TransformFunc(summarizeFilesystemUsage),
Writer: filesystemUsageByDaySummarizedStore,
},
transformer.PipelineStage{
Name: "WriteFilesystemUsageSummaryCsv",
Reader: filesystemUsageByDaySummarizedStore,
Writer: filesystemUsageSummaryCsv,
},
}
}
func UptimePipeline(levelDbManager, csvManager, sqliteManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
uptimeStore := levelDbManager.ReadingWriter("uptime")
var node string
var timestamp, uptime int64
csvStore := csvManager.Writer("uptime.csv", []string{"node", "timestamp"}, []string{"uptime"}, &node, ×tamp, &uptime)
sqliteStore := sqliteManager.Writer("uptime", []string{"node", "timestamp"}, []string{"uptime"}, &node, ×tamp, &uptime)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "Uptime",
Reader: ReadOnlySomeLogs(logsStore, "uptime"),
Transformer: transformer.MakeMapFunc(extractUptime),
Writer: uptimeStore,
},
transformer.PipelineStage{
Name: "WriteUptimeCsv",
Reader: uptimeStore,
Writer: csvStore,
},
transformer.PipelineStage{
Name: "WriteUptimeSqlite",
Reader: uptimeStore,
Writer: sqliteStore,
},
}
}
func AvailabilityPipeline(levelDbManager store.Manager, jsonWriter io.Writer, timestamp int64) transformer.Pipeline {
tracesStore := levelDbManager.Seeker("traces")
intervalsStore := levelDbManager.ReadingWriter("availability-intervals")
consolidatedStore := levelDbManager.ReadingDeleter("availability-consolidated")
nodesStore := levelDbManager.ReadingDeleter("availability-nodes")
excludeRangesStore := levelDbManager.ReadingDeleter("availability-done")
consistentRangesStore := levelDbManager.ReadingDeleter("consistent-ranges")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "AvailabilityIntervals",
Reader: store.NewRangeExcludingReader(tracesStore, excludeRangesStore),
Transformer: transformer.TransformFunc(availabilityIntervals),
Writer: intervalsStore,
},
transformer.PipelineStage{
Name: "ConsolidateAvailabilityIntervals",
Reader: intervalsStore,
Transformer: transformer.TransformFunc(consolidateAvailabilityIntervals),
Writer: store.NewTruncatingWriter(consolidatedStore),
},
transformer.PipelineStage{
Name: "AvailabilityReducer",
Reader: consolidatedStore,
Transformer: transformer.TransformFunc(availabilityReducer),
Writer: store.NewTruncatingWriter(nodesStore),
},
transformer.PipelineStage{
Name: "AvailabilityJson",
Reader: nodesStore,
Writer: &availabilityJsonStore{writer: jsonWriter, timestamp: timestamp},
},
transformer.PipelineStage{
Name: "GenerateExcludedRanges",
Reader: consolidatedStore,
Transformer: transformer.MakeMapFunc(generateExcludedRanges),
Writer: store.NewTruncatingWriter(excludeRangesStore),
},
transformer.PipelineStage{
Name: "GenerateConsistentRanges",
Reader: excludeRangesStore,
Transformer: transformer.MakeDoFunc(generateConsistentRanges),
Writer: store.NewTruncatingWriter(consistentRangesStore),
},
}
}
func AggregateStatisticsPipeline(levelDbManager store.Manager, jsonWriter io.Writer) transformer.Pipeline {
tracesStore := levelDbManager.Seeker("traces")
availabilityIntervalsStore := levelDbManager.Seeker("consistent-ranges")
traceAggregatesStore := levelDbManager.SeekingWriter("statistics-trace-aggregates")
sessionAggregatesStore := levelDbManager.ReadingWriter("statistics-session-aggregates")
nodeAggregatesStore := levelDbManager.ReadingWriter("statistics-node-aggregates")
sessionsStore := levelDbManager.ReadingDeleter("statistics-sessions")
traceKeyRangesStore := levelDbManager.ReadingDeleter("statistics-trace-key-ranges")
consolidatedTraceKeyRangesStore := levelDbManager.ReadingDeleter("statistics-consolidated-trace-key-ranges")
newTracesStore := store.NewRangeExcludingReader(store.NewRangeIncludingReader(tracesStore, availabilityIntervalsStore), traceKeyRangesStore)
return append([]transformer.PipelineStage{
transformer.PipelineStage{
Name: "AggregateStatisticsMapper",
Reader: store.NewRangeExcludingReader(tracesStore, traceKeyRangesStore),
Transformer: transformer.MakeMapFunc(aggregateStatisticsMapper),
Writer: traceAggregatesStore,
},
SessionPipelineStage(newTracesStore, sessionsStore),
transformer.PipelineStage{
Name: "AggregateStatisticsReduceBySession",
Reader: store.NewPrefixIncludingReader(traceAggregatesStore, sessionsStore),
Transformer: transformer.TransformFunc(aggregateStatisticsReduceBySession),
Writer: sessionAggregatesStore,
},
transformer.PipelineStage{
Name: "AggregateStatisticsReducer",
Reader: sessionAggregatesStore,
Transformer: transformer.TransformFunc(aggregateStatisticsReducer),
Writer: nodeAggregatesStore,
},
transformer.PipelineStage{
Name: "AggregateStatisticsJson",
Reader: nodeAggregatesStore,
Writer: &aggregateStatisticsJsonStore{writer: jsonWriter},
},
}, TraceKeyRangesPipeline(newTracesStore, traceKeyRangesStore, consolidatedTraceKeyRangesStore)...)
}
func SummarizePipeline(levelDbManager store.Manager, csvManager store.Manager) transformer.Pipeline {
statsStore := levelDbManager.Reader("stats")
statsWithHourStore := levelDbManager.ReadingDeleter("stats-with-hour")
statsWithDayStore := levelDbManager.ReadingDeleter("stats-with-day")
statsWithReceivedTimestampStore := levelDbManager.ReadingDeleter("stats-with-received-timestamp")
interarrivalTimesStore := levelDbManager.ReadingDeleter("interarrival-times")
sizeSummaryStore := levelDbManager.ReadingWriter("size-summary")
sizeSummaryByHourStore := levelDbManager.ReadingWriter("size-summary-by-hour")
sizeSummaryByDayStore := levelDbManager.ReadingWriter("size-summary-by-day")
interarrivalTimesSummaryStore := levelDbManager.ReadingWriter("interarrival-times-summary")
sizePerDayStore := levelDbManager.ReadingWriter("sizes-by-day")
sizeSummaryWriter := makeSummaryCsvWriter(csvManager, "size-summary.csv")
sizeSummaryByHourWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-hour.csv")
sizeSummaryByDayWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-day.csv")
interarrivalTimesSummaryWriter := makeSummaryCsvWriter(csvManager, "interarrival-times-summary.csv")
sizesPerDayWriter := csvManager.Writer("sizes-per-day.csv", []string{"experiment", "node", "timestamp"}, []string{"count"}, new(string), new(string), new(int64), new(int64))
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "SummarizeSizes",
Reader: statsStore,
Transformer: transformer.TransformFunc(summarizeSizes),
Writer: sizeSummaryStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByHour",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByHour),
Writer: store.NewTruncatingWriter(statsWithHourStore),
},
transformer.PipelineStage{
Name: "SummarizeSizesByHour",
Reader: statsWithHourStore,
Transformer: transformer.TransformFunc(summarizeSizesByTimestamp),
Writer: sizeSummaryByHourStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByDay",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByDay),
Writer: store.NewTruncatingWriter(statsWithDayStore),
},
transformer.PipelineStage{
Name: "SummarizeSizesByDay",
Reader: statsWithDayStore,
Transformer: transformer.TransformFunc(summarizeSizesByTimestamp),
Writer: sizeSummaryByDayStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByReceivedTimestamp",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByReceviedTimestamp),
Writer: store.NewTruncatingWriter(statsWithReceivedTimestampStore),
},
transformer.PipelineStage{
Name: "ComputeInterarrivalTimes",
Reader: statsWithReceivedTimestampStore,
Transformer: transformer.TransformFunc(computeInterarrivalTimes),
Writer: store.NewTruncatingWriter(interarrivalTimesStore),
},
transformer.PipelineStage{
Name: "SummarizeInterarrival",
Reader: interarrivalTimesStore,
Transformer: transformer.TransformFunc(summarizeInterarrivalTimes),
Writer: interarrivalTimesSummaryStore,
},
transformer.PipelineStage{
Name: "SummarizeSizesPerDay",
Reader: statsStore,
Transformer: transformer.TransformFunc(summarizeSizesPerDay),
Writer: sizePerDayStore,
},
transformer.PipelineStage{
Name: "AggregateExperimentsPerDay",
Reader: sizePerDayStore,
Transformer: transformer.TransformFunc(aggregateSizesPerDay),
Writer: sizePerDayStore,
},
transformer.PipelineStage{
Name: "WriteSizesSummary",
Reader: sizeSummaryStore,
Writer: sizeSummaryWriter,
},
transformer.PipelineStage{
Name: "WriteSizesSummaryByHour",
Reader: sizeSummaryByHourStore,
Writer: sizeSummaryByHourWriter,
},
transformer.PipelineStage{
Name: "WriteSizesSummaryByDay",
Reader: sizeSummaryByDayStore,
Writer: sizeSummaryByDayWriter,
},
transformer.PipelineStage{
Name: "WriteInterarrivalTimesSummary",
Reader: interarrivalTimesSummaryStore,
Writer: interarrivalTimesSummaryWriter,
},
transformer.PipelineStage{
Name: "WriteSizePerDaySummary",
Reader: sizePerDayStore,
Writer: sizesPerDayWriter,
},
}
}