func PackagesPipeline(levelDbManager, csvManager, sqliteManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
installedPackagesStore := levelDbManager.ReadingWriter("installed-packages")
versionChangesStore := levelDbManager.ReadingWriter("version-changes")
var node, packageName string
var timestamp int64
var version string
csvStore := csvManager.Writer("packages.csv", []string{"node", "package", "timestamp"}, []string{"version"}, &node, &packageName, ×tamp, &version)
sqliteStore := sqliteManager.Writer("packages", []string{"node", "package", "timestamp"}, []string{"version"}, &node, &packageName, ×tamp, &version)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "OpkgListInstalled",
Reader: ReadOnlySomeLogs(logsStore, "opkg_list-installed"),
Transformer: transformer.MakeDoFunc(extractInstalledPackages),
Writer: installedPackagesStore,
},
transformer.PipelineStage{
Name: "DetectVersionChanges",
Reader: installedPackagesStore,
Transformer: transformer.TransformFunc(detectChangedPackageVersions),
Writer: versionChangesStore,
},
transformer.PipelineStage{
Name: "WriteVersionChangesSqlite",
Reader: versionChangesStore,
Writer: sqliteStore,
},
transformer.PipelineStage{
Name: "WriteVersionChangesCsv",
Reader: versionChangesStore,
Writer: csvStore,
},
}
}
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
allTarballsPattern := filepath.Join(tarballsPath, "all", "health", "*", "*", "health_*.tar.gz")
dailyTarballsPattern := filepath.Join(tarballsPath, "by-date", "*", "health", "*", "health_*.tar.gz")
tarnamesStore := levelDbManager.ReadingWriter("tarnames")
tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
logsStore := levelDbManager.Writer("logs")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ScanLogTarballs",
Reader: store.NewGlobReader(allTarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "ScanDailyLogTarballs",
Reader: store.NewGlobReader(dailyTarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "ReadLogTarballs",
Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
Writer: store.NewMuxingWriter(logsStore, tarnamesIndexedStore),
},
}
}
func UptimePipeline(levelDbManager, csvManager, sqliteManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
uptimeStore := levelDbManager.ReadingWriter("uptime")
var node string
var timestamp, uptime int64
csvStore := csvManager.Writer("uptime.csv", []string{"node", "timestamp"}, []string{"uptime"}, &node, ×tamp, &uptime)
sqliteStore := sqliteManager.Writer("uptime", []string{"node", "timestamp"}, []string{"uptime"}, &node, ×tamp, &uptime)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "Uptime",
Reader: ReadOnlySomeLogs(logsStore, "uptime"),
Transformer: transformer.MakeMapFunc(extractUptime),
Writer: uptimeStore,
},
transformer.PipelineStage{
Name: "WriteUptimeCsv",
Reader: uptimeStore,
Writer: csvStore,
},
transformer.PipelineStage{
Name: "WriteUptimeSqlite",
Reader: uptimeStore,
Writer: sqliteStore,
},
}
}
func MemoryUsagePipeline(levelDbManager, csvManager, sqliteManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
memoryUsageStore := levelDbManager.ReadingWriter("memory")
var node string
var timestamp, used, free int64
csvStore := csvManager.Writer("memory.csv", []string{"node", "timestamp"}, []string{"used", "free"}, &node, ×tamp, &used, &free)
sqliteStore := sqliteManager.Writer("memory", []string{"node", "timestamp"}, []string{"used", "free"}, &node, ×tamp, &used, &free)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "Memory",
Reader: ReadOnlySomeLogs(logsStore, "top"),
Transformer: transformer.MakeDoFunc(extractMemoryUsage),
Writer: memoryUsageStore,
},
transformer.PipelineStage{
Name: "WriteMemoryUsageCsv",
Reader: memoryUsageStore,
Writer: csvStore,
},
transformer.PipelineStage{
Name: "WriteMemoryUsageSqlite",
Reader: memoryUsageStore,
Writer: sqliteStore,
},
}
}
func RebootsPipeline(levelDbManager, csvManager, sqliteManager store.Manager) transformer.Pipeline {
uptimeStore := levelDbManager.Seeker("uptime")
rebootsStore := levelDbManager.ReadingWriter("reboots")
var node string
var timestamp int64
rebootsCsvStore := csvManager.Writer("reboots.csv", []string{"node", "boot_timestamp"}, []string{}, &node, ×tamp)
rebootsSqliteStore := sqliteManager.Writer("reboots", []string{"node", "boot_timestamp"}, []string{}, &node, ×tamp)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "InferReboots",
Reader: uptimeStore,
Transformer: transformer.TransformFunc(inferReboots),
Writer: rebootsStore,
},
transformer.PipelineStage{
Name: "WriteRebootsCsv",
Reader: rebootsStore,
Writer: rebootsCsvStore,
},
transformer.PipelineStage{
Name: "WriteRebootsSqlite",
Reader: rebootsStore,
Writer: rebootsSqliteStore,
},
}
}
func makeSummaryByTimestampCsvWriter(manager store.Manager, name string) store.Writer {
keyNames := []string{
"experiment",
"node",
"timestamp",
}
valueNames := []string{
"count",
}
for _, i := range []int{0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100} {
valueNames = append(valueNames, humanize.Ordinal(i))
}
arguments := []interface{}{
name,
keyNames,
valueNames,
new(string), // experiment
new(string), // node
new(int64), // timestamp
new(int64), // count
}
for _ = range []int{0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100} {
arguments = append(arguments, new(int64))
}
return manager.Writer(arguments...)
}
func FilterNodesPipeline(nodeId string, levelDbManager store.Manager) transformer.Pipeline {
tracesStore := levelDbManager.Seeker("traces")
filteredStore := levelDbManager.Writer(fmt.Sprintf("filtered-%s", nodeId))
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "FilterNode",
Reader: FilterNodes(tracesStore, nodeId),
Writer: filteredStore,
},
}
}
func CsvPipeline(levelDbManager, csvManager store.Manager) transformer.Pipeline {
var experiment, node, filename string
var receivedTimestamp, creationTimestamp, size int64
csvStore := csvManager.Writer("stats.csv", []string{"experiment", "node", "filename"}, []string{"received_timestamp", "creation_timestamp", "size"}, &experiment, &node, &filename, &receivedTimestamp, &creationTimestamp, &size)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "WriteStatsCsv",
Reader: levelDbManager.Reader("stats"),
Writer: csvStore,
},
}
}
func SummarizeHealthPipeline(levelDbManager, csvManager store.Manager) transformer.Pipeline {
memoryStore := levelDbManager.Reader("memory")
memoryUsageByDayStore := levelDbManager.ReadingWriter("memory-usage-by-day")
memoryUsageByDaySummarizedStore := levelDbManager.ReadingWriter("memory-usage-by-day-summarized")
filesystemStore := levelDbManager.Reader("filesystem")
filesystemUsageByDayStore := levelDbManager.ReadingWriter("filesystem-usage-by-day")
filesystemUsageByDaySummarizedStore := levelDbManager.ReadingWriter("filesystem-usage-by-day-summarized")
var timestamp, usage int64
var filesystem, node string
memoryUsageSummaryCsv := csvManager.Writer("memory-usage-summary.csv", []string{"timestamp", "node"}, []string{"usage"}, ×tamp, &node, &usage)
filesystemUsageSummaryCsv := csvManager.Writer("filesystem-usage-summary.csv", []string{"filesystem", "timestamp", "node"}, []string{"usage"}, &filesystem, ×tamp, &node, &usage)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "OrderMemoryUsageByTimestamp",
Reader: memoryStore,
Transformer: transformer.MakeMapFunc(orderRecordsByDay),
Writer: memoryUsageByDayStore,
},
transformer.PipelineStage{
Name: "SummarizeMemoryUsage",
Reader: memoryUsageByDayStore,
Transformer: transformer.TransformFunc(summarizeMemoryUsage),
Writer: memoryUsageByDaySummarizedStore,
},
transformer.PipelineStage{
Name: "WriteMemoryUsageSummaryCsv",
Reader: memoryUsageByDaySummarizedStore,
Writer: memoryUsageSummaryCsv,
},
transformer.PipelineStage{
Name: "OrderFilesystemUsageByTimestamp",
Reader: filesystemStore,
Transformer: transformer.MakeMapFunc(orderFilesystemRecordsByDay),
Writer: filesystemUsageByDayStore,
},
transformer.PipelineStage{
Name: "SummarizeFilesystemUsage",
Reader: filesystemUsageByDayStore,
Transformer: transformer.TransformFunc(summarizeFilesystemUsage),
Writer: filesystemUsageByDaySummarizedStore,
},
transformer.PipelineStage{
Name: "WriteFilesystemUsageSummaryCsv",
Reader: filesystemUsageByDaySummarizedStore,
Writer: filesystemUsageSummaryCsv,
},
}
}
func FilterSessionsPipeline(sessionStartTime, sessionEndTime int64, levelDbManager store.Manager, outputName string) transformer.Pipeline {
tracesStore := levelDbManager.Reader("traces")
traceKeyRangesStore := levelDbManager.Reader("availability-done")
filteredStore := levelDbManager.Writer(outputName)
parameters := filterSessions{
SessionStartTime: sessionStartTime * 1000000,
SessionEndTime: sessionEndTime * 1000000,
}
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "FilterSessions",
Reader: store.NewDemuxingReader(traceKeyRangesStore, tracesStore),
Transformer: parameters,
Writer: filteredStore,
},
}
}
func DevicesCountPipeline(levelDbManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
devicesCountStore := levelDbManager.Writer("devices-count")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ExtractEthernetCount",
Reader: ReadOnlySomeLogs(logsStore, "swconfig_ports"),
Transformer: transformer.MakeDoFunc(extractEthernetCount),
Writer: devicesCountStore,
},
transformer.PipelineStage{
Name: "ExtractWirelessCount",
Reader: ReadOnlySomeLogs(logsStore, "iw_station_count"),
Transformer: transformer.MakeDoFunc(extractWirelessCount),
Writer: devicesCountStore,
},
}
}
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
tarballsPattern := filepath.Join(tarballsPath, "*", "*", "*.tar.gz")
tarnamesStore := levelDbManager.ReadingWriter("tarnames")
tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
tracesStore := levelDbManager.Writer("traces")
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ScanTraceTarballs",
Reader: store.NewGlobReader(tarballsPattern),
Writer: tarnamesStore,
},
transformer.PipelineStage{
Name: "IndexTraces",
Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
Reader: store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
Writer: store.NewMuxingWriter(tracesStore, tarnamesIndexedStore),
},
}
}
func DisjointPackagesPipeline(levelDbManager, csvManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
disjointPackagesStore := levelDbManager.ReadingWriter("disjoint-packages")
var filename, node string
var timestamp int64
csvStore := csvManager.Writer("not-disjoint.csv", []string{"filename", "node", "timestamp"}, []string{}, &filename, &node, ×tamp)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "DisjointPackages",
Reader: logsStore,
Transformer: transformer.MakeDoFunc(detectDisjointPackagesError),
Writer: disjointPackagesStore,
},
transformer.PipelineStage{
Name: "WriteDisjointPackagesCsv",
Reader: disjointPackagesStore,
Writer: csvStore,
},
}
}
func IpRoutePipeline(levelDbManager, sqliteManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
defaultRoutesStore := levelDbManager.ReadingWriter("default-routes")
var node string
var timestamp int64
var gateway string
sqliteStore := sqliteManager.Writer("defaultroutes", []string{"node", "timestamp"}, []string{"gateway"}, &node, ×tamp, &gateway)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "ExtractDefaultRoute",
Reader: ReadOnlySomeLogs(logsStore, "iproute"),
Transformer: transformer.MakeDoFunc(extractDefaultRoute),
Writer: defaultRoutesStore,
},
transformer.PipelineStage{
Name: "WriteDefaultRoutesSqlite",
Reader: defaultRoutesStore,
Writer: sqliteStore,
},
}
}
func FilesystemUsagePipeline(levelDbManager, csvManager store.Manager) transformer.Pipeline {
logsStore := levelDbManager.Seeker("logs")
filesystemUsageStore := levelDbManager.ReadingWriter("filesystem")
var mount, node string
var timestamp, used, free int64
csvStore := csvManager.Writer("filesystem.csv", []string{"mount", "node", "timestamp"}, []string{"used", "free"}, &mount, &node, ×tamp, &used, &free)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "Filesystem",
Reader: ReadOnlySomeLogs(logsStore, "df"),
Transformer: transformer.MakeDoFunc(extractFilesystemUsage),
Writer: filesystemUsageStore,
},
transformer.PipelineStage{
Name: "WriteFilesystemUsageCsv",
Reader: filesystemUsageStore,
Writer: csvStore,
},
}
}
func LookupsPerDevicePipeline(levelDbManager store.Manager) transformer.Pipeline {
tracesStore := levelDbManager.Seeker("traces")
availabilityIntervalsStore := levelDbManager.Seeker("consistent-ranges")
addressIdStore := levelDbManager.Seeker("bytesperdomain-address-id-table")
addressIdToDomainStore := levelDbManager.SeekingWriter("lookupsperdevice-address-id-to-domain")
lookupsPerDeviceSharded := levelDbManager.ReadingWriter("lookupsperdevice-sharded")
lookupsPerDeviceStore := levelDbManager.Writer("lookupsperdevice-lookups-per-device")
lookupsPerDevicePerHourStore := levelDbManager.Writer("lookupsperdevice-lookups-per-device-per-hour")
consistentTracesStore := store.NewRangeIncludingReader(tracesStore, availabilityIntervalsStore)
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "LookupsPerDeviceMapper",
Reader: consistentTracesStore,
Transformer: transformer.MakeDoFunc(lookupsPerDeviceMapper),
Writer: addressIdToDomainStore,
},
transformer.PipelineStage{
Name: "JoinMacWithLookups",
Reader: store.NewDemuxingSeeker(addressIdStore, addressIdToDomainStore),
Transformer: transformer.TransformFunc(joinMacWithLookups),
Writer: lookupsPerDeviceSharded,
},
transformer.PipelineStage{
Name: "FlattenLookupsToNodeAndMac",
Reader: lookupsPerDeviceSharded,
Transformer: transformer.TransformFunc(flattenLookupsToNodeAndMac),
Writer: lookupsPerDeviceStore,
},
transformer.PipelineStage{
Name: "FlattenLookupsToNodeMacAndTimestamp",
Reader: lookupsPerDeviceSharded,
Transformer: transformer.TransformFunc(flattenLookupsToNodeMacAndTimestamp),
Writer: lookupsPerDevicePerHourStore,
},
}
}
func makeSummaryCsvWriter(manager store.Manager, name string) store.Writer {
keyNames := []string{
"experiment",
"node",
}
valueNames := []string{
"count",
}
for i := 0; i <= 100; i += 5 {
valueNames = append(valueNames, humanize.Ordinal(i))
}
arguments := []interface{}{
name,
keyNames,
valueNames,
new(string), // experiment
new(string), // node
new(int64), // count
}
for i := 0; i <= 100; i += 5 {
arguments = append(arguments, new(int64))
}
return manager.Writer(arguments...)
}
func SummarizePipeline(levelDbManager store.Manager, csvManager store.Manager) transformer.Pipeline {
statsStore := levelDbManager.Reader("stats")
statsWithHourStore := levelDbManager.ReadingDeleter("stats-with-hour")
statsWithDayStore := levelDbManager.ReadingDeleter("stats-with-day")
statsWithReceivedTimestampStore := levelDbManager.ReadingDeleter("stats-with-received-timestamp")
interarrivalTimesStore := levelDbManager.ReadingDeleter("interarrival-times")
sizeSummaryStore := levelDbManager.ReadingWriter("size-summary")
sizeSummaryByHourStore := levelDbManager.ReadingWriter("size-summary-by-hour")
sizeSummaryByDayStore := levelDbManager.ReadingWriter("size-summary-by-day")
interarrivalTimesSummaryStore := levelDbManager.ReadingWriter("interarrival-times-summary")
sizePerDayStore := levelDbManager.ReadingWriter("sizes-by-day")
sizeSummaryWriter := makeSummaryCsvWriter(csvManager, "size-summary.csv")
sizeSummaryByHourWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-hour.csv")
sizeSummaryByDayWriter := makeSummaryByTimestampCsvWriter(csvManager, "size-summary-by-day.csv")
interarrivalTimesSummaryWriter := makeSummaryCsvWriter(csvManager, "interarrival-times-summary.csv")
sizesPerDayWriter := csvManager.Writer("sizes-per-day.csv", []string{"experiment", "node", "timestamp"}, []string{"count"}, new(string), new(string), new(int64), new(int64))
return []transformer.PipelineStage{
transformer.PipelineStage{
Name: "SummarizeSizes",
Reader: statsStore,
Transformer: transformer.TransformFunc(summarizeSizes),
Writer: sizeSummaryStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByHour",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByHour),
Writer: store.NewTruncatingWriter(statsWithHourStore),
},
transformer.PipelineStage{
Name: "SummarizeSizesByHour",
Reader: statsWithHourStore,
Transformer: transformer.TransformFunc(summarizeSizesByTimestamp),
Writer: sizeSummaryByHourStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByDay",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByDay),
Writer: store.NewTruncatingWriter(statsWithDayStore),
},
transformer.PipelineStage{
Name: "SummarizeSizesByDay",
Reader: statsWithDayStore,
Transformer: transformer.TransformFunc(summarizeSizesByTimestamp),
Writer: sizeSummaryByDayStore,
},
transformer.PipelineStage{
Name: "RekeyStatsByReceivedTimestamp",
Reader: statsStore,
Transformer: transformer.MakeMapFunc(rekeyStatsByReceviedTimestamp),
Writer: store.NewTruncatingWriter(statsWithReceivedTimestampStore),
},
transformer.PipelineStage{
Name: "ComputeInterarrivalTimes",
Reader: statsWithReceivedTimestampStore,
Transformer: transformer.TransformFunc(computeInterarrivalTimes),
Writer: store.NewTruncatingWriter(interarrivalTimesStore),
},
transformer.PipelineStage{
Name: "SummarizeInterarrival",
Reader: interarrivalTimesStore,
Transformer: transformer.TransformFunc(summarizeInterarrivalTimes),
Writer: interarrivalTimesSummaryStore,
},
transformer.PipelineStage{
Name: "SummarizeSizesPerDay",
Reader: statsStore,
Transformer: transformer.TransformFunc(summarizeSizesPerDay),
Writer: sizePerDayStore,
},
transformer.PipelineStage{
Name: "AggregateExperimentsPerDay",
Reader: sizePerDayStore,
Transformer: transformer.TransformFunc(aggregateSizesPerDay),
Writer: sizePerDayStore,
},
transformer.PipelineStage{
Name: "WriteSizesSummary",
Reader: sizeSummaryStore,
Writer: sizeSummaryWriter,
},
transformer.PipelineStage{
Name: "WriteSizesSummaryByHour",
Reader: sizeSummaryByHourStore,
Writer: sizeSummaryByHourWriter,
},
transformer.PipelineStage{
Name: "WriteSizesSummaryByDay",
Reader: sizeSummaryByDayStore,
Writer: sizeSummaryByDayWriter,
},
transformer.PipelineStage{
Name: "WriteInterarrivalTimesSummary",
Reader: interarrivalTimesSummaryStore,
Writer: interarrivalTimesSummaryWriter,
},
transformer.PipelineStage{
Name: "WriteSizePerDaySummary",
Reader: sizePerDayStore,
Writer: sizesPerDayWriter,
},
}
}