func pipelineStats() transformer.Pipeline {
flagset := flag.NewFlagSet("stats", flag.ExitOnError)
dbRoot := flagset.String("uploads_leveldb_root", "/data/users/sburnett/bismark-upload-stats-leveldb", "Write leveldbs in this directory.")
tarballsPath := flagset.String("tarballs_path", "/var/local/home/bismark-data-xfer/bismark_data_from_s3", "Read tarballs from this directory.")
flagset.Parse(flag.Args()[1:])
return stats.ExtractStatsPipeline(*tarballsPath, store.NewLevelDbManager(*dbRoot))
}
func pipelineTimesCsv() transformer.Pipeline {
flagset := flag.NewFlagSet("csv", flag.ExitOnError)
dbRoot := flagset.String("uploads_leveldb_root", "/data/users/sburnett/bismark-upload-stats-leveldb", "Write leveldbs in this directory.")
csvOutput := flagset.String("csv_output", "/dev/null", "Directory where we will write statistics in CSV format.")
flagset.Parse(flag.Args()[1:])
return stats.TimesCsvPipeline(store.NewLevelDbManager(*dbRoot), *csvOutput)
}
func pipelineFilesystem() transformer.Pipeline {
flagset := flag.NewFlagSet("filesystem", flag.ExitOnError)
dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
csvOutput := flagset.String("csv_output", "/dev/null", "Write filesystem usage in CSV format to this file.")
flagset.Parse(flag.Args()[1:])
return health.FilesystemUsagePipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
func pipelineCsv() transformer.Pipeline {
flagset := flag.NewFlagSet("csv", flag.ExitOnError)
csvOutput := flagset.String("csv_output", "/dev/null", "Write upload statistics to a file in this directory.")
dbRoot := flagset.String("uploads_leveldb_root", "/data/users/sburnett/bismark-upload-stats-leveldb", "Write leveldbs in this directory.")
flagset.Parse(flag.Args()[1:])
return stats.CsvPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
func pipelineIpRoute() transformer.Pipeline {
flagset := flag.NewFlagSet("iproute", flag.ExitOnError)
dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
sqliteFilename := flagset.String("sqlite_filename", "/dev/null", "Write to this sqlite database.")
flagset.Parse(flag.Args()[1:])
return health.IpRoutePipeline(store.NewLevelDbManager(*dbRoot), store.NewSqliteManager(*sqliteFilename))
}
func pipelineSummarize() transformer.Pipeline {
flagset := flag.NewFlagSet("summarize", flag.ExitOnError)
dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
csvOutput := flagset.String("csv_output", "/dev/null", "Write reboots to a CSV file in this directory.")
flagset.Parse(flag.Args()[1:])
return health.SummarizeHealthPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
func pipelineFilterNode() transformer.Pipeline {
flagset := flag.NewFlagSet("filter", flag.ExitOnError)
dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
nodeId := flagset.String("node_id", "OWC43DC7B0AE78", "Retain only data from this router.")
flagset.Parse(flag.Args()[1:])
return passive.FilterNodesPipeline(*nodeId, store.NewLevelDbManager(*dbRoot))
}
func pipelineIndex() transformer.Pipeline {
flagset := flag.NewFlagSet("index", flag.ExitOnError)
tarballsPath := flagset.String("tarballs_path", "/data/users/sburnett/bismark-health", "Read tarballs from this directory.")
dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
flagset.Parse(flag.Args()[1:])
return health.IndexTarballsPipeline(*tarballsPath, store.NewLevelDbManager(*dbRoot))
}
func pipelineDisjointPackages() transformer.Pipeline {
flagset := flag.NewFlagSet("disjoint", flag.ExitOnError)
dbRoot := flagset.String("logs_leveldb_root", "/data/users/sburnett/bismark-experiments-manager-logs-leveldb", "Write leveldbs in this directory.")
csvOutput := flagset.String("csv_output", "/dev/null", "Write disjoint packages statistics in CSV format to this file.")
flagset.Parse(flag.Args()[1:])
return experiments.DisjointPackagesPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
func pipelinePackages() transformer.Pipeline {
flagset := flag.NewFlagSet("packages", flag.ExitOnError)
dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
csvOutput := flagset.String("csv_output", "/dev/null", "Write reboots to a CSV file in this directory.")
sqliteFilename := flagset.String("sqlite_filename", "/dev/null", "Write to this sqlite database.")
flagset.Parse(flag.Args()[1:])
return health.PackagesPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput), store.NewSqliteManager(*sqliteFilename))
}
func pipelineStatistics() transformer.Pipeline {
flagset := flag.NewFlagSet("statistics", flag.ExitOnError)
dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
jsonOutput := flagset.String("json_output", "/dev/null", "Write statistics in JSON format to this file.")
flagset.Parse(flag.Args()[2:])
jsonHandle, err := os.Create(*jsonOutput)
if err != nil {
log.Fatalf("Error opening JSON output: %v", err)
}
return passive.AggregateStatisticsPipeline(store.NewLevelDbManager(*dbRoot), jsonHandle)
}
func pipelineDevicesCount() transformer.Pipeline {
flagset := flag.NewFlagSet("devicescount", flag.ExitOnError)
dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
flagset.Parse(flag.Args()[1:])
return health.DevicesCountPipeline(store.NewLevelDbManager(*dbRoot))
}
func pipelineLookupsPerDevice() transformer.Pipeline {
flagset := flag.NewFlagSet("lookupsperdevice", flag.ExitOnError)
dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
flagset.Parse(flag.Args()[1:])
return passive.LookupsPerDevicePipeline(store.NewLevelDbManager(*dbRoot))
}
func pipelineFilterDates() transformer.Pipeline {
flagset := flag.NewFlagSet("filter", flag.ExitOnError)
dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
sessionStartDate := flagset.String("session_start_date", "20120301", "Retain only session starting after this date, in YYYYMMDD format.")
sessionEndDate := flagset.String("session_end_date", "20120401", "Retain only session starting before this date, in YYYYMMDD format.")
flagset.Parse(flag.Args()[1:])
timeFormatString := "20060102"
sessionStartTime, err := time.Parse(timeFormatString, *sessionStartDate)
if err != nil {
panic(fmt.Errorf("Error parsing start date %s: %v", sessionStartDate, err))
}
sessionEndTime, err := time.Parse(timeFormatString, *sessionEndDate)
if err != nil {
panic(fmt.Errorf("Error parsing end date %s: %v", sessionEndDate, err))
}
outputName := fmt.Sprintf("filtered-%s-%s", *sessionStartDate, *sessionEndDate)
return passive.FilterSessionsPipeline(sessionStartTime.Unix(), sessionEndTime.Unix(), store.NewLevelDbManager(*dbRoot), outputName)
}
func pipelineBytesPerMinute() transformer.Pipeline {
flagset := flag.NewFlagSet("bytesperminute", flag.ExitOnError)
dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
flagset.Parse(flag.Args()[1:])
return passive.BytesPerMinutePipeline(store.NewLevelDbManager(*dbRoot), passive.NewBytesPerHourPostgresStore())
}