Пример #1
0
func pipelineStats() transformer.Pipeline {
	flagset := flag.NewFlagSet("stats", flag.ExitOnError)
	dbRoot := flagset.String("uploads_leveldb_root", "/data/users/sburnett/bismark-upload-stats-leveldb", "Write leveldbs in this directory.")
	tarballsPath := flagset.String("tarballs_path", "/var/local/home/bismark-data-xfer/bismark_data_from_s3", "Read tarballs from this directory.")
	flagset.Parse(flag.Args()[1:])
	return stats.ExtractStatsPipeline(*tarballsPath, store.NewLevelDbManager(*dbRoot))
}
Пример #2
0
func pipelineTimesCsv() transformer.Pipeline {
	flagset := flag.NewFlagSet("csv", flag.ExitOnError)
	dbRoot := flagset.String("uploads_leveldb_root", "/data/users/sburnett/bismark-upload-stats-leveldb", "Write leveldbs in this directory.")
	csvOutput := flagset.String("csv_output", "/dev/null", "Directory where we will write statistics in CSV format.")
	flagset.Parse(flag.Args()[1:])
	return stats.TimesCsvPipeline(store.NewLevelDbManager(*dbRoot), *csvOutput)
}
Пример #3
0
func pipelineFilesystem() transformer.Pipeline {
	flagset := flag.NewFlagSet("filesystem", flag.ExitOnError)
	dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
	csvOutput := flagset.String("csv_output", "/dev/null", "Write filesystem usage in CSV format to this file.")
	flagset.Parse(flag.Args()[1:])
	return health.FilesystemUsagePipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
Пример #4
0
func pipelineCsv() transformer.Pipeline {
	flagset := flag.NewFlagSet("csv", flag.ExitOnError)
	csvOutput := flagset.String("csv_output", "/dev/null", "Write upload statistics to a file in this directory.")
	dbRoot := flagset.String("uploads_leveldb_root", "/data/users/sburnett/bismark-upload-stats-leveldb", "Write leveldbs in this directory.")
	flagset.Parse(flag.Args()[1:])
	return stats.CsvPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
Пример #5
0
func pipelineIpRoute() transformer.Pipeline {
	flagset := flag.NewFlagSet("iproute", flag.ExitOnError)
	dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
	sqliteFilename := flagset.String("sqlite_filename", "/dev/null", "Write to this sqlite database.")
	flagset.Parse(flag.Args()[1:])
	return health.IpRoutePipeline(store.NewLevelDbManager(*dbRoot), store.NewSqliteManager(*sqliteFilename))
}
Пример #6
0
func pipelineSummarize() transformer.Pipeline {
	flagset := flag.NewFlagSet("summarize", flag.ExitOnError)
	dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
	csvOutput := flagset.String("csv_output", "/dev/null", "Write reboots to a CSV file in this directory.")
	flagset.Parse(flag.Args()[1:])
	return health.SummarizeHealthPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
Пример #7
0
func pipelineFilterNode() transformer.Pipeline {
	flagset := flag.NewFlagSet("filter", flag.ExitOnError)
	dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
	nodeId := flagset.String("node_id", "OWC43DC7B0AE78", "Retain only data from this router.")
	flagset.Parse(flag.Args()[1:])
	return passive.FilterNodesPipeline(*nodeId, store.NewLevelDbManager(*dbRoot))
}
Пример #8
0
func pipelineIndex() transformer.Pipeline {
	flagset := flag.NewFlagSet("index", flag.ExitOnError)
	tarballsPath := flagset.String("tarballs_path", "/data/users/sburnett/bismark-health", "Read tarballs from this directory.")
	dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
	flagset.Parse(flag.Args()[1:])
	return health.IndexTarballsPipeline(*tarballsPath, store.NewLevelDbManager(*dbRoot))
}
Пример #9
0
func pipelineDisjointPackages() transformer.Pipeline {
	flagset := flag.NewFlagSet("disjoint", flag.ExitOnError)
	dbRoot := flagset.String("logs_leveldb_root", "/data/users/sburnett/bismark-experiments-manager-logs-leveldb", "Write leveldbs in this directory.")
	csvOutput := flagset.String("csv_output", "/dev/null", "Write disjoint packages statistics in CSV format to this file.")
	flagset.Parse(flag.Args()[1:])
	return experiments.DisjointPackagesPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput))
}
Пример #10
0
func pipelinePackages() transformer.Pipeline {
	flagset := flag.NewFlagSet("packages", flag.ExitOnError)
	dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
	csvOutput := flagset.String("csv_output", "/dev/null", "Write reboots to a CSV file in this directory.")
	sqliteFilename := flagset.String("sqlite_filename", "/dev/null", "Write to this sqlite database.")
	flagset.Parse(flag.Args()[1:])
	return health.PackagesPipeline(store.NewLevelDbManager(*dbRoot), store.NewCsvFileManager(*csvOutput), store.NewSqliteManager(*sqliteFilename))
}
Пример #11
0
func pipelineStatistics() transformer.Pipeline {
	flagset := flag.NewFlagSet("statistics", flag.ExitOnError)
	dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
	jsonOutput := flagset.String("json_output", "/dev/null", "Write statistics in JSON format to this file.")
	flagset.Parse(flag.Args()[2:])
	jsonHandle, err := os.Create(*jsonOutput)
	if err != nil {
		log.Fatalf("Error opening JSON output: %v", err)
	}
	return passive.AggregateStatisticsPipeline(store.NewLevelDbManager(*dbRoot), jsonHandle)
}
Пример #12
0
func pipelineDevicesCount() transformer.Pipeline {
	flagset := flag.NewFlagSet("devicescount", flag.ExitOnError)
	dbRoot := flagset.String("health_leveldb_root", "/data/users/sburnett/bismark-health-leveldb", "Write leveldbs in this directory.")
	flagset.Parse(flag.Args()[1:])
	return health.DevicesCountPipeline(store.NewLevelDbManager(*dbRoot))
}
Пример #13
0
func pipelineLookupsPerDevice() transformer.Pipeline {
	flagset := flag.NewFlagSet("lookupsperdevice", flag.ExitOnError)
	dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
	flagset.Parse(flag.Args()[1:])
	return passive.LookupsPerDevicePipeline(store.NewLevelDbManager(*dbRoot))
}
Пример #14
0
func pipelineFilterDates() transformer.Pipeline {
	flagset := flag.NewFlagSet("filter", flag.ExitOnError)
	dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
	sessionStartDate := flagset.String("session_start_date", "20120301", "Retain only session starting after this date, in YYYYMMDD format.")
	sessionEndDate := flagset.String("session_end_date", "20120401", "Retain only session starting before this date, in YYYYMMDD format.")
	flagset.Parse(flag.Args()[1:])
	timeFormatString := "20060102"
	sessionStartTime, err := time.Parse(timeFormatString, *sessionStartDate)
	if err != nil {
		panic(fmt.Errorf("Error parsing start date %s: %v", sessionStartDate, err))
	}
	sessionEndTime, err := time.Parse(timeFormatString, *sessionEndDate)
	if err != nil {
		panic(fmt.Errorf("Error parsing end date %s: %v", sessionEndDate, err))
	}
	outputName := fmt.Sprintf("filtered-%s-%s", *sessionStartDate, *sessionEndDate)
	return passive.FilterSessionsPipeline(sessionStartTime.Unix(), sessionEndTime.Unix(), store.NewLevelDbManager(*dbRoot), outputName)
}
Пример #15
0
func pipelineBytesPerMinute() transformer.Pipeline {
	flagset := flag.NewFlagSet("bytesperminute", flag.ExitOnError)
	dbRoot := flagset.String("passive_leveldb_root", "/data/users/sburnett/passive-leveldb-new", "Write leveldbs in this directory.")
	flagset.Parse(flag.Args()[1:])
	return passive.BytesPerMinutePipeline(store.NewLevelDbManager(*dbRoot), passive.NewBytesPerHourPostgresStore())
}