func runConsolidateTraceKeyRanges(records, moreRecords []*store.Record) {
traces := store.SliceStore{}
traces.BeginWriting()
for _, record := range records {
traces.WriteRecord(record)
}
traces.EndWriting()
rangesStore := store.SliceStore{}
consolidatedStore := store.SliceStore{}
transformer.RunPipeline(TraceKeyRangesPipeline(&traces, &rangesStore, &consolidatedStore))
moreTraces := store.SliceStore{}
moreTraces.BeginWriting()
for _, record := range moreRecords {
moreTraces.WriteRecord(record)
}
moreTraces.EndWriting()
transformer.RunPipeline(TraceKeyRangesPipeline(&moreTraces, &rangesStore, &consolidatedStore))
rangesStore.BeginReading()
for {
record, err := rangesStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
fmt.Printf("%s %s\n", formatTraceKey(record.Key), formatTraceKey(record.Value))
}
rangesStore.EndReading()
}
func runAvailabilityPipelineAugmented(startTimestamp int64, timestamps map[string]int64, moreTimestamps map[string]int64) {
levelDbManager := store.NewSliceManager()
tracesStore := levelDbManager.Writer("traces")
tracesStore.BeginWriting()
for encodedKey, timestamp := range timestamps {
trace := Trace{
TraceCreationTimestamp: proto.Int64(timestamp),
}
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
tracesStore.EndWriting()
writer := bytes.NewBuffer([]byte{})
transformer.RunPipeline(AvailabilityPipeline(levelDbManager, writer, startTimestamp))
tracesStore.BeginWriting()
for encodedKey, timestamp := range moreTimestamps {
trace := Trace{
TraceCreationTimestamp: proto.Int64(timestamp),
}
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
tracesStore.EndWriting()
anotherTracesSlice := make([]*store.Record, 0)
for encodedKey, timestamp := range moreTimestamps {
trace := Trace{
TraceCreationTimestamp: proto.Int64(timestamp),
}
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
anotherTracesSlice = append(anotherTracesSlice, &store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
anotherWriter := bytes.NewBuffer([]byte{})
transformer.RunPipeline(AvailabilityPipeline(levelDbManager, anotherWriter, startTimestamp))
fmt.Printf("%s", anotherWriter.Bytes())
}
func runSessions(records []*store.Record) {
traces := store.SliceStore{}
traces.BeginWriting()
for _, record := range records {
traces.WriteRecord(record)
}
traces.EndWriting()
sessionsStore := store.SliceStore{}
transformer.RunPipeline([]transformer.PipelineStage{
SessionPipelineStage(&traces, &sessionsStore),
})
sessionsStore.BeginReading()
for {
record, err := sessionsStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
fmt.Printf("%s\n", formatSessionKey(record.Key))
}
sessionsStore.EndReading()
}
func runAggregateStatisticsPipeline(consistentRanges []*store.Record, allTraces ...map[string]Trace) {
levelDbManager := store.NewSliceManager()
availabilityIntervalsStore := levelDbManager.Writer("consistent-ranges")
availabilityIntervalsStore.BeginWriting()
for _, record := range consistentRanges {
availabilityIntervalsStore.WriteRecord(record)
}
availabilityIntervalsStore.EndWriting()
var writer *bytes.Buffer
tracesStore := levelDbManager.Writer("traces")
for _, traces := range allTraces {
tracesStore.BeginWriting()
for encodedKey, trace := range traces {
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
tracesStore.EndWriting()
writer = bytes.NewBuffer([]byte{})
transformer.RunPipeline(AggregateStatisticsPipeline(levelDbManager, writer))
}
fmt.Printf("%s", writer.Bytes())
}
func main() {
pipelineFuncs := map[string]transformer.PipelineThunk{
"index": pipelineIndex,
"disjoint": pipelineDisjointPackages,
}
name, pipeline := transformer.ParsePipelineChoice(pipelineFuncs)
go cube.Run(fmt.Sprintf("bismark_experiments_manager_logs_pipeline_%s", name))
transformer.RunPipeline(pipeline)
}
func main() {
pipelineFuncs := map[string]transformer.PipelineThunk{
"csv": pipelineCsv,
"stats": pipelineStats,
"summarize": pipelineSummarize,
"timescsv": pipelineTimesCsv,
}
name, pipeline := transformer.ParsePipelineChoice(pipelineFuncs)
go cube.Run(fmt.Sprintf("bismark_uploads_stats_%s", name))
transformer.RunPipeline(pipeline)
}
func runBytesPerDevicePipeline(consistentRanges []*store.Record, allTraces ...map[string]Trace) {
levelDbManager := store.NewSliceManager()
availabilityIntervalsStore := levelDbManager.Writer("consistent-ranges")
availabilityIntervalsStore.BeginWriting()
for _, record := range consistentRanges {
availabilityIntervalsStore.WriteRecord(record)
}
availabilityIntervalsStore.EndWriting()
bytesPerDevicePostgresStore := store.SliceStore{}
tracesStore := levelDbManager.Writer("traces")
for _, traces := range allTraces {
tracesStore.BeginWriting()
for encodedKey, trace := range traces {
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
tracesStore.EndWriting()
transformer.RunPipeline(BytesPerDevicePipeline(levelDbManager, &bytesPerDevicePostgresStore))
}
bytesPerDeviceStore := levelDbManager.Reader("bytesperdevice")
bytesPerDeviceStore.BeginReading()
for {
record, err := bytesPerDeviceStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
var nodeId, macAddress string
var timestamp, count int64
lex.DecodeOrDie(record.Key, &nodeId, &macAddress, ×tamp)
lex.DecodeOrDie(record.Value, &count)
fmt.Printf("%s,%s,%d: %d\n", nodeId, macAddress, timestamp, count)
}
bytesPerDeviceStore.EndReading()
}
func runFilesystemUsagePipeline(logs map[string]string) {
levelDbManager := store.NewSliceManager()
logsStore := levelDbManager.Writer("logs")
logsStore.BeginWriting()
for encodedKey, content := range logs {
record := store.Record{
Key: []byte(encodedKey),
Value: lex.EncodeOrDie(content),
}
logsStore.WriteRecord(&record)
}
logsStore.EndWriting()
csvManager := store.NewCsvStdoutManager()
transformer.RunPipeline(FilesystemUsagePipeline(levelDbManager, csvManager))
csvManager.PrintToStdout("filesystem.csv")
}
func main() {
pipelineFuncs := map[string]transformer.PipelineThunk{
"devicescount": pipelineDevicesCount,
"filesystem": pipelineFilesystem,
"index": pipelineIndex,
"iproute": pipelineIpRoute,
"memory": pipelineMemory,
"packages": pipelinePackages,
"reboots": pipelineReboots,
"summarize": pipelineSummarize,
"uptime": pipelineUptime,
}
name, pipeline := transformer.ParsePipelineChoice(pipelineFuncs)
go cube.Run(fmt.Sprintf("bismark_health_pipeline_%s", name))
transformer.RunPipeline(pipeline)
}
func main() {
pipelineFuncs := map[string]transformer.PipelineThunk{
"availability": pipelineAvailability,
"bytesperdevice": pipelineBytesPerDevice,
"bytesperdomain": pipelineBytesPerDomain,
"bytesperminute": pipelineBytesPerMinute,
"filternode": pipelineFilterNode,
"filterdates": pipelineFilterDates,
"index": pipelineIndex,
"lookupsperdevice": pipelineLookupsPerDevice,
"statistics": pipelineStatistics,
}
name, pipeline := transformer.ParsePipelineChoice(pipelineFuncs)
go cube.Run(fmt.Sprintf("bismark_passive_pipeline_%s", name))
transformer.RunPipeline(pipeline)
}
func runFilterSessionsPipeline(startSecs, endSecs int64, levelDbManager store.Manager) {
transformer.RunPipeline(FilterSessionsPipeline(startSecs, endSecs, levelDbManager, "test"))
filteredStore := levelDbManager.Reader("test")
filteredStore.BeginReading()
for {
record, err := filteredStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
var traceKey TraceKey
lex.DecodeOrDie(record.Key, &traceKey)
fmt.Printf("%s %d %d\n", traceKey.NodeId, traceKey.SessionId, traceKey.SequenceNumber)
}
filteredStore.EndReading()
}
func runBytesPerMinutePipeline(allTraces ...map[string]Trace) {
levelDbManager := store.NewSliceManager()
bytesPerHourPostgresStore := store.SliceStore{}
tracesStore := levelDbManager.Writer("traces")
for _, traces := range allTraces {
tracesStore.BeginWriting()
for encodedKey, trace := range traces {
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
tracesStore.EndWriting()
transformer.RunPipeline(BytesPerMinutePipeline(levelDbManager, &bytesPerHourPostgresStore))
}
bytesPerMinuteStore := levelDbManager.Reader("bytesperminute")
bytesPerMinuteStore.BeginReading()
for {
record, err := bytesPerMinuteStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
var nodeId string
var timestamp, count int64
lex.DecodeOrDie(record.Key, &nodeId, ×tamp)
lex.DecodeOrDie(record.Value, &count)
fmt.Printf("%s,%d: %d\n", nodeId, timestamp, count)
}
bytesPerMinuteStore.EndReading()
}
func runLookupsPerDevicePipeline(traces map[string]Trace, consistentRanges []*store.Record, addressIdToMac map[string]string) {
levelDbManager := store.NewSliceManager()
tracesStore := levelDbManager.Writer("traces")
tracesStore.BeginWriting()
for encodedKey, trace := range traces {
encodedTrace, err := proto.Marshal(&trace)
if err != nil {
panic(fmt.Errorf("Error encoding protocol buffer: %v", err))
}
tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace})
}
tracesStore.EndWriting()
availabilityIntervalsStore := levelDbManager.Writer("consistent-ranges")
availabilityIntervalsStore.BeginWriting()
for _, record := range consistentRanges {
availabilityIntervalsStore.WriteRecord(record)
}
availabilityIntervalsStore.EndWriting()
addressIdStore := levelDbManager.Writer("bytesperdomain-address-id-table")
addressIdStore.BeginWriting()
for encodedKey, encodedValue := range addressIdToMac {
addressIdStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: []byte(encodedValue)})
}
addressIdStore.EndWriting()
transformer.RunPipeline(LookupsPerDevicePipeline(levelDbManager))
fmt.Printf("LookupsPerDevice:\n")
lookupsPerDeviceStore := levelDbManager.Reader("lookupsperdevice-lookups-per-device")
lookupsPerDeviceStore.BeginReading()
for {
record, err := lookupsPerDeviceStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
var (
nodeId, macAddress, domain string
count int64
)
lex.DecodeOrDie(record.Key, &nodeId, &macAddress, &domain)
lex.DecodeOrDie(record.Value, &count)
fmt.Printf("%s,%s,%s: %d\n", nodeId, macAddress, domain, count)
}
lookupsPerDeviceStore.EndReading()
fmt.Printf("\nLookupsPerDevicePerHour:\n")
lookupsPerDevicePerHourStore := levelDbManager.Reader("lookupsperdevice-lookups-per-device-per-hour")
lookupsPerDevicePerHourStore.BeginReading()
for {
record, err := lookupsPerDevicePerHourStore.ReadRecord()
if err != nil {
panic(err)
}
if record == nil {
break
}
var (
nodeId, macAddress, domain string
timestamp, count int64
)
lex.DecodeOrDie(record.Key, &nodeId, &macAddress, &domain, ×tamp)
lex.DecodeOrDie(record.Value, &count)
fmt.Printf("%s,%s,%s,%d: %d\n", nodeId, macAddress, domain, timestamp, count)
}
lookupsPerDevicePerHourStore.EndReading()
}