func runAggregateStatisticsPipeline(consistentRanges []*store.Record, allTraces ...map[string]Trace) { levelDbManager := store.NewSliceManager() availabilityIntervalsStore := levelDbManager.Writer("consistent-ranges") availabilityIntervalsStore.BeginWriting() for _, record := range consistentRanges { availabilityIntervalsStore.WriteRecord(record) } availabilityIntervalsStore.EndWriting() var writer *bytes.Buffer tracesStore := levelDbManager.Writer("traces") for _, traces := range allTraces { tracesStore.BeginWriting() for encodedKey, trace := range traces { encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } tracesStore.EndWriting() writer = bytes.NewBuffer([]byte{}) transformer.RunPipeline(AggregateStatisticsPipeline(levelDbManager, writer)) } fmt.Printf("%s", writer.Bytes()) }
func ExampleFilterSessions() { usecs := int64(1000000) levelDbManager := store.NewSliceManager() traceKeyRangesStore := levelDbManager.Writer("availability-done") traceKeyRangesStore.BeginWriting() traceKeyRangesStore.WriteRecord(makeRangeRecord("node", 30*usecs, 0, 2)) traceKeyRangesStore.WriteRecord(makeRangeRecord("node", 31*usecs, 0, 1)) traceKeyRangesStore.WriteRecord(makeRangeRecord("node", 100*usecs, 0, 10)) traceKeyRangesStore.WriteRecord(makeRangeRecord("node", 200*usecs, 2, 8)) traceKeyRangesStore.BeginWriting() tracesStore := levelDbManager.Writer("traces") tracesStore.BeginWriting() tracesStore.WriteRecord(makeSessionRecord("node", 30*usecs, 1)) tracesStore.WriteRecord(makeSessionRecord("node", 31*usecs, 3)) tracesStore.WriteRecord(makeSessionRecord("node", 100*usecs, 2)) tracesStore.WriteRecord(makeSessionRecord("node", 200*usecs, 3)) tracesStore.EndWriting() runFilterSessionsPipeline(80, 120, levelDbManager) // Output: // node 30000000 1 // node 100000000 2 }
func runAvailabilityPipelineAugmented(startTimestamp int64, timestamps map[string]int64, moreTimestamps map[string]int64) { levelDbManager := store.NewSliceManager() tracesStore := levelDbManager.Writer("traces") tracesStore.BeginWriting() for encodedKey, timestamp := range timestamps { trace := Trace{ TraceCreationTimestamp: proto.Int64(timestamp), } encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } tracesStore.EndWriting() writer := bytes.NewBuffer([]byte{}) transformer.RunPipeline(AvailabilityPipeline(levelDbManager, writer, startTimestamp)) tracesStore.BeginWriting() for encodedKey, timestamp := range moreTimestamps { trace := Trace{ TraceCreationTimestamp: proto.Int64(timestamp), } encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } tracesStore.EndWriting() anotherTracesSlice := make([]*store.Record, 0) for encodedKey, timestamp := range moreTimestamps { trace := Trace{ TraceCreationTimestamp: proto.Int64(timestamp), } encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } anotherTracesSlice = append(anotherTracesSlice, &store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } anotherWriter := bytes.NewBuffer([]byte{}) transformer.RunPipeline(AvailabilityPipeline(levelDbManager, anotherWriter, startTimestamp)) fmt.Printf("%s", anotherWriter.Bytes()) }
func runBytesPerDevicePipeline(consistentRanges []*store.Record, allTraces ...map[string]Trace) { levelDbManager := store.NewSliceManager() availabilityIntervalsStore := levelDbManager.Writer("consistent-ranges") availabilityIntervalsStore.BeginWriting() for _, record := range consistentRanges { availabilityIntervalsStore.WriteRecord(record) } availabilityIntervalsStore.EndWriting() bytesPerDevicePostgresStore := store.SliceStore{} tracesStore := levelDbManager.Writer("traces") for _, traces := range allTraces { tracesStore.BeginWriting() for encodedKey, trace := range traces { encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } tracesStore.EndWriting() transformer.RunPipeline(BytesPerDevicePipeline(levelDbManager, &bytesPerDevicePostgresStore)) } bytesPerDeviceStore := levelDbManager.Reader("bytesperdevice") bytesPerDeviceStore.BeginReading() for { record, err := bytesPerDeviceStore.ReadRecord() if err != nil { panic(err) } if record == nil { break } var nodeId, macAddress string var timestamp, count int64 lex.DecodeOrDie(record.Key, &nodeId, &macAddress, ×tamp) lex.DecodeOrDie(record.Value, &count) fmt.Printf("%s,%s,%d: %d\n", nodeId, macAddress, timestamp, count) } bytesPerDeviceStore.EndReading() }
func runFilesystemUsagePipeline(logs map[string]string) { levelDbManager := store.NewSliceManager() logsStore := levelDbManager.Writer("logs") logsStore.BeginWriting() for encodedKey, content := range logs { record := store.Record{ Key: []byte(encodedKey), Value: lex.EncodeOrDie(content), } logsStore.WriteRecord(&record) } logsStore.EndWriting() csvManager := store.NewCsvStdoutManager() transformer.RunPipeline(FilesystemUsagePipeline(levelDbManager, csvManager)) csvManager.PrintToStdout("filesystem.csv") }
func runBytesPerMinutePipeline(allTraces ...map[string]Trace) { levelDbManager := store.NewSliceManager() bytesPerHourPostgresStore := store.SliceStore{} tracesStore := levelDbManager.Writer("traces") for _, traces := range allTraces { tracesStore.BeginWriting() for encodedKey, trace := range traces { encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } tracesStore.EndWriting() transformer.RunPipeline(BytesPerMinutePipeline(levelDbManager, &bytesPerHourPostgresStore)) } bytesPerMinuteStore := levelDbManager.Reader("bytesperminute") bytesPerMinuteStore.BeginReading() for { record, err := bytesPerMinuteStore.ReadRecord() if err != nil { panic(err) } if record == nil { break } var nodeId string var timestamp, count int64 lex.DecodeOrDie(record.Key, &nodeId, ×tamp) lex.DecodeOrDie(record.Value, &count) fmt.Printf("%s,%d: %d\n", nodeId, timestamp, count) } bytesPerMinuteStore.EndReading() }
func runLookupsPerDevicePipeline(traces map[string]Trace, consistentRanges []*store.Record, addressIdToMac map[string]string) { levelDbManager := store.NewSliceManager() tracesStore := levelDbManager.Writer("traces") tracesStore.BeginWriting() for encodedKey, trace := range traces { encodedTrace, err := proto.Marshal(&trace) if err != nil { panic(fmt.Errorf("Error encoding protocol buffer: %v", err)) } tracesStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: encodedTrace}) } tracesStore.EndWriting() availabilityIntervalsStore := levelDbManager.Writer("consistent-ranges") availabilityIntervalsStore.BeginWriting() for _, record := range consistentRanges { availabilityIntervalsStore.WriteRecord(record) } availabilityIntervalsStore.EndWriting() addressIdStore := levelDbManager.Writer("bytesperdomain-address-id-table") addressIdStore.BeginWriting() for encodedKey, encodedValue := range addressIdToMac { addressIdStore.WriteRecord(&store.Record{Key: []byte(encodedKey), Value: []byte(encodedValue)}) } addressIdStore.EndWriting() transformer.RunPipeline(LookupsPerDevicePipeline(levelDbManager)) fmt.Printf("LookupsPerDevice:\n") lookupsPerDeviceStore := levelDbManager.Reader("lookupsperdevice-lookups-per-device") lookupsPerDeviceStore.BeginReading() for { record, err := lookupsPerDeviceStore.ReadRecord() if err != nil { panic(err) } if record == nil { break } var ( nodeId, macAddress, domain string count int64 ) lex.DecodeOrDie(record.Key, &nodeId, &macAddress, &domain) lex.DecodeOrDie(record.Value, &count) fmt.Printf("%s,%s,%s: %d\n", nodeId, macAddress, domain, count) } lookupsPerDeviceStore.EndReading() fmt.Printf("\nLookupsPerDevicePerHour:\n") lookupsPerDevicePerHourStore := levelDbManager.Reader("lookupsperdevice-lookups-per-device-per-hour") lookupsPerDevicePerHourStore.BeginReading() for { record, err := lookupsPerDevicePerHourStore.ReadRecord() if err != nil { panic(err) } if record == nil { break } var ( nodeId, macAddress, domain string timestamp, count int64 ) lex.DecodeOrDie(record.Key, &nodeId, &macAddress, &domain, ×tamp) lex.DecodeOrDie(record.Value, &count) fmt.Printf("%s,%s,%s,%d: %d\n", nodeId, macAddress, domain, timestamp, count) } lookupsPerDevicePerHourStore.EndReading() }