Ejemplo n.º 1
0
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
	allTarballsPattern := filepath.Join(tarballsPath, "all", "health", "*", "*", "health_*.tar.gz")
	dailyTarballsPattern := filepath.Join(tarballsPath, "by-date", "*", "health", "*", "health_*.tar.gz")
	tarnamesStore := levelDbManager.ReadingWriter("tarnames")
	tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
	logsStore := levelDbManager.Writer("logs")
	return []transformer.PipelineStage{
		transformer.PipelineStage{
			Name:   "ScanLogTarballs",
			Reader: store.NewGlobReader(allTarballsPattern),
			Writer: tarnamesStore,
		},
		transformer.PipelineStage{
			Name:   "ScanDailyLogTarballs",
			Reader: store.NewGlobReader(dailyTarballsPattern),
			Writer: tarnamesStore,
		},
		transformer.PipelineStage{
			Name:        "ReadLogTarballs",
			Reader:      store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
			Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
			Writer:      store.NewMuxingWriter(logsStore, tarnamesIndexedStore),
		},
	}
}
Ejemplo n.º 2
0
func ExampleJoin_inner() {
	left := store.SliceStore{}
	left.BeginWriting()
	left.WriteRecord(store.NewRecord("1", "news", 0))
	left.WriteRecord(store.NewRecord("2", "book", 0))
	left.EndWriting()

	right := store.SliceStore{}
	right.BeginWriting()
	right.WriteRecord(store.NewRecord("1", "paper", 0))
	right.WriteRecord(store.NewRecord("3", "brush", 0))
	right.EndWriting()

	output := store.SliceStore{}

	joiner := Join(nil, nil)
	transformer.RunTransformer(transformer.MakeGroupDoFunc(joiner), store.NewDemuxingReader(&left, &right), &output)

	output.BeginReading()
	for {
		record, err := output.ReadRecord()
		if err != nil {
			panic(err)
		}
		if record == nil {
			break
		}
		fmt.Printf("%s: %s\n", record.Key, record.Value)
	}
	output.EndReading()

	// Output:
	//
	// 1: newspaper
}
func FilterSessionsPipeline(sessionStartTime, sessionEndTime int64, levelDbManager store.Manager, outputName string) transformer.Pipeline {
	tracesStore := levelDbManager.Reader("traces")
	traceKeyRangesStore := levelDbManager.Reader("availability-done")
	filteredStore := levelDbManager.Writer(outputName)
	parameters := filterSessions{
		SessionStartTime: sessionStartTime * 1000000,
		SessionEndTime:   sessionEndTime * 1000000,
	}
	return []transformer.PipelineStage{
		transformer.PipelineStage{
			Name:        "FilterSessions",
			Reader:      store.NewDemuxingReader(traceKeyRangesStore, tracesStore),
			Transformer: parameters,
			Writer:      filteredStore,
		},
	}
}
Ejemplo n.º 4
0
func IndexTarballsPipeline(tarballsPath string, levelDbManager store.Manager) transformer.Pipeline {
	tarballsPattern := filepath.Join(tarballsPath, "*", "*", "*.tar.gz")
	tarnamesStore := levelDbManager.ReadingWriter("tarnames")
	tarnamesIndexedStore := levelDbManager.ReadingWriter("tarnames-indexed")
	tracesStore := levelDbManager.Writer("traces")
	return []transformer.PipelineStage{
		transformer.PipelineStage{
			Name:   "ScanTraceTarballs",
			Reader: store.NewGlobReader(tarballsPattern),
			Writer: tarnamesStore,
		},
		transformer.PipelineStage{
			Name:        "IndexTraces",
			Transformer: transformer.MakeMultipleOutputsGroupDoFunc(IndexTarballs, 2),
			Reader:      store.NewDemuxingReader(tarnamesStore, tarnamesIndexedStore),
			Writer:      store.NewMuxingWriter(tracesStore, tarnamesIndexedStore),
		},
	}
}
Ejemplo n.º 5
0
func ExampleJoin_multiple() {
	left := store.SliceStore{}
	left.BeginWriting()
	left.WriteRecord(store.NewRecord("1", "what", 0))
	left.WriteRecord(store.NewRecord("2", "news", 0))
	left.EndWriting()

	middle := store.SliceStore{}
	middle.BeginWriting()
	middle.WriteRecord(store.NewRecord("1", "so", 0))
	middle.WriteRecord(store.NewRecord("3", "the", 0))
	middle.EndWriting()

	right := store.SliceStore{}
	right.BeginWriting()
	right.WriteRecord(store.NewRecord("1", "ever", 0))
	right.WriteRecord(store.NewRecord("3", "less", 0))
	right.EndWriting()

	output := store.SliceStore{}

	joiner := Join([]byte("never"), []byte("paper"), []byte("man"))
	transformer.RunTransformer(transformer.MakeGroupDoFunc(joiner), store.NewDemuxingReader(&left, &middle, &right), &output)

	output.BeginReading()
	for {
		record, err := output.ReadRecord()
		if err != nil {
			panic(err)
		}
		if record == nil {
			break
		}
		fmt.Printf("%s: %s\n", record.Key, record.Value)
	}
	output.EndReading()

	// Output:
	//
	// 1: whatsoever
	// 2: newspaperman
	// 3: nevertheless
}