Пример #1
0
// Test that consumer exhausts iterators even in the corner case where
// the write buffer fills up and all that is left on the iterator are
// types the writer doesn't support.
func TestBug1586(t *testing.T) {
	builder := pstore.NewConsumerWithMetricsBuilder(
		noStringsNilWriterType{})
	// Buffer size and number of int64 values have to match.
	builder.SetBufferSize(20)
	consumer := builder.Build()

	// 20 int64 values that can be written to pstore
	int64Iterator := newNamedIteratorSameValueType(
		"int64Iterator", 20, int64(37), kInt64MetricInfo)
	// 5 string values that cannot be written to pstore
	stringIterator := newNamedIteratorSameValueType(
		"stringIterator", 5, "foo", kStringMetricInfo)
	// Final iterator yields 20 int64 values followed by 5 string values.
	iterator := compoundIteratorType{
		int64Iterator, stringIterator}

	consumer.Write(iterator, "someHost", "someApp")
	consumer.Flush()

	// Writing to the consumer should exhaust all the values in the
	// iterator including the string values left over after the buffer
	// fills.
	int64Iterator.VerifyExhausted(t)
	stringIterator.VerifyExhausted(t)
}
Пример #2
0
func TestSubTypes(t *testing.T) {
	writer := &noListStringWriterType{}
	builder := pstore.NewConsumerWithMetricsBuilder(writer)
	consumer := builder.Build()

	// list int64 values that can be written to pstore
	listInt64Iterator := newNamedIteratorSameValueType(
		"listInt64Iterator",
		3,
		[]int64{},
		kListInt64MetricInfo)
	// list string values that cannot be written to pstore
	listStringIterator := newNamedIteratorSameValueType(
		"listStringIterator",
		2,
		[]string{},
		kListStringMetricInfo)
	// Final iterator yields 3 list int64 values
	// followed by 2 list string values.
	iterator := compoundIteratorType{
		listInt64Iterator, listStringIterator}

	consumer.Write(iterator, "someHost", "someApp")
	consumer.Flush()

	// three values should be written
	if assertValueEquals(t, 3, len(writer.Written)) {
		for i := 0; i < 3; i++ {
			assertValueEquals(
				t, types.List, writer.Written[i].Kind)
			assertValueEquals(
				t, types.Int64, writer.Written[i].SubType)
		}
	}
}
Пример #3
0
func (c ConsumerConfig) newConsumerBuilder(wf WriterFactory) (
	result *pstore.ConsumerWithMetricsBuilder, err error) {
	if c.Name == "" {
		return nil, errors.New("Name field is required.")
	}
	if c.BatchSize < 1 {
		c.BatchSize = kDefaultBatchSize
	}
	if c.Concurrency < 1 {
		c.Concurrency = 1
	}
	if c.RollUpSpan < 0 {
		c.RollUpSpan = 0
	}
	writer, err := wf.NewWriter()
	if err != nil {
		return
	}
	builder := pstore.NewConsumerWithMetricsBuilder(writer)
	if c.RecordsPerSecond > 0 {
		builder.SetRecordsPerSecond(c.RecordsPerSecond)
	}
	if c.DebugMetricRegex != "" || c.DebugHostRegex != "" {
		var hook pstore.RecordWriteHooker
		hook, err = newWriteHooker(
			c.DebugMetricRegex, c.DebugHostRegex, c.DebugFilePath)
		if err != nil {
			return
		}
		builder.AddHook(hook)
	}
	builder.SetConcurrency(c.Concurrency)
	builder.SetBufferSize(c.BatchSize)
	builder.SetRollUpSpan(c.RollUpSpan)
	builder.SetName(c.Name)
	return builder, nil
}