// Test that consumer exhausts iterators even in the corner case where
// the write buffer fills up and all that is left on the iterator are
// types the writer doesn't support.
func TestBug1586(t *testing.T) {
builder := pstore.NewConsumerWithMetricsBuilder(
noStringsNilWriterType{})
// Buffer size and number of int64 values have to match.
builder.SetBufferSize(20)
consumer := builder.Build()
// 20 int64 values that can be written to pstore
int64Iterator := newNamedIteratorSameValueType(
"int64Iterator", 20, int64(37), kInt64MetricInfo)
// 5 string values that cannot be written to pstore
stringIterator := newNamedIteratorSameValueType(
"stringIterator", 5, "foo", kStringMetricInfo)
// Final iterator yields 20 int64 values followed by 5 string values.
iterator := compoundIteratorType{
int64Iterator, stringIterator}
consumer.Write(iterator, "someHost", "someApp")
consumer.Flush()
// Writing to the consumer should exhaust all the values in the
// iterator including the string values left over after the buffer
// fills.
int64Iterator.VerifyExhausted(t)
stringIterator.VerifyExhausted(t)
}
func TestSubTypes(t *testing.T) {
writer := &noListStringWriterType{}
builder := pstore.NewConsumerWithMetricsBuilder(writer)
consumer := builder.Build()
// list int64 values that can be written to pstore
listInt64Iterator := newNamedIteratorSameValueType(
"listInt64Iterator",
3,
[]int64{},
kListInt64MetricInfo)
// list string values that cannot be written to pstore
listStringIterator := newNamedIteratorSameValueType(
"listStringIterator",
2,
[]string{},
kListStringMetricInfo)
// Final iterator yields 3 list int64 values
// followed by 2 list string values.
iterator := compoundIteratorType{
listInt64Iterator, listStringIterator}
consumer.Write(iterator, "someHost", "someApp")
consumer.Flush()
// three values should be written
if assertValueEquals(t, 3, len(writer.Written)) {
for i := 0; i < 3; i++ {
assertValueEquals(
t, types.List, writer.Written[i].Kind)
assertValueEquals(
t, types.Int64, writer.Written[i].SubType)
}
}
}
func (c ConsumerConfig) newConsumerBuilder(wf WriterFactory) (
result *pstore.ConsumerWithMetricsBuilder, err error) {
if c.Name == "" {
return nil, errors.New("Name field is required.")
}
if c.BatchSize < 1 {
c.BatchSize = kDefaultBatchSize
}
if c.Concurrency < 1 {
c.Concurrency = 1
}
if c.RollUpSpan < 0 {
c.RollUpSpan = 0
}
writer, err := wf.NewWriter()
if err != nil {
return
}
builder := pstore.NewConsumerWithMetricsBuilder(writer)
if c.RecordsPerSecond > 0 {
builder.SetRecordsPerSecond(c.RecordsPerSecond)
}
if c.DebugMetricRegex != "" || c.DebugHostRegex != "" {
var hook pstore.RecordWriteHooker
hook, err = newWriteHooker(
c.DebugMetricRegex, c.DebugHostRegex, c.DebugFilePath)
if err != nil {
return
}
builder.AddHook(hook)
}
builder.SetConcurrency(c.Concurrency)
builder.SetBufferSize(c.BatchSize)
builder.SetRollUpSpan(c.RollUpSpan)
builder.SetName(c.Name)
return builder, nil
}