func TestWriteIndexBlocksLarge(t *testing.T) {
w := new(bytes.Buffer)
index := &index{evs: make(events, 5000)}
for i := 0; i < 5000; i++ {
index.evs[i] = &Event{Timestamp: i, block: rand.Int63(), offset: rand.Intn(4096)}
}
writeIndexBlocks(index, w)
if index.length != 50040 {
t.Errorf("Wrong written length: wanted: 50040, found: %d", index.length)
}
reader := blocks.NewByteReader(w.Bytes(), 4096)
for i, event := range index.evs {
block := binary.ReadInt64(reader)
offset := binary.ReadInt16(reader)
if event.block != block || event.offset != int(offset) {
t.Errorf("Case %d: Wrong read event block/offset. wanted: %d,%d found: %d,%d", i, event.block, event.offset, block, offset)
}
}
}
func TestWriteIndexBlocksSmall(t *testing.T) {
w := new(bytes.Buffer)
e1 := &Event{Timestamp: 1, block: 0, offset: 2048}
e2 := &Event{Timestamp: 3, block: 512, offset: 128}
e3 := &Event{Timestamp: 2, block: 2048, offset: 512}
e4 := &Event{Timestamp: 4, block: 248, offset: 1024}
index := &index{evs: events{e1, e2, e3, e4}}
writeIndexBlocks(index, w)
if index.length != 36 {
t.Errorf("Wrong written length: wanted: 36, found: %d", index.length)
}
compressed, _ := csnappy.Encode(nil, []byte(
"\xF8\x00\x00\x00\x00\x00\x00\x00"+"\x00\x04"+
"\x00\x02\x00\x00\x00\x00\x00\x00"+"\x80\x00"+
"\x00\x08\x00\x00\x00\x00\x00\x00"+"\x00\x02"+
"\x00\x00\x00\x00\x00\x00\x00\x00"+"\x00\x08"+
"\x00"))
expected := append([]byte("\x21\x00\x01"), compressed...)
if !reflect.DeepEqual(w.Bytes(), expected) {
t.Errorf("Wrong event block bytecode:\n wanted: %x\n found: %x", expected, w.Bytes())
}
want := events{e4, e2, e3, e1}
if !reflect.DeepEqual(index.evs, want) {
t.Errorf("Wrongly sorted events: wanted: %v found: %v", want, index.evs)
}
reader := blocks.NewByteReader(w.Bytes(), 4096)
for i, event := range index.evs {
block := binary.ReadInt64(reader)
offset := binary.ReadInt16(reader)
if event.block != block || event.offset != int(offset) {
t.Errorf("Case %d: Wrong read event block/offset. wanted: %d,%d found: %d,%d", i, event.block, event.offset, block, offset)
}
}
}
func TestWriteEventBlocksMedium(t *testing.T) {
w := new(bytes.Buffer)
e1 := newEvent(generate(2600), 1)
e2 := newEvent(generate(2800), 3)
e3 := newEvent(generate(2200), 2)
e4 := newEvent(generate(3400), 4)
index := &index{evs: events{e1, e2, e3, e4}}
writeEventBlocks(index, w)
var tests = []struct {
event *Event
block int64
offset int
}{
{e1, 419, 226},
{e2, 0, 3406},
{e3, 211, 2116},
{e4, 0, 0},
}
for i, test := range tests {
event := test.event
if event.block != test.block || event.offset != test.offset {
t.Errorf("Case %d: Wrong event block/offset. wanted: %d,%d found: %d,%d", i, test.block, test.offset, event.block, event.offset)
}
}
r := blocks.NewByteReader(w.Bytes(), 4096)
for i, dataLen := range []int{3400, 2800, 2200, 2600} {
e := pullEvent(r)
if len(e.Data) != dataLen {
t.Errorf("Case %d: Wrong read data. wanted: %d bytes found: %d bytes", i, dataLen, len(e.Data))
}
}
if e := pullEvent(r); e != nil {
t.Errorf("Case %d: Found unexpected written event %v", e.Data)
}
}
func TestWriteSpaceIndexes(t *testing.T) {
w := new(bytes.Buffer)
writer := newSpace(w, []byte("a"))
e1data := []byte("abc")
e2data := []byte("b")
e3data := []byte("c")
e4data := []byte("def")
e1 := newEvent(e1data, 1)
e2 := newEvent(e2data, 3)
e3 := newEvent(e3data, 2)
e4 := newEvent(e4data, 4)
writer.add(e1, "1", map[string]string{"a": "1"})
writer.add(e2, "1", map[string]string{"a": "2"})
writer.add(e3, "1", map[string]string{"a": "2"})
writer.add(e4, "1", map[string]string{"a": "1"})
writer.write()
var tests = []struct {
key string
offset int
length int
evs [][]byte
indexed events
}{
{"g1", 1, 32, [][]byte{e4data, e2data, e3data, e1data}, nil},
{"ia:1", 33, 21, nil, events{e4, e1}},
{"ia:2", 54, 23, nil, events{e2, e3}},
}
sst, _ := findSpaceIndex(bytes.NewReader(w.Bytes()), 0, int64(w.Len()))
for i, test := range tests {
val, _ := sst.Get([]byte(test.key))
r := bytes.NewReader(val)
offset := binary.ReadUvarint(r)
length := binary.ReadUvarint(r)
if int(offset) != test.offset || int(length) != test.length {
t.Errorf("Case %d: Wrong grouping encoding: want: %d,%d found: %d,%d", i, test.offset, test.length, offset, length)
}
reader := blocks.NewByteReader(w.Bytes(), 4096)
reader.Seek(int64(offset), 0)
if len(test.evs) > 0 {
for j, data := range test.evs {
if e := pullEvent(reader); !reflect.DeepEqual(e.Data, data) {
t.Errorf("Case %d/%d: Wrong event found: want: %s found: %s", i, j, data, e.Data)
}
}
if e := pullEvent(reader); e != nil {
t.Errorf("Wrong event found: want: nil found: %s", e.Data)
}
}
for j, event := range test.indexed {
block := binary.ReadInt64(reader)
offset := binary.ReadInt16(reader)
if block != event.block || int(offset) != event.offset {
t.Errorf("Case %d/%d: Wrong event index: want: %d,%d found: %d,%d", i, j, event.block, event.offset, block, offset)
}
}
}
}
func TestWriteSpaceGrouping(t *testing.T) {
w := new(bytes.Buffer)
writer := newSpace(w, []byte("a"))
e1data := []byte("abc")
e2data := []byte("b")
e3data := []byte("c")
e4data := []byte("def")
e1 := newEvent(e1data, 1)
e2 := newEvent(e2data, 3)
e3 := newEvent(e3data, 2)
e4 := newEvent(e4data, 4)
writer.add(e1, "1", nil)
writer.add(e2, "2", nil)
writer.add(e3, "2", nil)
writer.add(e4, "1", nil)
writer.write()
sst, _ := findSpaceIndex(bytes.NewReader(w.Bytes()), 0, int64(w.Len()))
var tests = []struct {
key string
offset int
length int
data [][]byte
timestamps []int
}{
{"g1", 1, 20, [][]byte{e4data, e1data}, []int{4, 1}},
{"g2", 21, 16, [][]byte{e2data, e3data}, []int{3, 2}},
}
for i, test := range tests {
val, _ := sst.Get([]byte(test.key))
r := bytes.NewReader(val)
offset := binary.ReadUvarint(r)
length := binary.ReadUvarint(r)
if int(offset) != test.offset || int(length) != test.length {
t.Errorf("Case %d: Wrong grouping encoding: want: %d,%d found: %d,%d", i, test.offset, test.length, offset, length)
}
reader := blocks.NewByteReader(w.Bytes(), 4096)
reader.Seek(int64(offset), 0)
for j, data := range test.data {
found := pullEvent(reader)
if string(found.Data) != string(data) {
t.Errorf("Case %d/%d: Wrong event data found: want: %s found: %s", i, j, data, found.Data)
}
}
reader.Seek(int64(offset), 0)
for j, ts := range test.timestamps {
found := pullEvent(reader)
if found.Timestamp != ts {
t.Errorf("Case %d/%d: Wrong event timestamp found: want: %d found: %d", i, j, ts, found.Timestamp)
}
}
if e := pullEvent(reader); e != nil {
t.Errorf("Wrong event found: want: nil found: %s", e.Data)
}
}
}