Пример #1
0
func TestWriteIndexBlocksLarge(t *testing.T) {
	w := new(bytes.Buffer)

	index := &index{evs: make(events, 5000)}

	for i := 0; i < 5000; i++ {
		index.evs[i] = &Event{Timestamp: i, block: rand.Int63(), offset: rand.Intn(4096)}
	}

	writeIndexBlocks(index, w)

	if index.length != 50040 {
		t.Errorf("Wrong written length: wanted: 50040, found: %d", index.length)
	}

	reader := blocks.NewByteReader(w.Bytes(), 4096)

	for i, event := range index.evs {
		block := binary.ReadInt64(reader)
		offset := binary.ReadInt16(reader)

		if event.block != block || event.offset != int(offset) {
			t.Errorf("Case %d: Wrong read event block/offset. wanted: %d,%d found: %d,%d", i, event.block, event.offset, block, offset)
		}
	}
}
Пример #2
0
func (s *Space) ScanIndex(name, value string, scanner Scanner) {
	if reader := s.findIndex(name, value); reader != nil {
		for {
			// If the next byte is 0, then that's an empty event offset,
			// marking the end of the index. Nothing more to see here.
			if next := reader.Peek(1); len(next) == 0 || next[0] == 0 {
				return
			}

			// Each entry in the index is a 64 bit integer for the
			// event's block offset in the file, and a 16 bit integer
			// for the event's offset within the block (as each block
			// is 4096 bytes long)
			block := binary.ReadInt64(reader)
			offset := binary.ReadInt16(reader)

			// Move to the event's block
			s.blocks.Seek(s.offset+block, 0)

			// Read all data prior to the current event's offset.
			binary.ReadBytes(s.blocks, offset)

			event := pullEvent(s.blocks)

			if event == nil || !scanner(event) {
				return
			}
		}
	}
}
Пример #3
0
func TestWriteIndexBlocksSmall(t *testing.T) {
	w := new(bytes.Buffer)

	e1 := &Event{Timestamp: 1, block: 0, offset: 2048}
	e2 := &Event{Timestamp: 3, block: 512, offset: 128}
	e3 := &Event{Timestamp: 2, block: 2048, offset: 512}
	e4 := &Event{Timestamp: 4, block: 248, offset: 1024}

	index := &index{evs: events{e1, e2, e3, e4}}

	writeIndexBlocks(index, w)

	if index.length != 36 {
		t.Errorf("Wrong written length: wanted: 36, found: %d", index.length)
	}

	compressed, _ := csnappy.Encode(nil, []byte(
		"\xF8\x00\x00\x00\x00\x00\x00\x00"+"\x00\x04"+
			"\x00\x02\x00\x00\x00\x00\x00\x00"+"\x80\x00"+
			"\x00\x08\x00\x00\x00\x00\x00\x00"+"\x00\x02"+
			"\x00\x00\x00\x00\x00\x00\x00\x00"+"\x00\x08"+
			"\x00"))

	expected := append([]byte("\x21\x00\x01"), compressed...)

	if !reflect.DeepEqual(w.Bytes(), expected) {
		t.Errorf("Wrong event block bytecode:\n wanted: %x\n found:  %x", expected, w.Bytes())
	}

	want := events{e4, e2, e3, e1}

	if !reflect.DeepEqual(index.evs, want) {
		t.Errorf("Wrongly sorted events: wanted: %v found: %v", want, index.evs)
	}

	reader := blocks.NewByteReader(w.Bytes(), 4096)

	for i, event := range index.evs {
		block := binary.ReadInt64(reader)
		offset := binary.ReadInt16(reader)

		if event.block != block || event.offset != int(offset) {
			t.Errorf("Case %d: Wrong read event block/offset. wanted: %d,%d found: %d,%d", i, event.block, event.offset, block, offset)
		}
	}
}
Пример #4
0
func TestWriteSpaceIndexes(t *testing.T) {
	w := new(bytes.Buffer)

	writer := newSpace(w, []byte("a"))

	e1data := []byte("abc")
	e2data := []byte("b")
	e3data := []byte("c")
	e4data := []byte("def")
	e1 := newEvent(e1data, 1)
	e2 := newEvent(e2data, 3)
	e3 := newEvent(e3data, 2)
	e4 := newEvent(e4data, 4)

	writer.add(e1, "1", map[string]string{"a": "1"})
	writer.add(e2, "1", map[string]string{"a": "2"})
	writer.add(e3, "1", map[string]string{"a": "2"})
	writer.add(e4, "1", map[string]string{"a": "1"})

	writer.write()

	var tests = []struct {
		key     string
		offset  int
		length  int
		evs     [][]byte
		indexed events
	}{
		{"g1", 1, 32, [][]byte{e4data, e2data, e3data, e1data}, nil},
		{"ia:1", 33, 21, nil, events{e4, e1}},
		{"ia:2", 54, 23, nil, events{e2, e3}},
	}

	sst, _ := findSpaceIndex(bytes.NewReader(w.Bytes()), 0, int64(w.Len()))

	for i, test := range tests {
		val, _ := sst.Get([]byte(test.key))

		r := bytes.NewReader(val)

		offset := binary.ReadUvarint(r)
		length := binary.ReadUvarint(r)

		if int(offset) != test.offset || int(length) != test.length {
			t.Errorf("Case %d: Wrong grouping encoding: want: %d,%d found: %d,%d", i, test.offset, test.length, offset, length)
		}

		reader := blocks.NewByteReader(w.Bytes(), 4096)
		reader.Seek(int64(offset), 0)

		if len(test.evs) > 0 {
			for j, data := range test.evs {
				if e := pullEvent(reader); !reflect.DeepEqual(e.Data, data) {
					t.Errorf("Case %d/%d: Wrong event found: want: %s found: %s", i, j, data, e.Data)
				}
			}

			if e := pullEvent(reader); e != nil {
				t.Errorf("Wrong event found: want: nil found: %s", e.Data)
			}
		}

		for j, event := range test.indexed {
			block := binary.ReadInt64(reader)
			offset := binary.ReadInt16(reader)

			if block != event.block || int(offset) != event.offset {
				t.Errorf("Case %d/%d: Wrong event index: want: %d,%d found: %d,%d", i, j, event.block, event.offset, block, offset)
			}
		}
	}
}