func (db *DB) Recovery(b []byte) error { buf := bytes.NewBuffer(b) db.setCurrent(uint64(binary.ReadInt64(buf))) db.MostRecent = binary.ReadInt64(buf) count := int(binary.ReadUvarint(buf)) for i := 0; i < count; i++ { db.addClosed(uint64(binary.ReadInt64(buf))) } return nil }
func TestWriteIndexBlocksLarge(t *testing.T) { w := new(bytes.Buffer) index := &index{evs: make(events, 5000)} for i := 0; i < 5000; i++ { index.evs[i] = &Event{Timestamp: i, block: rand.Int63(), offset: rand.Intn(4096)} } writeIndexBlocks(index, w) if index.length != 50040 { t.Errorf("Wrong written length: wanted: 50040, found: %d", index.length) } reader := blocks.NewByteReader(w.Bytes(), 4096) for i, event := range index.evs { block := binary.ReadInt64(reader) offset := binary.ReadInt16(reader) if event.block != block || event.offset != int(offset) { t.Errorf("Case %d: Wrong read event block/offset. wanted: %d,%d found: %d,%d", i, event.block, event.offset, block, offset) } } }
func (s *Space) ScanIndex(name, value string, scanner Scanner) { if reader := s.findIndex(name, value); reader != nil { for { // If the next byte is 0, then that's an empty event offset, // marking the end of the index. Nothing more to see here. if next := reader.Peek(1); len(next) == 0 || next[0] == 0 { return } // Each entry in the index is a 64 bit integer for the // event's block offset in the file, and a 16 bit integer // for the event's offset within the block (as each block // is 4096 bytes long) block := binary.ReadInt64(reader) offset := binary.ReadInt16(reader) // Move to the event's block s.blocks.Seek(s.offset+block, 0) // Read all data prior to the current event's offset. binary.ReadBytes(s.blocks, offset) event := pullEvent(s.blocks) if event == nil || !scanner(event) { return } } } }
func findIndex(f *os.File) (*sst.Reader, error) { // The last 8 bytes in the file is the length // of the SSTable spaces index. f.Seek(-8, 2) indexLen := binary.ReadInt64(f) return sst.NewReader(bounded.New(f, -8-indexLen, -8), indexLen) }
func findSpaceIndex(r io.ReadSeeker, offset, length int64) (*sst.Reader, error) { footerOffset := offset + length - 8 // The last 8 bytes in the file is the length // of the SSTable grouping index. r.Seek(footerOffset, 0) indexLen := binary.ReadInt64(r) return sst.NewReader(bounded.New(r, footerOffset-indexLen, footerOffset), indexLen) }
// Finds and returns a space by it's id. func (db *Db) Find(id []byte) *Space { if val, err := db.index.Get(id); err == nil { b := bytes.NewReader(val) // The entry in the SSTable index is // the offset and length of the space // within the file. offset := binary.ReadInt64(b) length := binary.ReadInt64(b) return openSpace( db.file, id, offset, length, ) } return nil }
func TestWriteIndexBlocksSmall(t *testing.T) { w := new(bytes.Buffer) e1 := &Event{Timestamp: 1, block: 0, offset: 2048} e2 := &Event{Timestamp: 3, block: 512, offset: 128} e3 := &Event{Timestamp: 2, block: 2048, offset: 512} e4 := &Event{Timestamp: 4, block: 248, offset: 1024} index := &index{evs: events{e1, e2, e3, e4}} writeIndexBlocks(index, w) if index.length != 36 { t.Errorf("Wrong written length: wanted: 36, found: %d", index.length) } compressed, _ := csnappy.Encode(nil, []byte( "\xF8\x00\x00\x00\x00\x00\x00\x00"+"\x00\x04"+ "\x00\x02\x00\x00\x00\x00\x00\x00"+"\x80\x00"+ "\x00\x08\x00\x00\x00\x00\x00\x00"+"\x00\x02"+ "\x00\x00\x00\x00\x00\x00\x00\x00"+"\x00\x08"+ "\x00")) expected := append([]byte("\x21\x00\x01"), compressed...) if !reflect.DeepEqual(w.Bytes(), expected) { t.Errorf("Wrong event block bytecode:\n wanted: %x\n found: %x", expected, w.Bytes()) } want := events{e4, e2, e3, e1} if !reflect.DeepEqual(index.evs, want) { t.Errorf("Wrongly sorted events: wanted: %v found: %v", want, index.evs) } reader := blocks.NewByteReader(w.Bytes(), 4096) for i, event := range index.evs { block := binary.ReadInt64(reader) offset := binary.ReadInt16(reader) if event.block != block || event.offset != int(offset) { t.Errorf("Case %d: Wrong read event block/offset. wanted: %d,%d found: %d,%d", i, event.block, event.offset, block, offset) } } }
func TestWriteSpaceIndexes(t *testing.T) { w := new(bytes.Buffer) writer := newSpace(w, []byte("a")) e1data := []byte("abc") e2data := []byte("b") e3data := []byte("c") e4data := []byte("def") e1 := newEvent(e1data, 1) e2 := newEvent(e2data, 3) e3 := newEvent(e3data, 2) e4 := newEvent(e4data, 4) writer.add(e1, "1", map[string]string{"a": "1"}) writer.add(e2, "1", map[string]string{"a": "2"}) writer.add(e3, "1", map[string]string{"a": "2"}) writer.add(e4, "1", map[string]string{"a": "1"}) writer.write() var tests = []struct { key string offset int length int evs [][]byte indexed events }{ {"g1", 1, 32, [][]byte{e4data, e2data, e3data, e1data}, nil}, {"ia:1", 33, 21, nil, events{e4, e1}}, {"ia:2", 54, 23, nil, events{e2, e3}}, } sst, _ := findSpaceIndex(bytes.NewReader(w.Bytes()), 0, int64(w.Len())) for i, test := range tests { val, _ := sst.Get([]byte(test.key)) r := bytes.NewReader(val) offset := binary.ReadUvarint(r) length := binary.ReadUvarint(r) if int(offset) != test.offset || int(length) != test.length { t.Errorf("Case %d: Wrong grouping encoding: want: %d,%d found: %d,%d", i, test.offset, test.length, offset, length) } reader := blocks.NewByteReader(w.Bytes(), 4096) reader.Seek(int64(offset), 0) if len(test.evs) > 0 { for j, data := range test.evs { if e := pullEvent(reader); !reflect.DeepEqual(e.Data, data) { t.Errorf("Case %d/%d: Wrong event found: want: %s found: %s", i, j, data, e.Data) } } if e := pullEvent(reader); e != nil { t.Errorf("Wrong event found: want: nil found: %s", e.Data) } } for j, event := range test.indexed { block := binary.ReadInt64(reader) offset := binary.ReadInt16(reader) if block != event.block || int(offset) != event.offset { t.Errorf("Case %d/%d: Wrong event index: want: %d,%d found: %d,%d", i, j, event.block, event.offset, block, offset) } } } }