func (s *store) delete(key []byte, rev revision) { mainrev := s.currentRev.main + 1 ibytes := newRevBytes() revToBytes(revision{main: mainrev, sub: s.currentRev.sub}, ibytes) ibytes = appendMarkTombstone(ibytes) kv := mvccpb.KeyValue{ Key: key, } d, err := kv.Marshal() if err != nil { plog.Fatalf("cannot marshal event: %v", err) } s.tx.UnsafeSeqPut(keyBucketName, ibytes, d) err = s.kvindex.Tombstone(key, revision{main: mainrev, sub: s.currentRev.sub}) if err != nil { plog.Fatalf("cannot tombstone an existing key (%s): %v", string(key), err) } s.changes = append(s.changes, kv) s.currentRev.sub += 1 item := lease.LeaseItem{Key: string(key)} leaseID := s.le.GetLease(item) if leaseID != lease.NoLease { err = s.le.Detach(leaseID, []lease.LeaseItem{item}) if err != nil { plog.Errorf("cannot detach %v", err) } } }
func (s *store) put(key, value []byte, leaseID lease.LeaseID) { s.txnModify = true rev := s.currentRev.main + 1 c := rev oldLease := lease.NoLease // if the key exists before, use its previous created and // get its previous leaseID _, created, ver, err := s.kvindex.Get(key, rev) if err == nil { c = created.main oldLease = s.le.GetLease(lease.LeaseItem{Key: string(key)}) } ibytes := newRevBytes() revToBytes(revision{main: rev, sub: s.currentRev.sub}, ibytes) ver = ver + 1 kv := mvccpb.KeyValue{ Key: key, Value: value, CreateRevision: c, ModRevision: rev, Version: ver, Lease: int64(leaseID), } d, err := kv.Marshal() if err != nil { plog.Fatalf("cannot marshal event: %v", err) } s.tx.UnsafeSeqPut(keyBucketName, ibytes, d) s.kvindex.Put(key, revision{main: rev, sub: s.currentRev.sub}) s.changes = append(s.changes, kv) s.currentRev.sub += 1 if oldLease != lease.NoLease { if s.le == nil { panic("no lessor to detach lease") } err = s.le.Detach(oldLease, []lease.LeaseItem{{Key: string(key)}}) if err != nil { plog.Errorf("unexpected error from lease detach: %v", err) } } if leaseID != lease.NoLease { if s.le == nil { panic("no lessor to attach lease") } err = s.le.Attach(leaseID, []lease.LeaseItem{{Key: string(key)}}) if err != nil { panic("unexpected error from lease Attach") } } }
func WriteKV(be backend.Backend, kv mvccpb.KeyValue) { ibytes := newRevBytes() revToBytes(revision{main: kv.ModRevision}, ibytes) d, err := kv.Marshal() if err != nil { plog.Fatalf("cannot marshal event: %v", err) } be.BatchTx().Lock() be.BatchTx().UnsafePut(keyBucketName, ibytes, d) be.BatchTx().Unlock() }
// range is a keyword in Go, add Keys suffix. func (s *store) rangeKeys(key, end []byte, limit, rangeRev int64, countOnly bool) (kvs []mvccpb.KeyValue, count int, curRev int64, err error) { curRev = int64(s.currentRev.main) if s.currentRev.sub > 0 { curRev += 1 } if rangeRev > curRev { return nil, -1, s.currentRev.main, ErrFutureRev } var rev int64 if rangeRev <= 0 { rev = curRev } else { rev = rangeRev } if rev < s.compactMainRev { return nil, -1, 0, ErrCompacted } _, revpairs := s.kvindex.Range(key, end, int64(rev)) if len(revpairs) == 0 { return nil, 0, curRev, nil } if countOnly { return nil, len(revpairs), curRev, nil } for _, revpair := range revpairs { start, end := revBytesRange(revpair) _, vs := s.tx.UnsafeRange(keyBucketName, start, end, 0) if len(vs) != 1 { plog.Fatalf("range cannot find rev (%d,%d)", revpair.main, revpair.sub) } var kv mvccpb.KeyValue if err := kv.Unmarshal(vs[0]); err != nil { plog.Fatalf("cannot unmarshal event: %v", err) } kvs = append(kvs, kv) if limit > 0 && len(kvs) >= int(limit) { break } } return kvs, len(revpairs), curRev, nil }
// kvsToEvents gets all events for the watchers from all key-value pairs func kvsToEvents(wg *watcherGroup, revs, vals [][]byte) (evs []mvccpb.Event) { for i, v := range vals { var kv mvccpb.KeyValue if err := kv.Unmarshal(v); err != nil { plog.Panicf("cannot unmarshal event: %v", err) } if !wg.contains(string(kv.Key)) { continue } ty := mvccpb.PUT if isTombstone(revs[i]) { ty = mvccpb.DELETE // patch in mod revision so watchers won't skip kv.ModRevision = bytesToRev(revs[i]).main } evs = append(evs, mvccpb.Event{Kv: &kv, Type: ty}) } return evs }
func TestStoreRestore(t *testing.T) { s := newFakeStore() b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) putkey := newTestKeyBytes(revision{3, 0}, false) putkv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 4, ModRevision: 4, Version: 1, } putkvb, err := putkv.Marshal() if err != nil { t.Fatal(err) } delkey := newTestKeyBytes(revision{5, 0}, true) delkv := mvccpb.KeyValue{ Key: []byte("foo"), } delkvb, err := delkv.Marshal() if err != nil { t.Fatal(err) } b.tx.rangeRespc <- rangeResp{[][]byte{finishedCompactKeyName}, [][]byte{newTestRevBytes(revision{3, 0})}} b.tx.rangeRespc <- rangeResp{[][]byte{putkey, delkey}, [][]byte{putkvb, delkvb}} b.tx.rangeRespc <- rangeResp{[][]byte{scheduledCompactKeyName}, [][]byte{newTestRevBytes(revision{3, 0})}} s.restore() if s.compactMainRev != 3 { t.Errorf("compact rev = %d, want 5", s.compactMainRev) } wrev := revision{5, 0} if !reflect.DeepEqual(s.currentRev, wrev) { t.Errorf("current rev = %v, want %v", s.currentRev, wrev) } wact := []testutil.Action{ {"range", []interface{}{metaBucketName, finishedCompactKeyName, []byte(nil), int64(0)}}, {"range", []interface{}{keyBucketName, newTestRevBytes(revision{1, 0}), newTestRevBytes(revision{math.MaxInt64, math.MaxInt64}), int64(0)}}, {"range", []interface{}{metaBucketName, scheduledCompactKeyName, []byte(nil), int64(0)}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("tx actions = %+v, want %+v", g, wact) } gens := []generation{ {created: revision{4, 0}, ver: 2, revs: []revision{{3, 0}, {5, 0}}}, {created: revision{0, 0}, ver: 0, revs: nil}, } ki := &keyIndex{key: []byte("foo"), modified: revision{5, 0}, generations: gens} wact = []testutil.Action{ {"insert", []interface{}{ki}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("index action = %+v, want %+v", g, wact) } }
func (s *store) delete(key []byte, rev revision) { mainrev := s.currentRev.main + 1 ibytes := newRevBytes() revToBytes(revision{main: mainrev, sub: s.currentRev.sub}, ibytes) ibytes = appendMarkTombstone(ibytes) kv := mvccpb.KeyValue{ Key: key, } d, err := kv.Marshal() if err != nil { plog.Fatalf("cannot marshal event: %v", err) } s.tx.UnsafeSeqPut(keyBucketName, ibytes, d) err = s.kvindex.Tombstone(key, revision{main: mainrev, sub: s.currentRev.sub}) if err != nil { plog.Fatalf("cannot tombstone an existing key (%s): %v", string(key), err) } s.changes = append(s.changes, kv) s.currentRev.sub += 1 ibytes = newRevBytes() revToBytes(rev, ibytes) _, vs := s.tx.UnsafeRange(keyBucketName, ibytes, nil, 0) kv.Reset() if err = kv.Unmarshal(vs[0]); err != nil { plog.Fatalf("cannot unmarshal value: %v", err) } if lease.LeaseID(kv.Lease) != lease.NoLease { err = s.le.Detach(lease.LeaseID(kv.Lease), []lease.LeaseItem{{Key: string(kv.Key)}}) if err != nil { plog.Fatalf("cannot detach %v", err) } } }
func TestStorePut(t *testing.T) { kv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 2, Version: 1, } kvb, err := kv.Marshal() if err != nil { t.Fatal(err) } tests := []struct { rev revision r indexGetResp rr *rangeResp wrev revision wkey []byte wkv mvccpb.KeyValue wputrev revision }{ { revision{1, 0}, indexGetResp{revision{}, revision{}, 0, ErrRevisionNotFound}, nil, revision{1, 1}, newTestKeyBytes(revision{2, 0}, false), mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1, Lease: 1, }, revision{2, 0}, }, { revision{1, 1}, indexGetResp{revision{2, 0}, revision{2, 0}, 1, nil}, &rangeResp{[][]byte{newTestKeyBytes(revision{2, 1}, false)}, [][]byte{kvb}}, revision{1, 2}, newTestKeyBytes(revision{2, 1}, false), mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 2, Lease: 2, }, revision{2, 1}, }, { revision{2, 0}, indexGetResp{revision{2, 1}, revision{2, 0}, 2, nil}, &rangeResp{[][]byte{newTestKeyBytes(revision{2, 1}, false)}, [][]byte{kvb}}, revision{2, 1}, newTestKeyBytes(revision{3, 0}, false), mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 3, Lease: 3, }, revision{3, 0}, }, } for i, tt := range tests { s := newFakeStore() b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = tt.rev s.tx = b.BatchTx() fi.indexGetRespc <- tt.r if tt.rr != nil { b.tx.rangeRespc <- *tt.rr } s.put([]byte("foo"), []byte("bar"), lease.LeaseID(i+1)) data, err := tt.wkv.Marshal() if err != nil { t.Errorf("#%d: marshal err = %v, want nil", i, err) } wact := []testutil.Action{ {"seqput", []interface{}{keyBucketName, tt.wkey, data}}, } if tt.rr != nil { wact = []testutil.Action{ {"range", []interface{}{keyBucketName, newTestKeyBytes(tt.r.rev, false), []byte(nil), int64(0)}}, {"seqput", []interface{}{keyBucketName, tt.wkey, data}}, } } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact) } wact = []testutil.Action{ {"get", []interface{}{[]byte("foo"), tt.wputrev.main}}, {"put", []interface{}{[]byte("foo"), tt.wputrev}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: index action = %+v, want %+v", i, g, wact) } if s.currentRev != tt.wrev { t.Errorf("#%d: rev = %+v, want %+v", i, s.currentRev, tt.wrev) } s.Close() } }
func TestStoreDeleteRange(t *testing.T) { key := newTestKeyBytes(revision{2, 0}, false) kv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 2, Version: 1, } kvb, err := kv.Marshal() if err != nil { t.Fatal(err) } tests := []struct { rev revision r indexRangeResp rr rangeResp wkey []byte wrev revision wrrev int64 wdelrev revision }{ { revision{2, 0}, indexRangeResp{[][]byte{[]byte("foo")}, []revision{{2, 0}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, newTestKeyBytes(revision{3, 0}, true), revision{2, 1}, 2, revision{3, 0}, }, { revision{2, 1}, indexRangeResp{[][]byte{[]byte("foo")}, []revision{{2, 0}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, newTestKeyBytes(revision{3, 1}, true), revision{2, 2}, 3, revision{3, 1}, }, } for i, tt := range tests { s := newFakeStore() b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = tt.rev s.tx = b.BatchTx() fi.indexRangeRespc <- tt.r b.tx.rangeRespc <- tt.rr n := s.deleteRange([]byte("foo"), []byte("goo")) if n != 1 { t.Errorf("#%d: n = %d, want 1", i, n) } data, err := (&mvccpb.KeyValue{ Key: []byte("foo"), }).Marshal() if err != nil { t.Errorf("#%d: marshal err = %v, want nil", i, err) } wact := []testutil.Action{ {"seqput", []interface{}{keyBucketName, tt.wkey, data}}, {"range", []interface{}{keyBucketName, newTestKeyBytes(revision{2, 0}, false), []byte(nil), int64(0)}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact) } wact = []testutil.Action{ {"range", []interface{}{[]byte("foo"), []byte("goo"), tt.wrrev}}, {"tombstone", []interface{}{[]byte("foo"), tt.wdelrev}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: index action = %+v, want %+v", i, g, wact) } if s.currentRev != tt.wrev { t.Errorf("#%d: rev = %+v, want %+v", i, s.currentRev, tt.wrev) } } }
func TestStoreRange(t *testing.T) { key := newTestKeyBytes(revision{2, 0}, false) kv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 2, Version: 1, } kvb, err := kv.Marshal() if err != nil { t.Fatal(err) } currev := revision{1, 1} wrev := int64(2) tests := []struct { idxr indexRangeResp r rangeResp }{ { indexRangeResp{[][]byte{[]byte("foo")}, []revision{{2, 0}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, }, { indexRangeResp{[][]byte{[]byte("foo"), []byte("foo1")}, []revision{{2, 0}, {3, 0}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, }, } for i, tt := range tests { s := newFakeStore() b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = currev s.tx = b.BatchTx() b.tx.rangeRespc <- tt.r fi.indexRangeRespc <- tt.idxr kvs, rev, err := s.rangeKeys([]byte("foo"), []byte("goo"), 1, 0) if err != nil { t.Errorf("#%d: err = %v, want nil", i, err) } if w := []mvccpb.KeyValue{kv}; !reflect.DeepEqual(kvs, w) { t.Errorf("#%d: kvs = %+v, want %+v", i, kvs, w) } if rev != wrev { t.Errorf("#%d: rev = %d, want %d", i, rev, wrev) } wstart, wend := revBytesRange(tt.idxr.revs[0]) wact := []testutil.Action{ {"range", []interface{}{keyBucketName, wstart, wend, int64(0)}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact) } wact = []testutil.Action{ {"range", []interface{}{[]byte("foo"), []byte("goo"), wrev}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: index action = %+v, want %+v", i, g, wact) } if s.currentRev != currev { t.Errorf("#%d: current rev = %+v, want %+v", i, s.currentRev, currev) } s.Close() } }
func (s *store) restore() error { min, max := newRevBytes(), newRevBytes() revToBytes(revision{main: 1}, min) revToBytes(revision{main: math.MaxInt64, sub: math.MaxInt64}, max) // restore index tx := s.b.BatchTx() tx.Lock() _, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0) if len(finishedCompactBytes) != 0 { s.compactMainRev = bytesToRev(finishedCompactBytes[0]).main plog.Printf("restore compact to %d", s.compactMainRev) } // TODO: limit N to reduce max memory usage keys, vals := tx.UnsafeRange(keyBucketName, min, max, 0) for i, key := range keys { var kv mvccpb.KeyValue if err := kv.Unmarshal(vals[i]); err != nil { plog.Fatalf("cannot unmarshal event: %v", err) } rev := bytesToRev(key[:revBytesLen]) // restore index switch { case isTombstone(key): s.kvindex.Tombstone(kv.Key, rev) if lease.LeaseID(kv.Lease) != lease.NoLease { err := s.le.Detach(lease.LeaseID(kv.Lease), []lease.LeaseItem{{Key: string(kv.Key)}}) if err != nil && err != lease.ErrLeaseNotFound { plog.Fatalf("unexpected Detach error %v", err) } } default: s.kvindex.Restore(kv.Key, revision{kv.CreateRevision, 0}, rev, kv.Version) if lease.LeaseID(kv.Lease) != lease.NoLease { if s.le == nil { panic("no lessor to attach lease") } err := s.le.Attach(lease.LeaseID(kv.Lease), []lease.LeaseItem{{Key: string(kv.Key)}}) // We are walking through the kv history here. It is possible that we attached a key to // the lease and the lease was revoked later. // Thus attaching an old version of key to a none existing lease is possible here, and // we should just ignore the error. if err != nil && err != lease.ErrLeaseNotFound { panic("unexpected Attach error") } } } // update revision s.currentRev = rev } _, scheduledCompactBytes := tx.UnsafeRange(metaBucketName, scheduledCompactKeyName, nil, 0) scheduledCompact := int64(0) if len(scheduledCompactBytes) != 0 { scheduledCompact = bytesToRev(scheduledCompactBytes[0]).main if scheduledCompact <= s.compactMainRev { scheduledCompact = 0 } } tx.Unlock() if scheduledCompact != 0 { s.Compact(scheduledCompact) plog.Printf("resume scheduled compaction at %d", scheduledCompact) } return nil }
func (s *store) restore() error { min, max := newRevBytes(), newRevBytes() revToBytes(revision{main: 1}, min) revToBytes(revision{main: math.MaxInt64, sub: math.MaxInt64}, max) keyToLease := make(map[string]lease.LeaseID) // use an unordered map to hold the temp index data to speed up // the initial key index recovery. // we will convert this unordered map into the tree index later. unordered := make(map[string]*keyIndex, 100000) // restore index tx := s.b.BatchTx() tx.Lock() _, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0) if len(finishedCompactBytes) != 0 { s.compactMainRev = bytesToRev(finishedCompactBytes[0]).main plog.Printf("restore compact to %d", s.compactMainRev) } // TODO: limit N to reduce max memory usage keys, vals := tx.UnsafeRange(keyBucketName, min, max, 0) for i, key := range keys { var kv mvccpb.KeyValue if err := kv.Unmarshal(vals[i]); err != nil { plog.Fatalf("cannot unmarshal event: %v", err) } rev := bytesToRev(key[:revBytesLen]) // restore index switch { case isTombstone(key): if ki, ok := unordered[string(kv.Key)]; ok { ki.tombstone(rev.main, rev.sub) } delete(keyToLease, string(kv.Key)) default: ki, ok := unordered[string(kv.Key)] if ok { ki.put(rev.main, rev.sub) } else { ki = &keyIndex{key: kv.Key} ki.restore(revision{kv.CreateRevision, 0}, rev, kv.Version) unordered[string(kv.Key)] = ki } if lid := lease.LeaseID(kv.Lease); lid != lease.NoLease { keyToLease[string(kv.Key)] = lid } else { delete(keyToLease, string(kv.Key)) } } // update revision s.currentRev = rev } // restore the tree index from the unordered index. for _, v := range unordered { s.kvindex.Insert(v) } // keys in the range [compacted revision -N, compaction] might all be deleted due to compaction. // the correct revision should be set to compaction revision in the case, not the largest revision // we have seen. if s.currentRev.main < s.compactMainRev { s.currentRev.main = s.compactMainRev } for key, lid := range keyToLease { if s.le == nil { panic("no lessor to attach lease") } err := s.le.Attach(lid, []lease.LeaseItem{{Key: key}}) if err != nil { plog.Errorf("unexpected Attach error: %v", err) } } _, scheduledCompactBytes := tx.UnsafeRange(metaBucketName, scheduledCompactKeyName, nil, 0) scheduledCompact := int64(0) if len(scheduledCompactBytes) != 0 { scheduledCompact = bytesToRev(scheduledCompactBytes[0]).main if scheduledCompact <= s.compactMainRev { scheduledCompact = 0 } } tx.Unlock() if scheduledCompact != 0 { s.Compact(scheduledCompact) plog.Printf("resume scheduled compaction at %d", scheduledCompact) } return nil }
func (s *store) restore() error { min, max := newRevBytes(), newRevBytes() revToBytes(revision{main: 1}, min) revToBytes(revision{main: math.MaxInt64, sub: math.MaxInt64}, max) keyToLease := make(map[string]lease.LeaseID) // restore index tx := s.b.BatchTx() tx.Lock() _, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0) if len(finishedCompactBytes) != 0 { s.compactMainRev = bytesToRev(finishedCompactBytes[0]).main plog.Printf("restore compact to %d", s.compactMainRev) } // TODO: limit N to reduce max memory usage keys, vals := tx.UnsafeRange(keyBucketName, min, max, 0) for i, key := range keys { var kv mvccpb.KeyValue if err := kv.Unmarshal(vals[i]); err != nil { plog.Fatalf("cannot unmarshal event: %v", err) } rev := bytesToRev(key[:revBytesLen]) // restore index switch { case isTombstone(key): s.kvindex.Tombstone(kv.Key, rev) delete(keyToLease, string(kv.Key)) default: s.kvindex.Restore(kv.Key, revision{kv.CreateRevision, 0}, rev, kv.Version) if lid := lease.LeaseID(kv.Lease); lid != lease.NoLease { keyToLease[string(kv.Key)] = lid } else { delete(keyToLease, string(kv.Key)) } } // update revision s.currentRev = rev } for key, lid := range keyToLease { if s.le == nil { panic("no lessor to attach lease") } err := s.le.Attach(lid, []lease.LeaseItem{{Key: key}}) if err != nil { plog.Errorf("unexpected Attach error: %v", err) } } _, scheduledCompactBytes := tx.UnsafeRange(metaBucketName, scheduledCompactKeyName, nil, 0) scheduledCompact := int64(0) if len(scheduledCompactBytes) != 0 { scheduledCompact = bytesToRev(scheduledCompactBytes[0]).main if scheduledCompact <= s.compactMainRev { scheduledCompact = 0 } } tx.Unlock() if scheduledCompact != 0 { s.Compact(scheduledCompact) plog.Printf("resume scheduled compaction at %d", scheduledCompact) } return nil }