func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
var in proto.HistoryEntry
b, err := qs.db.Get(k.(Token), qs.readopts)
if err == leveldb.ErrNotFound {
// No harm, no foul.
return quad.Quad{}
} else if err != nil {
glog.Error("Error: could not get quad from DB.")
return quad.Quad{}
}
err = in.Unmarshal(b)
if err != nil {
glog.Error("Error: could not reconstruct history.", err)
return quad.Quad{}
}
b, err = qs.db.Get(createDeltaKeyFor(int64(in.History[len(in.History)-1])), qs.readopts)
if err == leveldb.ErrNotFound {
// No harm, no foul.
return quad.Quad{}
} else if err != nil {
glog.Error("Error: could not get quad from DB.")
return quad.Quad{}
}
var d proto.LogDelta
err = d.Unmarshal(b)
if err != nil {
glog.Error("Error: could not reconstruct quad.", err)
return quad.Quad{}
}
return d.Quad.ToNative()
}
func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
var d proto.LogDelta
tok := k.(*Token)
err := qs.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(tok.bucket)
data := b.Get(tok.key)
if data == nil {
return nil
}
var in proto.HistoryEntry
err := in.Unmarshal(data)
if err != nil {
return err
}
if len(in.History) == 0 {
return nil
}
b = tx.Bucket(logBucket)
data = b.Get(qs.createDeltaKeyFor(int64(in.History[len(in.History)-1])))
if data == nil {
// No harm, no foul.
return nil
}
return d.Unmarshal(data)
})
if err != nil {
glog.Error("Error getting quad: ", err)
return quad.Quad{}
}
return d.Quad.ToNative()
}
// Quad ??
func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
var d proto.LogDelta
tok := k.(*Token)
err := qs.env.View(func(tx *lmdb.Txn) (err error) {
tx.RawRead = true
dbi := qs.dbis[tok.db]
data, _ := tx.Get(dbi, tok.key)
if data == nil {
return nil
}
var in proto.HistoryEntry
err = in.Unmarshal(data)
if err != nil {
return err
}
if len(in.History) == 0 {
return nil
}
data, _ = tx.Get(qs.logDBI, qs.createDeltaKeyFor(int64(in.History[len(in.History)-1])))
if data == nil {
// No harm, no foul.
return nil
}
return d.Unmarshal(data)
})
if err != nil {
glog.Error("Error getting quad: ", err)
return quad.Quad{}
}
if d.Quad == nil {
glog.Error("Unable to get quad: ", err)
return quad.Quad{}
}
return quad.Quad{
d.Quad.Subject,
d.Quad.Predicate,
d.Quad.Object,
d.Quad.Label,
}
}
func (qs *QuadStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64, isAdd bool) error {
var entry proto.HistoryEntry
data, err := qs.db.Get(createKeyFor(spo, q), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Error("could not access DB to prepare index: ", err)
return err
}
if data != nil {
// We got something.
err = entry.Unmarshal(data)
if err != nil {
return err
}
}
if isAdd && len(entry.History)%2 == 1 {
if glog.V(2) {
glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
}
return graph.ErrQuadExists
}
if !isAdd && len(entry.History)%2 == 0 {
if glog.V(2) {
glog.Errorf("attempt to delete non-existent quad %v: %#c", entry, q)
}
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, uint64(id))
bytes, err := entry.Marshal()
if err != nil {
glog.Errorf("could not write to buffer for entry %#v: %s", entry, err)
return err
}
batch.Put(createKeyFor(spo, q), bytes)
batch.Put(createKeyFor(osp, q), bytes)
batch.Put(createKeyFor(pos, q), bytes)
if q.Get(quad.Label) != nil {
batch.Put(createKeyFor(cps, q), bytes)
}
return nil
}
func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bool) error {
var entry proto.HistoryEntry
b := tx.Bucket(spoBucket)
b.FillPercent = localFillPercent
data := b.Get(qs.createKeyFor(spo, q))
if data != nil {
// We got something.
err := entry.Unmarshal(data)
if err != nil {
return err
}
}
if isAdd && len(entry.History)%2 == 1 {
glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
return graph.ErrQuadExists
}
if !isAdd && len(entry.History)%2 == 0 {
glog.Errorf("attempt to delete non-existent quad %v: %#v", entry, q)
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, uint64(id))
bytes, err := entry.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
return err
}
for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
if index == cps && q.Get(quad.Label) == nil {
continue
}
b := tx.Bucket(bucketFor(index))
b.FillPercent = localFillPercent
err = b.Put(qs.createKeyFor(index, q), bytes)
if err != nil {
return err
}
}
return nil
}
func (qs *QuadStore) buildQuadWriteLMDB(tx *lmdb.Txn, q quad.Quad, id int64, isAdd bool) error {
var entry proto.HistoryEntry
dbi := qs.dbis[spoDB]
data, err := tx.Get(dbi, qs.createKeyFor(spo, q))
if err == nil {
// We got something.
err := entry.Unmarshal(data)
if err != nil {
return err
}
}
if isAdd && len(entry.History)%2 == 1 {
glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
return graph.ErrQuadExists
}
if !isAdd && len(entry.History)%2 == 0 {
glog.Errorf("attempt to delete non-existent quad %v: %#v", entry, q)
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, uint64(id))
bytes, err := entry.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
return err
}
for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
if index == cps && q.Get(quad.Label) == "" {
continue
}
dbi = qs.dbis[dbFor(index)]
err = tx.Put(dbi, qs.createKeyFor(index, q), bytes, 0)
if err != nil {
return err
}
}
return nil
}
func (it *Iterator) isLiveValue(val []byte) bool {
var entry proto.HistoryEntry
entry.Unmarshal(val)
return len(entry.History)%2 != 0
}
func upgrade1To2(db *leveldb.DB) error {
fmt.Println("Upgrading v1 to v2...")
type v1IndexEntry struct {
Subject string `json:"subject"`
Predicate string `json:"predicate"`
Object string `json:"object"`
Label string `json:"label,omitempty"`
History []int64 `json:"History"`
}
type v1ValueData struct {
Name string `json:"Name"`
Size int64 `json:"Size"`
}
var (
spoPref = []byte{spo[0].Prefix(), spo[1].Prefix()}
ospPref = []byte{osp[0].Prefix(), osp[1].Prefix()}
posPref = []byte{pos[0].Prefix(), pos[1].Prefix()}
cpsPref = []byte{cps[0].Prefix(), cps[1].Prefix()}
)
{
fmt.Println("Upgrading bucket z")
it := db.NewIterator(&util.Range{Start: []byte{'z'}, Limit: []byte{'z' + 1}}, nil)
for it.Next() {
k, v := it.Key(), it.Value()
var val v1ValueData
if err := json.Unmarshal(v, &val); err != nil {
return err
}
node := proto.NodeData{
Size: val.Size,
Value: proto.MakeValue(quad.Raw(val.Name)),
}
nv, err := node.Marshal()
if err != nil {
return err
}
if err = db.Put(k, nv, nil); err != nil {
return err
}
}
it.Release()
}
for _, pref := range [4][]byte{spoPref, ospPref, posPref, cpsPref} {
fmt.Println("Upgrading bucket", string(pref))
end := []byte{pref[0], pref[1] + 1}
it := db.NewIterator(&util.Range{Start: pref, Limit: end}, nil)
for it.Next() {
k, v := it.Key(), it.Value()
var entry v1IndexEntry
if err := json.Unmarshal(v, &entry); err != nil {
return err
}
var h proto.HistoryEntry
h.History = make([]uint64, len(entry.History))
for i, id := range entry.History {
h.History[i] = uint64(id)
}
nv, err := h.Marshal()
if err != nil {
return err
}
if err = db.Put(k, nv, nil); err != nil {
return err
}
}
it.Release()
}
{
fmt.Println("Upgrading bucket d")
it := db.NewIterator(&util.Range{Start: []byte{'d'}, Limit: []byte{'d' + 1}}, nil)
for it.Next() {
k, v := it.Key(), it.Value()
id, err := strconv.ParseInt(string(k[1:]), 16, 64)
if err != nil {
return err
}
nk := createDeltaKeyFor(id)
var val graph.Delta
if err := json.Unmarshal(v, &val); err != nil {
return err
}
p := deltaToProto(val)
nv, err := p.Marshal()
if err != nil {
return err
}
b := &leveldb.Batch{}
b.Put(nk, nv)
b.Delete(k)
if err = db.Write(b, nil); err != nil {
return err
}
}
it.Release()
}
return nil
}
func upgrade1To2LMDB(qs *QuadStore) error {
fmt.Println("Upgrading v1 to v2...")
err := qs.env.Update(func(tx *lmdb.Txn) (err error) {
fmt.Println("Upgrading DB", logDB)
s := lmdbscan.New(tx, qs.logDBI)
defer s.Close()
for s.Scan() {
var delta graph.Delta
err := json.Unmarshal(s.Val(), &delta)
if err != nil {
return err
}
newd := deltaToProto(delta)
data, err := newd.Marshal()
if err != nil {
return err
}
err = tx.Put(qs.logDBI, s.Key(), data, 0)
if err != nil {
return err
}
}
return s.Err()
})
if err != nil {
return err
}
err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
fmt.Println("Upgrading DB", nodeDB)
s := lmdbscan.New(tx, qs.nodeDBI)
defer s.Close()
for s.Scan() {
var vd proto.NodeData
err := json.Unmarshal(s.Val(), &vd)
if err != nil {
return err
}
data, err := vd.Marshal()
if err != nil {
return err
}
err = tx.Put(qs.nodeDBI, s.Key(), data, 0)
if err != nil {
return err
}
}
return s.Err()
})
if err != nil {
return err
}
for _, db := range [4]string{string(spoDB), string(ospDB), string(posDB), string(cpsDB)} {
err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
fmt.Println("Upgrading DB", db)
dbi := qs.dbis[db]
s := lmdbscan.New(tx, dbi)
defer s.Close()
for s.Scan() {
var h proto.HistoryEntry
err := json.Unmarshal(s.Val(), &h)
if err != nil {
return err
}
data, err := h.Marshal()
if err != nil {
return err
}
err = tx.Put(dbi, s.Key(), data, 0)
if err != nil {
return err
}
}
return s.Err()
})
if err != nil {
return err
}
return nil
}
return nil
}
func upgrade1To2(db *bolt.DB) error {
fmt.Println("Upgrading v1 to v2...")
tx, err := db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(logBucket))
lb := tx.Bucket(logBucket)
c := lb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var delta graph.Delta
err := json.Unmarshal(v, &delta)
if err != nil {
return err
}
newd := deltaToProto(delta)
data, err := newd.Marshal()
if err != nil {
return err
}
lb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(nodeBucket))
nb := tx.Bucket(nodeBucket)
c = nb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var vd proto.NodeData
err := json.Unmarshal(v, &vd)
if err != nil {
return err
}
data, err := vd.Marshal()
if err != nil {
return err
}
nb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
for _, bucket := range [4][]byte{spoBucket, ospBucket, posBucket, cpsBucket} {
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(bucket))
b := tx.Bucket(bucket)
cur := b.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
var h proto.HistoryEntry
err := json.Unmarshal(v, &h)
if err != nil {
return err
}
data, err := h.Marshal()
if err != nil {
return err
}
b.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
}
return nil
}