// UpdateValueKeyBy ??
func (qs *QuadStore) UpdateValueKeyBy(name string, amount int64, tx *lmdb.Txn) error {
value := proto.NodeData{
Name: name,
Size_: amount,
}
key := qs.createValueKeyFor(name)
data, err := tx.Get(qs.nodeDBI, key)
if err == nil {
// Node exists in the database -- unmarshal and update.
var oldvalue proto.NodeData
err := oldvalue.Unmarshal(data)
if err != nil {
glog.Errorf("Error: couldn't reconstruct value: %v", err)
return err
}
oldvalue.Size_ += amount
value = oldvalue
}
// Are we deleting something?
if value.Size_ <= 0 {
value.Size_ = 0
}
// Repackage and rewrite.
bytes, err := value.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
return err
}
err = tx.Put(qs.nodeDBI, key, bytes, 0)
return err
}
func (qs *QuadStore) valueData(t *Token) proto.NodeData {
var out proto.NodeData
if glog.V(3) {
glog.V(3).Infof("%s %v", string(t.bucket), t.key)
}
err := qs.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(t.bucket)
data := b.Get(t.key)
if data != nil {
return out.Unmarshal(data)
}
return nil
})
if err != nil {
glog.Errorln("Error: couldn't get value")
return proto.NodeData{}
}
return out
}
func (qs *QuadStore) valueData(key []byte) proto.NodeData {
var out proto.NodeData
if glog.V(3) {
glog.V(3).Infof("%c %v", key[0], key)
}
b, err := qs.db.Get(key, qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: could not get value from DB")
return out
}
if b != nil && err != leveldb.ErrNotFound {
err = out.Unmarshal(b)
if err != nil {
glog.Errorln("Error: could not reconstruct value")
return proto.NodeData{}
}
}
return out
}
func (qs *QuadStore) valueDataLMDB(t *Token) proto.NodeData {
var out proto.NodeData
if glog.V(3) {
glog.V(3).Infof("%s %v", string(t.db), t.key)
}
err := qs.env.View(func(tx *lmdb.Txn) (err error) {
tx.RawRead = true
dbi := qs.dbis[t.db]
data, err := tx.Get(dbi, t.key)
if err == nil {
return out.Unmarshal(data)
}
return nil
})
if err != nil {
glog.Errorln("Error: couldn't get value")
return proto.NodeData{}
}
return out
}
func upgrade2To3(db *bolt.DB) error {
fmt.Println("Upgrading v2 to v3...")
tx, err := db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(logBucket))
lb := tx.Bucket(logBucket)
c := lb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var delta proto.LogDelta
err := delta.Unmarshal(v)
if err != nil {
return err
}
delta.Quad.Upgrade()
data, err := delta.Marshal()
if err != nil {
return err
}
lb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(nodeBucket))
nb := tx.Bucket(nodeBucket)
c = nb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var vd proto.NodeData
err := vd.Unmarshal(v)
if err != nil {
return err
}
vd.Upgrade()
data, err := vd.Marshal()
if err != nil {
return err
}
nb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
return nil
}
func (qs *QuadStore) UpdateValueKeyBy(name quad.Value, amount int64, batch *leveldb.Batch) error {
value := proto.NodeData{
Value: proto.MakeValue(name),
Size: amount,
}
key := createValueKeyFor(name)
b, err := qs.db.Get(key, qs.readopts)
// Error getting the node from the database.
if err != nil && err != leveldb.ErrNotFound {
glog.Errorf("Error reading Value %s from the DB.", name)
return err
}
// Node exists in the database -- unmarshal and update.
if b != nil && err != leveldb.ErrNotFound {
var oldvalue proto.NodeData
err = oldvalue.Unmarshal(b)
if err != nil {
glog.Errorf("Error: could not reconstruct value: %v", err)
return err
}
oldvalue.Size += amount
value = oldvalue
}
// Are we deleting something?
if value.Size <= 0 {
value.Size = 0
}
// Repackage and rewrite.
bytes, err := value.Marshal()
if err != nil {
glog.Errorf("could not write to buffer for value %s: %s", name, err)
return err
}
if batch == nil {
qs.db.Put(key, bytes, qs.writeopts)
} else {
batch.Put(key, bytes)
}
return nil
}
func (qs *QuadStore) UpdateValueKeyBy(name quad.Value, amount int64, tx *bolt.Tx) error {
value := proto.NodeData{
Value: proto.MakeValue(name),
Size: amount,
}
b := tx.Bucket(nodeBucket)
b.FillPercent = localFillPercent
key := qs.createValueKeyFor(name)
data := b.Get(key)
if data != nil {
// Node exists in the database -- unmarshal and update.
var oldvalue proto.NodeData
err := oldvalue.Unmarshal(data)
if err != nil {
glog.Errorf("Error: couldn't reconstruct value: %v", err)
return err
}
oldvalue.Size += amount
value = oldvalue
}
// Are we deleting something?
if value.Size <= 0 {
value.Size = 0
}
// Repackage and rewrite.
bytes, err := value.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
return err
}
err = b.Put(key, bytes)
return err
}
func upgrade1To2(db *leveldb.DB) error {
fmt.Println("Upgrading v1 to v2...")
type v1IndexEntry struct {
Subject string `json:"subject"`
Predicate string `json:"predicate"`
Object string `json:"object"`
Label string `json:"label,omitempty"`
History []int64 `json:"History"`
}
type v1ValueData struct {
Name string `json:"Name"`
Size int64 `json:"Size"`
}
var (
spoPref = []byte{spo[0].Prefix(), spo[1].Prefix()}
ospPref = []byte{osp[0].Prefix(), osp[1].Prefix()}
posPref = []byte{pos[0].Prefix(), pos[1].Prefix()}
cpsPref = []byte{cps[0].Prefix(), cps[1].Prefix()}
)
{
fmt.Println("Upgrading bucket z")
it := db.NewIterator(&util.Range{Start: []byte{'z'}, Limit: []byte{'z' + 1}}, nil)
for it.Next() {
k, v := it.Key(), it.Value()
var val v1ValueData
if err := json.Unmarshal(v, &val); err != nil {
return err
}
node := proto.NodeData{
Size: val.Size,
Value: proto.MakeValue(quad.Raw(val.Name)),
}
nv, err := node.Marshal()
if err != nil {
return err
}
if err = db.Put(k, nv, nil); err != nil {
return err
}
}
it.Release()
}
for _, pref := range [4][]byte{spoPref, ospPref, posPref, cpsPref} {
fmt.Println("Upgrading bucket", string(pref))
end := []byte{pref[0], pref[1] + 1}
it := db.NewIterator(&util.Range{Start: pref, Limit: end}, nil)
for it.Next() {
k, v := it.Key(), it.Value()
var entry v1IndexEntry
if err := json.Unmarshal(v, &entry); err != nil {
return err
}
var h proto.HistoryEntry
h.History = make([]uint64, len(entry.History))
for i, id := range entry.History {
h.History[i] = uint64(id)
}
nv, err := h.Marshal()
if err != nil {
return err
}
if err = db.Put(k, nv, nil); err != nil {
return err
}
}
it.Release()
}
{
fmt.Println("Upgrading bucket d")
it := db.NewIterator(&util.Range{Start: []byte{'d'}, Limit: []byte{'d' + 1}}, nil)
for it.Next() {
k, v := it.Key(), it.Value()
id, err := strconv.ParseInt(string(k[1:]), 16, 64)
if err != nil {
return err
}
nk := createDeltaKeyFor(id)
var val graph.Delta
if err := json.Unmarshal(v, &val); err != nil {
return err
}
p := deltaToProto(val)
nv, err := p.Marshal()
if err != nil {
return err
}
b := &leveldb.Batch{}
b.Put(nk, nv)
b.Delete(k)
if err = db.Write(b, nil); err != nil {
return err
}
}
it.Release()
}
return nil
}
func upgrade1To2LMDB(qs *QuadStore) error {
fmt.Println("Upgrading v1 to v2...")
err := qs.env.Update(func(tx *lmdb.Txn) (err error) {
fmt.Println("Upgrading DB", logDB)
s := lmdbscan.New(tx, qs.logDBI)
defer s.Close()
for s.Scan() {
var delta graph.Delta
err := json.Unmarshal(s.Val(), &delta)
if err != nil {
return err
}
newd := deltaToProto(delta)
data, err := newd.Marshal()
if err != nil {
return err
}
err = tx.Put(qs.logDBI, s.Key(), data, 0)
if err != nil {
return err
}
}
return s.Err()
})
if err != nil {
return err
}
err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
fmt.Println("Upgrading DB", nodeDB)
s := lmdbscan.New(tx, qs.nodeDBI)
defer s.Close()
for s.Scan() {
var vd proto.NodeData
err := json.Unmarshal(s.Val(), &vd)
if err != nil {
return err
}
data, err := vd.Marshal()
if err != nil {
return err
}
err = tx.Put(qs.nodeDBI, s.Key(), data, 0)
if err != nil {
return err
}
}
return s.Err()
})
if err != nil {
return err
}
for _, db := range [4]string{string(spoDB), string(ospDB), string(posDB), string(cpsDB)} {
err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
fmt.Println("Upgrading DB", db)
dbi := qs.dbis[db]
s := lmdbscan.New(tx, dbi)
defer s.Close()
for s.Scan() {
var h proto.HistoryEntry
err := json.Unmarshal(s.Val(), &h)
if err != nil {
return err
}
data, err := h.Marshal()
if err != nil {
return err
}
err = tx.Put(dbi, s.Key(), data, 0)
if err != nil {
return err
}
}
return s.Err()
})
if err != nil {
return err
}
return nil
}
return nil
}
func upgrade1To2(db *bolt.DB) error {
fmt.Println("Upgrading v1 to v2...")
tx, err := db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(logBucket))
lb := tx.Bucket(logBucket)
c := lb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var delta graph.Delta
err := json.Unmarshal(v, &delta)
if err != nil {
return err
}
newd := deltaToProto(delta)
data, err := newd.Marshal()
if err != nil {
return err
}
lb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(nodeBucket))
nb := tx.Bucket(nodeBucket)
c = nb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var vd proto.NodeData
err := json.Unmarshal(v, &vd)
if err != nil {
return err
}
data, err := vd.Marshal()
if err != nil {
return err
}
nb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
for _, bucket := range [4][]byte{spoBucket, ospBucket, posBucket, cpsBucket} {
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(bucket))
b := tx.Bucket(bucket)
cur := b.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
var h proto.HistoryEntry
err := json.Unmarshal(v, &h)
if err != nil {
return err
}
data, err := h.Marshal()
if err != nil {
return err
}
b.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
}
return nil
}