Exemple #1
0
// UpdateValueKeyBy ??
func (qs *QuadStore) UpdateValueKeyBy(name string, amount int64, tx *lmdb.Txn) error {
	value := proto.NodeData{
		Name:  name,
		Size_: amount,
	}
	key := qs.createValueKeyFor(name)
	data, err := tx.Get(qs.nodeDBI, key)
	if err == nil {
		// Node exists in the database -- unmarshal and update.
		var oldvalue proto.NodeData
		err := oldvalue.Unmarshal(data)
		if err != nil {
			glog.Errorf("Error: couldn't reconstruct value: %v", err)
			return err
		}
		oldvalue.Size_ += amount
		value = oldvalue
	}

	// Are we deleting something?
	if value.Size_ <= 0 {
		value.Size_ = 0
	}

	// Repackage and rewrite.
	bytes, err := value.Marshal()
	if err != nil {
		glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
		return err
	}
	err = tx.Put(qs.nodeDBI, key, bytes, 0)
	return err
}
Exemple #2
0
func (qs *QuadStore) valueData(t *Token) proto.NodeData {
	var out proto.NodeData
	if glog.V(3) {
		glog.V(3).Infof("%s %v", string(t.bucket), t.key)
	}
	err := qs.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(t.bucket)
		data := b.Get(t.key)
		if data != nil {
			return out.Unmarshal(data)
		}
		return nil
	})
	if err != nil {
		glog.Errorln("Error: couldn't get value")
		return proto.NodeData{}
	}
	return out
}
Exemple #3
0
func (qs *QuadStore) valueData(key []byte) proto.NodeData {
	var out proto.NodeData
	if glog.V(3) {
		glog.V(3).Infof("%c %v", key[0], key)
	}
	b, err := qs.db.Get(key, qs.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorln("Error: could not get value from DB")
		return out
	}
	if b != nil && err != leveldb.ErrNotFound {
		err = out.Unmarshal(b)
		if err != nil {
			glog.Errorln("Error: could not reconstruct value")
			return proto.NodeData{}
		}
	}
	return out
}
Exemple #4
0
func (qs *QuadStore) valueDataLMDB(t *Token) proto.NodeData {
	var out proto.NodeData
	if glog.V(3) {
		glog.V(3).Infof("%s %v", string(t.db), t.key)
	}
	err := qs.env.View(func(tx *lmdb.Txn) (err error) {
		tx.RawRead = true

		dbi := qs.dbis[t.db]
		data, err := tx.Get(dbi, t.key)
		if err == nil {
			return out.Unmarshal(data)
		}
		return nil
	})
	if err != nil {
		glog.Errorln("Error: couldn't get value")
		return proto.NodeData{}
	}
	return out
}
Exemple #5
0
func upgrade2To3(db *bolt.DB) error {
	fmt.Println("Upgrading v2 to v3...")
	tx, err := db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	fmt.Println("Upgrading bucket", string(logBucket))
	lb := tx.Bucket(logBucket)
	c := lb.Cursor()
	for k, v := c.First(); k != nil; k, v = c.Next() {
		var delta proto.LogDelta
		err := delta.Unmarshal(v)
		if err != nil {
			return err
		}
		delta.Quad.Upgrade()
		data, err := delta.Marshal()
		if err != nil {
			return err
		}
		lb.Put(k, data)
	}
	if err := tx.Commit(); err != nil {
		return err
	}
	tx, err = db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	fmt.Println("Upgrading bucket", string(nodeBucket))
	nb := tx.Bucket(nodeBucket)
	c = nb.Cursor()
	for k, v := c.First(); k != nil; k, v = c.Next() {
		var vd proto.NodeData
		err := vd.Unmarshal(v)
		if err != nil {
			return err
		}
		vd.Upgrade()
		data, err := vd.Marshal()
		if err != nil {
			return err
		}
		nb.Put(k, data)
	}
	if err := tx.Commit(); err != nil {
		return err
	}
	return nil
}
Exemple #6
0
func (qs *QuadStore) UpdateValueKeyBy(name quad.Value, amount int64, batch *leveldb.Batch) error {
	value := proto.NodeData{
		Value: proto.MakeValue(name),
		Size:  amount,
	}
	key := createValueKeyFor(name)
	b, err := qs.db.Get(key, qs.readopts)

	// Error getting the node from the database.
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorf("Error reading Value %s from the DB.", name)
		return err
	}

	// Node exists in the database -- unmarshal and update.
	if b != nil && err != leveldb.ErrNotFound {
		var oldvalue proto.NodeData
		err = oldvalue.Unmarshal(b)
		if err != nil {
			glog.Errorf("Error: could not reconstruct value: %v", err)
			return err
		}
		oldvalue.Size += amount
		value = oldvalue
	}

	// Are we deleting something?
	if value.Size <= 0 {
		value.Size = 0
	}

	// Repackage and rewrite.
	bytes, err := value.Marshal()
	if err != nil {
		glog.Errorf("could not write to buffer for value %s: %s", name, err)
		return err
	}
	if batch == nil {
		qs.db.Put(key, bytes, qs.writeopts)
	} else {
		batch.Put(key, bytes)
	}
	return nil
}
Exemple #7
0
func (qs *QuadStore) UpdateValueKeyBy(name quad.Value, amount int64, tx *bolt.Tx) error {
	value := proto.NodeData{
		Value: proto.MakeValue(name),
		Size:  amount,
	}
	b := tx.Bucket(nodeBucket)
	b.FillPercent = localFillPercent
	key := qs.createValueKeyFor(name)
	data := b.Get(key)

	if data != nil {
		// Node exists in the database -- unmarshal and update.
		var oldvalue proto.NodeData
		err := oldvalue.Unmarshal(data)
		if err != nil {
			glog.Errorf("Error: couldn't reconstruct value: %v", err)
			return err
		}
		oldvalue.Size += amount
		value = oldvalue
	}

	// Are we deleting something?
	if value.Size <= 0 {
		value.Size = 0
	}

	// Repackage and rewrite.
	bytes, err := value.Marshal()
	if err != nil {
		glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
		return err
	}
	err = b.Put(key, bytes)
	return err
}
Exemple #8
0
func upgrade1To2(db *leveldb.DB) error {
	fmt.Println("Upgrading v1 to v2...")

	type v1IndexEntry struct {
		Subject   string  `json:"subject"`
		Predicate string  `json:"predicate"`
		Object    string  `json:"object"`
		Label     string  `json:"label,omitempty"`
		History   []int64 `json:"History"`
	}

	type v1ValueData struct {
		Name string `json:"Name"`
		Size int64  `json:"Size"`
	}

	var (
		spoPref = []byte{spo[0].Prefix(), spo[1].Prefix()}
		ospPref = []byte{osp[0].Prefix(), osp[1].Prefix()}
		posPref = []byte{pos[0].Prefix(), pos[1].Prefix()}
		cpsPref = []byte{cps[0].Prefix(), cps[1].Prefix()}
	)

	{
		fmt.Println("Upgrading bucket z")
		it := db.NewIterator(&util.Range{Start: []byte{'z'}, Limit: []byte{'z' + 1}}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			var val v1ValueData
			if err := json.Unmarshal(v, &val); err != nil {
				return err
			}
			node := proto.NodeData{
				Size:  val.Size,
				Value: proto.MakeValue(quad.Raw(val.Name)),
			}
			nv, err := node.Marshal()
			if err != nil {
				return err
			}
			if err = db.Put(k, nv, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	for _, pref := range [4][]byte{spoPref, ospPref, posPref, cpsPref} {
		fmt.Println("Upgrading bucket", string(pref))
		end := []byte{pref[0], pref[1] + 1}
		it := db.NewIterator(&util.Range{Start: pref, Limit: end}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			var entry v1IndexEntry
			if err := json.Unmarshal(v, &entry); err != nil {
				return err
			}
			var h proto.HistoryEntry
			h.History = make([]uint64, len(entry.History))
			for i, id := range entry.History {
				h.History[i] = uint64(id)
			}
			nv, err := h.Marshal()
			if err != nil {
				return err
			}
			if err = db.Put(k, nv, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	{
		fmt.Println("Upgrading bucket d")
		it := db.NewIterator(&util.Range{Start: []byte{'d'}, Limit: []byte{'d' + 1}}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			id, err := strconv.ParseInt(string(k[1:]), 16, 64)
			if err != nil {
				return err
			}
			nk := createDeltaKeyFor(id)
			var val graph.Delta
			if err := json.Unmarshal(v, &val); err != nil {
				return err
			}
			p := deltaToProto(val)
			nv, err := p.Marshal()
			if err != nil {
				return err
			}
			b := &leveldb.Batch{}
			b.Put(nk, nv)
			b.Delete(k)
			if err = db.Write(b, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	return nil
}
Exemple #9
0
func upgrade1To2LMDB(qs *QuadStore) error {
	fmt.Println("Upgrading v1 to v2...")
	err := qs.env.Update(func(tx *lmdb.Txn) (err error) {
		fmt.Println("Upgrading DB", logDB)

		s := lmdbscan.New(tx, qs.logDBI)
		defer s.Close()

		for s.Scan() {
			var delta graph.Delta
			err := json.Unmarshal(s.Val(), &delta)
			if err != nil {
				return err
			}
			newd := deltaToProto(delta)
			data, err := newd.Marshal()
			if err != nil {
				return err
			}
			err = tx.Put(qs.logDBI, s.Key(), data, 0)
			if err != nil {
				return err
			}
		}

		return s.Err()
	})
	if err != nil {
		return err
	}
	err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
		fmt.Println("Upgrading DB", nodeDB)

		s := lmdbscan.New(tx, qs.nodeDBI)
		defer s.Close()

		for s.Scan() {
			var vd proto.NodeData
			err := json.Unmarshal(s.Val(), &vd)
			if err != nil {
				return err
			}
			data, err := vd.Marshal()
			if err != nil {
				return err
			}
			err = tx.Put(qs.nodeDBI, s.Key(), data, 0)
			if err != nil {
				return err
			}
		}

		return s.Err()
	})
	if err != nil {
		return err
	}

	for _, db := range [4]string{string(spoDB), string(ospDB), string(posDB), string(cpsDB)} {
		err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
			fmt.Println("Upgrading DB", db)
			dbi := qs.dbis[db]
			s := lmdbscan.New(tx, dbi)
			defer s.Close()

			for s.Scan() {
				var h proto.HistoryEntry
				err := json.Unmarshal(s.Val(), &h)
				if err != nil {
					return err
				}
				data, err := h.Marshal()
				if err != nil {
					return err
				}
				err = tx.Put(dbi, s.Key(), data, 0)
				if err != nil {
					return err
				}
			}

			return s.Err()
		})
		if err != nil {
			return err
		}

		return nil
	}
	return nil
}
Exemple #10
0
func upgrade1To2(db *bolt.DB) error {
	fmt.Println("Upgrading v1 to v2...")
	tx, err := db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	fmt.Println("Upgrading bucket", string(logBucket))
	lb := tx.Bucket(logBucket)
	c := lb.Cursor()
	for k, v := c.First(); k != nil; k, v = c.Next() {
		var delta graph.Delta
		err := json.Unmarshal(v, &delta)
		if err != nil {
			return err
		}
		newd := deltaToProto(delta)
		data, err := newd.Marshal()
		if err != nil {
			return err
		}
		lb.Put(k, data)
	}
	if err := tx.Commit(); err != nil {
		return err
	}
	tx, err = db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	fmt.Println("Upgrading bucket", string(nodeBucket))
	nb := tx.Bucket(nodeBucket)
	c = nb.Cursor()
	for k, v := c.First(); k != nil; k, v = c.Next() {
		var vd proto.NodeData
		err := json.Unmarshal(v, &vd)
		if err != nil {
			return err
		}
		data, err := vd.Marshal()
		if err != nil {
			return err
		}
		nb.Put(k, data)
	}
	if err := tx.Commit(); err != nil {
		return err
	}

	for _, bucket := range [4][]byte{spoBucket, ospBucket, posBucket, cpsBucket} {
		tx, err = db.Begin(true)
		if err != nil {
			return err
		}
		defer tx.Rollback()
		fmt.Println("Upgrading bucket", string(bucket))
		b := tx.Bucket(bucket)
		cur := b.Cursor()
		for k, v := cur.First(); k != nil; k, v = cur.Next() {
			var h proto.HistoryEntry
			err := json.Unmarshal(v, &h)
			if err != nil {
				return err
			}
			data, err := h.Marshal()
			if err != nil {
				return err
			}
			b.Put(k, data)
		}
		if err := tx.Commit(); err != nil {
			return err
		}
	}
	return nil
}