Exemple #1
0
func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
	var in proto.HistoryEntry
	b, err := qs.db.Get(k.(Token), qs.readopts)
	if err == leveldb.ErrNotFound {
		// No harm, no foul.
		return quad.Quad{}
	} else if err != nil {
		glog.Error("Error: could not get quad from DB.")
		return quad.Quad{}
	}
	err = in.Unmarshal(b)
	if err != nil {
		glog.Error("Error: could not reconstruct history.", err)
		return quad.Quad{}
	}
	b, err = qs.db.Get(createDeltaKeyFor(int64(in.History[len(in.History)-1])), qs.readopts)
	if err == leveldb.ErrNotFound {
		// No harm, no foul.
		return quad.Quad{}
	} else if err != nil {
		glog.Error("Error: could not get quad from DB.")
		return quad.Quad{}
	}
	var d proto.LogDelta
	err = d.Unmarshal(b)
	if err != nil {
		glog.Error("Error: could not reconstruct quad.", err)
		return quad.Quad{}
	}
	return d.Quad.ToNative()
}
Exemple #2
0
func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
	var d proto.LogDelta
	tok := k.(*Token)
	err := qs.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(tok.bucket)
		data := b.Get(tok.key)
		if data == nil {
			return nil
		}
		var in proto.HistoryEntry
		err := in.Unmarshal(data)
		if err != nil {
			return err
		}
		if len(in.History) == 0 {
			return nil
		}
		b = tx.Bucket(logBucket)
		data = b.Get(qs.createDeltaKeyFor(int64(in.History[len(in.History)-1])))
		if data == nil {
			// No harm, no foul.
			return nil
		}
		return d.Unmarshal(data)
	})
	if err != nil {
		glog.Error("Error getting quad: ", err)
		return quad.Quad{}
	}
	return d.Quad.ToNative()
}
Exemple #3
0
// Quad ??
func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
	var d proto.LogDelta
	tok := k.(*Token)
	err := qs.env.View(func(tx *lmdb.Txn) (err error) {
		tx.RawRead = true

		dbi := qs.dbis[tok.db]
		data, _ := tx.Get(dbi, tok.key)
		if data == nil {
			return nil
		}
		var in proto.HistoryEntry
		err = in.Unmarshal(data)
		if err != nil {
			return err
		}
		if len(in.History) == 0 {
			return nil
		}
		data, _ = tx.Get(qs.logDBI, qs.createDeltaKeyFor(int64(in.History[len(in.History)-1])))
		if data == nil {
			// No harm, no foul.
			return nil
		}
		return d.Unmarshal(data)
	})
	if err != nil {
		glog.Error("Error getting quad: ", err)
		return quad.Quad{}
	}
	if d.Quad == nil {
		glog.Error("Unable to get quad: ", err)
		return quad.Quad{}
	}
	return quad.Quad{
		d.Quad.Subject,
		d.Quad.Predicate,
		d.Quad.Object,
		d.Quad.Label,
	}
}
Exemple #4
0
func (qs *QuadStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64, isAdd bool) error {
	var entry proto.HistoryEntry
	data, err := qs.db.Get(createKeyFor(spo, q), qs.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Error("could not access DB to prepare index: ", err)
		return err
	}
	if data != nil {
		// We got something.
		err = entry.Unmarshal(data)
		if err != nil {
			return err
		}
	}

	if isAdd && len(entry.History)%2 == 1 {
		if glog.V(2) {
			glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
		}
		return graph.ErrQuadExists
	}
	if !isAdd && len(entry.History)%2 == 0 {
		if glog.V(2) {
			glog.Errorf("attempt to delete non-existent quad %v: %#c", entry, q)
		}
		return graph.ErrQuadNotExist
	}

	entry.History = append(entry.History, uint64(id))

	bytes, err := entry.Marshal()
	if err != nil {
		glog.Errorf("could not write to buffer for entry %#v: %s", entry, err)
		return err
	}
	batch.Put(createKeyFor(spo, q), bytes)
	batch.Put(createKeyFor(osp, q), bytes)
	batch.Put(createKeyFor(pos, q), bytes)
	if q.Get(quad.Label) != nil {
		batch.Put(createKeyFor(cps, q), bytes)
	}
	return nil
}
Exemple #5
0
func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bool) error {
	var entry proto.HistoryEntry
	b := tx.Bucket(spoBucket)
	b.FillPercent = localFillPercent
	data := b.Get(qs.createKeyFor(spo, q))
	if data != nil {
		// We got something.
		err := entry.Unmarshal(data)
		if err != nil {
			return err
		}
	}

	if isAdd && len(entry.History)%2 == 1 {
		glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
		return graph.ErrQuadExists
	}
	if !isAdd && len(entry.History)%2 == 0 {
		glog.Errorf("attempt to delete non-existent quad %v: %#v", entry, q)
		return graph.ErrQuadNotExist
	}

	entry.History = append(entry.History, uint64(id))

	bytes, err := entry.Marshal()
	if err != nil {
		glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
		return err
	}
	for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
		if index == cps && q.Get(quad.Label) == nil {
			continue
		}
		b := tx.Bucket(bucketFor(index))
		b.FillPercent = localFillPercent
		err = b.Put(qs.createKeyFor(index, q), bytes)
		if err != nil {
			return err
		}
	}
	return nil
}
Exemple #6
0
func (qs *QuadStore) buildQuadWriteLMDB(tx *lmdb.Txn, q quad.Quad, id int64, isAdd bool) error {
	var entry proto.HistoryEntry
	dbi := qs.dbis[spoDB]
	data, err := tx.Get(dbi, qs.createKeyFor(spo, q))
	if err == nil {
		// We got something.
		err := entry.Unmarshal(data)
		if err != nil {
			return err
		}
	}

	if isAdd && len(entry.History)%2 == 1 {
		glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
		return graph.ErrQuadExists
	}
	if !isAdd && len(entry.History)%2 == 0 {
		glog.Errorf("attempt to delete non-existent quad %v: %#v", entry, q)
		return graph.ErrQuadNotExist
	}

	entry.History = append(entry.History, uint64(id))

	bytes, err := entry.Marshal()
	if err != nil {
		glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
		return err
	}
	for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
		if index == cps && q.Get(quad.Label) == "" {
			continue
		}
		dbi = qs.dbis[dbFor(index)]
		err = tx.Put(dbi, qs.createKeyFor(index, q), bytes, 0)
		if err != nil {
			return err
		}
	}
	return nil
}
Exemple #7
0
func (it *Iterator) isLiveValue(val []byte) bool {
	var entry proto.HistoryEntry
	entry.Unmarshal(val)
	return len(entry.History)%2 != 0
}
Exemple #8
0
func upgrade1To2(db *leveldb.DB) error {
	fmt.Println("Upgrading v1 to v2...")

	type v1IndexEntry struct {
		Subject   string  `json:"subject"`
		Predicate string  `json:"predicate"`
		Object    string  `json:"object"`
		Label     string  `json:"label,omitempty"`
		History   []int64 `json:"History"`
	}

	type v1ValueData struct {
		Name string `json:"Name"`
		Size int64  `json:"Size"`
	}

	var (
		spoPref = []byte{spo[0].Prefix(), spo[1].Prefix()}
		ospPref = []byte{osp[0].Prefix(), osp[1].Prefix()}
		posPref = []byte{pos[0].Prefix(), pos[1].Prefix()}
		cpsPref = []byte{cps[0].Prefix(), cps[1].Prefix()}
	)

	{
		fmt.Println("Upgrading bucket z")
		it := db.NewIterator(&util.Range{Start: []byte{'z'}, Limit: []byte{'z' + 1}}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			var val v1ValueData
			if err := json.Unmarshal(v, &val); err != nil {
				return err
			}
			node := proto.NodeData{
				Size:  val.Size,
				Value: proto.MakeValue(quad.Raw(val.Name)),
			}
			nv, err := node.Marshal()
			if err != nil {
				return err
			}
			if err = db.Put(k, nv, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	for _, pref := range [4][]byte{spoPref, ospPref, posPref, cpsPref} {
		fmt.Println("Upgrading bucket", string(pref))
		end := []byte{pref[0], pref[1] + 1}
		it := db.NewIterator(&util.Range{Start: pref, Limit: end}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			var entry v1IndexEntry
			if err := json.Unmarshal(v, &entry); err != nil {
				return err
			}
			var h proto.HistoryEntry
			h.History = make([]uint64, len(entry.History))
			for i, id := range entry.History {
				h.History[i] = uint64(id)
			}
			nv, err := h.Marshal()
			if err != nil {
				return err
			}
			if err = db.Put(k, nv, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	{
		fmt.Println("Upgrading bucket d")
		it := db.NewIterator(&util.Range{Start: []byte{'d'}, Limit: []byte{'d' + 1}}, nil)
		for it.Next() {
			k, v := it.Key(), it.Value()
			id, err := strconv.ParseInt(string(k[1:]), 16, 64)
			if err != nil {
				return err
			}
			nk := createDeltaKeyFor(id)
			var val graph.Delta
			if err := json.Unmarshal(v, &val); err != nil {
				return err
			}
			p := deltaToProto(val)
			nv, err := p.Marshal()
			if err != nil {
				return err
			}
			b := &leveldb.Batch{}
			b.Put(nk, nv)
			b.Delete(k)
			if err = db.Write(b, nil); err != nil {
				return err
			}
		}
		it.Release()
	}

	return nil
}
Exemple #9
0
func upgrade1To2LMDB(qs *QuadStore) error {
	fmt.Println("Upgrading v1 to v2...")
	err := qs.env.Update(func(tx *lmdb.Txn) (err error) {
		fmt.Println("Upgrading DB", logDB)

		s := lmdbscan.New(tx, qs.logDBI)
		defer s.Close()

		for s.Scan() {
			var delta graph.Delta
			err := json.Unmarshal(s.Val(), &delta)
			if err != nil {
				return err
			}
			newd := deltaToProto(delta)
			data, err := newd.Marshal()
			if err != nil {
				return err
			}
			err = tx.Put(qs.logDBI, s.Key(), data, 0)
			if err != nil {
				return err
			}
		}

		return s.Err()
	})
	if err != nil {
		return err
	}
	err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
		fmt.Println("Upgrading DB", nodeDB)

		s := lmdbscan.New(tx, qs.nodeDBI)
		defer s.Close()

		for s.Scan() {
			var vd proto.NodeData
			err := json.Unmarshal(s.Val(), &vd)
			if err != nil {
				return err
			}
			data, err := vd.Marshal()
			if err != nil {
				return err
			}
			err = tx.Put(qs.nodeDBI, s.Key(), data, 0)
			if err != nil {
				return err
			}
		}

		return s.Err()
	})
	if err != nil {
		return err
	}

	for _, db := range [4]string{string(spoDB), string(ospDB), string(posDB), string(cpsDB)} {
		err = qs.env.Update(func(tx *lmdb.Txn) (err error) {
			fmt.Println("Upgrading DB", db)
			dbi := qs.dbis[db]
			s := lmdbscan.New(tx, dbi)
			defer s.Close()

			for s.Scan() {
				var h proto.HistoryEntry
				err := json.Unmarshal(s.Val(), &h)
				if err != nil {
					return err
				}
				data, err := h.Marshal()
				if err != nil {
					return err
				}
				err = tx.Put(dbi, s.Key(), data, 0)
				if err != nil {
					return err
				}
			}

			return s.Err()
		})
		if err != nil {
			return err
		}

		return nil
	}
	return nil
}
Exemple #10
0
func upgrade1To2(db *bolt.DB) error {
	fmt.Println("Upgrading v1 to v2...")
	tx, err := db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	fmt.Println("Upgrading bucket", string(logBucket))
	lb := tx.Bucket(logBucket)
	c := lb.Cursor()
	for k, v := c.First(); k != nil; k, v = c.Next() {
		var delta graph.Delta
		err := json.Unmarshal(v, &delta)
		if err != nil {
			return err
		}
		newd := deltaToProto(delta)
		data, err := newd.Marshal()
		if err != nil {
			return err
		}
		lb.Put(k, data)
	}
	if err := tx.Commit(); err != nil {
		return err
	}
	tx, err = db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	fmt.Println("Upgrading bucket", string(nodeBucket))
	nb := tx.Bucket(nodeBucket)
	c = nb.Cursor()
	for k, v := c.First(); k != nil; k, v = c.Next() {
		var vd proto.NodeData
		err := json.Unmarshal(v, &vd)
		if err != nil {
			return err
		}
		data, err := vd.Marshal()
		if err != nil {
			return err
		}
		nb.Put(k, data)
	}
	if err := tx.Commit(); err != nil {
		return err
	}

	for _, bucket := range [4][]byte{spoBucket, ospBucket, posBucket, cpsBucket} {
		tx, err = db.Begin(true)
		if err != nil {
			return err
		}
		defer tx.Rollback()
		fmt.Println("Upgrading bucket", string(bucket))
		b := tx.Bucket(bucket)
		cur := b.Cursor()
		for k, v := cur.First(); k != nil; k, v = cur.Next() {
			var h proto.HistoryEntry
			err := json.Unmarshal(v, &h)
			if err != nil {
				return err
			}
			data, err := h.Marshal()
			if err != nil {
				return err
			}
			b.Put(k, data)
		}
		if err := tx.Commit(); err != nil {
			return err
		}
	}
	return nil
}