Beispiel #1
0
func (it *AllIterator) makeCursor() {
	var cursor *sql.Rows
	var err error
	if it.cursor != nil {
		it.cursor.Close()
	}
	if it.table == "quads" {
		cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
		if err != nil {
			glog.Errorln("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
	} else {
		glog.V(4).Infoln("sql: getting node query")
		cursor, err = it.qs.db.Query(`SELECT node FROM
			(
				SELECT subject FROM quads
				UNION
				SELECT predicate FROM quads
				UNION
				SELECT object FROM quads
				UNION
				SELECT label FROM quads
			) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
		if err != nil {
			glog.Errorln("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
		glog.V(4).Infoln("sql: got node query")
	}
	it.cursor = cursor
}
Beispiel #2
0
func upgradeLMDB(path string, opts graph.Options) error {
	env, err := createLMDB(path, opts)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return err
	}
	defer env.Close()

	qs := &QuadStore{}
	qs.env = env
	err = qs.openDBIs()
	if err != nil {
		return err
	}

	var version int64
	err = env.View(func(tx *lmdb.Txn) (err error) {
		version, err = qs.getInt64ForMetaKey(tx, "version", nilDataVersion)
		return err
	})
	if err != nil {
		glog.Errorln("error:", err)
		return err
	}

	if version == latestDataVersion {
		fmt.Printf("Already at latest version: %d\n", latestDataVersion)
		return nil
	}

	if version > latestDataVersion {
		err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version)
		glog.Errorln("error:", err)
		return err
	}

	for i := version; i < latestDataVersion; i++ {
		err := migrateFunctionsLMDB[i](qs)
		if err != nil {
			return err
		}
		err = setVersionLMDB(qs.env, qs.metaDBI, i+1)
		if err != nil {
			return err
		}
	}

	return nil
}
Beispiel #3
0
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	var qs QuadStore
	var err error
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return nil, err
	}
	qs.db = db
	// BoolKey returns false on non-existence. IE, Sync by default.
	qs.db.NoSync, _, err = options.BoolKey("nosync")
	if err != nil {
		return nil, err
	}
	err = qs.getMetadata()
	if err == errNoBucket {
		return nil, errors.New("bolt: quadstore has not been initialised")
	} else if err != nil {
		return nil, err
	}
	if qs.version != latestDataVersion {
		return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
	}
	return &qs, nil
}
Beispiel #4
0
func createNewLevelDB(path string, _ graph.Options) error {
	opts := &opt.Options{}
	db, err := leveldb.OpenFile(path, opts)
	if err != nil {
		glog.Errorf("Error: could not create database: %v", err)
		return err
	}
	defer db.Close()
	qs := &QuadStore{}
	qs.db = db
	qs.writeopts = &opt.WriteOptions{
		Sync: true,
	}
	qs.readopts = &opt.ReadOptions{}
	_, err = qs.db.Get([]byte(horizonKey), qs.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorln("couldn't read from leveldb during init")
		return err
	}
	if err != leveldb.ErrNotFound {
		return graph.ErrDatabaseExists
	}
	// Write some metadata
	qs.Close()
	return nil
}
Beispiel #5
0
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
	var qs TripleStore
	var err error
	qs.path = path
	cache_size := DefaultCacheSize
	if val, ok := options.IntKey("cache_size_mb"); ok {
		cache_size = val
	}
	qs.dbOpts = &opt.Options{
		BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
	}
	qs.dbOpts.ErrorIfMissing = true

	write_buffer_mb := DefaultWriteBufferSize
	if val, ok := options.IntKey("write_buffer_mb"); ok {
		write_buffer_mb = val
	}
	qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
	qs.writeopts = &opt.WriteOptions{
		Sync: false,
	}
	qs.readopts = &opt.ReadOptions{}
	db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return nil, err
	}
	qs.db = db
	glog.Infoln(qs.GetStats())
	err = qs.getMetadata()
	if err != nil {
		return nil, err
	}
	return &qs, nil
}
Beispiel #6
0
func NewIterator(ts *TripleStore, collection string, d graph.Direction, val graph.TSVal) *Iterator {
	var m Iterator
	iterator.BaseInit(&m.Base)

	m.name = ts.GetNameFor(val)
	m.collection = collection
	switch d {
	case graph.Subject:
		m.constraint = bson.M{"Subject": m.name}
	case graph.Predicate:
		m.constraint = bson.M{"Predicate": m.name}
	case graph.Object:
		m.constraint = bson.M{"Object": m.name}
	case graph.Provenance:
		m.constraint = bson.M{"Provenance": m.name}
	}

	m.ts = ts
	m.dir = d
	m.iter = ts.db.C(collection).Find(m.constraint).Iter()
	size, err := ts.db.C(collection).Find(m.constraint).Count()
	if err != nil {
		glog.Errorln("Trouble getting size for iterator! ", err)
		return nil
	}
	m.size = int64(size)
	m.hash = val.(string)
	m.isAll = false
	return &m
}
Beispiel #7
0
func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator {
	var m MongoIterator
	graph.BaseIteratorInit(&m.BaseIterator)

	m.name = ts.GetNameFor(val)
	m.collection = collection
	switch dir {

	case "s":
		m.constraint = bson.M{"Sub": m.name}
	case "p":
		m.constraint = bson.M{"Pred": m.name}
	case "o":
		m.constraint = bson.M{"Obj": m.name}
	case "c":
		m.constraint = bson.M{"Provenance": m.name}
	}

	m.ts = ts
	m.dir = dir
	m.iter = ts.db.C(collection).Find(m.constraint).Iter()
	size, err := ts.db.C(collection).Find(m.constraint).Count()
	if err != nil {
		glog.Errorln("Trouble getting size for iterator! ", err)
		return nil
	}
	m.size = int64(size)
	m.hash = val.(string)
	m.isAll = false
	return &m
}
Beispiel #8
0
// Next is a convenience function that conditionally calls the Next method
// of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next
// return a nil Value and false.
func Next(it Iterator) (Value, bool) {
	if n, ok := it.(Nexter); ok {
		return n.Next()
	}
	glog.Errorln("Nexting an un-nextable iterator")
	return nil, false
}
Beispiel #9
0
func getViaData(obj *otto.Object) (predicates []interface{}, tags []string, ok bool) {
	argList, _ := obj.Get("_gremlin_values")
	if argList.Class() != "GoArray" {
		glog.Errorln("How is arglist not an array? Return nothing.", argList.Class())
		return nil, nil, false
	}
	argArray := argList.Object()
	lengthVal, _ := argArray.Get("length")
	length, _ := lengthVal.ToInteger()
	if length == 0 {
		predicates = []interface{}{}
	} else {
		zero, _ := argArray.Get("0")
		predicates = buildPathFromValue(zero)
	}
	if length >= 2 {
		one, _ := argArray.Get("1")
		if one.IsString() {
			tags = append(tags, one.String())
		} else if one.Class() == "Array" {
			tags = stringsFrom(one.Object())
		}
	}
	ok = true
	return
}
Beispiel #10
0
func NewIterator(qs *TripleStore, collection string, d quad.Direction, val graph.Value) *Iterator {
	name := qs.NameOf(val)

	constraint := bson.M{d.String(): name}

	size, err := qs.db.C(collection).Find(constraint).Count()
	if err != nil {
		// FIXME(kortschak) This should be passed back rather than just logging.
		glog.Errorln("Trouble getting size for iterator! ", err)
		return nil
	}

	return &Iterator{
		uid:        iterator.NextUID(),
		name:       name,
		constraint: constraint,
		collection: collection,
		qs:         qs,
		dir:        d,
		iter:       qs.db.C(collection).Find(constraint).Iter(),
		size:       int64(size),
		hash:       val.(string),
		isAll:      false,
	}
}
Beispiel #11
0
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, val string) int64 {
	var err error
	if isAll {
		return qs.Size()
	}
	if qs.noSizes {
		if dir == quad.Predicate {
			return (qs.Size() / 100) + 1
		}
		return (qs.Size() / 1000) + 1
	}
	if val, ok := qs.lru.Get(val + string(dir.Prefix())); ok {
		return val
	}
	var size int64
	glog.V(4).Infoln("sql: getting size for select %s, %s", dir.String(), val)
	err = qs.db.QueryRow(
		fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hashOf(val)).Scan(&size)
	if err != nil {
		glog.Errorln("Error getting size from SQL database: %v", err)
		return 0
	}
	qs.lru.Put(val+string(dir.Prefix()), size)
	return size
}
Beispiel #12
0
func init() {
	glog.SetToStderr(true)
	cfg := config.ParseConfigFromFile("cayley_appengine.cfg")
	qs, _ := graph.NewQuadStore("memstore", "", nil)
	glog.Errorln(cfg)
	db.Load(qs, cfg, cfg.DatabasePath)
	http.SetupRoutes(qs, cfg)
}
Beispiel #13
0
func init() {
	glog.SetToStderr(true)
	cfg := config.ParseConfigFromFile("cayley_appengine.cfg")
	ts := memstore.NewMemTripleStore()
	glog.Errorln(cfg)
	LoadTriplesFromFileInto(ts, cfg.DatabasePath, cfg.LoadSize)
	http.SetupRoutes(ts, cfg)
}
Beispiel #14
0
func (ts *TripleStore) Triple(k graph.Value) *graph.Triple {
	var triple graph.Triple
	b, err := ts.db.Get(k.([]byte), ts.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorln("Error: couldn't get triple from DB")
		return &graph.Triple{}
	}
	if err == leveldb.ErrNotFound {
		// No harm, no foul.
		return &graph.Triple{}
	}
	err = json.Unmarshal(b, &triple)
	if err != nil {
		glog.Errorln("Error: couldn't reconstruct triple")
		return &graph.Triple{}
	}
	return &triple
}
Beispiel #15
0
func (ts *TripleStore) valueData(value_key []byte) ValueData {
	var out ValueData
	if glog.V(3) {
		glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key)
	}
	b, err := ts.db.Get(value_key, ts.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorln("Error: couldn't get value from DB")
		return out
	}
	if b != nil && err != leveldb.ErrNotFound {
		err = json.Unmarshal(b, &out)
		if err != nil {
			glog.Errorln("Error: couldn't reconstruct value")
			return ValueData{}
		}
	}
	return out
}
Beispiel #16
0
func (qs *TripleStore) getInt64ForKey(key string, empty int64) (int64, error) {
	var out int64
	b, err := qs.db.Get([]byte(key), qs.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorln("Couldn't read " + key + ": " + err.Error())
		return 0, err
	}
	if err == leveldb.ErrNotFound {
		// Must be a new database. Cool
		return empty, nil
	}
	buf := bytes.NewBuffer(b)
	err = binary.Read(buf, binary.LittleEndian, &out)
	if err != nil {
		glog.Errorln("Error: couldn't parse", key)
		return 0, err
	}
	return out, nil
}
Beispiel #17
0
func (qs *QuadStore) valueData(key []byte) ValueData {
	var out ValueData
	if glog.V(3) {
		glog.V(3).Infof("%c %v", key[0], key)
	}
	b, err := qs.db.Get(key, qs.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		glog.Errorln("Error: could not get value from DB")
		return out
	}
	if b != nil && err != leveldb.ErrNotFound {
		err = json.Unmarshal(b, &out)
		if err != nil {
			glog.Errorln("Error: could not reconstruct value")
			return ValueData{}
		}
	}
	return out
}
Beispiel #18
0
func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator {
	if val.IsNull() || val.IsUndefined() {
		return ts.GetNodesAllIterator()
	}
	if val.IsPrimitive() {
		thing, _ := val.Export()
		switch v := thing.(type) {
		case string:
			it := ts.MakeFixed()
			it.AddValue(ts.GetIdFor(v))
			return it
		default:
			glog.Errorln("Trying to build unknown primitive value.")
		}
	}
	switch val.Class() {
	case "Object":
		return buildIteratorTree(val.Object(), ts)
	case "Array":
		// Had better be an array of strings
		strings := makeListOfStringsFromArrayValue(val.Object())
		it := ts.MakeFixed()
		for _, x := range strings {
			it.AddValue(ts.GetIdFor(x))
		}
		return it
	case "Number":
		fallthrough
	case "Boolean":
		fallthrough
	case "Date":
		fallthrough
	case "String":
		it := ts.MakeFixed()
		str, _ := val.ToString()
		it.AddValue(ts.GetIdFor(str))
		return it
	default:
		glog.Errorln("Trying to handle unsupported Javascript value.")
		return graph.NewNullIterator()
	}
}
Beispiel #19
0
func (p *PrimaryKey) Int() int64 {
	switch p.keyType {
	case sequential:
		return p.sequentialID
	case unique:
		msg := "UUID cannot be converted to an int64"
		glog.Errorln(msg)
		panic(msg)
	}
	return -1
}
Beispiel #20
0
func buildIteratorFromValue(val otto.Value, qs graph.QuadStore) graph.Iterator {
	if val.IsNull() || val.IsUndefined() {
		return qs.NodesAllIterator()
	}
	if val.IsPrimitive() {
		thing, _ := val.Export()
		switch v := thing.(type) {
		case string:
			it := qs.FixedIterator()
			it.Add(qs.ValueOf(v))
			return it
		default:
			glog.Errorln("Trying to build unknown primitive value.")
		}
	}
	switch val.Class() {
	case "Object":
		return buildIteratorTree(val.Object(), qs)
	case "Array":
		// Had better be an array of strings
		strings := stringsFrom(val.Object())
		it := qs.FixedIterator()
		for _, x := range strings {
			it.Add(qs.ValueOf(x))
		}
		return it
	case "Number":
		fallthrough
	case "Boolean":
		fallthrough
	case "Date":
		fallthrough
	case "String":
		it := qs.FixedIterator()
		it.Add(qs.ValueOf(val.String()))
		return it
	default:
		glog.Errorln("Trying to handle unsupported Javascript value.")
		return iterator.NewNull()
	}
}
Beispiel #21
0
func (qs *QuadStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
	subs := it.SubIterators()
	if len(subs) != 1 {
		return it, false
	}
	primary := subs[0]
	switch primary.Type() {
	case graph.Fixed:
		size, _ := primary.Size()
		if size == 1 {
			if !graph.Next(primary) {
				panic("unexpected size during optimize")
			}
			val := primary.Result()
			newIt := qs.QuadIterator(it.Direction(), val)
			nt := newIt.Tagger()
			nt.CopyFrom(it)
			for _, tag := range primary.Tagger().Tags() {
				nt.AddFixed(tag, val)
			}
			it.Close()
			return newIt, true
		}
	case sqlType:
		p := primary.(*SQLIterator)
		newit, err := linksto(p.sql, it.Direction(), qs)
		if err != nil {
			glog.Errorln(err)
			return it, false
		}
		newit.Tagger().CopyFrom(it)
		return newit, true
	case graph.All:
		linkit := &SQLLinkIterator{
			tableName: newTableName(),
			size:      qs.Size(),
		}
		for _, t := range primary.Tagger().Tags() {
			linkit.tagdirs = append(linkit.tagdirs, tagDir{
				dir: it.Direction(),
				tag: t,
			})
		}
		for k, v := range primary.Tagger().Fixed() {
			linkit.tagger.AddFixed(k, v)
		}
		linkit.tagger.CopyFrom(it)
		newit := NewSQLIterator(qs, linkit)
		return newit, true
	}
	return it, false
}
Beispiel #22
0
func upgradeBolt(path string, opts graph.Options) error {
	db, err := bolt.Open(path, 0600, nil)
	defer db.Close()

	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return err
	}
	var version int64
	err = db.View(func(tx *bolt.Tx) error {
		version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
		return err
	})
	if err != nil {
		glog.Errorln("error:", err)
		return err
	}

	if version == latestDataVersion {
		fmt.Printf("Already at latest version: %d\n", latestDataVersion)
		return nil
	}

	if version > latestDataVersion {
		err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version)
		glog.Errorln("error:", err)
		return err
	}

	for i := version; i < latestDataVersion; i++ {
		err := migrateFunctions[i](db)
		if err != nil {
			return err
		}
		setVersion(db, i+1)
	}

	return nil
}
Beispiel #23
0
func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
	if api.config.ReadOnly {
		return jsonResponse(w, 400, "Database is read-only.")
	}

	formFile, _, err := r.FormFile("NQuadFile")
	if err != nil {
		glog.Errorln(err)
		return jsonResponse(w, 500, "Couldn't read file: "+err.Error())
	}
	defer formFile.Close()

	blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
	if blockErr != nil {
		blockSize = int64(api.config.LoadSize)
	}

	quadReader, err := internal.Decompressor(formFile)
	// TODO(kortschak) Make this configurable from the web UI.
	dec := cquads.NewDecoder(quadReader)

	h, err := api.GetHandleForRequest(r)
	if err != nil {
		return jsonResponse(w, 400, err)
	}

	var (
		n     int
		block = make([]quad.Quad, 0, blockSize)
	)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			glog.Fatalln("what can do this here?", err) // FIXME(kortschak)
		}
		block = append(block, t)
		n++
		if len(block) == cap(block) {
			h.QuadWriter.AddQuadSet(block)
			block = block[:0]
		}
	}
	h.QuadWriter.AddQuadSet(block)

	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)

	return 200
}
Beispiel #24
0
func buildPathFromValue(val otto.Value) (out []interface{}) {
	if val.IsNull() || val.IsUndefined() {
		return nil
	}
	if val.IsPrimitive() {
		thing, _ := val.Export()
		switch v := thing.(type) {
		case string:
			out = append(out, v)
			return
		default:
			glog.Errorln("Trying to build unknown primitive value.")
		}
	}
	switch val.Class() {
	case "Object":
		out = append(out, buildPathFromObject(val.Object()))
		return
	case "Array":
		// Had better be an array of strings
		for _, x := range stringsFrom(val.Object()) {
			out = append(out, x)
		}
		return
	case "Number":
		fallthrough
	case "Boolean":
		fallthrough
	case "Date":
		fallthrough
	case "String":
		out = append(out, val.String())
		return
	default:
		glog.Errorln("Trying to handle unsupported Javascript value.")
		return nil
	}
}
Beispiel #25
0
func createNewLevelDB(path string, _ graph.Options) error {
	opts := &opt.Options{}
	db, err := leveldb.OpenFile(path, opts)
	if err != nil {
		glog.Errorln("Error: couldn't create database: ", err)
		return err
	}
	defer db.Close()
	ts := &TripleStore{}
	ts.db = db
	ts.writeopts = &opt.WriteOptions{
		Sync: true,
	}
	ts.Close()
	return nil
}
Beispiel #26
0
func CreateNewLevelDB(path string) bool {
	opts := &opt.Options{}
	db, err := leveldb.OpenFile(path, opts)
	if err != nil {
		glog.Errorln("Error: couldn't create database", err)
		return false
	}
	defer db.Close()
	ts := &TripleStore{}
	ts.db = db
	ts.writeopts = &opt.WriteOptions{
		Sync: true,
	}
	ts.Close()
	return true
}
Beispiel #27
0
func Load(ts graph.TripleStore, cfg *config.Config, triplePath string) {
	tChan := make(chan *graph.Triple)
	go ReadTriplesFromFile(tChan, triplePath)

	bulker, canBulk := ts.(graph.BulkLoader)
	if canBulk {
		err := bulker.BulkLoad(tChan)
		if err == nil {
			return
		}
		if err != graph.ErrCannotBulkLoad {
			glog.Errorln("Error attempting to bulk load: ", err)
		}
	}

	LoadTriplesInto(tChan, ts, cfg.LoadSize)
}
Beispiel #28
0
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	var qs QuadStore
	var err error
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return nil, err
	}
	qs.db = db
	// BoolKey returns false on non-existence. IE, Sync by default.
	qs.db.NoSync, _ = options.BoolKey("nosync")
	err = qs.getMetadata()
	if err != nil {
		return nil, err
	}
	return &qs, nil
}
Beispiel #29
0
func NewAllIterator(ts *TripleStore, collection string) *Iterator {
	var m Iterator
	m.ts = ts
	m.dir = graph.Any
	m.constraint = nil
	m.collection = collection
	m.iter = ts.db.C(collection).Find(nil).Iter()
	size, err := ts.db.C(collection).Count()
	if err != nil {
		glog.Errorln("Trouble getting size for iterator! ", err)
		return nil
	}
	m.size = int64(size)
	m.hash = ""
	m.isAll = true
	return &m
}
Beispiel #30
0
func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
	if api.config.ReadOnly {
		return FormatJson400(w, "Database is read-only.")
	}

	formFile, _, err := r.FormFile("NQuadFile")
	if err != nil {
		glog.Errorln(err)
		return FormatJsonError(w, 500, "Couldn't read file: "+err.Error())
	}

	defer formFile.Close()

	blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
	if blockErr != nil {
		blockSize = int64(api.config.LoadSize)
	}

	dec := nquads.NewDecoder(formFile)

	var (
		n int

		block = make([]*quad.Quad, 0, blockSize)
	)
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			panic("what can do this here?") // FIXME(kortschak)
		}
		block = append(block, t)
		n++
		if len(block) == cap(block) {
			api.ts.AddTripleSet(block)
			block = block[:0]
		}
	}
	api.ts.AddTripleSet(block)

	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", n)

	return 200
}