func (it *AllIterator) makeCursor() { var cursor *sql.Rows var err error if it.cursor != nil { it.cursor.Close() } if it.table == "quads" { cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`) if err != nil { glog.Errorln("Couldn't get cursor from SQL database: %v", err) cursor = nil } } else { glog.V(4).Infoln("sql: getting node query") cursor, err = it.qs.db.Query(`SELECT node FROM ( SELECT subject FROM quads UNION SELECT predicate FROM quads UNION SELECT object FROM quads UNION SELECT label FROM quads ) AS DistinctNodes (node) WHERE node IS NOT NULL;`) if err != nil { glog.Errorln("Couldn't get cursor from SQL database: %v", err) cursor = nil } glog.V(4).Infoln("sql: got node query") } it.cursor = cursor }
func upgradeLMDB(path string, opts graph.Options) error { env, err := createLMDB(path, opts) if err != nil { glog.Errorln("Error, couldn't open! ", err) return err } defer env.Close() qs := &QuadStore{} qs.env = env err = qs.openDBIs() if err != nil { return err } var version int64 err = env.View(func(tx *lmdb.Txn) (err error) { version, err = qs.getInt64ForMetaKey(tx, "version", nilDataVersion) return err }) if err != nil { glog.Errorln("error:", err) return err } if version == latestDataVersion { fmt.Printf("Already at latest version: %d\n", latestDataVersion) return nil } if version > latestDataVersion { err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version) glog.Errorln("error:", err) return err } for i := version; i < latestDataVersion; i++ { err := migrateFunctionsLMDB[i](qs) if err != nil { return err } err = setVersionLMDB(qs.env, qs.metaDBI, i+1) if err != nil { return err } } return nil }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error db, err := bolt.Open(path, 0600, nil) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db // BoolKey returns false on non-existence. IE, Sync by default. qs.db.NoSync, _, err = options.BoolKey("nosync") if err != nil { return nil, err } err = qs.getMetadata() if err == errNoBucket { return nil, errors.New("bolt: quadstore has not been initialised") } else if err != nil { return nil, err } if qs.version != latestDataVersion { return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.") } return &qs, nil }
func createNewLevelDB(path string, _ graph.Options) error { opts := &opt.Options{} db, err := leveldb.OpenFile(path, opts) if err != nil { glog.Errorf("Error: could not create database: %v", err) return err } defer db.Close() qs := &QuadStore{} qs.db = db qs.writeopts = &opt.WriteOptions{ Sync: true, } qs.readopts = &opt.ReadOptions{} _, err = qs.db.Get([]byte(horizonKey), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Errorln("couldn't read from leveldb during init") return err } if err != leveldb.ErrNotFound { return graph.ErrDatabaseExists } // Write some metadata qs.Close() return nil }
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) { var qs TripleStore var err error qs.path = path cache_size := DefaultCacheSize if val, ok := options.IntKey("cache_size_mb"); ok { cache_size = val } qs.dbOpts = &opt.Options{ BlockCache: cache.NewLRUCache(cache_size * opt.MiB), } qs.dbOpts.ErrorIfMissing = true write_buffer_mb := DefaultWriteBufferSize if val, ok := options.IntKey("write_buffer_mb"); ok { write_buffer_mb = val } qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB qs.writeopts = &opt.WriteOptions{ Sync: false, } qs.readopts = &opt.ReadOptions{} db, err := leveldb.OpenFile(qs.path, qs.dbOpts) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db glog.Infoln(qs.GetStats()) err = qs.getMetadata() if err != nil { return nil, err } return &qs, nil }
func NewIterator(ts *TripleStore, collection string, d graph.Direction, val graph.TSVal) *Iterator { var m Iterator iterator.BaseInit(&m.Base) m.name = ts.GetNameFor(val) m.collection = collection switch d { case graph.Subject: m.constraint = bson.M{"Subject": m.name} case graph.Predicate: m.constraint = bson.M{"Predicate": m.name} case graph.Object: m.constraint = bson.M{"Object": m.name} case graph.Provenance: m.constraint = bson.M{"Provenance": m.name} } m.ts = ts m.dir = d m.iter = ts.db.C(collection).Find(m.constraint).Iter() size, err := ts.db.C(collection).Find(m.constraint).Count() if err != nil { glog.Errorln("Trouble getting size for iterator! ", err) return nil } m.size = int64(size) m.hash = val.(string) m.isAll = false return &m }
func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator { var m MongoIterator graph.BaseIteratorInit(&m.BaseIterator) m.name = ts.GetNameFor(val) m.collection = collection switch dir { case "s": m.constraint = bson.M{"Sub": m.name} case "p": m.constraint = bson.M{"Pred": m.name} case "o": m.constraint = bson.M{"Obj": m.name} case "c": m.constraint = bson.M{"Provenance": m.name} } m.ts = ts m.dir = dir m.iter = ts.db.C(collection).Find(m.constraint).Iter() size, err := ts.db.C(collection).Find(m.constraint).Count() if err != nil { glog.Errorln("Trouble getting size for iterator! ", err) return nil } m.size = int64(size) m.hash = val.(string) m.isAll = false return &m }
// Next is a convenience function that conditionally calls the Next method // of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next // return a nil Value and false. func Next(it Iterator) (Value, bool) { if n, ok := it.(Nexter); ok { return n.Next() } glog.Errorln("Nexting an un-nextable iterator") return nil, false }
func getViaData(obj *otto.Object) (predicates []interface{}, tags []string, ok bool) { argList, _ := obj.Get("_gremlin_values") if argList.Class() != "GoArray" { glog.Errorln("How is arglist not an array? Return nothing.", argList.Class()) return nil, nil, false } argArray := argList.Object() lengthVal, _ := argArray.Get("length") length, _ := lengthVal.ToInteger() if length == 0 { predicates = []interface{}{} } else { zero, _ := argArray.Get("0") predicates = buildPathFromValue(zero) } if length >= 2 { one, _ := argArray.Get("1") if one.IsString() { tags = append(tags, one.String()) } else if one.Class() == "Array" { tags = stringsFrom(one.Object()) } } ok = true return }
func NewIterator(qs *TripleStore, collection string, d quad.Direction, val graph.Value) *Iterator { name := qs.NameOf(val) constraint := bson.M{d.String(): name} size, err := qs.db.C(collection).Find(constraint).Count() if err != nil { // FIXME(kortschak) This should be passed back rather than just logging. glog.Errorln("Trouble getting size for iterator! ", err) return nil } return &Iterator{ uid: iterator.NextUID(), name: name, constraint: constraint, collection: collection, qs: qs, dir: d, iter: qs.db.C(collection).Find(constraint).Iter(), size: int64(size), hash: val.(string), isAll: false, } }
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, val string) int64 { var err error if isAll { return qs.Size() } if qs.noSizes { if dir == quad.Predicate { return (qs.Size() / 100) + 1 } return (qs.Size() / 1000) + 1 } if val, ok := qs.lru.Get(val + string(dir.Prefix())); ok { return val } var size int64 glog.V(4).Infoln("sql: getting size for select %s, %s", dir.String(), val) err = qs.db.QueryRow( fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hashOf(val)).Scan(&size) if err != nil { glog.Errorln("Error getting size from SQL database: %v", err) return 0 } qs.lru.Put(val+string(dir.Prefix()), size) return size }
func init() { glog.SetToStderr(true) cfg := config.ParseConfigFromFile("cayley_appengine.cfg") qs, _ := graph.NewQuadStore("memstore", "", nil) glog.Errorln(cfg) db.Load(qs, cfg, cfg.DatabasePath) http.SetupRoutes(qs, cfg) }
func init() { glog.SetToStderr(true) cfg := config.ParseConfigFromFile("cayley_appengine.cfg") ts := memstore.NewMemTripleStore() glog.Errorln(cfg) LoadTriplesFromFileInto(ts, cfg.DatabasePath, cfg.LoadSize) http.SetupRoutes(ts, cfg) }
func (ts *TripleStore) Triple(k graph.Value) *graph.Triple { var triple graph.Triple b, err := ts.db.Get(k.([]byte), ts.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Errorln("Error: couldn't get triple from DB") return &graph.Triple{} } if err == leveldb.ErrNotFound { // No harm, no foul. return &graph.Triple{} } err = json.Unmarshal(b, &triple) if err != nil { glog.Errorln("Error: couldn't reconstruct triple") return &graph.Triple{} } return &triple }
func (ts *TripleStore) valueData(value_key []byte) ValueData { var out ValueData if glog.V(3) { glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key) } b, err := ts.db.Get(value_key, ts.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Errorln("Error: couldn't get value from DB") return out } if b != nil && err != leveldb.ErrNotFound { err = json.Unmarshal(b, &out) if err != nil { glog.Errorln("Error: couldn't reconstruct value") return ValueData{} } } return out }
func (qs *TripleStore) getInt64ForKey(key string, empty int64) (int64, error) { var out int64 b, err := qs.db.Get([]byte(key), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Errorln("Couldn't read " + key + ": " + err.Error()) return 0, err } if err == leveldb.ErrNotFound { // Must be a new database. Cool return empty, nil } buf := bytes.NewBuffer(b) err = binary.Read(buf, binary.LittleEndian, &out) if err != nil { glog.Errorln("Error: couldn't parse", key) return 0, err } return out, nil }
func (qs *QuadStore) valueData(key []byte) ValueData { var out ValueData if glog.V(3) { glog.V(3).Infof("%c %v", key[0], key) } b, err := qs.db.Get(key, qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Errorln("Error: could not get value from DB") return out } if b != nil && err != leveldb.ErrNotFound { err = json.Unmarshal(b, &out) if err != nil { glog.Errorln("Error: could not reconstruct value") return ValueData{} } } return out }
func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator { if val.IsNull() || val.IsUndefined() { return ts.GetNodesAllIterator() } if val.IsPrimitive() { thing, _ := val.Export() switch v := thing.(type) { case string: it := ts.MakeFixed() it.AddValue(ts.GetIdFor(v)) return it default: glog.Errorln("Trying to build unknown primitive value.") } } switch val.Class() { case "Object": return buildIteratorTree(val.Object(), ts) case "Array": // Had better be an array of strings strings := makeListOfStringsFromArrayValue(val.Object()) it := ts.MakeFixed() for _, x := range strings { it.AddValue(ts.GetIdFor(x)) } return it case "Number": fallthrough case "Boolean": fallthrough case "Date": fallthrough case "String": it := ts.MakeFixed() str, _ := val.ToString() it.AddValue(ts.GetIdFor(str)) return it default: glog.Errorln("Trying to handle unsupported Javascript value.") return graph.NewNullIterator() } }
func (p *PrimaryKey) Int() int64 { switch p.keyType { case sequential: return p.sequentialID case unique: msg := "UUID cannot be converted to an int64" glog.Errorln(msg) panic(msg) } return -1 }
func buildIteratorFromValue(val otto.Value, qs graph.QuadStore) graph.Iterator { if val.IsNull() || val.IsUndefined() { return qs.NodesAllIterator() } if val.IsPrimitive() { thing, _ := val.Export() switch v := thing.(type) { case string: it := qs.FixedIterator() it.Add(qs.ValueOf(v)) return it default: glog.Errorln("Trying to build unknown primitive value.") } } switch val.Class() { case "Object": return buildIteratorTree(val.Object(), qs) case "Array": // Had better be an array of strings strings := stringsFrom(val.Object()) it := qs.FixedIterator() for _, x := range strings { it.Add(qs.ValueOf(x)) } return it case "Number": fallthrough case "Boolean": fallthrough case "Date": fallthrough case "String": it := qs.FixedIterator() it.Add(qs.ValueOf(val.String())) return it default: glog.Errorln("Trying to handle unsupported Javascript value.") return iterator.NewNull() } }
func (qs *QuadStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) { subs := it.SubIterators() if len(subs) != 1 { return it, false } primary := subs[0] switch primary.Type() { case graph.Fixed: size, _ := primary.Size() if size == 1 { if !graph.Next(primary) { panic("unexpected size during optimize") } val := primary.Result() newIt := qs.QuadIterator(it.Direction(), val) nt := newIt.Tagger() nt.CopyFrom(it) for _, tag := range primary.Tagger().Tags() { nt.AddFixed(tag, val) } it.Close() return newIt, true } case sqlType: p := primary.(*SQLIterator) newit, err := linksto(p.sql, it.Direction(), qs) if err != nil { glog.Errorln(err) return it, false } newit.Tagger().CopyFrom(it) return newit, true case graph.All: linkit := &SQLLinkIterator{ tableName: newTableName(), size: qs.Size(), } for _, t := range primary.Tagger().Tags() { linkit.tagdirs = append(linkit.tagdirs, tagDir{ dir: it.Direction(), tag: t, }) } for k, v := range primary.Tagger().Fixed() { linkit.tagger.AddFixed(k, v) } linkit.tagger.CopyFrom(it) newit := NewSQLIterator(qs, linkit) return newit, true } return it, false }
func upgradeBolt(path string, opts graph.Options) error { db, err := bolt.Open(path, 0600, nil) defer db.Close() if err != nil { glog.Errorln("Error, couldn't open! ", err) return err } var version int64 err = db.View(func(tx *bolt.Tx) error { version, err = getInt64ForMetaKey(tx, "version", nilDataVersion) return err }) if err != nil { glog.Errorln("error:", err) return err } if version == latestDataVersion { fmt.Printf("Already at latest version: %d\n", latestDataVersion) return nil } if version > latestDataVersion { err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version) glog.Errorln("error:", err) return err } for i := version; i < latestDataVersion; i++ { err := migrateFunctions[i](db) if err != nil { return err } setVersion(db, i+1) } return nil }
func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { if api.config.ReadOnly { return jsonResponse(w, 400, "Database is read-only.") } formFile, _, err := r.FormFile("NQuadFile") if err != nil { glog.Errorln(err) return jsonResponse(w, 500, "Couldn't read file: "+err.Error()) } defer formFile.Close() blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) if blockErr != nil { blockSize = int64(api.config.LoadSize) } quadReader, err := internal.Decompressor(formFile) // TODO(kortschak) Make this configurable from the web UI. dec := cquads.NewDecoder(quadReader) h, err := api.GetHandleForRequest(r) if err != nil { return jsonResponse(w, 400, err) } var ( n int block = make([]quad.Quad, 0, blockSize) ) for { t, err := dec.Unmarshal() if err != nil { if err == io.EOF { break } glog.Fatalln("what can do this here?", err) // FIXME(kortschak) } block = append(block, t) n++ if len(block) == cap(block) { h.QuadWriter.AddQuadSet(block) block = block[:0] } } h.QuadWriter.AddQuadSet(block) fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n) return 200 }
func buildPathFromValue(val otto.Value) (out []interface{}) { if val.IsNull() || val.IsUndefined() { return nil } if val.IsPrimitive() { thing, _ := val.Export() switch v := thing.(type) { case string: out = append(out, v) return default: glog.Errorln("Trying to build unknown primitive value.") } } switch val.Class() { case "Object": out = append(out, buildPathFromObject(val.Object())) return case "Array": // Had better be an array of strings for _, x := range stringsFrom(val.Object()) { out = append(out, x) } return case "Number": fallthrough case "Boolean": fallthrough case "Date": fallthrough case "String": out = append(out, val.String()) return default: glog.Errorln("Trying to handle unsupported Javascript value.") return nil } }
func createNewLevelDB(path string, _ graph.Options) error { opts := &opt.Options{} db, err := leveldb.OpenFile(path, opts) if err != nil { glog.Errorln("Error: couldn't create database: ", err) return err } defer db.Close() ts := &TripleStore{} ts.db = db ts.writeopts = &opt.WriteOptions{ Sync: true, } ts.Close() return nil }
func CreateNewLevelDB(path string) bool { opts := &opt.Options{} db, err := leveldb.OpenFile(path, opts) if err != nil { glog.Errorln("Error: couldn't create database", err) return false } defer db.Close() ts := &TripleStore{} ts.db = db ts.writeopts = &opt.WriteOptions{ Sync: true, } ts.Close() return true }
func Load(ts graph.TripleStore, cfg *config.Config, triplePath string) { tChan := make(chan *graph.Triple) go ReadTriplesFromFile(tChan, triplePath) bulker, canBulk := ts.(graph.BulkLoader) if canBulk { err := bulker.BulkLoad(tChan) if err == nil { return } if err != graph.ErrCannotBulkLoad { glog.Errorln("Error attempting to bulk load: ", err) } } LoadTriplesInto(tChan, ts, cfg.LoadSize) }
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) { var qs QuadStore var err error db, err := bolt.Open(path, 0600, nil) if err != nil { glog.Errorln("Error, couldn't open! ", err) return nil, err } qs.db = db // BoolKey returns false on non-existence. IE, Sync by default. qs.db.NoSync, _ = options.BoolKey("nosync") err = qs.getMetadata() if err != nil { return nil, err } return &qs, nil }
func NewAllIterator(ts *TripleStore, collection string) *Iterator { var m Iterator m.ts = ts m.dir = graph.Any m.constraint = nil m.collection = collection m.iter = ts.db.C(collection).Find(nil).Iter() size, err := ts.db.C(collection).Count() if err != nil { glog.Errorln("Trouble getting size for iterator! ", err) return nil } m.size = int64(size) m.hash = "" m.isAll = true return &m }
func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { if api.config.ReadOnly { return FormatJson400(w, "Database is read-only.") } formFile, _, err := r.FormFile("NQuadFile") if err != nil { glog.Errorln(err) return FormatJsonError(w, 500, "Couldn't read file: "+err.Error()) } defer formFile.Close() blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) if blockErr != nil { blockSize = int64(api.config.LoadSize) } dec := nquads.NewDecoder(formFile) var ( n int block = make([]*quad.Quad, 0, blockSize) ) for { t, err := dec.Unmarshal() if err != nil { if err == io.EOF { break } panic("what can do this here?") // FIXME(kortschak) } block = append(block, t) n++ if len(block) == cap(block) { api.ts.AddTripleSet(block) block = block[:0] } } api.ts.AddTripleSet(block) fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", n) return 200 }