func (it *AllIterator) makeCursor() {
var cursor *sql.Rows
var err error
if it.cursor != nil {
it.cursor.Close()
}
if it.table == "quads" {
cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
if err != nil {
glog.Errorln("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
} else {
glog.V(4).Infoln("sql: getting node query")
cursor, err = it.qs.db.Query(`SELECT node FROM
(
SELECT subject FROM quads
UNION
SELECT predicate FROM quads
UNION
SELECT object FROM quads
UNION
SELECT label FROM quads
) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
if err != nil {
glog.Errorln("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
glog.V(4).Infoln("sql: got node query")
}
it.cursor = cursor
}
func upgradeLMDB(path string, opts graph.Options) error {
env, err := createLMDB(path, opts)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return err
}
defer env.Close()
qs := &QuadStore{}
qs.env = env
err = qs.openDBIs()
if err != nil {
return err
}
var version int64
err = env.View(func(tx *lmdb.Txn) (err error) {
version, err = qs.getInt64ForMetaKey(tx, "version", nilDataVersion)
return err
})
if err != nil {
glog.Errorln("error:", err)
return err
}
if version == latestDataVersion {
fmt.Printf("Already at latest version: %d\n", latestDataVersion)
return nil
}
if version > latestDataVersion {
err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version)
glog.Errorln("error:", err)
return err
}
for i := version; i < latestDataVersion; i++ {
err := migrateFunctionsLMDB[i](qs)
if err != nil {
return err
}
err = setVersionLMDB(qs.env, qs.metaDBI, i+1)
if err != nil {
return err
}
}
return nil
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
db, err := bolt.Open(path, 0600, nil)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
// BoolKey returns false on non-existence. IE, Sync by default.
qs.db.NoSync, _, err = options.BoolKey("nosync")
if err != nil {
return nil, err
}
err = qs.getMetadata()
if err == errNoBucket {
return nil, errors.New("bolt: quadstore has not been initialised")
} else if err != nil {
return nil, err
}
if qs.version != latestDataVersion {
return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
}
return &qs, nil
}
func createNewLevelDB(path string, _ graph.Options) error {
opts := &opt.Options{}
db, err := leveldb.OpenFile(path, opts)
if err != nil {
glog.Errorf("Error: could not create database: %v", err)
return err
}
defer db.Close()
qs := &QuadStore{}
qs.db = db
qs.writeopts = &opt.WriteOptions{
Sync: true,
}
qs.readopts = &opt.ReadOptions{}
_, err = qs.db.Get([]byte(horizonKey), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("couldn't read from leveldb during init")
return err
}
if err != leveldb.ErrNotFound {
return graph.ErrDatabaseExists
}
// Write some metadata
qs.Close()
return nil
}
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var qs TripleStore
var err error
qs.path = path
cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val
}
qs.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
}
qs.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val
}
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
glog.Infoln(qs.GetStats())
err = qs.getMetadata()
if err != nil {
return nil, err
}
return &qs, nil
}
func NewIterator(ts *TripleStore, collection string, d graph.Direction, val graph.TSVal) *Iterator {
var m Iterator
iterator.BaseInit(&m.Base)
m.name = ts.GetNameFor(val)
m.collection = collection
switch d {
case graph.Subject:
m.constraint = bson.M{"Subject": m.name}
case graph.Predicate:
m.constraint = bson.M{"Predicate": m.name}
case graph.Object:
m.constraint = bson.M{"Object": m.name}
case graph.Provenance:
m.constraint = bson.M{"Provenance": m.name}
}
m.ts = ts
m.dir = d
m.iter = ts.db.C(collection).Find(m.constraint).Iter()
size, err := ts.db.C(collection).Find(m.constraint).Count()
if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
}
m.size = int64(size)
m.hash = val.(string)
m.isAll = false
return &m
}
func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator {
var m MongoIterator
graph.BaseIteratorInit(&m.BaseIterator)
m.name = ts.GetNameFor(val)
m.collection = collection
switch dir {
case "s":
m.constraint = bson.M{"Sub": m.name}
case "p":
m.constraint = bson.M{"Pred": m.name}
case "o":
m.constraint = bson.M{"Obj": m.name}
case "c":
m.constraint = bson.M{"Provenance": m.name}
}
m.ts = ts
m.dir = dir
m.iter = ts.db.C(collection).Find(m.constraint).Iter()
size, err := ts.db.C(collection).Find(m.constraint).Count()
if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
}
m.size = int64(size)
m.hash = val.(string)
m.isAll = false
return &m
}
// Next is a convenience function that conditionally calls the Next method
// of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next
// return a nil Value and false.
func Next(it Iterator) (Value, bool) {
if n, ok := it.(Nexter); ok {
return n.Next()
}
glog.Errorln("Nexting an un-nextable iterator")
return nil, false
}
func getViaData(obj *otto.Object) (predicates []interface{}, tags []string, ok bool) {
argList, _ := obj.Get("_gremlin_values")
if argList.Class() != "GoArray" {
glog.Errorln("How is arglist not an array? Return nothing.", argList.Class())
return nil, nil, false
}
argArray := argList.Object()
lengthVal, _ := argArray.Get("length")
length, _ := lengthVal.ToInteger()
if length == 0 {
predicates = []interface{}{}
} else {
zero, _ := argArray.Get("0")
predicates = buildPathFromValue(zero)
}
if length >= 2 {
one, _ := argArray.Get("1")
if one.IsString() {
tags = append(tags, one.String())
} else if one.Class() == "Array" {
tags = stringsFrom(one.Object())
}
}
ok = true
return
}
func NewIterator(qs *TripleStore, collection string, d quad.Direction, val graph.Value) *Iterator {
name := qs.NameOf(val)
constraint := bson.M{d.String(): name}
size, err := qs.db.C(collection).Find(constraint).Count()
if err != nil {
// FIXME(kortschak) This should be passed back rather than just logging.
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
}
return &Iterator{
uid: iterator.NextUID(),
name: name,
constraint: constraint,
collection: collection,
qs: qs,
dir: d,
iter: qs.db.C(collection).Find(constraint).Iter(),
size: int64(size),
hash: val.(string),
isAll: false,
}
}
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, val string) int64 {
var err error
if isAll {
return qs.Size()
}
if qs.noSizes {
if dir == quad.Predicate {
return (qs.Size() / 100) + 1
}
return (qs.Size() / 1000) + 1
}
if val, ok := qs.lru.Get(val + string(dir.Prefix())); ok {
return val
}
var size int64
glog.V(4).Infoln("sql: getting size for select %s, %s", dir.String(), val)
err = qs.db.QueryRow(
fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hashOf(val)).Scan(&size)
if err != nil {
glog.Errorln("Error getting size from SQL database: %v", err)
return 0
}
qs.lru.Put(val+string(dir.Prefix()), size)
return size
}
func init() {
glog.SetToStderr(true)
cfg := config.ParseConfigFromFile("cayley_appengine.cfg")
qs, _ := graph.NewQuadStore("memstore", "", nil)
glog.Errorln(cfg)
db.Load(qs, cfg, cfg.DatabasePath)
http.SetupRoutes(qs, cfg)
}
func init() {
glog.SetToStderr(true)
cfg := config.ParseConfigFromFile("cayley_appengine.cfg")
ts := memstore.NewMemTripleStore()
glog.Errorln(cfg)
LoadTriplesFromFileInto(ts, cfg.DatabasePath, cfg.LoadSize)
http.SetupRoutes(ts, cfg)
}
func (ts *TripleStore) Triple(k graph.Value) *graph.Triple {
var triple graph.Triple
b, err := ts.db.Get(k.([]byte), ts.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: couldn't get triple from DB")
return &graph.Triple{}
}
if err == leveldb.ErrNotFound {
// No harm, no foul.
return &graph.Triple{}
}
err = json.Unmarshal(b, &triple)
if err != nil {
glog.Errorln("Error: couldn't reconstruct triple")
return &graph.Triple{}
}
return &triple
}
func (ts *TripleStore) valueData(value_key []byte) ValueData {
var out ValueData
if glog.V(3) {
glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key)
}
b, err := ts.db.Get(value_key, ts.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: couldn't get value from DB")
return out
}
if b != nil && err != leveldb.ErrNotFound {
err = json.Unmarshal(b, &out)
if err != nil {
glog.Errorln("Error: couldn't reconstruct value")
return ValueData{}
}
}
return out
}
func (qs *TripleStore) getInt64ForKey(key string, empty int64) (int64, error) {
var out int64
b, err := qs.db.Get([]byte(key), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Couldn't read " + key + ": " + err.Error())
return 0, err
}
if err == leveldb.ErrNotFound {
// Must be a new database. Cool
return empty, nil
}
buf := bytes.NewBuffer(b)
err = binary.Read(buf, binary.LittleEndian, &out)
if err != nil {
glog.Errorln("Error: couldn't parse", key)
return 0, err
}
return out, nil
}
func (qs *QuadStore) valueData(key []byte) ValueData {
var out ValueData
if glog.V(3) {
glog.V(3).Infof("%c %v", key[0], key)
}
b, err := qs.db.Get(key, qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: could not get value from DB")
return out
}
if b != nil && err != leveldb.ErrNotFound {
err = json.Unmarshal(b, &out)
if err != nil {
glog.Errorln("Error: could not reconstruct value")
return ValueData{}
}
}
return out
}
func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator {
if val.IsNull() || val.IsUndefined() {
return ts.GetNodesAllIterator()
}
if val.IsPrimitive() {
thing, _ := val.Export()
switch v := thing.(type) {
case string:
it := ts.MakeFixed()
it.AddValue(ts.GetIdFor(v))
return it
default:
glog.Errorln("Trying to build unknown primitive value.")
}
}
switch val.Class() {
case "Object":
return buildIteratorTree(val.Object(), ts)
case "Array":
// Had better be an array of strings
strings := makeListOfStringsFromArrayValue(val.Object())
it := ts.MakeFixed()
for _, x := range strings {
it.AddValue(ts.GetIdFor(x))
}
return it
case "Number":
fallthrough
case "Boolean":
fallthrough
case "Date":
fallthrough
case "String":
it := ts.MakeFixed()
str, _ := val.ToString()
it.AddValue(ts.GetIdFor(str))
return it
default:
glog.Errorln("Trying to handle unsupported Javascript value.")
return graph.NewNullIterator()
}
}
func (p *PrimaryKey) Int() int64 {
switch p.keyType {
case sequential:
return p.sequentialID
case unique:
msg := "UUID cannot be converted to an int64"
glog.Errorln(msg)
panic(msg)
}
return -1
}
func buildIteratorFromValue(val otto.Value, qs graph.QuadStore) graph.Iterator {
if val.IsNull() || val.IsUndefined() {
return qs.NodesAllIterator()
}
if val.IsPrimitive() {
thing, _ := val.Export()
switch v := thing.(type) {
case string:
it := qs.FixedIterator()
it.Add(qs.ValueOf(v))
return it
default:
glog.Errorln("Trying to build unknown primitive value.")
}
}
switch val.Class() {
case "Object":
return buildIteratorTree(val.Object(), qs)
case "Array":
// Had better be an array of strings
strings := stringsFrom(val.Object())
it := qs.FixedIterator()
for _, x := range strings {
it.Add(qs.ValueOf(x))
}
return it
case "Number":
fallthrough
case "Boolean":
fallthrough
case "Date":
fallthrough
case "String":
it := qs.FixedIterator()
it.Add(qs.ValueOf(val.String()))
return it
default:
glog.Errorln("Trying to handle unsupported Javascript value.")
return iterator.NewNull()
}
}
func (qs *QuadStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
subs := it.SubIterators()
if len(subs) != 1 {
return it, false
}
primary := subs[0]
switch primary.Type() {
case graph.Fixed:
size, _ := primary.Size()
if size == 1 {
if !graph.Next(primary) {
panic("unexpected size during optimize")
}
val := primary.Result()
newIt := qs.QuadIterator(it.Direction(), val)
nt := newIt.Tagger()
nt.CopyFrom(it)
for _, tag := range primary.Tagger().Tags() {
nt.AddFixed(tag, val)
}
it.Close()
return newIt, true
}
case sqlType:
p := primary.(*SQLIterator)
newit, err := linksto(p.sql, it.Direction(), qs)
if err != nil {
glog.Errorln(err)
return it, false
}
newit.Tagger().CopyFrom(it)
return newit, true
case graph.All:
linkit := &SQLLinkIterator{
tableName: newTableName(),
size: qs.Size(),
}
for _, t := range primary.Tagger().Tags() {
linkit.tagdirs = append(linkit.tagdirs, tagDir{
dir: it.Direction(),
tag: t,
})
}
for k, v := range primary.Tagger().Fixed() {
linkit.tagger.AddFixed(k, v)
}
linkit.tagger.CopyFrom(it)
newit := NewSQLIterator(qs, linkit)
return newit, true
}
return it, false
}
func upgradeBolt(path string, opts graph.Options) error {
db, err := bolt.Open(path, 0600, nil)
defer db.Close()
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return err
}
var version int64
err = db.View(func(tx *bolt.Tx) error {
version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
return err
})
if err != nil {
glog.Errorln("error:", err)
return err
}
if version == latestDataVersion {
fmt.Printf("Already at latest version: %d\n", latestDataVersion)
return nil
}
if version > latestDataVersion {
err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version)
glog.Errorln("error:", err)
return err
}
for i := version; i < latestDataVersion; i++ {
err := migrateFunctions[i](db)
if err != nil {
return err
}
setVersion(db, i+1)
}
return nil
}
func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
if api.config.ReadOnly {
return jsonResponse(w, 400, "Database is read-only.")
}
formFile, _, err := r.FormFile("NQuadFile")
if err != nil {
glog.Errorln(err)
return jsonResponse(w, 500, "Couldn't read file: "+err.Error())
}
defer formFile.Close()
blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
if blockErr != nil {
blockSize = int64(api.config.LoadSize)
}
quadReader, err := internal.Decompressor(formFile)
// TODO(kortschak) Make this configurable from the web UI.
dec := cquads.NewDecoder(quadReader)
h, err := api.GetHandleForRequest(r)
if err != nil {
return jsonResponse(w, 400, err)
}
var (
n int
block = make([]quad.Quad, 0, blockSize)
)
for {
t, err := dec.Unmarshal()
if err != nil {
if err == io.EOF {
break
}
glog.Fatalln("what can do this here?", err) // FIXME(kortschak)
}
block = append(block, t)
n++
if len(block) == cap(block) {
h.QuadWriter.AddQuadSet(block)
block = block[:0]
}
}
h.QuadWriter.AddQuadSet(block)
fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)
return 200
}
func buildPathFromValue(val otto.Value) (out []interface{}) {
if val.IsNull() || val.IsUndefined() {
return nil
}
if val.IsPrimitive() {
thing, _ := val.Export()
switch v := thing.(type) {
case string:
out = append(out, v)
return
default:
glog.Errorln("Trying to build unknown primitive value.")
}
}
switch val.Class() {
case "Object":
out = append(out, buildPathFromObject(val.Object()))
return
case "Array":
// Had better be an array of strings
for _, x := range stringsFrom(val.Object()) {
out = append(out, x)
}
return
case "Number":
fallthrough
case "Boolean":
fallthrough
case "Date":
fallthrough
case "String":
out = append(out, val.String())
return
default:
glog.Errorln("Trying to handle unsupported Javascript value.")
return nil
}
}
func createNewLevelDB(path string, _ graph.Options) error {
opts := &opt.Options{}
db, err := leveldb.OpenFile(path, opts)
if err != nil {
glog.Errorln("Error: couldn't create database: ", err)
return err
}
defer db.Close()
ts := &TripleStore{}
ts.db = db
ts.writeopts = &opt.WriteOptions{
Sync: true,
}
ts.Close()
return nil
}
func CreateNewLevelDB(path string) bool {
opts := &opt.Options{}
db, err := leveldb.OpenFile(path, opts)
if err != nil {
glog.Errorln("Error: couldn't create database", err)
return false
}
defer db.Close()
ts := &TripleStore{}
ts.db = db
ts.writeopts = &opt.WriteOptions{
Sync: true,
}
ts.Close()
return true
}
func Load(ts graph.TripleStore, cfg *config.Config, triplePath string) {
tChan := make(chan *graph.Triple)
go ReadTriplesFromFile(tChan, triplePath)
bulker, canBulk := ts.(graph.BulkLoader)
if canBulk {
err := bulker.BulkLoad(tChan)
if err == nil {
return
}
if err != graph.ErrCannotBulkLoad {
glog.Errorln("Error attempting to bulk load: ", err)
}
}
LoadTriplesInto(tChan, ts, cfg.LoadSize)
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
db, err := bolt.Open(path, 0600, nil)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
qs.db = db
// BoolKey returns false on non-existence. IE, Sync by default.
qs.db.NoSync, _ = options.BoolKey("nosync")
err = qs.getMetadata()
if err != nil {
return nil, err
}
return &qs, nil
}
func NewAllIterator(ts *TripleStore, collection string) *Iterator {
var m Iterator
m.ts = ts
m.dir = graph.Any
m.constraint = nil
m.collection = collection
m.iter = ts.db.C(collection).Find(nil).Iter()
size, err := ts.db.C(collection).Count()
if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
}
m.size = int64(size)
m.hash = ""
m.isAll = true
return &m
}
func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
if api.config.ReadOnly {
return FormatJson400(w, "Database is read-only.")
}
formFile, _, err := r.FormFile("NQuadFile")
if err != nil {
glog.Errorln(err)
return FormatJsonError(w, 500, "Couldn't read file: "+err.Error())
}
defer formFile.Close()
blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
if blockErr != nil {
blockSize = int64(api.config.LoadSize)
}
dec := nquads.NewDecoder(formFile)
var (
n int
block = make([]*quad.Quad, 0, blockSize)
)
for {
t, err := dec.Unmarshal()
if err != nil {
if err == io.EOF {
break
}
panic("what can do this here?") // FIXME(kortschak)
}
block = append(block, t)
n++
if len(block) == cap(block) {
api.ts.AddTripleSet(block)
block = block[:0]
}
}
api.ts.AddTripleSet(block)
fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", n)
return 200
}