func (it *AllIterator) makeCursor() {
var cursor *sql.Rows
var err error
if it.cursor != nil {
it.cursor.Close()
}
if it.table == "quads" {
cursor, err = it.qs.db.Query(`SELECT
subject_hash,
predicate_hash,
object_hash,
label_hash
FROM quads;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
} else {
if clog.V(4) {
clog.Infof("sql: getting node query")
}
cursor, err = it.qs.db.Query(`SELECT hash FROM nodes;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
if clog.V(4) {
clog.Infof("sql: got node query")
}
}
it.cursor = cursor
}
// Get the next result that matches this branch.
func (it *HasA) NextPath() bool {
// Order here is important. If the subiterator has a NextPath, then we
// need do nothing -- there is a next result, and we shouldn't move forward.
// However, we then need to get the next result from our last Contains().
//
// The upshot is, the end of NextPath() bubbles up from the bottom of the
// iterator tree up, and we need to respect that.
if clog.V(4) {
clog.Infof("HASA %v NextPath", it.UID())
}
if it.primaryIt.NextPath() {
return true
}
it.err = it.primaryIt.Err()
if it.err != nil {
return false
}
result := it.NextContains() // Sets it.err if there's an error
if it.err != nil {
return false
}
if clog.V(4) {
clog.Infof("HASA %v NextPath Returns %v", it.UID(), result)
}
return result
}
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
block := make([]quad.Quad, 0, cfg.LoadSize)
count := 0
for {
t, err := dec.Unmarshal()
if err != nil {
if err == io.EOF {
break
}
return err
}
block = append(block, t)
if len(block) == cap(block) {
count += len(block)
err := qw.AddQuadSet(block)
if err != nil {
return fmt.Errorf("db: failed to load data: %v", err)
}
block = block[:0]
if clog.V(2) {
clog.Infof("Wrote %d quads.", count)
}
}
}
count += len(block)
err := qw.AddQuadSet(block)
if err != nil {
return fmt.Errorf("db: failed to load data: %v", err)
}
if clog.V(2) {
clog.Infof("Wrote %d quads.", count)
}
return nil
}
// Size is the number of values stored, if we've got them all.
// Otherwise, guess based on the size of the subiterator.
func (it *Materialize) Size() (int64, bool) {
if it.hasRun && !it.aborted {
if clog.V(2) {
clog.Infof("returning size %v", it.actualSize)
}
return it.actualSize, true
}
if clog.V(2) {
clog.Infof("bailing size %v", it.actualSize)
}
return it.subIt.Size()
}
func (s *Session) Execute(input string, c chan interface{}, _ int) {
defer close(c)
var mqlQuery interface{}
err := json.Unmarshal([]byte(input), &mqlQuery)
if err != nil {
return
}
s.currentQuery = NewQuery(s)
s.currentQuery.BuildIteratorTree(mqlQuery)
if s.currentQuery.isError() {
return
}
it, _ := s.currentQuery.it.Optimize()
if clog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
for it.NextPath() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
}
}
}
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, hash NodeHash) int64 {
var err error
if isAll {
return qs.Size()
}
if qs.noSizes {
if dir == quad.Predicate {
return (qs.Size() / 100) + 1
}
return (qs.Size() / 1000) + 1
}
if val, ok := qs.sizes.Get(hash.String() + string(dir.Prefix())); ok {
return val.(int64)
}
var size int64
if clog.V(4) {
clog.Infof("sql: getting size for select %s, %v", dir.String(), hash)
}
err = qs.db.QueryRow(
fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hash.toSQL()).Scan(&size)
if err != nil {
clog.Errorf("Error getting size from SQL database: %v", err)
return 0
}
qs.sizes.Put(hash.String()+string(dir.Prefix()), size)
return size
}
func (wk *worker) runIterator(it graph.Iterator) {
if wk.wantShape() {
iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
return
}
it, _ = it.Optimize()
if clog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
}
}
if clog.V(2) {
bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", " ")
clog.Infof(string(bytes))
}
it.Close()
}
func (qs *QuadStore) NameOf(v graph.Value) string {
if v == nil {
if clog.V(2) {
clog.Infof("NameOf was nil")
}
return ""
}
return v.(string)
}
func (l *SQLLinkIterator) buildSQL(next bool, val graph.Value) (string, sqlArgs) {
query := "SELECT "
t := []string{
fmt.Sprintf("%s.subject_hash AS subject", l.tableName),
fmt.Sprintf("%s.predicate_hash AS predicate", l.tableName),
fmt.Sprintf("%s.object_hash AS object", l.tableName),
fmt.Sprintf("%s.label_hash AS label", l.tableName),
}
for _, v := range l.getTags() {
t = append(t, v.String())
}
query += strings.Join(t, ", ")
query += " FROM "
t = []string{}
var values sqlArgs
for _, k := range l.getTables() {
values = append(values, k.values...)
t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
}
query += strings.Join(t, ", ")
constraint, wherevalues := l.buildWhere()
if constraint != "" {
query += " WHERE "
}
values = append(values, wherevalues...)
if !next {
h := val.(QuadHashes)
if constraint != "" {
constraint += " AND "
} else {
constraint += " WHERE "
}
t = []string{
fmt.Sprintf("%s.subject_hash = ?", l.tableName),
fmt.Sprintf("%s.predicate_hash = ?", l.tableName),
fmt.Sprintf("%s.object_hash = ?", l.tableName),
fmt.Sprintf("%s.label_hash = ?", l.tableName),
}
constraint += strings.Join(t, " AND ")
values = append(values, h[0].toSQL())
values = append(values, h[1].toSQL())
values = append(values, h[2].toSQL())
values = append(values, h[3].toSQL())
}
query += constraint
query += ";"
if clog.V(4) {
dstr := query
for i := 1; i <= len(values); i++ {
dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
}
clog.Infof("%v", dstr)
}
return query, values
}
func (qs *QuadStore) NameOf(k graph.Value) string {
if k == nil {
if clog.V(2) {
clog.Infof("k was nil")
}
return ""
}
return qs.valueData(k.(*Token)).Name
}
func (qs *QuadStore) DebugPrint() {
for i, l := range qs.log {
if i == 0 {
continue
}
if clog.V(2) {
clog.Infof("%d: %#v", i, l)
}
}
}
func NextLogOut(it Iterator, val Value, ok bool) bool {
if clog.V(4) {
if ok {
clog.Infof("%s %d NEXT IS %v", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
}
}
return ok
}
func ContainsLogOut(it Iterator, val Value, good bool) bool {
if clog.V(4) {
if good {
clog.Infof("%s %d CHECK CONTAINS %v GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
clog.Infof("%s %d CHECK CONTAINS %v BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
}
}
return good
}
func (qs *QuadStore) NameOf(k graph.Value) quad.Value {
if k == nil {
if clog.V(2) {
clog.Infof("k was nil")
}
return nil
}
v := qs.valueData(k.(*Token))
return v.GetNativeValue()
}
func (c *IterateChain) end() {
c.it.Close()
if !clog.V(2) {
return
}
if b, err := json.MarshalIndent(DumpStats(c.it), "", " "); err != nil {
clog.Infof("failed to format stats: %v", err)
} else {
clog.Infof("%s", b)
}
}
func NextLogOut(it Iterator, ok bool) bool {
if clog.V(4) {
if ok {
val := it.Result()
clog.Infof("%s %d NEXT IS %v (%T)", strings.ToUpper(it.Type().String()), it.UID(), val, val)
} else {
clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
}
}
return ok
}
func debugChain(obj *otto.Object) bool {
val, _ := obj.Get("_gremlin_type")
if clog.V(2) {
clog.Infof("%v", val)
}
val, _ = obj.Get("_gremlin_prev")
if val.IsObject() {
return debugChain(val.Object())
}
return false
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
qs.path = path
cacheSize := DefaultCacheSize
val, ok, err := options.IntKey("cache_size_mb")
if err != nil {
return nil, err
} else if ok {
cacheSize = val
}
qs.dbOpts = &opt.Options{
BlockCacheCapacity: cacheSize * opt.MiB,
}
qs.dbOpts.ErrorIfMissing = true
writeBufferSize := DefaultWriteBufferSize
val, ok, err = options.IntKey("writeBufferSize")
if err != nil {
return nil, err
} else if ok {
writeBufferSize = val
}
qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
clog.Errorf("Error, could not open! %v", err)
return nil, err
}
qs.db = db
if clog.V(1) {
clog.Infof("%v", qs.GetStats())
}
vers, err := getVersion(qs.db)
if err != nil {
clog.Errorf("Error, could not read version info! %v", err)
db.Close()
return nil, err
} else if vers != latestDataVersion {
db.Close()
return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion)
}
err = qs.getMetadata()
if err != nil {
db.Close()
return nil, err
}
return &qs, nil
}
func (it *SQLIterator) Contains(v graph.Value) bool {
var err error
if ok, res := it.sql.quickContains(v); ok {
return res
}
err = it.makeCursor(false, v)
if err != nil {
clog.Errorf("Couldn't make query: %v", err)
it.err = err
if it.cursor != nil {
it.cursor.Close()
}
return false
}
it.cols, err = it.cursor.Columns()
if err != nil {
clog.Errorf("Couldn't get columns")
it.err = err
it.cursor.Close()
return false
}
it.resultList = nil
for {
if !it.cursor.Next() {
if clog.V(4) {
clog.Infof("sql: No next")
}
err := it.cursor.Err()
if err != nil {
clog.Errorf("Cursor error in SQL: %v", err)
it.err = err
}
it.cursor.Close()
break
}
s, err := scan(it.cursor, len(it.cols))
if err != nil {
it.err = err
it.cursor.Close()
return false
}
it.resultList = append(it.resultList, s)
}
it.cursor.Close()
it.cursor = nil
if len(it.resultList) != 0 {
it.resultIndex = 0
it.buildResult(0)
return true
}
return false
}
func (it *AllIterator) makeCursor() {
var cursor *sql.Rows
var err error
if it.cursor != nil {
it.cursor.Close()
}
if it.table == "quads" {
cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
} else {
if clog.V(4) {
clog.Infof("sql: getting node query")
}
cursor, err = it.qs.db.Query(`SELECT node FROM
(
SELECT subject FROM quads
UNION
SELECT predicate FROM quads
UNION
SELECT object FROM quads
UNION
SELECT label FROM quads
) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
if clog.V(4) {
clog.Infof("sql: got node query")
}
}
it.cursor = cursor
}
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a quad iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
for it.resultIt.Next() {
it.runstats.ContainsNext += 1
link := it.resultIt.Result()
if clog.V(4) {
clog.Infof("Quad is %v", it.qs.Quad(link))
}
if it.primaryIt.Contains(link) {
it.result = it.qs.QuadDirection(link, it.dir)
return true
}
}
it.err = it.resultIt.Err()
return false
}
func (c *IterateChain) start() {
if c.optimize {
c.it, _ = c.it.Optimize()
if c.qs != nil {
c.it, _ = c.qs.OptimizeIterator(c.it)
}
}
if !clog.V(2) {
return
}
if b, err := json.MarshalIndent(c.it.Describe(), "", " "); err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
n := 0
it, _ = it.Optimize()
if clog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
}
}
it.Close()
}
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "quads that have `val` in our direction", given to us by the quad store,
// and then Next() values out of that iterator and Contains() them against our subiterator.
func (it *HasA) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
it.runstats.Contains += 1
if clog.V(4) {
clog.Infof("Id is %v", it.qs.NameOf(val))
}
// TODO(barakmich): Optimize this
if it.resultIt != nil {
it.resultIt.Close()
}
it.resultIt = it.qs.QuadIterator(it.dir, val)
ok := it.NextContains()
if it.err != nil {
return false
}
return graph.ContainsLogOut(it, val, ok)
}
func (n *SQLNodeIterator) buildSQL(next bool, val graph.Value) (string, []string) {
topData := n.tableID()
tags := []tagDir{topData}
tags = append(tags, n.getTags()...)
query := "SELECT "
var t []string
for _, v := range tags {
t = append(t, v.String())
}
query += strings.Join(t, ", ")
query += " FROM "
t = []string{}
var values []string
for _, k := range n.getTables() {
values = append(values, k.values...)
t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
}
query += strings.Join(t, ", ")
query += " WHERE "
constraint, wherevalues := n.buildWhere()
values = append(values, wherevalues...)
if !next {
v := val.(string)
if constraint != "" {
constraint += " AND "
}
constraint += fmt.Sprintf("%s.%s_hash = ?", topData.table, topData.dir)
values = append(values, hashOf(v))
}
query += constraint
query += ";"
if clog.V(4) {
dstr := query
for i := 1; i <= len(values); i++ {
dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
}
clog.Infof("%v", dstr)
}
return query, values
}
func (it *Materialize) materializeSet() {
i := 0
for graph.Next(it.subIt) {
i++
if i > abortMaterializeAt {
it.aborted = true
break
}
id := it.subIt.Result()
val := id
if h, ok := id.(Keyer); ok {
val = h.Key()
}
if _, ok := it.containsMap[val]; !ok {
it.containsMap[val] = len(it.values)
it.values = append(it.values, nil)
}
index := it.containsMap[val]
tags := make(map[string]graph.Value)
it.subIt.TagResults(tags)
it.values[index] = append(it.values[index], result{id: id, tags: tags})
it.actualSize += 1
for it.subIt.NextPath() {
i++
if i > abortMaterializeAt {
it.aborted = true
break
}
tags := make(map[string]graph.Value)
it.subIt.TagResults(tags)
it.values[index] = append(it.values[index], result{id: id, tags: tags})
it.actualSize += 1
}
}
it.err = it.subIt.Err()
if it.err == nil && it.aborted {
if clog.V(2) {
clog.Infof("Aborting subiterator")
}
it.values = nil
it.containsMap = nil
it.subIt.Reset()
}
it.hasRun = true
}
func (qs *QuadStore) valueData(t *Token) proto.NodeData {
var out proto.NodeData
if clog.V(3) {
clog.Infof("%s %v", string(t.bucket), t.key)
}
err := qs.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(t.bucket)
data := b.Get(t.key)
if data != nil {
return out.Unmarshal(data)
}
return nil
})
if err != nil {
clog.Errorf("Error: couldn't get value")
return proto.NodeData{}
}
return out
}
func (qs *QuadStore) valueData(key []byte) ValueData {
var out ValueData
if clog.V(3) {
clog.Infof("%c %v", key[0], key)
}
b, err := qs.db.Get(key, qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
clog.Errorf("Error: could not get value from DB")
return out
}
if b != nil && err != leveldb.ErrNotFound {
err = json.Unmarshal(b, &out)
if err != nil {
clog.Errorf("Error: could not reconstruct value")
return ValueData{}
}
}
return out
}
func (it *AllIterator) Next() bool {
graph.NextLogIn(it)
if it.cursor == nil {
it.makeCursor()
if it.cursor == nil {
return false
}
}
if !it.cursor.Next() {
if clog.V(4) {
clog.Infof("sql: No next")
}
err := it.cursor.Err()
if err != nil {
clog.Errorf("Cursor error in SQL: %v", err)
it.err = err
}
it.cursor.Close()
return false
}
if it.table == "nodes" {
var hash NodeHash
err := it.cursor.Scan(&hash)
if err != nil {
clog.Errorf("Error nexting node iterator: %v", err)
it.err = err
return false
}
it.result = NodeHash(hash)
return true
}
var q QuadHashes
err := it.cursor.Scan(&q[0], &q[1], &q[2], &q[3])
if err != nil {
clog.Errorf("Error scanning sql iterator: %v", err)
it.err = err
return false
}
it.result = q
return graph.NextLogOut(it, true)
}
func SetupRoutes(handle *graph.Handle, cfg *config.Config) {
r := httprouter.New()
assets := findAssetsPath()
if clog.V(2) {
clog.Infof("Found assets at %v", assets)
}
var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl")))
templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html"))
root := &TemplateRequestHandler{templates: templates}
docs := &DocRequestHandler{assets: assets}
api := &API{config: cfg, handle: handle}
api.APIv1(r)
//m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true}))
//r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/"))))
r.GET("/docs/:docpage", docs.ServeHTTP)
r.GET("/ui/:ui_type", root.ServeHTTP)
r.GET("/", root.ServeHTTP)
http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/")))))
http.Handle("/", r)
}
