// Get the next result that matches this branch.
func (it *HasA) NextPath() bool {
// Order here is important. If the subiterator has a NextPath, then we
// need do nothing -- there is a next result, and we shouldn't move forward.
// However, we then need to get the next result from our last Contains().
//
// The upshot is, the end of NextPath() bubbles up from the bottom of the
// iterator tree up, and we need to respect that.
if clog.V(4) {
clog.Infof("HASA %v NextPath", it.UID())
}
if it.primaryIt.NextPath() {
return true
}
it.err = it.primaryIt.Err()
if it.err != nil {
return false
}
result := it.NextContains() // Sets it.err if there's an error
if it.err != nil {
return false
}
if clog.V(4) {
clog.Infof("HASA %v NextPath Returns %v", it.UID(), result)
}
return result
}
func (s *Session) Execute(input string, c chan interface{}, _ int) {
defer close(c)
var mqlQuery interface{}
err := json.Unmarshal([]byte(input), &mqlQuery)
if err != nil {
return
}
s.currentQuery = NewQuery(s)
s.currentQuery.BuildIteratorTree(mqlQuery)
if s.currentQuery.isError() {
return
}
it, _ := s.currentQuery.it.Optimize()
if clog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
for it.NextPath() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
c <- tags
}
}
}
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
block := make([]quad.Quad, 0, cfg.LoadSize)
count := 0
for {
t, err := dec.Unmarshal()
if err != nil {
if err == io.EOF {
break
}
return err
}
block = append(block, t)
if len(block) == cap(block) {
count += len(block)
err := qw.AddQuadSet(block)
if err != nil {
return fmt.Errorf("db: failed to load data: %v", err)
}
block = block[:0]
if clog.V(2) {
clog.Infof("Wrote %d quads.", count)
}
}
}
count += len(block)
err := qw.AddQuadSet(block)
if err != nil {
return fmt.Errorf("db: failed to load data: %v", err)
}
if clog.V(2) {
clog.Infof("Wrote %d quads.", count)
}
return nil
}
func (it *AllIterator) makeCursor() {
var cursor *sql.Rows
var err error
if it.cursor != nil {
it.cursor.Close()
}
if it.table == "quads" {
cursor, err = it.qs.db.Query(`SELECT
subject_hash,
predicate_hash,
object_hash,
label_hash
FROM quads;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
} else {
if clog.V(4) {
clog.Infof("sql: getting node query")
}
cursor, err = it.qs.db.Query(`SELECT hash FROM nodes;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
if clog.V(4) {
clog.Infof("sql: got node query")
}
}
it.cursor = cursor
}
func NextLogOut(it Iterator, val Value, ok bool) bool {
if clog.V(4) {
if ok {
clog.Infof("%s %d NEXT IS %v", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
}
}
return ok
}
func ContainsLogOut(it Iterator, val Value, good bool) bool {
if clog.V(4) {
if good {
clog.Infof("%s %d CHECK CONTAINS %v GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
clog.Infof("%s %d CHECK CONTAINS %v BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
}
}
return good
}
func NextLogOut(it Iterator, ok bool) bool {
if clog.V(4) {
if ok {
val := it.Result()
clog.Infof("%s %d NEXT IS %v (%T)", strings.ToUpper(it.Type().String()), it.UID(), val, val)
} else {
clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
}
}
return ok
}
func (c *IterateChain) end() {
c.it.Close()
if !clog.V(2) {
return
}
if b, err := json.MarshalIndent(DumpStats(c.it), "", " "); err != nil {
clog.Infof("failed to format stats: %v", err)
} else {
clog.Infof("%s", b)
}
}
// Size is the number of values stored, if we've got them all.
// Otherwise, guess based on the size of the subiterator.
func (it *Materialize) Size() (int64, bool) {
if it.hasRun && !it.aborted {
if clog.V(2) {
clog.Infof("returning size %v", it.actualSize)
}
return it.actualSize, true
}
if clog.V(2) {
clog.Infof("bailing size %v", it.actualSize)
}
return it.subIt.Size()
}
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, hash NodeHash) int64 {
var err error
if isAll {
return qs.Size()
}
if qs.noSizes {
if dir == quad.Predicate {
return (qs.Size() / 100) + 1
}
return (qs.Size() / 1000) + 1
}
if val, ok := qs.sizes.Get(hash.String() + string(dir.Prefix())); ok {
return val.(int64)
}
var size int64
if clog.V(4) {
clog.Infof("sql: getting size for select %s, %v", dir.String(), hash)
}
err = qs.db.QueryRow(
fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hash.toSQL()).Scan(&size)
if err != nil {
clog.Errorf("Error getting size from SQL database: %v", err)
return 0
}
qs.sizes.Put(hash.String()+string(dir.Prefix()), size)
return size
}
func (c *IterateChain) start() {
if c.optimize {
c.it, _ = c.it.Optimize()
if c.qs != nil {
c.it, _ = c.qs.OptimizeIterator(c.it)
}
}
if !clog.V(2) {
return
}
if b, err := json.MarshalIndent(c.it.Describe(), "", " "); err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
func (qs *QuadStore) NameOf(k graph.Value) string {
if k == nil {
clog.Infof("k was nil")
return ""
}
return qs.valueData(k.(Token)).Name
}
func LogRequest(handler ResponseHandler) httprouter.Handle {
return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
start := time.Now()
addr := req.Header.Get("X-Real-IP")
if addr == "" {
addr = req.Header.Get("X-Forwarded-For")
if addr == "" {
addr = req.RemoteAddr
}
}
clog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr)
code := handler(w, req, params)
clog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start))
}
}
func configFrom(file string) *config.Config {
// Find the file...
if file != "" {
if _, err := os.Stat(file); os.IsNotExist(err) {
clog.Fatalf("Cannot find specified configuration file '%s', aborting.", file)
}
} else if _, err := os.Stat(os.Getenv("CAYLEY_CFG")); err == nil {
file = os.Getenv("CAYLEY_CFG")
} else if _, err := os.Stat("/etc/cayley.cfg"); err == nil {
file = "/etc/cayley.cfg"
}
if file == "" {
clog.Infof("Couldn't find a config file in either $CAYLEY_CFG or /etc/cayley.cfg. Going by flag defaults only.")
}
cfg, err := config.Load(file)
if err != nil {
clog.Fatalf("%v", err)
}
if cfg.DatabasePath == "" {
cfg.DatabasePath = *databasePath
}
if cfg.DatabaseType == "" {
cfg.DatabaseType = *databaseBackend
}
return cfg
}
func (wk *worker) runIterator(it graph.Iterator) {
if wk.wantShape() {
iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
return
}
it, _ = it.Optimize()
if clog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if !wk.send(&Result{actualResults: tags}) {
break
}
}
}
if clog.V(2) {
bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", " ")
clog.Infof(string(bytes))
}
it.Close()
}
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
n := 0
it, _ = it.Optimize()
if clog.V(2) {
b, err := json.MarshalIndent(it.Describe(), "", " ")
if err != nil {
clog.Infof("failed to format description: %v", err)
} else {
clog.Infof("%s", b)
}
}
for {
select {
case <-wk.kill:
return
default:
}
if !graph.Next(it) {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
for it.NextPath() {
select {
case <-wk.kill:
return
default:
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
val, _ = callback.Call(this.This, val)
n++
if limit >= 0 && n >= limit {
break
}
}
}
it.Close()
}
func TestIteratorsAndNextResultOrderA(t *testing.T) {
clog.Infof("\n-----------\n")
inst, opts, err := createInstance()
defer inst.Close()
if err != nil {
t.Fatalf("failed to create instance: %v", err)
}
qs, _, _ := makeTestStore(simpleGraph, opts)
if qs.Size() != 11 {
t.Fatal("Incorrect number of quads")
}
fixed := qs.FixedIterator()
fixed.Add(qs.ValueOf("C"))
fixed2 := qs.FixedIterator()
fixed2.Add(qs.ValueOf("follows"))
all := qs.NodesAllIterator()
innerAnd := iterator.NewAnd(qs)
innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed2, quad.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Object))
hasa := iterator.NewHasA(qs, innerAnd, quad.Subject)
outerAnd := iterator.NewAnd(qs)
outerAnd.AddSubIterator(fixed)
outerAnd.AddSubIterator(hasa)
if !outerAnd.Next() {
t.Error("Expected one matching subtree")
}
val := outerAnd.Result()
if qs.NameOf(val) != "C" {
t.Errorf("Matching subtree should be %s, got %s", "barak", qs.NameOf(val))
}
var (
got []string
expect = []string{"B", "D"}
)
for {
got = append(got, qs.NameOf(all.Result()))
if !outerAnd.NextPath() {
break
}
}
sort.Strings(got)
if !reflect.DeepEqual(got, expect) {
t.Errorf("Unexpected result, got:%q expect:%q", got, expect)
}
if outerAnd.Next() {
t.Error("More than one possible top level output?")
}
}
func (qs *QuadStore) NameOf(v graph.Value) string {
if v == nil {
if clog.V(2) {
clog.Infof("NameOf was nil")
}
return ""
}
return v.(string)
}
func OpenQuadWriter(qs graph.QuadStore, cfg *config.Config) (graph.QuadWriter, error) {
clog.Infof("Opening replication method %q", cfg.ReplicationType)
w, err := graph.NewQuadWriter(cfg.ReplicationType, qs, cfg.ReplicationOptions)
if err != nil {
return nil, err
}
return w, nil
}
func OpenQuadStore(cfg *config.Config) (graph.QuadStore, error) {
clog.Infof("Opening quad store %q at %s", cfg.DatabaseType, cfg.DatabasePath)
qs, err := graph.NewQuadStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions)
if err != nil {
return nil, err
}
return qs, nil
}
func (l *SQLLinkIterator) buildSQL(next bool, val graph.Value) (string, sqlArgs) {
query := "SELECT "
t := []string{
fmt.Sprintf("%s.subject_hash AS subject", l.tableName),
fmt.Sprintf("%s.predicate_hash AS predicate", l.tableName),
fmt.Sprintf("%s.object_hash AS object", l.tableName),
fmt.Sprintf("%s.label_hash AS label", l.tableName),
}
for _, v := range l.getTags() {
t = append(t, v.String())
}
query += strings.Join(t, ", ")
query += " FROM "
t = []string{}
var values sqlArgs
for _, k := range l.getTables() {
values = append(values, k.values...)
t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
}
query += strings.Join(t, ", ")
constraint, wherevalues := l.buildWhere()
if constraint != "" {
query += " WHERE "
}
values = append(values, wherevalues...)
if !next {
h := val.(QuadHashes)
if constraint != "" {
constraint += " AND "
} else {
constraint += " WHERE "
}
t = []string{
fmt.Sprintf("%s.subject_hash = ?", l.tableName),
fmt.Sprintf("%s.predicate_hash = ?", l.tableName),
fmt.Sprintf("%s.object_hash = ?", l.tableName),
fmt.Sprintf("%s.label_hash = ?", l.tableName),
}
constraint += strings.Join(t, " AND ")
values = append(values, h[0].toSQL())
values = append(values, h[1].toSQL())
values = append(values, h[2].toSQL())
values = append(values, h[3].toSQL())
}
query += constraint
query += ";"
if clog.V(4) {
dstr := query
for i := 1; i <= len(values); i++ {
dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
}
clog.Infof("%v", dstr)
}
return query, values
}
func Serve(handle *graph.Handle, cfg *config.Config) {
SetupRoutes(handle, cfg)
clog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort)
fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort)
err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil)
if err != nil {
clog.Fatalf("ListenAndServe: %v", err)
}
}
func (qs *QuadStore) NameOf(k graph.Value) quad.Value {
if k == nil {
if clog.V(2) {
clog.Infof("k was nil")
}
return nil
}
v := qs.valueData(k.(*Token))
return v.GetNativeValue()
}
func (qs *QuadStore) DebugPrint() {
for i, l := range qs.log {
if i == 0 {
continue
}
if clog.V(2) {
clog.Infof("%d: %#v", i, l)
}
}
}
func debugChain(obj *otto.Object) bool {
val, _ := obj.Get("_gremlin_type")
if clog.V(2) {
clog.Infof("%v", val)
}
val, _ = obj.Get("_gremlin_prev")
if val.IsObject() {
return debugChain(val.Object())
}
return false
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
var qs QuadStore
var err error
qs.path = path
cacheSize := DefaultCacheSize
val, ok, err := options.IntKey("cache_size_mb")
if err != nil {
return nil, err
} else if ok {
cacheSize = val
}
qs.dbOpts = &opt.Options{
BlockCacheCapacity: cacheSize * opt.MiB,
}
qs.dbOpts.ErrorIfMissing = true
writeBufferSize := DefaultWriteBufferSize
val, ok, err = options.IntKey("writeBufferSize")
if err != nil {
return nil, err
} else if ok {
writeBufferSize = val
}
qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
clog.Errorf("Error, could not open! %v", err)
return nil, err
}
qs.db = db
if clog.V(1) {
clog.Infof("%v", qs.GetStats())
}
vers, err := getVersion(qs.db)
if err != nil {
clog.Errorf("Error, could not read version info! %v", err)
db.Close()
return nil, err
} else if vers != latestDataVersion {
db.Close()
return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion)
}
err = qs.getMetadata()
if err != nil {
db.Close()
return nil, err
}
return &qs, nil
}
func (it *SQLIterator) Contains(v graph.Value) bool {
var err error
if ok, res := it.sql.quickContains(v); ok {
return res
}
err = it.makeCursor(false, v)
if err != nil {
clog.Errorf("Couldn't make query: %v", err)
it.err = err
if it.cursor != nil {
it.cursor.Close()
}
return false
}
it.cols, err = it.cursor.Columns()
if err != nil {
clog.Errorf("Couldn't get columns")
it.err = err
it.cursor.Close()
return false
}
it.resultList = nil
for {
if !it.cursor.Next() {
if clog.V(4) {
clog.Infof("sql: No next")
}
err := it.cursor.Err()
if err != nil {
clog.Errorf("Cursor error in SQL: %v", err)
it.err = err
}
it.cursor.Close()
break
}
s, err := scan(it.cursor, len(it.cols))
if err != nil {
it.err = err
it.cursor.Close()
return false
}
it.resultList = append(it.resultList, s)
}
it.cursor.Close()
it.cursor = nil
if len(it.resultList) != 0 {
it.resultIndex = 0
it.buildResult(0)
return true
}
return false
}
func configFrom(file string) (*config.Config, error) {
// Find the file...
if file != "" {
if _, err := os.Stat(file); os.IsNotExist(err) {
return nil, fmt.Errorf("Cannot find specified configuration file", file)
}
} else if _, err := os.Stat("/cayley_appengine.cfg"); err == nil {
file = "/cayley_appengine.cfg"
}
if file == "" {
clog.Infof("Couldn't find a config file appengine.cfg. Going by flag defaults only.")
}
cfg, err := config.Load(file)
if err != nil {
return nil, err
}
if cfg.DatabasePath == "" {
cfg.DatabasePath = databasePath
}
if cfg.DatabaseType == "" {
cfg.DatabaseType = databaseBackend
}
if cfg.ReplicationType == "" {
cfg.ReplicationType = replicationBackend
}
if cfg.ListenHost == "" {
cfg.ListenHost = host
}
if cfg.ListenPort == "" {
cfg.ListenPort = port
}
if cfg.Timeout == 0 {
cfg.Timeout = timeout
}
if cfg.LoadSize == 0 {
cfg.LoadSize = loadSize
}
cfg.ReadOnly = cfg.ReadOnly || readOnly
return cfg, nil
}
func (it *AllIterator) makeCursor() {
var cursor *sql.Rows
var err error
if it.cursor != nil {
it.cursor.Close()
}
if it.table == "quads" {
cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
} else {
if clog.V(4) {
clog.Infof("sql: getting node query")
}
cursor, err = it.qs.db.Query(`SELECT node FROM
(
SELECT subject FROM quads
UNION
SELECT predicate FROM quads
UNION
SELECT object FROM quads
UNION
SELECT label FROM quads
) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
if err != nil {
clog.Errorf("Couldn't get cursor from SQL database: %v", err)
cursor = nil
}
if clog.V(4) {
clog.Infof("sql: got node query")
}
}
it.cursor = cursor
}
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a quad iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
for it.resultIt.Next() {
it.runstats.ContainsNext += 1
link := it.resultIt.Result()
if clog.V(4) {
clog.Infof("Quad is %v", it.qs.Quad(link))
}
if it.primaryIt.Contains(link) {
it.result = it.qs.QuadDirection(link, it.dir)
return true
}
}
it.err = it.resultIt.Err()
return false
}