Exemple #1
0
// Get the next result that matches this branch.
func (it *HasA) NextPath() bool {
	// Order here is important. If the subiterator has a NextPath, then we
	// need do nothing -- there is a next result, and we shouldn't move forward.
	// However, we then need to get the next result from our last Contains().
	//
	// The upshot is, the end of NextPath() bubbles up from the bottom of the
	// iterator tree up, and we need to respect that.
	if clog.V(4) {
		clog.Infof("HASA %v NextPath", it.UID())
	}
	if it.primaryIt.NextPath() {
		return true
	}
	it.err = it.primaryIt.Err()
	if it.err != nil {
		return false
	}

	result := it.NextContains() // Sets it.err if there's an error
	if it.err != nil {
		return false
	}
	if clog.V(4) {
		clog.Infof("HASA %v NextPath Returns %v", it.UID(), result)
	}
	return result
}
Exemple #2
0
func (s *Session) Execute(input string, c chan interface{}, _ int) {
	defer close(c)
	var mqlQuery interface{}
	err := json.Unmarshal([]byte(input), &mqlQuery)
	if err != nil {
		return
	}
	s.currentQuery = NewQuery(s)
	s.currentQuery.BuildIteratorTree(mqlQuery)
	if s.currentQuery.isError() {
		return
	}
	it, _ := s.currentQuery.it.Optimize()
	if clog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			clog.Infof("failed to format description: %v", err)
		} else {
			clog.Infof("%s", b)
		}
	}
	for graph.Next(it) {
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		c <- tags
		for it.NextPath() == true {
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			c <- tags
		}
	}
}
Exemple #3
0
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	block := make([]quad.Quad, 0, cfg.LoadSize)
	count := 0
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		block = append(block, t)
		if len(block) == cap(block) {
			count += len(block)
			err := qw.AddQuadSet(block)
			if err != nil {
				return fmt.Errorf("db: failed to load data: %v", err)
			}
			block = block[:0]
			if clog.V(2) {
				clog.Infof("Wrote %d quads.", count)
			}
		}
	}
	count += len(block)
	err := qw.AddQuadSet(block)
	if err != nil {
		return fmt.Errorf("db: failed to load data: %v", err)
	}
	if clog.V(2) {
		clog.Infof("Wrote %d quads.", count)
	}

	return nil
}
Exemple #4
0
func (it *AllIterator) makeCursor() {
	var cursor *sql.Rows
	var err error
	if it.cursor != nil {
		it.cursor.Close()
	}
	if it.table == "quads" {
		cursor, err = it.qs.db.Query(`SELECT
			subject_hash,
			predicate_hash,
			object_hash,
			label_hash
			FROM quads;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
	} else {
		if clog.V(4) {
			clog.Infof("sql: getting node query")
		}
		cursor, err = it.qs.db.Query(`SELECT hash FROM nodes;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
		if clog.V(4) {
			clog.Infof("sql: got node query")
		}
	}
	it.cursor = cursor
}
Exemple #5
0
func NextLogOut(it Iterator, val Value, ok bool) bool {
	if clog.V(4) {
		if ok {
			clog.Infof("%s %d NEXT IS %v", strings.ToUpper(it.Type().String()), it.UID(), val)
		} else {
			clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
		}
	}
	return ok
}
Exemple #6
0
func ContainsLogOut(it Iterator, val Value, good bool) bool {
	if clog.V(4) {
		if good {
			clog.Infof("%s %d CHECK CONTAINS %v GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
		} else {
			clog.Infof("%s %d CHECK CONTAINS %v BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
		}
	}
	return good
}
Exemple #7
0
func NextLogOut(it Iterator, ok bool) bool {
	if clog.V(4) {
		if ok {
			val := it.Result()
			clog.Infof("%s %d NEXT IS %v (%T)", strings.ToUpper(it.Type().String()), it.UID(), val, val)
		} else {
			clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
		}
	}
	return ok
}
Exemple #8
0
func (c *IterateChain) end() {
	c.it.Close()
	if !clog.V(2) {
		return
	}
	if b, err := json.MarshalIndent(DumpStats(c.it), "", "  "); err != nil {
		clog.Infof("failed to format stats: %v", err)
	} else {
		clog.Infof("%s", b)
	}
}
// Size is the number of values stored, if we've got them all.
// Otherwise, guess based on the size of the subiterator.
func (it *Materialize) Size() (int64, bool) {
	if it.hasRun && !it.aborted {
		if clog.V(2) {
			clog.Infof("returning size %v", it.actualSize)
		}
		return it.actualSize, true
	}
	if clog.V(2) {
		clog.Infof("bailing size %v", it.actualSize)
	}
	return it.subIt.Size()
}
Exemple #10
0
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, hash NodeHash) int64 {
	var err error
	if isAll {
		return qs.Size()
	}
	if qs.noSizes {
		if dir == quad.Predicate {
			return (qs.Size() / 100) + 1
		}
		return (qs.Size() / 1000) + 1
	}
	if val, ok := qs.sizes.Get(hash.String() + string(dir.Prefix())); ok {
		return val.(int64)
	}
	var size int64
	if clog.V(4) {
		clog.Infof("sql: getting size for select %s, %v", dir.String(), hash)
	}
	err = qs.db.QueryRow(
		fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hash.toSQL()).Scan(&size)
	if err != nil {
		clog.Errorf("Error getting size from SQL database: %v", err)
		return 0
	}
	qs.sizes.Put(hash.String()+string(dir.Prefix()), size)
	return size
}
Exemple #11
0
func (c *IterateChain) start() {
	if c.optimize {
		c.it, _ = c.it.Optimize()
		if c.qs != nil {
			c.it, _ = c.qs.OptimizeIterator(c.it)
		}
	}
	if !clog.V(2) {
		return
	}
	if b, err := json.MarshalIndent(c.it.Describe(), "", "  "); err != nil {
		clog.Infof("failed to format description: %v", err)
	} else {
		clog.Infof("%s", b)
	}
}
Exemple #12
0
func (qs *QuadStore) NameOf(k graph.Value) string {
	if k == nil {
		clog.Infof("k was nil")
		return ""
	}
	return qs.valueData(k.(Token)).Name
}
Exemple #13
0
func LogRequest(handler ResponseHandler) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		start := time.Now()
		addr := req.Header.Get("X-Real-IP")
		if addr == "" {
			addr = req.Header.Get("X-Forwarded-For")
			if addr == "" {
				addr = req.RemoteAddr
			}
		}
		clog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr)
		code := handler(w, req, params)
		clog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start))

	}
}
Exemple #14
0
func configFrom(file string) *config.Config {
	// Find the file...
	if file != "" {
		if _, err := os.Stat(file); os.IsNotExist(err) {
			clog.Fatalf("Cannot find specified configuration file '%s', aborting.", file)
		}
	} else if _, err := os.Stat(os.Getenv("CAYLEY_CFG")); err == nil {
		file = os.Getenv("CAYLEY_CFG")
	} else if _, err := os.Stat("/etc/cayley.cfg"); err == nil {
		file = "/etc/cayley.cfg"
	}
	if file == "" {
		clog.Infof("Couldn't find a config file in either $CAYLEY_CFG or /etc/cayley.cfg. Going by flag defaults only.")
	}
	cfg, err := config.Load(file)
	if err != nil {
		clog.Fatalf("%v", err)
	}
	if cfg.DatabasePath == "" {
		cfg.DatabasePath = *databasePath
	}

	if cfg.DatabaseType == "" {
		cfg.DatabaseType = *databaseBackend
	}
	return cfg
}
Exemple #15
0
func (wk *worker) runIterator(it graph.Iterator) {
	if wk.wantShape() {
		iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
		return
	}
	it, _ = it.Optimize()
	if clog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			clog.Infof("failed to format description: %v", err)
		} else {
			clog.Infof("%s", b)
		}
	}
	for {
		select {
		case <-wk.kill:
			return
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		if !wk.send(&Result{actualResults: tags}) {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			if !wk.send(&Result{actualResults: tags}) {
				break
			}
		}
	}
	if clog.V(2) {
		bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", "  ")
		clog.Infof(string(bytes))
	}
	it.Close()
}
Exemple #16
0
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
	n := 0
	it, _ = it.Optimize()
	if clog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			clog.Infof("failed to format description: %v", err)
		} else {
			clog.Infof("%s", b)
		}
	}
	for {
		select {
		case <-wk.kill:
			return
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
		val, _ = callback.Call(this.This, val)
		n++
		if limit >= 0 && n >= limit {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
			val, _ = callback.Call(this.This, val)
			n++
			if limit >= 0 && n >= limit {
				break
			}
		}
	}
	it.Close()
}
Exemple #17
0
func TestIteratorsAndNextResultOrderA(t *testing.T) {
	clog.Infof("\n-----------\n")
	inst, opts, err := createInstance()
	defer inst.Close()

	if err != nil {
		t.Fatalf("failed to create instance: %v", err)
	}
	qs, _, _ := makeTestStore(simpleGraph, opts)
	if qs.Size() != 11 {
		t.Fatal("Incorrect number of quads")
	}

	fixed := qs.FixedIterator()
	fixed.Add(qs.ValueOf("C"))

	fixed2 := qs.FixedIterator()
	fixed2.Add(qs.ValueOf("follows"))

	all := qs.NodesAllIterator()

	innerAnd := iterator.NewAnd(qs)
	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed2, quad.Predicate))
	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Object))

	hasa := iterator.NewHasA(qs, innerAnd, quad.Subject)
	outerAnd := iterator.NewAnd(qs)
	outerAnd.AddSubIterator(fixed)
	outerAnd.AddSubIterator(hasa)

	if !outerAnd.Next() {
		t.Error("Expected one matching subtree")
	}
	val := outerAnd.Result()
	if qs.NameOf(val) != "C" {
		t.Errorf("Matching subtree should be %s, got %s", "barak", qs.NameOf(val))
	}

	var (
		got    []string
		expect = []string{"B", "D"}
	)
	for {
		got = append(got, qs.NameOf(all.Result()))
		if !outerAnd.NextPath() {
			break
		}
	}
	sort.Strings(got)

	if !reflect.DeepEqual(got, expect) {
		t.Errorf("Unexpected result, got:%q expect:%q", got, expect)
	}

	if outerAnd.Next() {
		t.Error("More than one possible top level output?")
	}
}
Exemple #18
0
func (qs *QuadStore) NameOf(v graph.Value) string {
	if v == nil {
		if clog.V(2) {
			clog.Infof("NameOf was nil")
		}
		return ""
	}
	return v.(string)
}
Exemple #19
0
func OpenQuadWriter(qs graph.QuadStore, cfg *config.Config) (graph.QuadWriter, error) {
	clog.Infof("Opening replication method %q", cfg.ReplicationType)
	w, err := graph.NewQuadWriter(cfg.ReplicationType, qs, cfg.ReplicationOptions)
	if err != nil {
		return nil, err
	}

	return w, nil
}
Exemple #20
0
func OpenQuadStore(cfg *config.Config) (graph.QuadStore, error) {
	clog.Infof("Opening quad store %q at %s", cfg.DatabaseType, cfg.DatabasePath)
	qs, err := graph.NewQuadStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions)
	if err != nil {
		return nil, err
	}

	return qs, nil
}
Exemple #21
0
func (l *SQLLinkIterator) buildSQL(next bool, val graph.Value) (string, sqlArgs) {
	query := "SELECT "
	t := []string{
		fmt.Sprintf("%s.subject_hash AS subject", l.tableName),
		fmt.Sprintf("%s.predicate_hash AS predicate", l.tableName),
		fmt.Sprintf("%s.object_hash AS object", l.tableName),
		fmt.Sprintf("%s.label_hash AS label", l.tableName),
	}
	for _, v := range l.getTags() {
		t = append(t, v.String())
	}
	query += strings.Join(t, ", ")
	query += " FROM "
	t = []string{}
	var values sqlArgs
	for _, k := range l.getTables() {
		values = append(values, k.values...)
		t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
	}
	query += strings.Join(t, ", ")
	constraint, wherevalues := l.buildWhere()
	if constraint != "" {
		query += " WHERE "
	}

	values = append(values, wherevalues...)
	if !next {
		h := val.(QuadHashes)
		if constraint != "" {
			constraint += " AND "
		} else {
			constraint += " WHERE "
		}
		t = []string{
			fmt.Sprintf("%s.subject_hash = ?", l.tableName),
			fmt.Sprintf("%s.predicate_hash = ?", l.tableName),
			fmt.Sprintf("%s.object_hash = ?", l.tableName),
			fmt.Sprintf("%s.label_hash = ?", l.tableName),
		}
		constraint += strings.Join(t, " AND ")
		values = append(values, h[0].toSQL())
		values = append(values, h[1].toSQL())
		values = append(values, h[2].toSQL())
		values = append(values, h[3].toSQL())
	}
	query += constraint
	query += ";"

	if clog.V(4) {
		dstr := query
		for i := 1; i <= len(values); i++ {
			dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
		}
		clog.Infof("%v", dstr)
	}
	return query, values
}
Exemple #22
0
func Serve(handle *graph.Handle, cfg *config.Config) {
	SetupRoutes(handle, cfg)
	clog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort)
	fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort)
	err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil)
	if err != nil {
		clog.Fatalf("ListenAndServe: %v", err)
	}
}
Exemple #23
0
func (qs *QuadStore) NameOf(k graph.Value) quad.Value {
	if k == nil {
		if clog.V(2) {
			clog.Infof("k was nil")
		}
		return nil
	}
	v := qs.valueData(k.(*Token))
	return v.GetNativeValue()
}
Exemple #24
0
func (qs *QuadStore) DebugPrint() {
	for i, l := range qs.log {
		if i == 0 {
			continue
		}
		if clog.V(2) {
			clog.Infof("%d: %#v", i, l)
		}
	}
}
Exemple #25
0
func debugChain(obj *otto.Object) bool {
	val, _ := obj.Get("_gremlin_type")
	if clog.V(2) {
		clog.Infof("%v", val)
	}
	val, _ = obj.Get("_gremlin_prev")
	if val.IsObject() {
		return debugChain(val.Object())
	}
	return false
}
Exemple #26
0
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	var qs QuadStore
	var err error
	qs.path = path
	cacheSize := DefaultCacheSize
	val, ok, err := options.IntKey("cache_size_mb")
	if err != nil {
		return nil, err
	} else if ok {
		cacheSize = val
	}
	qs.dbOpts = &opt.Options{
		BlockCacheCapacity: cacheSize * opt.MiB,
	}
	qs.dbOpts.ErrorIfMissing = true

	writeBufferSize := DefaultWriteBufferSize
	val, ok, err = options.IntKey("writeBufferSize")
	if err != nil {
		return nil, err
	} else if ok {
		writeBufferSize = val
	}
	qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB
	qs.writeopts = &opt.WriteOptions{
		Sync: false,
	}
	qs.readopts = &opt.ReadOptions{}
	db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
	if err != nil {
		clog.Errorf("Error, could not open! %v", err)
		return nil, err
	}
	qs.db = db
	if clog.V(1) {
		clog.Infof("%v", qs.GetStats())
	}
	vers, err := getVersion(qs.db)
	if err != nil {
		clog.Errorf("Error, could not read version info! %v", err)
		db.Close()
		return nil, err
	} else if vers != latestDataVersion {
		db.Close()
		return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion)
	}
	err = qs.getMetadata()
	if err != nil {
		db.Close()
		return nil, err
	}
	return &qs, nil
}
Exemple #27
0
func (it *SQLIterator) Contains(v graph.Value) bool {
	var err error
	if ok, res := it.sql.quickContains(v); ok {
		return res
	}
	err = it.makeCursor(false, v)
	if err != nil {
		clog.Errorf("Couldn't make query: %v", err)
		it.err = err
		if it.cursor != nil {
			it.cursor.Close()
		}
		return false
	}
	it.cols, err = it.cursor.Columns()
	if err != nil {
		clog.Errorf("Couldn't get columns")
		it.err = err
		it.cursor.Close()
		return false
	}
	it.resultList = nil
	for {
		if !it.cursor.Next() {
			if clog.V(4) {
				clog.Infof("sql: No next")
			}
			err := it.cursor.Err()
			if err != nil {
				clog.Errorf("Cursor error in SQL: %v", err)
				it.err = err
			}
			it.cursor.Close()
			break
		}
		s, err := scan(it.cursor, len(it.cols))
		if err != nil {
			it.err = err
			it.cursor.Close()
			return false
		}
		it.resultList = append(it.resultList, s)
	}
	it.cursor.Close()
	it.cursor = nil
	if len(it.resultList) != 0 {
		it.resultIndex = 0
		it.buildResult(0)
		return true
	}
	return false
}
Exemple #28
0
func configFrom(file string) (*config.Config, error) {
	// Find the file...
	if file != "" {
		if _, err := os.Stat(file); os.IsNotExist(err) {
			return nil, fmt.Errorf("Cannot find specified configuration file", file)
		}
	} else if _, err := os.Stat("/cayley_appengine.cfg"); err == nil {
		file = "/cayley_appengine.cfg"
	}
	if file == "" {
		clog.Infof("Couldn't find a config file appengine.cfg. Going by flag defaults only.")
	}
	cfg, err := config.Load(file)
	if err != nil {
		return nil, err
	}

	if cfg.DatabasePath == "" {
		cfg.DatabasePath = databasePath
	}

	if cfg.DatabaseType == "" {
		cfg.DatabaseType = databaseBackend
	}

	if cfg.ReplicationType == "" {
		cfg.ReplicationType = replicationBackend
	}

	if cfg.ListenHost == "" {
		cfg.ListenHost = host
	}

	if cfg.ListenPort == "" {
		cfg.ListenPort = port
	}

	if cfg.Timeout == 0 {
		cfg.Timeout = timeout
	}

	if cfg.LoadSize == 0 {
		cfg.LoadSize = loadSize
	}

	cfg.ReadOnly = cfg.ReadOnly || readOnly

	return cfg, nil
}
Exemple #29
0
func (it *AllIterator) makeCursor() {
	var cursor *sql.Rows
	var err error
	if it.cursor != nil {
		it.cursor.Close()
	}
	if it.table == "quads" {
		cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
	} else {
		if clog.V(4) {
			clog.Infof("sql: getting node query")
		}
		cursor, err = it.qs.db.Query(`SELECT node FROM
			(
				SELECT subject FROM quads
				UNION
				SELECT predicate FROM quads
				UNION
				SELECT object FROM quads
				UNION
				SELECT label FROM quads
			) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
		if clog.V(4) {
			clog.Infof("sql: got node query")
		}
	}
	it.cursor = cursor
}
Exemple #30
0
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a quad iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
	for it.resultIt.Next() {
		it.runstats.ContainsNext += 1
		link := it.resultIt.Result()
		if clog.V(4) {
			clog.Infof("Quad is %v", it.qs.Quad(link))
		}
		if it.primaryIt.Contains(link) {
			it.result = it.qs.QuadDirection(link, it.dir)
			return true
		}
	}
	it.err = it.resultIt.Err()
	return false
}