示例#1
0
func (it *AllIterator) makeCursor() {
	var cursor *sql.Rows
	var err error
	if it.cursor != nil {
		it.cursor.Close()
	}
	if it.table == "quads" {
		cursor, err = it.qs.db.Query(`SELECT
			subject_hash,
			predicate_hash,
			object_hash,
			label_hash
			FROM quads;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
	} else {
		if clog.V(4) {
			clog.Infof("sql: getting node query")
		}
		cursor, err = it.qs.db.Query(`SELECT hash FROM nodes;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
		if clog.V(4) {
			clog.Infof("sql: got node query")
		}
	}
	it.cursor = cursor
}
示例#2
0
// Get the next result that matches this branch.
func (it *HasA) NextPath() bool {
	// Order here is important. If the subiterator has a NextPath, then we
	// need do nothing -- there is a next result, and we shouldn't move forward.
	// However, we then need to get the next result from our last Contains().
	//
	// The upshot is, the end of NextPath() bubbles up from the bottom of the
	// iterator tree up, and we need to respect that.
	if clog.V(4) {
		clog.Infof("HASA %v NextPath", it.UID())
	}
	if it.primaryIt.NextPath() {
		return true
	}
	it.err = it.primaryIt.Err()
	if it.err != nil {
		return false
	}

	result := it.NextContains() // Sets it.err if there's an error
	if it.err != nil {
		return false
	}
	if clog.V(4) {
		clog.Infof("HASA %v NextPath Returns %v", it.UID(), result)
	}
	return result
}
示例#3
0
文件: db.go 项目: RamboWANG/cayley
func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
	block := make([]quad.Quad, 0, cfg.LoadSize)
	count := 0
	for {
		t, err := dec.Unmarshal()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		block = append(block, t)
		if len(block) == cap(block) {
			count += len(block)
			err := qw.AddQuadSet(block)
			if err != nil {
				return fmt.Errorf("db: failed to load data: %v", err)
			}
			block = block[:0]
			if clog.V(2) {
				clog.Infof("Wrote %d quads.", count)
			}
		}
	}
	count += len(block)
	err := qw.AddQuadSet(block)
	if err != nil {
		return fmt.Errorf("db: failed to load data: %v", err)
	}
	if clog.V(2) {
		clog.Infof("Wrote %d quads.", count)
	}

	return nil
}
示例#4
0
// Size is the number of values stored, if we've got them all.
// Otherwise, guess based on the size of the subiterator.
func (it *Materialize) Size() (int64, bool) {
	if it.hasRun && !it.aborted {
		if clog.V(2) {
			clog.Infof("returning size %v", it.actualSize)
		}
		return it.actualSize, true
	}
	if clog.V(2) {
		clog.Infof("bailing size %v", it.actualSize)
	}
	return it.subIt.Size()
}
示例#5
0
func (s *Session) Execute(input string, c chan interface{}, _ int) {
	defer close(c)
	var mqlQuery interface{}
	err := json.Unmarshal([]byte(input), &mqlQuery)
	if err != nil {
		return
	}
	s.currentQuery = NewQuery(s)
	s.currentQuery.BuildIteratorTree(mqlQuery)
	if s.currentQuery.isError() {
		return
	}
	it, _ := s.currentQuery.it.Optimize()
	if clog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			clog.Infof("failed to format description: %v", err)
		} else {
			clog.Infof("%s", b)
		}
	}
	for graph.Next(it) {
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		c <- tags
		for it.NextPath() == true {
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			c <- tags
		}
	}
}
示例#6
0
func (qs *QuadStore) sizeForIterator(isAll bool, dir quad.Direction, hash NodeHash) int64 {
	var err error
	if isAll {
		return qs.Size()
	}
	if qs.noSizes {
		if dir == quad.Predicate {
			return (qs.Size() / 100) + 1
		}
		return (qs.Size() / 1000) + 1
	}
	if val, ok := qs.sizes.Get(hash.String() + string(dir.Prefix())); ok {
		return val.(int64)
	}
	var size int64
	if clog.V(4) {
		clog.Infof("sql: getting size for select %s, %v", dir.String(), hash)
	}
	err = qs.db.QueryRow(
		fmt.Sprintf("SELECT count(*) FROM quads WHERE %s_hash = $1;", dir.String()), hash.toSQL()).Scan(&size)
	if err != nil {
		clog.Errorf("Error getting size from SQL database: %v", err)
		return 0
	}
	qs.sizes.Put(hash.String()+string(dir.Prefix()), size)
	return size
}
示例#7
0
func (wk *worker) runIterator(it graph.Iterator) {
	if wk.wantShape() {
		iterator.OutputQueryShapeForIterator(it, wk.qs, wk.shape)
		return
	}
	it, _ = it.Optimize()
	if clog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			clog.Infof("failed to format description: %v", err)
		} else {
			clog.Infof("%s", b)
		}
	}
	for {
		select {
		case <-wk.kill:
			return
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		if !wk.send(&Result{actualResults: tags}) {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			if !wk.send(&Result{actualResults: tags}) {
				break
			}
		}
	}
	if clog.V(2) {
		bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", "  ")
		clog.Infof(string(bytes))
	}
	it.Close()
}
示例#8
0
func (qs *QuadStore) NameOf(v graph.Value) string {
	if v == nil {
		if clog.V(2) {
			clog.Infof("NameOf was nil")
		}
		return ""
	}
	return v.(string)
}
示例#9
0
func (l *SQLLinkIterator) buildSQL(next bool, val graph.Value) (string, sqlArgs) {
	query := "SELECT "
	t := []string{
		fmt.Sprintf("%s.subject_hash AS subject", l.tableName),
		fmt.Sprintf("%s.predicate_hash AS predicate", l.tableName),
		fmt.Sprintf("%s.object_hash AS object", l.tableName),
		fmt.Sprintf("%s.label_hash AS label", l.tableName),
	}
	for _, v := range l.getTags() {
		t = append(t, v.String())
	}
	query += strings.Join(t, ", ")
	query += " FROM "
	t = []string{}
	var values sqlArgs
	for _, k := range l.getTables() {
		values = append(values, k.values...)
		t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
	}
	query += strings.Join(t, ", ")
	constraint, wherevalues := l.buildWhere()
	if constraint != "" {
		query += " WHERE "
	}

	values = append(values, wherevalues...)
	if !next {
		h := val.(QuadHashes)
		if constraint != "" {
			constraint += " AND "
		} else {
			constraint += " WHERE "
		}
		t = []string{
			fmt.Sprintf("%s.subject_hash = ?", l.tableName),
			fmt.Sprintf("%s.predicate_hash = ?", l.tableName),
			fmt.Sprintf("%s.object_hash = ?", l.tableName),
			fmt.Sprintf("%s.label_hash = ?", l.tableName),
		}
		constraint += strings.Join(t, " AND ")
		values = append(values, h[0].toSQL())
		values = append(values, h[1].toSQL())
		values = append(values, h[2].toSQL())
		values = append(values, h[3].toSQL())
	}
	query += constraint
	query += ";"

	if clog.V(4) {
		dstr := query
		for i := 1; i <= len(values); i++ {
			dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
		}
		clog.Infof("%v", dstr)
	}
	return query, values
}
示例#10
0
func (qs *QuadStore) NameOf(k graph.Value) string {
	if k == nil {
		if clog.V(2) {
			clog.Infof("k was nil")
		}
		return ""
	}
	return qs.valueData(k.(*Token)).Name
}
示例#11
0
func (qs *QuadStore) DebugPrint() {
	for i, l := range qs.log {
		if i == 0 {
			continue
		}
		if clog.V(2) {
			clog.Infof("%d: %#v", i, l)
		}
	}
}
示例#12
0
func NextLogOut(it Iterator, val Value, ok bool) bool {
	if clog.V(4) {
		if ok {
			clog.Infof("%s %d NEXT IS %v", strings.ToUpper(it.Type().String()), it.UID(), val)
		} else {
			clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
		}
	}
	return ok
}
示例#13
0
func ContainsLogOut(it Iterator, val Value, good bool) bool {
	if clog.V(4) {
		if good {
			clog.Infof("%s %d CHECK CONTAINS %v GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
		} else {
			clog.Infof("%s %d CHECK CONTAINS %v BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
		}
	}
	return good
}
示例#14
0
func (qs *QuadStore) NameOf(k graph.Value) quad.Value {
	if k == nil {
		if clog.V(2) {
			clog.Infof("k was nil")
		}
		return nil
	}
	v := qs.valueData(k.(*Token))
	return v.GetNativeValue()
}
示例#15
0
文件: iterate.go 项目: rlugojr/cayley
func (c *IterateChain) end() {
	c.it.Close()
	if !clog.V(2) {
		return
	}
	if b, err := json.MarshalIndent(DumpStats(c.it), "", "  "); err != nil {
		clog.Infof("failed to format stats: %v", err)
	} else {
		clog.Infof("%s", b)
	}
}
示例#16
0
func NextLogOut(it Iterator, ok bool) bool {
	if clog.V(4) {
		if ok {
			val := it.Result()
			clog.Infof("%s %d NEXT IS %v (%T)", strings.ToUpper(it.Type().String()), it.UID(), val, val)
		} else {
			clog.Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
		}
	}
	return ok
}
示例#17
0
func debugChain(obj *otto.Object) bool {
	val, _ := obj.Get("_gremlin_type")
	if clog.V(2) {
		clog.Infof("%v", val)
	}
	val, _ = obj.Get("_gremlin_prev")
	if val.IsObject() {
		return debugChain(val.Object())
	}
	return false
}
示例#18
0
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	var qs QuadStore
	var err error
	qs.path = path
	cacheSize := DefaultCacheSize
	val, ok, err := options.IntKey("cache_size_mb")
	if err != nil {
		return nil, err
	} else if ok {
		cacheSize = val
	}
	qs.dbOpts = &opt.Options{
		BlockCacheCapacity: cacheSize * opt.MiB,
	}
	qs.dbOpts.ErrorIfMissing = true

	writeBufferSize := DefaultWriteBufferSize
	val, ok, err = options.IntKey("writeBufferSize")
	if err != nil {
		return nil, err
	} else if ok {
		writeBufferSize = val
	}
	qs.dbOpts.WriteBuffer = writeBufferSize * opt.MiB
	qs.writeopts = &opt.WriteOptions{
		Sync: false,
	}
	qs.readopts = &opt.ReadOptions{}
	db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
	if err != nil {
		clog.Errorf("Error, could not open! %v", err)
		return nil, err
	}
	qs.db = db
	if clog.V(1) {
		clog.Infof("%v", qs.GetStats())
	}
	vers, err := getVersion(qs.db)
	if err != nil {
		clog.Errorf("Error, could not read version info! %v", err)
		db.Close()
		return nil, err
	} else if vers != latestDataVersion {
		db.Close()
		return nil, fmt.Errorf("leveldb: data version is out of date (%d vs %d). Run cayleyupgrade for your config to update the data.", vers, latestDataVersion)
	}
	err = qs.getMetadata()
	if err != nil {
		db.Close()
		return nil, err
	}
	return &qs, nil
}
示例#19
0
func (it *SQLIterator) Contains(v graph.Value) bool {
	var err error
	if ok, res := it.sql.quickContains(v); ok {
		return res
	}
	err = it.makeCursor(false, v)
	if err != nil {
		clog.Errorf("Couldn't make query: %v", err)
		it.err = err
		if it.cursor != nil {
			it.cursor.Close()
		}
		return false
	}
	it.cols, err = it.cursor.Columns()
	if err != nil {
		clog.Errorf("Couldn't get columns")
		it.err = err
		it.cursor.Close()
		return false
	}
	it.resultList = nil
	for {
		if !it.cursor.Next() {
			if clog.V(4) {
				clog.Infof("sql: No next")
			}
			err := it.cursor.Err()
			if err != nil {
				clog.Errorf("Cursor error in SQL: %v", err)
				it.err = err
			}
			it.cursor.Close()
			break
		}
		s, err := scan(it.cursor, len(it.cols))
		if err != nil {
			it.err = err
			it.cursor.Close()
			return false
		}
		it.resultList = append(it.resultList, s)
	}
	it.cursor.Close()
	it.cursor = nil
	if len(it.resultList) != 0 {
		it.resultIndex = 0
		it.buildResult(0)
		return true
	}
	return false
}
示例#20
0
func (it *AllIterator) makeCursor() {
	var cursor *sql.Rows
	var err error
	if it.cursor != nil {
		it.cursor.Close()
	}
	if it.table == "quads" {
		cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
	} else {
		if clog.V(4) {
			clog.Infof("sql: getting node query")
		}
		cursor, err = it.qs.db.Query(`SELECT node FROM
			(
				SELECT subject FROM quads
				UNION
				SELECT predicate FROM quads
				UNION
				SELECT object FROM quads
				UNION
				SELECT label FROM quads
			) AS DistinctNodes (node) WHERE node IS NOT NULL;`)
		if err != nil {
			clog.Errorf("Couldn't get cursor from SQL database: %v", err)
			cursor = nil
		}
		if clog.V(4) {
			clog.Infof("sql: got node query")
		}
	}
	it.cursor = cursor
}
示例#21
0
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a quad iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) NextContains() bool {
	for it.resultIt.Next() {
		it.runstats.ContainsNext += 1
		link := it.resultIt.Result()
		if clog.V(4) {
			clog.Infof("Quad is %v", it.qs.Quad(link))
		}
		if it.primaryIt.Contains(link) {
			it.result = it.qs.QuadDirection(link, it.dir)
			return true
		}
	}
	it.err = it.resultIt.Err()
	return false
}
示例#22
0
文件: iterate.go 项目: rlugojr/cayley
func (c *IterateChain) start() {
	if c.optimize {
		c.it, _ = c.it.Optimize()
		if c.qs != nil {
			c.it, _ = c.qs.OptimizeIterator(c.it)
		}
	}
	if !clog.V(2) {
		return
	}
	if b, err := json.MarshalIndent(c.it.Describe(), "", "  "); err != nil {
		clog.Infof("failed to format description: %v", err)
	} else {
		clog.Infof("%s", b)
	}
}
示例#23
0
func (wk *worker) runIteratorWithCallback(it graph.Iterator, callback otto.Value, this otto.FunctionCall, limit int) {
	n := 0
	it, _ = it.Optimize()
	if clog.V(2) {
		b, err := json.MarshalIndent(it.Describe(), "", "  ")
		if err != nil {
			clog.Infof("failed to format description: %v", err)
		} else {
			clog.Infof("%s", b)
		}
	}
	for {
		select {
		case <-wk.kill:
			return
		default:
		}
		if !graph.Next(it) {
			break
		}
		tags := make(map[string]graph.Value)
		it.TagResults(tags)
		val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
		val, _ = callback.Call(this.This, val)
		n++
		if limit >= 0 && n >= limit {
			break
		}
		for it.NextPath() {
			select {
			case <-wk.kill:
				return
			default:
			}
			tags := make(map[string]graph.Value)
			it.TagResults(tags)
			val, _ := this.Otto.ToValue(wk.tagsToValueMap(tags))
			val, _ = callback.Call(this.This, val)
			n++
			if limit >= 0 && n >= limit {
				break
			}
		}
	}
	it.Close()
}
示例#24
0
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "quads that have `val` in our direction", given to us by the quad store,
// and then Next() values out of that iterator and Contains() them against our subiterator.
func (it *HasA) Contains(val graph.Value) bool {
	graph.ContainsLogIn(it, val)
	it.runstats.Contains += 1
	if clog.V(4) {
		clog.Infof("Id is %v", it.qs.NameOf(val))
	}
	// TODO(barakmich): Optimize this
	if it.resultIt != nil {
		it.resultIt.Close()
	}
	it.resultIt = it.qs.QuadIterator(it.dir, val)
	ok := it.NextContains()
	if it.err != nil {
		return false
	}
	return graph.ContainsLogOut(it, val, ok)
}
示例#25
0
func (n *SQLNodeIterator) buildSQL(next bool, val graph.Value) (string, []string) {
	topData := n.tableID()
	tags := []tagDir{topData}
	tags = append(tags, n.getTags()...)
	query := "SELECT "

	var t []string
	for _, v := range tags {
		t = append(t, v.String())
	}
	query += strings.Join(t, ", ")
	query += " FROM "
	t = []string{}
	var values []string
	for _, k := range n.getTables() {
		values = append(values, k.values...)
		t = append(t, fmt.Sprintf("%s as %s", k.table, k.name))
	}
	query += strings.Join(t, ", ")
	query += " WHERE "

	constraint, wherevalues := n.buildWhere()
	values = append(values, wherevalues...)

	if !next {
		v := val.(string)
		if constraint != "" {
			constraint += " AND "
		}
		constraint += fmt.Sprintf("%s.%s_hash = ?", topData.table, topData.dir)
		values = append(values, hashOf(v))
	}

	query += constraint
	query += ";"

	if clog.V(4) {
		dstr := query
		for i := 1; i <= len(values); i++ {
			dstr = strings.Replace(dstr, "?", fmt.Sprintf("'%s'", values[i-1]), 1)
		}
		clog.Infof("%v", dstr)
	}
	return query, values
}
示例#26
0
func (it *Materialize) materializeSet() {
	i := 0
	for graph.Next(it.subIt) {
		i++
		if i > abortMaterializeAt {
			it.aborted = true
			break
		}
		id := it.subIt.Result()
		val := id
		if h, ok := id.(Keyer); ok {
			val = h.Key()
		}
		if _, ok := it.containsMap[val]; !ok {
			it.containsMap[val] = len(it.values)
			it.values = append(it.values, nil)
		}
		index := it.containsMap[val]
		tags := make(map[string]graph.Value)
		it.subIt.TagResults(tags)
		it.values[index] = append(it.values[index], result{id: id, tags: tags})
		it.actualSize += 1
		for it.subIt.NextPath() {
			i++
			if i > abortMaterializeAt {
				it.aborted = true
				break
			}
			tags := make(map[string]graph.Value)
			it.subIt.TagResults(tags)
			it.values[index] = append(it.values[index], result{id: id, tags: tags})
			it.actualSize += 1
		}
	}
	it.err = it.subIt.Err()
	if it.err == nil && it.aborted {
		if clog.V(2) {
			clog.Infof("Aborting subiterator")
		}
		it.values = nil
		it.containsMap = nil
		it.subIt.Reset()
	}
	it.hasRun = true
}
示例#27
0
func (qs *QuadStore) valueData(t *Token) proto.NodeData {
	var out proto.NodeData
	if clog.V(3) {
		clog.Infof("%s %v", string(t.bucket), t.key)
	}
	err := qs.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(t.bucket)
		data := b.Get(t.key)
		if data != nil {
			return out.Unmarshal(data)
		}
		return nil
	})
	if err != nil {
		clog.Errorf("Error: couldn't get value")
		return proto.NodeData{}
	}
	return out
}
示例#28
0
func (qs *QuadStore) valueData(key []byte) ValueData {
	var out ValueData
	if clog.V(3) {
		clog.Infof("%c %v", key[0], key)
	}
	b, err := qs.db.Get(key, qs.readopts)
	if err != nil && err != leveldb.ErrNotFound {
		clog.Errorf("Error: could not get value from DB")
		return out
	}
	if b != nil && err != leveldb.ErrNotFound {
		err = json.Unmarshal(b, &out)
		if err != nil {
			clog.Errorf("Error: could not reconstruct value")
			return ValueData{}
		}
	}
	return out
}
示例#29
0
func (it *AllIterator) Next() bool {
	graph.NextLogIn(it)
	if it.cursor == nil {
		it.makeCursor()
		if it.cursor == nil {
			return false
		}
	}
	if !it.cursor.Next() {
		if clog.V(4) {
			clog.Infof("sql: No next")
		}
		err := it.cursor.Err()
		if err != nil {
			clog.Errorf("Cursor error in SQL: %v", err)
			it.err = err
		}
		it.cursor.Close()
		return false
	}
	if it.table == "nodes" {
		var hash NodeHash
		err := it.cursor.Scan(&hash)
		if err != nil {
			clog.Errorf("Error nexting node iterator: %v", err)
			it.err = err
			return false
		}
		it.result = NodeHash(hash)
		return true
	}
	var q QuadHashes
	err := it.cursor.Scan(&q[0], &q[1], &q[2], &q[3])
	if err != nil {
		clog.Errorf("Error scanning sql iterator: %v", err)
		it.err = err
		return false
	}
	it.result = q
	return graph.NextLogOut(it, true)
}
示例#30
0
文件: http.go 项目: RamboWANG/cayley
func SetupRoutes(handle *graph.Handle, cfg *config.Config) {
	r := httprouter.New()
	assets := findAssetsPath()
	if clog.V(2) {
		clog.Infof("Found assets at %v", assets)
	}
	var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl")))
	templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html"))
	root := &TemplateRequestHandler{templates: templates}
	docs := &DocRequestHandler{assets: assets}
	api := &API{config: cfg, handle: handle}
	api.APIv1(r)

	//m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true}))
	//r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/"))))
	r.GET("/docs/:docpage", docs.ServeHTTP)
	r.GET("/ui/:ui_type", root.ServeHTTP)
	r.GET("/", root.ServeHTTP)
	http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/")))))
	http.Handle("/", r)
}