func Load(ts graph.TripleStore, cfg *config.Config, dec quad.Unmarshaler) error { bulker, canBulk := ts.(graph.BulkLoader) if canBulk { switch err := bulker.BulkLoad(dec); err { case nil: return nil case graph.ErrCannotBulkLoad: // Try individual loading. default: return err } } block := make([]quad.Quad, 0, cfg.LoadSize) for { t, err := dec.Unmarshal() if err != nil { if err == io.EOF { break } return err } block = append(block, t) if len(block) == cap(block) { ts.AddTripleSet(block) block = block[:0] } } ts.AddTripleSet(block) return nil }
func checkIteratorContains(ts graph.TripleStore, it graph.Iterator, expected []string, t *testing.T) { var actual []string actual = nil for { val, ok := it.Next() if !ok { break } actual = append(actual, ts.GetNameFor(val)) } actualSet := actual[:] for _, a := range expected { found := false for j, b := range actualSet { if a == b { actualSet = append(actualSet[:j], actualSet[j+1:]...) found = true break } } if !found { t.Error("Couldn't find", a, "in actual output.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet) return } } if len(actualSet) != 0 { t.Error("Actual output has more than expected.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet) } }
func NewSingleReplication(ts graph.TripleStore, opts graph.Options) (graph.QuadWriter, error) { horizon := ts.Horizon() rep := &Single{nextID: horizon + 1, ts: ts} if horizon <= 0 { rep.nextID = 1 } return rep, nil }
func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string { var res []string for graph.Next(it) { res = append(res, qs.NameOf(it.Result())) } sort.Strings(res) return res }
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []quad.Quad { var res ordered for graph.Next(it) { res = append(res, qs.Quad(it.Result())) } sort.Sort(res) return res }
func extractTripleFromIterator(ts graph.TripleStore, it graph.Iterator) []string { var output []string for { val, ok := it.Next() if !ok { break } output = append(output, ts.GetTriple(val).ToString()) } return output }
func extractValuesFromIterator(ts graph.TripleStore, it graph.Iterator) []string { var output []string for { val, ok := it.Next() if !ok { break } output = append(output, ts.GetNameFor(val)) } return output }
func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string { var res []string for { val, ok := graph.Next(it) if !ok { break } res = append(res, qs.NameOf(val)) } sort.Strings(res) return res }
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad { var res ordered for { val, ok := graph.Next(it) if !ok { break } res = append(res, qs.Quad(val)) } sort.Sort(res) return res }
func iteratedTriples(ts graph.TripleStore, it graph.Iterator) []*graph.Triple { var res ordered for { val, ok := it.Next() if !ok { break } res = append(res, ts.Triple(val)) } sort.Sort(res) return res }
func LoadTriplesInto(tChan chan *graph.Triple, ts graph.TripleStore, loadSize int) { tripleblock := make([]*graph.Triple, loadSize) i := 0 for t := range tChan { tripleblock[i] = t i++ if i == loadSize { ts.AddTripleSet(tripleblock) i = 0 } } ts.AddTripleSet(tripleblock[0:i]) }
func LoadTriplesFromFileInto(ts graph.TripleStore, filename string, loadSize int) { tChan := make(chan *graph.Triple) go ReadTriplesFromFile(tChan, filename) tripleblock := make([]*graph.Triple, loadSize) i := 0 for t := range tChan { tripleblock[i] = t i++ if i == loadSize { ts.AddTripleSet(tripleblock) i = 0 } } ts.AddTripleSet(tripleblock[0:i]) }
func Load(ts graph.TripleStore, cfg *config.Config, path string) error { f, err := os.Open(path) if err != nil { return fmt.Errorf("could not open file %q: %v", path, err) } defer f.Close() r, err := decompressor(f) if err != nil { glog.Fatalln(err) } dec := cquads.NewDecoder(r) bulker, canBulk := ts.(graph.BulkLoader) if canBulk { err = bulker.BulkLoad(dec) if err == nil { return nil } if err == graph.ErrCannotBulkLoad { err = nil } } if err != nil { return err } block := make([]*quad.Quad, 0, cfg.LoadSize) for { t, err := dec.Unmarshal() if err != nil { if err == io.EOF { break } return err } block = append(block, t) if len(block) == cap(block) { ts.AddTripleSet(block) block = block[:0] } } ts.AddTripleSet(block) return nil }
func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA { and := NewAnd() obj := ts.FixedIterator() obj.Add(ts.ValueOf(target)) obj.Tagger().Add(tag) and.AddSubIterator(NewLinksTo(ts, obj, quad.Object)) pred := ts.FixedIterator() pred.Add(ts.ValueOf("status")) and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate)) return NewHasA(ts, and, quad.Subject) }
func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Iterator, isReverse bool) graph.Iterator { argList, _ := obj.Get("_gremlin_values") if argList.Class() != "GoArray" { glog.Errorln("How is arglist not an array? Return nothing.", argList.Class()) return graph.NewNullIterator() } argArray := argList.Object() lengthVal, _ := argArray.Get("length") length, _ := lengthVal.ToInteger() var predicateNodeIterator graph.Iterator if length == 0 { predicateNodeIterator = ts.GetNodesAllIterator() } else { zero, _ := argArray.Get("0") predicateNodeIterator = buildIteratorFromValue(zero, ts) } if length >= 2 { var tags []string one, _ := argArray.Get("1") if one.IsString() { s, _ := one.ToString() tags = append(tags, s) } else if one.Class() == "Array" { tags = makeListOfStringsFromArrayValue(one.Object()) } for _, tag := range tags { predicateNodeIterator.AddTag(tag) } } in, out := graph.Subject, graph.Object if isReverse { in, out = out, in } lto := graph.NewLinksToIterator(ts, base, in) and := graph.NewAndIterator() and.AddSubIterator(graph.NewLinksToIterator(ts, predicateNodeIterator, graph.Predicate)) and.AddSubIterator(lto) return graph.NewHasaIterator(ts, and, out) }
func buildHasaWithTag(ts graph.TripleStore, tag string, target string) *HasA { fixed_obj := ts.FixedIterator() fixed_pred := ts.FixedIterator() fixed_obj.Add(ts.ValueOf(target)) fixed_pred.Add(ts.ValueOf("status")) fixed_obj.AddTag(tag) lto1 := NewLinksTo(ts, fixed_obj, graph.Object) lto2 := NewLinksTo(ts, fixed_pred, graph.Predicate) and := NewAnd() and.AddSubIterator(lto1) and.AddSubIterator(lto2) hasa := NewHasA(ts, and, graph.Subject) return hasa }
func main() { // No command? It's time for usage. if len(os.Args) == 1 { Usage() os.Exit(1) } cmd := os.Args[1] newargs := make([]string, 0) newargs = append(newargs, os.Args[0]) newargs = append(newargs, os.Args[2:]...) os.Args = newargs flag.Parse() var ts graph.TripleStore cfg := config.ParseConfigFromFlagsAndFile(*configFile) if os.Getenv("GOMAXPROCS") == "" { runtime.GOMAXPROCS(runtime.NumCPU()) glog.Infoln("Setting GOMAXPROCS to", runtime.NumCPU()) } else { glog.Infoln("GOMAXPROCS currently", os.Getenv("GOMAXPROCS"), " -- not adjusting") } switch cmd { case "init": db.Init(cfg, *tripleFile) case "load": ts = db.Open(cfg) db.Load(ts, cfg, *tripleFile, false) ts.Close() case "repl": ts = db.Open(cfg) db.Repl(ts, *queryLanguage, cfg) ts.Close() case "http": ts = db.Open(cfg) http.Serve(ts, cfg) ts.Close() default: fmt.Println("No command", cmd) flag.Usage() } }
func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error { var ses query.Session switch queryLanguage { case "sexp": ses = sexp.NewSession(ts) case "mql": ses = mql.NewSession(ts) case "gremlin": fallthrough default: ses = gremlin.NewSession(ts, cfg.Timeout, true) } buf := bufio.NewReader(os.Stdin) var line []byte for { if len(line) == 0 { fmt.Print("cayley> ") } else { fmt.Print("... ") } l, prefix, err := buf.ReadLine() if err == io.EOF { if len(line) != 0 { line = line[:0] } else { return nil } } if err != nil { line = line[:0] } if prefix { return errors.New("line too long") } line = append(line, l...) if len(line) == 0 { continue } line = bytes.TrimSpace(line) if len(line) == 0 || line[0] == '#' { line = line[:0] continue } if bytes.HasPrefix(line, []byte(":debug")) { ses.ToggleDebug() fmt.Println("Debug Toggled") line = line[:0] continue } if bytes.HasPrefix(line, []byte(":a")) { var tripleStmt = line[3:] triple, err := cquads.Parse(string(tripleStmt)) if !triple.IsValid() { if err != nil { fmt.Printf("not a valid triple: %v\n", err) } line = line[:0] continue } ts.AddTriple(triple) line = line[:0] continue } if bytes.HasPrefix(line, []byte(":d")) { var tripleStmt = line[3:] triple, err := cquads.Parse(string(tripleStmt)) if !triple.IsValid() { if err != nil { fmt.Printf("not a valid triple: %v\n", err) } line = line[:0] continue } ts.RemoveTriple(triple) line = line[:0] continue } result, err := ses.InputParses(string(line)) switch result { case query.Parsed: Run(string(line), ses) line = line[:0] case query.ParseFail: fmt.Println("Error: ", err) line = line[:0] case query.ParseMore: } } }
func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator { if val.IsNull() || val.IsUndefined() { return ts.GetNodesAllIterator() } if val.IsPrimitive() { thing, _ := val.Export() switch v := thing.(type) { case string: it := ts.MakeFixed() it.AddValue(ts.GetIdFor(v)) return it default: glog.Errorln("Trying to build unknown primitive value.") } } switch val.Class() { case "Object": return buildIteratorTree(val.Object(), ts) case "Array": // Had better be an array of strings strings := makeListOfStringsFromArrayValue(val.Object()) it := ts.MakeFixed() for _, x := range strings { it.AddValue(ts.GetIdFor(x)) } return it case "Number": fallthrough case "Boolean": fallthrough case "Date": fallthrough case "String": it := ts.MakeFixed() str, _ := val.ToString() it.AddValue(ts.GetIdFor(str)) return it default: glog.Errorln("Trying to handle unsupported Javascript value.") return graph.NewNullIterator() } }
func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.Iterator) graph.Iterator { var it graph.Iterator it = base // TODO: Better error handling kindVal, _ := obj.Get("_gremlin_type") stringArgs := getStringArgs(obj) var subIt graph.Iterator prevVal, _ := obj.Get("_gremlin_prev") if !prevVal.IsObject() { subIt = base } else { subIt = buildIteratorTreeHelper(prevVal.Object(), ts, base) } kind, _ := kindVal.ToString() switch kind { case "vertex": if len(stringArgs) == 0 { it = ts.GetNodesAllIterator() } else { fixed := ts.MakeFixed() for _, name := range stringArgs { fixed.AddValue(ts.GetIdFor(name)) } it = fixed } case "tag": it = subIt for _, tag := range stringArgs { it.AddTag(tag) } case "save": all := ts.GetNodesAllIterator() if len(stringArgs) > 2 || len(stringArgs) == 0 { return graph.NewNullIterator() } if len(stringArgs) == 2 { all.AddTag(stringArgs[1]) } else { all.AddTag(stringArgs[0]) } predFixed := ts.MakeFixed() predFixed.AddValue(ts.GetIdFor(stringArgs[0])) subAnd := graph.NewAndIterator() subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, graph.Object)) hasa := graph.NewHasaIterator(ts, subAnd, graph.Subject) and := graph.NewAndIterator() and.AddSubIterator(hasa) and.AddSubIterator(subIt) it = and case "saver": all := ts.GetNodesAllIterator() if len(stringArgs) > 2 || len(stringArgs) == 0 { return graph.NewNullIterator() } if len(stringArgs) == 2 { all.AddTag(stringArgs[1]) } else { all.AddTag(stringArgs[0]) } predFixed := ts.MakeFixed() predFixed.AddValue(ts.GetIdFor(stringArgs[0])) subAnd := graph.NewAndIterator() subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, graph.Subject)) hasa := graph.NewHasaIterator(ts, subAnd, graph.Object) and := graph.NewAndIterator() and.AddSubIterator(hasa) and.AddSubIterator(subIt) it = and case "has": fixed := ts.MakeFixed() if len(stringArgs) < 2 { return graph.NewNullIterator() } for _, name := range stringArgs[1:] { fixed.AddValue(ts.GetIdFor(name)) } predFixed := ts.MakeFixed() predFixed.AddValue(ts.GetIdFor(stringArgs[0])) subAnd := graph.NewAndIterator() subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(graph.NewLinksToIterator(ts, fixed, graph.Object)) hasa := graph.NewHasaIterator(ts, subAnd, graph.Subject) and := graph.NewAndIterator() and.AddSubIterator(hasa) and.AddSubIterator(subIt) it = and case "morphism": it = base case "and": arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") if !isVertexChain(firstArg.Object()) { return graph.NewNullIterator() } argIt := buildIteratorTree(firstArg.Object(), ts) and := graph.NewAndIterator() and.AddSubIterator(subIt) and.AddSubIterator(argIt) it = and case "back": arg, _ := obj.Get("_gremlin_back_chain") argIt := buildIteratorTree(arg.Object(), ts) and := graph.NewAndIterator() and.AddSubIterator(subIt) and.AddSubIterator(argIt) it = and case "is": fixed := ts.MakeFixed() for _, name := range stringArgs { fixed.AddValue(ts.GetIdFor(name)) } and := graph.NewAndIterator() and.AddSubIterator(fixed) and.AddSubIterator(subIt) it = and case "or": arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") if !isVertexChain(firstArg.Object()) { return graph.NewNullIterator() } argIt := buildIteratorTree(firstArg.Object(), ts) or := graph.NewOrIterator() or.AddSubIterator(subIt) or.AddSubIterator(argIt) it = or case "both": // Hardly the most efficient pattern, but the most general. // Worth looking into an Optimize() optimization here. clone := subIt.Clone() it1 := buildInOutIterator(obj, ts, subIt, false) it2 := buildInOutIterator(obj, ts, clone, true) or := graph.NewOrIterator() or.AddSubIterator(it1) or.AddSubIterator(it2) it = or case "out": it = buildInOutIterator(obj, ts, subIt, false) case "follow": // Follow a morphism arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") if isVertexChain(firstArg.Object()) { return graph.NewNullIterator() } it = buildIteratorTreeHelper(firstArg.Object(), ts, subIt) case "followr": // Follow a morphism arg, _ := obj.Get("_gremlin_followr") if isVertexChain(arg.Object()) { return graph.NewNullIterator() } it = buildIteratorTreeHelper(arg.Object(), ts, subIt) case "in": it = buildInOutIterator(obj, ts, subIt, true) } return it }
func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Iterator { switch tree.Name { case "Start": return buildIteratorTree(tree.Children[0], ts) case "NodeIdentifier": var out graph.Iterator nodeID := getIdentString(tree) if tree.Children[0].Name == "Variable" { allIt := ts.GetNodesAllIterator() allIt.AddTag(nodeID) out = allIt } else { n := nodeID if tree.Children[0].Children[0].Name == "ColonIdentifier" { n = nodeID[1:] } fixed := ts.MakeFixed() fixed.AddValue(ts.GetIdFor(n)) out = fixed } return out case "PredIdentifier": i := 0 if tree.Children[0].Name == "Reverse" { //Taken care of below i++ } it := buildIteratorTree(tree.Children[i], ts) lto := graph.NewLinksToIterator(ts, it, "p") return lto case "RootConstraint": constraintCount := 0 and := graph.NewAndIterator() for _, c := range tree.Children { switch c.Name { case "NodeIdentifier": fallthrough case "Constraint": it := buildIteratorTree(c, ts) and.AddSubIterator(it) constraintCount++ continue default: continue } } return and case "Constraint": var hasa *graph.HasaIterator topLevelDir := "s" subItDir := "o" subAnd := graph.NewAndIterator() isOptional := false for _, c := range tree.Children { switch c.Name { case "PredIdentifier": if c.Children[0].Name == "Reverse" { topLevelDir = "o" subItDir = "s" } it := buildIteratorTree(c, ts) subAnd.AddSubIterator(it) continue case "PredicateKeyword": switch c.Children[0].Name { case "OptionalKeyword": isOptional = true } case "NodeIdentifier": fallthrough case "RootConstraint": it := buildIteratorTree(c, ts) l := graph.NewLinksToIterator(ts, it, subItDir) subAnd.AddSubIterator(l) continue default: continue } } hasa = graph.NewHasaIterator(ts, subAnd, topLevelDir) if isOptional { optional := graph.NewOptionalIterator(hasa) return optional } return hasa default: return &graph.NullIterator{} } panic("Not reached") }
func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) { var ses graph.Session switch queryLanguage { case "sexp": ses = sexp.NewSession(ts) case "mql": ses = mql.NewSession(ts) case "gremlin": fallthrough default: ses = gremlin.NewSession(ts, cfg.GremlinTimeout, true) } inputBf := bufio.NewReader(os.Stdin) line := "" for { if line == "" { fmt.Print("cayley> ") } else { fmt.Print("... ") } l, pre, err := inputBf.ReadLine() if err == io.EOF { if line != "" { line = "" } else { break } } if err != nil { line = "" } if pre { panic("Line too long") } line += string(l) if line == "" { continue } if strings.HasPrefix(line, ":debug") { ses.ToggleDebug() fmt.Println("Debug Toggled") line = "" continue } if strings.HasPrefix(line, ":a") { var tripleStmt = line[3:] triple := nquads.Parse(tripleStmt) if triple == nil { fmt.Println("Not a valid triple.") line = "" continue } ts.AddTriple(triple) line = "" continue } if strings.HasPrefix(line, ":d") { var tripleStmt = line[3:] triple := nquads.Parse(tripleStmt) if triple == nil { fmt.Println("Not a valid triple.") line = "" continue } ts.RemoveTriple(triple) line = "" continue } result, err := ses.InputParses(line) switch result { case graph.Parsed: Run(line, ses) line = "" case graph.ParseFail: fmt.Println("Error: ", err) line = "" case graph.ParseMore: default: } } }
func main() { // No command? It's time for usage. if len(os.Args) == 1 { Usage() os.Exit(1) } cmd := os.Args[1] var newargs []string newargs = append(newargs, os.Args[0]) newargs = append(newargs, os.Args[2:]...) os.Args = newargs flag.Parse() var buildString string if VERSION != "" { buildString = fmt.Sprint("Cayley ", VERSION, " built ", BUILD_DATE) glog.Infoln(buildString) } cfg := config.ParseConfigFromFlagsAndFile(*configFile) if os.Getenv("GOMAXPROCS") == "" { runtime.GOMAXPROCS(runtime.NumCPU()) glog.Infoln("Setting GOMAXPROCS to", runtime.NumCPU()) } else { glog.Infoln("GOMAXPROCS currently", os.Getenv("GOMAXPROCS"), " -- not adjusting") } var ( ts graph.TripleStore err error ) switch cmd { case "version": if VERSION != "" { fmt.Println(buildString) } else { fmt.Println("Cayley snapshot") } os.Exit(0) case "init": err = db.Init(cfg, *tripleFile) case "load": ts, err = db.Open(cfg) if err != nil { break } err = db.Load(ts, cfg, *tripleFile) if err != nil { break } ts.Close() case "repl": ts, err = db.Open(cfg) if err != nil { break } err = db.Repl(ts, *queryLanguage, cfg) if err != nil { break } ts.Close() case "http": ts, err = db.Open(cfg) if err != nil { break } http.Serve(ts, cfg) ts.Close() default: fmt.Println("No command", cmd) flag.Usage() } if err != nil { glog.Fatalln(err) } }
func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error { var ses query.Session switch queryLanguage { case "sexp": ses = sexp.NewSession(ts) case "mql": ses = mql.NewSession(ts) case "gremlin": fallthrough default: ses = gremlin.NewSession(ts, cfg.Timeout, true) } term, err := terminal(history) if os.IsNotExist(err) { fmt.Printf("creating new history file: %q\n", history) } defer persist(term, history) var ( prompt = ps1 code string ) for { if len(code) == 0 { prompt = ps1 } else { prompt = ps2 } line, err := term.Prompt(prompt) if err != nil { if err == io.EOF { return nil } return err } term.AppendHistory(line) line = strings.TrimSpace(line) if len(line) == 0 || line[0] == '#' { continue } if code == "" { switch { case strings.HasPrefix(line, ":debug"): ses.ToggleDebug() fmt.Println("Debug Toggled") continue case strings.HasPrefix(line, ":a"): triple, err := cquads.Parse(line[3:]) if !triple.IsValid() { if err != nil { fmt.Printf("not a valid triple: %v\n", err) } continue } ts.AddTriple(triple) continue case strings.HasPrefix(line, ":d"): triple, err := cquads.Parse(line[3:]) if !triple.IsValid() { if err != nil { fmt.Printf("not a valid triple: %v\n", err) } continue } ts.RemoveTriple(triple) continue } } code += line result, err := ses.InputParses(code) switch result { case query.Parsed: Run(code, ses) code = "" case query.ParseFail: fmt.Println("Error: ", err) code = "" case query.ParseMore: } } }