func main() {
rand.Seed(time.Now().UnixNano())
x.Init()
checkFlagsAndInitDirs()
ps, err := store.NewStore(*postingDir)
x.Checkf(err, "Error initializing postings store")
defer ps.Close()
if len(*schemaFile) > 0 {
err = schema.Parse(*schemaFile)
x.Checkf(err, "Error while loading schema: %s", *schemaFile)
}
// Posting will initialize index which requires schema. Hence, initialize
// schema before calling posting.Init().
posting.Init(ps)
worker.Init(ps)
x.Check(group.ParseGroupConfig(*conf))
// Setup external communication.
che := make(chan error, 1)
go setupServer(che)
go worker.StartRaftNodes(*walDir)
if err := <-che; !strings.Contains(err.Error(),
"use of closed network connection") {
log.Fatal(err)
}
}
func prepare() (dir1, dir2 string, ps *store.Store, clog *commit.Logger, rerr error) {
var err error
dir1, err = ioutil.TempDir("", "storetest_")
if err != nil {
return "", "", nil, nil, err
}
ps = new(store.Store)
ps.Init(dir1)
dir2, err = ioutil.TempDir("", "storemuts_")
if err != nil {
return dir1, "", nil, nil, err
}
clog = commit.NewLogger(dir2, "mutations", 50<<20)
clog.Init()
posting.Init(clog)
worker.Init(ps)
uid.Init(ps)
loader.Init(ps, ps)
f, err := os.Open("testdata.nq")
if err != nil {
return dir1, dir2, nil, clog, err
}
defer f.Close()
_, err = loader.HandleRdfReader(f, 0, 1)
if err != nil {
return dir1, dir2, nil, clog, err
}
return dir1, dir2, ps, clog, nil
}
func main() {
flag.Parse()
if !flag.Parsed() {
glog.Fatal("Unable to parse flags")
}
logrus.SetLevel(logrus.InfoLevel)
numCpus := runtime.NumCPU()
prev := runtime.GOMAXPROCS(numCpus)
glog.WithField("num_cpu", numCpus).
WithField("prev_maxprocs", prev).
Info("Set max procs to num cpus")
ps := new(store.Store)
ps.Init(*postingDir)
defer ps.Close()
clog := commit.NewLogger(*mutationDir, "dgraph", 50<<20)
clog.SyncEvery = 1
clog.Init()
defer clog.Close()
posting.Init(ps, clog)
http.HandleFunc("/query", queryHandler)
glog.WithField("port", *port).Info("Listening for requests...")
if err := http.ListenAndServe(":"+*port, nil); err != nil {
x.Err(glog, err).Fatal("ListenAndServe")
}
}
func main() {
flag.Parse()
if !flag.Parsed() {
glog.Fatal("Unable to parse flags")
}
if len(*cpuprofile) > 0 {
f, err := os.Create(*cpuprofile)
if err != nil {
glog.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
logrus.SetLevel(logrus.InfoLevel)
numCpus := *numcpu
prevProcs := runtime.GOMAXPROCS(numCpus)
glog.WithField("num_cpus", numCpus).
WithField("prev_maxprocs", prevProcs).
Info("Set max procs to num cpus")
if len(*rdfGzips) == 0 {
glog.Fatal("No RDF GZIP files specified")
}
ps := new(store.Store)
ps.Init(*uidDir)
defer ps.Close()
posting.Init(ps, nil)
files := strings.Split(*rdfGzips, ",")
for _, path := range files {
if len(path) == 0 {
continue
}
glog.WithField("path", path).Info("Handling...")
f, err := os.Open(path)
if err != nil {
glog.WithError(err).Fatal("Unable to open rdf file.")
}
r, err := gzip.NewReader(f)
if err != nil {
glog.WithError(err).Fatal("Unable to create gzip reader.")
}
count, err := loader.HandleRdfReaderWhileAssign(r, *mod)
if err != nil {
glog.WithError(err).Fatal("While handling rdf reader.")
}
glog.WithField("count", count).Info("RDFs parsed")
r.Close()
f.Close()
}
glog.Info("Calling merge lists")
posting.MergeLists(100 * numCpus) // 100 per core.
}
func createTestStore(t *testing.T) (string, *store.Store) {
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
ps, err := store.NewStore(dir)
require.NoError(t, err)
schema.ParseBytes([]byte(`scalar geometry:geo @index`))
posting.Init(ps)
return dir, ps
}
func TestGenerateGroup(t *testing.T) {
dir, err := ioutil.TempDir("", "store3")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
r := bytes.NewReader([]byte("default: fp % 3"))
require.NoError(t, group.ParseConfig(r), "Unable to parse config.")
ps, err := store.NewStore(dir)
if err != nil {
t.Fatal(err)
}
defer ps.Close()
posting.Init(ps)
Init(ps)
require.Equal(t, uint32(0), group.BelongsTo("pred0"))
writePLs(t, "pred0", 33, 1, ps)
require.Equal(t, uint32(1), group.BelongsTo("p1"))
writePLs(t, "p1", 34, 1, ps)
require.Equal(t, uint32(2), group.BelongsTo("pr2"))
writePLs(t, "pr2", 35, 1, ps)
time.Sleep(time.Second)
g, err := generateGroup(0)
if err != nil {
t.Error(err)
}
require.Equal(t, 33, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("pred0", uint64(i)), k.Key)
}
g, err = generateGroup(1)
if err != nil {
t.Error(err)
}
require.Equal(t, 34, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("p1", uint64(i)), k.Key)
}
g, err = generateGroup(2)
if err != nil {
t.Error(err)
}
require.Equal(t, 35, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("pr2", uint64(i)), k.Key)
}
}
func initTest(t *testing.T, schemaStr string) (string, *store.Store) {
schema.ParseBytes([]byte(schemaStr))
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
ps, err := store.NewStore(dir)
require.NoError(t, err)
posting.Init(ps)
populateGraph(t)
time.Sleep(200 * time.Millisecond) // Let the index process jobs from channel.
return dir, ps
}
func TestQuery(t *testing.T) {
var numInstances uint64 = 2
mod := math.MaxUint64 / numInstances
minIdx0 := 0 * mod
minIdx1 := 1 * mod
logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
posting.Init(clog)
uid.Init(ps)
list := []string{"alice", "bob", "mallory", "ash", "man", "dgraph"}
for _, str := range list {
if farm.Fingerprint64([]byte(str))%numInstances == 0 {
uid, err := rdf.GetUid(str, 0, numInstances)
if uid < minIdx0 || uid > minIdx0+mod-1 {
t.Error("Not the correct UID", err)
}
t.Logf("Instance-0 Correct UID", str, uid)
} else {
uid, err := rdf.GetUid(str, 1, numInstances)
if uid < minIdx1 || uid > minIdx1+mod-1 {
t.Error("Not the correct UID", err)
}
t.Logf("Instance-1 Correct UID", str, uid)
}
}
}
func TestNewGraph(t *testing.T) {
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
gq := &gql.GraphQuery{
UID: 101,
Attr: "me",
}
ps, err := store.NewStore(dir)
require.NoError(t, err)
posting.Init(ps)
ctx := context.Background()
sg, err := newGraph(ctx, gq)
require.NoError(t, err)
require.EqualValues(t,
[][]uint64{
[]uint64{101},
}, algo.ToUintsListForTest(sg.uidMatrix))
}
func prepare() (dir1, dir2 string, ps *store.Store, rerr error) {
var err error
dir1, err = ioutil.TempDir("", "storetest_")
if err != nil {
return "", "", nil, err
}
ps, err = store.NewStore(dir1)
if err != nil {
return "", "", nil, err
}
dir2, err = ioutil.TempDir("", "wal_")
if err != nil {
return dir1, "", nil, err
}
posting.Init(ps)
loader.Init(ps)
group.ParseGroupConfig("groups.conf")
worker.StartRaftNodes(dir2)
{
// Then load data.
f, err := os.Open("testdata.nq")
if err != nil {
return dir1, dir2, nil, err
}
gm := map[uint32]bool{
0: true,
}
_, err = loader.LoadEdges(f, gm)
f.Close()
if err != nil {
return dir1, dir2, nil, err
}
}
return dir1, dir2, ps, nil
}
func populateGraph(t *testing.T) (string, string, *store.Store) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
ps, err := store.NewStore(dir)
require.NoError(t, err)
schema.ParseBytes([]byte(schemaStr))
posting.Init(ps)
worker.Init(ps)
group.ParseGroupConfig("")
dir2, err := ioutil.TempDir("", "wal_")
require.NoError(t, err)
worker.StartRaftNodes(dir2)
// So, user we're interested in has uid: 1.
// She has 5 friends: 23, 24, 25, 31, and 101
addEdgeToUID(t, ps, "friend", 1, 23)
addEdgeToUID(t, ps, "friend", 1, 24)
addEdgeToUID(t, ps, "friend", 1, 25)
addEdgeToUID(t, ps, "friend", 1, 31)
addEdgeToUID(t, ps, "friend", 1, 101)
// Now let's add a few properties for the main user.
addEdgeToValue(t, ps, "name", 1, "Michonne")
addEdgeToValue(t, ps, "gender", 1, "female")
var coord types.Geo
err = coord.UnmarshalText([]byte("{\"Type\":\"Point\", \"Coordinates\":[1.1,2.0]}"))
require.NoError(t, err)
gData, err := coord.MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "loc", 1, types.GeoID, gData)
data, err := types.Int32(15).MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "age", 1, types.Int32ID, data)
addEdgeToValue(t, ps, "address", 1, "31, 32 street, Jupiter")
data, err = types.Bool(true).MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "alive", 1, types.BoolID, data)
addEdgeToValue(t, ps, "age", 1, "38")
addEdgeToValue(t, ps, "survival_rate", 1, "98.99")
addEdgeToValue(t, ps, "sword_present", 1, "true")
addEdgeToValue(t, ps, "_xid_", 1, "mich")
// Now let's add a name for each of the friends, except 101.
addEdgeToTypedValue(t, ps, "name", 23, types.StringID, []byte("Rick Grimes"))
addEdgeToValue(t, ps, "age", 23, "15")
err = coord.UnmarshalText([]byte(`{"Type":"Polygon", "Coordinates":[[[0.0,0.0], [2.0,0.0], [2.0, 2.0], [0.0, 2.0]]]}`))
require.NoError(t, err)
gData, err = coord.MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "loc", 23, types.GeoID, gData)
addEdgeToValue(t, ps, "address", 23, "21, mark street, Mars")
addEdgeToValue(t, ps, "name", 24, "Glenn Rhee")
addEdgeToValue(t, ps, "name", 25, "Daryl Dixon")
addEdgeToValue(t, ps, "name", 31, "Andrea")
addEdgeToValue(t, ps, "dob", 23, "1910-01-02")
addEdgeToValue(t, ps, "dob", 24, "1909-05-05")
addEdgeToValue(t, ps, "dob", 25, "1909-01-10")
addEdgeToValue(t, ps, "dob", 31, "1901-01-15")
return dir, dir2, ps
}
func TestGetOrAssign(t *testing.T) {
logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
posting.Init(ps, clog)
var u1, u2 uint64
{
uid, err := GetOrAssign("externalid0")
if err != nil {
t.Error(err)
}
t.Logf("Found uid: [%x]", uid)
u1 = uid
}
{
uid, err := GetOrAssign("externalid1")
if err != nil {
t.Error(err)
}
t.Logf("Found uid: [%x]", uid)
u2 = uid
}
if u1 == u2 {
t.Error("Uid1 and Uid2 shouldn't be the same")
}
// return
{
uid, err := GetOrAssign("externalid0")
if err != nil {
t.Error(err)
}
t.Logf("Found uid: [%x]", uid)
if u1 != uid {
t.Error("Uid should be the same.")
}
}
// return
{
xid, err := ExternalId(u1)
if err != nil {
t.Error(err)
}
if xid != "externalid0" {
t.Errorf("Expected externalid0. Found: [%q]", xid)
}
}
return
{
xid, err := ExternalId(u2)
if err != nil {
t.Error(err)
}
if xid != "externalid1" {
t.Errorf("Expected externalid1. Found: [%q]", xid)
}
}
}
func TestProcessTask(t *testing.T) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
posting.Init(clog)
Init(ps)
edge := x.DirectedEdge{
ValueId: 23,
Source: "author0",
Timestamp: time.Now(),
}
addEdge(t, edge, posting.GetOrCreate(posting.Key(10, "friend"), ps))
addEdge(t, edge, posting.GetOrCreate(posting.Key(11, "friend"), ps))
addEdge(t, edge, posting.GetOrCreate(posting.Key(12, "friend"), ps))
edge.ValueId = 25
addEdge(t, edge, posting.GetOrCreate(posting.Key(12, "friend"), ps))
edge.ValueId = 26
addEdge(t, edge, posting.GetOrCreate(posting.Key(12, "friend"), ps))
edge.ValueId = 31
addEdge(t, edge, posting.GetOrCreate(posting.Key(10, "friend"), ps))
addEdge(t, edge, posting.GetOrCreate(posting.Key(12, "friend"), ps))
edge.Value = "photon"
addEdge(t, edge, posting.GetOrCreate(posting.Key(12, "friend"), ps))
query := NewQuery("friend", []uint64{10, 11, 12})
result, err := ProcessTask(query)
if err != nil {
t.Error(err)
}
ro := flatbuffers.GetUOffsetT(result)
r := new(task.Result)
r.Init(result, ro)
if r.UidmatrixLength() != 3 {
t.Errorf("Expected 3. Got uidmatrix length: %v", r.UidmatrixLength())
}
if err := check(r, 0, []uint64{23, 31}); err != nil {
t.Error(err)
}
if err := check(r, 1, []uint64{23}); err != nil {
t.Error(err)
}
if err := check(r, 2, []uint64{23, 25, 26, 31}); err != nil {
t.Error(err)
}
if r.ValuesLength() != 3 {
t.Errorf("Expected 3. Got values length: %v", r.ValuesLength())
}
var tval task.Value
if ok := r.Values(&tval, 0); !ok {
t.Errorf("Unable to retrieve value")
}
if tval.ValLength() != 1 ||
tval.ValBytes()[0] != 0x00 {
t.Errorf("Invalid byte value at index 0")
}
if ok := r.Values(&tval, 1); !ok {
t.Errorf("Unable to retrieve value")
}
if tval.ValLength() != 1 ||
tval.ValBytes()[0] != 0x00 {
t.Errorf("Invalid byte value at index 0")
}
if ok := r.Values(&tval, 2); !ok {
t.Errorf("Unable to retrieve value")
}
var iout interface{}
if err := posting.ParseValue(&iout, tval.ValBytes()); err != nil {
t.Error(err)
}
v := iout.(string)
if v != "photon" {
t.Errorf("Expected photon. Got: %q", v)
}
}
func populateGraph(t *testing.T) (string, *store.Store) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return "", nil
}
ps := new(store.Store)
ps.Init(dir)
worker.Init(ps)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
posting.Init(clog)
// So, user we're interested in has uid: 1.
// She has 4 friends: 23, 24, 25, 31, and 101
edge := x.DirectedEdge{
ValueId: 23,
Source: "testing",
Timestamp: time.Now(),
}
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 24
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 25
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 31
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 101
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
// Now let's add a few properties for the main user.
edge.Value = "Michonne"
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "name"), ps))
edge.Value = "female"
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "gender"), ps))
edge.Value = "alive"
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "status"), ps))
// Now let's add a name for each of the friends, except 101.
edge.Value = "Rick Grimes"
addEdge(t, edge, posting.GetOrCreate(posting.Key(23, "name"), ps))
edge.Value = "Glenn Rhee"
addEdge(t, edge, posting.GetOrCreate(posting.Key(24, "name"), ps))
edge.Value = "Daryl Dixon"
addEdge(t, edge, posting.GetOrCreate(posting.Key(25, "name"), ps))
edge.Value = "Andrea"
addEdge(t, edge, posting.GetOrCreate(posting.Key(31, "name"), ps))
return dir, ps
}