func TestAddMutation_Value(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
log.Println("Init successful.")
edge := &task.DirectedEdge{
Value: []byte("oh hey there"),
Label: "new-testing",
}
addMutation(t, ol, edge, Set)
checkValue(t, ol, "oh hey there")
// Run the same check after committing.
_, err = ol.CommitIfDirty(context.Background())
require.NoError(t, err)
checkValue(t, ol, "oh hey there")
// The value made it to the posting list. Changing it now.
edge.Value = []byte(strconv.Itoa(119))
addMutation(t, ol, edge, Set)
checkValue(t, ol, "119")
}
func BenchmarkAddMutations(b *testing.B) {
// logrus.SetLevel(logrus.DebugLevel)
key := x.DataKey("name", 1)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
b.Error(err)
return
}
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
if err != nil {
b.Error(err)
return
}
l := getNew(key, ps)
b.ResetTimer()
ctx := context.Background()
for i := 0; i < b.N; i++ {
if err != nil {
b.Error(err)
return
}
edge := &task.DirectedEdge{
ValueId: uint64(rand.Intn(b.N) + 1),
Label: "testing",
}
if _, err := l.AddMutation(ctx, edge, Set); err != nil {
b.Error(err)
}
}
}
func main() {
rand.Seed(time.Now().UnixNano())
x.Init()
checkFlagsAndInitDirs()
ps, err := store.NewStore(*postingDir)
x.Checkf(err, "Error initializing postings store")
defer ps.Close()
if len(*schemaFile) > 0 {
err = schema.Parse(*schemaFile)
x.Checkf(err, "Error while loading schema: %s", *schemaFile)
}
// Posting will initialize index which requires schema. Hence, initialize
// schema before calling posting.Init().
posting.Init(ps)
worker.Init(ps)
x.Check(group.ParseGroupConfig(*conf))
// Setup external communication.
che := make(chan error, 1)
go setupServer(che)
go worker.StartRaftNodes(*walDir)
if err := <-che; !strings.Contains(err.Error(),
"use of closed network connection") {
log.Fatal(err)
}
}
func TestAfterUIDCount2(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
ol := getNew(key, ps)
// Set value to cars and merge to RocksDB.
edge := &task.DirectedEdge{
Label: "jchiu",
}
for i := 100; i < 300; i++ {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 200, ol.Length(0))
require.EqualValues(t, 100, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Re-insert 1/4 of the edges. Counts should not change.
edge.Label = "somethingelse"
for i := 100; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 200, ol.Length(0))
require.EqualValues(t, 100, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
}
func TestAddMutation_jchiu3(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Set value to cars and merge to RocksDB.
edge := &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
require.Equal(t, 1, ol.Length(0))
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "cars")
// Del a value cars and but don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
// Set value to newcars, but don't merge yet.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "newcars")
// Del a value othercars and but don't merge.
edge = &task.DirectedEdge{
Value: []byte("othercars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.NotEqual(t, 0, ol.Length(0))
checkValue(t, ol, "newcars")
// Del a value newcars and but don't merge.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
}
func TestAddMutation_gru2(t *testing.T) {
key := x.DataKey("question.tag", 0x01)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
{
// Set two tag ids and merge.
edge := &task.DirectedEdge{
ValueId: 0x02,
Label: "gru",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 0x03,
Label: "gru",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
{
// Lets set a new tag and delete the two older ones.
edge := &task.DirectedEdge{
ValueId: 0x02,
Label: "gru",
}
addMutation(t, ol, edge, Del)
edge = &task.DirectedEdge{
ValueId: 0x03,
Label: "gru",
}
addMutation(t, ol, edge, Del)
edge = &task.DirectedEdge{
ValueId: 0x04,
Label: "gru",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
// Posting list should just have the new tag.
uids := []uint64{0x04}
require.Equal(t, uids, listToArray(t, 0, ol))
}
func createTestStore(t *testing.T) (string, *store.Store) {
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
ps, err := store.NewStore(dir)
require.NoError(t, err)
schema.ParseBytes([]byte(`scalar geometry:geo @index`))
posting.Init(ps)
return dir, ps
}
func TestGenerateGroup(t *testing.T) {
dir, err := ioutil.TempDir("", "store3")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
r := bytes.NewReader([]byte("default: fp % 3"))
require.NoError(t, group.ParseConfig(r), "Unable to parse config.")
ps, err := store.NewStore(dir)
if err != nil {
t.Fatal(err)
}
defer ps.Close()
posting.Init(ps)
Init(ps)
require.Equal(t, uint32(0), group.BelongsTo("pred0"))
writePLs(t, "pred0", 33, 1, ps)
require.Equal(t, uint32(1), group.BelongsTo("p1"))
writePLs(t, "p1", 34, 1, ps)
require.Equal(t, uint32(2), group.BelongsTo("pr2"))
writePLs(t, "pr2", 35, 1, ps)
time.Sleep(time.Second)
g, err := generateGroup(0)
if err != nil {
t.Error(err)
}
require.Equal(t, 33, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("pred0", uint64(i)), k.Key)
}
g, err = generateGroup(1)
if err != nil {
t.Error(err)
}
require.Equal(t, 34, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("p1", uint64(i)), k.Key)
}
g, err = generateGroup(2)
if err != nil {
t.Error(err)
}
require.Equal(t, 35, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("pr2", uint64(i)), k.Key)
}
}
func initTest(t *testing.T, schemaStr string) (string, *store.Store) {
schema.ParseBytes([]byte(schemaStr))
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
ps, err := store.NewStore(dir)
require.NoError(t, err)
posting.Init(ps)
populateGraph(t)
time.Sleep(200 * time.Millisecond) // Let the index process jobs from channel.
return dir, ps
}
func TestAddMutation_gru(t *testing.T) {
key := x.DataKey("question.tag", 0x01)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
{
// Set two tag ids and merge.
edge := &task.DirectedEdge{
ValueId: 0x2b693088816b04b7,
Label: "gru",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 0x29bf442b48a772e0,
Label: "gru",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
{
edge := &task.DirectedEdge{
ValueId: 0x38dec821d2ac3a79,
Label: "gru",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 0x2b693088816b04b7,
Label: "gru",
}
addMutation(t, ol, edge, Del)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
}
func TestNewGraph(t *testing.T) {
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
gq := &gql.GraphQuery{
UID: 101,
Attr: "me",
}
ps, err := store.NewStore(dir)
require.NoError(t, err)
posting.Init(ps)
ctx := context.Background()
sg, err := newGraph(ctx, gq)
require.NoError(t, err)
require.EqualValues(t,
[][]uint64{
[]uint64{101},
}, algo.ToUintsListForTest(sg.uidMatrix))
}
func prepare() (dir1, dir2 string, ps *store.Store, rerr error) {
var err error
dir1, err = ioutil.TempDir("", "storetest_")
if err != nil {
return "", "", nil, err
}
ps, err = store.NewStore(dir1)
if err != nil {
return "", "", nil, err
}
dir2, err = ioutil.TempDir("", "wal_")
if err != nil {
return dir1, "", nil, err
}
posting.Init(ps)
loader.Init(ps)
group.ParseGroupConfig("groups.conf")
worker.StartRaftNodes(dir2)
{
// Then load data.
f, err := os.Open("testdata.nq")
if err != nil {
return dir1, dir2, nil, err
}
gm := map[uint32]bool{
0: true,
}
_, err = loader.LoadEdges(f, gm)
f.Close()
if err != nil {
return dir1, dir2, nil, err
}
}
return dir1, dir2, ps, nil
}
func TestAddMutation_jchiu2(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Del a value cars and but don't merge.
edge := &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.EqualValues(t, 0, ol.Length(0))
// Set value to newcars, but don't merge yet.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
require.NoError(t, err)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "newcars")
// Del a value cars. This operation should be ignored.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "newcars")
}
func populateGraph(t *testing.T) (string, string, *store.Store) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
ps, err := store.NewStore(dir)
require.NoError(t, err)
schema.ParseBytes([]byte(schemaStr))
posting.Init(ps)
worker.Init(ps)
group.ParseGroupConfig("")
dir2, err := ioutil.TempDir("", "wal_")
require.NoError(t, err)
worker.StartRaftNodes(dir2)
// So, user we're interested in has uid: 1.
// She has 5 friends: 23, 24, 25, 31, and 101
addEdgeToUID(t, ps, "friend", 1, 23)
addEdgeToUID(t, ps, "friend", 1, 24)
addEdgeToUID(t, ps, "friend", 1, 25)
addEdgeToUID(t, ps, "friend", 1, 31)
addEdgeToUID(t, ps, "friend", 1, 101)
// Now let's add a few properties for the main user.
addEdgeToValue(t, ps, "name", 1, "Michonne")
addEdgeToValue(t, ps, "gender", 1, "female")
var coord types.Geo
err = coord.UnmarshalText([]byte("{\"Type\":\"Point\", \"Coordinates\":[1.1,2.0]}"))
require.NoError(t, err)
gData, err := coord.MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "loc", 1, types.GeoID, gData)
data, err := types.Int32(15).MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "age", 1, types.Int32ID, data)
addEdgeToValue(t, ps, "address", 1, "31, 32 street, Jupiter")
data, err = types.Bool(true).MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "alive", 1, types.BoolID, data)
addEdgeToValue(t, ps, "age", 1, "38")
addEdgeToValue(t, ps, "survival_rate", 1, "98.99")
addEdgeToValue(t, ps, "sword_present", 1, "true")
addEdgeToValue(t, ps, "_xid_", 1, "mich")
// Now let's add a name for each of the friends, except 101.
addEdgeToTypedValue(t, ps, "name", 23, types.StringID, []byte("Rick Grimes"))
addEdgeToValue(t, ps, "age", 23, "15")
err = coord.UnmarshalText([]byte(`{"Type":"Polygon", "Coordinates":[[[0.0,0.0], [2.0,0.0], [2.0, 2.0], [0.0, 2.0]]]}`))
require.NoError(t, err)
gData, err = coord.MarshalBinary()
require.NoError(t, err)
addEdgeToTypedValue(t, ps, "loc", 23, types.GeoID, gData)
addEdgeToValue(t, ps, "address", 23, "21, mark street, Mars")
addEdgeToValue(t, ps, "name", 24, "Glenn Rhee")
addEdgeToValue(t, ps, "name", 25, "Daryl Dixon")
addEdgeToValue(t, ps, "name", 31, "Andrea")
addEdgeToValue(t, ps, "dob", 23, "1910-01-02")
addEdgeToValue(t, ps, "dob", 24, "1909-05-05")
addEdgeToValue(t, ps, "dob", 25, "1909-01-10")
addEdgeToValue(t, ps, "dob", 31, "1901-01-15")
return dir, dir2, ps
}
func TestAddMutation_mrjn1(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Set a value cars and merge.
edge := &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
// Delete a non-existent value newcars. This should have no effect.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
checkValue(t, ol, "cars")
// Delete the previously committed value cars. But don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
// Do this again to cover Del, muid == curUid, inPlist test case.
// Delete the previously committed value cars. But don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
// Set the value again to cover Set, muid == curUid, inPlist test case.
// Set the previously committed value cars. But don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
checkValue(t, ol, "cars")
// Delete it again, just for fun.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
}
func TestAfterUIDCountWithCommit(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Set value to cars and merge to RocksDB.
edge := &task.DirectedEdge{
Label: "jchiu",
}
for i := 100; i < 300; i++ {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 200, ol.Length(0))
require.EqualValues(t, 100, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Commit to database.
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
// Mutation layer starts afresh from here.
// Delete half of the edges.
for i := 100; i < 300; i += 2 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Del)
}
require.EqualValues(t, 100, ol.Length(0))
require.EqualValues(t, 50, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Try to delete half of the edges. Redundant deletes.
for i := 100; i < 300; i += 2 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Del)
}
require.EqualValues(t, 100, ol.Length(0))
require.EqualValues(t, 50, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Delete everything.
for i := 100; i < 300; i++ {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Del)
}
require.EqualValues(t, 0, ol.Length(0))
require.EqualValues(t, 0, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Insert 1/4 of the edges.
for i := 100; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 50, ol.Length(0))
require.EqualValues(t, 25, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Insert 1/4 of the edges.
edge.Label = "somethingelse"
for i := 100; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 50, ol.Length(0)) // Expect no change.
require.EqualValues(t, 25, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Insert 1/4 of the edges.
for i := 103; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 100, ol.Length(0))
require.EqualValues(t, 50, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
}
func TestAddMutation_checksum(t *testing.T) {
var c1, c2, c3 []byte
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
{
key := x.DataKey("value", 10)
ol := getNew(key, ps)
edge := &task.DirectedEdge{
ValueId: 1,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 3,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
pl := ol.PostingList()
c1 = pl.Checksum
}
{
key := x.DataKey("value2", 10)
ol := getNew(key, ps)
// Add in reverse.
edge := &task.DirectedEdge{
ValueId: 3,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 1,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
pl := ol.PostingList()
c2 = pl.Checksum
}
require.Equal(t, c1, c2)
{
key := x.DataKey("value3", 10)
ol := getNew(key, ps)
// Add in reverse.
edge := &task.DirectedEdge{
ValueId: 3,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 1,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 4,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
pl := ol.PostingList()
c3 = pl.Checksum
}
require.NotEqual(t, c3, c1)
}
func TestAddMutation(t *testing.T) {
key := x.DataKey("name", 1)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
l := getNew(key, ps)
edge := &task.DirectedEdge{
ValueId: 9,
Label: "testing",
}
addMutation(t, l, edge, Set)
require.Equal(t, listToArray(t, 0, l), []uint64{9})
p := getFirst(l)
require.NotNil(t, p, "Unable to retrieve posting")
require.EqualValues(t, p.Label, "testing")
// Add another edge now.
edge.ValueId = 81
addMutation(t, l, edge, Set)
require.Equal(t, listToArray(t, 0, l), []uint64{9, 81})
// Add another edge, in between the two above.
edge.ValueId = 49
addMutation(t, l, edge, Set)
require.Equal(t, listToArray(t, 0, l), []uint64{9, 49, 81})
checkUids(t, l, []uint64{9, 49, 81})
// Delete an edge, add an edge, replace an edge
edge.ValueId = 49
addMutation(t, l, edge, Del)
edge.ValueId = 69
addMutation(t, l, edge, Set)
edge.ValueId = 9
edge.Label = "anti-testing"
addMutation(t, l, edge, Set)
uids := []uint64{9, 69, 81}
checkUids(t, l, uids)
p = getFirst(l)
require.NotNil(t, p, "Unable to retrieve posting")
require.EqualValues(t, p.Label, "anti-testing")
l.CommitIfDirty(context.Background())
l.WaitForCommit()
// Try reading the same data in another PostingList.
dl := getNew(key, ps)
checkUids(t, dl, uids)
return
_, err = dl.CommitIfDirty(context.Background())
require.NoError(t, err)
checkUids(t, dl, uids)
}