func TestGenerateGroup(t *testing.T) {
dir, err := ioutil.TempDir("", "store3")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
r := bytes.NewReader([]byte("default: fp % 3"))
require.NoError(t, group.ParseConfig(r), "Unable to parse config.")
ps, err := store.NewStore(dir)
if err != nil {
t.Fatal(err)
}
defer ps.Close()
posting.Init(ps)
Init(ps)
require.Equal(t, uint32(0), group.BelongsTo("pred0"))
writePLs(t, "pred0", 33, 1, ps)
require.Equal(t, uint32(1), group.BelongsTo("p1"))
writePLs(t, "p1", 34, 1, ps)
require.Equal(t, uint32(2), group.BelongsTo("pr2"))
writePLs(t, "pr2", 35, 1, ps)
time.Sleep(time.Second)
g, err := generateGroup(0)
if err != nil {
t.Error(err)
}
require.Equal(t, 33, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("pred0", uint64(i)), k.Key)
}
g, err = generateGroup(1)
if err != nil {
t.Error(err)
}
require.Equal(t, 34, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("p1", uint64(i)), k.Key)
}
g, err = generateGroup(2)
if err != nil {
t.Error(err)
}
require.Equal(t, 35, len(g.Keys))
for i, k := range g.Keys {
require.Equal(t, x.DataKey("pr2", uint64(i)), k.Key)
}
}
func BenchmarkAddMutations(b *testing.B) {
// logrus.SetLevel(logrus.DebugLevel)
key := x.DataKey("name", 1)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
b.Error(err)
return
}
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
if err != nil {
b.Error(err)
return
}
l := getNew(key, ps)
b.ResetTimer()
ctx := context.Background()
for i := 0; i < b.N; i++ {
if err != nil {
b.Error(err)
return
}
edge := &task.DirectedEdge{
ValueId: uint64(rand.Intn(b.N) + 1),
Label: "testing",
}
if _, err := l.AddMutation(ctx, edge, Set); err != nil {
b.Error(err)
}
}
}
func TestAddMutation_Value(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
log.Println("Init successful.")
edge := &task.DirectedEdge{
Value: []byte("oh hey there"),
Label: "new-testing",
}
addMutation(t, ol, edge, Set)
checkValue(t, ol, "oh hey there")
// Run the same check after committing.
_, err = ol.CommitIfDirty(context.Background())
require.NoError(t, err)
checkValue(t, ol, "oh hey there")
// The value made it to the posting list. Changing it now.
edge.Value = []byte(strconv.Itoa(119))
addMutation(t, ol, edge, Set)
checkValue(t, ol, "119")
}
func TestAfterUIDCount2(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
ol := getNew(key, ps)
// Set value to cars and merge to RocksDB.
edge := &task.DirectedEdge{
Label: "jchiu",
}
for i := 100; i < 300; i++ {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 200, ol.Length(0))
require.EqualValues(t, 100, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Re-insert 1/4 of the edges. Counts should not change.
edge.Label = "somethingelse"
for i := 100; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 200, ol.Length(0))
require.EqualValues(t, 100, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
}
func populateGraphForSort(t *testing.T, ps *store.Store) {
edge := &task.DirectedEdge{
Label: "author1",
Attr: "dob",
}
dobs := []string{
"1980-05-05", // 10 (1980)
"1980-04-05", // 11
"1979-05-05", // 12 (1979)
"1979-02-05", // 13
"1979-03-05", // 14
"1965-05-05", // 15 (1965)
"1965-04-05", // 16
"1965-03-05", // 17
"1970-05-05", // 18 (1970)
"1970-04-05", // 19
"1970-01-05", // 20
"1970-02-05", // 21
}
// The sorted UIDs are: (17 16 15) (20 21 19 18) (13 14 12) (11 10)
for i, dob := range dobs {
edge.Entity = uint64(i + 10)
edge.Value = []byte(dob)
addEdge(t, edge,
getOrCreate(x.DataKey(edge.Attr, edge.Entity)))
}
time.Sleep(200 * time.Millisecond) // Let indexing finish.
}
// allocateUniqueUid returns an integer in range:
// [minIdx, maxIdx] derived based on numInstances and instanceIdx.
// which hasn't already been allocated to other xids. It does this by
// taking the fingerprint of the xid appended with zero or more spaces
// until the obtained integer is unique.
func allocateUniqueUid(instanceIdx uint64, numInstances uint64) uint64 {
mod := math.MaxUint64 / numInstances
minIdx := instanceIdx * mod
buf := make([]byte, 128)
for {
_, err := rand.Read(buf)
x.Checkf(err, "rand.Read shouldn't throw an error")
uidb := farm.Fingerprint64(buf) // Generate from hash.
uid := (uidb % mod) + minIdx
if uid == math.MaxUint64 || !lmgr.isNew(uid) {
continue
}
// Check if this uid has already been allocated.
key := x.DataKey("_uid_", uid)
pl, decr := posting.GetOrCreate(key)
defer decr()
if pl.Length(0) == 0 {
return uid
}
}
log.Fatalf("This shouldn't be reached.")
return 0
}
func TestAddMutation_jchiu3(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Set value to cars and merge to RocksDB.
edge := &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
require.Equal(t, 1, ol.Length(0))
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "cars")
// Del a value cars and but don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
// Set value to newcars, but don't merge yet.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "newcars")
// Del a value othercars and but don't merge.
edge = &task.DirectedEdge{
Value: []byte("othercars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.NotEqual(t, 0, ol.Length(0))
checkValue(t, ol, "newcars")
// Del a value newcars and but don't merge.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
}
func TestAddMutation_gru2(t *testing.T) {
key := x.DataKey("question.tag", 0x01)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
{
// Set two tag ids and merge.
edge := &task.DirectedEdge{
ValueId: 0x02,
Label: "gru",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 0x03,
Label: "gru",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
{
// Lets set a new tag and delete the two older ones.
edge := &task.DirectedEdge{
ValueId: 0x02,
Label: "gru",
}
addMutation(t, ol, edge, Del)
edge = &task.DirectedEdge{
ValueId: 0x03,
Label: "gru",
}
addMutation(t, ol, edge, Del)
edge = &task.DirectedEdge{
ValueId: 0x04,
Label: "gru",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
// Posting list should just have the new tag.
uids := []uint64{0x04}
require.Equal(t, uids, listToArray(t, 0, ol))
}
func addEdgeToUID(t *testing.T, ps *store.Store, attr string, src uint64, dst uint64) {
edge := &task.DirectedEdge{
ValueId: dst,
Label: "testing",
Attr: attr,
Entity: src,
}
l, _ := posting.GetOrCreate(x.DataKey(attr, src))
require.NoError(t,
l.AddMutationWithIndex(context.Background(), edge, posting.Set))
}
func markTaken(ctx context.Context, uid uint64) {
mu := &task.DirectedEdge{
Entity: uid,
Attr: "_uid_",
Value: []byte("_"), // not txid
Label: "_loader_",
}
key := x.DataKey("_uid_", uid)
plist, decr := posting.GetOrCreate(key)
plist.AddMutation(ctx, mu, posting.Set)
decr()
}
func populateGraphBackup(t *testing.T) {
edge := &task.DirectedEdge{
ValueId: 5,
Label: "author0",
Attr: "friend",
}
edge.Entity = 1
addEdge(t, edge, getOrCreate(x.DataKey("friend", 1)))
edge.Entity = 2
addEdge(t, edge, getOrCreate(x.DataKey("friend", 2)))
edge.Entity = 3
addEdge(t, edge, getOrCreate(x.DataKey("friend", 3)))
edge.Entity = 4
addEdge(t, edge, getOrCreate(x.DataKey("friend", 4)))
edge.Entity = 1
edge.ValueId = 0
edge.Value = []byte("pho\\ton")
edge.Attr = "name"
addEdge(t, edge, getOrCreate(x.DataKey("name", 1)))
edge.Entity = 2
addEdge(t, edge, getOrCreate(x.DataKey("name", 2)))
}
func addEdgeToTypedValue(t *testing.T, ps *store.Store, attr string, src uint64,
typ types.TypeID, value []byte) {
edge := &task.DirectedEdge{
Value: value,
ValueType: uint32(typ),
Label: "testing",
Attr: attr,
Entity: src,
}
l, _ := posting.GetOrCreate(x.DataKey(attr, src))
require.NoError(t,
l.AddMutationWithIndex(context.Background(), edge, posting.Set))
}
// handleNQuads converts the nQuads that satisfy the modulo
// rule into posting lists.
func (s *state) handleNQuads(wg *sync.WaitGroup) {
defer wg.Done()
// Check if we need to mark used UIDs.
markUids := s.groupsMap[group.BelongsTo("_uid_")]
ctx := context.Background()
for nq := range s.cnq {
if s.Error() != nil {
return
}
// Only handle this edge if the attribute satisfies the modulo rule
if !s.groupsMap[group.BelongsTo(nq.Predicate)] {
atomic.AddUint64(&s.ctr.ignored, 1)
continue
}
edge, err := nq.ToEdge()
for err != nil {
// Just put in a retry loop to tackle temporary errors.
if err == posting.ErrRetry {
time.Sleep(time.Microsecond)
} else {
s.SetError(err)
glog.WithError(err).WithField("nq", nq).
Error("While converting to edge")
return
}
edge, err = nq.ToEdge()
}
key := x.DataKey(edge.Attr, edge.Entity)
plist, decr := posting.GetOrCreate(key)
plist.AddMutationWithIndex(ctx, edge, posting.Set)
decr() // Don't defer, just call because we're in a channel loop.
// Mark UIDs and XIDs as taken
if markUids {
// Mark entity UID.
markTaken(ctx, edge.Entity)
// Mark the Value UID.
if edge.ValueId != 0 {
markTaken(ctx, edge.ValueId)
}
}
atomic.AddUint64(&s.ctr.processed, 1)
}
}
func writePLs(t *testing.T, pred string, count int, vid uint64, ps *store.Store) {
for i := 0; i < count; i++ {
k := x.DataKey(pred, uint64(i))
list, _ := posting.GetOrCreate(k)
de := &task.DirectedEdge{
ValueId: vid,
Label: "test",
}
list.AddMutation(context.TODO(), de, posting.Set)
if merged, err := list.CommitIfDirty(context.TODO()); err != nil {
t.Errorf("While merging: %v", err)
} else if !merged {
t.Errorf("No merge happened")
}
}
}
// runMutations goes through all the edges and applies them. It returns the
// mutations which were not applied in left.
func runMutations(ctx context.Context, edges []*task.DirectedEdge, op uint32) error {
for _, edge := range edges {
if !groups().ServesGroup(group.BelongsTo(edge.Attr)) {
return x.Errorf("Predicate fingerprint doesn't match this instance")
}
key := x.DataKey(edge.Attr, edge.Entity)
plist, decr := posting.GetOrCreate(key)
defer decr()
if err := plist.AddMutationWithIndex(ctx, edge, op); err != nil {
x.Printf("Error while adding mutation: %v %v", edge, err)
return err // abort applying the rest of them.
}
}
return nil
}
func TestAddMutation_gru(t *testing.T) {
key := x.DataKey("question.tag", 0x01)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
{
// Set two tag ids and merge.
edge := &task.DirectedEdge{
ValueId: 0x2b693088816b04b7,
Label: "gru",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 0x29bf442b48a772e0,
Label: "gru",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
{
edge := &task.DirectedEdge{
ValueId: 0x38dec821d2ac3a79,
Label: "gru",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 0x2b693088816b04b7,
Label: "gru",
}
addMutation(t, ol, edge, Del)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
}
}
func populateGraph(t *testing.T) {
edge := &task.DirectedEdge{
ValueId: 23,
Label: "author0",
Attr: "friend",
}
edge.Entity = 10
addEdge(t, edge, getOrCreate(x.DataKey("friend", 10)))
edge.Entity = 11
addEdge(t, edge, getOrCreate(x.DataKey("friend", 11)))
edge.Entity = 12
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.ValueId = 25
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.ValueId = 26
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.Entity = 10
edge.ValueId = 31
addEdge(t, edge, getOrCreate(x.DataKey("friend", 10)))
edge.Entity = 12
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.Entity = 12
edge.Value = []byte("photon")
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.Entity = 10
addEdge(t, edge, getOrCreate(x.DataKey("friend", 10)))
}
func TestAddMutation_jchiu2(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Del a value cars and but don't merge.
edge := &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.EqualValues(t, 0, ol.Length(0))
// Set value to newcars, but don't merge yet.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
require.NoError(t, err)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "newcars")
// Del a value cars. This operation should be ignored.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.EqualValues(t, 1, ol.Length(0))
checkValue(t, ol, "newcars")
}
// fetchValue gets the value for a given UID.
func fetchValue(uid uint64, attr string, scalar types.Scalar) (types.Value, error) {
pl, decr := posting.GetOrCreate(x.DataKey(attr, uid))
defer decr()
valBytes, vType, err := pl.Value()
if err != nil {
return nil, err
}
val := types.ValueForType(types.TypeID(vType))
if val == nil {
return nil, x.Errorf("Invalid type: %v", vType)
}
err = val.UnmarshalBinary(valBytes)
if err != nil {
return nil, err
}
schemaVal, err := scalar.Convert(val)
if err != nil {
return nil, err
}
return schemaVal, nil
}
// processTask processes the query, accumulates and returns the result.
func processTask(q *task.Query) (*task.Result, error) {
attr := q.Attr
useFunc := len(q.SrcFunc) != 0
var n int
var tokens []string
var geoQuery *geo.QueryData
var err error
var intersectDest bool
var ineqValue types.Value
var ineqValueToken string
var isGeq, isLeq bool
if useFunc {
f := q.SrcFunc[0]
isGeq = f == "geq"
isLeq = f == "leq"
switch {
case isGeq:
fallthrough
case isLeq:
if len(q.SrcFunc) != 2 {
return nil, x.Errorf("Function requires 2 arguments, but got %d %v",
len(q.SrcFunc), q.SrcFunc)
}
ineqValue, err = getValue(attr, q.SrcFunc[1])
if err != nil {
return nil, err
}
// Tokenizing RHS value of inequality.
ineqTokens, err := posting.IndexTokens(attr, ineqValue)
if err != nil {
return nil, err
}
if len(ineqTokens) != 1 {
return nil, x.Errorf("Expected only 1 token but got: %v", ineqTokens)
}
ineqValueToken = ineqTokens[0]
// Get tokens geq / leq ineqValueToken.
tokens, err = getInequalityTokens(attr, ineqValueToken, isGeq)
if err != nil {
return nil, err
}
case geo.IsGeoFunc(q.SrcFunc[0]):
// For geo functions, we get extra information used for filtering.
tokens, geoQuery, err = geo.GetTokens(q.SrcFunc)
if err != nil {
return nil, err
}
default:
tokens, err = getTokens(q.SrcFunc)
if err != nil {
return nil, err
}
intersectDest = (strings.ToLower(q.SrcFunc[0]) == "allof")
}
n = len(tokens)
} else {
n = len(q.Uids)
}
var out task.Result
for i := 0; i < n; i++ {
var key []byte
if useFunc {
key = x.IndexKey(attr, tokens[i])
} else {
key = x.DataKey(attr, q.Uids[i])
}
// Get or create the posting list for an entity, attribute combination.
pl, decr := posting.GetOrCreate(key)
defer decr()
// If a posting list contains a value, we store that or else we store a nil
// byte so that processing is consistent later.
vbytes, vtype, err := pl.Value()
newValue := &task.Value{ValType: uint32(vtype)}
if err == nil {
newValue.Val = vbytes
} else {
newValue.Val = x.Nilbyte
}
out.Values = append(out.Values, newValue)
if q.DoCount {
out.Counts = append(out.Counts, uint32(pl.Length(0)))
// Add an empty UID list to make later processing consistent
out.UidMatrix = append(out.UidMatrix, &emptyUIDList)
continue
}
// The more usual case: Getting the UIDs.
opts := posting.ListOptions{
AfterUID: uint64(q.AfterUid),
}
// If we have srcFunc and Uids, it means its a filter. So we intersect.
if useFunc && len(q.Uids) > 0 {
opts.Intersect = &task.List{Uids: q.Uids}
}
out.UidMatrix = append(out.UidMatrix, pl.Uids(opts))
}
if (isGeq || isLeq) && len(tokens) > 0 && ineqValueToken == tokens[0] {
// Need to evaluate inequality for entries in the first bucket.
typ := schema.TypeOf(attr)
if typ == nil || !typ.IsScalar() {
return nil, x.Errorf("Attribute not scalar: %s %v", attr, typ)
}
scalarType := typ.(types.Scalar)
x.AssertTrue(len(out.UidMatrix) > 0)
// Filter the first row of UidMatrix. Since ineqValue != nil, we may
// assume that ineqValue is equal to the first token found in TokensTable.
algo.ApplyFilter(out.UidMatrix[0], func(uid uint64, i int) bool {
key := x.DataKey(attr, uid)
sv := getPostingValue(key, scalarType)
if sv == nil {
return false
}
if isGeq {
return !scalarType.Less(*sv, ineqValue)
}
return !scalarType.Less(ineqValue, *sv)
})
}
// If geo filter, do value check for correctness.
var values []*task.Value
if geoQuery != nil {
uids := algo.MergeSorted(out.UidMatrix)
for _, uid := range uids.Uids {
key := x.DataKey(attr, uid)
pl, decr := posting.GetOrCreate(key)
vbytes, vtype, err := pl.Value()
newValue := &task.Value{ValType: uint32(vtype)}
if err == nil {
newValue.Val = vbytes
} else {
newValue.Val = x.Nilbyte
}
values = append(values, newValue)
decr() // Decrement the reference count of the pl.
}
filtered := geo.FilterUids(uids, values, geoQuery)
for i := 0; i < len(out.UidMatrix); i++ {
out.UidMatrix[i] = algo.IntersectSorted([]*task.List{out.UidMatrix[i], filtered})
}
}
out.IntersectDest = intersectDest
return &out, nil
}
// Index-related test. Similar to TestProcessTaskIndeMLayer except we call
// MergeLists in between a lot of updates.
func TestProcessTaskIndex(t *testing.T) {
dir, ps := initTest(t, `scalar friend:string @index`)
defer os.RemoveAll(dir)
defer ps.Close()
query := newQuery("friend", nil, []string{"anyof", "hey photon"})
r, err := processTask(query)
require.NoError(t, err)
require.EqualValues(t, [][]uint64{
[]uint64{},
[]uint64{10, 12},
}, algo.ToUintsListForTest(r.UidMatrix))
posting.CommitLists(10)
time.Sleep(200 * time.Millisecond) // Let the index process jobs from channel.
// Now try changing 12's friend value from "photon" to "notphotonExtra" to
// "notphoton".
edge := &task.DirectedEdge{
Value: []byte("notphotonExtra"),
Label: "author0",
Attr: "friend",
Entity: 12,
}
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.Value = []byte("notphoton")
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
time.Sleep(200 * time.Millisecond) // Let the index process jobs from channel.
// Issue a similar query.
query = newQuery("friend", nil, []string{"anyof", "hey photon notphoton notphotonExtra"})
r, err = processTask(query)
require.NoError(t, err)
require.EqualValues(t, [][]uint64{
[]uint64{},
[]uint64{10},
[]uint64{12},
[]uint64{},
}, algo.ToUintsListForTest(r.UidMatrix))
posting.CommitLists(10)
time.Sleep(200 * time.Millisecond) // Let the index process jobs from channel.
// Try deleting.
edge = &task.DirectedEdge{
Value: []byte("photon"),
Label: "author0",
Attr: "friend",
Entity: 10,
}
// Redundant deletes.
delEdge(t, edge, getOrCreate(x.DataKey("friend", 10)))
delEdge(t, edge, getOrCreate(x.DataKey("friend", 10)))
// Delete followed by set.
edge.Entity = 12
edge.Value = []byte("notphoton")
delEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
edge.Value = []byte("ignored")
addEdge(t, edge, getOrCreate(x.DataKey("friend", 12)))
time.Sleep(200 * time.Millisecond) // Let indexing finish.
// Issue a similar query.
query = newQuery("friend", nil, []string{"anyof", "photon notphoton ignored"})
r, err = processTask(query)
require.NoError(t, err)
require.EqualValues(t, [][]uint64{
[]uint64{},
[]uint64{},
[]uint64{12},
}, algo.ToUintsListForTest(r.UidMatrix))
}
func TestAddMutation_mrjn1(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Set a value cars and merge.
edge := &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
// Delete a non-existent value newcars. This should have no effect.
edge = &task.DirectedEdge{
Value: []byte("newcars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
checkValue(t, ol, "cars")
// Delete the previously committed value cars. But don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
// Do this again to cover Del, muid == curUid, inPlist test case.
// Delete the previously committed value cars. But don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
// Set the value again to cover Set, muid == curUid, inPlist test case.
// Set the previously committed value cars. But don't merge.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
checkValue(t, ol, "cars")
// Delete it again, just for fun.
edge = &task.DirectedEdge{
Value: []byte("cars"),
Label: "jchiu",
}
addMutation(t, ol, edge, Del)
require.Equal(t, 0, ol.Length(0))
}
func TestAddMutation_checksum(t *testing.T) {
var c1, c2, c3 []byte
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
{
key := x.DataKey("value", 10)
ol := getNew(key, ps)
edge := &task.DirectedEdge{
ValueId: 1,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 3,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
pl := ol.PostingList()
c1 = pl.Checksum
}
{
key := x.DataKey("value2", 10)
ol := getNew(key, ps)
// Add in reverse.
edge := &task.DirectedEdge{
ValueId: 3,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 1,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
pl := ol.PostingList()
c2 = pl.Checksum
}
require.Equal(t, c1, c2)
{
key := x.DataKey("value3", 10)
ol := getNew(key, ps)
// Add in reverse.
edge := &task.DirectedEdge{
ValueId: 3,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 1,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
edge = &task.DirectedEdge{
ValueId: 4,
Label: "jchiu",
}
addMutation(t, ol, edge, Set)
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
pl := ol.PostingList()
c3 = pl.Checksum
}
require.NotEqual(t, c3, c1)
}
func TestAddMutation(t *testing.T) {
key := x.DataKey("name", 1)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
l := getNew(key, ps)
edge := &task.DirectedEdge{
ValueId: 9,
Label: "testing",
}
addMutation(t, l, edge, Set)
require.Equal(t, listToArray(t, 0, l), []uint64{9})
p := getFirst(l)
require.NotNil(t, p, "Unable to retrieve posting")
require.EqualValues(t, p.Label, "testing")
// Add another edge now.
edge.ValueId = 81
addMutation(t, l, edge, Set)
require.Equal(t, listToArray(t, 0, l), []uint64{9, 81})
// Add another edge, in between the two above.
edge.ValueId = 49
addMutation(t, l, edge, Set)
require.Equal(t, listToArray(t, 0, l), []uint64{9, 49, 81})
checkUids(t, l, []uint64{9, 49, 81})
// Delete an edge, add an edge, replace an edge
edge.ValueId = 49
addMutation(t, l, edge, Del)
edge.ValueId = 69
addMutation(t, l, edge, Set)
edge.ValueId = 9
edge.Label = "anti-testing"
addMutation(t, l, edge, Set)
uids := []uint64{9, 69, 81}
checkUids(t, l, uids)
p = getFirst(l)
require.NotNil(t, p, "Unable to retrieve posting")
require.EqualValues(t, p.Label, "anti-testing")
l.CommitIfDirty(context.Background())
l.WaitForCommit()
// Try reading the same data in another PostingList.
dl := getNew(key, ps)
checkUids(t, dl, uids)
return
_, err = dl.CommitIfDirty(context.Background())
require.NoError(t, err)
checkUids(t, dl, uids)
}
func TestAfterUIDCountWithCommit(t *testing.T) {
key := x.DataKey("value", 10)
dir, err := ioutil.TempDir("", "storetest_")
require.NoError(t, err)
defer os.RemoveAll(dir)
ps, err := store.NewStore(dir)
require.NoError(t, err)
Init(ps)
ol := getNew(key, ps)
// Set value to cars and merge to RocksDB.
edge := &task.DirectedEdge{
Label: "jchiu",
}
for i := 100; i < 300; i++ {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 200, ol.Length(0))
require.EqualValues(t, 100, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Commit to database.
merged, err := ol.CommitIfDirty(context.Background())
require.NoError(t, err)
require.True(t, merged)
// Mutation layer starts afresh from here.
// Delete half of the edges.
for i := 100; i < 300; i += 2 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Del)
}
require.EqualValues(t, 100, ol.Length(0))
require.EqualValues(t, 50, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Try to delete half of the edges. Redundant deletes.
for i := 100; i < 300; i += 2 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Del)
}
require.EqualValues(t, 100, ol.Length(0))
require.EqualValues(t, 50, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Delete everything.
for i := 100; i < 300; i++ {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Del)
}
require.EqualValues(t, 0, ol.Length(0))
require.EqualValues(t, 0, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Insert 1/4 of the edges.
for i := 100; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 50, ol.Length(0))
require.EqualValues(t, 25, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Insert 1/4 of the edges.
edge.Label = "somethingelse"
for i := 100; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 50, ol.Length(0)) // Expect no change.
require.EqualValues(t, 25, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
// Insert 1/4 of the edges.
for i := 103; i < 300; i += 4 {
edge.ValueId = uint64(i)
addMutation(t, ol, edge, Set)
}
require.EqualValues(t, 100, ol.Length(0))
require.EqualValues(t, 50, ol.Length(199))
require.EqualValues(t, 0, ol.Length(300))
}