// In benchmarks, the time taken per AddMutation before was
// plateauing at 2.5 ms with sync per 10 log entries, and increasing
// for sync per 100 log entries (to 3 ms per AddMutation), largely because
// of how index generation was being done.
//
// With this change, the benchmarks perform as good as benchmarks for
// commit.Logger, where the less frequently file sync happens, the faster
// AddMutations run.
//
// PASS
// BenchmarkAddMutations_SyncEveryLogEntry-6 100 24712455 ns/op
// BenchmarkAddMutations_SyncEvery10LogEntry-6 500 2485961 ns/op
// BenchmarkAddMutations_SyncEvery100LogEntry-6 10000 298352 ns/op
// BenchmarkAddMutations_SyncEvery1000LogEntry-6 30000 63544 ns/op
// ok github.com/dgraph-io/dgraph/posting 10.291s
func (l *List) AddMutation(t x.DirectedEdge, op byte) error {
l.wg.Wait()
l.Lock()
defer l.Unlock()
if l.deleteMe {
return E_TMP_ERROR
}
if t.Timestamp.UnixNano() < l.maxMutationTs {
return fmt.Errorf("Mutation ts lower than committed ts.")
}
// Mutation arrives:
// - Check if we had any(SET/DEL) before this, stored in the mutation list.
// - If yes, then replace that mutation. Jump to a)
// a) check if the entity exists in main posting list.
// - If yes, store the mutation.
// - If no, disregard this mutation.
// All edges with a value set, have the same uid. In other words,
// an (entity, attribute) can only have one interface{} value.
if t.Value != nil {
t.ValueId = math.MaxUint64
}
if t.ValueId == 0 {
return fmt.Errorf("ValueId cannot be zero.")
}
mbuf := newPosting(t, op)
uo := flatbuffers.GetUOffsetT(mbuf)
mpost := new(types.Posting)
mpost.Init(mbuf, uo)
glog.WithFields(logrus.Fields{
"uid": mpost.Uid(),
"source": string(mpost.Source()),
"ts": mpost.Ts(),
}).Debug("Add mutation")
l.mergeMutation(mpost)
l.maxMutationTs = t.Timestamp.UnixNano()
if len(l.mindex)+len(l.mlayer) > 0 {
atomic.StoreInt64(&l.dirtyTs, time.Now().UnixNano())
if dirtymap != nil {
dirtymap.Put(l.ghash, true)
}
}
if l.clog == nil {
return nil
}
return l.clog.AddLog(t.Timestamp.UnixNano(), l.hash, mbuf)
}
func TestAddMutation(t *testing.T) {
// logrus.SetLevel(logrus.DebugLevel)
l := NewList()
key := Key(1, "name")
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
l.init(key, ps, clog)
edge := x.DirectedEdge{
ValueId: 9,
Source: "testing",
Timestamp: time.Now(),
}
if err := l.AddMutation(edge, Set); err != nil {
t.Error(err)
}
/*
if err := l.CommitIfDirty(); err != nil {
t.Error(err)
}
*/
if l.Length() != 1 {
t.Error("Unable to find added elements in posting list")
}
var p types.Posting
if ok := l.Get(&p, 0); !ok {
t.Error("Unable to retrieve posting at 1st iter")
t.Fail()
}
if p.Uid() != 9 {
t.Errorf("Expected 9. Got: %v", p.Uid)
}
if string(p.Source()) != "testing" {
t.Errorf("Expected testing. Got: %v", string(p.Source()))
}
// return // Test 1.
// Add another edge now.
edge.ValueId = 81
l.AddMutation(edge, Set)
// l.CommitIfDirty()
if l.Length() != 2 {
t.Errorf("Length: %d", l.Length())
t.Fail()
}
var uid uint64
uid = 1
for i := 0; i < l.Length(); i++ {
if ok := l.Get(&p, i); !ok {
t.Error("Unable to retrieve posting at 2nd iter")
}
uid *= 9
if p.Uid() != uid {
t.Logf("Expected: %v. Got: %v", uid, p.Uid())
}
}
// return // Test 2.
// Add another edge, in between the two above.
uids := []uint64{
9, 49, 81,
}
edge.ValueId = 49
if err := l.AddMutation(edge, Set); err != nil {
t.Error(err)
}
/*
if err := l.CommitIfDirty(); err != nil {
t.Error(err)
}
*/
if err := checkUids(t, l, uids...); err != nil {
t.Error(err)
}
// return // Test 3.
// Delete an edge, add an edge, replace an edge
edge.ValueId = 49
if err := l.AddMutation(edge, Del); err != nil {
t.Error(err)
}
edge.ValueId = 69
if err := l.AddMutation(edge, Set); err != nil {
t.Error(err)
}
edge.ValueId = 9
edge.Source = "anti-testing"
if err := l.AddMutation(edge, Set); err != nil {
t.Error(err)
}
/*
if err := l.CommitIfDirty(); err != nil {
t.Error(err)
}
*/
uids = []uint64{9, 69, 81}
if err := checkUids(t, l, uids...); err != nil {
t.Error(err)
}
l.Get(&p, 0)
if string(p.Source()) != "anti-testing" {
t.Errorf("Expected: anti-testing. Got: %v", string(p.Source()))
}
/*
if err := l.CommitIfDirty(); err != nil {
t.Error(err)
}
*/
// Try reading the same data in another PostingList.
dl := NewList()
dl.init(key, ps, clog)
if err := checkUids(t, dl, uids...); err != nil {
t.Error(err)
}
if _, err := dl.MergeIfDirty(); err != nil {
t.Error(err)
}
if err := checkUids(t, dl, uids...); err != nil {
t.Error(err)
}
}
func TestProcessTask(t *testing.T) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
Init(ps, clog)
edge := x.DirectedEdge{
ValueId: 23,
Source: "author0",
Timestamp: time.Now(),
}
addEdge(t, edge, GetOrCreate(Key(10, "friend")))
addEdge(t, edge, GetOrCreate(Key(11, "friend")))
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.ValueId = 25
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.ValueId = 26
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.ValueId = 31
addEdge(t, edge, GetOrCreate(Key(10, "friend")))
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
edge.Value = "photon"
addEdge(t, edge, GetOrCreate(Key(12, "friend")))
query := NewQuery("friend", []uint64{10, 11, 12})
result, err := ProcessTask(query)
if err != nil {
t.Error(err)
}
ro := flatbuffers.GetUOffsetT(result)
r := new(task.Result)
r.Init(result, ro)
if r.UidmatrixLength() != 3 {
t.Errorf("Expected 3. Got uidmatrix length: %v", r.UidmatrixLength())
}
if err := check(r, 0, []uint64{23, 31}); err != nil {
t.Error(err)
}
if err := check(r, 1, []uint64{23}); err != nil {
t.Error(err)
}
if err := check(r, 2, []uint64{23, 25, 26, 31}); err != nil {
t.Error(err)
}
if r.ValuesLength() != 3 {
t.Errorf("Expected 3. Got values length: %v", r.ValuesLength())
}
var tval task.Value
if ok := r.Values(&tval, 0); !ok {
t.Errorf("Unable to retrieve value")
}
if tval.ValLength() != 1 ||
tval.ValBytes()[0] != 0x00 {
t.Errorf("Invalid byte value at index 0")
}
if ok := r.Values(&tval, 1); !ok {
t.Errorf("Unable to retrieve value")
}
if tval.ValLength() != 1 ||
tval.ValBytes()[0] != 0x00 {
t.Errorf("Invalid byte value at index 0")
}
if ok := r.Values(&tval, 2); !ok {
t.Errorf("Unable to retrieve value")
}
var iout interface{}
if err := ParseValue(&iout, tval.ValBytes()); err != nil {
t.Error(err)
}
v := iout.(string)
if v != "photon" {
t.Errorf("Expected photon. Got: %q", v)
}
}
func TestAddMutation_Value(t *testing.T) {
// logrus.SetLevel(logrus.DebugLevel)
glog.Debug("Running init...")
ol := NewList()
key := Key(10, "value")
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(dir)
ps := new(store.Store)
ps.Init(dir)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
defer clog.Close()
ol.init(key, ps, clog)
glog.Debug("Init successful.")
edge := x.DirectedEdge{
Value: "oh hey there",
Source: "new-testing",
Timestamp: time.Now(),
}
if err := ol.AddMutation(edge, Set); err != nil {
t.Error(err)
}
var p types.Posting
ol.Get(&p, 0)
if p.Uid() != math.MaxUint64 {
t.Errorf("All value uids should go to MaxUint64. Got: %v", p.Uid())
}
var iout interface{}
if err := ParseValue(&iout, p.ValueBytes()); err != nil {
t.Error(err)
}
out := iout.(string)
if out != "oh hey there" {
t.Errorf("Expected a value. Got: [%q]", out)
}
// Run the same check after committing.
if _, err := ol.MergeIfDirty(); err != nil {
t.Error(err)
}
{
var tp types.Posting
if ok := ol.Get(&tp, 0); !ok {
t.Error("While retrieving posting")
}
if err := ParseValue(&iout, tp.ValueBytes()); err != nil {
t.Error(err)
}
out := iout.(string)
if out != "oh hey there" {
t.Errorf("Expected a value. Got: [%q]", out)
}
}
// The value made it to the posting list. Changing it now.
edge.Value = 119
if err := ol.AddMutation(edge, Set); err != nil {
t.Error(err)
}
if ol.Length() != 1 {
t.Errorf("Length should be one. Got: %v", ol.Length())
}
if ok := ol.Get(&p, 0); !ok {
t.Error("While retrieving posting")
}
if err := ParseValue(&iout, p.ValueBytes()); err != nil {
t.Error(err)
}
intout := iout.(float64)
if intout != 119 {
t.Errorf("Expected 119. Got: %v", intout)
}
}
func populateGraph(t *testing.T) (string, *store.Store) {
// logrus.SetLevel(logrus.DebugLevel)
dir, err := ioutil.TempDir("", "storetest_")
if err != nil {
t.Error(err)
return "", nil
}
ps := new(store.Store)
ps.Init(dir)
worker.Init(ps)
clog := commit.NewLogger(dir, "mutations", 50<<20)
clog.Init()
posting.Init(clog)
// So, user we're interested in has uid: 1.
// She has 4 friends: 23, 24, 25, 31, and 101
edge := x.DirectedEdge{
ValueId: 23,
Source: "testing",
Timestamp: time.Now(),
}
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 24
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 25
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 31
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
edge.ValueId = 101
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "friend"), ps))
// Now let's add a few properties for the main user.
edge.Value = "Michonne"
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "name"), ps))
edge.Value = "female"
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "gender"), ps))
edge.Value = "alive"
addEdge(t, edge, posting.GetOrCreate(posting.Key(1, "status"), ps))
// Now let's add a name for each of the friends, except 101.
edge.Value = "Rick Grimes"
addEdge(t, edge, posting.GetOrCreate(posting.Key(23, "name"), ps))
edge.Value = "Glenn Rhee"
addEdge(t, edge, posting.GetOrCreate(posting.Key(24, "name"), ps))
edge.Value = "Daryl Dixon"
addEdge(t, edge, posting.GetOrCreate(posting.Key(25, "name"), ps))
edge.Value = "Andrea"
addEdge(t, edge, posting.GetOrCreate(posting.Key(31, "name"), ps))
return dir, ps
}
