func TestSegmentMethods(t *testing.T) {
engine, _ := origins.Init("memory", nil)
id := uuid.NewV4()
next := uuid.NewV4()
now := chrono.Norm(time.Now().UTC())
s := Segment{
UUID: &id,
Transaction: 10,
Domain: "testing",
Time: now,
Blocks: 5,
Count: 4800,
Bytes: 460800,
Next: &next,
Base: &next,
}
_, err := SetSegment(engine, "testing", &s)
if err != nil {
t.Error(err)
}
s2, err := GetSegment(engine, "testing", &id)
if err != nil {
t.Error(err)
}
assert.Equal(t, s.Transaction, s2.Transaction)
assert.Equal(t, *s.Next, *s2.Next)
}
func BenchmarkGetSegment(b *testing.B) {
engine, _ := origins.Init("memory", nil)
id := uuid.NewV4()
next := uuid.NewV4()
now := chrono.Norm(time.Now().UTC())
s := &Segment{
UUID: &id,
Transaction: 10,
Domain: "testing",
Time: now,
Blocks: 5,
Count: 4800,
Bytes: 460800,
Next: &next,
Base: &next,
}
SetSegment(engine, "testing", s)
b.ResetTimer()
for i := 0; i < b.N; i++ {
GetSegment(engine, "testing", &id)
}
}
func TestLogMethods(t *testing.T) {
engine, _ := origins.Init("memory", nil)
id := uuid.NewV4()
l := Log{
Name: "commit",
Domain: "testing",
Head: &id,
}
_, err := SetLog(engine, "testing", &l)
if err != nil {
t.Error(err)
}
l2, err := GetLog(engine, "testing", "commit")
if err != nil {
t.Error(err)
}
assert.Equal(t, l.Name, l2.Name)
assert.Equal(t, l.Domain, l2.Domain)
assert.Equal(t, *l.Head, *l2.Head)
}
func TestSegment(t *testing.T) {
engine, err := origins.Init("mem", nil)
segment := NewSegment(engine, "test", 1)
gen := testutil.NewRandGenerator("test", 1, 0)
var (
f *origins.Fact
n = 10000
)
// Write facts to segment.
for i := 0; i < n; i++ {
f = gen.Next()
if err = segment.Write(f); err != nil {
t.Fatal(err)
}
}
// Compare counts.
if segment.Count != n {
t.Errorf("segment: expected %d count, got %d", n, segment.Count)
}
// Compare expected number of blocks.
if segment.Blocks != n/blockSize {
t.Errorf("segment: expected %d blocks, got %d", n/blockSize, segment.Blocks)
}
// Validate the blocks have been written to storage (even though the
// data has been committed).
id := segment.UUID
for i := 0; i < segment.Blocks; i++ {
bl, _ := dal.GetBlock(engine, "test", id, i)
if bl == nil {
t.Errorf("segment: block %s.%d is nil", id, i)
}
}
// Abort the segment.
if err = segment.Abort(engine); err != nil {
t.Error("segment: abort failed %s", err)
}
for i := 0; i < segment.Blocks; i++ {
bl, _ := dal.GetBlock(engine, "test", id, i)
if bl != nil {
t.Fatalf("segment: %s.%d should be nil", id, i)
}
}
}
func BenchmarkSetLog(b *testing.B) {
engine, _ := origins.Init("memory", nil)
id := uuid.NewV4()
l := &Log{
Name: "commit",
Head: &id,
}
for i := 0; i < b.N; i++ {
SetLog(engine, "testing", l)
}
}
func TestBlockMethods(t *testing.T) {
engine, _ := origins.Init("memory", nil)
id := uuid.NewV4()
idx := 0
f := &origins.Fact{
Domain: "testing",
Operation: origins.Assertion,
Transaction: 5,
Time: chrono.Norm(time.Now()),
Entity: &origins.Ident{
Domain: "testing",
Name: "field",
},
Attribute: &origins.Ident{
Domain: "testing",
Name: "dataType",
},
Value: &origins.Ident{
Name: "string",
},
}
encoder := NewBlockEncoder()
for i := 0; i < 1000; i++ {
encoder.Write(f)
}
_, err := SetBlock(engine, "testing", &id, idx, encoder.Bytes())
if err != nil {
t.Error(err)
}
_, err = GetBlock(engine, "testing", &id, idx)
if err != nil {
t.Error(err)
}
}
// Initializes an in-memory store and generates n transactions each with m
// randomly generated facts that belong to one of the specified domains.
func randMultidomainStorage(domains []string, n, m int) storage.Engine {
engine, _ := origins.Init("memory", nil)
for i := 0; i < m; i++ {
tx, _ := transactor.New(engine, transactor.Options{
AllowDuplicates: true,
})
gen := testutil.NewMultidomainGenerator(domains, tx.ID, n)
origins.Copy(gen, tx)
tx.Commit()
}
return engine
}
// Initializes an in-memory store and generates n transactions each with m
// facts randomly generated from the same dictionary of possible E, A, V values.
// Varying the size of the dictionary relative to the size of the store
// allows to guarantee repeating facts.
func randStorageWRepeats(domain string, n, m, eLen, aLen, vLen int) storage.Engine {
engine, _ := origins.Init("memory", nil)
dictionary := testutil.NewEAVDictionary(eLen, aLen, vLen)
for i := 0; i < m; i++ {
tx, _ := transactor.New(engine, transactor.Options{
AllowDuplicates: true,
})
gen := testutil.NewDictionaryBasedGenerator(dictionary, domain, tx.ID, n)
origins.Copy(gen, tx)
tx.Commit()
}
return engine
}
func setup() storage.Engine {
engine, _ := origins.Init("memory", nil)
data, _ := testutil.Asset("assets/origins.csv")
iter := origins.NewCSVReader(bytes.NewBuffer(data))
tx, _ := transactor.New(engine, transactor.Options{})
// Write the facts.
if _, err := origins.Copy(iter, tx); err != nil {
panic(err)
}
tx.Commit()
return engine
}
func BenchmarkGetBlock(b *testing.B) {
engine, _ := origins.Init("memory", nil)
id := uuid.NewV4()
idx := 0
f := &origins.Fact{
Domain: "testing",
Operation: origins.Assertion,
Transaction: 5,
Time: chrono.Norm(time.Now()),
Entity: &origins.Ident{
Domain: "testing",
Name: "field",
},
Attribute: &origins.Ident{
Domain: "testing",
Name: "dataType",
},
Value: &origins.Ident{
Name: "string",
},
}
encoder := NewBlockEncoder()
for i := 0; i < 1000; i++ {
encoder.Write(f)
}
SetBlock(engine, "testing", &id, idx, encoder.Bytes())
b.ResetTimer()
for i := 0; i < b.N; i++ {
GetBlock(engine, "testing", &id, idx)
}
}
func benchSegmentSize(b *testing.B, n int) {
engine, _ := origins.Init("mem", nil)
var f *origins.Fact
for i := 0; i < b.N; i++ {
gen := testutil.NewRandGenerator("test", 1, n)
segment := NewSegment(engine, "test", 1)
for j := 0; j < n; j++ {
f = gen.Next()
segment.Write(f)
}
engine.Multi(func(tx storage.Tx) error {
return segment.Commit(tx)
})
}
}
func benchSegmentBlockSize(b *testing.B, bs int) {
engine, _ := origins.Init("mem", nil)
blockSize = bs
var f *origins.Fact
gen := testutil.NewRandGenerator("test", 1, 0)
segment := NewSegment(engine, "test", 1)
for i := 0; i < b.N; i++ {
f = gen.Next()
segment.Write(f)
}
engine.Multi(func(tx storage.Tx) error {
return segment.Commit(tx)
})
blockSize = 1000
}
// Commands can call this if it requires use of the store.
func initStorage() storage.Engine {
// Name of the storage engine.
name := viper.GetString("storage")
path := viper.GetString("path")
// Directory of the config file. Ensure the storage engine
// path is resolved relative to the config file.
dir := filepath.Dir(viper.ConfigFileUsed())
path = filepath.Join(dir, path)
// Supported options.
opts := storage.Options{
"path": path,
}
// Initialize the storage engine.
engine, err := origins.Init(name, &opts)
if err != nil {
logrus.Fatal("storage:", err)
}
return engine
}