func TestFileStore(t *testing.T) {
var store chainstore.Store
var err error
ctx := context.Background()
store = chainstore.New(New(tempDir(), 0755))
assert := assert.New(t)
err = store.Open()
assert.Nil(err)
// Put/Get/Del basic data
err = store.Put(ctx, "test.txt", []byte{1, 2, 3, 4})
assert.Nil(err)
data, err := store.Get(ctx, "test.txt")
assert.Nil(err)
assert.Equal(data, []byte{1, 2, 3, 4})
// Auto-creating directories on put
err = store.Put(ctx, "hello/there/everyone.txt", []byte{1, 2, 3, 4})
assert.Nil(err)
}
func (cf *Config) GetChainstore() (chainstore.Store, error) {
// First, reset cache storage path
err := filepath.Walk(
cf.Chainstore.Path,
func(path string, info os.FileInfo, err error) error {
if cf.Chainstore.Path == path {
return nil // skip the root
}
if err = os.RemoveAll(path); err != nil {
return fmt.Errorf("Failed to remove or clean the directory: %s, because: %s", path, err)
}
return nil
},
)
if err != nil {
return nil, err
}
// TODO: impl another kind of lrumgr (or option) to be based on number of keys, not filesize
// at which point, we can add a method called .Keys() that will return the keys
// matching some query from a Store, and we can seed the LRU this way, and keep
// the bolt data..
// Build the stores and setup the chain
memStore := memstore.New(cf.Chainstore.MemCacheSize * 1024 * 1024)
diskStore := lrumgr.New(cf.Chainstore.DiskCacheSize*1024*1024,
metricsmgr.New("fn.store.bolt", nil,
levelstore.New(cf.Chainstore.Path),
),
)
var store chainstore.Store
if cf.Chainstore.S3AccessKey != "" && cf.Chainstore.S3SecretKey != "" {
s3Store := metricsmgr.New("fn.store.s3", nil,
s3store.New(cf.Chainstore.S3Bucket, cf.Chainstore.S3AccessKey, cf.Chainstore.S3SecretKey),
)
store = chainstore.New(memStore, chainstore.Async(diskStore, s3Store))
} else {
store = chainstore.New(memStore, chainstore.Async(diskStore))
}
if err := store.Open(); err != nil {
return nil, err
}
return store, nil
}
func (cf *Config) GetChainstore() (chainstore.Store, error) {
// chainstore.DefaultTimeout = 60 * time.Second // TODO: ....
// First, reset cache storage path
err := filepath.Walk(
cf.Chainstore.Path,
func(path string, info os.FileInfo, err error) error {
if cf.Chainstore.Path == path {
return nil // skip the root
}
if err = os.RemoveAll(path); err != nil {
return fmt.Errorf("Failed to remove or clean the directory: %s, because: %s", path, err)
}
return nil
},
)
if err != nil {
return nil, err
}
// Build the stores and setup the chain
memStore := metricsmgr.New("fn.store.mem",
memstore.New(cf.Chainstore.MemCacheSize*1024*1024),
)
diskStore := lrumgr.New(cf.Chainstore.DiskCacheSize*1024*1024,
metricsmgr.New("fn.store.bolt",
boltstore.New(cf.Chainstore.Path+"store.db", "imgry"),
),
)
var store chainstore.Store
if cf.Chainstore.S3AccessKey != "" && cf.Chainstore.S3SecretKey != "" {
s3Store := metricsmgr.New("fn.store.s3",
s3store.New(cf.Chainstore.S3Bucket, cf.Chainstore.S3AccessKey, cf.Chainstore.S3SecretKey),
)
// store = chainstore.New(memStore, chainstore.Async(diskStore, s3Store))
store = chainstore.New(memStore, chainstore.Async(nil, s3Store))
} else {
store = chainstore.New(memStore, chainstore.Async(nil, diskStore))
}
if err := store.Open(); err != nil {
return nil, err
}
return store, nil
}
func TestFileStore(t *testing.T) {
var store chainstore.Store
var err error
Convey("Fsdb Open", t, func() {
store = filestore.New(chainstore.TempDir(), 0755)
err = nil
So(err, ShouldEqual, nil)
Convey("Put/Get/Del basic data", func() {
err = store.Put("test.txt", []byte{1, 2, 3, 4})
So(err, ShouldEqual, nil)
data, err := store.Get("test.txt")
So(err, ShouldEqual, nil)
So(data, ShouldResemble, []byte{1, 2, 3, 4})
})
Convey("Auto-creating directories on put", func() {
err = store.Put("hello/there/everyone.txt", []byte{1, 2, 3, 4})
So(err, ShouldEqual, nil)
})
})
}
// TestMockStoreCancelWithDefaultTimeout tests automatic operation cancellation
// and posterior recover.
func TestMockStoreCancelWithDefaultTimeout(t *testing.T) {
var store chainstore.Store
var err error
ctx := context.Background()
cfg := &Config{
Capacity: 100,
SuccessRate: 1.0, // always succeeds.
Delay: time.Second * 1, // any operation takes 0.5s.
}
store = chainstore.New(New(cfg))
assert := assert.New(t)
// This is going to fail because the timeout is lower than the delay.
cfg.Timeout = time.Millisecond * 500
err = store.Put(ctx, "notnil", []byte("something"))
assert.Equal(chainstore.ErrTimeout, err)
// This is going to succeed because the timeout is greater than the delay.
cfg.Timeout = time.Millisecond * 1500
err = store.Put(ctx, "notnil", []byte("something"))
assert.Nil(err)
// This is going to fail because the timeout is lower than the delay.
cfg.Timeout = time.Millisecond * 500
_, err = store.Get(ctx, "notnil")
assert.Equal(chainstore.ErrTimeout, err)
// This is going to succeed because the timeout is greater than the delay.
cfg.Timeout = time.Millisecond * 1500
_, err = store.Get(ctx, "notnil")
assert.Nil(err)
}
func TestMemCacheStore(t *testing.T) {
var store chainstore.Store
store = memstore.New(10)
Convey("MemCacheStore", t, func() {
e := store.Put("hi", []byte{1, 2, 3})
So(e, ShouldEqual, nil)
obj, e := store.Get("hi")
So(e, ShouldEqual, nil)
So(obj, ShouldResemble, []byte{1, 2, 3})
e = store.Put("bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
So(e, ShouldEqual, nil)
obj, e = store.Get("hi")
So(len(obj), ShouldEqual, 0)
obj, e = store.Get("bye")
So(len(obj), ShouldEqual, 8)
})
}
// TestMockStoreCancelWithTimeout creates and test a mockstore with that after
// a while gets cancelled.
func TestMockStoreCancelWithTimeout(t *testing.T) {
var store chainstore.Store
var err error
assert := assert.New(t)
store = chainstore.New(New(&Config{
Capacity: 100,
SuccessRate: 1.0, // always succeeds.
Delay: time.Second * 1, // any operation takes 1s.
}))
ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*500)
// After 0.5s this all is going to fail, because the context timed out.
err = store.Put(ctx, "notnil", []byte("something"))
assert.NotNil(err)
err = store.Put(ctx, "hi", []byte{1, 2, 3})
assert.NotNil(err)
_, err = store.Get(ctx, "hi")
assert.NotNil(err)
err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
assert.NotNil(err)
err = store.Del(ctx, "hi")
assert.NotNil(err)
_, err = store.Get(ctx, "hi")
assert.NotNil(err)
_, err = store.Get(ctx, "bye")
assert.NotNil(err)
_, err = store.Get(ctx, "notnil")
assert.NotNil(err)
}
// TestMockStoreFail creates and test a mockstore that always fails.
func TestMockStoreFail(t *testing.T) {
var store chainstore.Store
var err error
ctx := context.Background()
store = chainstore.New(New(&Config{
Capacity: 100,
SuccessRate: 0.0, // always succeeds.
}))
assert := assert.New(t)
err = store.Put(ctx, "notnil", []byte("something"))
assert.NotNil(err)
err = store.Put(ctx, "hi", []byte{1, 2, 3})
assert.NotNil(err)
_, err = store.Get(ctx, "hi")
assert.NotNil(err)
err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
assert.NotNil(err)
err = store.Del(ctx, "hi")
assert.NotNil(err)
_, err = store.Get(ctx, "hi")
assert.NotNil(err)
_, err = store.Get(ctx, "bye")
assert.NotNil(err)
_, err = store.Get(ctx, "notnil")
assert.NotNil(err)
}
func TestBasicChain(t *testing.T) {
var ms, fs, chain chainstore.Store
var err error
logger := log.New(os.Stdout, "", log.LstdFlags)
Convey("Basic chain", t, func() {
storeDir := chainstore.TempDir()
err = nil
ms = memstore.New(100)
fs = filestore.New(storeDir+"/filestore", 0755)
chain = chainstore.New(
logmgr.New(logger, ""),
ms,
fs,
)
err = chain.Open()
So(err, ShouldEqual, nil)
Convey("Put/Get/Del", func() {
v := []byte("value")
err = chain.Put("k", v)
So(err, ShouldEqual, nil)
val, err := chain.Get("k")
So(err, ShouldEqual, nil)
So(v, ShouldResemble, v)
val, err = ms.Get("k")
So(err, ShouldEqual, nil)
So(val, ShouldResemble, v)
val, err = fs.Get("k")
So(err, ShouldEqual, nil)
So(val, ShouldResemble, v)
err = chain.Del("k")
So(err, ShouldEqual, nil)
val, err = fs.Get("k")
So(err, ShouldEqual, nil)
So(len(val), ShouldEqual, 0)
val, err = chain.Get("woo!@#")
So(err, ShouldNotBeNil)
})
})
}
func TestS3Store(t *testing.T) {
var store chainstore.Store
var err error
ctx := context.Background()
assert := assert.New(t)
store = chainstore.New(New(bucketID, accessKey, secretKey))
err = store.Open()
assert.Nil(err)
defer store.Close()
// Put a bunch of objects
e1 := store.Put(ctx, "hi", []byte{1, 2, 3})
e2 := store.Put(ctx, "bye", []byte{4, 5, 6})
assert.Nil(e1)
assert.Nil(e2)
// Get those objects
v1, _ := store.Get(ctx, "hi")
v2, _ := store.Get(ctx, "bye")
assert.Equal(v1, []byte{1, 2, 3})
assert.Equal(v2, []byte{4, 5, 6})
// Delete those objects
e1 = store.Del(ctx, "hi")
e2 = store.Del(ctx, "bye")
assert.Equal(e1, nil)
assert.Equal(e2, nil)
}
func TestBasicChain(t *testing.T) {
var ms, fs, chain chainstore.Store
var err error
ctx := context.Background()
logger := log.New(os.Stdout, "", log.LstdFlags)
storeDir := tempDir()
err = nil
ms = memstore.New(100)
fs = filestore.New(storeDir+"/filestore", 0755)
chain = chainstore.New(
logmgr.New(logger, ""),
ms,
fs,
)
assert := assert.New(t)
err = chain.Open()
assert.Nil(err)
v := []byte("value")
err = chain.Put(ctx, "k", v)
assert.Nil(err)
val, err := chain.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
val, err = ms.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
val, err = fs.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
err = chain.Del(ctx, "k")
assert.Nil(err)
val, err = fs.Get(ctx, "k")
assert.Nil(err)
assert.Equal(len(val), 0)
val, err = chain.Get(ctx, "woo!@#")
assert.NotNil(err)
}
func TestMetricsMgrStore(t *testing.T) {
var store chainstore.Store
var err error
ctx := context.Background()
assert := assert.New(t)
store = chainstore.New(New("ns"))
err = store.Open()
assert.Nil(err)
defer store.Close()
// Put a bunch of objects
e1 := store.Put(ctx, "hi", []byte{1, 2, 3})
e2 := store.Put(ctx, "bye", []byte{4, 5, 6})
assert.Nil(e1)
assert.Nil(e2)
// Delete those objects
e1 = store.Del(ctx, "hi")
e2 = store.Del(ctx, "bye")
assert.Equal(e1, nil)
assert.Equal(e2, nil)
}
func TestBoltStore(t *testing.T) {
var store chainstore.Store
var err error
store = boltstore.New(chainstore.TempDir()+"/test.db", "test")
err = store.Open()
if err != nil {
t.Error(err)
}
defer store.Close() // does this get called?
Convey("Boltdb Open", t, func() {
Convey("Put a bunch of objects", func() {
e1 := store.Put("hi", []byte{1, 2, 3})
e2 := store.Put("bye", []byte{4, 5, 6})
So(e1, ShouldEqual, nil)
So(e2, ShouldEqual, nil)
})
Convey("Get those objects", func() {
v1, _ := store.Get("hi")
v2, _ := store.Get("bye")
So(v1, ShouldResemble, []byte{1, 2, 3})
So(v2, ShouldResemble, []byte{4, 5, 6})
})
Convey("Delete those objects", func() {
e1 := store.Del("hi")
e2 := store.Del("bye")
So(e1, ShouldEqual, nil)
So(e2, ShouldEqual, nil)
v, _ := store.Get("hi")
So(len(v), ShouldEqual, 0)
})
})
}
func TestAsyncChain(t *testing.T) {
var ms, fs, bs, chain chainstore.Store
var err error
logger := log.New(os.Stdout, "", log.LstdFlags)
Convey("Async chain", t, func() {
storeDir := chainstore.TempDir()
err = nil
ms = memstore.New(100)
fs = filestore.New(storeDir+"/filestore", 0755)
bs = boltstore.New(storeDir+"/boltstore/bolt.db", "test")
chain = chainstore.New(
logmgr.New(logger, ""),
ms,
chainstore.Async(
logmgr.New(logger, "async"),
metricsmgr.New("chaintest", nil,
fs,
lrumgr.New(100, bs),
),
),
)
err = chain.Open()
So(err, ShouldEqual, nil)
Convey("Put/Get/Del", func() {
v := []byte("value")
err = chain.Put("k", v)
So(err, ShouldEqual, nil)
val, err := chain.Get("k")
So(err, ShouldEqual, nil)
So(v, ShouldResemble, v)
val, err = ms.Get("k")
So(err, ShouldEqual, nil)
So(val, ShouldResemble, v)
time.Sleep(10e6) // wait for async operation..
val, err = fs.Get("k")
So(err, ShouldEqual, nil)
So(val, ShouldResemble, v)
val, err = bs.Get("k")
So(err, ShouldEqual, nil)
So(val, ShouldResemble, v)
})
})
}
func TestLevelStore(t *testing.T) {
var store chainstore.Store
var err error
store = levelstore.New(chainstore.TempDir())
err = store.Open()
if err != nil {
t.Error(err)
}
defer store.Close()
Convey("Leveldb Open", t, func() {
Convey("Put a bunch of objects", func() {
e1 := store.Put("hi", []byte{1, 2, 3})
e2 := store.Put("bye", []byte{4, 5, 6})
So(e1, ShouldEqual, nil)
So(e2, ShouldEqual, nil)
})
Convey("Get those objects", func() {
v1, _ := store.Get("hi")
v2, _ := store.Get("bye")
So(v1, ShouldResemble, []byte{1, 2, 3})
So(v2, ShouldResemble, []byte{4, 5, 6})
})
Convey("Delete those objects", func() {
e1 := store.Del("hi")
e2 := store.Del("bye")
So(e1, ShouldEqual, nil)
So(e2, ShouldEqual, nil)
})
})
}
func TestMemCacheStore(t *testing.T) {
var store chainstore.Store
var err error
var obj []byte
ctx := context.Background()
store = chainstore.New(New(10))
assert := assert.New(t)
err = store.Put(ctx, "hi", []byte{1, 2, 3})
assert.Nil(err)
obj, err = store.Get(ctx, "hi")
assert.Nil(err)
assert.Equal(obj, []byte{1, 2, 3})
err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
assert.Nil(err)
obj, err = store.Get(ctx, "hi")
assert.Equal(len(obj), 0)
obj, err = store.Get(ctx, "bye")
assert.Equal(len(obj), 8)
}
func TestAsyncChain(t *testing.T) {
var ms, fs, bs, chain chainstore.Store
var err error
logger := log.New(os.Stdout, "", log.LstdFlags)
storeDir := tempDir()
var errored atomic.Value
ms = memstore.New(100)
fs = filestore.New(storeDir+"/filestore", 0755)
bs = boltstore.New(storeDir+"/boltstore/bolt.db", "test")
chain = chainstore.New(
logmgr.New(logger, ""),
ms,
chainstore.Async(
func(err error) {
log.Println("async error:", err)
errored.Store(true)
},
logmgr.New(logger, "async"),
&testStore{},
metricsmgr.New("chaintest",
fs,
lrumgr.New(100, bs),
),
),
)
ctx := context.Background()
assert := assert.New(t)
err = chain.Open()
assert.Nil(err)
v := []byte("value")
err = chain.Put(ctx, "k", v)
assert.Nil(err)
val, err := chain.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
val, err = ms.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
time.Sleep(time.Second * 1) // wait for async operation..
val, err = fs.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
val, err = bs.Get(ctx, "k")
assert.Nil(err)
assert.Equal(val, v)
//--
// Lets make an error in async store
assert.Nil(errored.Load())
err = chain.Put(ctx, "bad", []byte("v"))
assert.Nil(err) // no error because sync store took it fine
time.Sleep(time.Second * 1) // wait for async operation..
assert.NotEmpty(errored.Load())
}
// TestMockStoreCancelWithFunc creates and test a mockstore that succeeds at
// first but then is cancelled.
func TestMockStoreCancelWithFunc(t *testing.T) {
var store chainstore.Store
var err error
ctx, cancel := context.WithCancel(context.Background())
store = chainstore.New(New(&Config{
Capacity: 100,
SuccessRate: 1.0, // always succeeds.
Delay: time.Second * 1, // any operation takes 1s.
}))
go func() {
time.Sleep(time.Millisecond * 1500)
cancel()
}()
assert := assert.New(t)
// This is going to succeed.
err = store.Put(ctx, "notnil", []byte("something"))
assert.Nil(err)
// This will fail because after 1.5s the context will send a cancellation signal.
err = store.Put(ctx, "hi", []byte{1, 2, 3})
assert.NotNil(err)
_, err = store.Get(ctx, "hi")
assert.NotNil(err)
err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
assert.NotNil(err)
err = store.Del(ctx, "hi")
assert.NotNil(err)
_, err = store.Get(ctx, "hi")
assert.NotNil(err)
_, err = store.Get(ctx, "bye")
assert.NotNil(err)
_, err = store.Get(ctx, "notnil")
assert.NotNil(err)
}
func TestBoltStore(t *testing.T) {
var store chainstore.Store
var err error
ctx := context.Background()
store = chainstore.New(New(tempDir()+"/test.db", "test"))
assert := assert.New(t)
err = store.Open()
assert.Nil(err)
defer store.Close() // does this get called?
// Put a bunch of objects
e1 := store.Put(ctx, "hi", []byte{1, 2, 3})
e2 := store.Put(ctx, "bye", []byte{4, 5, 6})
assert.Equal(e1, nil)
assert.Equal(e2, nil)
// Get those objects
v1, _ := store.Get(ctx, "hi")
v2, _ := store.Get(ctx, "bye")
assert.Equal(v1, []byte{1, 2, 3})
assert.Equal(v2, []byte{4, 5, 6})
// Delete those objects
e1 = store.Del(ctx, "hi")
e2 = store.Del(ctx, "bye")
assert.Equal(e1, nil)
assert.Equal(e2, nil)
v, _ := store.Get(ctx, "hi")
assert.Equal(len(v), 0)
}