func TestFileStore(t *testing.T) { var store chainstore.Store var err error ctx := context.Background() store = chainstore.New(New(tempDir(), 0755)) assert := assert.New(t) err = store.Open() assert.Nil(err) // Put/Get/Del basic data err = store.Put(ctx, "test.txt", []byte{1, 2, 3, 4}) assert.Nil(err) data, err := store.Get(ctx, "test.txt") assert.Nil(err) assert.Equal(data, []byte{1, 2, 3, 4}) // Auto-creating directories on put err = store.Put(ctx, "hello/there/everyone.txt", []byte{1, 2, 3, 4}) assert.Nil(err) }
func (cf *Config) GetChainstore() (chainstore.Store, error) { // First, reset cache storage path err := filepath.Walk( cf.Chainstore.Path, func(path string, info os.FileInfo, err error) error { if cf.Chainstore.Path == path { return nil // skip the root } if err = os.RemoveAll(path); err != nil { return fmt.Errorf("Failed to remove or clean the directory: %s, because: %s", path, err) } return nil }, ) if err != nil { return nil, err } // TODO: impl another kind of lrumgr (or option) to be based on number of keys, not filesize // at which point, we can add a method called .Keys() that will return the keys // matching some query from a Store, and we can seed the LRU this way, and keep // the bolt data.. // Build the stores and setup the chain memStore := memstore.New(cf.Chainstore.MemCacheSize * 1024 * 1024) diskStore := lrumgr.New(cf.Chainstore.DiskCacheSize*1024*1024, metricsmgr.New("fn.store.bolt", nil, levelstore.New(cf.Chainstore.Path), ), ) var store chainstore.Store if cf.Chainstore.S3AccessKey != "" && cf.Chainstore.S3SecretKey != "" { s3Store := metricsmgr.New("fn.store.s3", nil, s3store.New(cf.Chainstore.S3Bucket, cf.Chainstore.S3AccessKey, cf.Chainstore.S3SecretKey), ) store = chainstore.New(memStore, chainstore.Async(diskStore, s3Store)) } else { store = chainstore.New(memStore, chainstore.Async(diskStore)) } if err := store.Open(); err != nil { return nil, err } return store, nil }
func (cf *Config) GetChainstore() (chainstore.Store, error) { // chainstore.DefaultTimeout = 60 * time.Second // TODO: .... // First, reset cache storage path err := filepath.Walk( cf.Chainstore.Path, func(path string, info os.FileInfo, err error) error { if cf.Chainstore.Path == path { return nil // skip the root } if err = os.RemoveAll(path); err != nil { return fmt.Errorf("Failed to remove or clean the directory: %s, because: %s", path, err) } return nil }, ) if err != nil { return nil, err } // Build the stores and setup the chain memStore := metricsmgr.New("fn.store.mem", memstore.New(cf.Chainstore.MemCacheSize*1024*1024), ) diskStore := lrumgr.New(cf.Chainstore.DiskCacheSize*1024*1024, metricsmgr.New("fn.store.bolt", boltstore.New(cf.Chainstore.Path+"store.db", "imgry"), ), ) var store chainstore.Store if cf.Chainstore.S3AccessKey != "" && cf.Chainstore.S3SecretKey != "" { s3Store := metricsmgr.New("fn.store.s3", s3store.New(cf.Chainstore.S3Bucket, cf.Chainstore.S3AccessKey, cf.Chainstore.S3SecretKey), ) // store = chainstore.New(memStore, chainstore.Async(diskStore, s3Store)) store = chainstore.New(memStore, chainstore.Async(nil, s3Store)) } else { store = chainstore.New(memStore, chainstore.Async(nil, diskStore)) } if err := store.Open(); err != nil { return nil, err } return store, nil }
func TestFileStore(t *testing.T) { var store chainstore.Store var err error Convey("Fsdb Open", t, func() { store = filestore.New(chainstore.TempDir(), 0755) err = nil So(err, ShouldEqual, nil) Convey("Put/Get/Del basic data", func() { err = store.Put("test.txt", []byte{1, 2, 3, 4}) So(err, ShouldEqual, nil) data, err := store.Get("test.txt") So(err, ShouldEqual, nil) So(data, ShouldResemble, []byte{1, 2, 3, 4}) }) Convey("Auto-creating directories on put", func() { err = store.Put("hello/there/everyone.txt", []byte{1, 2, 3, 4}) So(err, ShouldEqual, nil) }) }) }
// TestMockStoreCancelWithDefaultTimeout tests automatic operation cancellation // and posterior recover. func TestMockStoreCancelWithDefaultTimeout(t *testing.T) { var store chainstore.Store var err error ctx := context.Background() cfg := &Config{ Capacity: 100, SuccessRate: 1.0, // always succeeds. Delay: time.Second * 1, // any operation takes 0.5s. } store = chainstore.New(New(cfg)) assert := assert.New(t) // This is going to fail because the timeout is lower than the delay. cfg.Timeout = time.Millisecond * 500 err = store.Put(ctx, "notnil", []byte("something")) assert.Equal(chainstore.ErrTimeout, err) // This is going to succeed because the timeout is greater than the delay. cfg.Timeout = time.Millisecond * 1500 err = store.Put(ctx, "notnil", []byte("something")) assert.Nil(err) // This is going to fail because the timeout is lower than the delay. cfg.Timeout = time.Millisecond * 500 _, err = store.Get(ctx, "notnil") assert.Equal(chainstore.ErrTimeout, err) // This is going to succeed because the timeout is greater than the delay. cfg.Timeout = time.Millisecond * 1500 _, err = store.Get(ctx, "notnil") assert.Nil(err) }
func TestMemCacheStore(t *testing.T) { var store chainstore.Store store = memstore.New(10) Convey("MemCacheStore", t, func() { e := store.Put("hi", []byte{1, 2, 3}) So(e, ShouldEqual, nil) obj, e := store.Get("hi") So(e, ShouldEqual, nil) So(obj, ShouldResemble, []byte{1, 2, 3}) e = store.Put("bye", []byte{5, 6, 7, 8, 9, 10, 11, 12}) So(e, ShouldEqual, nil) obj, e = store.Get("hi") So(len(obj), ShouldEqual, 0) obj, e = store.Get("bye") So(len(obj), ShouldEqual, 8) }) }
// TestMockStoreCancelWithTimeout creates and test a mockstore with that after // a while gets cancelled. func TestMockStoreCancelWithTimeout(t *testing.T) { var store chainstore.Store var err error assert := assert.New(t) store = chainstore.New(New(&Config{ Capacity: 100, SuccessRate: 1.0, // always succeeds. Delay: time.Second * 1, // any operation takes 1s. })) ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*500) // After 0.5s this all is going to fail, because the context timed out. err = store.Put(ctx, "notnil", []byte("something")) assert.NotNil(err) err = store.Put(ctx, "hi", []byte{1, 2, 3}) assert.NotNil(err) _, err = store.Get(ctx, "hi") assert.NotNil(err) err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12}) assert.NotNil(err) err = store.Del(ctx, "hi") assert.NotNil(err) _, err = store.Get(ctx, "hi") assert.NotNil(err) _, err = store.Get(ctx, "bye") assert.NotNil(err) _, err = store.Get(ctx, "notnil") assert.NotNil(err) }
// TestMockStoreFail creates and test a mockstore that always fails. func TestMockStoreFail(t *testing.T) { var store chainstore.Store var err error ctx := context.Background() store = chainstore.New(New(&Config{ Capacity: 100, SuccessRate: 0.0, // always succeeds. })) assert := assert.New(t) err = store.Put(ctx, "notnil", []byte("something")) assert.NotNil(err) err = store.Put(ctx, "hi", []byte{1, 2, 3}) assert.NotNil(err) _, err = store.Get(ctx, "hi") assert.NotNil(err) err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12}) assert.NotNil(err) err = store.Del(ctx, "hi") assert.NotNil(err) _, err = store.Get(ctx, "hi") assert.NotNil(err) _, err = store.Get(ctx, "bye") assert.NotNil(err) _, err = store.Get(ctx, "notnil") assert.NotNil(err) }
func TestBasicChain(t *testing.T) { var ms, fs, chain chainstore.Store var err error logger := log.New(os.Stdout, "", log.LstdFlags) Convey("Basic chain", t, func() { storeDir := chainstore.TempDir() err = nil ms = memstore.New(100) fs = filestore.New(storeDir+"/filestore", 0755) chain = chainstore.New( logmgr.New(logger, ""), ms, fs, ) err = chain.Open() So(err, ShouldEqual, nil) Convey("Put/Get/Del", func() { v := []byte("value") err = chain.Put("k", v) So(err, ShouldEqual, nil) val, err := chain.Get("k") So(err, ShouldEqual, nil) So(v, ShouldResemble, v) val, err = ms.Get("k") So(err, ShouldEqual, nil) So(val, ShouldResemble, v) val, err = fs.Get("k") So(err, ShouldEqual, nil) So(val, ShouldResemble, v) err = chain.Del("k") So(err, ShouldEqual, nil) val, err = fs.Get("k") So(err, ShouldEqual, nil) So(len(val), ShouldEqual, 0) val, err = chain.Get("woo!@#") So(err, ShouldNotBeNil) }) }) }
func TestS3Store(t *testing.T) { var store chainstore.Store var err error ctx := context.Background() assert := assert.New(t) store = chainstore.New(New(bucketID, accessKey, secretKey)) err = store.Open() assert.Nil(err) defer store.Close() // Put a bunch of objects e1 := store.Put(ctx, "hi", []byte{1, 2, 3}) e2 := store.Put(ctx, "bye", []byte{4, 5, 6}) assert.Nil(e1) assert.Nil(e2) // Get those objects v1, _ := store.Get(ctx, "hi") v2, _ := store.Get(ctx, "bye") assert.Equal(v1, []byte{1, 2, 3}) assert.Equal(v2, []byte{4, 5, 6}) // Delete those objects e1 = store.Del(ctx, "hi") e2 = store.Del(ctx, "bye") assert.Equal(e1, nil) assert.Equal(e2, nil) }
func TestBasicChain(t *testing.T) { var ms, fs, chain chainstore.Store var err error ctx := context.Background() logger := log.New(os.Stdout, "", log.LstdFlags) storeDir := tempDir() err = nil ms = memstore.New(100) fs = filestore.New(storeDir+"/filestore", 0755) chain = chainstore.New( logmgr.New(logger, ""), ms, fs, ) assert := assert.New(t) err = chain.Open() assert.Nil(err) v := []byte("value") err = chain.Put(ctx, "k", v) assert.Nil(err) val, err := chain.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) val, err = ms.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) val, err = fs.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) err = chain.Del(ctx, "k") assert.Nil(err) val, err = fs.Get(ctx, "k") assert.Nil(err) assert.Equal(len(val), 0) val, err = chain.Get(ctx, "woo!@#") assert.NotNil(err) }
func TestMetricsMgrStore(t *testing.T) { var store chainstore.Store var err error ctx := context.Background() assert := assert.New(t) store = chainstore.New(New("ns")) err = store.Open() assert.Nil(err) defer store.Close() // Put a bunch of objects e1 := store.Put(ctx, "hi", []byte{1, 2, 3}) e2 := store.Put(ctx, "bye", []byte{4, 5, 6}) assert.Nil(e1) assert.Nil(e2) // Delete those objects e1 = store.Del(ctx, "hi") e2 = store.Del(ctx, "bye") assert.Equal(e1, nil) assert.Equal(e2, nil) }
func TestBoltStore(t *testing.T) { var store chainstore.Store var err error store = boltstore.New(chainstore.TempDir()+"/test.db", "test") err = store.Open() if err != nil { t.Error(err) } defer store.Close() // does this get called? Convey("Boltdb Open", t, func() { Convey("Put a bunch of objects", func() { e1 := store.Put("hi", []byte{1, 2, 3}) e2 := store.Put("bye", []byte{4, 5, 6}) So(e1, ShouldEqual, nil) So(e2, ShouldEqual, nil) }) Convey("Get those objects", func() { v1, _ := store.Get("hi") v2, _ := store.Get("bye") So(v1, ShouldResemble, []byte{1, 2, 3}) So(v2, ShouldResemble, []byte{4, 5, 6}) }) Convey("Delete those objects", func() { e1 := store.Del("hi") e2 := store.Del("bye") So(e1, ShouldEqual, nil) So(e2, ShouldEqual, nil) v, _ := store.Get("hi") So(len(v), ShouldEqual, 0) }) }) }
func TestAsyncChain(t *testing.T) { var ms, fs, bs, chain chainstore.Store var err error logger := log.New(os.Stdout, "", log.LstdFlags) Convey("Async chain", t, func() { storeDir := chainstore.TempDir() err = nil ms = memstore.New(100) fs = filestore.New(storeDir+"/filestore", 0755) bs = boltstore.New(storeDir+"/boltstore/bolt.db", "test") chain = chainstore.New( logmgr.New(logger, ""), ms, chainstore.Async( logmgr.New(logger, "async"), metricsmgr.New("chaintest", nil, fs, lrumgr.New(100, bs), ), ), ) err = chain.Open() So(err, ShouldEqual, nil) Convey("Put/Get/Del", func() { v := []byte("value") err = chain.Put("k", v) So(err, ShouldEqual, nil) val, err := chain.Get("k") So(err, ShouldEqual, nil) So(v, ShouldResemble, v) val, err = ms.Get("k") So(err, ShouldEqual, nil) So(val, ShouldResemble, v) time.Sleep(10e6) // wait for async operation.. val, err = fs.Get("k") So(err, ShouldEqual, nil) So(val, ShouldResemble, v) val, err = bs.Get("k") So(err, ShouldEqual, nil) So(val, ShouldResemble, v) }) }) }
func TestLevelStore(t *testing.T) { var store chainstore.Store var err error store = levelstore.New(chainstore.TempDir()) err = store.Open() if err != nil { t.Error(err) } defer store.Close() Convey("Leveldb Open", t, func() { Convey("Put a bunch of objects", func() { e1 := store.Put("hi", []byte{1, 2, 3}) e2 := store.Put("bye", []byte{4, 5, 6}) So(e1, ShouldEqual, nil) So(e2, ShouldEqual, nil) }) Convey("Get those objects", func() { v1, _ := store.Get("hi") v2, _ := store.Get("bye") So(v1, ShouldResemble, []byte{1, 2, 3}) So(v2, ShouldResemble, []byte{4, 5, 6}) }) Convey("Delete those objects", func() { e1 := store.Del("hi") e2 := store.Del("bye") So(e1, ShouldEqual, nil) So(e2, ShouldEqual, nil) }) }) }
func TestMemCacheStore(t *testing.T) { var store chainstore.Store var err error var obj []byte ctx := context.Background() store = chainstore.New(New(10)) assert := assert.New(t) err = store.Put(ctx, "hi", []byte{1, 2, 3}) assert.Nil(err) obj, err = store.Get(ctx, "hi") assert.Nil(err) assert.Equal(obj, []byte{1, 2, 3}) err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12}) assert.Nil(err) obj, err = store.Get(ctx, "hi") assert.Equal(len(obj), 0) obj, err = store.Get(ctx, "bye") assert.Equal(len(obj), 8) }
func TestAsyncChain(t *testing.T) { var ms, fs, bs, chain chainstore.Store var err error logger := log.New(os.Stdout, "", log.LstdFlags) storeDir := tempDir() var errored atomic.Value ms = memstore.New(100) fs = filestore.New(storeDir+"/filestore", 0755) bs = boltstore.New(storeDir+"/boltstore/bolt.db", "test") chain = chainstore.New( logmgr.New(logger, ""), ms, chainstore.Async( func(err error) { log.Println("async error:", err) errored.Store(true) }, logmgr.New(logger, "async"), &testStore{}, metricsmgr.New("chaintest", fs, lrumgr.New(100, bs), ), ), ) ctx := context.Background() assert := assert.New(t) err = chain.Open() assert.Nil(err) v := []byte("value") err = chain.Put(ctx, "k", v) assert.Nil(err) val, err := chain.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) val, err = ms.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) time.Sleep(time.Second * 1) // wait for async operation.. val, err = fs.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) val, err = bs.Get(ctx, "k") assert.Nil(err) assert.Equal(val, v) //-- // Lets make an error in async store assert.Nil(errored.Load()) err = chain.Put(ctx, "bad", []byte("v")) assert.Nil(err) // no error because sync store took it fine time.Sleep(time.Second * 1) // wait for async operation.. assert.NotEmpty(errored.Load()) }
// TestMockStoreCancelWithFunc creates and test a mockstore that succeeds at // first but then is cancelled. func TestMockStoreCancelWithFunc(t *testing.T) { var store chainstore.Store var err error ctx, cancel := context.WithCancel(context.Background()) store = chainstore.New(New(&Config{ Capacity: 100, SuccessRate: 1.0, // always succeeds. Delay: time.Second * 1, // any operation takes 1s. })) go func() { time.Sleep(time.Millisecond * 1500) cancel() }() assert := assert.New(t) // This is going to succeed. err = store.Put(ctx, "notnil", []byte("something")) assert.Nil(err) // This will fail because after 1.5s the context will send a cancellation signal. err = store.Put(ctx, "hi", []byte{1, 2, 3}) assert.NotNil(err) _, err = store.Get(ctx, "hi") assert.NotNil(err) err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12}) assert.NotNil(err) err = store.Del(ctx, "hi") assert.NotNil(err) _, err = store.Get(ctx, "hi") assert.NotNil(err) _, err = store.Get(ctx, "bye") assert.NotNil(err) _, err = store.Get(ctx, "notnil") assert.NotNil(err) }
func TestBoltStore(t *testing.T) { var store chainstore.Store var err error ctx := context.Background() store = chainstore.New(New(tempDir()+"/test.db", "test")) assert := assert.New(t) err = store.Open() assert.Nil(err) defer store.Close() // does this get called? // Put a bunch of objects e1 := store.Put(ctx, "hi", []byte{1, 2, 3}) e2 := store.Put(ctx, "bye", []byte{4, 5, 6}) assert.Equal(e1, nil) assert.Equal(e2, nil) // Get those objects v1, _ := store.Get(ctx, "hi") v2, _ := store.Get(ctx, "bye") assert.Equal(v1, []byte{1, 2, 3}) assert.Equal(v2, []byte{4, 5, 6}) // Delete those objects e1 = store.Del(ctx, "hi") e2 = store.Del(ctx, "bye") assert.Equal(e1, nil) assert.Equal(e2, nil) v, _ := store.Get(ctx, "hi") assert.Equal(len(v), 0) }