func BenchmarkConsecutivePut(b *testing.B) {
r := rand.New()
var blocks [][]byte
var keys []datastore.Key
for i := 0; i < b.N; i++ {
blk := make([]byte, 256*1024)
r.Read(blk)
blocks = append(blocks, blk)
key := base32.StdEncoding.EncodeToString(blk[:8])
keys = append(keys, datastore.NewKey(key))
}
temp, cleanup := tempdir(b)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
b.Fatalf("New fail: %v\n", err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := fs.Put(keys[i], blocks[i])
if err != nil {
b.Fatal(err)
}
}
}
func TestGet(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
const input = "foobar"
err = fs.Put(datastore.NewKey("quux"), []byte(input))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
data, err := fs.Get(datastore.NewKey("quux"))
if err != nil {
t.Fatalf("Get failed: %v", err)
}
buf, ok := data.([]byte)
if !ok {
t.Fatalf("expected []byte from Get, got %T: %v", data, data)
}
if g, e := string(buf), input; g != e {
t.Fatalf("Get gave wrong content: %q != %q", g, e)
}
}
func TestPutOverwrite(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
const (
loser = "foobar"
winner = "xyzzy"
)
err = fs.Put(datastore.NewKey("quux"), []byte(loser))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte(winner))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
data, err := fs.Get(datastore.NewKey("quux"))
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if g, e := string(data.([]byte)), winner; g != e {
t.Fatalf("Get gave wrong content: %q != %q", g, e)
}
}
func TestQuerySimple(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
const myKey = "quux"
err = fs.Put(datastore.NewKey(myKey), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
res, err := fs.Query(query.Query{KeysOnly: true})
if err != nil {
t.Fatalf("Query fail: %v\n", err)
}
entries, err := res.Rest()
if err != nil {
t.Fatalf("Query Results.Rest fail: %v\n", err)
}
seen := false
for _, e := range entries {
switch e.Key {
case datastore.NewKey(myKey).String():
seen = true
default:
t.Errorf("saw unexpected key: %q", e.Key)
}
}
if !seen {
t.Errorf("did not see wanted key %q in %+v", myKey, entries)
}
}
func TestStorage(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
const prefixLen = 2
const prefix = "7175"
const target = prefix + string(os.PathSeparator) + "71757578.data"
fs, err := flatfs.New(temp, prefixLen)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
seen := false
walk := func(absPath string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
path, err := filepath.Rel(temp, absPath)
if err != nil {
return err
}
switch path {
case ".", "..":
// ignore
case prefix:
if !fi.IsDir() {
t.Errorf("prefix directory is not a file? %v", fi.Mode())
}
// we know it's there if we see the file, nothing more to
// do here
case target:
seen = true
if !fi.Mode().IsRegular() {
t.Errorf("expected a regular file, mode: %04o", fi.Mode())
}
if runtime.GOOS != "windows" {
if g, e := fi.Mode()&os.ModePerm&0007, os.FileMode(0000); g != e {
t.Errorf("file should not be world accessible: %04o", fi.Mode())
}
}
default:
t.Errorf("saw unexpected directory entry: %q %v", path, fi.Mode())
}
return nil
}
if err := filepath.Walk(temp, walk); err != nil {
t.Fatal("walk: %v", err)
}
if !seen {
t.Error("did not see the data file")
}
}
// openDatastore returns an error if the config file is not present.
func (r *FSRepo) openDatastore() error {
leveldbPath := path.Join(r.path, leveldbDirectory)
var err error
// save leveldb reference so it can be neatly closed afterward
leveldbDS, err := levelds.NewDatastore(leveldbPath, &levelds.Options{
Compression: ldbopts.NoCompression,
})
if err != nil {
return errors.New("unable to open leveldb datastore")
}
// 4TB of 256kB objects ~=17M objects, splitting that 256-way
// leads to ~66k objects per dir, splitting 256*256-way leads to
// only 256.
//
// The keys seen by the block store have predictable prefixes,
// including "/" from datastore.Key and 2 bytes from multihash. To
// reach a uniform 256-way split, we need approximately 4 bytes of
// prefix.
blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
if err != nil {
return errors.New("unable to open flatfs datastore")
}
// Add our PeerID to metrics paths to keep them unique
//
// As some tests just pass a zero-value Config to fsrepo.Init,
// cope with missing PeerID.
id := r.config.Identity.PeerID
if id == "" {
// the tests pass in a zero Config; cope with it
id = fmt.Sprintf("uninitialized_%p", r)
}
prefix := "fsrepo." + id + ".datastore."
metricsBlocks := measure.New(prefix+"blocks", blocksDS)
metricsLevelDB := measure.New(prefix+"leveldb", leveldbDS)
mountDS := mount.New([]mount.Mount{
{
Prefix: ds.NewKey("/blocks"),
Datastore: metricsBlocks,
},
{
Prefix: ds.NewKey("/"),
Datastore: metricsLevelDB,
},
})
// Make sure it's ok to claim the virtual datastore from mount as
// threadsafe. There's no clean way to make mount itself provide
// this information without copy-pasting the code into two
// variants. This is the same dilemma as the `[].byte` attempt at
// introducing const types to Go.
var _ ds.ThreadSafeDatastore = blocksDS
var _ ds.ThreadSafeDatastore = leveldbDS
r.ds = ds2.ClaimThreadSafe{mountDS}
return nil
}
func TestBatchDelete(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
dstest.RunBatchDeleteTest(t, fs)
}
func TestBadPrefixLen(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
for i := 0; i > -3; i-- {
_, err := flatfs.New(temp, 0)
if g, e := err, flatfs.ErrBadPrefixLen; g != e {
t.Errorf("expected ErrBadPrefixLen, got: %v", g)
}
}
}
func TestPut(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
}
func TestPutBadValueType(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), 22)
if g, e := err, datastore.ErrInvalidType; g != e {
t.Fatalf("expected ErrInvalidType, got: %v\n", g)
}
}
func TestDeleteNotFound(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Delete(datastore.NewKey("quux"))
if g, e := err, datastore.ErrNotFound; g != e {
t.Fatalf("expected ErrNotFound, got: %v\n", g)
}
}
func TestHasNotFound(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
found, err := fs.Has(datastore.NewKey("quux"))
if err != nil {
t.Fatalf("Has fail: %v\n", err)
}
if g, e := found, false; g != e {
t.Fatalf("wrong Has: %v != %v", g, e)
}
}
func TestHasFound(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
found, err := fs.Has(datastore.NewKey("quux"))
if err != nil {
t.Fatalf("Has fail: %v\n", err)
}
if g, e := found, true; g != e {
t.Fatalf("wrong Has: %v != %v", g, e)
}
}
func TestDeleteFound(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.New(temp, 2)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
err = fs.Delete(datastore.NewKey("quux"))
if err != nil {
t.Fatalf("Delete fail: %v\n", err)
}
// check that it's gone
_, err = fs.Get(datastore.NewKey("quux"))
if g, e := err, datastore.ErrNotFound; g != e {
t.Fatalf("expected Get after Delete to give ErrNotFound, got: %v\n", g)
}
}