func TestGet(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } const input = "foobar" err = fs.Put(datastore.NewKey("quux"), []byte(input)) if err != nil { t.Fatalf("Put fail: %v\n", err) } data, err := fs.Get(datastore.NewKey("quux")) if err != nil { t.Fatalf("Get failed: %v", err) } buf, ok := data.([]byte) if !ok { t.Fatalf("expected []byte from Get, got %T: %v", data, data) } if g, e := string(buf), input; g != e { t.Fatalf("Get gave wrong content: %q != %q", g, e) } }
func TestQuerySimple(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } const myKey = "quux" err = fs.Put(datastore.NewKey(myKey), []byte("foobar")) if err != nil { t.Fatalf("Put fail: %v\n", err) } res, err := fs.Query(query.Query{KeysOnly: true}) if err != nil { t.Fatalf("Query fail: %v\n", err) } entries, err := res.Rest() if err != nil { t.Fatalf("Query Results.Rest fail: %v\n", err) } seen := false for _, e := range entries { switch e.Key { case datastore.NewKey(myKey).String(): seen = true default: t.Errorf("saw unexpected key: %q", e.Key) } } if !seen { t.Errorf("did not see wanted key %q in %+v", myKey, entries) } }
func BenchmarkConsecutivePut(b *testing.B) { r := rand.New() var blocks [][]byte var keys []datastore.Key for i := 0; i < b.N; i++ { blk := make([]byte, 256*1024) r.Read(blk) blocks = append(blocks, blk) key := base32.StdEncoding.EncodeToString(blk[:8]) keys = append(keys, datastore.NewKey(key)) } temp, cleanup := tempdir(b) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { b.Fatalf("New fail: %v\n", err) } b.ResetTimer() for i := 0; i < b.N; i++ { err := fs.Put(keys[i], blocks[i]) if err != nil { b.Fatal(err) } } }
func TestPutOverwrite(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } const ( loser = "foobar" winner = "xyzzy" ) err = fs.Put(datastore.NewKey("quux"), []byte(loser)) if err != nil { t.Fatalf("Put fail: %v\n", err) } err = fs.Put(datastore.NewKey("quux"), []byte(winner)) if err != nil { t.Fatalf("Put fail: %v\n", err) } data, err := fs.Get(datastore.NewKey("quux")) if err != nil { t.Fatalf("Get failed: %v", err) } if g, e := string(data.([]byte)), winner; g != e { t.Fatalf("Get gave wrong content: %q != %q", g, e) } }
func TestStorage(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() const prefixLen = 2 const prefix = "7175" const target = prefix + string(os.PathSeparator) + "71757578.data" fs, err := flatfs.New(temp, prefixLen) if err != nil { t.Fatalf("New fail: %v\n", err) } err = fs.Put(datastore.NewKey("quux"), []byte("foobar")) if err != nil { t.Fatalf("Put fail: %v\n", err) } seen := false walk := func(absPath string, fi os.FileInfo, err error) error { if err != nil { return err } path, err := filepath.Rel(temp, absPath) if err != nil { return err } switch path { case ".", "..": // ignore case prefix: if !fi.IsDir() { t.Errorf("prefix directory is not a file? %v", fi.Mode()) } // we know it's there if we see the file, nothing more to // do here case target: seen = true if !fi.Mode().IsRegular() { t.Errorf("expected a regular file, mode: %04o", fi.Mode()) } if runtime.GOOS != "windows" { if g, e := fi.Mode()&os.ModePerm&0007, os.FileMode(0000); g != e { t.Errorf("file should not be world accessible: %04o", fi.Mode()) } } default: t.Errorf("saw unexpected directory entry: %q %v", path, fi.Mode()) } return nil } if err := filepath.Walk(temp, walk); err != nil { t.Fatal("walk: %v", err) } if !seen { t.Error("did not see the data file") } }
// openDatastore returns an error if the config file is not present. func (r *FSRepo) openDatastore() error { leveldbPath := path.Join(r.path, leveldbDirectory) var err error // save leveldb reference so it can be neatly closed afterward r.leveldbDS, err = levelds.NewDatastore(leveldbPath, &levelds.Options{ Compression: ldbopts.NoCompression, }) if err != nil { return errors.New("unable to open leveldb datastore") } // 4TB of 256kB objects ~=17M objects, splitting that 256-way // leads to ~66k objects per dir, splitting 256*256-way leads to // only 256. // // The keys seen by the block store have predictable prefixes, // including "/" from datastore.Key and 2 bytes from multihash. To // reach a uniform 256-way split, we need approximately 4 bytes of // prefix. blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4) if err != nil { return errors.New("unable to open flatfs datastore") } // Add our PeerID to metrics paths to keep them unique // // As some tests just pass a zero-value Config to fsrepo.Init, // cope with missing PeerID. id := r.config.Identity.PeerID if id == "" { // the tests pass in a zero Config; cope with it id = fmt.Sprintf("uninitialized_%p", r) } prefix := "fsrepo." + id + ".datastore." r.metricsBlocks = measure.New(prefix+"blocks", blocksDS) r.metricsLevelDB = measure.New(prefix+"leveldb", r.leveldbDS) mountDS := mount.New([]mount.Mount{ { Prefix: ds.NewKey("/blocks"), Datastore: r.metricsBlocks, }, { Prefix: ds.NewKey("/"), Datastore: r.metricsLevelDB, }, }) // Make sure it's ok to claim the virtual datastore from mount as // threadsafe. There's no clean way to make mount itself provide // this information without copy-pasting the code into two // variants. This is the same dilemma as the `[].byte` attempt at // introducing const types to Go. var _ ds.ThreadSafeDatastore = blocksDS var _ ds.ThreadSafeDatastore = r.leveldbDS r.ds = ds2.ClaimThreadSafe{mountDS} return nil }
func TestBatchDelete(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } dstest.RunBatchDeleteTest(t, fs) }
func TestBadPrefixLen(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() for i := 0; i > -3; i-- { _, err := flatfs.New(temp, 0) if g, e := err, flatfs.ErrBadPrefixLen; g != e { t.Errorf("expected ErrBadPrefixLen, got: %v", g) } } }
func TestPut(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } err = fs.Put(datastore.NewKey("quux"), []byte("foobar")) if err != nil { t.Fatalf("Put fail: %v\n", err) } }
func TestPutBadValueType(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } err = fs.Put(datastore.NewKey("quux"), 22) if g, e := err, datastore.ErrInvalidType; g != e { t.Fatalf("expected ErrInvalidType, got: %v\n", g) } }
func TestDeleteNotFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } err = fs.Delete(datastore.NewKey("quux")) if g, e := err, datastore.ErrNotFound; g != e { t.Fatalf("expected ErrNotFound, got: %v\n", g) } }
func TestHasNotFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } found, err := fs.Has(datastore.NewKey("quux")) if err != nil { t.Fatalf("Has fail: %v\n", err) } if g, e := found, false; g != e { t.Fatalf("wrong Has: %v != %v", g, e) } }
func TestHasFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } err = fs.Put(datastore.NewKey("quux"), []byte("foobar")) if err != nil { t.Fatalf("Put fail: %v\n", err) } found, err := fs.Has(datastore.NewKey("quux")) if err != nil { t.Fatalf("Has fail: %v\n", err) } if g, e := found, true; g != e { t.Fatalf("wrong Has: %v != %v", g, e) } }
func TestDeleteFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() fs, err := flatfs.New(temp, 2) if err != nil { t.Fatalf("New fail: %v\n", err) } err = fs.Put(datastore.NewKey("quux"), []byte("foobar")) if err != nil { t.Fatalf("Put fail: %v\n", err) } err = fs.Delete(datastore.NewKey("quux")) if err != nil { t.Fatalf("Delete fail: %v\n", err) } // check that it's gone _, err = fs.Get(datastore.NewKey("quux")) if g, e := err, datastore.ErrNotFound; g != e { t.Fatalf("expected Get after Delete to give ErrNotFound, got: %v\n", g) } }