func main() {
d := diskv.New(diskv.Options{
BasePath: "data",
Transform: blockTransform,
CacheSizeMax: 1024 * 1024, // 1MB
})
for _, valueStr := range []string{
"I am the very model of a modern Major-General",
"I've information vegetable, animal, and mineral",
"I know the kings of England, and I quote the fights historical",
"From Marathon to Waterloo, in order categorical",
"I'm very well acquainted, too, with matters mathematical",
"I understand equations, both the simple and quadratical",
"About binomial theorem I'm teeming with a lot o' news",
"With many cheerful facts about the square of the hypotenuse",
} {
d.Write(md5sum(valueStr), []byte(valueStr))
}
var keyCount int
for key := range d.Keys(nil) {
val, err := d.Read(key)
if err != nil {
panic(fmt.Sprintf("key %s had no value", key))
}
fmt.Printf("%s: %s\n", key, val)
keyCount++
}
fmt.Printf("%d total keys\n", keyCount)
// d.EraseAll() // leave it commented out to see how data is kept on disk
}
func TestImportCopy(t *testing.T) {
b := []byte(`¡åéîòü!`)
f, err := ioutil.TempFile("", "temp-test")
if err != nil {
t.Fatal(err)
}
if _, err := f.Write(b); err != nil {
t.Fatal(err)
}
f.Close()
d := diskv.New(diskv.Options{
BasePath: "test-import-copy",
})
defer d.EraseAll()
if err := d.Import(f.Name(), "key", false); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(f.Name()); err != nil {
t.Errorf("expected temp file to remain, but got err = %v", err)
}
}
func TestKeysCancel(t *testing.T) {
d := diskv.New(diskv.Options{
BasePath: "test-data",
})
defer d.EraseAll()
for k, v := range keysTestData {
d.Write(k, []byte(v))
}
var (
cancel = make(chan struct{})
received = 0
cancelAfter = len(keysTestData) / 2
)
for key := range d.Keys(cancel) {
received++
if received >= cancelAfter {
close(cancel)
runtime.Gosched() // allow walker to detect cancel
}
t.Logf("received %d: %q", received, key)
}
if want, have := cancelAfter, received; want != have {
t.Errorf("want %d, have %d")
}
}
func TestKeysNested(t *testing.T) {
d := diskv.New(diskv.Options{
BasePath: "test-data",
Transform: blockTransform(2),
})
defer d.EraseAll()
for k, v := range keysTestData {
d.Write(k, []byte(v))
}
checkKeys(t, d.Keys(nil), keysTestData)
}
func TestKeysPrefixFlat(t *testing.T) {
d := diskv.New(diskv.Options{
BasePath: "test-data",
})
defer d.EraseAll()
for k, v := range keysTestData {
d.Write(k, []byte(v))
}
for _, prefix := range prefixes {
checkKeys(t, d.KeysPrefix(prefix, nil), filterPrefix(keysTestData, prefix))
}
}
func TestKeysFlat(t *testing.T) {
transform := func(s string) []string {
if s == "" {
t.Fatalf(`transform should not be called with ""`)
}
return []string{}
}
d := diskv.New(diskv.Options{
BasePath: "test-data",
Transform: transform,
})
defer d.EraseAll()
for k, v := range keysTestData {
d.Write(k, []byte(v))
}
checkKeys(t, d.Keys(nil), keysTestData)
}
func main() {
d := diskv.New(diskv.Options{
BasePath: "my-diskv-data-directory",
Transform: func(s string) []string { return []string{} },
CacheSizeMax: 1024 * 1024, // 1MB
})
key := "alpha"
if err := d.Write(key, []byte{'1', '2', '3'}); err != nil {
panic(err)
}
value, err := d.Read(key)
if err != nil {
panic(err)
}
fmt.Printf("%v\n", value)
if err := d.Erase(key); err != nil {
panic(err)
}
}
func TestImportMove(t *testing.T) {
b := []byte(`0123456789`)
f, err := ioutil.TempFile("", "temp-test")
if err != nil {
t.Fatal(err)
}
if _, err := f.Write(b); err != nil {
t.Fatal(err)
}
f.Close()
d := diskv.New(diskv.Options{
BasePath: "test-import-move",
})
defer d.EraseAll()
key := "key"
if err := d.Write(key, []byte(`TBD`)); err != nil {
t.Fatal(err)
}
if err := d.Import(f.Name(), key, true); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(f.Name()); err == nil || !os.IsNotExist(err) {
t.Errorf("expected temp file to be gone, but err = %v", err)
}
if !d.Has(key) {
t.Errorf("%q not present", key)
}
if buf, err := d.Read(key); err != nil || bytes.Compare(b, buf) != 0 {
t.Errorf("want %q, have %q (err = %v)", string(b), string(buf), err)
}
}
func NewStore(baseDir string) (*Store, error) {
storeDir := filepath.Join(baseDir, "cas")
s := &Store{
dir: storeDir,
stores: make([]*diskv.Diskv, len(diskvStores)),
}
s.imageLockDir = filepath.Join(storeDir, "imagelocks")
err := os.MkdirAll(s.imageLockDir, defaultPathPerm)
if err != nil {
return nil, err
}
s.treeStoreLockDir = filepath.Join(storeDir, "treestorelocks")
err = os.MkdirAll(s.treeStoreLockDir, defaultPathPerm)
if err != nil {
return nil, err
}
// Take a shared cas lock
s.storeLock, err = lock.NewLock(storeDir, lock.Dir)
if err != nil {
return nil, err
}
for i, p := range diskvStores {
s.stores[i] = diskv.New(diskv.Options{
BasePath: filepath.Join(storeDir, p),
Transform: blockTransform,
})
}
db, err := NewDB(filepath.Join(storeDir, "db"))
if err != nil {
return nil, err
}
s.db = db
s.treestore = &TreeStore{path: filepath.Join(storeDir, "tree")}
needsMigrate := false
fn := func(tx *sql.Tx) error {
var err error
ok, err := dbIsPopulated(tx)
if err != nil {
return err
}
// populate the db
if !ok {
for _, stmt := range dbCreateStmts {
_, err = tx.Exec(stmt)
if err != nil {
return err
}
}
return nil
}
// if db is populated check its version
version, err := getDBVersion(tx)
if err != nil {
return err
}
if version < dbVersion {
needsMigrate = true
}
if version > dbVersion {
return fmt.Errorf("Current store db version: %d greater than the current rkt expected version: %d", version, dbVersion)
}
return nil
}
if err = db.Do(fn); err != nil {
return nil, err
}
// migration is done in another transaction as it must take an exclusive
// store lock. If, in the meantime, another process has already done the
// migration, between the previous db version check and the below
// migration code, the migration will do nothing as it'll start
// migration from the current version.
if needsMigrate {
// Take an exclusive store lock
err := s.storeLock.ExclusiveLock()
if err != nil {
return nil, err
}
if err := s.backupDB(); err != nil {
return nil, err
}
fn := func(tx *sql.Tx) error {
return migrate(tx, dbVersion)
}
if err = db.Do(fn); err != nil {
return nil, err
}
}
return s, nil
}