func TestLayerConfig(t *testing.T) {
db := ram.New()
err := db.RunTx(func(ctx kvl.Ctx) error {
l, err := Open(ctx)
if err != nil {
return err
}
value, err := l.GetConfig("a")
if err != nil {
t.Errorf("Nonexistent config returned unexpected error %v", err)
return err
}
if string(value) != "" {
t.Errorf("Nonexistent config returned non-empty string %#v",
string(value))
}
err = l.SetConfig("a", []byte("hello there"))
if err != nil {
t.Errorf("Couldn't set config variable: %v", err)
return err
}
value, err = l.GetConfig("a")
if err != nil {
t.Errorf("Couldn't get config variable \"a\": %v", err)
return err
}
if string(value) != "hello there" {
t.Errorf("GetConfig returned %#v, wanted %#v",
string(value), "hello there")
}
return nil
})
if err != nil {
t.Errorf("Couldn't run transaction: %v", err)
}
}
func prepareMultiTest(t testing.TB, need, total, serverCount int) ([]*killHandler, *Multi, []*storetests.MockStore, func()) {
var killers []*killHandler
var mockstores []*storetests.MockStore
var servers []*httptest.Server
var chunkServers []*chunkserver.Handler
var finder *Finder
var multi *Multi
done := func() {
if multi != nil {
multi.Close()
}
if finder != nil {
finder.Stop()
}
for _, srv := range servers {
srv.Close()
}
for _, cs := range chunkServers {
cs.Close()
}
for _, mock := range mockstores {
mock.Close()
}
}
for i := 0; i < serverCount; i++ {
mock := storetests.NewMockStore(0)
mockstores = append(mockstores, mock)
cs, err := chunkserver.New([]store.Store{mock})
if err != nil {
done()
t.Fatalf("Couldn't create chunkserver: %v", err)
}
chunkServers = append(chunkServers, cs)
killer := newKillHandler(cs)
srv := httptest.NewServer(killer)
killers = append(killers, killer)
servers = append(servers, srv)
}
db := ram.New()
finder, err := NewFinder(db)
if err != nil {
done()
t.Fatalf("Couldn't create new finder: %v", err)
}
for _, srv := range servers {
err = finder.Scan(srv.URL)
if err != nil {
done()
t.Fatalf("Couldn't scan %v: %v", srv.URL, err)
}
}
if len(finder.Stores()) != len(servers) {
done()
t.Fatalf("Finder did not find all stores")
}
multi, err = NewMulti(db, finder, 0)
if err != nil {
done()
t.Fatalf("Couldn't create multi: %v", err)
}
err = multi.SetRedundancy(need, total)
if err != nil {
done()
t.Fatalf("Couldn't set redundancy levels: %v", err)
}
return killers, multi, mockstores, done
}
func TestLayerFileGetSetRemove(t *testing.T) {
db := ram.New()
err := db.RunTx(func(ctx kvl.Ctx) error {
l, err := Open(ctx)
if err != nil {
return err
}
fileA := File{
Path: "path to a",
Size: 4,
SHA256: sha256.Sum256([]byte("abcd")),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 2,
MappingValue: 0,
Locations: [][16]byte{uuid.Gen4(), uuid.Gen4()},
}
err = l.SetFile(&fileA)
if err != nil {
t.Errorf("Couldn't SetFile(%#v): %v", fileA, err)
return err
}
f, err := l.GetFile("path to a")
if err != nil {
t.Errorf("Couldn't GetFile(path to a): %v", err)
return err
}
if !reflect.DeepEqual(f, &fileA) {
t.Errorf("GetFile(path to a) returned %#v, but wanted %#v",
f, &fileA)
}
f, err = l.GetFile("nonexistent")
if err != nil {
t.Errorf("Couldn't GetFile(nonexistent): %v", err)
return err
}
if f != nil {
t.Errorf("GetFile(nonexistent) returned %#v, but wanted nil", f)
}
err = l.RemoveFilePath("nonexistent")
if err != kvl.ErrNotFound {
t.Errorf("RemoveFilePath(nonexistent) returned %v, but wanted %v",
err, kvl.ErrNotFound)
if err != nil {
return err
}
}
err = l.RemoveFilePath("path to a")
if err != nil {
t.Errorf("Couldn't RemoveFilePath(path to a): %v",
err)
return err
}
f, err = l.GetFile("path to a")
if err != nil {
t.Errorf("Couldn't GetFile(path to a): %v", err)
return err
}
if f != nil {
t.Errorf("GetFile(path to a) returned %#v, but wnated nil", f)
}
return nil
})
if err != nil {
t.Errorf("Couldn't run transaction: %v", err)
}
}
func TestLayerWALConcurrent(t *testing.T) {
db := ram.New()
err := db.RunTx(func(ctx kvl.Ctx) error {
l, err := Open(ctx)
if err != nil {
return err
}
id := uuid.Gen4()
out, err := l.WALCheck(id)
if err != nil {
return err
}
if out != false {
t.Errorf("WALCheck returned %v on an empty database", out)
}
for i := 0; i < 5; i++ {
err = l.WALMark(id)
if err != nil {
return err
}
out, err = l.WALCheck(id)
if err != nil {
return err
}
if out != true {
t.Errorf("WALCheck returned %v after marking %v times", out, i+1)
}
}
for i := 0; i < 5; i++ {
err = l.WALClear(id)
if err != nil {
return err
}
out, err = l.WALCheck(id)
if err != nil {
return err
}
if out != (i < 4) {
t.Errorf("WALCheck returned %v after unmarking %v times (should have %v marks)", out, i+1, 5-(i+1))
}
}
out, err = l.WALCheck(id)
if err != nil {
return err
}
if out != false {
t.Errorf("WALCheck returned %v on an empty database after unmarking", out)
}
return nil
})
if err != nil {
t.Fatalf("Couldn't run transaction: %v", err)
}
}
func TestLayerLocationContents(t *testing.T) {
db := ram.New()
err := db.RunTx(func(ctx kvl.Ctx) error {
l, err := Open(ctx)
if err != nil {
return err
}
locA := uuid.Gen4()
locB := uuid.Gen4()
locC := uuid.Gen4()
files := []File{
{
Path: "file-a",
Size: 1,
SHA256: sha256.Sum256([]byte{'a'}),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 2,
MappingValue: 0,
Locations: [][16]byte{locA, locB, locC},
},
{
Path: "file-b",
Size: 1,
SHA256: sha256.Sum256([]byte{'b'}),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 2,
MappingValue: 0,
Locations: [][16]byte{locB, locA, locC},
},
{
Path: "file-c",
Size: 1,
SHA256: sha256.Sum256([]byte{'c'}),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 2,
MappingValue: 0,
Locations: [][16]byte{locC, locA},
},
}
for _, file := range files {
err := l.SetFile(&file)
if err != nil {
t.Errorf("Couldn't SetFile: %v", err)
return err
}
}
localKeyFor := func(file File, idx int) string {
return fmt.Sprintf("%v_%x_%v",
uuid.Fmt(file.PrefixID), file.SHA256[:8], idx)
}
tests := []struct {
ID [16]byte
Entries []string
}{
{
ID: locA,
Entries: []string{
localKeyFor(files[0], 0),
localKeyFor(files[1], 1),
localKeyFor(files[2], 1),
},
},
{
ID: locB,
Entries: []string{
localKeyFor(files[0], 1),
localKeyFor(files[1], 0),
},
},
{
ID: locC,
Entries: []string{
localKeyFor(files[0], 2),
localKeyFor(files[1], 2),
localKeyFor(files[2], 0),
},
},
}
for _, test := range tests {
sort.Strings(test.Entries)
names, err := l.GetLocationContents(test.ID, "", 0)
if err != nil {
t.Errorf("Couldn't GetLocationContents(%#v, \"\", 0): %v",
uuid.Fmt(test.ID), err)
return err
}
if !reflect.DeepEqual(names, test.Entries) {
t.Errorf("GetLocationContents(%#v, \"\", 0) returned %#v, wanted %#v",
uuid.Fmt(test.ID), names, test.Entries)
}
names = nil
after := ""
for {
localNames, err := l.GetLocationContents(test.ID, after, 1)
if err != nil {
t.Errorf("Couldn't GetLocationContents(%#v, %#v, %v): %v",
uuid.Fmt(test.ID), after, 1, err)
return err
}
if len(localNames) > 1 {
t.Errorf("GetLocationContents(%#v, %#v, %v) = %#v, too many results",
uuid.Fmt(test.ID), after, 1, localNames)
}
if len(localNames) == 0 {
break
}
names = append(names, localNames...)
after = localNames[0]
}
if !reflect.DeepEqual(names, test.Entries) {
t.Errorf("GetLocationContents on %v one by one returned %#v, wanted %#v",
uuid.Fmt(test.ID), names, test.Entries)
}
}
return nil
})
if err != nil {
t.Errorf("Couldn't run transaction: %v", err)
}
}
func TestLayerFileList(t *testing.T) {
db := ram.New()
err := db.RunTx(func(ctx kvl.Ctx) error {
l, err := Open(ctx)
if err != nil {
return err
}
for c := 'a'; c <= 'z'; c++ {
f := File{
Path: string(c),
Size: 1,
SHA256: sha256.Sum256([]byte{byte(c)}),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 1,
MappingValue: 0,
Locations: [][16]byte{uuid.Gen4()},
}
err := l.SetFile(&f)
if err != nil {
t.Errorf("Couldn't SetFile: %v", err)
return err
}
}
tests := []struct {
After string
Limit int
Paths []string
}{
{"", 2, []string{"a", "b"}},
{"a", 4, []string{"b", "c", "d", "e"}},
{"azz", 3, []string{"b", "c", "d"}},
{"q", 0, []string{"r", "s", "t", "u", "v", "w", "x", "y", "z"}},
{"w", 6, []string{"x", "y", "z"}},
}
for _, test := range tests {
fs, err := l.ListFiles(test.After, test.Limit)
if err != nil {
t.Errorf("Couldn't ListFiles(%#v, %v): %v",
test.After, test.Limit, err)
continue
}
bad := false
if len(fs) != len(test.Paths) {
bad = true
} else {
for i, f := range fs {
if f.Path != test.Paths[i] {
bad = true
break
}
}
}
if bad {
t.Errorf("ListFiles(%#v, %v) returned %#v, but wanted paths %v",
test.After, test.Limit, fs, test.Paths)
}
}
return nil
})
if err != nil {
t.Errorf("Couldn't run transaction: %v", err)
}
}
func TestLayerFileGetByLocation(t *testing.T) {
db := ram.New()
err := db.RunTx(func(ctx kvl.Ctx) error {
l, err := Open(ctx)
if err != nil {
return err
}
idA := [16]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
idB := [16]byte{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
idC := [16]byte{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
fileA := File{
Path: "path to a",
Size: 4,
SHA256: sha256.Sum256([]byte("abcd")),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 2,
MappingValue: 0,
Locations: [][16]byte{idB, idC},
}
fileB := File{
Path: "path to b",
Size: 12,
SHA256: sha256.Sum256([]byte("goodbye dude")),
WriteTime: time.Now().Unix(),
PrefixID: uuid.Gen4(),
DataChunks: 1,
MappingValue: 0,
Locations: [][16]byte{idA, idB, idC},
}
err = l.SetFile(&fileA)
if err != nil {
t.Errorf("Couldn't SetFile(%#v): %v", fileA, err)
return err
}
err = l.SetFile(&fileB)
if err != nil {
t.Errorf("Couldn't SetFile(%#v): %v", fileB, err)
return err
}
fs, err := l.GetFilesByLocation(idB, 1)
if err != nil {
t.Errorf("Couldn't GetFilesByLocation: %v", err)
return err
}
if len(fs) != 1 || !reflect.DeepEqual(fs[0], fileA) {
t.Errorf("GetFilesByLocation(%v, 1) returned %#v, but wanted %#v",
uuid.Fmt(idB), fs, []File{fileA})
}
fs, err = l.GetFilesByLocation(idA, 3)
if err != nil {
t.Errorf("Couldn't GetFilesByLocation: %v", err)
return err
}
if len(fs) != 1 || !reflect.DeepEqual(fs[0], fileB) {
t.Errorf("GetFilesByLocation(%v, 1) returned %#v, but wanted %#v",
uuid.Fmt(idA), fs, []File{fileB})
}
fs, err = l.GetFilesByLocation(idC, 3)
if err != nil {
t.Errorf("Couldn't GetFilesByLocation: %v", err)
return err
}
if len(fs) != 2 ||
!reflect.DeepEqual(fs[0], fileA) ||
!reflect.DeepEqual(fs[1], fileB) {
t.Errorf("GetFilesByLocation(%v, 3) returned %#v, but wanted %#v",
uuid.Fmt(idC), fs, []File{fileA, fileB})
}
return nil
})
if err != nil {
t.Errorf("Couldn't run transaction: %v", err)
}
}
func TestFinderScan(t *testing.T) {
db := ram.New()
mock := storetests.NewMockStore(0)
cs, err := chunkserver.New([]store.Store{mock})
if err != nil {
t.Fatalf("Couldn't create chunkserver: %v", err)
}
defer cs.Close()
cs.WaitAllAvailable()
killer := newKillHandler(cs)
srv := httptest.NewServer(killer)
defer srv.Close()
f, err := NewFinder(db)
if err != nil {
t.Fatalf("Couldn't create new finder: %v", err)
}
defer f.Stop()
err = f.Scan(srv.URL)
if err != nil {
t.Fatalf("Couldn't scan %v: %v", srv.URL, err)
}
// newly scanned store should be in the Finder
stores := f.Stores()
if _, ok := stores[mock.UUID()]; !ok {
t.Fatalf("Finder did not find uuid of directory store")
}
// kill the store and update the Finder
killer.setKilled(true)
f.test(0)
// should have been removed
stores = f.Stores()
if len(stores) > 0 {
t.Fatalf("Finder did not remove dead store")
}
// but should stay in the DB
err = db.RunReadTx(func(ctx kvl.Ctx) error {
layer, err := meta.Open(ctx)
if err != nil {
return err
}
loc, err := layer.GetLocation(mock.UUID())
if err != nil {
return err
}
if loc == nil {
return fmt.Errorf("No location in database")
}
return nil
})
if err != nil {
t.Fatalf("Couldn't verify locations: %v", err)
}
// when the store comes back
killer.setKilled(false)
err = f.Rescan()
if err != nil {
t.Fatalf("Couldn't Rescan: %v", err)
}
// it should be there again
stores = f.Stores()
if _, ok := stores[mock.UUID()]; !ok {
t.Fatalf("Finder did not find uuid of directory store after resurrection")
}
}