func ShouldGetError(t testing.TB, s store.Store, key string, wantErr error) {
got, _, err := s.Get(key, store.GetOptions{})
if err != wantErr {
t.Errorf("Get(%#v) = (%#v, %v), but wanted err = %v",
key, got, err, wantErr)
}
}
func ShouldCASError(t testing.TB, s store.Store, key string, from, to store.CASV, wantErr error) {
err := s.CAS(key, from, to, nil)
if err != wantErr {
t.Errorf("CAS(%#v, %v, %v) returned error %v, but wanted %v",
key, from, to, err, wantErr)
}
}
func ShouldStatMiss(t testing.TB, s store.Store, key string) {
st, err := s.Stat(key, nil)
if err != store.ErrNotFound {
t.Errorf("Stat(%#v) returned (%v, %v), but wanted %v",
key, st, err, store.ErrNotFound)
}
}
func ShouldFreeSpace(t testing.TB, s store.Store) {
free, err := s.FreeSpace(nil)
if err != nil {
t.Errorf("FreeSpace() returned unexpected error %v", err)
return
}
if free <= 0 {
t.Errorf("FreeSpace() returned nonpositive %v", free)
}
}
func ShouldStatNoTime(t testing.TB, s store.Store, key string, stat store.Stat) {
st, err := s.Stat(key, nil)
if err != nil {
t.Errorf("Stat(%#v) returned unexpected error %v", key, err)
return
}
stat.WriteTime = st.WriteTime
if st != stat {
t.Errorf("Stat(%#v) = %#v, but wanted %#v", key, st, stat)
}
}
func ShouldList(t testing.TB, s store.Store, after string, limit int, expect []string) {
got, err := s.List(after, limit, nil)
if err != nil {
t.Errorf("Unexpected error from List(%#v, %v): %v", after, limit, err)
return
}
if !((len(got) == 0 && len(expect) == 0) ||
reflect.DeepEqual(got, expect)) {
t.Errorf("List(%#v, %v) = %#v, but wanted %#v",
after, limit, got, expect)
}
}
func ShouldGet(t testing.TB, s store.Store, key string, data []byte) {
got, st, err := s.Get(key, store.GetOptions{})
if err != nil {
t.Errorf("Get(%#v) returned unexpected error %v", key, err)
return
}
wantStat := store.Stat{
SHA256: sha256.Sum256(data),
Size: int64(len(got)),
WriteTime: st.WriteTime,
}
if !bytes.Equal(got, data) || st != wantStat {
t.Errorf("Get(%#v) = (%#v, %#v), but wanted (%#v, %#v)",
key, got, st, data, wantStat)
}
}
func TestStoreWriteTime(t *testing.T, s store.Store) {
t.Logf("TestStoreWriteTime()")
ShouldCAS(t, s, "key", store.AnyV, store.DataV([]byte("one")))
now := time.Now().Unix()
st, err := s.Stat("key", nil)
if err != nil {
t.Fatalf("Couldn't stat key: %v", err)
}
diff := st.WriteTime - now
if diff < 0 {
diff = -diff
}
if diff > 2 {
t.Fatalf("Store returned timestamp %v, but wanted %v", st.WriteTime, now)
}
ShouldCAS(t, s, "key", store.AnyV, store.MissingV)
}
func ShouldListCount(t testing.TB, s store.Store, count int) {
actualCount := 0
from := ""
for {
list, err := s.List(from, 100, nil)
if err != nil {
t.Errorf("Couldn't List(%#v, 100): %v", from, err)
return
}
actualCount += len(list)
if len(list) < 100 {
break
}
}
if actualCount != count {
t.Errorf("Full list returned %v elements but wanted %v", actualCount, count)
}
}
func (m *Multi) rebalanceFile(f meta.File, finderEntries map[[16]byte]FinderEntry) (bool, error) {
// search for the lowest free location that this file is stored on
var minI int
var minF int64
var minS store.Store
for i, l := range f.Locations {
fe, ok := finderEntries[l]
if !ok {
continue
}
if minS == nil || minF > fe.Free {
minS = fe.Store
minF = fe.Free
minI = i
}
}
if minS == nil {
return false, nil
}
// search for the highest free location that this file is NOT stored on
var maxF int64
var maxS store.Store
for id, fe := range finderEntries {
if fe.Dead {
continue
}
found := false
for _, locid := range f.Locations {
if locid == id {
found = true
break
}
}
if found {
continue
}
if maxS == nil || maxF < fe.Free {
maxF = fe.Free
maxS = fe.Store
}
}
if maxS == nil {
return false, nil
}
if maxF-minF < rebalanceMinDifference {
return false, nil
}
// we should move chunk minI on minS to maxS
err := m.db.RunTx(func(ctx kvl.Ctx) error {
l, err := meta.Open(ctx)
if err != nil {
return err
}
return l.WALMark(f.PrefixID)
})
if err != nil {
return false, err
}
defer func() {
// TODO: how to handle errors here?
m.db.RunTx(func(ctx kvl.Ctx) error {
l, err := meta.Open(ctx)
if err != nil {
return err
}
return l.WALClear(f.PrefixID)
})
}()
localKey := localKeyFor(&f, minI)
data, st, err := minS.Get(localKey, store.GetOptions{})
if err != nil {
return false, err
}
setCASV := store.CASV{
Present: true,
SHA256: st.SHA256,
Data: data,
}
err = maxS.CAS(localKey, store.MissingV, setCASV, nil)
if err != nil {
return false, err
}
newF := f
newF.Locations = make([][16]byte, len(f.Locations))
copy(newF.Locations, f.Locations)
newF.Locations[minI] = maxS.UUID()
err = m.db.RunTx(func(ctx kvl.Ctx) error {
l, err := meta.Open(ctx)
if err != nil {
return err
}
f2, err := l.GetFile(f.Path)
if err != nil {
return err
}
if f2.PrefixID != f.PrefixID {
return errModifiedDuringBalance
}
if len(f2.Locations) != len(f.Locations) {
return errModifiedDuringBalance
}
for i, floc := range f.Locations {
if floc != f2.Locations[i] {
return errModifiedDuringBalance
}
}
err = l.SetFile(&newF)
if err != nil {
return err
}
return nil
})
if err != nil {
maxS.CAS(localKey, setCASV, store.MissingV, nil) // ignore error
return false, err
}
err = minS.CAS(localKey, setCASV, store.MissingV, nil)
if err != nil {
log.Printf("Couldn't remove rebalanced chunk from old location %v: %v",
uuid.Fmt(maxS.UUID()), err)
}
fe := finderEntries[minS.UUID()]
fe.Free += int64(len(data))
finderEntries[minS.UUID()] = fe
fe = finderEntries[maxS.UUID()]
fe.Free -= int64(len(data))
finderEntries[maxS.UUID()] = fe
return true, nil
}
func TestStoreCASCountRace(t *testing.T, s store.Store) {
t.Logf("TestStoreCASCountRace()")
const (
goroutines = 4
iterations = 15
)
ShouldFullList(t, s, nil)
ShouldCAS(t, s, "key", store.AnyV, store.DataV([]byte("0")))
errs := make(chan error)
casFailures := uint64(0)
for i := 0; i < goroutines; i++ {
go func(i int) {
for j := 0; j < iterations; j++ {
for {
data, st, err := s.Get("key", store.GetOptions{})
if err != nil {
t.Logf("Routine %v: Couldn't get key: %v", i, err)
errs <- err
return
}
num, err := strconv.ParseInt(string(data), 10, 64)
if err != nil {
t.Logf("Routine %v: Couldn't parse int: %v", i, err)
errs <- err
return
}
num++
data = strconv.AppendInt(data[:0], num, 10)
err = s.CAS("key",
store.CASV{Present: true, SHA256: st.SHA256},
store.DataV(data),
nil)
if err != nil {
if err == store.ErrCASFailure {
atomic.AddUint64(&casFailures, 1)
continue
}
t.Logf("Routine %v: Couldn't cas: %v", i, err)
errs <- err
return
}
break
}
}
errs <- nil
}(i)
}
for i := 0; i < goroutines; i++ {
err := <-errs
if err != nil {
t.Errorf("Got error from goroutine: %v", err)
}
}
t.Logf("%v cas failures", casFailures)
ShouldGet(t, s, "key", []byte(strconv.FormatInt(goroutines*iterations, 10)))
ShouldCAS(t, s, "key", store.AnyV, store.MissingV)
}