func ShouldGetError(t testing.TB, s store.Store, key string, wantErr error) {
got, _, err := s.Get(key, store.GetOptions{})
if err != wantErr {
t.Errorf("Get(%#v) = (%#v, %v), but wanted err = %v",
key, got, err, wantErr)
}
}
func ShouldGet(t testing.TB, s store.Store, key string, data []byte) {
got, st, err := s.Get(key, store.GetOptions{})
if err != nil {
t.Errorf("Get(%#v) returned unexpected error %v", key, err)
return
}
wantStat := store.Stat{
SHA256: sha256.Sum256(data),
Size: int64(len(got)),
WriteTime: st.WriteTime,
}
if !bytes.Equal(got, data) || st != wantStat {
t.Errorf("Get(%#v) = (%#v, %#v), but wanted (%#v, %#v)",
key, got, st, data, wantStat)
}
}
func (m *Multi) rebalanceFile(f meta.File, finderEntries map[[16]byte]FinderEntry) (bool, error) {
// search for the lowest free location that this file is stored on
var minI int
var minF int64
var minS store.Store
for i, l := range f.Locations {
fe, ok := finderEntries[l]
if !ok {
continue
}
if minS == nil || minF > fe.Free {
minS = fe.Store
minF = fe.Free
minI = i
}
}
if minS == nil {
return false, nil
}
// search for the highest free location that this file is NOT stored on
var maxF int64
var maxS store.Store
for id, fe := range finderEntries {
if fe.Dead {
continue
}
found := false
for _, locid := range f.Locations {
if locid == id {
found = true
break
}
}
if found {
continue
}
if maxS == nil || maxF < fe.Free {
maxF = fe.Free
maxS = fe.Store
}
}
if maxS == nil {
return false, nil
}
if maxF-minF < rebalanceMinDifference {
return false, nil
}
// we should move chunk minI on minS to maxS
err := m.db.RunTx(func(ctx kvl.Ctx) error {
l, err := meta.Open(ctx)
if err != nil {
return err
}
return l.WALMark(f.PrefixID)
})
if err != nil {
return false, err
}
defer func() {
// TODO: how to handle errors here?
m.db.RunTx(func(ctx kvl.Ctx) error {
l, err := meta.Open(ctx)
if err != nil {
return err
}
return l.WALClear(f.PrefixID)
})
}()
localKey := localKeyFor(&f, minI)
data, st, err := minS.Get(localKey, store.GetOptions{})
if err != nil {
return false, err
}
setCASV := store.CASV{
Present: true,
SHA256: st.SHA256,
Data: data,
}
err = maxS.CAS(localKey, store.MissingV, setCASV, nil)
if err != nil {
return false, err
}
newF := f
newF.Locations = make([][16]byte, len(f.Locations))
copy(newF.Locations, f.Locations)
newF.Locations[minI] = maxS.UUID()
err = m.db.RunTx(func(ctx kvl.Ctx) error {
l, err := meta.Open(ctx)
if err != nil {
return err
}
f2, err := l.GetFile(f.Path)
if err != nil {
return err
}
if f2.PrefixID != f.PrefixID {
return errModifiedDuringBalance
}
if len(f2.Locations) != len(f.Locations) {
return errModifiedDuringBalance
}
for i, floc := range f.Locations {
if floc != f2.Locations[i] {
return errModifiedDuringBalance
}
}
err = l.SetFile(&newF)
if err != nil {
return err
}
return nil
})
if err != nil {
maxS.CAS(localKey, setCASV, store.MissingV, nil) // ignore error
return false, err
}
err = minS.CAS(localKey, setCASV, store.MissingV, nil)
if err != nil {
log.Printf("Couldn't remove rebalanced chunk from old location %v: %v",
uuid.Fmt(maxS.UUID()), err)
}
fe := finderEntries[minS.UUID()]
fe.Free += int64(len(data))
finderEntries[minS.UUID()] = fe
fe = finderEntries[maxS.UUID()]
fe.Free -= int64(len(data))
finderEntries[maxS.UUID()] = fe
return true, nil
}
func TestStoreCASCountRace(t *testing.T, s store.Store) {
t.Logf("TestStoreCASCountRace()")
const (
goroutines = 4
iterations = 15
)
ShouldFullList(t, s, nil)
ShouldCAS(t, s, "key", store.AnyV, store.DataV([]byte("0")))
errs := make(chan error)
casFailures := uint64(0)
for i := 0; i < goroutines; i++ {
go func(i int) {
for j := 0; j < iterations; j++ {
for {
data, st, err := s.Get("key", store.GetOptions{})
if err != nil {
t.Logf("Routine %v: Couldn't get key: %v", i, err)
errs <- err
return
}
num, err := strconv.ParseInt(string(data), 10, 64)
if err != nil {
t.Logf("Routine %v: Couldn't parse int: %v", i, err)
errs <- err
return
}
num++
data = strconv.AppendInt(data[:0], num, 10)
err = s.CAS("key",
store.CASV{Present: true, SHA256: st.SHA256},
store.DataV(data),
nil)
if err != nil {
if err == store.ErrCASFailure {
atomic.AddUint64(&casFailures, 1)
continue
}
t.Logf("Routine %v: Couldn't cas: %v", i, err)
errs <- err
return
}
break
}
}
errs <- nil
}(i)
}
for i := 0; i < goroutines; i++ {
err := <-errs
if err != nil {
t.Errorf("Got error from goroutine: %v", err)
}
}
t.Logf("%v cas failures", casFailures)
ShouldGet(t, s, "key", []byte(strconv.FormatInt(goroutines*iterations, 10)))
ShouldCAS(t, s, "key", store.AnyV, store.MissingV)
}