func TestSorted(t *testing.T, kv sorted.KeyValue) {
defer test.TLog(t)()
if !isEmpty(t, kv) {
t.Fatal("kv for test is expected to be initially empty")
}
set := func(k, v string) {
if err := kv.Set(k, v); err != nil {
t.Fatalf("Error setting %q to %q: %v", k, v, err)
}
}
set("foo", "bar")
if isEmpty(t, kv) {
t.Fatalf("iterator reports the kv is empty after adding foo=bar; iterator must be broken")
}
if v, err := kv.Get("foo"); err != nil || v != "bar" {
t.Errorf("get(foo) = %q, %v; want bar", v, err)
}
if v, err := kv.Get("NOT_EXIST"); err != sorted.ErrNotFound {
t.Errorf("get(NOT_EXIST) = %q, %v; want error sorted.ErrNotFound", v, err)
}
for i := 0; i < 2; i++ {
if err := kv.Delete("foo"); err != nil {
t.Errorf("Delete(foo) (on loop %d/2) returned error %v", i+1, err)
}
}
set("a", "av")
set("b", "bv")
set("c", "cv")
testEnumerate(t, kv, "", "", "av", "bv", "cv")
testEnumerate(t, kv, "a", "", "av", "bv", "cv")
testEnumerate(t, kv, "b", "", "bv", "cv")
testEnumerate(t, kv, "a", "c", "av", "bv")
testEnumerate(t, kv, "a", "b", "av")
testEnumerate(t, kv, "a", "a")
testEnumerate(t, kv, "d", "")
testEnumerate(t, kv, "d", "e")
// Verify that < comparison works identically for all DBs (because it is affected by collation rules)
// http://postgresql.1045698.n5.nabble.com/String-comparison-and-the-SQL-standard-td5740721.html
set("foo|abc", "foo|abcv")
testEnumerate(t, kv, "foo|", "", "foo|abcv")
testEnumerate(t, kv, "foo|", "foo}", "foo|abcv")
// Verify that the value isn't being used instead of the key in the range comparison.
set("y", "x:foo")
testEnumerate(t, kv, "x:", "x~")
testInsertLarge(t, kv)
testInsertTooLarge(t, kv)
// TODO: test batch commits
// Deleting a non-existent item in a batch should not be an error
testDeleteNotFoundBatch(t, kv)
testDeletePartialNotFoundBatch(t, kv)
}
func testDeletePartialNotFoundBatch(t *testing.T, kv sorted.KeyValue) {
if err := kv.Set(butIExistKey, "whatever"); err != nil {
t.Fatal(err)
}
b := kv.BeginBatch()
b.Delete(notExistKey)
b.Delete(butIExistKey)
if err := kv.CommitBatch(b); err != nil {
t.Fatalf("Batch deletion with one non existing key returned an error: %v", err)
}
if val, err := kv.Get(butIExistKey); err != sorted.ErrNotFound || val != "" {
t.Fatalf("Key %q should have been batch deleted", butIExistKey)
}
}
func (s *storage) reindexOne(ctx *context.Context, index sorted.KeyValue, overwrite bool, packID int) error {
var batch sorted.BatchMutation
if overwrite {
batch = index.BeginBatch()
}
allOk := true
// TODO(tgulacsi): proper verbose from context
verbose := camliDebug
err := s.walkPack(verbose, packID,
func(packID int, ref blob.Ref, offset int64, size uint32) error {
if !ref.Valid() {
if camliDebug {
log.Printf("found deleted blob in %d at %d with size %d", packID, offset, size)
}
return nil
}
meta := blobMeta{packID, offset, size}.String()
if overwrite && batch != nil {
batch.Set(ref.String(), meta)
} else {
if old, err := index.Get(ref.String()); err != nil {
allOk = false
if err == sorted.ErrNotFound {
log.Println(ref.String() + ": cannot find in index!")
} else {
log.Println(ref.String()+": error getting from index: ", err.Error())
}
} else if old != meta {
allOk = false
log.Printf("%s: index mismatch - index=%s data=%s", ref.String(), old, meta)
}
}
return nil
})
if err != nil {
return err
}
if overwrite && batch != nil {
log.Printf("overwriting %s from %d", index, packID)
if err = index.CommitBatch(batch); err != nil {
return err
}
} else if !allOk {
return fmt.Errorf("index does not match data in %d", packID)
}
return nil
}
func testInsertLarge(t *testing.T, kv sorted.KeyValue) {
largeKey := make([]byte, sorted.MaxKeySize-1)
// setting all the bytes because postgres whines about an invalid byte sequence
// otherwise
for k, _ := range largeKey {
largeKey[k] = 'A'
}
largeKey[sorted.MaxKeySize-2] = 'B'
largeValue := make([]byte, sorted.MaxValueSize-1)
for k, _ := range largeValue {
largeValue[k] = 'A'
}
largeValue[sorted.MaxValueSize-2] = 'B'
// insert with large key
if err := kv.Set(string(largeKey), "whatever"); err != nil {
t.Fatalf("Insertion of large key failed: %v", err)
}
// and verify we can get it back, i.e. that the key hasn't been truncated.
it := kv.Find(string(largeKey), "")
if !it.Next() || it.Key() != string(largeKey) || it.Value() != "whatever" {
it.Close()
t.Fatalf("Find(largeKey) = %q, %q; want %q, %q", it.Key(), it.Value(), largeKey, "whatever")
}
it.Close()
// insert with large value
if err := kv.Set("whatever", string(largeValue)); err != nil {
t.Fatalf("Insertion of large value failed: %v", err)
}
// and verify we can get it back, i.e. that the value hasn't been truncated.
if v, err := kv.Get("whatever"); err != nil || v != string(largeValue) {
t.Fatalf("get(\"whatever\") = %q, %v; want %q", v, err, largeValue)
}
// insert with large key and large value
if err := kv.Set(string(largeKey), string(largeValue)); err != nil {
t.Fatalf("Insertion of large key and value failed: %v", err)
}
// and verify we can get them back
it = kv.Find(string(largeKey), "")
defer it.Close()
if !it.Next() || it.Key() != string(largeKey) || it.Value() != string(largeValue) {
t.Fatalf("Find(largeKey) = %q, %q; want %q, %q", it.Key(), it.Value(), largeKey, largeValue)
}
}
func TestSorted(t *testing.T, kv sorted.KeyValue) {
defer test.TLog(t)()
if !isEmpty(t, kv) {
t.Fatal("kv for test is expected to be initially empty")
}
set := func(k, v string) {
if err := kv.Set(k, v); err != nil {
t.Fatalf("Error setting %q to %q: %v", k, v, err)
}
}
set("foo", "bar")
if isEmpty(t, kv) {
t.Fatalf("iterator reports the kv is empty after adding foo=bar; iterator must be broken")
}
if v, err := kv.Get("foo"); err != nil || v != "bar" {
t.Errorf("get(foo) = %q, %v; want bar", v, err)
}
if v, err := kv.Get("NOT_EXIST"); err != sorted.ErrNotFound {
t.Errorf("get(NOT_EXIST) = %q, %v; want error sorted.ErrNotFound", v, err)
}
for i := 0; i < 2; i++ {
if err := kv.Delete("foo"); err != nil {
t.Errorf("Delete(foo) (on loop %d/2) returned error %v", i+1, err)
}
}
set("a", "av")
set("b", "bv")
set("c", "cv")
testEnumerate(t, kv, "", "", "av", "bv", "cv")
testEnumerate(t, kv, "a", "", "av", "bv", "cv")
testEnumerate(t, kv, "b", "", "bv", "cv")
testEnumerate(t, kv, "a", "c", "av", "bv")
testEnumerate(t, kv, "a", "b", "av")
testEnumerate(t, kv, "a", "a")
testEnumerate(t, kv, "d", "")
testEnumerate(t, kv, "d", "e")
// Verify that the value isn't being used instead of the key in the range comparison.
set("y", "x:foo")
testEnumerate(t, kv, "x:", "x~")
// TODO: test batch commits
}
func testInsertTooLarge(t *testing.T, kv sorted.KeyValue) {
largeKey := make([]byte, sorted.MaxKeySize+1)
largeValue := make([]byte, sorted.MaxValueSize+1)
err := kv.Set(string(largeKey), "whatever")
if err != nil {
t.Fatalf("Insertion of too large a key should have skipped with some logging, but err was %v", err)
}
_, err = kv.Get(string(largeKey))
if err == nil {
t.Fatal("large key should not have been inserted")
}
err = kv.Set("testInsertTooLarge", string(largeValue))
if err != nil {
t.Fatalf("Insertion of too large a value should have skipped with some logging, but err was %v", err)
}
_, err = kv.Get("testInsertTooLarge")
if err == nil {
t.Fatal("large value should not have been inserted")
}
}
func (s *storage) reindexOne(ctx *context.Context, index sorted.KeyValue, overwrite bool, packID int) error {
var batch sorted.BatchMutation
if overwrite {
batch = index.BeginBatch()
}
allOk := true
// TODO(tgulacsi): proper verbose from context
verbose := env.IsDebug()
misses := make(map[blob.Ref]string, 8)
err := s.walkPack(verbose, packID,
func(packID int, ref blob.Ref, offset int64, size uint32) error {
if !ref.Valid() {
if verbose {
log.Printf("found deleted blob in %d at %d with size %d", packID, offset, size)
}
return nil
}
meta := blobMeta{packID, offset, size}.String()
if overwrite && batch != nil {
batch.Set(ref.String(), meta)
return nil
}
if _, ok := misses[ref]; ok { // maybe this is the last of this blob.
delete(misses, ref)
}
if old, err := index.Get(ref.String()); err != nil {
allOk = false
if err == sorted.ErrNotFound {
log.Println(ref.String() + ": cannot find in index!")
} else {
log.Println(ref.String()+": error getting from index: ", err.Error())
}
} else if old != meta {
if old > meta {
misses[ref] = meta
log.Printf("WARN: possible duplicate blob %s", ref.String())
} else {
allOk = false
log.Printf("ERROR: index mismatch for %s - index=%s, meta=%s!", ref.String(), old, meta)
}
}
return nil
})
if err != nil {
return err
}
for ref, meta := range misses {
log.Printf("ERROR: index mismatch for %s (%s)!", ref.String(), meta)
allOk = false
}
if overwrite && batch != nil {
if err := index.CommitBatch(batch); err != nil {
return err
}
} else if !allOk {
return fmt.Errorf("index does not match data in %d", packID)
}
return nil
}
func reindexOne(index sorted.KeyValue, overwrite, verbose bool, r io.ReadSeeker, name string, packId int64) error {
l, err := lock.Lock(name + ".lock")
defer l.Close()
var pos, size int64
errAt := func(prefix, suffix string) error {
if prefix != "" {
prefix = prefix + " "
}
if suffix != "" {
suffix = " " + suffix
}
return fmt.Errorf(prefix+"at %d (0x%x) in %q:"+suffix, pos, pos, name)
}
var batch sorted.BatchMutation
if overwrite {
batch = index.BeginBatch()
}
allOk := true
br := bufio.NewReaderSize(r, 512)
for {
if b, err := br.ReadByte(); err != nil {
if err == io.EOF {
break
}
return errAt("error while reading", err.Error())
} else if b != '[' {
return errAt(fmt.Sprintf("found byte 0x%x", b), "but '[' should be here!")
}
chunk, err := br.ReadSlice(']')
if err != nil {
if err == io.EOF {
break
}
return errAt("error reading blob header", err.Error())
}
m := len(chunk)
chunk = chunk[:m-1]
i := bytes.IndexByte(chunk, byte(' '))
if i <= 0 {
return errAt("", fmt.Sprintf("bad header format (no space in %q)", chunk))
}
if size, err = strconv.ParseInt(string(chunk[i+1:]), 10, 64); err != nil {
return errAt(fmt.Sprintf("cannot parse size %q as int", chunk[i+1:]), err.Error())
}
ref, ok := blob.Parse(string(chunk[:i]))
if !ok {
return errAt("", fmt.Sprintf("cannot parse %q as blobref", chunk[:i]))
}
if verbose {
log.Printf("found %s at %d", ref, pos)
}
meta := blobMeta{packId, pos + 1 + int64(m), size}.String()
if overwrite && batch != nil {
batch.Set(ref.String(), meta)
} else {
if old, err := index.Get(ref.String()); err != nil {
allOk = false
if err == sorted.ErrNotFound {
log.Println(ref.String() + ": cannot find in index!")
} else {
log.Println(ref.String()+": error getting from index: ", err.Error())
}
} else if old != meta {
allOk = false
log.Printf("%s: index mismatch - index=%s data=%s", ref.String(), old, meta)
}
}
pos += 1 + int64(m)
// TODO(tgulacsi78): not just seek, but check the hashes of the files
// maybe with a different command-line flag, only.
if pos, err = r.Seek(pos+size, 0); err != nil {
return errAt("", "cannot seek +"+strconv.FormatInt(size, 10)+" bytes")
}
// drain the buffer after the underlying reader Seeks
io.CopyN(ioutil.Discard, br, int64(br.Buffered()))
}
if overwrite && batch != nil {
log.Printf("overwriting %s from %s", index, name)
if err = index.CommitBatch(batch); err != nil {
return err
}
} else if !allOk {
return fmt.Errorf("index does not match data in %q", name)
}
return nil
}