func init() {
server := &RESTServer{
Validator: NobodyValidator{},
Items: items.NewWithCache(store.NewMemory(), items.NewMemoryCache()),
TxStore: transaction.New(store.NewMemory()),
FileStore: fragment.New(store.NewMemory()),
Cache: blobcache.NewLRU(store.NewMemory(), 400),
useTape: true,
}
server.txgate = util.NewGate(MaxConcurrentCommits)
server.TxStore.Load()
testServer = httptest.NewServer(server.addRoutes())
}
func TestJSON(t *testing.T) {
memory := store.NewMemory()
js := NewJSON(memory)
first := JTest{Name: "Petra", Age: 30}
err := js.Save("petra", &first)
if err != nil {
t.Fatalf("Got err = %s, expected nil", err.Error())
}
second := new(JTest)
err = js.Open("petra", &second)
if err != nil {
t.Fatalf("Got err = %s, expected nil", err.Error())
}
if second.Name != "Petra" || second.Age != 30 {
t.Fatalf("Got %#v, expected %#v", second, first)
}
// directly inject some JSON and see if it is read correctly
thirdString := `{"Name":"John","Age":50}`
w, _ := memory.Create("john-2")
w.Write([]byte(thirdString))
w.Close()
third := new(JTest)
err = js.Open("john-2", &third)
if err != nil {
t.Fatalf("Got err = %s, expected nil", err.Error())
}
if third.Name != "John" || third.Age != 50 {
t.Fatalf("Got %#v, expected %#v", third, JTest{"John", 50})
}
}
func TestEviction(t *testing.T) {
cache := NewLRU(store.NewMemory(), 100)
// "hello world" is 11 bytes. so 10 should cause a cache eviction
for i := 0; i < 10; i++ {
key := fmt.Sprintf("hello-%d", i)
w, err := cache.Put(key)
if err != nil {
t.Fatalf("received %s", err.Error())
}
w.Write([]byte("hello world"))
w.Close()
}
// see if one was evicted. This does not assume an eviction strategy.
var nEvicted int
for i := 0; i < 10; i++ {
key := fmt.Sprintf("hello-%d", i)
r, size, err := cache.Get(key)
if err != nil {
t.Fatalf("received %s", err.Error())
}
if r == nil {
nEvicted++
continue
}
if size != 11 {
t.Errorf("Received size %d, expected %d", size, 11)
}
r.Close()
}
t.Logf("nEvicted = %d", nEvicted)
if nEvicted == 0 {
t.Errorf("No items evicted")
}
}
func TestDeleteBlob(t *testing.T) {
ms := store.NewMemory()
s := New(ms)
w, err := s.Open("abc", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
w.SetCreator("nobody")
for i := 0; i < 10; i++ {
istring := strconv.Itoa(i)
bid := writedata(t, w, "hello "+istring)
w.SetSlot("slot"+istring, bid)
}
err = w.Close()
if err != nil {
t.Fatalf("Got %s, expected nil", err.Error())
}
// now delete the first five blobs
w, err = s.Open("abc", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
w.DeleteBlob(1)
w.DeleteBlob(2)
w.DeleteBlob(3)
w.DeleteBlob(4)
w.DeleteBlob(5)
w.DeleteBlob(1) // delete this one twice!!
err = w.Close()
if err != nil {
t.Fatalf("Got %s, expected nil", err.Error())
}
// first bundle should be missing
_, _, err = ms.Open(sugar("abc", 1))
if err == nil {
t.Errorf("Received nil, expected error")
}
// is the 10th blob readable?
rc, _, err := s.Blob("abc", 10)
if err != nil {
t.Errorf("Received %s, expected nil", err.Error())
}
var p = make([]byte, 32)
n, _ := rc.Read(p)
rc.Close()
if string(p[:n]) != "hello 9" {
t.Errorf("Received %v, expected 'hello 9'", p)
}
// is the 1st blob deleted?
_, _, err = s.Blob("abc", 1)
if err == nil {
t.Errorf("Received nil, expected error")
}
}
func TestWriteBlob(t *testing.T) {
ms := store.NewMemory()
s := New(ms)
w, err := s.Open("abc", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
// first use the wrong lengths and hashes
bid, err := w.WriteBlob(strings.NewReader("hello world"),
3, // wrong length
nil, // wrong md5
nil) // wrong sha256
if err == nil {
t.Fatalf("Got nil err, expected an error")
}
if bid != 0 {
t.Fatalf("Got blob id %d, expected 0", bid)
}
// now use correct information
bid = writedata(t, w, "hello")
w.SetSlot("slotname", bid)
err = w.Close()
if err != nil {
t.Fatalf("Got %s, expected nil", err.Error())
}
// try opening a second time and see if slot information is copied
w, err = s.Open("abc", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
w.Close()
checkslots(t, s, "abc", []slotTriple{{2, "slotname", bid}})
// now add a new blob and remove the slot entry for the old one
w, err = s.Open("abc", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
w.SetSlot("slotname", 0)
newBid := writedata(t, w, "The cow jumpes over the moon")
w.SetSlot("poem", newBid)
w.Close()
checkslots(t, s, "abc", []slotTriple{
{1, "slotname", bid},
{2, "slotname", bid},
{3, "slotname", 0},
{3, "poem", newBid},
})
}
func TestTagParser(t *testing.T) {
var table = []struct {
name string
contents zdata
tags map[string]string
}{
// Parse normal tag file
{"ok-1",
zdata{
"bag-info.txt": "a-tag: some text\nanother-tag: more text\n extended line",
},
map[string]string{
"a-tag": "some text",
"another-tag": "more text extended line",
}},
{"ok-2",
zdata{
"bag-info.txt": "first tag:important\nthis line is skipped\n\n this line continues the first\n",
},
map[string]string{
"first tag": "important this line continues the first",
}},
}
mstore := store.NewMemory()
for _, tab := range table {
t.Logf("Doing %s", tab.name)
f, err := mstore.Create(tab.name)
if err != nil {
t.Fatal(err)
}
makezipfile(f, tab.contents)
f.Close()
f2, size, err := mstore.Open(tab.name)
if err != nil {
t.Fatal(err)
}
r, err := NewReader(f2, size)
if err != nil {
t.Error(err)
}
tags := r.Tags()
if !mapsEqual(tags, tab.tags) {
t.Errorf("tags unequal received %#v, expected %#v",
tags,
tab.tags)
}
f2.Close()
}
}
func TestChecksum(t *testing.T) {
// Test Data for Checksums tests
cksum := zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}
mstore := store.NewMemory()
f, err := mstore.Create("cksum")
if err != nil {
t.Fatal(err)
}
makezipfile(f, cksum)
f.Close()
f2, size, err := mstore.Open("cksum")
if err != nil {
t.Fatal(err)
}
r, err := NewReader(f2, size)
if err != nil {
t.Error(err)
}
// Verify Existing Key can be retrieved
if r.Checksum("hello1") == nil {
t.Error("Checksum for file 'data/hello1' returns nil")
}
// Verify Existing Key can be retrieved
if r.Checksum("hello2") == nil {
t.Error("Checksum for file 'data/hello2' returns nil")
}
// Verify Checksum of Existing Key is correct
if hex.EncodeToString(r.Checksum("hello2").SHA256) != "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" {
t.Error("Checksum.SHA256 for file 'data/hello2' returns nil")
}
// Verify NonExistent Key Fails
if r.Checksum("hello3") != nil {
t.Error("Checksum for nonexistent file 'data/hello3' returns value")
}
f2.Close()
}
func TestScan(t *testing.T) {
mem := store.NewMemory()
// populate the store
var table = []struct {
key, contents string
}{
{"qwerty", "1234567890"},
{"asdf", "1234567890-="},
{"zxcv", "abcdefghijklmnopqrstuvwxyz"},
}
for _, elem := range table {
w, err := mem.Create(elem.key)
if err != nil {
t.Fatal(err)
}
w.Write([]byte(elem.contents))
w.Close()
}
// now set up the cache and scan it
cache := NewLRU(mem, 100)
cache.Scan()
for _, elem := range table {
r, _, _ := cache.Get(elem.key)
if r == nil {
t.Logf("key %s: nil", elem.key)
continue
}
r.Close()
}
// now set up a small cache and scan that
cache = NewLRU(mem, 15)
cache.Scan()
for _, elem := range table {
r, _, _ := cache.Get(elem.key)
if r == nil {
t.Logf("key %s: nil", elem.key)
continue
}
r.Close()
}
}
func TestWriteDuplicate(t *testing.T) {
ms := store.NewMemory()
s := New(ms)
w, err := s.Open("item-name", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
// write a blob and remember its id
bid := writedata(t, w, "hello")
w.SetSlot("slotname", bid)
// try writing the blob again...see if we get the same id
bid2 := writedata(t, w, "hello")
if bid != bid2 {
t.Errorf("Received %d and expected %d for the blob id", bid2, bid)
}
// try writing a blob without hash values...it should make a new one
bid2, err = w.WriteBlob(strings.NewReader("hello"), 0, nil, nil)
if err != nil {
t.Fatalf("Got %s, expected nil", err.Error())
}
if bid == bid2 {
t.Errorf("Received %d and expected something different", bid2)
}
err = w.Close()
if err != nil {
t.Fatalf("Got %s, expected nil", err.Error())
}
// try opening a second time and see if the same id is still returned
w, err = s.Open("item-name", "nobody")
if err != nil {
t.Fatalf("Unexpected error %s", err.Error())
}
bid2 = writedata(t, w, "hello")
if bid != bid2 {
t.Errorf("Received %d and expected %d for the blob id", bid2, bid)
}
err = w.Close()
if err != nil {
t.Fatalf("Got %s, expected nil", err.Error())
}
}
func TestOpenCorrupt(t *testing.T) {
ms := store.NewMemory()
s := New(ms)
// a bad bundle should cause an error when opening for writing
// make a bad bundle file
out, err := ms.Create(sugar("abc", 1))
if err != nil {
t.Fatalf("Received error %s", err.Error())
}
out.Write([]byte("not a valid zip file")) // never fails for memory store
out.Close() // never fails for memory store
// now try to open
_, err = s.Open("abc", "nobody")
if err != zip.ErrFormat {
t.Fatalf("Received error %s, expected %s", err.Error(), zip.ErrFormat)
}
}
func TestValidate(t *testing.T) {
// Testing validation is difficult since the data needs to be in bundles.
// We spread a our items over two bundles for testing.
// make good item
ms := store.NewMemory()
s := New(ms)
err := createBundledItem(t, s, "gooditem", []itemData{
{bundle: 1,
slot: "stuff/hello",
data: "hello"},
{bundle: 1,
slot: "stuff/hello2",
data: "hello2"},
{bundle: 2,
slot: "morestuff/hello",
data: "hello"},
{bundle: 2,
slot: "stuff/hello3",
data: "hello3"},
})
if err != nil {
t.Fatalf("Received %s, expected nil", err.Error())
}
nb, problems, err := s.Validate("gooditem")
t.Logf("nb = %d", nb)
if len(problems) > 0 {
t.Errorf("Received problems %v", problems)
}
if err != nil {
t.Errorf("Received error %s", err.Error())
}
// making bad items requires mucking with the bundle innards.
// TODO(dbrower): add tests for bad items.
}
func TestTooLargeItem(t *testing.T) {
cache := NewLRU(store.NewMemory(), 100)
key := "qwerty"
w, err := cache.Put(key)
if err != nil {
t.Fatalf("received %s", err.Error())
}
// write this in pieces. should error on last one
for i := 0; i < 10; i++ {
_, err = w.Write([]byte("hello world"))
if err != nil {
t.Logf("Received error %s", err.Error())
break
}
}
if err != ErrCacheFull {
t.Errorf("Did not receive ErrCacheFull")
}
w.Close()
size := cache.size
if size != 0 {
t.Errorf("Cache size is %d. Expected %d", size, 0)
}
}
func TestDelete(t *testing.T) {
cache := NewLRU(store.NewMemory(), 100)
key := "1234"
w, err := cache.Put(key)
if err != nil {
t.Fatalf("received %s", err.Error())
}
w.Write([]byte("abcdefghijklmnopqrstuvwxyz"))
w.Close()
if !cache.Contains(key) {
t.Errorf("Cache does not contain item, expected it to.")
}
if cache.size != 26 {
t.Errorf("Cache size is %d, expected 26", cache.size)
}
err = cache.Delete(key)
if err != nil {
t.Errorf("Error deleting key, %s", err.Error())
}
if cache.Contains(key) {
t.Errorf("Cache contains item, expected it to be deleted.")
}
if cache.size != 0 {
t.Errorf("Cache size is %d, expected 0", cache.size)
}
// delete it a second time. should not get an error
err = cache.Delete(key)
if err != nil {
t.Errorf("Error deleting key, %s", err.Error())
}
}
func TestRoundtrip(t *testing.T) {
// first save a bag
mstore := store.NewMemory()
f, err := mstore.Create("test-bag.zip")
if err != nil {
t.Fatal(err)
}
w := NewWriter(f, "zzz-test-bag")
w.SetTag("Contact-Name", "Nobody")
out, err := w.Create("hello")
if err != nil {
t.Fatal(err)
}
out.Write([]byte("hello there"))
w.Close()
f.Close()
// now read it and see if it matches what was written
f2, size, err := mstore.Open("test-bag.zip")
if err != nil {
t.Fatal(err)
}
r, err := NewReader(f2, size)
if err != nil {
t.Fatal(err)
}
contactName := r.Tags()["Contact-Name"]
if contactName != "Nobody" {
t.Errorf("Read contact name %s, expected %s\n", contactName, "Nobody")
}
version := r.Tags()["BagIt-Version"]
if version != Version {
t.Errorf("Read version %s, expected %s\n", version, Version)
}
// does the hello payload file match?
in, err := r.Open("hello")
if err != nil {
t.Fatal(err)
}
buf := new(bytes.Buffer)
size, err = buf.ReadFrom(in)
if err != nil {
t.Error(err)
}
in.Close()
data := buf.String()
if data != "hello there" {
t.Errorf("Read %s, expected %s\n", data, "hello there")
}
// does the bag verification work?
err = r.Verify()
if err != nil {
t.Errorf("Valid returned %s\n", err.Error())
}
// does the file listing work?
filelist := r.Files()
if len(filelist) != 1 || filelist[0] != "hello" {
t.Errorf("File list is %v, expected [hello]", filelist)
}
f2.Close()
}
func TestVerify(t *testing.T) {
var table = []struct {
name string
contents zdata
ok bool
}{
// payload files split between two manifests
{"ok-1", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, true},
// extra payload file
{"extra-1", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\n",
}, false},
// missing payload file
{"extra-2", zdata{
"data/hello1": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, false},
// missing tag file
{"extra-3", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\nabcdef missing.txt\n",
}, true},
// mismatch payload file
{"checksum-1", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "00000000000000000000000000000000 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "d0d355c1ef01ef6a24b68112d62b1700 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, false},
// mismatch tag file
{"checksum-2", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "00000000000000000000000000000000 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, false},
// extra tag file
{"checksum-3", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"tagfile.txt": "extra tag file",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, true},
// manifest not hex
{"manifest-1", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"tagfile.txt": "extra tag file",
"manifest-md5.txt": "thisisnothexdata0000000000000000 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "f6c4e3fa0e551b551b1fc171f01c1bdf manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, false},
// malformed manifest -- missing final newline
{"manifest-2", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
"tagmanifest-md5.txt": "2afc9fa64386fe74f0500bc6f83b9d9c manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, true},
// manifest line only has hash
{"manifest-3", zdata{
"data/hello1": "hello",
"data/hello2": "hello",
"manifest-md5.txt": "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\n",
"tagmanifest-md5.txt": "2afc9fa64386fe74f0500bc6f83b9d9c manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
}, false},
}
mstore := store.NewMemory()
for _, tab := range table {
t.Logf("Doing %s", tab.name)
f, err := mstore.Create(tab.name)
if err != nil {
t.Fatal(err)
}
makezipfile(f, tab.contents)
f.Close()
f2, size, err := mstore.Open(tab.name)
if err != nil {
t.Fatal(err)
}
r, err := NewReader(f2, size)
if err != nil {
t.Error(err)
}
err = r.Verify()
if tab.ok && err != nil {
t.Errorf("Error, valid returned %s", err.Error())
} else if !tab.ok && err == nil {
t.Errorf("Error, valid returned nil")
}
f2.Close()
}
}
