Пример #1
0
func init() {
	server := &RESTServer{
		Validator: NobodyValidator{},
		Items:     items.NewWithCache(store.NewMemory(), items.NewMemoryCache()),
		TxStore:   transaction.New(store.NewMemory()),
		FileStore: fragment.New(store.NewMemory()),
		Cache:     blobcache.NewLRU(store.NewMemory(), 400),
		useTape:   true,
	}
	server.txgate = util.NewGate(MaxConcurrentCommits)
	server.TxStore.Load()
	testServer = httptest.NewServer(server.addRoutes())
}
Пример #2
0
func TestJSON(t *testing.T) {
	memory := store.NewMemory()
	js := NewJSON(memory)

	first := JTest{Name: "Petra", Age: 30}
	err := js.Save("petra", &first)
	if err != nil {
		t.Fatalf("Got err = %s, expected nil", err.Error())
	}
	second := new(JTest)
	err = js.Open("petra", &second)
	if err != nil {
		t.Fatalf("Got err = %s, expected nil", err.Error())
	}
	if second.Name != "Petra" || second.Age != 30 {
		t.Fatalf("Got %#v, expected %#v", second, first)
	}
	// directly inject some JSON and see if it is read correctly
	thirdString := `{"Name":"John","Age":50}`
	w, _ := memory.Create("john-2")
	w.Write([]byte(thirdString))
	w.Close()
	third := new(JTest)
	err = js.Open("john-2", &third)
	if err != nil {
		t.Fatalf("Got err = %s, expected nil", err.Error())
	}
	if third.Name != "John" || third.Age != 50 {
		t.Fatalf("Got %#v, expected %#v", third, JTest{"John", 50})
	}
}
Пример #3
0
func TestEviction(t *testing.T) {
	cache := NewLRU(store.NewMemory(), 100)
	// "hello world" is 11 bytes. so 10 should cause a cache eviction
	for i := 0; i < 10; i++ {
		key := fmt.Sprintf("hello-%d", i)
		w, err := cache.Put(key)
		if err != nil {
			t.Fatalf("received %s", err.Error())
		}
		w.Write([]byte("hello world"))
		w.Close()
	}

	// see if one was evicted. This does not assume an eviction strategy.
	var nEvicted int
	for i := 0; i < 10; i++ {
		key := fmt.Sprintf("hello-%d", i)
		r, size, err := cache.Get(key)
		if err != nil {
			t.Fatalf("received %s", err.Error())
		}
		if r == nil {
			nEvicted++
			continue
		}
		if size != 11 {
			t.Errorf("Received size %d, expected %d", size, 11)
		}
		r.Close()
	}
	t.Logf("nEvicted = %d", nEvicted)
	if nEvicted == 0 {
		t.Errorf("No items evicted")
	}
}
Пример #4
0
func TestDeleteBlob(t *testing.T) {
	ms := store.NewMemory()
	s := New(ms)
	w, err := s.Open("abc", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}
	w.SetCreator("nobody")

	for i := 0; i < 10; i++ {
		istring := strconv.Itoa(i)
		bid := writedata(t, w, "hello "+istring)
		w.SetSlot("slot"+istring, bid)
	}
	err = w.Close()
	if err != nil {
		t.Fatalf("Got %s, expected nil", err.Error())
	}

	// now delete the first five blobs
	w, err = s.Open("abc", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}
	w.DeleteBlob(1)
	w.DeleteBlob(2)
	w.DeleteBlob(3)
	w.DeleteBlob(4)
	w.DeleteBlob(5)
	w.DeleteBlob(1) // delete this one twice!!
	err = w.Close()
	if err != nil {
		t.Fatalf("Got %s, expected nil", err.Error())
	}

	// first bundle should be missing
	_, _, err = ms.Open(sugar("abc", 1))
	if err == nil {
		t.Errorf("Received nil, expected error")
	}

	// is the 10th blob readable?
	rc, _, err := s.Blob("abc", 10)
	if err != nil {
		t.Errorf("Received %s, expected nil", err.Error())
	}
	var p = make([]byte, 32)
	n, _ := rc.Read(p)
	rc.Close()
	if string(p[:n]) != "hello 9" {
		t.Errorf("Received %v, expected 'hello 9'", p)
	}

	// is the 1st blob deleted?
	_, _, err = s.Blob("abc", 1)
	if err == nil {
		t.Errorf("Received nil, expected error")
	}
}
Пример #5
0
func TestWriteBlob(t *testing.T) {
	ms := store.NewMemory()
	s := New(ms)
	w, err := s.Open("abc", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}

	// first use the wrong lengths and hashes
	bid, err := w.WriteBlob(strings.NewReader("hello world"),
		3,   // wrong length
		nil, // wrong md5
		nil) // wrong sha256
	if err == nil {
		t.Fatalf("Got nil err, expected an error")
	}
	if bid != 0 {
		t.Fatalf("Got blob id %d, expected 0", bid)
	}

	// now use correct information
	bid = writedata(t, w, "hello")
	w.SetSlot("slotname", bid)
	err = w.Close()
	if err != nil {
		t.Fatalf("Got %s, expected nil", err.Error())
	}

	// try opening a second time and see if slot information is copied
	w, err = s.Open("abc", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}
	w.Close()

	checkslots(t, s, "abc", []slotTriple{{2, "slotname", bid}})

	// now add a new blob and remove the slot entry for the old one
	w, err = s.Open("abc", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}
	w.SetSlot("slotname", 0)
	newBid := writedata(t, w, "The cow jumpes over the moon")
	w.SetSlot("poem", newBid)
	w.Close()

	checkslots(t, s, "abc", []slotTriple{
		{1, "slotname", bid},
		{2, "slotname", bid},
		{3, "slotname", 0},
		{3, "poem", newBid},
	})
}
Пример #6
0
func TestTagParser(t *testing.T) {
	var table = []struct {
		name     string
		contents zdata
		tags     map[string]string
	}{
		// Parse normal tag file
		{"ok-1",
			zdata{
				"bag-info.txt": "a-tag: some text\nanother-tag: more text\n  extended line",
			},
			map[string]string{
				"a-tag":       "some text",
				"another-tag": "more text extended line",
			}},
		{"ok-2",
			zdata{
				"bag-info.txt": "first tag:important\nthis line is skipped\n\n this line continues the first\n",
			},
			map[string]string{
				"first tag": "important this line continues the first",
			}},
	}

	mstore := store.NewMemory()
	for _, tab := range table {
		t.Logf("Doing %s", tab.name)
		f, err := mstore.Create(tab.name)
		if err != nil {
			t.Fatal(err)
		}
		makezipfile(f, tab.contents)
		f.Close()

		f2, size, err := mstore.Open(tab.name)
		if err != nil {
			t.Fatal(err)
		}

		r, err := NewReader(f2, size)
		if err != nil {
			t.Error(err)
		}
		tags := r.Tags()
		if !mapsEqual(tags, tab.tags) {
			t.Errorf("tags unequal received %#v, expected %#v",
				tags,
				tab.tags)
		}
		f2.Close()
	}
}
Пример #7
0
func TestChecksum(t *testing.T) {

	// Test Data for Checksums tests
	cksum := zdata{
		"data/hello1":         "hello",
		"data/hello2":         "hello",
		"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
		"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
		"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
	}

	mstore := store.NewMemory()
	f, err := mstore.Create("cksum")
	if err != nil {
		t.Fatal(err)
	}
	makezipfile(f, cksum)
	f.Close()

	f2, size, err := mstore.Open("cksum")
	if err != nil {
		t.Fatal(err)
	}

	r, err := NewReader(f2, size)
	if err != nil {
		t.Error(err)
	}

	// Verify Existing Key can be retrieved
	if r.Checksum("hello1") == nil {
		t.Error("Checksum for file 'data/hello1' returns nil")
	}

	// Verify Existing Key can be retrieved
	if r.Checksum("hello2") == nil {
		t.Error("Checksum for file 'data/hello2' returns nil")
	}

	// Verify Checksum of Existing Key is correct
	if hex.EncodeToString(r.Checksum("hello2").SHA256) != "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" {
		t.Error("Checksum.SHA256 for file 'data/hello2' returns nil")
	}

	// Verify NonExistent Key Fails
	if r.Checksum("hello3") != nil {
		t.Error("Checksum for nonexistent file 'data/hello3' returns value")
	}

	f2.Close()
}
Пример #8
0
func TestScan(t *testing.T) {
	mem := store.NewMemory()

	// populate the store
	var table = []struct {
		key, contents string
	}{
		{"qwerty", "1234567890"},
		{"asdf", "1234567890-="},
		{"zxcv", "abcdefghijklmnopqrstuvwxyz"},
	}

	for _, elem := range table {
		w, err := mem.Create(elem.key)
		if err != nil {
			t.Fatal(err)
		}
		w.Write([]byte(elem.contents))
		w.Close()
	}

	// now set up the cache and scan it
	cache := NewLRU(mem, 100)
	cache.Scan()

	for _, elem := range table {
		r, _, _ := cache.Get(elem.key)
		if r == nil {
			t.Logf("key %s: nil", elem.key)
			continue
		}
		r.Close()
	}

	// now set up a small cache and scan that
	cache = NewLRU(mem, 15)
	cache.Scan()

	for _, elem := range table {
		r, _, _ := cache.Get(elem.key)
		if r == nil {
			t.Logf("key %s: nil", elem.key)
			continue
		}
		r.Close()
	}
}
Пример #9
0
func TestWriteDuplicate(t *testing.T) {
	ms := store.NewMemory()
	s := New(ms)
	w, err := s.Open("item-name", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}

	// write a blob and remember its id
	bid := writedata(t, w, "hello")
	w.SetSlot("slotname", bid)

	// try writing the blob again...see if we get the same id
	bid2 := writedata(t, w, "hello")
	if bid != bid2 {
		t.Errorf("Received %d and expected %d for the blob id", bid2, bid)
	}

	// try writing a blob without hash values...it should make a new one
	bid2, err = w.WriteBlob(strings.NewReader("hello"), 0, nil, nil)
	if err != nil {
		t.Fatalf("Got %s, expected nil", err.Error())
	}
	if bid == bid2 {
		t.Errorf("Received %d and expected something different", bid2)
	}

	err = w.Close()
	if err != nil {
		t.Fatalf("Got %s, expected nil", err.Error())
	}

	// try opening a second time and see if the same id is still returned
	w, err = s.Open("item-name", "nobody")
	if err != nil {
		t.Fatalf("Unexpected error %s", err.Error())
	}
	bid2 = writedata(t, w, "hello")
	if bid != bid2 {
		t.Errorf("Received %d and expected %d for the blob id", bid2, bid)
	}
	err = w.Close()
	if err != nil {
		t.Fatalf("Got %s, expected nil", err.Error())
	}
}
Пример #10
0
func TestOpenCorrupt(t *testing.T) {
	ms := store.NewMemory()
	s := New(ms)

	// a bad bundle should cause an error when opening for writing
	// make a bad bundle file
	out, err := ms.Create(sugar("abc", 1))
	if err != nil {
		t.Fatalf("Received error %s", err.Error())
	}
	out.Write([]byte("not a valid zip file")) // never fails for memory store
	out.Close()                               // never fails for memory store

	// now try to open
	_, err = s.Open("abc", "nobody")
	if err != zip.ErrFormat {
		t.Fatalf("Received error %s, expected %s", err.Error(), zip.ErrFormat)
	}
}
Пример #11
0
func TestValidate(t *testing.T) {
	// Testing validation is difficult since the data needs to be in bundles.
	// We spread a our items over two bundles for testing.

	// make good item
	ms := store.NewMemory()
	s := New(ms)
	err := createBundledItem(t, s, "gooditem", []itemData{
		{bundle: 1,
			slot: "stuff/hello",
			data: "hello"},
		{bundle: 1,
			slot: "stuff/hello2",
			data: "hello2"},
		{bundle: 2,
			slot: "morestuff/hello",
			data: "hello"},
		{bundle: 2,
			slot: "stuff/hello3",
			data: "hello3"},
	})

	if err != nil {
		t.Fatalf("Received %s, expected nil", err.Error())
	}

	nb, problems, err := s.Validate("gooditem")

	t.Logf("nb = %d", nb)
	if len(problems) > 0 {
		t.Errorf("Received problems %v", problems)
	}
	if err != nil {
		t.Errorf("Received error %s", err.Error())
	}

	// making bad items requires mucking with the bundle innards.
	// TODO(dbrower): add tests for bad items.
}
Пример #12
0
func TestTooLargeItem(t *testing.T) {
	cache := NewLRU(store.NewMemory(), 100)
	key := "qwerty"
	w, err := cache.Put(key)
	if err != nil {
		t.Fatalf("received %s", err.Error())
	}
	// write this in pieces. should error on last one
	for i := 0; i < 10; i++ {
		_, err = w.Write([]byte("hello world"))
		if err != nil {
			t.Logf("Received error %s", err.Error())
			break
		}
	}
	if err != ErrCacheFull {
		t.Errorf("Did not receive ErrCacheFull")
	}
	w.Close()
	size := cache.size
	if size != 0 {
		t.Errorf("Cache size is %d. Expected %d", size, 0)
	}
}
Пример #13
0
func TestDelete(t *testing.T) {
	cache := NewLRU(store.NewMemory(), 100)
	key := "1234"
	w, err := cache.Put(key)
	if err != nil {
		t.Fatalf("received %s", err.Error())
	}
	w.Write([]byte("abcdefghijklmnopqrstuvwxyz"))
	w.Close()

	if !cache.Contains(key) {
		t.Errorf("Cache does not contain item, expected it to.")
	}
	if cache.size != 26 {
		t.Errorf("Cache size is %d, expected 26", cache.size)
	}

	err = cache.Delete(key)
	if err != nil {
		t.Errorf("Error deleting key, %s", err.Error())
	}

	if cache.Contains(key) {
		t.Errorf("Cache contains item, expected it to be deleted.")
	}
	if cache.size != 0 {
		t.Errorf("Cache size is %d, expected 0", cache.size)
	}

	// delete it a second time. should not get an error
	err = cache.Delete(key)
	if err != nil {
		t.Errorf("Error deleting key, %s", err.Error())
	}

}
Пример #14
0
func TestRoundtrip(t *testing.T) {
	// first save a bag
	mstore := store.NewMemory()
	f, err := mstore.Create("test-bag.zip")
	if err != nil {
		t.Fatal(err)
	}
	w := NewWriter(f, "zzz-test-bag")
	w.SetTag("Contact-Name", "Nobody")
	out, err := w.Create("hello")
	if err != nil {
		t.Fatal(err)
	}
	out.Write([]byte("hello there"))
	w.Close()
	f.Close()

	// now read it and see if it matches what was written
	f2, size, err := mstore.Open("test-bag.zip")
	if err != nil {
		t.Fatal(err)
	}
	r, err := NewReader(f2, size)
	if err != nil {
		t.Fatal(err)
	}
	contactName := r.Tags()["Contact-Name"]
	if contactName != "Nobody" {
		t.Errorf("Read contact name %s, expected %s\n", contactName, "Nobody")
	}
	version := r.Tags()["BagIt-Version"]
	if version != Version {
		t.Errorf("Read version %s, expected %s\n", version, Version)
	}

	// does the hello payload file match?
	in, err := r.Open("hello")
	if err != nil {
		t.Fatal(err)
	}
	buf := new(bytes.Buffer)
	size, err = buf.ReadFrom(in)
	if err != nil {
		t.Error(err)
	}
	in.Close()
	data := buf.String()
	if data != "hello there" {
		t.Errorf("Read %s, expected %s\n", data, "hello there")
	}

	// does the bag verification work?
	err = r.Verify()
	if err != nil {
		t.Errorf("Valid returned %s\n", err.Error())
	}

	// does the file listing work?
	filelist := r.Files()
	if len(filelist) != 1 || filelist[0] != "hello" {
		t.Errorf("File list is %v, expected [hello]", filelist)
	}

	f2.Close()
}
Пример #15
0
func TestVerify(t *testing.T) {
	var table = []struct {
		name     string
		contents zdata
		ok       bool
	}{
		// payload files split between two manifests
		{"ok-1", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, true},
		// extra payload file
		{"extra-1", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\n",
		}, false},
		// missing payload file
		{"extra-2", zdata{
			"data/hello1":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, false},
		// missing tag file
		{"extra-3", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\nabcdef missing.txt\n",
		}, true},
		// mismatch payload file
		{"checksum-1", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "00000000000000000000000000000000 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "d0d355c1ef01ef6a24b68112d62b1700 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, false},
		// mismatch tag file
		{"checksum-2", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "00000000000000000000000000000000 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, false},
		// extra tag file
		{"checksum-3", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"tagfile.txt":         "extra tag file",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "49ce66cef8d32ec33eca290c2c731185 manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, true},
		// manifest not hex
		{"manifest-1", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"tagfile.txt":         "extra tag file",
			"manifest-md5.txt":    "thisisnothexdata0000000000000000 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "f6c4e3fa0e551b551b1fc171f01c1bdf manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, false},
		// malformed manifest -- missing final newline
		{"manifest-2", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 data/hello2\n",
			"tagmanifest-md5.txt": "2afc9fa64386fe74f0500bc6f83b9d9c manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, true},
		// manifest line only has hash
		{"manifest-3", zdata{
			"data/hello1":         "hello",
			"data/hello2":         "hello",
			"manifest-md5.txt":    "5d41402abc4b2a76b9719d911017c592 data/hello1\n",
			"manifest-sha256.txt": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\n",
			"tagmanifest-md5.txt": "2afc9fa64386fe74f0500bc6f83b9d9c manifest-md5.txt\nbd41f3fc8aa771760265275d3576a30a manifest-sha256.txt\n",
		}, false},
	}

	mstore := store.NewMemory()
	for _, tab := range table {
		t.Logf("Doing %s", tab.name)
		f, err := mstore.Create(tab.name)
		if err != nil {
			t.Fatal(err)
		}
		makezipfile(f, tab.contents)
		f.Close()

		f2, size, err := mstore.Open(tab.name)
		if err != nil {
			t.Fatal(err)
		}

		r, err := NewReader(f2, size)
		if err != nil {
			t.Error(err)
		}
		err = r.Verify()
		if tab.ok && err != nil {
			t.Errorf("Error, valid returned %s", err.Error())
		} else if !tab.ok && err == nil {
			t.Errorf("Error, valid returned nil")
		}
		f2.Close()
	}
}