Пример #1
0
func (server *isolatedFake) finalizeGSUpload(r *http.Request) interface{} {
	data := &isolated.FinalizeRequest{}
	if err := json.NewDecoder(r.Body).Decode(data); err != nil {
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}
	prefix := "ticket:"
	if !strings.HasPrefix(data.UploadTicket, prefix) {
		err := fmt.Errorf("unexpected ticket %#v", data.UploadTicket)
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}
	digest := isolated.HexDigest(data.UploadTicket[len(prefix):])
	if !digest.Validate() {
		err := fmt.Errorf("invalid digest %#v", digest)
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}

	server.lock.Lock()
	defer server.lock.Unlock()
	if _, ok := server.staging[digest]; !ok {
		err := fmt.Errorf("finalizing non uploaded file")
		server.failLocked(err)
		return map[string]string{"err": err.Error()}
	}
	server.contents[digest] = server.staging[digest]
	delete(server.staging, digest)
	return map[string]string{"ok": "true"}
}
Пример #2
0
func (server *isolatedFake) fakeCloudStorage(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	if r.Header.Get("Content-Type") != "application/octet-stream" {
		w.WriteHeader(400)
		server.Fail(fmt.Errorf("invalid content type: %s", r.Header.Get("Content-Type")))
		return
	}
	if r.Method != "PUT" {
		w.WriteHeader(405)
		server.Fail(fmt.Errorf("invalid method: %s", r.Method))
		return
	}
	raw, err := ioutil.ReadAll(isolated.GetDecompressor(r.Body))
	if err != nil {
		w.WriteHeader(500)
		server.Fail(err)
		return
	}
	digest := isolated.HexDigest(r.URL.Query().Get("digest"))
	if digest != isolated.HashBytes(raw) {
		w.WriteHeader(400)
		server.Fail(fmt.Errorf("invalid digest %#v", digest))
		return
	}

	server.lock.Lock()
	defer server.lock.Unlock()
	server.staging[digest] = raw
	w.WriteHeader(200)
}
Пример #3
0
func TestArchiverFile(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)

	fEmpty, err := ioutil.TempFile("", "archiver")
	ut.AssertEqual(t, nil, err)
	future1 := a.PushFile(fEmpty.Name(), fEmpty.Name(), 0)
	ut.AssertEqual(t, fEmpty.Name(), future1.DisplayName())
	fFoo, err := ioutil.TempFile("", "archiver")
	ut.AssertEqual(t, nil, err)
	ut.AssertEqual(t, nil, ioutil.WriteFile(fFoo.Name(), []byte("foo"), 0600))
	future2 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
	// Push the same file another time. It'll get linked to the first.
	future3 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
	future1.WaitForHashed()
	future2.WaitForHashed()
	future3.WaitForHashed()
	ut.AssertEqual(t, nil, a.Close())

	stats := a.Stats()
	ut.AssertEqual(t, 0, stats.TotalHits())
	// Only 2 lookups, not 3.
	ut.AssertEqual(t, 2, stats.TotalMisses())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
	ut.AssertEqual(t, common.Size(3), stats.TotalBytesPushed())
	expected := map[isolated.HexDigest][]byte{
		"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"),
		"da39a3ee5e6b4b0d3255bfef95601890afd80709": {},
	}
	ut.AssertEqual(t, expected, server.Contents())
	ut.AssertEqual(t, isolated.HexDigest("da39a3ee5e6b4b0d3255bfef95601890afd80709"), future1.Digest())
	ut.AssertEqual(t, nil, future1.Error())
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future2.Digest())
	ut.AssertEqual(t, nil, future2.Error())
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future3.Digest())
	ut.AssertEqual(t, nil, future3.Error())
	ut.AssertEqual(t, nil, server.Error())
}
Пример #4
0
func TestArchiverPushSeeked(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
	misplaced := bytes.NewReader([]byte("foo"))
	_, _ = misplaced.Seek(1, os.SEEK_SET)
	future := a.Push("works", misplaced, 0)
	future.WaitForHashed()
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
	ut.AssertEqual(t, nil, a.Close())
}
Пример #5
0
func TestArchiverFileHit(t *testing.T) {
	t.Parallel()
	server := isolatedfake.New()
	ts := httptest.NewServer(server)
	defer ts.Close()
	a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
	server.Inject([]byte("foo"))
	future := a.Push("foo", bytes.NewReader([]byte("foo")), 0)
	future.WaitForHashed()
	ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
	ut.AssertEqual(t, nil, a.Close())

	stats := a.Stats()
	ut.AssertEqual(t, 1, stats.TotalHits())
	ut.AssertEqual(t, 0, stats.TotalMisses())
	ut.AssertEqual(t, common.Size(3), stats.TotalBytesHits())
	ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed())
}
Пример #6
0
func (e *entry) UnmarshalJSON(data []byte) error {
	// decode from tuple.
	var elems []interface{}
	if err := json.Unmarshal(data, &elems); err != nil {
		return fmt.Errorf("invalid entry: %s: %s", err, string(data))
	}
	if len(elems) != 2 {
		return fmt.Errorf("invalid entry: expected 2 items: %s", string(data))
	}
	if key, ok := elems[0].(string); ok {
		e.key = isolated.HexDigest(key)
		if value, ok := elems[1].(float64); ok {
			e.value = common.Size(value)
		} else {
			return fmt.Errorf("invalid entry: expected value to be number: %s", string(data))
		}
	} else {
		return fmt.Errorf("invalid entry: expected key to be string: %s", string(data))
	}
	return nil
}
Пример #7
0
func (server *isolatedFake) storeInline(r *http.Request) interface{} {
	data := &isolated.StorageRequest{}
	if err := json.NewDecoder(r.Body).Decode(data); err != nil {
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}

	prefix := "ticket:"
	if !strings.HasPrefix(data.UploadTicket, prefix) {
		err := fmt.Errorf("unexpected ticket %#v", data.UploadTicket)
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}

	digest := isolated.HexDigest(data.UploadTicket[len(prefix):])
	if !digest.Validate() {
		err := fmt.Errorf("invalid digest %#v", digest)
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}
	raw, err := ioutil.ReadAll(isolated.GetDecompressor(bytes.NewBuffer(data.Content)))
	if err != nil {
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}
	if digest != isolated.HashBytes(raw) {
		err := fmt.Errorf("invalid digest %#v", digest)
		server.Fail(err)
		return map[string]string{"err": err.Error()}
	}

	server.lock.Lock()
	defer server.lock.Unlock()
	server.contents[digest] = raw
	return map[string]string{"ok": "true"}
}
Пример #8
0
func testCache(t *testing.T, c Cache) []isolated.HexDigest {
	// c's policies must have MaxItems == 2 and MaxSize == 1024.
	td, err := ioutil.TempDir("", "cache")
	ut.AssertEqual(t, nil, err)
	defer func() {
		if err := os.RemoveAll(td); err != nil {
			t.Fail()
		}
	}()

	fakeDigest := isolated.HexDigest("0123456789012345678901234567890123456789")
	badDigest := isolated.HexDigest("012345678901234567890123456789012345678")
	emptyContent := []byte{}
	emptyDigest := isolated.HashBytes(emptyContent)
	file1Content := []byte("foo")
	file1Digest := isolated.HashBytes(file1Content)
	file2Content := []byte("foo bar")
	file2Digest := isolated.HashBytes(file2Content)
	largeContent := bytes.Repeat([]byte("A"), 1023)
	largeDigest := isolated.HashBytes(largeContent)
	tooLargeContent := bytes.Repeat([]byte("A"), 1025)
	tooLargeDigest := isolated.HashBytes(tooLargeContent)

	ut.AssertEqual(t, []isolated.HexDigest{}, c.Keys())

	ut.AssertEqual(t, false, c.Touch(fakeDigest))
	ut.AssertEqual(t, false, c.Touch(badDigest))

	c.Evict(fakeDigest)
	c.Evict(badDigest)

	r, err := c.Read(fakeDigest)
	ut.AssertEqual(t, nil, r)
	ut.AssertEqual(t, true, err != nil)
	r, err = c.Read(badDigest)
	ut.AssertEqual(t, nil, r)
	ut.AssertEqual(t, true, err != nil)

	// It's too large to fit in the cache.
	ut.AssertEqual(t, true, nil != c.Add(tooLargeDigest, bytes.NewBuffer(tooLargeContent)))

	// It gets discarded because it's too large.
	ut.AssertEqual(t, nil, c.Add(largeDigest, bytes.NewBuffer(largeContent)))
	ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent)))
	ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent)))
	ut.AssertEqual(t, []isolated.HexDigest{emptyDigest, largeDigest}, c.Keys())
	c.Evict(emptyDigest)
	ut.AssertEqual(t, []isolated.HexDigest{largeDigest}, c.Keys())
	ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent)))

	ut.AssertEqual(t, nil, c.Add(file1Digest, bytes.NewBuffer(file1Content)))
	ut.AssertEqual(t, true, c.Touch(emptyDigest))
	ut.AssertEqual(t, nil, c.Add(file2Digest, bytes.NewBuffer(file2Content)))

	r, err = c.Read(file1Digest)
	ut.AssertEqual(t, nil, r)
	ut.AssertEqual(t, true, nil != err)
	r, err = c.Read(file2Digest)
	ut.AssertEqual(t, nil, err)
	actual, err := ioutil.ReadAll(r)
	ut.AssertEqual(t, nil, r.Close())
	ut.AssertEqual(t, nil, err)
	ut.AssertEqual(t, file2Content, actual)

	expected := []isolated.HexDigest{file2Digest, emptyDigest}
	ut.AssertEqual(t, expected, c.Keys())

	dest := filepath.Join(td, "foo")
	ut.AssertEqual(t, true, nil != c.Hardlink(fakeDigest, dest, os.FileMode(0600)))
	ut.AssertEqual(t, true, nil != c.Hardlink(badDigest, dest, os.FileMode(0600)))
	ut.AssertEqual(t, nil, c.Hardlink(file2Digest, dest, os.FileMode(0600)))
	// See comment about the fact that it may or may not work.
	_ = c.Hardlink(file2Digest, dest, os.FileMode(0600))
	actual, err = ioutil.ReadFile(dest)
	ut.AssertEqual(t, nil, err)
	ut.AssertEqual(t, file2Content, actual)

	ut.AssertEqual(t, nil, c.Close())
	return expected
}