func (server *isolatedFake) finalizeGSUpload(r *http.Request) interface{} {
data := &isolated.FinalizeRequest{}
if err := json.NewDecoder(r.Body).Decode(data); err != nil {
server.Fail(err)
return map[string]string{"err": err.Error()}
}
prefix := "ticket:"
if !strings.HasPrefix(data.UploadTicket, prefix) {
err := fmt.Errorf("unexpected ticket %#v", data.UploadTicket)
server.Fail(err)
return map[string]string{"err": err.Error()}
}
digest := isolated.HexDigest(data.UploadTicket[len(prefix):])
if !digest.Validate() {
err := fmt.Errorf("invalid digest %#v", digest)
server.Fail(err)
return map[string]string{"err": err.Error()}
}
server.lock.Lock()
defer server.lock.Unlock()
if _, ok := server.staging[digest]; !ok {
err := fmt.Errorf("finalizing non uploaded file")
server.failLocked(err)
return map[string]string{"err": err.Error()}
}
server.contents[digest] = server.staging[digest]
delete(server.staging, digest)
return map[string]string{"ok": "true"}
}
func (server *isolatedFake) fakeCloudStorage(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if r.Header.Get("Content-Type") != "application/octet-stream" {
w.WriteHeader(400)
server.Fail(fmt.Errorf("invalid content type: %s", r.Header.Get("Content-Type")))
return
}
if r.Method != "PUT" {
w.WriteHeader(405)
server.Fail(fmt.Errorf("invalid method: %s", r.Method))
return
}
raw, err := ioutil.ReadAll(isolated.GetDecompressor(r.Body))
if err != nil {
w.WriteHeader(500)
server.Fail(err)
return
}
digest := isolated.HexDigest(r.URL.Query().Get("digest"))
if digest != isolated.HashBytes(raw) {
w.WriteHeader(400)
server.Fail(fmt.Errorf("invalid digest %#v", digest))
return
}
server.lock.Lock()
defer server.lock.Unlock()
server.staging[digest] = raw
w.WriteHeader(200)
}
func TestArchiverFile(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
fEmpty, err := ioutil.TempFile("", "archiver")
ut.AssertEqual(t, nil, err)
future1 := a.PushFile(fEmpty.Name(), fEmpty.Name(), 0)
ut.AssertEqual(t, fEmpty.Name(), future1.DisplayName())
fFoo, err := ioutil.TempFile("", "archiver")
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, nil, ioutil.WriteFile(fFoo.Name(), []byte("foo"), 0600))
future2 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
// Push the same file another time. It'll get linked to the first.
future3 := a.PushFile(fFoo.Name(), fFoo.Name(), 0)
future1.WaitForHashed()
future2.WaitForHashed()
future3.WaitForHashed()
ut.AssertEqual(t, nil, a.Close())
stats := a.Stats()
ut.AssertEqual(t, 0, stats.TotalHits())
// Only 2 lookups, not 3.
ut.AssertEqual(t, 2, stats.TotalMisses())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits())
ut.AssertEqual(t, common.Size(3), stats.TotalBytesPushed())
expected := map[isolated.HexDigest][]byte{
"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"),
"da39a3ee5e6b4b0d3255bfef95601890afd80709": {},
}
ut.AssertEqual(t, expected, server.Contents())
ut.AssertEqual(t, isolated.HexDigest("da39a3ee5e6b4b0d3255bfef95601890afd80709"), future1.Digest())
ut.AssertEqual(t, nil, future1.Error())
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future2.Digest())
ut.AssertEqual(t, nil, future2.Error())
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future3.Digest())
ut.AssertEqual(t, nil, future3.Error())
ut.AssertEqual(t, nil, server.Error())
}
func TestArchiverPushSeeked(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
misplaced := bytes.NewReader([]byte("foo"))
_, _ = misplaced.Seek(1, os.SEEK_SET)
future := a.Push("works", misplaced, 0)
future.WaitForHashed()
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
ut.AssertEqual(t, nil, a.Close())
}
func TestArchiverFileHit(t *testing.T) {
t.Parallel()
server := isolatedfake.New()
ts := httptest.NewServer(server)
defer ts.Close()
a := New(isolatedclient.New(ts.URL, "default-gzip"), nil)
server.Inject([]byte("foo"))
future := a.Push("foo", bytes.NewReader([]byte("foo")), 0)
future.WaitForHashed()
ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest())
ut.AssertEqual(t, nil, a.Close())
stats := a.Stats()
ut.AssertEqual(t, 1, stats.TotalHits())
ut.AssertEqual(t, 0, stats.TotalMisses())
ut.AssertEqual(t, common.Size(3), stats.TotalBytesHits())
ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed())
}
func (e *entry) UnmarshalJSON(data []byte) error {
// decode from tuple.
var elems []interface{}
if err := json.Unmarshal(data, &elems); err != nil {
return fmt.Errorf("invalid entry: %s: %s", err, string(data))
}
if len(elems) != 2 {
return fmt.Errorf("invalid entry: expected 2 items: %s", string(data))
}
if key, ok := elems[0].(string); ok {
e.key = isolated.HexDigest(key)
if value, ok := elems[1].(float64); ok {
e.value = common.Size(value)
} else {
return fmt.Errorf("invalid entry: expected value to be number: %s", string(data))
}
} else {
return fmt.Errorf("invalid entry: expected key to be string: %s", string(data))
}
return nil
}
func (server *isolatedFake) storeInline(r *http.Request) interface{} {
data := &isolated.StorageRequest{}
if err := json.NewDecoder(r.Body).Decode(data); err != nil {
server.Fail(err)
return map[string]string{"err": err.Error()}
}
prefix := "ticket:"
if !strings.HasPrefix(data.UploadTicket, prefix) {
err := fmt.Errorf("unexpected ticket %#v", data.UploadTicket)
server.Fail(err)
return map[string]string{"err": err.Error()}
}
digest := isolated.HexDigest(data.UploadTicket[len(prefix):])
if !digest.Validate() {
err := fmt.Errorf("invalid digest %#v", digest)
server.Fail(err)
return map[string]string{"err": err.Error()}
}
raw, err := ioutil.ReadAll(isolated.GetDecompressor(bytes.NewBuffer(data.Content)))
if err != nil {
server.Fail(err)
return map[string]string{"err": err.Error()}
}
if digest != isolated.HashBytes(raw) {
err := fmt.Errorf("invalid digest %#v", digest)
server.Fail(err)
return map[string]string{"err": err.Error()}
}
server.lock.Lock()
defer server.lock.Unlock()
server.contents[digest] = raw
return map[string]string{"ok": "true"}
}
func testCache(t *testing.T, c Cache) []isolated.HexDigest {
// c's policies must have MaxItems == 2 and MaxSize == 1024.
td, err := ioutil.TempDir("", "cache")
ut.AssertEqual(t, nil, err)
defer func() {
if err := os.RemoveAll(td); err != nil {
t.Fail()
}
}()
fakeDigest := isolated.HexDigest("0123456789012345678901234567890123456789")
badDigest := isolated.HexDigest("012345678901234567890123456789012345678")
emptyContent := []byte{}
emptyDigest := isolated.HashBytes(emptyContent)
file1Content := []byte("foo")
file1Digest := isolated.HashBytes(file1Content)
file2Content := []byte("foo bar")
file2Digest := isolated.HashBytes(file2Content)
largeContent := bytes.Repeat([]byte("A"), 1023)
largeDigest := isolated.HashBytes(largeContent)
tooLargeContent := bytes.Repeat([]byte("A"), 1025)
tooLargeDigest := isolated.HashBytes(tooLargeContent)
ut.AssertEqual(t, []isolated.HexDigest{}, c.Keys())
ut.AssertEqual(t, false, c.Touch(fakeDigest))
ut.AssertEqual(t, false, c.Touch(badDigest))
c.Evict(fakeDigest)
c.Evict(badDigest)
r, err := c.Read(fakeDigest)
ut.AssertEqual(t, nil, r)
ut.AssertEqual(t, true, err != nil)
r, err = c.Read(badDigest)
ut.AssertEqual(t, nil, r)
ut.AssertEqual(t, true, err != nil)
// It's too large to fit in the cache.
ut.AssertEqual(t, true, nil != c.Add(tooLargeDigest, bytes.NewBuffer(tooLargeContent)))
// It gets discarded because it's too large.
ut.AssertEqual(t, nil, c.Add(largeDigest, bytes.NewBuffer(largeContent)))
ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent)))
ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent)))
ut.AssertEqual(t, []isolated.HexDigest{emptyDigest, largeDigest}, c.Keys())
c.Evict(emptyDigest)
ut.AssertEqual(t, []isolated.HexDigest{largeDigest}, c.Keys())
ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent)))
ut.AssertEqual(t, nil, c.Add(file1Digest, bytes.NewBuffer(file1Content)))
ut.AssertEqual(t, true, c.Touch(emptyDigest))
ut.AssertEqual(t, nil, c.Add(file2Digest, bytes.NewBuffer(file2Content)))
r, err = c.Read(file1Digest)
ut.AssertEqual(t, nil, r)
ut.AssertEqual(t, true, nil != err)
r, err = c.Read(file2Digest)
ut.AssertEqual(t, nil, err)
actual, err := ioutil.ReadAll(r)
ut.AssertEqual(t, nil, r.Close())
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, file2Content, actual)
expected := []isolated.HexDigest{file2Digest, emptyDigest}
ut.AssertEqual(t, expected, c.Keys())
dest := filepath.Join(td, "foo")
ut.AssertEqual(t, true, nil != c.Hardlink(fakeDigest, dest, os.FileMode(0600)))
ut.AssertEqual(t, true, nil != c.Hardlink(badDigest, dest, os.FileMode(0600)))
ut.AssertEqual(t, nil, c.Hardlink(file2Digest, dest, os.FileMode(0600)))
// See comment about the fact that it may or may not work.
_ = c.Hardlink(file2Digest, dest, os.FileMode(0600))
actual, err = ioutil.ReadFile(dest)
ut.AssertEqual(t, nil, err)
ut.AssertEqual(t, file2Content, actual)
ut.AssertEqual(t, nil, c.Close())
return expected
}