func (server *isolatedFake) finalizeGSUpload(r *http.Request) interface{} { data := &isolated.FinalizeRequest{} if err := json.NewDecoder(r.Body).Decode(data); err != nil { server.Fail(err) return map[string]string{"err": err.Error()} } prefix := "ticket:" if !strings.HasPrefix(data.UploadTicket, prefix) { err := fmt.Errorf("unexpected ticket %#v", data.UploadTicket) server.Fail(err) return map[string]string{"err": err.Error()} } digest := isolated.HexDigest(data.UploadTicket[len(prefix):]) if !digest.Validate() { err := fmt.Errorf("invalid digest %#v", digest) server.Fail(err) return map[string]string{"err": err.Error()} } server.lock.Lock() defer server.lock.Unlock() if _, ok := server.staging[digest]; !ok { err := fmt.Errorf("finalizing non uploaded file") server.failLocked(err) return map[string]string{"err": err.Error()} } server.contents[digest] = server.staging[digest] delete(server.staging, digest) return map[string]string{"ok": "true"} }
func (server *isolatedFake) fakeCloudStorage(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if r.Header.Get("Content-Type") != "application/octet-stream" { w.WriteHeader(400) server.Fail(fmt.Errorf("invalid content type: %s", r.Header.Get("Content-Type"))) return } if r.Method != "PUT" { w.WriteHeader(405) server.Fail(fmt.Errorf("invalid method: %s", r.Method)) return } raw, err := ioutil.ReadAll(isolated.GetDecompressor(r.Body)) if err != nil { w.WriteHeader(500) server.Fail(err) return } digest := isolated.HexDigest(r.URL.Query().Get("digest")) if digest != isolated.HashBytes(raw) { w.WriteHeader(400) server.Fail(fmt.Errorf("invalid digest %#v", digest)) return } server.lock.Lock() defer server.lock.Unlock() server.staging[digest] = raw w.WriteHeader(200) }
func TestArchiverFile(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) fEmpty, err := ioutil.TempFile("", "archiver") ut.AssertEqual(t, nil, err) future1 := a.PushFile(fEmpty.Name(), fEmpty.Name(), 0) ut.AssertEqual(t, fEmpty.Name(), future1.DisplayName()) fFoo, err := ioutil.TempFile("", "archiver") ut.AssertEqual(t, nil, err) ut.AssertEqual(t, nil, ioutil.WriteFile(fFoo.Name(), []byte("foo"), 0600)) future2 := a.PushFile(fFoo.Name(), fFoo.Name(), 0) // Push the same file another time. It'll get linked to the first. future3 := a.PushFile(fFoo.Name(), fFoo.Name(), 0) future1.WaitForHashed() future2.WaitForHashed() future3.WaitForHashed() ut.AssertEqual(t, nil, a.Close()) stats := a.Stats() ut.AssertEqual(t, 0, stats.TotalHits()) // Only 2 lookups, not 3. ut.AssertEqual(t, 2, stats.TotalMisses()) ut.AssertEqual(t, common.Size(0), stats.TotalBytesHits()) ut.AssertEqual(t, common.Size(3), stats.TotalBytesPushed()) expected := map[isolated.HexDigest][]byte{ "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33": []byte("foo"), "da39a3ee5e6b4b0d3255bfef95601890afd80709": {}, } ut.AssertEqual(t, expected, server.Contents()) ut.AssertEqual(t, isolated.HexDigest("da39a3ee5e6b4b0d3255bfef95601890afd80709"), future1.Digest()) ut.AssertEqual(t, nil, future1.Error()) ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future2.Digest()) ut.AssertEqual(t, nil, future2.Error()) ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future3.Digest()) ut.AssertEqual(t, nil, future3.Error()) ut.AssertEqual(t, nil, server.Error()) }
func TestArchiverPushSeeked(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) misplaced := bytes.NewReader([]byte("foo")) _, _ = misplaced.Seek(1, os.SEEK_SET) future := a.Push("works", misplaced, 0) future.WaitForHashed() ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest()) ut.AssertEqual(t, nil, a.Close()) }
func TestArchiverFileHit(t *testing.T) { t.Parallel() server := isolatedfake.New() ts := httptest.NewServer(server) defer ts.Close() a := New(isolatedclient.New(ts.URL, "default-gzip"), nil) server.Inject([]byte("foo")) future := a.Push("foo", bytes.NewReader([]byte("foo")), 0) future.WaitForHashed() ut.AssertEqual(t, isolated.HexDigest("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), future.Digest()) ut.AssertEqual(t, nil, a.Close()) stats := a.Stats() ut.AssertEqual(t, 1, stats.TotalHits()) ut.AssertEqual(t, 0, stats.TotalMisses()) ut.AssertEqual(t, common.Size(3), stats.TotalBytesHits()) ut.AssertEqual(t, common.Size(0), stats.TotalBytesPushed()) }
func (e *entry) UnmarshalJSON(data []byte) error { // decode from tuple. var elems []interface{} if err := json.Unmarshal(data, &elems); err != nil { return fmt.Errorf("invalid entry: %s: %s", err, string(data)) } if len(elems) != 2 { return fmt.Errorf("invalid entry: expected 2 items: %s", string(data)) } if key, ok := elems[0].(string); ok { e.key = isolated.HexDigest(key) if value, ok := elems[1].(float64); ok { e.value = common.Size(value) } else { return fmt.Errorf("invalid entry: expected value to be number: %s", string(data)) } } else { return fmt.Errorf("invalid entry: expected key to be string: %s", string(data)) } return nil }
func (server *isolatedFake) storeInline(r *http.Request) interface{} { data := &isolated.StorageRequest{} if err := json.NewDecoder(r.Body).Decode(data); err != nil { server.Fail(err) return map[string]string{"err": err.Error()} } prefix := "ticket:" if !strings.HasPrefix(data.UploadTicket, prefix) { err := fmt.Errorf("unexpected ticket %#v", data.UploadTicket) server.Fail(err) return map[string]string{"err": err.Error()} } digest := isolated.HexDigest(data.UploadTicket[len(prefix):]) if !digest.Validate() { err := fmt.Errorf("invalid digest %#v", digest) server.Fail(err) return map[string]string{"err": err.Error()} } raw, err := ioutil.ReadAll(isolated.GetDecompressor(bytes.NewBuffer(data.Content))) if err != nil { server.Fail(err) return map[string]string{"err": err.Error()} } if digest != isolated.HashBytes(raw) { err := fmt.Errorf("invalid digest %#v", digest) server.Fail(err) return map[string]string{"err": err.Error()} } server.lock.Lock() defer server.lock.Unlock() server.contents[digest] = raw return map[string]string{"ok": "true"} }
func testCache(t *testing.T, c Cache) []isolated.HexDigest { // c's policies must have MaxItems == 2 and MaxSize == 1024. td, err := ioutil.TempDir("", "cache") ut.AssertEqual(t, nil, err) defer func() { if err := os.RemoveAll(td); err != nil { t.Fail() } }() fakeDigest := isolated.HexDigest("0123456789012345678901234567890123456789") badDigest := isolated.HexDigest("012345678901234567890123456789012345678") emptyContent := []byte{} emptyDigest := isolated.HashBytes(emptyContent) file1Content := []byte("foo") file1Digest := isolated.HashBytes(file1Content) file2Content := []byte("foo bar") file2Digest := isolated.HashBytes(file2Content) largeContent := bytes.Repeat([]byte("A"), 1023) largeDigest := isolated.HashBytes(largeContent) tooLargeContent := bytes.Repeat([]byte("A"), 1025) tooLargeDigest := isolated.HashBytes(tooLargeContent) ut.AssertEqual(t, []isolated.HexDigest{}, c.Keys()) ut.AssertEqual(t, false, c.Touch(fakeDigest)) ut.AssertEqual(t, false, c.Touch(badDigest)) c.Evict(fakeDigest) c.Evict(badDigest) r, err := c.Read(fakeDigest) ut.AssertEqual(t, nil, r) ut.AssertEqual(t, true, err != nil) r, err = c.Read(badDigest) ut.AssertEqual(t, nil, r) ut.AssertEqual(t, true, err != nil) // It's too large to fit in the cache. ut.AssertEqual(t, true, nil != c.Add(tooLargeDigest, bytes.NewBuffer(tooLargeContent))) // It gets discarded because it's too large. ut.AssertEqual(t, nil, c.Add(largeDigest, bytes.NewBuffer(largeContent))) ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent))) ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent))) ut.AssertEqual(t, []isolated.HexDigest{emptyDigest, largeDigest}, c.Keys()) c.Evict(emptyDigest) ut.AssertEqual(t, []isolated.HexDigest{largeDigest}, c.Keys()) ut.AssertEqual(t, nil, c.Add(emptyDigest, bytes.NewBuffer(emptyContent))) ut.AssertEqual(t, nil, c.Add(file1Digest, bytes.NewBuffer(file1Content))) ut.AssertEqual(t, true, c.Touch(emptyDigest)) ut.AssertEqual(t, nil, c.Add(file2Digest, bytes.NewBuffer(file2Content))) r, err = c.Read(file1Digest) ut.AssertEqual(t, nil, r) ut.AssertEqual(t, true, nil != err) r, err = c.Read(file2Digest) ut.AssertEqual(t, nil, err) actual, err := ioutil.ReadAll(r) ut.AssertEqual(t, nil, r.Close()) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, file2Content, actual) expected := []isolated.HexDigest{file2Digest, emptyDigest} ut.AssertEqual(t, expected, c.Keys()) dest := filepath.Join(td, "foo") ut.AssertEqual(t, true, nil != c.Hardlink(fakeDigest, dest, os.FileMode(0600))) ut.AssertEqual(t, true, nil != c.Hardlink(badDigest, dest, os.FileMode(0600))) ut.AssertEqual(t, nil, c.Hardlink(file2Digest, dest, os.FileMode(0600))) // See comment about the fact that it may or may not work. _ = c.Hardlink(file2Digest, dest, os.FileMode(0600)) actual, err = ioutil.ReadFile(dest) ut.AssertEqual(t, nil, err) ut.AssertEqual(t, file2Content, actual) ut.AssertEqual(t, nil, c.Close()) return expected }