func downloadFile(writer io.Writer, ptr *Pointer, workingfile, mediafile string, cb progress.CopyCallback) error {
fmt.Fprintf(os.Stderr, "Downloading %s (%s)\n", workingfile, pb.FormatBytes(ptr.Size))
xfers := transfer.GetDownloadAdapterNames()
obj, adapterName, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: ptr.Oid, Size: ptr.Size}, "download", xfers)
if err != nil {
return errutil.Errorf(err, "Error downloading %s: %s", filepath.Base(mediafile), err)
}
if ptr.Size == 0 {
ptr.Size = obj.Size
}
adapter := transfer.NewDownloadAdapter(adapterName)
var tcb transfer.TransferProgressCallback
if cb != nil {
tcb = func(name string, totalSize, readSoFar int64, readSinceLast int) error {
return cb(totalSize, readSoFar, readSinceLast)
}
}
// Single download
adapterResultChan := make(chan transfer.TransferResult, 1)
adapter.Begin(1, tcb, adapterResultChan)
adapter.Add(transfer.NewTransfer(filepath.Base(workingfile), obj, mediafile))
adapter.End()
res := <-adapterResultChan
if res.Error != nil {
return errutil.Errorf(err, "Error buffering media file: %s", res.Error)
}
return readLocalFile(writer, ptr, mediafile, workingfile, nil)
}
func TestUploadApiError(t *testing.T) {
SetupTestCredentialsFunc()
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
postCalled := false
mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
postCalled = true
w.WriteHeader(404)
})
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.url": server.URL + "/media",
},
})
oidPath, _ := lfs.LocalMediaPath("988881adc9fc3655077dc2d4d757d480b5ea0e11")
if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
t.Fatal(err)
}
oid := filepath.Base(oidPath)
stat, _ := os.Stat(oidPath)
_, _, err := api.BatchOrLegacySingle(cfg, &api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
if err == nil {
t.Fatal(err)
}
if errors.IsFatalError(err) {
t.Fatal("should not panic")
}
if isDockerConnectionError(err) {
return
}
expected := "LFS: " + fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/media/objects")
if err.Error() != expected {
t.Fatalf("Expected: %s\nGot: %s", expected, err.Error())
}
if !postCalled {
t.Errorf("POST not called")
}
}
func TestDownloadAPIError(t *testing.T) {
SetupTestCredentialsFunc()
defer func() {
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
defer server.Close()
tmp := tempdir(t)
defer os.RemoveAll(tmp)
mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(404)
})
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.batch": "false",
"lfs.url": server.URL + "/media",
},
})
_, _, err := api.BatchOrLegacySingle(cfg, &api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
if err == nil {
t.Fatal("no error?")
}
if errors.IsFatalError(err) {
t.Fatal("should not panic")
}
if isDockerConnectionError(err) {
return
}
expected := fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/media/objects/oid")
if err.Error() != expected {
t.Fatalf("Expected: %s\nGot: %s", expected, err.Error())
}
}
func TestDownloadAPIError(t *testing.T) {
SetupTestCredentialsFunc()
defer func() {
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
defer server.Close()
tmp := tempdir(t)
defer os.RemoveAll(tmp)
mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(404)
})
defer config.Config.ResetConfig()
config.Config.SetConfig("lfs.batch", "false")
config.Config.SetConfig("lfs.url", server.URL+"/media")
_, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
if err == nil {
t.Fatal("no error?")
}
if errutil.IsFatalError(err) {
t.Fatal("should not panic")
}
if isDockerConnectionError(err) {
return
}
if err.Error() != fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/media/objects/oid") {
t.Fatalf("Unexpected error: %s", err.Error())
}
}
// nearly identical to TestSuccessfulDownload
// called multiple times to return different 3xx status codes
func TestSuccessfulDownloadWithRedirects(t *testing.T) {
SetupTestCredentialsFunc()
defer func() {
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
defer server.Close()
tmp := tempdir(t)
defer os.RemoveAll(tmp)
// all of these should work for GET requests
redirectCodes := []int{301, 302, 303, 307}
redirectIndex := 0
mux.HandleFunc("/redirect/objects/oid", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
t.Logf("request header: %v", r.Header)
if r.Method != "GET" {
w.WriteHeader(405)
return
}
w.Header().Set("Location", server.URL+"/redirect2/objects/oid")
w.WriteHeader(redirectCodes[redirectIndex])
t.Logf("redirect with %d", redirectCodes[redirectIndex])
})
mux.HandleFunc("/redirect2/objects/oid", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
t.Logf("request header: %v", r.Header)
if r.Method != "GET" {
w.WriteHeader(405)
return
}
w.Header().Set("Location", server.URL+"/media/objects/oid")
w.WriteHeader(redirectCodes[redirectIndex])
t.Logf("redirect again with %d", redirectCodes[redirectIndex])
redirectIndex += 1
})
mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
t.Logf("request header: %v", r.Header)
if r.Method != "GET" {
w.WriteHeader(405)
return
}
if r.Header.Get("Accept") != api.MediaType {
t.Error("Invalid Accept")
}
if r.Header.Get("Authorization") != expectedAuth(t, server) {
t.Error("Invalid Authorization")
}
obj := &api.ObjectResource{
Oid: "oid",
Size: 4,
Actions: map[string]*api.LinkRelation{
"download": &api.LinkRelation{
Href: server.URL + "/download",
Header: map[string]string{"A": "1"},
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Fatal(err)
}
head := w.Header()
head.Set("Content-Type", api.MediaType)
head.Set("Content-Length", strconv.Itoa(len(by)))
w.WriteHeader(200)
w.Write(by)
})
defer config.Config.ResetConfig()
config.Config.SetConfig("lfs.batch", "false")
config.Config.SetConfig("lfs.url", server.URL+"/redirect")
for _, redirect := range redirectCodes {
obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
if err != nil {
if isDockerConnectionError(err) {
return
}
t.Fatalf("unexpected error for %d status: %s", redirect, err)
}
if obj.Size != 4 {
t.Errorf("unexpected size for %d status: %d", redirect, obj.Size)
}
}
}
// nearly identical to TestSuccessfulDownload
// the api request returns a custom Authorization header
func TestSuccessfulDownloadWithAuthorization(t *testing.T) {
SetupTestCredentialsFunc()
defer func() {
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
defer server.Close()
tmp := tempdir(t)
defer os.RemoveAll(tmp)
mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
t.Logf("request header: %v", r.Header)
if r.Method != "GET" {
w.WriteHeader(405)
return
}
if r.Header.Get("Accept") != api.MediaType {
t.Error("Invalid Accept")
}
if r.Header.Get("Authorization") != expectedAuth(t, server) {
t.Error("Invalid Authorization")
}
obj := &api.ObjectResource{
Oid: "oid",
Size: 4,
Actions: map[string]*api.LinkRelation{
"download": &api.LinkRelation{
Href: server.URL + "/download",
Header: map[string]string{
"A": "1",
"Authorization": "custom",
},
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Fatal(err)
}
head := w.Header()
head.Set("Content-Type", "application/json; charset=utf-8")
head.Set("Content-Length", strconv.Itoa(len(by)))
w.WriteHeader(200)
w.Write(by)
})
defer config.Config.ResetConfig()
config.Config.SetConfig("lfs.batch", "false")
config.Config.SetConfig("lfs.url", server.URL+"/media")
obj, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: "oid"}, "download", []string{"basic"})
if err != nil {
if isDockerConnectionError(err) {
return
}
t.Fatalf("unexpected error: %s", err)
}
if obj.Size != 4 {
t.Errorf("unexpected size: %d", obj.Size)
}
}
func TestUploadVerifyError(t *testing.T) {
SetupTestCredentialsFunc()
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
postCalled := false
verifyCalled := false
mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
if r.Header.Get("Accept") != api.MediaType {
t.Errorf("Invalid Accept")
}
if r.Header.Get("Content-Type") != api.MediaType {
t.Errorf("Invalid Content-Type")
}
buf := &bytes.Buffer{}
tee := io.TeeReader(r.Body, buf)
reqObj := &api.ObjectResource{}
err := json.NewDecoder(tee).Decode(reqObj)
t.Logf("request header: %v", r.Header)
t.Logf("request body: %s", buf.String())
if err != nil {
t.Fatal(err)
}
if reqObj.Oid != "988881adc9fc3655077dc2d4d757d480b5ea0e11" {
t.Errorf("invalid oid from request: %s", reqObj.Oid)
}
if reqObj.Size != 4 {
t.Errorf("invalid size from request: %d", reqObj.Size)
}
obj := &api.ObjectResource{
Oid: reqObj.Oid,
Size: reqObj.Size,
Actions: map[string]*api.LinkRelation{
"upload": &api.LinkRelation{
Href: server.URL + "/upload",
Header: map[string]string{"A": "1"},
},
"verify": &api.LinkRelation{
Href: server.URL + "/verify",
Header: map[string]string{"B": "2"},
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Fatal(err)
}
postCalled = true
head := w.Header()
head.Set("Content-Type", api.MediaType)
head.Set("Content-Length", strconv.Itoa(len(by)))
w.WriteHeader(202)
w.Write(by)
})
mux.HandleFunc("/verify", func(w http.ResponseWriter, r *http.Request) {
verifyCalled = true
w.WriteHeader(404)
})
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.url": server.URL + "/media",
},
})
oidPath, _ := lfs.LocalMediaPath("988881adc9fc3655077dc2d4d757d480b5ea0e11")
if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
t.Fatal(err)
}
oid := filepath.Base(oidPath)
stat, _ := os.Stat(oidPath)
o, _, err := api.BatchOrLegacySingle(cfg, &api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
if err != nil {
if isDockerConnectionError(err) {
return
}
t.Fatal(err)
}
err = api.VerifyUpload(cfg, o)
if err == nil {
t.Fatal("verify should fail")
}
if errors.IsFatalError(err) {
t.Fatal("should not panic")
}
expected := fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/verify")
if err.Error() != expected {
t.Fatalf("Expected: %s\nGot: %s", expected, err.Error())
}
if !postCalled {
t.Errorf("POST not called")
}
if !verifyCalled {
t.Errorf("verify not called")
}
}
func TestUploadWithRedirect(t *testing.T) {
SetupTestCredentialsFunc()
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
mux.HandleFunc("/redirect/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
w.Header().Set("Location", server.URL+"/redirect2/objects")
w.WriteHeader(307)
})
mux.HandleFunc("/redirect2/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
w.Header().Set("Location", server.URL+"/media/objects")
w.WriteHeader(307)
})
mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
if r.Header.Get("Accept") != api.MediaType {
t.Errorf("Invalid Accept")
}
if r.Header.Get("Content-Type") != api.MediaType {
t.Errorf("Invalid Content-Type")
}
buf := &bytes.Buffer{}
tee := io.TeeReader(r.Body, buf)
reqObj := &api.ObjectResource{}
err := json.NewDecoder(tee).Decode(reqObj)
t.Logf("request header: %v", r.Header)
t.Logf("request body: %s", buf.String())
if err != nil {
t.Fatal(err)
}
if reqObj.Oid != "988881adc9fc3655077dc2d4d757d480b5ea0e11" {
t.Errorf("invalid oid from request: %s", reqObj.Oid)
}
if reqObj.Size != 4 {
t.Errorf("invalid size from request: %d", reqObj.Size)
}
obj := &api.ObjectResource{
Actions: map[string]*api.LinkRelation{
"upload": &api.LinkRelation{
Href: server.URL + "/upload",
Header: map[string]string{"A": "1"},
},
"verify": &api.LinkRelation{
Href: server.URL + "/verify",
Header: map[string]string{"B": "2"},
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Fatal(err)
}
head := w.Header()
head.Set("Content-Type", api.MediaType)
head.Set("Content-Length", strconv.Itoa(len(by)))
w.WriteHeader(200)
w.Write(by)
})
defer config.Config.ResetConfig()
config.Config.SetConfig("lfs.url", server.URL+"/redirect")
oidPath, _ := lfs.LocalMediaPath("988881adc9fc3655077dc2d4d757d480b5ea0e11")
if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
t.Fatal(err)
}
oid := filepath.Base(oidPath)
stat, _ := os.Stat(oidPath)
o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
if err != nil {
if isDockerConnectionError(err) {
return
}
t.Fatal(err)
}
if o != nil {
t.Fatal("Received an object")
}
}