func pruneDeleteFiles(prunableObjects []string) {
spinner := lfs.NewSpinner()
var problems bytes.Buffer
// In case we fail to delete some
var deletedFiles int
for i, oid := range prunableObjects {
spinner.Print(OutputWriter, fmt.Sprintf("Deleting object %d/%d", i, len(prunableObjects)))
mediaFile, err := lfs.LocalMediaPath(oid)
if err != nil {
problems.WriteString(fmt.Sprintf("Unable to find media path for %v: %v\n", oid, err))
continue
}
err = os.Remove(mediaFile)
if err != nil {
problems.WriteString(fmt.Sprintf("Failed to remove file %v: %v\n", mediaFile, err))
continue
}
deletedFiles++
}
spinner.Finish(OutputWriter, fmt.Sprintf("Deleted %d files", deletedFiles))
if problems.Len() > 0 {
LoggedError(fmt.Errorf("Failed to delete some files"), problems.String())
Exit("Prune failed, see errors above")
}
}
func smudgeCommand(cmd *cobra.Command, args []string) {
requireStdin("This command should be run by the Git 'smudge' filter")
lfs.InstallHooks(false)
// keeps the initial buffer from lfs.DecodePointer
b := &bytes.Buffer{}
r := io.TeeReader(os.Stdin, b)
ptr, err := lfs.DecodePointer(r)
if err != nil {
mr := io.MultiReader(b, os.Stdin)
_, err := io.Copy(os.Stdout, mr)
if err != nil {
Panic(err, "Error writing data to stdout:")
}
return
}
if smudgeInfo {
localPath, err := lfs.LocalMediaPath(ptr.Oid)
if err != nil {
Exit(err.Error())
}
stat, err := os.Stat(localPath)
if err != nil {
Print("%d --", ptr.Size)
} else {
Print("%d %s", stat.Size(), localPath)
}
return
}
filename := smudgeFilename(args, err)
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
if err != nil {
Error(err.Error())
}
cfg := lfs.Config
download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
if smudgeSkip || lfs.Config.GetenvBool("GIT_LFS_SKIP_SMUDGE", false) {
download = false
}
err = ptr.Smudge(os.Stdout, filename, download, cb)
if file != nil {
file.Close()
}
if err != nil {
ptr.Encode(os.Stdout)
// Download declined error is ok to skip if we weren't requesting download
if !(lfs.IsDownloadDeclinedError(err) && !download) {
LoggedError(err, "Error accessing media: %s (%s)", filename, ptr.Oid)
os.Exit(2)
}
}
}
func TestUploadApiError(t *testing.T) {
SetupTestCredentialsFunc()
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
postCalled := false
mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
postCalled = true
w.WriteHeader(404)
})
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.url": server.URL + "/media",
},
})
oidPath, _ := lfs.LocalMediaPath("988881adc9fc3655077dc2d4d757d480b5ea0e11")
if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
t.Fatal(err)
}
oid := filepath.Base(oidPath)
stat, _ := os.Stat(oidPath)
_, _, err := api.BatchOrLegacySingle(cfg, &api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
if err == nil {
t.Fatal(err)
}
if errors.IsFatalError(err) {
t.Fatal("should not panic")
}
if isDockerConnectionError(err) {
return
}
expected := "LFS: " + fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/media/objects")
if err.Error() != expected {
t.Fatalf("Expected: %s\nGot: %s", expected, err.Error())
}
if !postCalled {
t.Errorf("POST not called")
}
}
func smudgeCommand(cmd *cobra.Command, args []string) {
requireStdin("This command should be run by the Git 'smudge' filter")
lfs.InstallHooks(false)
b := &bytes.Buffer{}
r := io.TeeReader(os.Stdin, b)
ptr, err := lfs.DecodePointer(r)
if err != nil {
mr := io.MultiReader(b, os.Stdin)
_, err := io.Copy(os.Stdout, mr)
if err != nil {
Panic(err, "Error writing data to stdout:")
}
return
}
if smudgeInfo {
localPath, err := lfs.LocalMediaPath(ptr.Oid)
if err != nil {
Exit(err.Error())
}
stat, err := os.Stat(localPath)
if err != nil {
Print("%d --", ptr.Size)
} else {
Print("%d %s", stat.Size(), localPath)
}
return
}
filename := smudgeFilename(args, err)
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
if err != nil {
Error(err.Error())
}
cfg := lfs.Config
download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
err = ptr.Smudge(os.Stdout, filename, download, cb)
if file != nil {
file.Close()
}
if err != nil {
ptr.Encode(os.Stdout)
LoggedError(err, "Error accessing media: %s (%s)", filename, ptr.Oid)
}
}
func cleanCommand(cmd *cobra.Command, args []string) {
requireStdin("This command should be run by the Git 'clean' filter")
lfs.InstallHooks(false)
var fileName string
var cb progress.CopyCallback
var file *os.File
var fileSize int64
if len(args) > 0 {
fileName = args[0]
stat, err := os.Stat(fileName)
if err == nil && stat != nil {
fileSize = stat.Size()
localCb, localFile, err := lfs.CopyCallbackFile("clean", fileName, 1, 1)
if err != nil {
Error(err.Error())
} else {
cb = localCb
file = localFile
}
}
}
cleaned, err := lfs.PointerClean(os.Stdin, fileName, fileSize, cb)
if file != nil {
file.Close()
}
if cleaned != nil {
defer cleaned.Teardown()
}
if errors.IsCleanPointerError(err) {
os.Stdout.Write(errors.GetContext(err, "bytes").([]byte))
return
}
if err != nil {
Panic(err, "Error cleaning asset.")
}
tmpfile := cleaned.Filename
mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
if err != nil {
Panic(err, "Unable to get local media path.")
}
if stat, _ := os.Stat(mediafile); stat != nil {
if stat.Size() != cleaned.Size && len(cleaned.Pointer.Extensions) == 0 {
Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
}
Debug("%s exists", mediafile)
} else {
if err := os.Rename(tmpfile, mediafile); err != nil {
Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
}
Debug("Writing %s", mediafile)
}
lfs.EncodePointer(os.Stdout, cleaned.Pointer)
}
func (repo *Repo) AddCommits(inputs []*CommitInput) []*CommitOutput {
if repo.Settings.RepoType == RepoTypeBare {
repo.callback.Fatalf("Cannot use AddCommits on a bare repo; clone it & push changes instead")
}
// Change to repo working dir
oldwd, err := os.Getwd()
if err != nil {
repo.callback.Fatalf("Can't get cwd %v", err)
}
err = os.Chdir(repo.Path)
if err != nil {
repo.callback.Fatalf("Can't chdir to repo %v", err)
}
// Used to check whether we need to checkout another commit before
lastBranch := "master"
outputs := make([]*CommitOutput, 0, len(inputs))
for i, input := range inputs {
output := &CommitOutput{}
// first, are we on the correct branch
if len(input.ParentBranches) > 0 {
if input.ParentBranches[0] != lastBranch {
RunGitCommand(repo.callback, true, "checkout", input.ParentBranches[0])
lastBranch = input.ParentBranches[0]
}
}
// Is this a merge?
if len(input.ParentBranches) > 1 {
// Always take the *other* side in a merge so we adopt changes
// also don't automatically commit, we'll do that below
args := []string{"merge", "--no-ff", "--no-commit", "--strategy-option=theirs"}
args = append(args, input.ParentBranches[1:]...)
RunGitCommand(repo.callback, false, args...)
} else if input.NewBranch != "" {
RunGitCommand(repo.callback, true, "checkout", "-b", input.NewBranch)
lastBranch = input.NewBranch
}
// Any files to write?
for fi, infile := range input.Files {
inputData := infile.DataReader
if inputData == nil && infile.Data != "" {
inputData = strings.NewReader(infile.Data)
}
if inputData == nil {
// Different data for each file but deterministic
inputData = NewPlaceholderDataReader(int64(i*fi), infile.Size)
}
cleaned, err := lfs.PointerClean(inputData, infile.Filename, infile.Size, nil)
if err != nil {
repo.callback.Errorf("Error creating pointer file: %v", err)
continue
}
// this only created the temp file, move to final location
tmpfile := cleaned.Filename
mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
if err != nil {
repo.callback.Errorf("Unable to get local media path: %v", err)
continue
}
if _, err := os.Stat(mediafile); err != nil {
if err := os.Rename(tmpfile, mediafile); err != nil {
repo.callback.Errorf("Unable to move %s to %s: %v", tmpfile, mediafile, err)
continue
}
}
output.Files = append(output.Files, cleaned.Pointer)
// Write pointer to local filename for adding (not using clean filter)
os.MkdirAll(filepath.Dir(infile.Filename), 0755)
f, err := os.Create(infile.Filename)
if err != nil {
repo.callback.Errorf("Error creating pointer file: %v", err)
continue
}
_, err = cleaned.Pointer.Encode(f)
if err != nil {
f.Close()
repo.callback.Errorf("Error encoding pointer file: %v", err)
continue
}
f.Close() // early close in a loop, don't defer
RunGitCommand(repo.callback, true, "add", infile.Filename)
}
// Now commit
err = commitAtDate(input.CommitDate, input.CommitterName, input.CommitterEmail,
fmt.Sprintf("Test commit %d", i))
if err != nil {
repo.callback.Fatalf("Error committing: %v", err)
}
commit, err := git.GetCommitSummary("HEAD")
if err != nil {
repo.callback.Fatalf("Error determining commit SHA: %v", err)
}
// tags
for _, tag := range input.Tags {
// Use annotated tags, assume full release tags (also tag objects have edge cases)
RunGitCommand(repo.callback, true, "tag", "-a", "-m", "Added tag", tag)
}
output.Sha = commit.Sha
output.Parents = commit.Parents
outputs = append(outputs, output)
}
// Restore cwd
err = os.Chdir(oldwd)
if err != nil {
repo.callback.Fatalf("Can't restore old cwd %v", err)
}
return outputs
}
func TestUploadVerifyError(t *testing.T) {
SetupTestCredentialsFunc()
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
postCalled := false
verifyCalled := false
mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
if r.Header.Get("Accept") != api.MediaType {
t.Errorf("Invalid Accept")
}
if r.Header.Get("Content-Type") != api.MediaType {
t.Errorf("Invalid Content-Type")
}
buf := &bytes.Buffer{}
tee := io.TeeReader(r.Body, buf)
reqObj := &api.ObjectResource{}
err := json.NewDecoder(tee).Decode(reqObj)
t.Logf("request header: %v", r.Header)
t.Logf("request body: %s", buf.String())
if err != nil {
t.Fatal(err)
}
if reqObj.Oid != "988881adc9fc3655077dc2d4d757d480b5ea0e11" {
t.Errorf("invalid oid from request: %s", reqObj.Oid)
}
if reqObj.Size != 4 {
t.Errorf("invalid size from request: %d", reqObj.Size)
}
obj := &api.ObjectResource{
Oid: reqObj.Oid,
Size: reqObj.Size,
Actions: map[string]*api.LinkRelation{
"upload": &api.LinkRelation{
Href: server.URL + "/upload",
Header: map[string]string{"A": "1"},
},
"verify": &api.LinkRelation{
Href: server.URL + "/verify",
Header: map[string]string{"B": "2"},
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Fatal(err)
}
postCalled = true
head := w.Header()
head.Set("Content-Type", api.MediaType)
head.Set("Content-Length", strconv.Itoa(len(by)))
w.WriteHeader(202)
w.Write(by)
})
mux.HandleFunc("/verify", func(w http.ResponseWriter, r *http.Request) {
verifyCalled = true
w.WriteHeader(404)
})
cfg := config.NewFrom(config.Values{
Git: map[string]string{
"lfs.url": server.URL + "/media",
},
})
oidPath, _ := lfs.LocalMediaPath("988881adc9fc3655077dc2d4d757d480b5ea0e11")
if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
t.Fatal(err)
}
oid := filepath.Base(oidPath)
stat, _ := os.Stat(oidPath)
o, _, err := api.BatchOrLegacySingle(cfg, &api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
if err != nil {
if isDockerConnectionError(err) {
return
}
t.Fatal(err)
}
err = api.VerifyUpload(cfg, o)
if err == nil {
t.Fatal("verify should fail")
}
if errors.IsFatalError(err) {
t.Fatal("should not panic")
}
expected := fmt.Sprintf(httputil.GetDefaultError(404), server.URL+"/verify")
if err.Error() != expected {
t.Fatalf("Expected: %s\nGot: %s", expected, err.Error())
}
if !postCalled {
t.Errorf("POST not called")
}
if !verifyCalled {
t.Errorf("verify not called")
}
}
func TestUploadWithRedirect(t *testing.T) {
SetupTestCredentialsFunc()
repo := test.NewRepo(t)
repo.Pushd()
defer func() {
repo.Popd()
repo.Cleanup()
RestoreCredentialsFunc()
}()
mux := http.NewServeMux()
server := httptest.NewServer(mux)
tmp := tempdir(t)
defer server.Close()
defer os.RemoveAll(tmp)
mux.HandleFunc("/redirect/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
w.Header().Set("Location", server.URL+"/redirect2/objects")
w.WriteHeader(307)
})
mux.HandleFunc("/redirect2/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
w.Header().Set("Location", server.URL+"/media/objects")
w.WriteHeader(307)
})
mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
t.Logf("Server: %s %s", r.Method, r.URL)
if r.Method != "POST" {
w.WriteHeader(405)
return
}
if r.Header.Get("Accept") != api.MediaType {
t.Errorf("Invalid Accept")
}
if r.Header.Get("Content-Type") != api.MediaType {
t.Errorf("Invalid Content-Type")
}
buf := &bytes.Buffer{}
tee := io.TeeReader(r.Body, buf)
reqObj := &api.ObjectResource{}
err := json.NewDecoder(tee).Decode(reqObj)
t.Logf("request header: %v", r.Header)
t.Logf("request body: %s", buf.String())
if err != nil {
t.Fatal(err)
}
if reqObj.Oid != "988881adc9fc3655077dc2d4d757d480b5ea0e11" {
t.Errorf("invalid oid from request: %s", reqObj.Oid)
}
if reqObj.Size != 4 {
t.Errorf("invalid size from request: %d", reqObj.Size)
}
obj := &api.ObjectResource{
Actions: map[string]*api.LinkRelation{
"upload": &api.LinkRelation{
Href: server.URL + "/upload",
Header: map[string]string{"A": "1"},
},
"verify": &api.LinkRelation{
Href: server.URL + "/verify",
Header: map[string]string{"B": "2"},
},
},
}
by, err := json.Marshal(obj)
if err != nil {
t.Fatal(err)
}
head := w.Header()
head.Set("Content-Type", api.MediaType)
head.Set("Content-Length", strconv.Itoa(len(by)))
w.WriteHeader(200)
w.Write(by)
})
defer config.Config.ResetConfig()
config.Config.SetConfig("lfs.url", server.URL+"/redirect")
oidPath, _ := lfs.LocalMediaPath("988881adc9fc3655077dc2d4d757d480b5ea0e11")
if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
t.Fatal(err)
}
oid := filepath.Base(oidPath)
stat, _ := os.Stat(oidPath)
o, _, err := api.BatchOrLegacySingle(&api.ObjectResource{Oid: oid, Size: stat.Size()}, "upload", []string{"basic"})
if err != nil {
if isDockerConnectionError(err) {
return
}
t.Fatal(err)
}
if o != nil {
t.Fatal("Received an object")
}
}