func shouldDeleteTempObject(path string) bool {
info, err := os.Stat(path)
if err != nil {
return false
}
if info.IsDir() {
return false
}
base := filepath.Base(path)
parts := strings.SplitN(base, "-", 2)
oid := parts[0]
if len(parts) < 2 || len(oid) != 64 {
tracerx.Printf("Removing invalid tmp object file: %s", path)
return true
}
if FileExists(localMediaPathNoCreate(oid)) {
tracerx.Printf("Removing existing tmp object file: %s", path)
return true
}
if time.Since(info.ModTime()) > time.Hour {
tracerx.Printf("Removing old tmp object file: %s", path)
return true
}
return false
}
func Batch(objects []*ObjectResource, operation string) ([]*ObjectResource, error) {
if len(objects) == 0 {
return nil, nil
}
o := map[string]interface{}{"objects": objects, "operation": operation}
by, err := json.Marshal(o)
if err != nil {
return nil, Error(err)
}
req, err := newBatchApiRequest(operation)
if err != nil {
return nil, Error(err)
}
req.Header.Set("Content-Type", mediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = &byteCloser{bytes.NewReader(by)}
tracerx.Printf("api: batch %d files", len(objects))
res, objs, err := doApiBatchRequest(req)
if err != nil {
if res == nil {
return nil, newRetriableError(err)
}
if res.StatusCode == 0 {
return nil, newRetriableError(err)
}
if IsAuthError(err) {
setAuthType(req, res)
return Batch(objects, operation)
}
switch res.StatusCode {
case 404, 410:
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
return nil, newNotImplementedError(nil)
}
tracerx.Printf("api error: %s", err)
return nil, Error(err)
}
LogTransfer("lfs.api.batch", res)
if res.StatusCode != 200 {
return nil, Error(fmt.Errorf("Invalid status for %s: %d", traceHttpReq(req), res.StatusCode))
}
return objs, nil
}
func UploadCheck(oidPath string) (*objectResource, error) {
oid := filepath.Base(oidPath)
stat, err := os.Stat(oidPath)
if err != nil {
return nil, Error(err)
}
reqObj := &objectResource{
Oid: oid,
Size: stat.Size(),
}
by, err := json.Marshal(reqObj)
if err != nil {
return nil, Error(err)
}
req, err := newApiRequest("POST", oid)
if err != nil {
return nil, Error(err)
}
req.Header.Set("Content-Type", mediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = &byteCloser{bytes.NewReader(by)}
tracerx.Printf("api: uploading (%s)", oid)
res, obj, err := doLegacyApiRequest(req)
if err != nil {
if IsAuthError(err) {
Config.SetAccess("basic")
tracerx.Printf("api: upload check not authorized, submitting with auth")
return UploadCheck(oidPath)
}
return nil, newRetriableError(err)
}
LogTransfer("lfs.api.upload", res)
if res.StatusCode == 200 {
return nil, nil
}
if obj.Oid == "" {
obj.Oid = oid
}
if obj.Size == 0 {
obj.Size = reqObj.Size
}
return obj, nil
}
// RecentBranches returns branches with commit dates on or after the given date/time
// Return full Ref type for easier detection of duplicate SHAs etc
// since: refs with commits on or after this date will be included
// includeRemoteBranches: true to include refs on remote branches
// onlyRemote: set to non-blank to only include remote branches on a single remote
func RecentBranches(since time.Time, includeRemoteBranches bool, onlyRemote string) ([]*Ref, error) {
cmd := subprocess.ExecCommand("git", "for-each-ref",
`--sort=-committerdate`,
`--format=%(refname) %(objectname) %(committerdate:iso)`,
"refs")
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git for-each-ref: %v", err)
}
cmd.Start()
defer cmd.Wait()
scanner := bufio.NewScanner(outp)
// Output is like this:
// refs/heads/master f03686b324b29ff480591745dbfbbfa5e5ac1bd5 2015-08-19 16:50:37 +0100
// refs/remotes/origin/master ad3b29b773e46ad6870fdf08796c33d97190fe93 2015-08-13 16:50:37 +0100
// Output is ordered by latest commit date first, so we can stop at the threshold
regex := regexp.MustCompile(`^(refs/[^/]+/\S+)\s+([0-9A-Za-z]{40})\s+(\d{4}-\d{2}-\d{2}\s+\d{2}\:\d{2}\:\d{2}\s+[\+\-]\d{4})`)
tracerx.Printf("RECENT: Getting refs >= %v", since)
var ret []*Ref
for scanner.Scan() {
line := scanner.Text()
if match := regex.FindStringSubmatch(line); match != nil {
fullref := match[1]
sha := match[2]
reftype, ref := ParseRefToTypeAndName(fullref)
if reftype == RefTypeRemoteBranch || reftype == RefTypeRemoteTag {
if !includeRemoteBranches {
continue
}
if onlyRemote != "" && !strings.HasPrefix(ref, onlyRemote+"/") {
continue
}
}
// This is a ref we might use
// Check the date
commitDate, err := ParseGitDate(match[3])
if err != nil {
return ret, err
}
if commitDate.Before(since) {
// the end
break
}
tracerx.Printf("RECENT: %v (%v)", ref, commitDate)
ret = append(ret, &Ref{ref, reftype, sha})
}
}
return ret, nil
}
func Batch(objects []*objectResource, operation string) ([]*objectResource, error) {
if len(objects) == 0 {
return nil, nil
}
o := map[string]interface{}{"objects": objects, "operation": operation}
by, err := json.Marshal(o)
if err != nil {
return nil, Error(err)
}
req, err := newBatchApiRequest(operation)
if err != nil {
return nil, Error(err)
}
req.Header.Set("Content-Type", mediaType)
req.Header.Set("Content-Length", strconv.Itoa(len(by)))
req.ContentLength = int64(len(by))
req.Body = &byteCloser{bytes.NewReader(by)}
tracerx.Printf("api: batch %d files", len(objects))
res, objs, err := doApiBatchRequest(req)
if err != nil {
if res == nil {
return nil, err
}
switch res.StatusCode {
case 401:
Config.SetAccess("basic")
tracerx.Printf("api: batch not authorized, submitting with auth")
return Batch(objects, operation)
case 404, 410:
tracerx.Printf("api: batch not implemented: %d", res.StatusCode)
return nil, newNotImplementedError(nil)
}
tracerx.Printf("api error: %s", err)
}
LogTransfer("lfs.api.batch", res)
if res.StatusCode != 200 {
return nil, Error(fmt.Errorf("Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode))
}
return objs, nil
}
// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
if q.batcher != nil {
q.batcher.Exit()
}
q.wait.Wait()
// Handle any retries
close(q.retriesc)
q.retrywait.Wait()
atomic.StoreUint32(&q.retrying, 1)
if len(q.retries) > 0 && q.batcher != nil {
tracerx.Printf("tq: retrying %d failed transfers", len(q.retries))
for _, t := range q.retries {
q.Add(t)
}
q.batcher.Exit()
q.wait.Wait()
}
atomic.StoreUint32(&q.retrying, 0)
close(q.apic)
close(q.transferc)
close(q.errorc)
for _, watcher := range q.watchers {
close(watcher)
}
q.meter.Finish()
q.errorwait.Wait()
}
func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s [%s]", pointer.Name, pointer.Oid)
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s", pointer.Name)
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, wErr := lfs.NewUploadable(pointer.Oid, pointer.Name)
if wErr != nil {
if Debugging || wErr.Panic {
Panic(wErr.Err, wErr.Error())
} else {
Exit(wErr.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func fillCredentials(req *http.Request, u *url.URL) (Creds, error) {
path := strings.TrimPrefix(u.Path, "/")
input := Creds{"protocol": u.Scheme, "host": u.Host, "path": path}
if u.User != nil && u.User.Username() != "" {
input["username"] = u.User.Username()
}
creds, err := execCreds(input, "fill")
if creds == nil || len(creds) < 1 {
errmsg := fmt.Sprintf("Git credentials for %s not found", u)
if err != nil {
errmsg = errmsg + ":\n" + err.Error()
} else {
errmsg = errmsg + "."
}
err = errors.New(errmsg)
}
if err != nil {
return nil, err
}
tracerx.Printf("Filled credentials for %s", u)
setRequestAuth(req, creds["username"], creds["password"])
return creds, err
}
func (c *countingReadCloser) Read(b []byte) (int, error) {
n, err := c.ReadCloser.Read(b)
if err != nil && err != io.EOF {
return n, err
}
c.Count += n
if c.isTraceableType {
chunk := string(b[0:n])
if c.useGitTrace {
tracerx.Printf("HTTP: %s", chunk)
}
if Config.isTracingHttp {
fmt.Fprint(os.Stderr, chunk)
}
}
if err == io.EOF && Config.isLoggingStats {
// This transfer is done, we're checking it this way so we can also
// catch transfers where the caller forgets to Close() the Body.
if c.response != nil {
transfersLock.Lock()
if transfer, ok := transfers[c.response]; ok {
transfer.responseStats.BodySize = c.Count
transfer.responseStats.Stop = time.Now()
}
transfersLock.Unlock()
}
}
return n, err
}
// Refs returns all of the local and remote branches and tags for the current
// repository. Other refs (HEAD, refs/stash, git notes) are ignored.
func LocalRefs() ([]*Ref, error) {
cmd := subprocess.ExecCommand("git", "show-ref", "--heads", "--tags")
outp, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("Failed to call git show-ref: %v", err)
}
cmd.Start()
var refs []*Ref
scanner := bufio.NewScanner(outp)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
parts := strings.SplitN(line, " ", 2)
if len(parts) != 2 || len(parts[0]) != 40 || len(parts[1]) < 1 {
tracerx.Printf("Invalid line from git show-ref: %q", line)
continue
}
rtype, name := ParseRefToTypeAndName(parts[1])
if rtype != RefTypeLocalBranch && rtype != RefTypeLocalTag {
continue
}
refs = append(refs, &Ref{name, rtype, parts[0]})
}
return refs, cmd.Wait()
}
// startCommand starts up a command and creates a stdin pipe and a buffered
// stdout & stderr pipes, wrapped in a wrappedCmd. The stdout buffer will be of stdoutBufSize
// bytes.
func startCommand(command string, args ...string) (*wrappedCmd, error) {
cmd := exec.Command(command, args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
stdin, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
tracerx.Printf("run_command: %s %s", command, strings.Join(args, " "))
if err := cmd.Start(); err != nil {
return nil, err
}
return &wrappedCmd{
stdin,
bufio.NewReaderSize(stdout, stdoutBufSize),
bufio.NewReaderSize(stderr, stdoutBufSize),
cmd,
}, nil
}
func scanStorageDir(dir string, c chan *Pointer) {
// ioutil.ReadDir and filepath.Walk do sorting which is unnecessary & inefficient
dirf, err := os.Open(dir)
if err != nil {
return
}
defer dirf.Close()
direntries, err := dirf.Readdir(0)
if err != nil {
tracerx.Printf("Problem with Readdir in %v: %v", dir, err)
return
}
for _, dirfi := range direntries {
if dirfi.IsDir() {
subpath := filepath.Join(dir, dirfi.Name())
scanStorageDir(subpath, c)
} else {
// Make sure it's really an object file & not .DS_Store etc
if oidRE.MatchString(dirfi.Name()) {
c <- NewPointer(dirfi.Name(), dirfi.Size(), nil)
}
}
}
}
func ResolveDirs() {
var err error
LocalGitDir, LocalWorkingDir, err = git.GitAndRootDirs()
if err == nil {
LocalGitStorageDir = resolveGitStorageDir(LocalGitDir)
LocalMediaDir = filepath.Join(LocalGitStorageDir, "lfs", "objects")
LocalLogDir = filepath.Join(LocalMediaDir, "logs")
TempDir = filepath.Join(LocalGitDir, "lfs", "tmp") // temp files per worktree
if err := os.MkdirAll(LocalMediaDir, localMediaDirPerms); err != nil {
panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
}
if err := os.MkdirAll(LocalLogDir, localLogDirPerms); err != nil {
panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
}
LocalObjectTempDir = filepath.Join(TempDir, "objects")
if err := os.MkdirAll(LocalObjectTempDir, tempDirPerms); err != nil {
panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
}
} else {
errMsg := err.Error()
tracerx.Printf("Error running 'git rev-parse': %s", errMsg)
if !strings.Contains(errMsg, "Not a git repository") {
fmt.Fprintf(os.Stderr, "Error: %s\n", errMsg)
}
}
}
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, cb CopyCallback) error {
mediafile, err := LocalMediaPath(ptr.Oid)
if err != nil {
return err
}
stat, statErr := os.Stat(mediafile)
if statErr == nil && stat != nil {
fileSize := stat.Size()
if fileSize == 0 || fileSize != ptr.Size {
tracerx.Printf("Removing %s, size %d is invalid", mediafile, fileSize)
os.RemoveAll(mediafile)
stat = nil
}
}
var wErr *WrappedError
if statErr != nil || stat == nil {
wErr = downloadFile(writer, ptr, workingfile, mediafile, cb)
} else {
wErr = readLocalFile(writer, ptr, mediafile, cb)
}
if wErr != nil {
return &SmudgeError{ptr.Oid, mediafile, wErr}
}
return nil
}
func traceHttpResponse(c *Configuration, res *http.Response) {
if res == nil {
return
}
tracerx.Printf("HTTP: %d", res.StatusCode)
if c.isTracingHttp == false {
return
}
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "< %s %s\n", res.Proto, res.Status)
for key, _ := range res.Header {
fmt.Fprintf(os.Stderr, "< %s: %s\n", key, res.Header.Get(key))
}
traceBody := false
ctype := strings.ToLower(strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0])
for _, tracedType := range tracedTypes {
if strings.Contains(ctype, tracedType) {
traceBody = true
}
}
res.Body = newCountedResponse(res)
if traceBody {
res.Body = newTracedBody(res.Body)
}
fmt.Fprintf(os.Stderr, "\n")
}
func (q *TransferQueue) transferWorker() {
for transfer := range q.transferc {
cb := func(total, read int64, current int) error {
q.meter.TransferBytes(q.transferKind, transfer.Name(), read, total, current)
return nil
}
if err := transfer.Transfer(cb); err != nil {
if q.canRetry(err) {
tracerx.Printf("tq: retrying object %s", transfer.Oid())
q.retry(transfer)
} else {
q.errorc <- err
}
} else {
oid := transfer.Oid()
for _, c := range q.watchers {
c <- oid
}
}
q.meter.FinishTransfer(transfer.Name())
q.wait.Done()
}
}
func newBatchApiRequest(operation string) (*http.Request, error) {
endpoint := Config.Endpoint()
res, err := sshAuthenticate(endpoint, operation, "")
if err != nil {
tracerx.Printf("ssh: %s attempted with %s. Error: %s",
operation, endpoint.SshUserAndHost, err.Error(),
)
}
if len(res.Href) > 0 {
endpoint.Url = res.Href
}
u, err := ObjectUrl(endpoint, "batch")
if err != nil {
return nil, err
}
req, err := newBatchClientRequest("POST", u.String())
if err != nil {
return nil, err
}
req.Header.Set("Accept", mediaType)
if res.Header != nil {
for key, value := range res.Header {
req.Header.Set(key, value)
}
}
return req, nil
}
func scanObjects(dir string, ch chan<- Object) {
dirf, err := os.Open(dir)
if err != nil {
return
}
defer dirf.Close()
direntries, err := dirf.Readdir(0)
if err != nil {
tracerx.Printf("Problem with Readdir in %q: %s", dir, err)
return
}
for _, dirfi := range direntries {
if dirfi.IsDir() {
subpath := filepath.Join(dir, dirfi.Name())
scanObjects(subpath, ch)
} else {
// Make sure it's really an object file & not .DS_Store etc
if oidRE.MatchString(dirfi.Name()) {
ch <- Object{dirfi.Name(), dirfi.Size()}
}
}
}
}
func newApiRequest(method, oid string) (*http.Request, error) {
endpoint := Config.Endpoint()
objectOid := oid
operation := "download"
if method == "POST" {
if oid != "batch" {
objectOid = ""
operation = "upload"
}
}
res, err := sshAuthenticate(endpoint, operation, oid)
if err != nil {
tracerx.Printf("ssh: attempted with %s. Error: %s",
endpoint.SshUserAndHost, err.Error(),
)
}
if len(res.Href) > 0 {
endpoint.Url = res.Href
}
u, err := ObjectUrl(endpoint, objectOid)
if err != nil {
return nil, err
}
req, err := newClientRequest(method, u.String(), res.Header)
if err != nil {
return nil, err
}
req.Header.Set("Accept", mediaType)
return req, nil
}
func sshAuthenticate(endpoint Endpoint, operation, oid string) (sshAuthResponse, error) {
// This is only used as a fallback where the Git URL is SSH but server doesn't support a full SSH binary protocol
// and therefore we derive a HTTPS endpoint for binaries instead; but check authentication here via SSH
res := sshAuthResponse{}
if len(endpoint.SshUserAndHost) == 0 {
return res, nil
}
tracerx.Printf("ssh: %s git-lfs-authenticate %s %s %s",
endpoint.SshUserAndHost, endpoint.SshPath, operation, oid)
exe, args := sshGetExeAndArgs(endpoint)
args = append(args,
"git-lfs-authenticate",
endpoint.SshPath,
operation, oid)
cmd := exec.Command(exe, args...)
out, err := cmd.CombinedOutput()
if err != nil {
res.Message = string(out)
} else {
err = json.Unmarshal(out, &res)
}
return res, err
}
func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
skipObjects := prePushCheckForMissingObjects(pointers)
uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}
if _, skip := skipObjects[pointer.Oid]; skip {
// object missing locally but on server, don't bother
continue
}
tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploadQueue.Add(u)
}
return uploadQueue
}
func uploadsBetweenRefAndRemote(ctx *uploadContext, refnames []string) {
tracerx.Printf("Upload refs %v to remote %v", refnames, lfs.Config.CurrentRemote)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote
if pushAll {
scanOpt.ScanMode = lfs.ScanRefsMode
}
refs, err := refsByNames(refnames)
if err != nil {
Error(err.Error())
Exit("Error getting local refs.")
}
for _, ref := range refs {
pointers, err := lfs.ScanRefs(ref.Name, "", scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files in the %q ref", ref.Name)
}
upload(ctx, pointers)
}
}
func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
uploads := []*lfs.Uploadable{}
totalSize := int64(0)
for i, oid := range oids {
if pushDryRun {
Print("push object ID %s", oid)
continue
}
tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))
u, err := lfs.NewUploadable(oid, "")
if err != nil {
if Debugging || lfs.IsFatalError(err) {
Panic(err, err.Error())
} else {
Exit(err.Error())
}
}
uploads = append(uploads, u)
}
uploadQueue := lfs.NewUploadQueue(len(oids), totalSize, pushDryRun)
for _, u := range uploads {
uploadQueue.Add(u)
}
return uploadQueue
}
// PointerSmudgeObject uses a Pointer and ObjectResource to download the object to the
// media directory. It does not write the file to the working directory.
func PointerSmudgeObject(ptr *Pointer, obj *ObjectResource, cb CopyCallback) error {
mediafile, err := LocalMediaPath(obj.Oid)
if err != nil {
return err
}
stat, statErr := os.Stat(mediafile)
if statErr == nil && stat != nil {
fileSize := stat.Size()
if fileSize == 0 || fileSize != obj.Size {
tracerx.Printf("Removing %s, size %d is invalid", mediafile, fileSize)
os.RemoveAll(mediafile)
stat = nil
}
}
if statErr != nil || stat == nil {
err := downloadObject(ptr, obj, mediafile, cb)
if err != nil {
return newSmudgeError(err, obj.Oid, mediafile)
}
}
return nil
}
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download bool, cb CopyCallback) error {
mediafile, err := LocalMediaPath(ptr.Oid)
if err != nil {
return err
}
stat, statErr := os.Stat(mediafile)
if statErr == nil && stat != nil {
fileSize := stat.Size()
if fileSize == 0 || fileSize != ptr.Size {
tracerx.Printf("Removing %s, size %d is invalid", mediafile, fileSize)
os.RemoveAll(mediafile)
stat = nil
}
}
if statErr != nil || stat == nil {
if download {
err = downloadFile(writer, ptr, workingfile, mediafile, cb)
} else {
return newDownloadDeclinedError(nil)
}
} else {
err = readLocalFile(writer, ptr, mediafile, workingfile, cb)
}
if err != nil {
return newSmudgeError(err, ptr.Oid, mediafile)
}
return nil
}
// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
var startProgress sync.Once
for {
batch := q.batcher.Next()
if batch == nil {
break
}
tracerx.Printf("tq: sending batch of size %d", len(batch))
transfers := make([]*objectResource, 0, len(batch))
for _, t := range batch {
transfers = append(transfers, &objectResource{Oid: t.Oid(), Size: t.Size()})
}
objects, err := Batch(transfers, q.transferKind)
if err != nil {
if IsNotImplementedError(err) {
configFile := filepath.Join(LocalGitDir, "config")
git.Config.SetLocal(configFile, "lfs.batch", "false")
go q.legacyFallback(batch)
return
}
q.errorc <- err
q.wait.Add(-len(transfers))
continue
}
startProgress.Do(q.meter.Start)
for _, o := range objects {
if _, ok := o.Rel(q.transferKind); ok {
// This object has an error
if o.Error != nil {
q.errorc <- Error(o.Error)
q.meter.Skip(o.Size)
q.wait.Done()
continue
}
// This object needs to be transferred
if transfer, ok := q.transferables[o.Oid]; ok {
transfer.SetObject(o)
q.meter.Add(transfer.Name())
q.transferc <- transfer
} else {
q.meter.Skip(transfer.Size())
q.wait.Done()
}
} else {
q.meter.Skip(o.Size)
q.wait.Done()
}
}
}
}
// IsVersionAtLeast returns whether the git version is the one specified or higher
// argument is plain version string separated by '.' e.g. "2.3.1" but can omit minor/patch
func (c *gitConfig) IsGitVersionAtLeast(ver string) bool {
gitver, err := c.Version()
if err != nil {
tracerx.Printf("Error getting git version: %v", err)
return false
}
return IsVersionAtLeast(gitver, ver)
}
// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
go q.errorCollector()
tracerx.Printf("tq: starting %d transfer workers", q.workers)
for i := 0; i < q.workers; i++ {
go q.transferWorker()
}
if Config.BatchTransfer() {
tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
q.batcher = NewBatcher(batchSize)
go q.batchApiRoutine()
} else {
tracerx.Printf("tq: running as individual queue")
q.launchIndividualApiRoutines()
}
}
// GetAllWorkTreeHEADs returns the refs that all worktrees are using as HEADs
// This returns all worktrees plus the master working copy, and works even if
// working dir is actually in a worktree right now
// Pass in the git storage dir (parent of 'objects') to work from
func GetAllWorkTreeHEADs(storageDir string) ([]*Ref, error) {
worktreesdir := filepath.Join(storageDir, "worktrees")
dirf, err := os.Open(worktreesdir)
if err != nil && !os.IsNotExist(err) {
return nil, err
}
var worktrees []*Ref
if err == nil {
// There are some worktrees
defer dirf.Close()
direntries, err := dirf.Readdir(0)
if err != nil {
return nil, err
}
for _, dirfi := range direntries {
if dirfi.IsDir() {
// to avoid having to chdir and run git commands to identify the commit
// just read the HEAD file & git rev-parse if necessary
// Since the git repo is shared the same rev-parse will work from this location
headfile := filepath.Join(worktreesdir, dirfi.Name(), "HEAD")
ref, err := parseRefFile(headfile)
if err != nil {
tracerx.Printf("Error reading %v for worktree, skipping: %v", headfile, err)
continue
}
worktrees = append(worktrees, ref)
}
}
}
// This has only established the separate worktrees, not the original checkout
// If the storageDir contains a HEAD file then there is a main checkout
// as well; this mus tbe resolveable whether you're in the main checkout or
// a worktree
headfile := filepath.Join(storageDir, "HEAD")
ref, err := parseRefFile(headfile)
if err == nil {
worktrees = append(worktrees, ref)
} else if !os.IsNotExist(err) { // ok if not exists, probably bare repo
tracerx.Printf("Error reading %v for main checkout, skipping: %v", headfile, err)
}
return worktrees, nil
}