func setCredURLFromNetrc(cfg *config.Configuration, req *http.Request) bool {
hostname := req.URL.Host
var host string
if strings.Contains(hostname, ":") {
var err error
host, _, err = net.SplitHostPort(hostname)
if err != nil {
tracerx.Printf("netrc: error parsing %q: %s", hostname, err)
return false
}
} else {
host = hostname
}
machine, err := cfg.FindNetrcHost(host)
if err != nil {
tracerx.Printf("netrc: error finding match for %q: %s", hostname, err)
return false
}
if machine == nil {
return false
}
setRequestAuth(cfg, req, machine.Login, machine.Password)
return true
}
// Initialise custom adapters based on current config
func configureCustomAdapters(cfg *config.Configuration, m *Manifest) {
pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
for k, v := range cfg.AllGitConfig() {
match := pathRegex.FindStringSubmatch(k)
if match == nil {
continue
}
name := match[1]
path := v
// retrieve other values
args, _ := cfg.Git.Get(fmt.Sprintf("lfs.customtransfer.%s.args", name))
concurrent := cfg.Git.Bool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
direction, _ := cfg.Git.Get(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
if len(direction) == 0 {
direction = "both"
} else {
direction = strings.ToLower(direction)
}
// Separate closure for each since we need to capture vars above
newfunc := func(name string, dir Direction) TransferAdapter {
return newCustomAdapter(name, dir, path, args, concurrent)
}
if direction == "download" || direction == "both" {
m.RegisterNewTransferAdapterFunc(name, Download, newfunc)
}
if direction == "upload" || direction == "both" {
m.RegisterNewTransferAdapterFunc(name, Upload, newfunc)
}
}
}
// LogHttpStats is intended to be called after all HTTP operations for the
// commmand have finished. It dumps k/v logs, one line per httpTransfer into
// a log file with the current timestamp.
func LogHttpStats(cfg *config.Configuration) {
if !cfg.IsLoggingStats {
return
}
file, err := statsLogFile()
if err != nil {
fmt.Fprintf(os.Stderr, "Error logging http stats: %s\n", err)
return
}
fmt.Fprintf(file, "concurrent=%d batch=%v time=%d version=%s\n", cfg.ConcurrentTransfers(), cfg.BatchTransfer(), time.Now().Unix(), config.Version)
for key, responses := range httpTransferBuckets {
for _, response := range responses {
stats := httpTransfers[response]
fmt.Fprintf(file, "key=%s reqheader=%d reqbody=%d resheader=%d resbody=%d restime=%d status=%d url=%s\n",
key,
stats.requestStats.HeaderSize,
stats.requestStats.BodySize,
stats.responseStats.HeaderSize,
stats.responseStats.BodySize,
stats.responseStats.Stop.Sub(stats.responseStats.Start).Nanoseconds(),
response.StatusCode,
response.Request.URL)
}
}
fmt.Fprintf(os.Stderr, "HTTP Stats logged to file %s\n", file.Name())
}
func setRequestAuthFromUrl(cfg *config.Configuration, req *http.Request, u *url.URL) bool {
if !cfg.NtlmAccess(GetOperationForRequest(req)) && u.User != nil {
if pass, ok := u.User.Password(); ok {
fmt.Fprintln(os.Stderr, "warning: current Git remote contains credentials")
setRequestAuth(cfg, req, u.User.Username(), pass)
return true
}
}
return false
}
func ConfigureManifest(m *Manifest, cfg *config.Configuration) *Manifest {
m.basicTransfersOnly = cfg.BasicTransfersOnly()
configureBasicDownloadAdapter(m)
configureBasicUploadAdapter(m)
if cfg.TusTransfersAllowed() {
configureTusAdapter(m)
}
configureCustomAdapters(cfg, m)
return m
}
func skipCredsCheck(cfg *config.Configuration, req *http.Request) bool {
if cfg.NtlmAccess(GetOperationForRequest(req)) {
return false
}
if len(req.Header.Get("Authorization")) > 0 {
return true
}
q := req.URL.Query()
return len(q["token"]) > 0
}
func determineIncludeExcludePaths(config *config.Configuration, includeArg, excludeArg *string) (include, exclude []string) {
if includeArg == nil {
include = config.FetchIncludePaths()
} else {
include = tools.CleanPaths(*includeArg, ",")
}
if excludeArg == nil {
exclude = config.FetchExcludePaths()
} else {
exclude = tools.CleanPaths(*excludeArg, ",")
}
return
}
func setRequestAuth(cfg *config.Configuration, req *http.Request, user, pass string) {
if cfg.NtlmAccess(GetOperationForRequest(req)) {
return
}
if len(user) == 0 && len(pass) == 0 {
return
}
token := fmt.Sprintf("%s:%s", user, pass)
auth := "Basic " + strings.TrimSpace(base64.StdEncoding.EncodeToString([]byte(token)))
req.Header.Set("Authorization", auth)
}
// BatchOrLegacy calls the Batch API and falls back on the Legacy API
// This is for simplicity, legacy route is not most optimal (serial)
// TODO LEGACY API: remove when legacy API removed
func BatchOrLegacy(cfg *config.Configuration, objects []*ObjectResource, operation string, transferAdapters []string) (objs []*ObjectResource, transferAdapter string, e error) {
if !cfg.BatchTransfer() {
objs, err := Legacy(cfg, objects, operation)
return objs, "", err
}
objs, adapterName, err := Batch(cfg, objects, operation, transferAdapters)
if err != nil {
if errors.IsNotImplementedError(err) {
git.Config.SetLocal("", "lfs.batch", "false")
objs, err := Legacy(cfg, objects, operation)
return objs, "", err
}
return nil, "", err
}
return objs, adapterName, nil
}
// Internal http request management
func doHttpRequest(cfg *config.Configuration, req *http.Request, creds auth.Creds) (*http.Response, error) {
var (
res *http.Response
cause string
err error
)
if cfg.NtlmAccess(auth.GetOperationForRequest(req)) {
cause = "ntlm"
res, err = doNTLMRequest(cfg, req, true)
} else {
cause = "http"
res, err = NewHttpClient(cfg, req.Host).Do(req)
}
if res == nil {
res = &http.Response{
StatusCode: 0,
Header: make(http.Header),
Request: req,
Body: ioutil.NopCloser(bytes.NewBufferString("")),
}
}
if err != nil {
if errors.IsAuthError(err) {
SetAuthType(cfg, req, res)
doHttpRequest(cfg, req, creds)
} else {
err = errors.Wrap(err, cause)
}
} else {
err = handleResponse(cfg, res, creds)
}
if err != nil {
if res != nil {
SetErrorResponseContext(cfg, err, res)
} else {
setErrorRequestContext(cfg, err, req)
}
}
return res, err
}
// doApiBatchRequest runs the request to the LFS batch API. If the API returns a
// 401, the repo will be marked as having private access and the request will be
// re-run. When the repo is marked as having private access, credentials will
// be retrieved.
func DoBatchRequest(cfg *config.Configuration, req *http.Request) (*http.Response, *batchResponse, error) {
res, err := DoRequest(req, cfg.PrivateAccess(auth.GetOperationForRequest(req)))
if err != nil {
if res != nil && res.StatusCode == 401 {
return res, nil, errors.NewAuthError(err)
}
return res, nil, err
}
resp := &batchResponse{}
err = httputil.DecodeResponse(res, resp)
if err != nil {
httputil.SetErrorResponseContext(cfg, err, res)
}
return res, resp, err
}
// NewHttpClient returns a new HttpClient for the given host (which may be "host:port")
func NewHttpClient(c *config.Configuration, host string) *HttpClient {
httpClientsMutex.Lock()
defer httpClientsMutex.Unlock()
if httpClients == nil {
httpClients = make(map[string]*HttpClient)
}
if client, ok := httpClients[host]; ok {
return client
}
dialtime := c.Git.Int("lfs.dialtimeout", 30)
keepalivetime := c.Git.Int("lfs.keepalive", 1800) // 30 minutes
tlstime := c.Git.Int("lfs.tlstimeout", 30)
tr := &http.Transport{
Proxy: ProxyFromGitConfigOrEnvironment(c),
Dial: (&net.Dialer{
Timeout: time.Duration(dialtime) * time.Second,
KeepAlive: time.Duration(keepalivetime) * time.Second,
}).Dial,
TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
MaxIdleConnsPerHost: c.ConcurrentTransfers(),
}
tr.TLSClientConfig = &tls.Config{}
if isCertVerificationDisabledForHost(c, host) {
tr.TLSClientConfig.InsecureSkipVerify = true
} else {
tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
}
client := &HttpClient{
Config: c,
Client: &http.Client{Transport: tr, CheckRedirect: CheckRedirect},
}
httpClients[host] = client
return client
}
func SshAuthenticate(cfg *config.Configuration, operation, oid string) (SshAuthResponse, config.Endpoint, error) {
// This is only used as a fallback where the Git URL is SSH but server doesn't support a full SSH binary protocol
// and therefore we derive a HTTPS endpoint for binaries instead; but check authentication here via SSH
endpoint := cfg.Endpoint(operation)
res := SshAuthResponse{}
if len(endpoint.SshUserAndHost) == 0 {
return res, endpoint, nil
}
tracerx.Printf("ssh: %s git-lfs-authenticate %s %s %s",
endpoint.SshUserAndHost, endpoint.SshPath, operation, oid)
exe, args := sshGetExeAndArgs(cfg, endpoint)
args = append(args,
fmt.Sprintf("git-lfs-authenticate %s %s %s", endpoint.SshPath, operation, oid))
cmd := exec.Command(exe, args...)
// Save stdout and stderr in separate buffers
var outbuf, errbuf bytes.Buffer
cmd.Stdout = &outbuf
cmd.Stderr = &errbuf
// Execute command
err := cmd.Start()
if err == nil {
err = cmd.Wait()
}
// Processing result
if err != nil {
res.Message = errbuf.String()
} else {
err = json.Unmarshal(outbuf.Bytes(), &res)
}
return res, endpoint, err
}
func getCredURLForAPI(cfg *config.Configuration, req *http.Request) (*url.URL, error) {
operation := GetOperationForRequest(req)
apiUrl, err := url.Parse(cfg.Endpoint(operation).Url)
if err != nil {
return nil, err
}
// if the LFS request doesn't match the current LFS url, don't bother
// attempting to set the Authorization header from the LFS or Git remote URLs.
if req.URL.Scheme != apiUrl.Scheme ||
req.URL.Host != apiUrl.Host {
return req.URL, nil
}
if setRequestAuthFromUrl(cfg, req, apiUrl) {
return nil, nil
}
credsUrl := apiUrl
if len(cfg.CurrentRemote) > 0 {
if u := cfg.GitRemoteUrl(cfg.CurrentRemote, operation == "upload"); u != "" {
gitRemoteUrl, err := url.Parse(u)
if err != nil {
return nil, err
}
if gitRemoteUrl.Scheme == apiUrl.Scheme &&
gitRemoteUrl.Host == apiUrl.Host {
if setRequestAuthFromUrl(cfg, req, gitRemoteUrl) {
return nil, nil
}
credsUrl = gitRemoteUrl
}
}
}
return credsUrl, nil
}
func ntlmClientSession(c *config.Configuration, creds auth.Creds) (ntlm.ClientSession, error) {
if c.NtlmSession != nil {
return c.NtlmSession, nil
}
splits := strings.Split(creds["username"], "\\")
if len(splits) != 2 {
errorMessage := fmt.Sprintf("Your user name must be of the form DOMAIN\\user. It is currently %s", creds["username"])
return nil, errors.New(errorMessage)
}
session, err := ntlm.CreateClientSession(ntlm.Version2, ntlm.ConnectionOrientedMode)
if err != nil {
return nil, err
}
session.SetUserInfo(splits[1], creds["password"], strings.ToUpper(splits[0]))
c.NtlmSession = session
return session, nil
}
func determineIncludeExcludePaths(config *config.Configuration, includeArg, excludeArg string) (include, exclude []string) {
return tools.CleanPathsDefault(includeArg, ",", config.FetchIncludePaths()),
tools.CleanPathsDefault(excludeArg, ",", config.FetchExcludePaths())
}
func Environ(cfg *config.Configuration, manifest *transfer.Manifest) []string {
osEnviron := os.Environ()
env := make([]string, 0, len(osEnviron)+7)
dltransfers := manifest.GetDownloadAdapterNames()
sort.Strings(dltransfers)
ultransfers := manifest.GetUploadAdapterNames()
sort.Strings(ultransfers)
fetchPruneConfig := cfg.FetchPruneConfig()
env = append(env,
fmt.Sprintf("LocalWorkingDir=%s", config.LocalWorkingDir),
fmt.Sprintf("LocalGitDir=%s", config.LocalGitDir),
fmt.Sprintf("LocalGitStorageDir=%s", config.LocalGitStorageDir),
fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir()),
fmt.Sprintf("LocalReferenceDir=%s", config.LocalReferenceDir),
fmt.Sprintf("TempDir=%s", TempDir()),
fmt.Sprintf("ConcurrentTransfers=%d", cfg.ConcurrentTransfers()),
fmt.Sprintf("TusTransfers=%v", cfg.TusTransfersAllowed()),
fmt.Sprintf("BasicTransfersOnly=%v", cfg.BasicTransfersOnly()),
fmt.Sprintf("BatchTransfer=%v", cfg.BatchTransfer()),
fmt.Sprintf("SkipDownloadErrors=%v", cfg.SkipDownloadErrors()),
fmt.Sprintf("FetchRecentAlways=%v", fetchPruneConfig.FetchRecentAlways),
fmt.Sprintf("FetchRecentRefsDays=%d", fetchPruneConfig.FetchRecentRefsDays),
fmt.Sprintf("FetchRecentCommitsDays=%d", fetchPruneConfig.FetchRecentCommitsDays),
fmt.Sprintf("FetchRecentRefsIncludeRemotes=%v", fetchPruneConfig.FetchRecentRefsIncludeRemotes),
fmt.Sprintf("PruneOffsetDays=%d", fetchPruneConfig.PruneOffsetDays),
fmt.Sprintf("PruneVerifyRemoteAlways=%v", fetchPruneConfig.PruneVerifyRemoteAlways),
fmt.Sprintf("PruneRemoteName=%s", fetchPruneConfig.PruneRemoteName),
fmt.Sprintf("AccessDownload=%s", cfg.Access("download")),
fmt.Sprintf("AccessUpload=%s", cfg.Access("upload")),
fmt.Sprintf("DownloadTransfers=%s", strings.Join(dltransfers, ",")),
fmt.Sprintf("UploadTransfers=%s", strings.Join(ultransfers, ",")),
)
if len(cfg.FetchExcludePaths()) > 0 {
env = append(env, fmt.Sprintf("FetchExclude=%s", strings.Join(cfg.FetchExcludePaths(), ", ")))
}
if len(cfg.FetchIncludePaths()) > 0 {
env = append(env, fmt.Sprintf("FetchInclude=%s", strings.Join(cfg.FetchIncludePaths(), ", ")))
}
for _, ext := range cfg.Extensions() {
env = append(env, fmt.Sprintf("Extension[%d]=%s", ext.Priority, ext.Name))
}
for _, e := range osEnviron {
if !strings.Contains(e, "GIT_") {
continue
}
env = append(env, e)
}
return env
}
func SetAuthType(cfg *config.Configuration, req *http.Request, res *http.Response) {
authType := GetAuthType(res)
operation := auth.GetOperationForRequest(req)
cfg.SetAccess(operation, authType)
tracerx.Printf("api: http response indicates %q authentication. Resubmitting...", authType)
}
func setErrorRequestContext(cfg *config.Configuration, err error, req *http.Request) {
errors.SetContext(err, "Endpoint", cfg.Endpoint(auth.GetOperationForRequest(req)).Url)
errors.SetContext(err, "URL", TraceHttpReq(req))
setErrorHeaderContext(err, "Response", req.Header)
}