// DialTLSFunc returns the adequate dial function, when using SSL, depending on
// whether we're using insecure TLS (certificate verification is disabled), or we
// have some trusted certs, or we're on android.
// If the client's config has some trusted certs, the server's certificate will
// be checked against those in the config after the TLS handshake.
func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
if !c.useTLS() {
return nil
}
trustedCerts := c.getTrustedCerts()
var stdTLS bool
if !c.InsecureTLS && len(trustedCerts) == 0 {
// TLS with normal/full verification
stdTLS = true
if !android.IsChild() {
// Not android, so let the stdlib deal with it
return nil
}
}
return func(network, addr string) (net.Conn, error) {
var conn *tls.Conn
var err error
if android.IsChild() {
con, err := android.Dial(network, addr)
if err != nil {
return nil, err
}
var tlsConfig *tls.Config
if stdTLS {
tlsConfig, err = android.TLSConfig()
if err != nil {
return nil, err
}
} else {
tlsConfig = &tls.Config{InsecureSkipVerify: true}
}
conn = tls.Client(con, tlsConfig)
if err = conn.Handshake(); err != nil {
return nil, err
}
} else {
conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
if err != nil {
return nil, err
}
}
if c.InsecureTLS {
return conn, nil
}
certs := conn.ConnectionState().PeerCertificates
if certs == nil || len(certs) < 1 {
return nil, errors.New("Could not get server's certificate from the TLS connection.")
}
sig := hashutil.SHA256Prefix(certs[0].Raw)
for _, v := range trustedCerts {
if v == sig {
return conn, nil
}
}
return nil, fmt.Errorf("Server's certificate %v is not in the trusted list", sig)
}
}
// DialTLSFunc returns the adequate dial function, when using SSL, depending on
// whether we're using insecure TLS (certificate verification is disabled), or we
// have some trusted certs, or we're on android.1
// If the client's config has some trusted certs, the server's certificate will
// be checked against those in the config after the TLS handshake.
func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
if !c.useTLS() {
return nil
}
trustedCerts := c.getTrustedCerts()
var stdTLS bool
if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
// TLS with normal/full verification.
stdTLS = true
if !android.IsChild() {
// Not android, so let the stdlib deal with it
return nil
}
}
return func(network, addr string) (net.Conn, error) {
var conn *tls.Conn
var err error
if android.IsChild() {
ac, err := android.Dial(network, addr)
if err != nil {
return nil, err
}
var tlsConfig *tls.Config
if stdTLS {
tlsConfig, err = android.TLSConfig()
if err != nil {
return nil, err
}
} else {
tlsConfig = &tls.Config{InsecureSkipVerify: true}
}
conn = tls.Client(ac, tlsConfig)
if err := conn.Handshake(); err != nil {
return nil, err
}
} else {
conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
if err != nil {
return nil, err
}
}
if c.insecureAnyTLSCert {
return conn, nil
}
certs := conn.ConnectionState().PeerCertificates
if len(certs) < 1 {
return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
}
sig := hashutil.SHA256Prefix(certs[0].Raw)
for _, v := range trustedCerts {
if v == sig {
return conn, nil
}
}
return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
}
}
func init() {
cmdmain.RegisterCommand("file", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(fileCmd)
flags.BoolVar(&cmd.makePermanode, "permanode", false, "Create an associate a new permanode for the uploaded file or directory.")
flags.BoolVar(&cmd.filePermanodes, "filenodes", false, "Create (if necessary) content-based permanodes for each uploaded file.")
flags.BoolVar(&cmd.deleteAfterUpload, "delete_after_upload", false, "If using -filenodes, deletes files once they're uploaded, of if they've already been uploaded.")
flags.BoolVar(&cmd.vivify, "vivify", false,
"If true, ask the server to create and sign permanode(s) associated with each uploaded"+
" file. This permits the server to have your signing key. Used mostly with untrusted"+
" or at-risk clients, such as phones.")
flags.BoolVar(&cmd.exifTime, "exiftime", false, "Try to use metadata (such as EXIF) to get a stable creation time. If found, used as the replacement for the modtime. Mainly useful with vivify or filenodes.")
flags.StringVar(&cmd.title, "title", "", "Optional title attribute to set on permanode when using -permanode.")
flags.StringVar(&cmd.tag, "tag", "", "Optional tag(s) to set on permanode when using -permanode or -filenodes. Single value or comma separated.")
flags.BoolVar(&cmd.diskUsage, "du", false, "Dry run mode: only show disk usage information, without upload or statting dest. Used for testing skipDirs configs, mostly.")
if debug, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG")); debug {
flags.BoolVar(&cmd.statcache, "statcache", true, "Use the stat cache, assuming unchanged files already uploaded in the past are still there. Fast, but potentially dangerous.")
flags.BoolVar(&cmd.memstats, "debug-memstats", false, "Enter debug in-memory mode; collecting stats only. Doesn't upload anything.")
flags.StringVar(&cmd.histo, "debug-histogram-file", "", "Optional file to create and write the blob size for each file uploaded. For use with GNU R and hist(read.table(\"filename\")$V1). Requires debug-memstats.")
flags.BoolVar(&cmd.capCtime, "capctime", false, "For file blobs use file modification time as creation time if it would be bigger (newer) than modification time. For stable filenode creation (you can forge mtime, but can't forge ctime).")
flags.BoolVar(&flagUseSQLiteChildCache, "sqlitecache", false, "Use sqlite for the statcache and havecache instead of a flat cache.")
} else {
cmd.statcache = true
}
if android.IsChild() {
flags.BoolVar(&cmd.argsFromInput, "stdinargs", false, "If true, filenames to upload are sent one-per-line on stdin. EOF means to quit the process with exit status 0.")
}
flagCacheLog = flags.Bool("logcache", false, "log caching details")
return cmd
})
}
// transportForConfig returns a transport for the client, setting the correct
// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
// It is the caller's responsibility to then use that transport to set
// the client's httpClient with SetHTTPClient.
func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
if c == nil {
return nil
}
var transport http.RoundTripper
proxy := http.ProxyFromEnvironment
if tc != nil && tc.Proxy != nil {
proxy = tc.Proxy
}
if c.useHTTP2(tc) {
transport = &http2.Transport{
DialTLS: c.http2DialTLSFunc(),
}
} else {
transport = &http.Transport{
DialTLS: c.DialTLSFunc(),
Dial: c.DialFunc(),
Proxy: proxy,
MaxIdleConnsPerHost: maxParallelHTTP_h1,
}
}
httpStats := &httputil.StatsTransport{
Transport: transport,
}
if tc != nil {
httpStats.VerboseLog = tc.Verbose
}
transport = httpStats
if android.IsChild() {
transport = &android.StatsTransport{transport}
}
return transport
}
// TransportForConfig returns a transport for the client, setting the correct
// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
// It is the caller's responsibility to then use that transport to set
// the client's httpClient with SetHTTPClient.
func (c *Client) TransportForConfig(tc *TransportConfig) http.RoundTripper {
if c == nil {
return nil
}
tlsConfig, err := c.TLSConfig()
if err != nil {
log.Fatalf("Error while configuring TLS for client: %v", err)
}
var transport http.RoundTripper
proxy := http.ProxyFromEnvironment
if tc != nil && tc.Proxy != nil {
proxy = tc.Proxy
}
transport = &http.Transport{
Dial: c.DialFunc(),
TLSClientConfig: tlsConfig,
Proxy: proxy,
MaxIdleConnsPerHost: maxParallelHTTP,
}
httpStats := &httputil.StatsTransport{
Transport: transport,
}
if tc != nil {
httpStats.VerboseLog = tc.Verbose
}
transport = httpStats
if android.IsChild() {
transport = &android.StatsTransport{transport}
}
return transport
}
// DialFunc returns the adequate dial function, depending on
// whether SSL is required, the client's config has some trusted
// certs, and we're on android.
// If the client's config has some trusted certs, the server's
// certificate will be checked against those in the config after
// the TLS handshake.
func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
trustedCerts := c.getTrustedCerts()
if !c.useTLS() || (!c.InsecureTLS && len(trustedCerts) == 0) {
// No TLS, or TLS with normal/full verification
if android.IsChild() {
return func(network, addr string) (net.Conn, error) {
return android.Dial(network, addr)
}
}
return nil
}
return func(network, addr string) (net.Conn, error) {
var conn *tls.Conn
var err error
if android.IsChild() {
con, err := android.Dial(network, addr)
if err != nil {
return nil, err
}
conn = tls.Client(con, &tls.Config{InsecureSkipVerify: true})
if err = conn.Handshake(); err != nil {
return nil, err
}
} else {
conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
if err != nil {
return nil, err
}
}
if c.InsecureTLS {
return conn, nil
}
certs := conn.ConnectionState().PeerCertificates
if certs == nil || len(certs) < 1 {
return nil, errors.New("Could not get server's certificate from the TLS connection.")
}
sig := hashutil.SHA256Prefix(certs[0].Raw)
for _, v := range trustedCerts {
if v == sig {
return conn, nil
}
}
return nil, fmt.Errorf("Server's certificate %v is not in the trusted list", sig)
}
}
// DialFunc returns the adequate dial function when we're on android.
func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
if c.useTLS() {
return nil
}
if android.IsChild() {
return func(network, addr string) (net.Conn, error) {
return android.Dial(network, addr)
}
}
return nil
}
// statReceiver returns the StatReceiver used for checking for and uploading blobs.
//
// The optional provided node is only used for conditionally printing out status info to stdout.
func (up *Uploader) statReceiver(n *node) blobserver.StatReceiver {
statReceiver := up.altStatReceiver
if statReceiver == nil {
// TODO(mpl): simplify the altStatReceiver situation as well,
// see TODO in cmd/camput/uploader.go
statReceiver = up.Client
}
if android.IsChild() && n != nil && n.fi.Mode()&os.ModeType == 0 {
return android.StatusReceiver{Sr: statReceiver, Path: n.fullPath}
}
return statReceiver
}
func (c *Client) useHTTP2(tc *TransportConfig) bool {
if !c.useTLS() {
return false
}
if android.IsChild() {
// No particular reason; just untested so far.
return false
}
if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
(tc != nil && tc.Proxy != nil) {
// Also just untested. Which proxies support h2 anyway?
return false
}
return true
}
func (t *TreeUpload) run() {
defer close(t.donec)
// Kick off scanning all files, eventually learning the root
// node (which references all its children).
var root *node // nil until received and set in loop below.
rootc := make(chan *node, 1)
if !t.rootless {
go func() {
n, err := t.statPath(t.base, nil)
if err != nil {
log.Fatalf("Error scanning files under %s: %v", t.base, err)
}
close(t.stattedc)
rootc <- n
}()
}
var lastStat, lastUpload string
dumpStats := func() {
if android.IsChild() {
printAndroidCamputStatus(t)
return
}
statStatus := ""
if root == nil {
statStatus = fmt.Sprintf("last stat: %s", lastStat)
}
blobStats := t.up.Stats()
log.Printf("FILES: Total: %+v Skipped: %+v Uploaded: %+v %s BLOBS: %s Digested: %d last upload: %s",
t.total, t.skipped, t.uploaded,
statStatus,
blobStats.String(),
atomic.LoadInt64(&atomicDigestOps),
lastUpload)
}
// Channels for stats & progress bars. These are never closed:
uploadedc := make(chan *node) // at least tried to upload; server might have had blob
skippedc := make(chan *node) // didn't even hit blobserver; trusted our stat cache
uploadsdonec := make(chan bool)
var upload chan<- interface{}
withPermanode := t.up.fileOpts.wantFilePermanode()
if t.DiskUsageMode {
upload = chanworker.NewWorker(1, func(el interface{}, ok bool) {
if !ok {
uploadsdonec <- true
return
}
n := el.(*node)
if n.fi.IsDir() {
fmt.Printf("%d\t%s\n", n.SumBytes()>>10, n.fullPath)
}
})
} else {
// dirUpload is unbounded because directories can depend on directories.
// We bound the number of HTTP requests in flight instead.
// TODO(bradfitz): remove this chanworker stuff?
dirUpload := chanworker.NewWorker(-1, func(el interface{}, ok bool) {
if !ok {
log.Printf("done uploading directories - done with all uploads.")
uploadsdonec <- true
return
}
n := el.(*node)
put, err := t.up.uploadNode(n)
if err != nil {
log.Fatalf("Error uploading %s: %v", n.fullPath, err)
}
n.SetPutResult(put, nil)
uploadedc <- n
})
upload = chanworker.NewWorker(uploadWorkers, func(el interface{}, ok bool) {
if !ok {
log.Printf("done with all uploads.")
close(dirUpload)
log.Printf("closed dirUpload")
return
}
n := el.(*node)
if n.fi.IsDir() {
dirUpload <- n
return
}
put, err := t.up.uploadNode(n)
if err != nil {
log.Fatalf("Error uploading %s: %v", n.fullPath, err)
}
n.SetPutResult(put, nil)
if c := t.up.statCache; c != nil {
c.AddCachedPutResult(
t.up.pwd, n.fullPath, n.fi, put, withPermanode)
}
uploadedc <- n
})
}
checkStatCache := chanworker.NewWorker(statCacheWorkers, func(el interface{}, ok bool) {
if hook := testHookStatCache; hook != nil {
hook(el, ok)
}
if !ok {
if t.up.statCache != nil {
log.Printf("done checking stat cache")
}
close(upload)
return
}
n := el.(*node)
if t.DiskUsageMode || t.up.statCache == nil {
upload <- n
return
}
if !n.fi.IsDir() {
cachedRes, err := t.up.statCache.CachedPutResult(
t.up.pwd, n.fullPath, n.fi, withPermanode)
if err == nil {
n.SetPutResult(cachedRes, nil)
cachelog.Printf("Cache HIT on %q -> %v", n.fullPath, cachedRes)
android.NoteFileUploaded(n.fullPath, false)
skippedc <- n
return
}
}
upload <- n
})
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
stattedc := t.stattedc
Loop:
for {
select {
case <-uploadsdonec:
break Loop
case n := <-rootc:
root = n
case n := <-uploadedc:
t.uploaded.incr(n)
lastUpload = n.fullPath
case n := <-skippedc:
t.skipped.incr(n)
case n, ok := <-stattedc:
if !ok {
log.Printf("done statting:")
dumpStats()
close(checkStatCache)
stattedc = nil
continue
}
lastStat = n.fullPath
t.total.incr(n)
checkStatCache <- n
case <-ticker.C:
dumpStats()
}
}
log.Printf("tree upload finished. final stats:")
dumpStats()
if root == nil {
panic("unexpected nil root node")
}
var err error
log.Printf("Waiting on root node %q", root.fullPath)
t.finalPutRes, err = root.PutResult()
log.Printf("Waited on root node %q: %v", root.fullPath, t.finalPutRes)
if err != nil {
t.err = err
}
}