func (up *Uploader) UploadFile(filename string, rollSplits bool) (respr *client.PutResult, outerr os.Error) {
up.getUploadToken()
defer up.releaseUploadToken()
fi, err := os.Lstat(filename)
if err != nil {
return nil, err
}
if up.statCache != nil && fi.IsRegular() {
cachedRes, err := up.statCache.CachedPutResult(up.pwd, filename, fi)
if err == nil {
cachelog.Printf("Cache HIT on %q -> %v", filename, cachedRes)
return cachedRes, nil
}
defer func() {
if respr != nil && outerr == nil {
up.statCache.AddCachedPutResult(up.pwd, filename, fi, respr)
}
}()
}
m := schema.NewCommonFileMap(filename, fi)
switch {
case fi.IsRegular():
m["camliType"] = "file"
file, err := os.Open(filename)
if err != nil {
return nil, err
}
defer file.Close()
statReceiver := up.altStatReceiver
if statReceiver == nil {
// TODO(bradfitz): just make Client be a
// StatReceiver? move remote's ReceiveBlob ->
// Upload wrapper into Client itself?
statReceiver = remote.NewFromClient(up.Client)
}
schemaWriteFileMap := schema.WriteFileMap
if rollSplits {
schemaWriteFileMap = schema.WriteFileMapRolling
}
blobref, err := schemaWriteFileMap(statReceiver, m, io.LimitReader(file, fi.Size))
if err != nil {
return nil, err
}
// TODO(bradfitz): taking a PutResult here is kinda
// gross. should instead make a blobserver.Storage
// wrapper type that can track some of this? or that
// updates the client stats directly or something.
{
json, _ := schema.MapToCamliJson(m)
pr := &client.PutResult{BlobRef: blobref, Size: int64(len(json)), Skipped: false}
return pr, nil
}
case fi.IsSymlink():
if err = schema.PopulateSymlinkMap(m, filename); err != nil {
return nil, err
}
case fi.IsDirectory():
ss := new(schema.StaticSet)
dir, err := os.Open(filename)
if err != nil {
return nil, err
}
dirNames, err := dir.Readdirnames(-1)
if err != nil {
return nil, err
}
dir.Close()
sort.Strings(dirNames)
// Temporarily give up our upload token while we
// process all our children. The defer function makes
// sure we re-acquire it (keeping balance in the
// world) before we return.
up.releaseUploadToken()
tokenTookBack := false
defer func() {
if !tokenTookBack {
up.getUploadToken()
}
}()
rate := make(chan bool, 100) // max outstanding goroutines, further limited by filecapc
type nameResult struct {
name string
putres *client.PutResult
err os.Error
}
resc := make(chan nameResult, buffered)
go func() {
for _, name := range dirNames {
rate <- true
go func(dirEntName string) {
pr, err := up.UploadFile(filename+"/"+dirEntName, rollSplits)
if pr == nil && err == nil {
log.Fatalf("nil/nil from up.UploadFile on %q", filename+"/"+dirEntName)
}
resc <- nameResult{dirEntName, pr, err}
<-rate
}(name)
}
}()
resm := make(map[string]*client.PutResult)
var entUploadErr os.Error
for _ = range dirNames {
r := <-resc
if r.err != nil {
entUploadErr = fmt.Errorf("error uploading %s: %v", r.name, r.err)
continue
}
resm[r.name] = r.putres
}
if entUploadErr != nil {
return nil, entUploadErr
}
for _, name := range dirNames {
ss.Add(resm[name].BlobRef)
}
// Re-acquire the upload token that we temporarily yielded up above.
up.getUploadToken()
tokenTookBack = true
sspr, err := up.UploadMap(ss.Map())
if err != nil {
return nil, err
}
schema.PopulateDirectoryMap(m, sspr.BlobRef)
case fi.IsBlock():
fallthrough
case fi.IsChar():
fallthrough
case fi.IsSocket():
fallthrough
case fi.IsFifo():
fallthrough
default:
return nil, schema.ErrUnimplemented
}
mappr, err := up.UploadMap(m)
if err == nil {
vlog.Printf("Uploaded %q, %s for %s", m["camliType"], mappr.BlobRef, filename)
} else {
vlog.Printf("Error uploading map %v: %v", m, err)
}
return mappr, err
}
func (up *Uploader) UploadFile(filename string) (*client.PutResult, os.Error) {
fi, err := os.Lstat(filename)
if err != nil {
return nil, err
}
m := schema.NewCommonFileMap(filename, fi)
switch {
case fi.IsRegular():
// Put the blob of the file itself. (TODO: smart boundary chunking)
// For now we just store it as one range.
blobpr, err := up.UploadFileBlob(filename)
if err != nil {
return nil, err
}
parts := []schema.ContentPart{{BlobRef: blobpr.BlobRef, Size: blobpr.Size}}
if blobpr.Size != fi.Size {
// TODO: handle races of file changing while reading it
// after the stat.
}
if err = schema.PopulateRegularFileMap(m, fi, parts); err != nil {
return nil, err
}
case fi.IsSymlink():
if err = schema.PopulateSymlinkMap(m, filename); err != nil {
return nil, err
}
case fi.IsDirectory():
ss := new(schema.StaticSet)
dir, err := os.Open(filename, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
dirNames, err := dir.Readdirnames(-1)
if err != nil {
return nil, err
}
dir.Close()
sort.SortStrings(dirNames)
// TODO: process dirName entries in parallel
for _, dirEntName := range dirNames {
pr, err := up.UploadFile(filename + "/" + dirEntName)
if err != nil {
return nil, err
}
ss.Add(pr.BlobRef)
}
sspr, err := up.UploadMap(ss.Map())
if err != nil {
return nil, err
}
schema.PopulateDirectoryMap(m, sspr.BlobRef)
case fi.IsBlock():
fallthrough
case fi.IsChar():
fallthrough
case fi.IsSocket():
fallthrough
case fi.IsFifo():
fallthrough
default:
return nil, schema.UnimplementedError
}
mappr, err := up.UploadMap(m)
return mappr, err
}