func TestUploadDirExecute(t *testing.T) {
th.SetupHTTP()
defer th.TeardownHTTP()
_, filename, _, _ := runtime.Caller(0)
rootDirFix := path.Dir(filename)
var count int64
count = 0
filepath.Walk(rootDirFix, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
urlPath := "/foo" + strings.TrimPrefix(path, rootDirFix)
th.Mux.HandleFunc(urlPath, func(w http.ResponseWriter, r *http.Request) {
th.TestMethod(t, r, "PUT")
str, err := ioutil.ReadFile(path)
if err == nil {
th.TestBody(t, r, string(str))
count++
}
w.WriteHeader(201)
})
return nil
})
fs := flag.NewFlagSet("flags", 1)
fs.String("container", "", "")
fs.String("dir", "", "")
fs.String("quiet", "", "")
fs.Set("container", "foo")
fs.Set("dir", rootDirFix)
fs.Set("quiet", "true")
cmd := newUpDirCmd(fs)
cmd.Ctx.ServiceClient = client.ServiceClient()
res := &handler.Resource{}
cmd.HandleFlags(res)
cmd.Execute(res)
th.AssertNoErr(t, res.Err)
if !strings.Contains(res.Result.(string), fmt.Sprintf("Uploaded %d %s ", count, util.Pluralize("object", count))) {
t.Fatalf("Unexpected result message: %s", res.Result)
}
}
func handleEmpty(command handler.Commander, resource *handler.Resource, params *handleEmptyParams) {
var totalFiles int64
var wg sync.WaitGroup
// bump thread count to number of available CPUs
runtime.GOMAXPROCS(runtime.NumCPU())
// get the names of all the objects in the container
allPages, err := objects.List(command.Context().ServiceClient, params.container, nil).AllPages()
if err != nil {
resource.Err = err
return
}
names, err := objects.ExtractNames(allPages)
if err != nil {
resource.Err = err
return
}
// send the object names into the `jobs` channel
jobs := make(chan string, len(names))
for i := 0; i < len(names); i++ {
wg.Add(1)
jobs <- names[i]
}
close(jobs)
// default the number of goroutines to spawn if the `concurrency` flag
// wasn't provided
if params.concurrency == 0 {
params.concurrency = 100
}
start := time.Now()
for i := 0; i < params.concurrency; i++ {
go func(totalFiles *int64) {
for objectName := range jobs {
ticker := backoff.NewTicker(backoff.NewExponentialBackOff())
for _ = range ticker.C {
rawResponse := objects.Delete(command.Context().ServiceClient, params.container, objectName, nil)
if rawResponse.Err != nil {
continue
}
ticker.Stop()
break
}
*totalFiles++
if !params.quiet {
re := &handler.Resource{
Result: fmt.Sprintf("Successfully deleted object [%s] from container [%s]\n", objectName, params.container),
}
command.Context().Results <- re
}
wg.Done()
}
}(&totalFiles)
}
wg.Wait()
resource.Result = fmt.Sprintf("Finished! Deleted %s %s in %s", humanize.Comma(totalFiles), util.Pluralize("object", totalFiles), humanize.RelTime(start, time.Now(), "", ""))
}
func (command *commandUploadDir) Execute(resource *handler.Resource) {
params := resource.Params.(*paramsUploadDir)
stat, err := os.Stat(params.dir)
if err != nil {
resource.Err = err
return
}
if !stat.IsDir() {
resource.Err = fmt.Errorf("%s is not a directory, ignoring", params.dir)
return
}
// bump thread count to number of available CPUs
runtime.GOMAXPROCS(runtime.NumCPU())
jobs := make(chan string)
results := make(chan *handler.Resource)
var wg sync.WaitGroup
var totalSize uint64
var totalFiles int64
start := time.Now()
for i := 0; i < params.concurrency; i++ {
wg.Add(1)
go func(totalSize *uint64, totalFiles *int64) {
for p := range jobs {
var re *handler.Resource
ticker := backoff.NewTicker(backoff.NewExponentialBackOff())
for _ = range ticker.C {
re = command.handle(p, params)
if re.Err != nil {
continue
}
ticker.Stop()
break
}
fi, err := os.Stat(p)
if err == nil {
*totalSize += uint64(fi.Size())
*totalFiles++
}
if !params.quiet {
command.Ctx.Results <- re
}
}
wg.Done()
}(&totalSize, &totalFiles)
}
filepath.Walk(params.dir, func(path string, info os.FileInfo, err error) error {
pathSep := string(os.PathSeparator)
parent := filepath.Clean(params.dir)
if !params.recurse && strings.Contains(strings.TrimPrefix(path, parent+pathSep), pathSep) {
return nil
}
if !info.IsDir() {
jobs <- path
}
return nil
})
close(jobs)
wg.Wait()
close(results)
resource.Result = fmt.Sprintf("Finished! Uploaded %s %s totaling %s in %s", humanize.Comma(totalFiles), util.Pluralize("object", totalFiles), humanize.Bytes(totalSize), humanize.RelTime(start, time.Now(), "", ""))
}
