Ejemplo n.º 1
0
func TestUploadDirExecute(t *testing.T) {
	th.SetupHTTP()
	defer th.TeardownHTTP()

	_, filename, _, _ := runtime.Caller(0)
	rootDirFix := path.Dir(filename)

	var count int64
	count = 0

	filepath.Walk(rootDirFix, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		urlPath := "/foo" + strings.TrimPrefix(path, rootDirFix)
		th.Mux.HandleFunc(urlPath, func(w http.ResponseWriter, r *http.Request) {
			th.TestMethod(t, r, "PUT")
			str, err := ioutil.ReadFile(path)
			if err == nil {
				th.TestBody(t, r, string(str))
				count++
			}
			w.WriteHeader(201)
		})
		return nil
	})

	fs := flag.NewFlagSet("flags", 1)

	fs.String("container", "", "")
	fs.String("dir", "", "")
	fs.String("quiet", "", "")

	fs.Set("container", "foo")
	fs.Set("dir", rootDirFix)
	fs.Set("quiet", "true")

	cmd := newUpDirCmd(fs)
	cmd.Ctx.ServiceClient = client.ServiceClient()

	res := &handler.Resource{}
	cmd.HandleFlags(res)
	cmd.Execute(res)

	th.AssertNoErr(t, res.Err)

	if !strings.Contains(res.Result.(string), fmt.Sprintf("Uploaded %d %s ", count, util.Pluralize("object", count))) {
		t.Fatalf("Unexpected result message: %s", res.Result)
	}
}
Ejemplo n.º 2
0
func handleEmpty(command handler.Commander, resource *handler.Resource, params *handleEmptyParams) {
	var totalFiles int64
	var wg sync.WaitGroup

	// bump thread count to number of available CPUs
	runtime.GOMAXPROCS(runtime.NumCPU())

	// get the names of all the objects in the container
	allPages, err := objects.List(command.Context().ServiceClient, params.container, nil).AllPages()
	if err != nil {
		resource.Err = err
		return
	}
	names, err := objects.ExtractNames(allPages)
	if err != nil {
		resource.Err = err
		return
	}

	// send the object names into the `jobs` channel
	jobs := make(chan string, len(names))
	for i := 0; i < len(names); i++ {
		wg.Add(1)
		jobs <- names[i]
	}
	close(jobs)

	// default the number of goroutines to spawn if the `concurrency` flag
	// wasn't provided
	if params.concurrency == 0 {
		params.concurrency = 100
	}

	start := time.Now()

	for i := 0; i < params.concurrency; i++ {
		go func(totalFiles *int64) {
			for objectName := range jobs {
				ticker := backoff.NewTicker(backoff.NewExponentialBackOff())
				for _ = range ticker.C {
					rawResponse := objects.Delete(command.Context().ServiceClient, params.container, objectName, nil)
					if rawResponse.Err != nil {
						continue
					}
					ticker.Stop()
					break
				}

				*totalFiles++

				if !params.quiet {
					re := &handler.Resource{
						Result: fmt.Sprintf("Successfully deleted object [%s] from container [%s]\n", objectName, params.container),
					}
					command.Context().Results <- re
				}
				wg.Done()
			}
		}(&totalFiles)
	}

	wg.Wait()

	resource.Result = fmt.Sprintf("Finished! Deleted %s %s in %s", humanize.Comma(totalFiles), util.Pluralize("object", totalFiles), humanize.RelTime(start, time.Now(), "", ""))
}
Ejemplo n.º 3
0
func (command *commandUploadDir) Execute(resource *handler.Resource) {
	params := resource.Params.(*paramsUploadDir)

	stat, err := os.Stat(params.dir)
	if err != nil {
		resource.Err = err
		return
	}
	if !stat.IsDir() {
		resource.Err = fmt.Errorf("%s is not a directory, ignoring", params.dir)
		return
	}

	// bump thread count to number of available CPUs
	runtime.GOMAXPROCS(runtime.NumCPU())

	jobs := make(chan string)
	results := make(chan *handler.Resource)

	var wg sync.WaitGroup
	var totalSize uint64
	var totalFiles int64
	start := time.Now()

	for i := 0; i < params.concurrency; i++ {
		wg.Add(1)
		go func(totalSize *uint64, totalFiles *int64) {
			for p := range jobs {
				var re *handler.Resource

				ticker := backoff.NewTicker(backoff.NewExponentialBackOff())
				for _ = range ticker.C {
					re = command.handle(p, params)
					if re.Err != nil {
						continue
					}

					ticker.Stop()
					break
				}

				fi, err := os.Stat(p)
				if err == nil {
					*totalSize += uint64(fi.Size())
					*totalFiles++
				}

				if !params.quiet {
					command.Ctx.Results <- re
				}
			}
			wg.Done()
		}(&totalSize, &totalFiles)
	}

	filepath.Walk(params.dir, func(path string, info os.FileInfo, err error) error {
		pathSep := string(os.PathSeparator)
		parent := filepath.Clean(params.dir)
		if !params.recurse && strings.Contains(strings.TrimPrefix(path, parent+pathSep), pathSep) {
			return nil
		}
		if !info.IsDir() {
			jobs <- path
		}
		return nil
	})
	close(jobs)

	wg.Wait()
	close(results)

	resource.Result = fmt.Sprintf("Finished! Uploaded %s %s totaling %s in %s", humanize.Comma(totalFiles), util.Pluralize("object", totalFiles), humanize.Bytes(totalSize), humanize.RelTime(start, time.Now(), "", ""))
}