Exemple #1
0
// GetType returns the type of the "object" behind the path
// It can be : container, object, vfolder, (more)
func (p *osPath) GetType() (string, error) {
	if p.Ptype != "" {
		return p.Ptype, nil
	}
	resp, err := p.client.Call(&gopenstack.CallOptions{
		Method:    "HEAD",
		Ressource: p.Name,
	})

	err = resp.HandleErr(err, []int{200, 204, 404})
	if err != nil {
		return "", err
	}

	// If 404 it must be a vfolder or nothing
	if resp.StatusCode == 404 {
		// Search vpath in container
		resp, err := p.client.Call(&gopenstack.CallOptions{
			Method:    "GET",
			Ressource: p.GetContainer() + "?format=json",
		})

		err = resp.HandleErr(err, []int{200, 404})

		if resp.StatusCode == 404 {
			return "", gopenstack.ErrPathNotFound(p.Name)
		}
		var objects []object
		if err = json.Unmarshal(resp.Body, &objects); err != nil {
			return "", err
		}
		prefix := p.GetPrefix()
		for _, o := range objects {
			if strings.HasPrefix(o.Name, prefix) {
				p.Ptype = "vfolder"
				return p.Ptype, nil
			}
		}
		return "", gopenstack.ErrPathNotFound(p.Name)
	}

	// Region root
	if resp.Headers["X-Account-Container-Count"] != nil {
		p.Ptype = "root"
		return p.Ptype, nil
	}

	// Container
	if resp.Headers["X-Container-Bytes-Used"] != nil || resp.Headers["X-Container-Object-Count"] != nil {
		p.Ptype = "container"
		return p.Ptype, nil
	}
	// Object
	if resp.Headers["Etag"] != nil {
		p.Ptype = "object"
		return p.Ptype, nil
	}
	return "", gopenstack.ErrPathNotFound(p.Name)
}
Exemple #2
0
// GetChildrenObjects return children object of a given path
func (p *osPath) GetChildrenObjects() (children []object, err error) {
	resp, err := p.client.Call(&gopenstack.CallOptions{
		Method:    "GET",
		Ressource: p.GetContainer() + "?format=json",
	})
	err = resp.HandleErr(err, []int{200, 404})
	if resp.StatusCode == 404 {
		return children, gopenstack.ErrPathNotFound(p.Name)
	}
	var tObjects []object
	if err = json.Unmarshal(resp.Body, &tObjects); err != nil {
		return children, err
	}
	prefix := p.GetPrefix()
	for _, o := range tObjects {
		if strings.HasPrefix(o.Name, prefix) {
			children = append(children, o)
		}
	}
	return
}
Exemple #3
0
// DeletePath delete path & his children (helper)
func (s *Swift) DeletePath(path string) error {
	var err error
	hasTrailingSlash := false
	objectToremovePaths := []string{}
	if strings.HasSuffix(path, "/") {
		path = path[:len(path)-1]
		hasTrailingSlash = true
	}

	// Container to remove (last job)
	containerToRemove := ""

	// get path type (container, object, vpath)
	dPath := NewOsPath(s.client, path)
	pathType, err := dPath.GetType()
	if err != nil {
		return err
	}

	switch pathType {
	case "object":
		objectToremovePaths = append(objectToremovePaths, path)
	case "container", "vfolder":
		objectsToRemove, err := dPath.GetChildrenObjects()
		if err != nil {
			return err
		}
		if pathType == "container" {
			for _, o := range objectsToRemove {
				objectToremovePaths = append(objectToremovePaths, path+"/"+o.Name)
			}
			if !hasTrailingSlash {
				containerToRemove = path
			}
		} else {
			if len(objectsToRemove) == 0 {
				return gopenstack.ErrPathNotFound(path)
			}
			container := dPath.GetContainer()
			for _, o := range objectsToRemove {
				objectToremovePaths = append(objectToremovePaths, container+"/"+o.Name)
			}
		}
	default:
		err = gopenstack.ErrUnsuportedPathType(pathType)
		return err
	}

	// Remove objects
	chanDone := make(chan bool)
	chanAJobIsDone := make(chan bool)
	chanThreadCount := make(chan int)
	threadsCount := 0
	remainingJobs := len(objectToremovePaths)

	// Count concurrent threads
	go func() {
		for {
			threadsCount += <-chanThreadCount
		}
	}()

	// Update remainingJobs and send "all jobs are done"  signal
	go func() {
		for {
			<-chanAJobIsDone
			remainingJobs--
			if remainingJobs < 1 {
				chanDone <- true
				break
			}
		}
	}()

	// Delete
	if len(objectToremovePaths) > 0 {
		for _, p := range objectToremovePaths {
			for {
				if threadsCount < 10 {
					threadsCount++
					break
				}
				time.Sleep(1 * time.Second)
			}
			go func(path string) {
				err = s.DeleteObject(path)
				if err != nil {
					chanDone <- true
				}
				threadsCount--
				chanAJobIsDone <- true
			}(p)
		}
		// Waiting for all jobs
		<-chanDone
	}
	// remove container if needed
	if len(containerToRemove) != 0 {
		err = s.DeleteObject(containerToRemove)
	}
	return err
}
Exemple #4
0
// GetAndStore recursively gets objects from srcPath and write them under destPath
func (s *Swift) DownloadPath(srcPath, destPath string) error {

	// we must have a container specified
	if srcPath == "" || srcPath == "/" {
		return gopenstack.ErrNoContainerSpecified
	}

	// check if local path exist
	if _, err := os.Stat(destPath); err != nil {
		return gopenstack.ErrPathNotFound(destPath)
	}
	// Is RA path exists ?
	dPath := NewOsPath(s.client, srcPath)
	pathType, err := dPath.GetType()
	if err != nil {
		return err
	}

	hasTrailingSlash := false
	container := dPath.GetContainer()

	// clean paths
	if strings.HasSuffix(srcPath, "/") {
		srcPath = srcPath[0 : len(srcPath)-1]
		hasTrailingSlash = true
	}
	if strings.HasSuffix(destPath, "/") {
		destPath = destPath[:len(destPath)-1]
	}

	// is container
	isContainer := pathType == "container"
	prefix := ""
	if !isContainer {
		prefix = srcPath[len(container):]
	}
	pPrefix := strings.Split(prefix, "/")

	objectsToDownload, err := dPath.GetChildrenObjects()
	if err != nil {
		return err
	}

	// Upload files
	chanDone := make(chan bool)
	chanAJobIsDone := make(chan bool)
	chanThreadCount := make(chan int)
	threadsCount := 0
	remainingJobs := len(objectsToDownload)

	// Count concurrent threads
	go func() {
		for {
			threadsCount += <-chanThreadCount
		}
	}()

	// Update remainingJobs and send "all jobs are done" signal
	go func() {
		for {
			<-chanAJobIsDone
			remainingJobs--
			if remainingJobs < 1 {
				chanDone <- true
				break
			}
		}
	}()
	exitAsap := false
	for _, o := range objectsToDownload {

		for {
			if threadsCount < 5 {
				threadsCount++
				break
			}
			time.Sleep(1 * time.Second)
		}
		if exitAsap {
			// Wait for running process to finish their task
			for {
				if threadsCount == 1 {
					break
				}
				time.Sleep(1 * time.Second)
			}
			break
		}

		go func(o object) {
			src := container + "/" + o.Name
			dest := destPath + "/"
			if !hasTrailingSlash {
				if isContainer {
					dest += container + "/" + o.Name
				} else {
					dest += pPrefix[len(pPrefix)-1] + "/" + o.Name[len(prefix):]
				}
			} else {
				if isContainer {
					dest += o.Name
				} else {
					dest += o.Name[len(prefix):]
				}
			}

			//fmt.Println(o)
			//fmt.Println(o.Name, src+" -> "+dest)
			err = s.DownloadObject(src, dest)
			if err != nil {
				threadsCount--
				exitAsap = true
				chanDone <- true
				return
			}
			threadsCount--
			chanAJobIsDone <- true
		}(o)
	}
	// Waiting for all jobs to finish
	<-chanDone
	return nil
}
Exemple #5
0
// Put recursively upload files under srcPath to destPath
func (s *Swift) Put(srcPath, destPath string) error {
	srcPath, err := filepath.Abs(filepath.Clean(srcPath))
	if err != nil {
		return err
	}
	fmt.Println(srcPath + " -> " + destPath)
	if strings.HasSuffix(destPath, "/") {
		destPath = destPath[:len(destPath)-1]
	}

	// we must have a container specified
	if destPath == "" || destPath == "/" {
		return gopenstack.ErrNoContainerSpecified
	}

	childrenPaths := []string{}
	if _, err = os.Stat(srcPath); err != nil {
		return gopenstack.ErrPathNotFound(srcPath)
	}

	err = filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			childrenPaths = append(childrenPaths, path)
		}
		return err
	})
	if err != nil {
		return err
	}

	// Upload files
	chanDone := make(chan bool)
	chanAJobIsDone := make(chan bool)
	chanThreadCount := make(chan int)
	threadsCount := 0
	remainingJobs := len(childrenPaths)

	// Count concurrent threads
	go func() {
		for {
			threadsCount += <-chanThreadCount
		}
	}()

	// Update remainingJobs and send "all jobs are done" signal
	go func() {
		for {
			<-chanAJobIsDone
			remainingJobs--
			if remainingJobs < 1 {
				chanDone <- true
				break
			}
		}
	}()

	// PUT
	ps := strings.Split(srcPath, "/")
	exitAsap := false
	for _, p := range childrenPaths {
		for {
			if threadsCount < 5 {
				threadsCount++
				break
			}
			time.Sleep(1 * time.Second)
		}
		if exitAsap {
			// Wait for running process to finish their task
			for {
				if threadsCount == 1 {
					break
				}
				time.Sleep(1 * time.Second)
			}
			break
		}
		go func(p string) {

			destination := "/"

			if !strings.HasSuffix(srcPath, "/") {
				destination += ps[len(ps)-1]
			}

			destination += p[len(srcPath):]

			err = s.PutFile(p, destination)
			if err != nil {
				threadsCount--
				exitAsap = true
				chanDone <- true
				return
			}
			threadsCount--
			chanAJobIsDone <- true
		}(p)
	}
	// Waiting for all jobs to finish
	<-chanDone
	return err
}