// GetType returns the type of the "object" behind the path
// It can be : container, object, vfolder, (more)
func (p *osPath) GetType() (string, error) {
if p.Ptype != "" {
return p.Ptype, nil
}
resp, err := p.client.Call(&gopenstack.CallOptions{
Method: "HEAD",
Ressource: p.Name,
})
err = resp.HandleErr(err, []int{200, 204, 404})
if err != nil {
return "", err
}
// If 404 it must be a vfolder or nothing
if resp.StatusCode == 404 {
// Search vpath in container
resp, err := p.client.Call(&gopenstack.CallOptions{
Method: "GET",
Ressource: p.GetContainer() + "?format=json",
})
err = resp.HandleErr(err, []int{200, 404})
if resp.StatusCode == 404 {
return "", gopenstack.ErrPathNotFound(p.Name)
}
var objects []object
if err = json.Unmarshal(resp.Body, &objects); err != nil {
return "", err
}
prefix := p.GetPrefix()
for _, o := range objects {
if strings.HasPrefix(o.Name, prefix) {
p.Ptype = "vfolder"
return p.Ptype, nil
}
}
return "", gopenstack.ErrPathNotFound(p.Name)
}
// Region root
if resp.Headers["X-Account-Container-Count"] != nil {
p.Ptype = "root"
return p.Ptype, nil
}
// Container
if resp.Headers["X-Container-Bytes-Used"] != nil || resp.Headers["X-Container-Object-Count"] != nil {
p.Ptype = "container"
return p.Ptype, nil
}
// Object
if resp.Headers["Etag"] != nil {
p.Ptype = "object"
return p.Ptype, nil
}
return "", gopenstack.ErrPathNotFound(p.Name)
}
// GetChildrenObjects return children object of a given path
func (p *osPath) GetChildrenObjects() (children []object, err error) {
resp, err := p.client.Call(&gopenstack.CallOptions{
Method: "GET",
Ressource: p.GetContainer() + "?format=json",
})
err = resp.HandleErr(err, []int{200, 404})
if resp.StatusCode == 404 {
return children, gopenstack.ErrPathNotFound(p.Name)
}
var tObjects []object
if err = json.Unmarshal(resp.Body, &tObjects); err != nil {
return children, err
}
prefix := p.GetPrefix()
for _, o := range tObjects {
if strings.HasPrefix(o.Name, prefix) {
children = append(children, o)
}
}
return
}
// DeletePath delete path & his children (helper)
func (s *Swift) DeletePath(path string) error {
var err error
hasTrailingSlash := false
objectToremovePaths := []string{}
if strings.HasSuffix(path, "/") {
path = path[:len(path)-1]
hasTrailingSlash = true
}
// Container to remove (last job)
containerToRemove := ""
// get path type (container, object, vpath)
dPath := NewOsPath(s.client, path)
pathType, err := dPath.GetType()
if err != nil {
return err
}
switch pathType {
case "object":
objectToremovePaths = append(objectToremovePaths, path)
case "container", "vfolder":
objectsToRemove, err := dPath.GetChildrenObjects()
if err != nil {
return err
}
if pathType == "container" {
for _, o := range objectsToRemove {
objectToremovePaths = append(objectToremovePaths, path+"/"+o.Name)
}
if !hasTrailingSlash {
containerToRemove = path
}
} else {
if len(objectsToRemove) == 0 {
return gopenstack.ErrPathNotFound(path)
}
container := dPath.GetContainer()
for _, o := range objectsToRemove {
objectToremovePaths = append(objectToremovePaths, container+"/"+o.Name)
}
}
default:
err = gopenstack.ErrUnsuportedPathType(pathType)
return err
}
// Remove objects
chanDone := make(chan bool)
chanAJobIsDone := make(chan bool)
chanThreadCount := make(chan int)
threadsCount := 0
remainingJobs := len(objectToremovePaths)
// Count concurrent threads
go func() {
for {
threadsCount += <-chanThreadCount
}
}()
// Update remainingJobs and send "all jobs are done" signal
go func() {
for {
<-chanAJobIsDone
remainingJobs--
if remainingJobs < 1 {
chanDone <- true
break
}
}
}()
// Delete
if len(objectToremovePaths) > 0 {
for _, p := range objectToremovePaths {
for {
if threadsCount < 10 {
threadsCount++
break
}
time.Sleep(1 * time.Second)
}
go func(path string) {
err = s.DeleteObject(path)
if err != nil {
chanDone <- true
}
threadsCount--
chanAJobIsDone <- true
}(p)
}
// Waiting for all jobs
<-chanDone
}
// remove container if needed
if len(containerToRemove) != 0 {
err = s.DeleteObject(containerToRemove)
}
return err
}
// GetAndStore recursively gets objects from srcPath and write them under destPath
func (s *Swift) DownloadPath(srcPath, destPath string) error {
// we must have a container specified
if srcPath == "" || srcPath == "/" {
return gopenstack.ErrNoContainerSpecified
}
// check if local path exist
if _, err := os.Stat(destPath); err != nil {
return gopenstack.ErrPathNotFound(destPath)
}
// Is RA path exists ?
dPath := NewOsPath(s.client, srcPath)
pathType, err := dPath.GetType()
if err != nil {
return err
}
hasTrailingSlash := false
container := dPath.GetContainer()
// clean paths
if strings.HasSuffix(srcPath, "/") {
srcPath = srcPath[0 : len(srcPath)-1]
hasTrailingSlash = true
}
if strings.HasSuffix(destPath, "/") {
destPath = destPath[:len(destPath)-1]
}
// is container
isContainer := pathType == "container"
prefix := ""
if !isContainer {
prefix = srcPath[len(container):]
}
pPrefix := strings.Split(prefix, "/")
objectsToDownload, err := dPath.GetChildrenObjects()
if err != nil {
return err
}
// Upload files
chanDone := make(chan bool)
chanAJobIsDone := make(chan bool)
chanThreadCount := make(chan int)
threadsCount := 0
remainingJobs := len(objectsToDownload)
// Count concurrent threads
go func() {
for {
threadsCount += <-chanThreadCount
}
}()
// Update remainingJobs and send "all jobs are done" signal
go func() {
for {
<-chanAJobIsDone
remainingJobs--
if remainingJobs < 1 {
chanDone <- true
break
}
}
}()
exitAsap := false
for _, o := range objectsToDownload {
for {
if threadsCount < 5 {
threadsCount++
break
}
time.Sleep(1 * time.Second)
}
if exitAsap {
// Wait for running process to finish their task
for {
if threadsCount == 1 {
break
}
time.Sleep(1 * time.Second)
}
break
}
go func(o object) {
src := container + "/" + o.Name
dest := destPath + "/"
if !hasTrailingSlash {
if isContainer {
dest += container + "/" + o.Name
} else {
dest += pPrefix[len(pPrefix)-1] + "/" + o.Name[len(prefix):]
}
} else {
if isContainer {
dest += o.Name
} else {
dest += o.Name[len(prefix):]
}
}
//fmt.Println(o)
//fmt.Println(o.Name, src+" -> "+dest)
err = s.DownloadObject(src, dest)
if err != nil {
threadsCount--
exitAsap = true
chanDone <- true
return
}
threadsCount--
chanAJobIsDone <- true
}(o)
}
// Waiting for all jobs to finish
<-chanDone
return nil
}
// Put recursively upload files under srcPath to destPath
func (s *Swift) Put(srcPath, destPath string) error {
srcPath, err := filepath.Abs(filepath.Clean(srcPath))
if err != nil {
return err
}
fmt.Println(srcPath + " -> " + destPath)
if strings.HasSuffix(destPath, "/") {
destPath = destPath[:len(destPath)-1]
}
// we must have a container specified
if destPath == "" || destPath == "/" {
return gopenstack.ErrNoContainerSpecified
}
childrenPaths := []string{}
if _, err = os.Stat(srcPath); err != nil {
return gopenstack.ErrPathNotFound(srcPath)
}
err = filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
childrenPaths = append(childrenPaths, path)
}
return err
})
if err != nil {
return err
}
// Upload files
chanDone := make(chan bool)
chanAJobIsDone := make(chan bool)
chanThreadCount := make(chan int)
threadsCount := 0
remainingJobs := len(childrenPaths)
// Count concurrent threads
go func() {
for {
threadsCount += <-chanThreadCount
}
}()
// Update remainingJobs and send "all jobs are done" signal
go func() {
for {
<-chanAJobIsDone
remainingJobs--
if remainingJobs < 1 {
chanDone <- true
break
}
}
}()
// PUT
ps := strings.Split(srcPath, "/")
exitAsap := false
for _, p := range childrenPaths {
for {
if threadsCount < 5 {
threadsCount++
break
}
time.Sleep(1 * time.Second)
}
if exitAsap {
// Wait for running process to finish their task
for {
if threadsCount == 1 {
break
}
time.Sleep(1 * time.Second)
}
break
}
go func(p string) {
destination := "/"
if !strings.HasSuffix(srcPath, "/") {
destination += ps[len(ps)-1]
}
destination += p[len(srcPath):]
err = s.PutFile(p, destination)
if err != nil {
threadsCount--
exitAsap = true
chanDone <- true
return
}
threadsCount--
chanAJobIsDone <- true
}(p)
}
// Waiting for all jobs to finish
<-chanDone
return err
}
