Beispiel #1
0
// parseContent parse client Content container into printer struct
func parseContent(c *client.Content) Content {
	content := Content{}
	content.Time = c.Time.Local().Format(printDate)

	// guess file type
	content.Filetype = func() string {
		if c.Type.IsDir() {
			return "directory"
		}
		return "file"
	}()

	content.Size = humanize.IBytes(uint64(c.Size))

	// Convert OS Type to match console file printing style
	content.Name = func() string {
		switch {
		case runtime.GOOS == "windows":
			c.Name = strings.Replace(c.Name, "/", "\\", -1)
			c.Name = strings.TrimSuffix(c.Name, "\\")
		default:
			c.Name = strings.TrimSuffix(c.Name, "/")
		}
		if c.Type.IsDir() {
			switch {
			case runtime.GOOS == "windows":
				return fmt.Sprintf("%s\\", c.Name)
			default:
				return fmt.Sprintf("%s/", c.Name)
			}
		}
		return c.Name
	}()
	return content
}
Beispiel #2
0
// parseContent parse client Content container into printer struct.
func parseContent(c *client.Content) ContentMessage {
	content := ContentMessage{}
	content.Time = c.Time.Local()

	// guess file type
	content.Filetype = func() string {
		if c.Type.IsDir() {
			return "folder"
		}
		return "file"
	}()

	content.Size = c.Size
	// Convert OS Type to match console file printing style.
	content.Name = func() string {
		switch {
		case runtime.GOOS == "windows":
			c.Name = strings.Replace(c.Name, "/", "\\", -1)
			c.Name = strings.TrimSuffix(c.Name, "\\")
		default:
			c.Name = strings.TrimSuffix(c.Name, "/")
		}
		if c.Type.IsDir() {
			switch {
			case runtime.GOOS == "windows":
				return fmt.Sprintf("%s\\", c.Name)
			default:
				return fmt.Sprintf("%s/", c.Name)
			}
		}
		return c.Name
	}()
	return content
}
Beispiel #3
0
Datei: s3.go Projekt: fwessels/mc
func (c *s3Client) listIncompleteRecursiveInRoutine(contentCh chan *client.Content) {
	defer close(contentCh)
	// get bucket and object from URL.
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(bucket.Err),
				}
				return
			}
			for object := range c.api.ListIncompleteUploads(bucket.Name, o, true) {
				if object.Err != nil {
					contentCh <- &client.Content{
						Err: probe.NewError(object.Err),
					}
					return
				}
				content := new(client.Content)
				url := *c.hostURL
				url.Path = filepath.Join(url.Path, bucket.Name, object.Key)
				content.URL = url
				content.Size = object.Size
				content.Time = object.Initiated
				content.Type = os.ModeTemporary
				contentCh <- content
			}
		}
	default:
		for object := range c.api.ListIncompleteUploads(b, o, true) {
			if object.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(object.Err),
				}
				return
			}
			url := *c.hostURL
			// Join bucket and incoming object key.
			url.Path = filepath.Join(string(url.Separator), b, object.Key)
			if c.virtualStyle {
				url.Path = filepath.Join(string(url.Separator), object.Key)
			}
			content := new(client.Content)
			content.URL = url
			content.Size = object.Size
			content.Time = object.Initiated
			content.Type = os.ModeTemporary
			contentCh <- content
		}
	}
}
Beispiel #4
0
// Stat - send a 'HEAD' on a bucket or object to get its metadata
func (c *s3Client) Stat() (*client.Content, *probe.Error) {
	c.mu.Lock()
	objectMetadata := new(client.Content)
	bucket, object := c.url2BucketAndObject()
	switch {
	// valid case for s3/...
	case bucket == "" && object == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				c.mu.Unlock()
				return nil, probe.NewError(bucket.Err)
			}
		}
		c.mu.Unlock()
		return &client.Content{URL: *c.hostURL, Type: os.ModeDir}, nil
	}
	if object != "" {
		metadata, err := c.api.StatObject(bucket, object)
		if err != nil {
			c.mu.Unlock()
			errResponse := minio.ToErrorResponse(err)
			if errResponse != nil {
				if errResponse.Code == "NoSuchKey" {
					for content := range c.List(false, false) {
						if content.Err != nil {
							return nil, content.Err.Trace()
						}
						content.Content.URL = *c.hostURL
						content.Content.Type = os.ModeDir
						content.Content.Size = 0
						return content.Content, nil
					}
				}
			}
			return nil, probe.NewError(err)
		}
		objectMetadata.URL = *c.hostURL
		objectMetadata.Time = metadata.LastModified
		objectMetadata.Size = metadata.Size
		objectMetadata.Type = os.FileMode(0664)
		c.mu.Unlock()
		return objectMetadata, nil
	}
	err := c.api.BucketExists(bucket)
	if err != nil {
		c.mu.Unlock()
		return nil, probe.NewError(err)
	}
	bucketMetadata := new(client.Content)
	bucketMetadata.URL = *c.hostURL
	bucketMetadata.Type = os.ModeDir
	c.mu.Unlock()
	return bucketMetadata, nil
}
Beispiel #5
0
// Stat - send a 'HEAD' on a bucket or object to get its metadata
func (c *s3Client) Stat() (*client.Content, error) {
	objectMetadata := new(client.Content)
	bucket, object := c.url2BucketAndObject()
	switch {
	// valid case for s3:...
	case bucket == "" && object == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				return nil, iodine.New(bucket.Err, nil)
			}
			return &client.Content{Type: os.ModeDir}, nil
		}
	}
	if object != "" {
		metadata, err := c.api.StatObject(bucket, object)
		if err != nil {
			errResponse := minio.ToErrorResponse(err)
			if errResponse != nil {
				if errResponse.Code == "NoSuchKey" {
					for content := range c.List(false) {
						if content.Err != nil {
							return nil, iodine.New(err, nil)
						}
						content.Content.Type = os.ModeDir
						content.Content.Name = object
						content.Content.Size = 0
						return content.Content, nil
					}
				}
			}
			return nil, iodine.New(err, nil)
		}
		objectMetadata.Name = metadata.Key
		objectMetadata.Time = metadata.LastModified
		objectMetadata.Size = metadata.Size
		objectMetadata.Type = os.FileMode(0664)
		return objectMetadata, nil
	}
	err := c.api.BucketExists(bucket)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	bucketMetadata := new(client.Content)
	bucketMetadata.Name = bucket
	bucketMetadata.Type = os.ModeDir
	return bucketMetadata, nil
}
Beispiel #6
0
func (c *s3Client) listRecursiveInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     probe.NewError(bucket.Err),
				}
				return
			}
			for object := range c.api.ListObjects(bucket.Stat.Name, o, true) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     probe.NewError(object.Err),
					}
					return
				}
				content := new(client.Content)
				content.Name = filepath.Join(bucket.Stat.Name, object.Stat.Key)
				content.Size = object.Stat.Size
				content.Time = object.Stat.LastModified
				content.Type = os.FileMode(0664)
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	default:
		for object := range c.api.ListObjects(b, o, true) {
			if object.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     probe.NewError(object.Err),
				}
				return
			}
			content := new(client.Content)
			normalizedKey := object.Stat.Key
			switch {
			case o == "":
				// if no prefix provided and also URL is not delimited then we add bucket back into object name
				if strings.LastIndex(c.hostURL.Path, string(c.hostURL.Separator)) == 0 {
					if c.hostURL.String()[:strings.LastIndex(c.hostURL.String(), string(c.hostURL.Separator))+1] != b {
						normalizedKey = filepath.Join(b, object.Stat.Key)
					}
				}
			default:
				if strings.HasSuffix(o, string(c.hostURL.Separator)) {
					normalizedKey = strings.TrimPrefix(object.Stat.Key, o)
				}
			}
			content.Name = normalizedKey
			content.Size = object.Stat.Size
			content.Time = object.Stat.LastModified
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	}
}
Beispiel #7
0
func (c *s3Client) listInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     probe.NewError(bucket.Err),
				}
				return
			}
			content := new(client.Content)
			content.Name = bucket.Stat.Name
			content.Size = 0
			content.Time = bucket.Stat.CreationDate
			content.Type = os.ModeDir
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	default:
		metadata, err := c.api.StatObject(b, o)
		switch err.(type) {
		case nil:
			content := new(client.Content)
			content.Name = metadata.Key
			content.Time = metadata.LastModified
			content.Size = metadata.Size
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		default:
			for object := range c.api.ListObjects(b, o, false) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     probe.NewError(object.Err),
					}
					return
				}
				content := new(client.Content)
				normalizedPrefix := strings.TrimSuffix(o, string(c.hostURL.Separator)) + string(c.hostURL.Separator)
				normalizedKey := object.Stat.Key
				if normalizedPrefix != object.Stat.Key && strings.HasPrefix(object.Stat.Key, normalizedPrefix) {
					normalizedKey = strings.TrimPrefix(object.Stat.Key, normalizedPrefix)
				}
				content.Name = normalizedKey
				switch {
				case strings.HasSuffix(object.Stat.Key, string(c.hostURL.Separator)):
					content.Time = time.Now()
					content.Type = os.ModeDir
				default:
					content.Size = object.Stat.Size
					content.Time = object.Stat.LastModified
					content.Type = os.FileMode(0664)
				}
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	}
}
Beispiel #8
0
Datei: s3.go Projekt: fwessels/mc
func (c *s3Client) listRecursiveInRoutine(contentCh chan *client.Content) {
	defer close(contentCh)
	// get bucket and object from URL.
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(bucket.Err),
				}
				return
			}
			bucketURL := *c.hostURL
			bucketURL.Path = filepath.Join(bucketURL.Path, bucket.Name)
			contentCh <- &client.Content{
				URL:  bucketURL,
				Type: os.ModeDir,
				Time: bucket.CreationDate,
			}
			for object := range c.api.ListObjects(bucket.Name, o, true) {
				if object.Err != nil {
					contentCh <- &client.Content{
						Err: probe.NewError(object.Err),
					}
					continue
				}
				content := new(client.Content)
				objectURL := *c.hostURL
				objectURL.Path = filepath.Join(objectURL.Path, bucket.Name, object.Key)
				content.URL = objectURL
				content.Size = object.Size
				content.Time = object.LastModified
				content.Type = os.FileMode(0664)
				contentCh <- content
			}
		}
	default:
		for object := range c.api.ListObjects(b, o, true) {
			if object.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(object.Err),
				}
				continue
			}
			content := new(client.Content)
			url := *c.hostURL
			// Join bucket and incoming object key.
			url.Path = filepath.Join(string(url.Separator), b, object.Key)
			// If virtualStyle replace the url.Path back.
			if c.virtualStyle {
				url.Path = filepath.Join(string(url.Separator), object.Key)
			}
			content.URL = url
			content.Size = object.Size
			content.Time = object.LastModified
			content.Type = os.FileMode(0664)
			contentCh <- content
		}
	}
}
Beispiel #9
0
Datei: s3.go Projekt: fwessels/mc
func (c *s3Client) listInRoutine(contentCh chan *client.Content) {
	defer close(contentCh)
	// get bucket and object from URL.
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(bucket.Err),
				}
				return
			}
			url := *c.hostURL
			url.Path = filepath.Join(url.Path, bucket.Name)
			content := new(client.Content)
			content.URL = url
			content.Size = 0
			content.Time = bucket.CreationDate
			content.Type = os.ModeDir
			contentCh <- content
		}
	case b != "" && !strings.HasSuffix(c.hostURL.Path, string(c.hostURL.Separator)) && o == "":
		e := c.api.BucketExists(b)
		if e != nil {
			contentCh <- &client.Content{
				Err: probe.NewError(e),
			}
		}
		content := new(client.Content)
		content.URL = *c.hostURL
		content.Type = os.ModeDir
		contentCh <- content
	default:
		metadata, e := c.api.StatObject(b, o)
		switch e.(type) {
		case nil:
			content := new(client.Content)
			content.URL = *c.hostURL
			content.Time = metadata.LastModified
			content.Size = metadata.Size
			content.Type = os.FileMode(0664)
			contentCh <- content
		default:
			for object := range c.api.ListObjects(b, o, false) {
				if object.Err != nil {
					contentCh <- &client.Content{
						Err: probe.NewError(object.Err),
					}
					return
				}
				content := new(client.Content)
				url := *c.hostURL
				// Join bucket and incoming object key.
				url.Path = filepath.Join(string(url.Separator), b, object.Key)
				if c.virtualStyle {
					url.Path = filepath.Join(string(url.Separator), object.Key)
				}
				switch {
				case strings.HasSuffix(object.Key, string(c.hostURL.Separator)):
					// We need to keep the trailing Separator, do not use filepath.Join().
					content.URL = url
					content.Time = time.Now()
					content.Type = os.ModeDir
				default:
					content.URL = url
					content.Size = object.Size
					content.Time = object.LastModified
					content.Type = os.FileMode(0664)
				}
				contentCh <- content
			}
		}
	}
}
Beispiel #10
0
Datei: s3.go Projekt: fwessels/mc
func (c *s3Client) listIncompleteInRoutine(contentCh chan *client.Content) {
	defer close(contentCh)
	// get bucket and object from URL.
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(bucket.Err),
				}
				return
			}
			for object := range c.api.ListIncompleteUploads(bucket.Name, o, false) {
				if object.Err != nil {
					contentCh <- &client.Content{
						Err: probe.NewError(object.Err),
					}
					return
				}
				content := new(client.Content)
				url := *c.hostURL
				// Join bucket with - incoming object key.
				url.Path = filepath.Join(string(url.Separator), bucket.Name, object.Key)
				if c.virtualStyle {
					url.Path = filepath.Join(string(url.Separator), object.Key)
				}
				switch {
				case strings.HasSuffix(object.Key, string(c.hostURL.Separator)):
					// We need to keep the trailing Separator, do not use filepath.Join().
					content.URL = url
					content.Time = time.Now()
					content.Type = os.ModeDir
				default:
					content.URL = url
					content.Size = object.Size
					content.Time = object.Initiated
					content.Type = os.ModeTemporary
				}
				contentCh <- content
			}
		}
	default:
		for object := range c.api.ListIncompleteUploads(b, o, false) {
			if object.Err != nil {
				contentCh <- &client.Content{
					Err: probe.NewError(object.Err),
				}
				return
			}
			content := new(client.Content)
			url := *c.hostURL
			// Join bucket with - incoming object key.
			url.Path = filepath.Join(string(url.Separator), b, object.Key)
			if c.virtualStyle {
				url.Path = filepath.Join(string(url.Separator), object.Key)
			}
			switch {
			case strings.HasSuffix(object.Key, string(c.hostURL.Separator)):
				// We need to keep the trailing Separator, do not use filepath.Join().
				content.URL = url
				content.Time = time.Now()
				content.Type = os.ModeDir
			default:
				content.URL = url
				content.Size = object.Size
				content.Time = object.Initiated
				content.Type = os.ModeTemporary
			}
			contentCh <- content
		}
	}
}
Beispiel #11
0
Datei: s3.go Projekt: fwessels/mc
// Stat - send a 'HEAD' on a bucket or object to fetch its metadata.
func (c *s3Client) Stat() (*client.Content, *probe.Error) {
	c.mu.Lock()
	objectMetadata := new(client.Content)
	bucket, object := c.url2BucketAndObject()
	switch {
	// valid case for 'ls -r s3/'
	case bucket == "" && object == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				c.mu.Unlock()
				return nil, probe.NewError(bucket.Err)
			}
		}
		c.mu.Unlock()
		return &client.Content{URL: *c.hostURL, Type: os.ModeDir}, nil
	}
	if object != "" {
		metadata, e := c.api.StatObject(bucket, object)
		if e != nil {
			c.mu.Unlock()
			errResponse := minio.ToErrorResponse(e)
			if errResponse != nil {
				if errResponse.Code == "NoSuchKey" {
					// Append "/" to the object name proactively and see if the Listing
					// produces an output. If yes, then we treat it as a directory.
					prefixName := object
					// Trim any trailing separators and add it.
					prefixName = strings.TrimSuffix(prefixName, string(c.hostURL.Separator)) + string(c.hostURL.Separator)
					for objectStat := range c.api.ListObjects(bucket, prefixName, false) {
						if objectStat.Err != nil {
							return nil, probe.NewError(objectStat.Err)
						}
						content := client.Content{}
						content.URL = *c.hostURL
						content.Type = os.ModeDir
						return &content, nil
					}
					return nil, probe.NewError(client.PathNotFound{Path: c.hostURL.Path})
				}
			}
			return nil, probe.NewError(e)
		}
		objectMetadata.URL = *c.hostURL
		objectMetadata.Time = metadata.LastModified
		objectMetadata.Size = metadata.Size
		objectMetadata.Type = os.FileMode(0664)
		c.mu.Unlock()
		return objectMetadata, nil
	}
	e := c.api.BucketExists(bucket)
	if e != nil {
		c.mu.Unlock()
		return nil, probe.NewError(e)
	}
	bucketMetadata := new(client.Content)
	bucketMetadata.URL = *c.hostURL
	bucketMetadata.Type = os.ModeDir
	c.mu.Unlock()
	return bucketMetadata, nil
}
Beispiel #12
0
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
	mirrorURLsCh := make(chan mirrorURLs)
	go func() {
		defer close(mirrorURLsCh)
		sourceTrie := patricia.NewTrie()
		targetTries := make([]*patricia.Trie, len(targetClnts))
		wg := new(sync.WaitGroup)

		wg.Add(1)
		go func() {
			defer wg.Done()
			for sourceContentCh := range sourceClnt.List(true) {
				if sourceContentCh.Err != nil {
					mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
					return
				}
				if sourceContentCh.Content.Type.IsRegular() {
					sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
				}
			}
		}()

		for i, targetClnt := range targetClnts {
			wg.Add(1)
			go func(i int, targetClnt client.Client) {
				defer wg.Done()
				targetTrie := patricia.NewTrie()
				for targetContentCh := range targetClnt.List(true) {
					if targetContentCh.Err != nil {
						mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
						return
					}
					if targetContentCh.Content.Type.IsRegular() {
						targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
					}
				}
				targetTries[i] = targetTrie
			}(i, targetClnt)
		}
		wg.Wait()

		matchNameCh := make(chan string, 10000)
		go func(matchNameCh chan<- string) {
			itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
				matchNameCh <- string(prefix)
				return nil
			}
			sourceTrie.Visit(itemFunc)
			defer close(matchNameCh)
		}(matchNameCh)
		for matchName := range matchNameCh {
			sourceContent := new(client.Content)
			var targetContents []*client.Content
			for i, targetTrie := range targetTries {
				if !targetTrie.Match(patricia.Prefix(matchName)) {
					sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
						string(sourceClnt.URL().Separator))+1]
					newTargetURLParse := *targetClnts[i].URL()
					newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
					sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
					sourceContent.Name = sourceURLDelimited + matchName
					targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
				}
			}
			mirrorURLsCh <- mirrorURLs{
				SourceContent:  sourceContent,
				TargetContents: targetContents,
			}
		}
	}()
	return mirrorURLsCh
}
Beispiel #13
0
func (c *s3Client) listRecursiveInRoutine(contentCh chan client.ContentOnChannel) {
	defer close(contentCh)
	// get bucket and object from URL
	b, o := c.url2BucketAndObject()
	switch {
	case b == "" && o == "":
		for bucket := range c.api.ListBuckets() {
			if bucket.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     probe.NewError(bucket.Err),
				}
				return
			}
			for object := range c.api.ListObjects(bucket.Stat.Name, o, true) {
				if object.Err != nil {
					contentCh <- client.ContentOnChannel{
						Content: nil,
						Err:     probe.NewError(object.Err),
					}
					return
				}
				content := new(client.Content)
				url := *c.hostURL
				url.Path = filepath.Join(url.Path, bucket.Stat.Name, object.Stat.Key)
				content.URL = url
				content.Size = object.Stat.Size
				content.Time = object.Stat.LastModified
				content.Type = os.FileMode(0664)
				contentCh <- client.ContentOnChannel{
					Content: content,
					Err:     nil,
				}
			}
		}
	default:
		for object := range c.api.ListObjects(b, o, true) {
			if object.Err != nil {
				contentCh <- client.ContentOnChannel{
					Content: nil,
					Err:     probe.NewError(object.Err),
				}
				return
			}
			content := new(client.Content)
			url := *c.hostURL
			// join bucket and incoming object key.
			url.Path = filepath.Join(string(url.Separator), b, object.Stat.Key)
			// if virtualStyle replace the url.Path back.
			if c.virtualStyle {
				url.Path = filepath.Join(string(url.Separator), object.Stat.Key)
			}
			content.URL = url
			content.Size = object.Stat.Size
			content.Time = object.Stat.LastModified
			content.Type = os.FileMode(0664)
			contentCh <- client.ContentOnChannel{
				Content: content,
				Err:     nil,
			}
		}
	}
}