// parseContent parse client Content container into printer struct
func parseContent(c *client.Content) Content {
content := Content{}
content.Time = c.Time.Local().Format(printDate)
// guess file type
content.Filetype = func() string {
if c.Type.IsDir() {
return "directory"
}
return "file"
}()
content.Size = humanize.IBytes(uint64(c.Size))
// Convert OS Type to match console file printing style
content.Name = func() string {
switch {
case runtime.GOOS == "windows":
c.Name = strings.Replace(c.Name, "/", "\\", -1)
c.Name = strings.TrimSuffix(c.Name, "\\")
default:
c.Name = strings.TrimSuffix(c.Name, "/")
}
if c.Type.IsDir() {
switch {
case runtime.GOOS == "windows":
return fmt.Sprintf("%s\\", c.Name)
default:
return fmt.Sprintf("%s/", c.Name)
}
}
return c.Name
}()
return content
}
// parseContent parse client Content container into printer struct.
func parseContent(c *client.Content) ContentMessage {
content := ContentMessage{}
content.Time = c.Time.Local()
// guess file type
content.Filetype = func() string {
if c.Type.IsDir() {
return "folder"
}
return "file"
}()
content.Size = c.Size
// Convert OS Type to match console file printing style.
content.Name = func() string {
switch {
case runtime.GOOS == "windows":
c.Name = strings.Replace(c.Name, "/", "\\", -1)
c.Name = strings.TrimSuffix(c.Name, "\\")
default:
c.Name = strings.TrimSuffix(c.Name, "/")
}
if c.Type.IsDir() {
switch {
case runtime.GOOS == "windows":
return fmt.Sprintf("%s\\", c.Name)
default:
return fmt.Sprintf("%s/", c.Name)
}
}
return c.Name
}()
return content
}
func (c *s3Client) listIncompleteInRoutine(contentCh chan client.ContentOnChannel) {
defer close(contentCh)
b, o := c.url2BucketAndObject()
switch {
case b == "" && o == "":
for bucket := range c.api.ListBuckets() {
if bucket.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: probe.NewError(bucket.Err),
}
return
}
content := new(client.Content)
content.Name = bucket.Stat.Name
content.Size = 0
content.Time = bucket.Stat.CreationDate
content.Type = os.ModeDir
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
}
default:
for object := range c.api.ListIncompleteUploads(b, o, false) {
if object.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: probe.NewError(object.Err),
}
return
}
content := new(client.Content)
normalizedPrefix := strings.TrimSuffix(o, string(c.hostURL.Separator)) + string(c.hostURL.Separator)
normalizedKey := object.Stat.Key
if normalizedPrefix != object.Stat.Key && strings.HasPrefix(object.Stat.Key, normalizedPrefix) {
normalizedKey = strings.TrimPrefix(object.Stat.Key, normalizedPrefix)
}
content.Name = normalizedKey
switch {
case strings.HasSuffix(object.Stat.Key, string(c.hostURL.Separator)):
content.Time = time.Now()
content.Type = os.ModeDir
default:
content.Size = object.Stat.Size
content.Time = object.Stat.Initiated
content.Type = os.ModeTemporary
}
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
}
}
}
// Stat - send a 'HEAD' on a bucket or object to get its metadata
func (c *s3Client) Stat() (*client.Content, *probe.Error) {
objectMetadata := new(client.Content)
bucket, object := c.url2BucketAndObject()
switch {
// valid case for s3/...
case bucket == "" && object == "":
for bucket := range c.api.ListBuckets() {
if bucket.Err != nil {
return nil, probe.NewError(bucket.Err)
}
}
return &client.Content{Type: os.ModeDir}, nil
}
if object != "" {
metadata, err := c.api.StatObject(bucket, object)
if err != nil {
errResponse := minio.ToErrorResponse(err)
if errResponse != nil {
if errResponse.Code == "NoSuchKey" {
for content := range c.List(false) {
if content.Err != nil {
return nil, content.Err.Trace()
}
content.Content.Type = os.ModeDir
content.Content.Name = object
content.Content.Size = 0
return content.Content, nil
}
}
}
return nil, probe.NewError(err)
}
objectMetadata.Name = metadata.Key
objectMetadata.Time = metadata.LastModified
objectMetadata.Size = metadata.Size
objectMetadata.Type = os.FileMode(0664)
return objectMetadata, nil
}
err := c.api.BucketExists(bucket)
if err != nil {
return nil, probe.NewError(err)
}
bucketMetadata := new(client.Content)
bucketMetadata.Name = bucket
bucketMetadata.Type = os.ModeDir
return bucketMetadata, nil
}
func (c *s3Client) listRecursiveInRoutine(contentCh chan client.ContentOnChannel) {
defer close(contentCh)
b, o := c.url2BucketAndObject()
switch {
case b == "" && o == "":
for bucket := range c.api.ListBuckets() {
if bucket.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: probe.NewError(bucket.Err),
}
return
}
for object := range c.api.ListObjects(bucket.Stat.Name, o, true) {
if object.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: probe.NewError(object.Err),
}
return
}
content := new(client.Content)
content.Name = filepath.Join(bucket.Stat.Name, object.Stat.Key)
content.Size = object.Stat.Size
content.Time = object.Stat.LastModified
content.Type = os.FileMode(0664)
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
}
}
default:
for object := range c.api.ListObjects(b, o, true) {
if object.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: probe.NewError(object.Err),
}
return
}
content := new(client.Content)
normalizedKey := object.Stat.Key
switch {
case o == "":
// if no prefix provided and also URL is not delimited then we add bucket back into object name
if strings.LastIndex(c.hostURL.Path, string(c.hostURL.Separator)) == 0 {
if c.hostURL.String()[:strings.LastIndex(c.hostURL.String(), string(c.hostURL.Separator))+1] != b {
normalizedKey = filepath.Join(b, object.Stat.Key)
}
}
default:
if strings.HasSuffix(o, string(c.hostURL.Separator)) {
normalizedKey = strings.TrimPrefix(object.Stat.Key, o)
}
}
content.Name = normalizedKey
content.Size = object.Stat.Size
content.Time = object.Stat.LastModified
content.Type = os.FileMode(0664)
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
}
}
}
func (c *s3Client) listInRoutine(contentCh chan client.ContentOnChannel) {
defer close(contentCh)
b, o := c.url2BucketAndObject()
switch {
case b == "" && o == "":
for bucket := range c.api.ListBuckets() {
if bucket.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: bucket.Err,
}
return
}
content := new(client.Content)
content.Name = bucket.Stat.Name
content.Size = 0
content.Time = bucket.Stat.CreationDate
content.Type = os.ModeDir
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
}
default:
metadata, err := c.api.StatObject(b, o)
switch err.(type) {
case nil:
content := new(client.Content)
content.Name = metadata.Key
content.Time = metadata.LastModified
content.Size = metadata.Size
content.Type = os.FileMode(0664)
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
default:
for object := range c.api.ListObjects(b, o, false) {
if object.Err != nil {
contentCh <- client.ContentOnChannel{
Content: nil,
Err: object.Err,
}
return
}
content := new(client.Content)
normalizedPrefix := strings.TrimSuffix(o, string(c.hostURL.Separator)) + string(c.hostURL.Separator)
normalizedKey := object.Stat.Key
if normalizedPrefix != object.Stat.Key && strings.HasPrefix(object.Stat.Key, normalizedPrefix) {
normalizedKey = strings.TrimPrefix(object.Stat.Key, normalizedPrefix)
}
content.Name = normalizedKey
switch {
case strings.HasSuffix(object.Stat.Key, string(c.hostURL.Separator)):
content.Time = time.Now()
content.Type = os.ModeDir
default:
content.Size = object.Stat.Size
content.Time = object.Stat.LastModified
content.Type = os.FileMode(0664)
}
contentCh <- client.ContentOnChannel{
Content: content,
Err: nil,
}
}
}
}
}
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
mirrorURLsCh := make(chan mirrorURLs)
go func() {
defer close(mirrorURLsCh)
sourceTrie := patricia.NewTrie()
targetTries := make([]*patricia.Trie, len(targetClnts))
wg := new(sync.WaitGroup)
wg.Add(1)
go func() {
defer wg.Done()
for sourceContentCh := range sourceClnt.List(true) {
if sourceContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
return
}
if sourceContentCh.Content.Type.IsRegular() {
sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
}
}
}()
for i, targetClnt := range targetClnts {
wg.Add(1)
go func(i int, targetClnt client.Client) {
defer wg.Done()
targetTrie := patricia.NewTrie()
for targetContentCh := range targetClnt.List(true) {
if targetContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
return
}
if targetContentCh.Content.Type.IsRegular() {
targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
}
}
targetTries[i] = targetTrie
}(i, targetClnt)
}
wg.Wait()
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
sourceTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
sourceContent := new(client.Content)
var targetContents []*client.Content
for i, targetTrie := range targetTries {
if !targetTrie.Match(patricia.Prefix(matchName)) {
sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
string(sourceClnt.URL().Separator))+1]
newTargetURLParse := *targetClnts[i].URL()
newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
sourceContent.Name = sourceURLDelimited + matchName
targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
}
}
mirrorURLsCh <- mirrorURLs{
SourceContent: sourceContent,
TargetContents: targetContents,
}
}
}()
return mirrorURLsCh
}
