// doList - list all entities inside a folder
func doList(clnt client.Client, recursive bool) error {
var err error
for contentCh := range clnt.List(recursive) {
if contentCh.Err != nil {
switch err := iodine.ToError(contentCh.Err).(type) {
// handle this specifically for filesystem
case client.ISBrokenSymlink:
console.Errors(ErrorMessage{
Message: "Failed with",
Error: iodine.New(err, nil),
})
continue
}
if os.IsNotExist(iodine.ToError(contentCh.Err)) || os.IsPermission(iodine.ToError(contentCh.Err)) {
console.Errors(ErrorMessage{
Message: "Failed with",
Error: iodine.New(contentCh.Err, nil),
})
continue
}
err = contentCh.Err
break
}
console.Prints(parseContent(contentCh.Content))
}
if err != nil {
return iodine.New(err, map[string]string{"Target": clnt.URL().String()})
}
return nil
}
// mainList - is a handler for mc ls command
func mainList(ctx *cli.Context) {
checkListSyntax(ctx)
args := ctx.Args()
// Operating system tool behavior
if globalMimicFlag && !ctx.Args().Present() {
args = []string{"."}
}
console.SetCustomTheme(map[string]*color.Color{
"File": color.New(color.FgWhite),
"Dir": color.New(color.FgCyan, color.Bold),
"Size": color.New(color.FgYellow),
"Time": color.New(color.FgGreen),
})
targetURLs, err := args2URLs(args)
fatalIf(err.Trace(args...), "One or more unknown URL types passed.")
for _, targetURL := range targetURLs {
// if recursive strip off the "..."
var clnt client.Client
clnt, err = target2Client(stripRecursiveURL(targetURL))
fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")
err = doList(clnt, isURLRecursive(targetURL), len(targetURLs) > 1)
fatalIf(err.Trace(clnt.URL().String()), "Unable to list target ‘"+clnt.URL().String()+"’.")
}
}
// doMakeBucket - wrapper around MakeBucket() API
func doMakeBucket(clnt client.Client) (string, error) {
err := clnt.MakeBucket()
if err != nil {
msg := fmt.Sprintf("Failed to create bucket for URL ‘%s’", clnt.URL().String())
return msg, iodine.New(err, nil)
}
return "Bucket created successfully : " + clnt.URL().String(), nil
}
func doUpdateAccess(clnt client.Client, targetACL bucketACL) (string, error) {
err := clnt.SetBucketACL(targetACL.String())
if err != nil {
msg := fmt.Sprintf("Failed to add bucket access policy for URL ‘%s’", clnt.URL().String())
return msg, iodine.New(err, nil)
}
return "Bucket access policy updated successfully : " + clnt.URL().String(), nil
}
func dodiff(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
for contentCh := range firstClnt.List(false) {
if contentCh.Err != nil {
ch <- DiffMessage{
Error: contentCh.Err.Trace(firstClnt.URL().String()),
}
return
}
newFirstURL, err := urlJoinPath(firstClnt.URL().String(), contentCh.Content.Name)
if err != nil {
ch <- DiffMessage{
Error: err.Trace(firstClnt.URL().String()),
}
return
}
newSecondURL, err := urlJoinPath(secondClnt.URL().String(), contentCh.Content.Name)
if err != nil {
ch <- DiffMessage{
Error: err.Trace(secondClnt.URL().String()),
}
return
}
_, newFirstContent, errFirst := url2Stat(newFirstURL)
_, newSecondContent, errSecond := url2Stat(newSecondURL)
switch {
case errFirst == nil && errSecond != nil:
ch <- DiffMessage{
FirstURL: newFirstURL,
SecondURL: newSecondURL,
Diff: "only-in-first",
}
continue
case errFirst == nil && errSecond == nil:
switch {
case newFirstContent.Type.IsDir():
if !newSecondContent.Type.IsDir() {
ch <- DiffMessage{
FirstURL: newFirstURL,
SecondURL: newSecondURL,
Diff: "type",
}
}
continue
case newFirstContent.Type.IsRegular():
if !newSecondContent.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: newFirstURL,
SecondURL: newSecondURL,
Diff: "type",
}
continue
}
doDiffObjects(newFirstURL, newSecondURL, ch)
}
}
} // End of for-loop
}
// doShareURL share files from target
func doShareURL(targetURL string, recursive bool, expires time.Duration) *probe.Error {
shareDate := time.Now().UTC()
sURLs, err := loadSharedURLsV2()
if err != nil {
return err.Trace()
}
var clnt client.Client
clnt, err = target2Client(targetURL)
if err != nil {
return err.Trace()
}
if expires.Seconds() < 1 {
return probe.NewError(errors.New("Too low expires, expiration cannot be less than 1 second."))
}
if expires.Seconds() > 604800 {
return probe.NewError(errors.New("Too high expires, expiration cannot be larger than 7 days."))
}
for contentCh := range clnt.List(recursive) {
if contentCh.Err != nil {
return contentCh.Err.Trace()
}
var newClnt client.Client
newClnt, err = url2Client(getNewTargetURL(clnt.URL(), contentCh.Content.Name))
if err != nil {
return err.Trace()
}
var sharedURL string
sharedURL, err = newClnt.Share(expires)
if err != nil {
return err.Trace()
}
shareMessage := ShareMessage{
Expiry: expires,
URL: sharedURL,
Key: newClnt.URL().String(),
}
sURLs.URLs = append(sURLs.URLs, struct {
Date time.Time
Message ShareMessage
}{
Date: shareDate,
Message: shareMessage,
})
Prints("%s\n", shareMessage)
}
saveSharedURLsV2(sURLs)
return nil
}
// doListIncomplete - list all incomplete uploads entities inside a folder.
func doListIncomplete(clnt client.Client, recursive, multipleArgs bool) *probe.Error {
var err *probe.Error
var parentContent *client.Content
parentContent, err = clnt.Stat()
if err != nil {
return err.Trace(clnt.URL().String())
}
for contentCh := range clnt.List(recursive, true) {
if contentCh.Err != nil {
switch contentCh.Err.ToGoError().(type) {
// handle this specifically for filesystem
case client.BrokenSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
continue
case client.TooManyLevelsSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
continue
}
if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
if contentCh.Content != nil {
if contentCh.Content.Type.IsDir() && (contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink) {
errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
continue
}
}
errorIf(contentCh.Err.Trace(), "Unable to list.")
continue
}
err = contentCh.Err.Trace()
break
}
if multipleArgs && parentContent.Type.IsDir() {
contentCh.Content.Name = filepath.Join(parentContent.Name, strings.TrimPrefix(contentCh.Content.Name, parentContent.Name))
}
Prints("%s\n", parseContent(contentCh.Content))
}
if err != nil {
return err.Trace()
}
return nil
}
// mainList - is a handler for mc ls command
func mainList(ctx *cli.Context) {
checkListSyntax(ctx)
args := ctx.Args()
// Operating system tool behavior
if globalMimicFlag && !ctx.Args().Present() {
args = []string{"."}
}
setListPalette(ctx.GlobalString("colors"))
targetURLs, err := args2URLs(args)
fatalIf(err.Trace(args...), "One or more unknown URL types passed.")
for _, targetURL := range targetURLs {
// if recursive strip off the "..."
var clnt client.Client
clnt, err = url2Client(stripRecursiveURL(targetURL))
fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")
err = doList(clnt, isURLRecursive(targetURL), len(targetURLs) > 1)
fatalIf(err.Trace(clnt.URL().String()), "Unable to list target ‘"+clnt.URL().String()+"’.")
}
}
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
firstTrie := patricia.NewTrie()
secondTrie := patricia.NewTrie()
wg := new(sync.WaitGroup)
type urlAttr struct {
Size int64
Type os.FileMode
}
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for firstContentCh := range firstClnt.List(true) {
if firstContentCh.Err != nil {
ch <- DiffMessage{
Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
}
return
}
firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
}
}(ch)
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for secondContentCh := range secondClnt.List(true) {
if secondContentCh.Err != nil {
ch <- DiffMessage{
Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
}
return
}
secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
}
}(ch)
doneCh := make(chan struct{})
defer close(doneCh)
go func(doneCh <-chan struct{}) {
cursorCh := cursorAnimate()
for {
select {
case <-time.Tick(100 * time.Millisecond):
if !globalQuietFlag && !globalJSONFlag {
console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
}
case <-doneCh:
return
}
}
}(doneCh)
wg.Wait()
doneCh <- struct{}{}
if !globalQuietFlag && !globalJSONFlag {
console.Printf("%c[2K\n", 27)
console.Printf("%c[A", 27)
}
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
firstTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
firstURL := firstURLDelimited + matchName
secondURL := secondURLDelimited + matchName
if !secondTrie.Match(patricia.Prefix(matchName)) {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
} else {
firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)
if firstURLAttr.Type.IsRegular() {
if !secondURLAttr.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Type.IsDir() {
if !secondURLAttr.Type.IsDir() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Size != secondURLAttr.Size {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "size",
}
}
}
}
}
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
mirrorURLsCh := make(chan mirrorURLs)
go func() {
defer close(mirrorURLsCh)
id := newRandomID(8)
doneCh := make(chan bool)
defer close(doneCh)
go func(doneCh <-chan bool) {
cursorCh := cursorAnimate()
for {
select {
case <-time.Tick(100 * time.Millisecond):
if !globalQuietFlag && !globalJSONFlag {
console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
}
case <-doneCh:
return
}
}
}(doneCh)
sourceSortedList := sortedList{}
targetSortedList := make([]*sortedList, len(targetClnts))
surldelimited := sourceClnt.URL().String()
err := sourceSortedList.Create(sourceClnt, id+".src")
if err != nil {
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
return
}
turldelimited := make([]string, len(targetClnts))
for i := range targetClnts {
turldelimited[i] = targetClnts[i].URL().String()
targetSortedList[i] = &sortedList{}
err := targetSortedList[i].Create(targetClnts[i], id+"."+strconv.Itoa(i))
if err != nil {
// FIXME: do cleanup by calling Delete()
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
return
}
}
for source := range sourceSortedList.List(true) {
if source.Content.Type.IsDir() {
continue
}
targetContents := make([]*client.Content, 0, len(targetClnts))
for i, t := range targetSortedList {
match, err := t.Match(source.Content)
if err != nil || match {
// continue on io.EOF or if the keys matches
// FIXME: handle other errors and ignore this target for future calls
continue
}
targetContents = append(targetContents, &client.Content{Name: turldelimited[i] + source.Content.Name})
}
source.Content.Name = surldelimited + source.Content.Name
if len(targetContents) > 0 {
mirrorURLsCh <- mirrorURLs{
SourceContent: source.Content,
TargetContents: targetContents,
}
}
}
if err := sourceSortedList.Delete(); err != nil {
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
}
for _, t := range targetSortedList {
if err := t.Delete(); err != nil {
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
}
}
doneCh <- true
if !globalQuietFlag && !globalJSONFlag {
console.Eraseline()
}
}()
return mirrorURLsCh
}
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
mirrorURLsCh := make(chan mirrorURLs)
go func() {
defer close(mirrorURLsCh)
sourceTrie := patricia.NewTrie()
targetTries := make([]*patricia.Trie, len(targetClnts))
wg := new(sync.WaitGroup)
wg.Add(1)
go func() {
defer wg.Done()
for sourceContentCh := range sourceClnt.List(true) {
if sourceContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
return
}
if sourceContentCh.Content.Type.IsRegular() {
sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
}
}
}()
for i, targetClnt := range targetClnts {
wg.Add(1)
go func(i int, targetClnt client.Client) {
defer wg.Done()
targetTrie := patricia.NewTrie()
for targetContentCh := range targetClnt.List(true) {
if targetContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
return
}
if targetContentCh.Content.Type.IsRegular() {
targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
}
}
targetTries[i] = targetTrie
}(i, targetClnt)
}
wg.Wait()
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
sourceTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
sourceContent := new(client.Content)
var targetContents []*client.Content
for i, targetTrie := range targetTries {
if !targetTrie.Match(patricia.Prefix(matchName)) {
sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
string(sourceClnt.URL().Separator))+1]
newTargetURLParse := *targetClnts[i].URL()
newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
sourceContent.Name = sourceURLDelimited + matchName
targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
}
}
mirrorURLsCh <- mirrorURLs{
SourceContent: sourceContent,
TargetContents: targetContents,
}
}
}()
return mirrorURLsCh
}
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
firstURLDelimited := firstClnt.URL().String()
secondURLDelimited := secondClnt.URL().String()
if strings.HasSuffix(firstURLDelimited, "/") == false {
firstURLDelimited = firstURLDelimited + "/"
}
if strings.HasSuffix(secondURLDelimited, "/") == false {
secondURLDelimited = secondURLDelimited + "/"
}
firstClnt, err := url2Client(firstURLDelimited)
if err != nil {
ch <- DiffMessage{Error: err.Trace()}
return
}
secondClnt, err = url2Client(secondURLDelimited)
if err != nil {
ch <- DiffMessage{Error: err.Trace()}
return
}
fch := firstClnt.List(true, false)
sch := secondClnt.List(true, false)
f, fok := <-fch
s, sok := <-sch
for {
if fok == false {
break
}
if f.Err != nil {
ch <- DiffMessage{Error: f.Err.Trace()}
continue
}
if f.Content.Type.IsDir() {
// skip directories
// there is no concept of directories on S3
f, fok = <-fch
continue
}
firstURL := firstURLDelimited + f.Content.Name
secondURL := secondURLDelimited + f.Content.Name
if sok == false {
// Second list reached EOF
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
f, fok = <-fch
continue
}
if s.Err != nil {
ch <- DiffMessage{Error: s.Err.Trace()}
continue
}
if s.Content.Type.IsDir() {
// skip directories
s, sok = <-sch
continue
}
fC := f.Content
sC := s.Content
if fC.Name == sC.Name {
if fC.Type.IsRegular() {
if !sC.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
}
} else if fC.Type.IsDir() {
if !sC.Type.IsDir() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
}
} else if fC.Size != sC.Size {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "size",
}
}
f, fok = <-fch
s, sok = <-sch
}
if fC.Name < sC.Name {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
f, fok = <-fch
}
if fC.Name > sC.Name {
s, sok = <-sch
}
}
}