Example #1
0
File: ls.go Project: bosky101/mc
// doList - list all entities inside a folder
func doList(clnt client.Client, recursive bool) error {
	var err error
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			switch err := iodine.ToError(contentCh.Err).(type) {
			// handle this specifically for filesystem
			case client.ISBrokenSymlink:
				console.Errors(ErrorMessage{
					Message: "Failed with",
					Error:   iodine.New(err, nil),
				})
				continue
			}
			if os.IsNotExist(iodine.ToError(contentCh.Err)) || os.IsPermission(iodine.ToError(contentCh.Err)) {
				console.Errors(ErrorMessage{
					Message: "Failed with",
					Error:   iodine.New(contentCh.Err, nil),
				})
				continue
			}
			err = contentCh.Err
			break
		}
		console.Prints(parseContent(contentCh.Content))
	}
	if err != nil {
		return iodine.New(err, map[string]string{"Target": clnt.URL().String()})
	}
	return nil
}
Example #2
0
File: ls.go Project: fwessels/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, isRecursive, isIncomplete bool) *probe.Error {
	prefixPath := clnt.GetURL().Path
	separator := string(clnt.GetURL().Separator)
	if !strings.HasSuffix(prefixPath, separator) {
		prefixPath = prefixPath[:strings.LastIndex(prefixPath, separator)+1]
	}
	for content := range clnt.List(isRecursive, isIncomplete) {
		if content.Err != nil {
			switch content.Err.ToGoError().(type) {
			// handle this specifically for filesystem related errors.
			case client.BrokenSymlink:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list too many levels link.")
				continue
			case client.PathNotFound:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
				continue
			case client.PathInsufficientPermission:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
				continue
			}
			errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
			continue
		}
		contentURL := content.URL.Path
		contentURL = strings.TrimPrefix(contentURL, prefixPath)
		content.URL.Path = contentURL
		parsedContent := parseContent(content)
		// print colorized or jsonized content info.
		printMsg(parsedContent)
	}
	return nil
}
Example #3
0
// diffFolders - diff of contents of two folders only top level content.
//
// 3: diff(d1, d2) -> []diff(d1/f, d2/f) -> VALID
func diffFolders(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) {
	recursive := false
	// Range on the List to consume incoming content
	for contentCh := range firstClnt.List(recursive, false) {
		if contentCh.Err != nil {
			outCh <- diffMessage{
				Error: contentCh.Err.Trace(firstClnt.GetURL().String()),
			}
			continue
		}
		// Store incoming content
		newFirstContent := contentCh.Content
		newFirstURLStr := newFirstContent.URL.String()
		// Construct the second URL.
		newSecondURL := secondClnt.GetURL()
		// Need to verify the same path from first URL, construct the second URL
		newSecondURL.Path = filepath.Join(newSecondURL.Path, filepath.Base(contentCh.Content.URL.Path))
		newSecondURLStr := newSecondURL.String()
		// Send a stat to verify
		_, newSecondContent, err := url2Stat(newSecondURLStr)
		if err != nil {
			outCh <- diffMessage{
				FirstURL:  newFirstURLStr,
				SecondURL: newSecondURLStr,
				Diff:      "only-in-first",
			}
			continue
		}
		diffMsg := diffObjects(newFirstContent, newSecondContent)
		if diffMsg != nil {
			outCh <- *diffMsg
			continue
		}
	} // Reached EOF
}
Example #4
0
func dodiff(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	for contentCh := range firstClnt.List(false) {
		if contentCh.Err != nil {
			ch <- DiffMessage{
				Error: contentCh.Err.Trace(firstClnt.URL().String()),
			}
			return
		}
		newFirstURL, err := urlJoinPath(firstClnt.URL().String(), contentCh.Content.Name)
		if err != nil {
			ch <- DiffMessage{
				Error: err.Trace(firstClnt.URL().String()),
			}
			return
		}
		newSecondURL, err := urlJoinPath(secondClnt.URL().String(), contentCh.Content.Name)
		if err != nil {
			ch <- DiffMessage{
				Error: err.Trace(secondClnt.URL().String()),
			}
			return
		}
		_, newFirstContent, errFirst := url2Stat(newFirstURL)
		_, newSecondContent, errSecond := url2Stat(newSecondURL)
		switch {
		case errFirst == nil && errSecond != nil:
			ch <- DiffMessage{
				FirstURL:  newFirstURL,
				SecondURL: newSecondURL,
				Diff:      "only-in-first",
			}
			continue
		case errFirst == nil && errSecond == nil:
			switch {
			case newFirstContent.Type.IsDir():
				if !newSecondContent.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  newFirstURL,
						SecondURL: newSecondURL,
						Diff:      "type",
					}
				}
				continue
			case newFirstContent.Type.IsRegular():
				if !newSecondContent.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  newFirstURL,
						SecondURL: newSecondURL,
						Diff:      "type",
					}
					continue
				}
				doDiffObjects(newFirstURL, newSecondURL, ch)
			}
		}
	} // End of for-loop
}
Example #5
0
// doShareURL share files from target
func doShareDownloadURL(targetURL string, recursive bool, expires time.Duration) *probe.Error {
	shareDate := time.Now().UTC()
	sURLs, err := loadSharedURLsV3()
	if err != nil {
		return err.Trace()
	}
	var clnt client.Client
	clnt, err = url2Client(targetURL)
	if err != nil {
		return err.Trace()
	}
	if expires.Seconds() < 1 {
		return probe.NewError(errors.New("Too low expires, expiration cannot be less than 1 second."))
	}
	if expires.Seconds() > 604800 {
		return probe.NewError(errors.New("Too high expires, expiration cannot be larger than 7 days."))
	}
	for contentCh := range clnt.List(recursive, false) {
		if contentCh.Err != nil {
			return contentCh.Err.Trace()
		}
		var newClnt client.Client
		newClnt, err = url2Client(getNewTargetURL(clnt.URL(), contentCh.Content.Name))
		if err != nil {
			return err.Trace()
		}
		var sharedURL string
		sharedURL, err = newClnt.ShareDownload(expires)
		if err != nil {
			return err.Trace()
		}
		shareMessage := ShareMessage{
			Expiry:      expires,
			DownloadURL: sharedURL,
			Key:         newClnt.URL().String(),
		}
		shareMessageV3 := ShareMessageV3{
			Expiry:      expires,
			DownloadURL: sharedURL,
			Key:         newClnt.URL().String(),
		}
		sURLs.URLs = append(sURLs.URLs, struct {
			Date    time.Time
			Message ShareMessageV3
		}{
			Date:    shareDate,
			Message: shareMessageV3,
		})
		Prints("%s\n", shareMessage)
	}
	saveSharedURLsV3(sURLs)
	return nil
}
Example #6
0
File: ls.go Project: henrylee2cn/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive, multipleArgs bool) *probe.Error {
	var err *probe.Error
	var parentContent *client.Content
	urlStr := clnt.URL().String()
	parentDir := url2Dir(urlStr)
	parentClnt, err := url2Client(parentDir)
	if err != nil {
		return err.Trace(clnt.URL().String())
	}
	parentContent, err = parentClnt.Stat()
	if err != nil {
		return err.Trace(clnt.URL().String())
	}
	for contentCh := range clnt.List(recursive, false) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.BrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				if contentCh.Content != nil {
					if contentCh.Content.Type.IsDir() {
						if contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink {
							errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
							continue
						}
						errorIf(contentCh.Err.Trace(), "Unable to list folder.")
					}
				} else {
					errorIf(contentCh.Err.Trace(), "Unable to list.")
					continue
				}
			}
			err = contentCh.Err.Trace()
			break
		}
		if multipleArgs && parentContent.Type.IsDir() {
			contentCh.Content.Name = filepath.Join(parentContent.Name, strings.TrimPrefix(contentCh.Content.Name, parentContent.Name))
		}
		Prints("%s\n", parseContent(contentCh.Content))
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #7
0
// Create create an on disk sorted file from clnt
func (sl *sortedList) Create(clnt client.Client, id string) *probe.Error {
	var e error
	if err := createSortedListDir(); err != nil {
		return err.Trace()
	}
	sortedListDir, err := getSortedListDir()
	if err != nil {
		return err.Trace()
	}
	sl.name = filepath.Join(sortedListDir, id)
	sl.file, e = os.OpenFile(sl.name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
	if e != nil {
		return probe.NewError(e)
	}
	sl.enc = gob.NewEncoder(sl.file)
	sl.dec = gob.NewDecoder(sl.file)
	for content := range clnt.List(true) {
		if content.Err != nil {
			switch err := content.Err.ToGoError().(type) {
			case client.BrokenSymlink:
				// FIXME: send the error to caller using channel
				errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken symlink ‘%s’.", err.Path))
				continue
			case client.TooManyLevelsSymlink:
				// FIXME: send the error to caller using channel
				errorIf(content.Err.Trace(), fmt.Sprintf("Skipping too many levels symlink ‘%s’.", err.Path))
				continue
			}
			if os.IsNotExist(content.Err.ToGoError()) || os.IsPermission(content.Err.ToGoError()) {
				// FIXME: abstract this at fs.go layer
				if content.Content != nil {
					if content.Content.Type.IsDir() && (content.Content.Type&os.ModeSymlink == os.ModeSymlink) {
						errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken folder symlink ‘%s’.", content.Content.Name))
						continue
					}
					errorIf(content.Err.Trace(), fmt.Sprintf("Skipping ‘%s’.", content.Content.Name))
					continue
				}
				errorIf(content.Err.Trace(), "Skipping unknown file.")
				continue
			}
			return content.Err.Trace()
		}
		sl.enc.Encode(*content.Content)
	}
	if _, err := sl.file.Seek(0, os.SEEK_SET); err != nil {
		return probe.NewError(err)
	}
	return nil
}
Example #8
0
File: ls.go Project: koolhead17/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, isRecursive, isIncomplete bool) *probe.Error {
	_, parentContent, err := url2Stat(clnt.GetURL().String())
	if err != nil {
		return err.Trace(clnt.GetURL().String())
	}

	for contentCh := range clnt.List(isRecursive, isIncomplete) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.BrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				if contentCh.Content != nil {
					if contentCh.Content.Type.IsDir() {
						if contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink {
							errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
							continue
						}
						errorIf(contentCh.Err.Trace(), "Unable to list folder.")
					}
				} else {
					errorIf(contentCh.Err.Trace(), "Unable to list.")
					continue
				}
			}
			err = contentCh.Err.Trace()
			break
		}
		trimmedContent := trimContent(parentContent, contentCh.Content, isRecursive)
		parsedContent := parseContent(trimmedContent)
		printMsg(parsedContent)
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #9
0
// diffFoldersRecursive diff folders for all files recursively.
//
// 4: diff(d1..., d2) -> []diff(d1/f, d2/f) -> VALID.
func diffFoldersRecursive(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) {
	var scanBar scanBarFunc
	if !globalQuietFlag && !globalJSONFlag { // set up progress bar
		scanBar = scanBarFactory()
	}
	recursive := true
	firstListCh := firstClnt.List(recursive, false) // Copy first list channel.
	for firstContentCh := range firstListCh {
		if firstContentCh.Err != nil {
			outCh <- diffMessage{Error: firstContentCh.Err.Trace()}
			continue
		}
		if firstContentCh.Content.Type.IsDir() {
			// Skip directories there is no concept of directories on S3.
			continue
		}
		firstContent := firstContentCh.Content
		secondURL := secondClnt.GetURL()
		secondURL.Path = filepath.Join(secondURL.Path,
			strings.TrimPrefix(firstContent.URL.Path, url2Dir(firstClnt.GetURL().Path)))
		_, secondContent, err := url2Stat(secondURL.String())
		if err != nil {
			outCh <- diffMessage{
				FirstURL:  firstContent.URL.String(),
				SecondURL: secondURL.String(),
				Diff:      "only-in-first",
			}
			continue
		}
		if diffMsg := diffObjects(firstContent, secondContent); diffMsg != nil {
			outCh <- *diffMsg
			continue
		}
		if !globalQuietFlag && !globalJSONFlag { // set up progress bar
			scanBar(firstContent.URL.String())
		}
	}
}
Example #10
0
File: ls.go Project: axcoto-lab/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive bool) *probe.Error {
	var err *probe.Error
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.ISBrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				errorIf(contentCh.Err.Trace(), "Unable to list.")
				continue
			}
			err = contentCh.Err.Trace()
			break
		}
		console.Println(parseContent(contentCh.Content))
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #11
0
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	firstTrie := patricia.NewTrie()
	secondTrie := patricia.NewTrie()
	wg := new(sync.WaitGroup)

	type urlAttr struct {
		Size int64
		Type os.FileMode
	}

	wg.Add(1)
	go func(ch chan<- DiffMessage) {
		defer wg.Done()
		for firstContentCh := range firstClnt.List(true) {
			if firstContentCh.Err != nil {
				ch <- DiffMessage{
					Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
				}
				return
			}
			firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
		}
	}(ch)
	wg.Add(1)
	go func(ch chan<- DiffMessage) {
		defer wg.Done()
		for secondContentCh := range secondClnt.List(true) {
			if secondContentCh.Err != nil {
				ch <- DiffMessage{
					Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
				}
				return
			}
			secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
		}
	}(ch)

	doneCh := make(chan struct{})
	defer close(doneCh)
	go func(doneCh <-chan struct{}) {
		cursorCh := cursorAnimate()
		for {
			select {
			case <-time.Tick(100 * time.Millisecond):
				if !globalQuietFlag && !globalJSONFlag {
					console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
				}
			case <-doneCh:
				return
			}
		}
	}(doneCh)
	wg.Wait()
	doneCh <- struct{}{}
	if !globalQuietFlag && !globalJSONFlag {
		console.Printf("%c[2K\n", 27)
		console.Printf("%c[A", 27)
	}

	matchNameCh := make(chan string, 10000)
	go func(matchNameCh chan<- string) {
		itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
			matchNameCh <- string(prefix)
			return nil
		}
		firstTrie.Visit(itemFunc)
		defer close(matchNameCh)
	}(matchNameCh)
	for matchName := range matchNameCh {
		firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
		secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
		firstURL := firstURLDelimited + matchName
		secondURL := secondURLDelimited + matchName
		if !secondTrie.Match(patricia.Prefix(matchName)) {
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
		} else {
			firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
			secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)

			if firstURLAttr.Type.IsRegular() {
				if !secondURLAttr.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
					continue
				}
			}

			if firstURLAttr.Type.IsDir() {
				if !secondURLAttr.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
					continue
				}
			}

			if firstURLAttr.Size != secondURLAttr.Size {
				ch <- DiffMessage{
					FirstURL:  firstURL,
					SecondURL: secondURL,
					Diff:      "size",
				}
			}
		}
	}
}
Example #12
0
File: diff.go Project: bosky101/mc
func dodiffdirs(firstClnt client.Client, firstURL, secondURL string, recursive bool, ch chan diff) {
	for contentCh := range firstClnt.List(recursive) {
		if contentCh.Err != nil {
			ch <- diff{
				message: "Failed to list ‘" + firstURL + "’",
				err:     iodine.New(contentCh.Err, nil),
			}
			return
		}
		newFirstURL, err := urlJoinPath(firstURL, contentCh.Content.Name)
		if err != nil {
			ch <- diff{
				message: "Unable to construct new URL from ‘" + firstURL + "’ using ‘" + contentCh.Content.Name + "’",
				err:     iodine.New(err, nil),
			}
			return
		}
		newSecondURL, err := urlJoinPath(secondURL, contentCh.Content.Name)
		if err != nil {
			ch <- diff{
				message: "Unable to construct new URL from ‘" + secondURL + "’ using ‘" + contentCh.Content.Name + "’",
				err:     iodine.New(err, nil),
			}
			return
		}
		_, newFirstContent, errFirst := url2Stat(newFirstURL)
		_, newSecondContent, errSecond := url2Stat(newSecondURL)
		switch {
		case errFirst != nil && errSecond == nil:
			ch <- diff{
				message: "‘" + newSecondURL + "’ Only in ‘" + secondURL + "’",
				err:     nil,
			}
			continue
		case errFirst == nil && errSecond != nil:
			ch <- diff{
				message: "‘" + newFirstURL + "’ Only in ‘" + firstURL + "’",
				err:     nil,
			}
			continue
		case errFirst == nil && errSecond == nil:
			switch {
			case newFirstContent.Type.IsDir():
				if !newSecondContent.Type.IsDir() {
					ch <- diff{
						message: newFirstURL + " and " + newSecondURL + " differs in type.",
						err:     nil,
					}
				}
				continue
			case newFirstContent.Type.IsRegular():
				if !newSecondContent.Type.IsRegular() {
					ch <- diff{
						message: newFirstURL + " and " + newSecondURL + " differs in type.",
						err:     nil,
					}
					continue
				}
				doDiffObjects(newFirstURL, newSecondURL, ch)
			}
		}
	} // End of for-loop
}
Example #13
0
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
	mirrorURLsCh := make(chan mirrorURLs)
	go func() {
		defer close(mirrorURLsCh)
		sourceTrie := patricia.NewTrie()
		targetTries := make([]*patricia.Trie, len(targetClnts))
		wg := new(sync.WaitGroup)

		wg.Add(1)
		go func() {
			defer wg.Done()
			for sourceContentCh := range sourceClnt.List(true) {
				if sourceContentCh.Err != nil {
					mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
					return
				}
				if sourceContentCh.Content.Type.IsRegular() {
					sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
				}
			}
		}()

		for i, targetClnt := range targetClnts {
			wg.Add(1)
			go func(i int, targetClnt client.Client) {
				defer wg.Done()
				targetTrie := patricia.NewTrie()
				for targetContentCh := range targetClnt.List(true) {
					if targetContentCh.Err != nil {
						mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
						return
					}
					if targetContentCh.Content.Type.IsRegular() {
						targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
					}
				}
				targetTries[i] = targetTrie
			}(i, targetClnt)
		}
		wg.Wait()

		matchNameCh := make(chan string, 10000)
		go func(matchNameCh chan<- string) {
			itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
				matchNameCh <- string(prefix)
				return nil
			}
			sourceTrie.Visit(itemFunc)
			defer close(matchNameCh)
		}(matchNameCh)
		for matchName := range matchNameCh {
			sourceContent := new(client.Content)
			var targetContents []*client.Content
			for i, targetTrie := range targetTries {
				if !targetTrie.Match(patricia.Prefix(matchName)) {
					sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
						string(sourceClnt.URL().Separator))+1]
					newTargetURLParse := *targetClnts[i].URL()
					newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
					sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
					sourceContent.Name = sourceURLDelimited + matchName
					targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
				}
			}
			mirrorURLsCh <- mirrorURLs{
				SourceContent:  sourceContent,
				TargetContents: targetContents,
			}
		}
	}()
	return mirrorURLsCh
}
Example #14
0
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	firstURLDelimited := firstClnt.URL().String()
	secondURLDelimited := secondClnt.URL().String()
	if strings.HasSuffix(firstURLDelimited, "/") == false {
		firstURLDelimited = firstURLDelimited + "/"
	}
	if strings.HasSuffix(secondURLDelimited, "/") == false {
		secondURLDelimited = secondURLDelimited + "/"
	}
	firstClnt, err := url2Client(firstURLDelimited)
	if err != nil {
		ch <- DiffMessage{Error: err.Trace()}
		return
	}
	secondClnt, err = url2Client(secondURLDelimited)
	if err != nil {
		ch <- DiffMessage{Error: err.Trace()}
		return
	}

	fch := firstClnt.List(true, false)
	sch := secondClnt.List(true, false)
	f, fok := <-fch
	s, sok := <-sch
	for {
		if fok == false {
			break
		}
		if f.Err != nil {
			ch <- DiffMessage{Error: f.Err.Trace()}
			continue
		}
		if f.Content.Type.IsDir() {
			// skip directories
			// there is no concept of directories on S3
			f, fok = <-fch
			continue
		}
		firstURL := firstURLDelimited + f.Content.Name
		secondURL := secondURLDelimited + f.Content.Name
		if sok == false {
			// Second list reached EOF
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
			f, fok = <-fch
			continue
		}
		if s.Err != nil {
			ch <- DiffMessage{Error: s.Err.Trace()}
			continue
		}
		if s.Content.Type.IsDir() {
			// skip directories
			s, sok = <-sch
			continue
		}
		fC := f.Content
		sC := s.Content
		if fC.Name == sC.Name {
			if fC.Type.IsRegular() {
				if !sC.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
				}
			} else if fC.Type.IsDir() {
				if !sC.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
				}
			} else if fC.Size != sC.Size {
				ch <- DiffMessage{
					FirstURL:  firstURL,
					SecondURL: secondURL,
					Diff:      "size",
				}
			}
			f, fok = <-fch
			s, sok = <-sch
		}
		if fC.Name < sC.Name {
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
			f, fok = <-fch
		}
		if fC.Name > sC.Name {
			s, sok = <-sch
		}
	}
}