// doList - list all entities inside a folder
func doList(clnt client.Client, recursive bool) error {
var err error
for contentCh := range clnt.List(recursive) {
if contentCh.Err != nil {
switch err := iodine.ToError(contentCh.Err).(type) {
// handle this specifically for filesystem
case client.ISBrokenSymlink:
console.Errors(ErrorMessage{
Message: "Failed with",
Error: iodine.New(err, nil),
})
continue
}
if os.IsNotExist(iodine.ToError(contentCh.Err)) || os.IsPermission(iodine.ToError(contentCh.Err)) {
console.Errors(ErrorMessage{
Message: "Failed with",
Error: iodine.New(contentCh.Err, nil),
})
continue
}
err = contentCh.Err
break
}
console.Prints(parseContent(contentCh.Content))
}
if err != nil {
return iodine.New(err, map[string]string{"Target": clnt.URL().String()})
}
return nil
}
// doList - list all entities inside a folder.
func doList(clnt client.Client, isRecursive, isIncomplete bool) *probe.Error {
prefixPath := clnt.GetURL().Path
separator := string(clnt.GetURL().Separator)
if !strings.HasSuffix(prefixPath, separator) {
prefixPath = prefixPath[:strings.LastIndex(prefixPath, separator)+1]
}
for content := range clnt.List(isRecursive, isIncomplete) {
if content.Err != nil {
switch content.Err.ToGoError().(type) {
// handle this specifically for filesystem related errors.
case client.BrokenSymlink:
errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list broken link.")
continue
case client.TooManyLevelsSymlink:
errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list too many levels link.")
continue
case client.PathNotFound:
errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
continue
case client.PathInsufficientPermission:
errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
continue
}
errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
continue
}
contentURL := content.URL.Path
contentURL = strings.TrimPrefix(contentURL, prefixPath)
content.URL.Path = contentURL
parsedContent := parseContent(content)
// print colorized or jsonized content info.
printMsg(parsedContent)
}
return nil
}
// diffFolders - diff of contents of two folders only top level content.
//
// 3: diff(d1, d2) -> []diff(d1/f, d2/f) -> VALID
func diffFolders(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) {
recursive := false
// Range on the List to consume incoming content
for contentCh := range firstClnt.List(recursive, false) {
if contentCh.Err != nil {
outCh <- diffMessage{
Error: contentCh.Err.Trace(firstClnt.GetURL().String()),
}
continue
}
// Store incoming content
newFirstContent := contentCh.Content
newFirstURLStr := newFirstContent.URL.String()
// Construct the second URL.
newSecondURL := secondClnt.GetURL()
// Need to verify the same path from first URL, construct the second URL
newSecondURL.Path = filepath.Join(newSecondURL.Path, filepath.Base(contentCh.Content.URL.Path))
newSecondURLStr := newSecondURL.String()
// Send a stat to verify
_, newSecondContent, err := url2Stat(newSecondURLStr)
if err != nil {
outCh <- diffMessage{
FirstURL: newFirstURLStr,
SecondURL: newSecondURLStr,
Diff: "only-in-first",
}
continue
}
diffMsg := diffObjects(newFirstContent, newSecondContent)
if diffMsg != nil {
outCh <- *diffMsg
continue
}
} // Reached EOF
}
func dodiff(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
for contentCh := range firstClnt.List(false) {
if contentCh.Err != nil {
ch <- DiffMessage{
Error: contentCh.Err.Trace(firstClnt.URL().String()),
}
return
}
newFirstURL, err := urlJoinPath(firstClnt.URL().String(), contentCh.Content.Name)
if err != nil {
ch <- DiffMessage{
Error: err.Trace(firstClnt.URL().String()),
}
return
}
newSecondURL, err := urlJoinPath(secondClnt.URL().String(), contentCh.Content.Name)
if err != nil {
ch <- DiffMessage{
Error: err.Trace(secondClnt.URL().String()),
}
return
}
_, newFirstContent, errFirst := url2Stat(newFirstURL)
_, newSecondContent, errSecond := url2Stat(newSecondURL)
switch {
case errFirst == nil && errSecond != nil:
ch <- DiffMessage{
FirstURL: newFirstURL,
SecondURL: newSecondURL,
Diff: "only-in-first",
}
continue
case errFirst == nil && errSecond == nil:
switch {
case newFirstContent.Type.IsDir():
if !newSecondContent.Type.IsDir() {
ch <- DiffMessage{
FirstURL: newFirstURL,
SecondURL: newSecondURL,
Diff: "type",
}
}
continue
case newFirstContent.Type.IsRegular():
if !newSecondContent.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: newFirstURL,
SecondURL: newSecondURL,
Diff: "type",
}
continue
}
doDiffObjects(newFirstURL, newSecondURL, ch)
}
}
} // End of for-loop
}
// doShareURL share files from target
func doShareDownloadURL(targetURL string, recursive bool, expires time.Duration) *probe.Error {
shareDate := time.Now().UTC()
sURLs, err := loadSharedURLsV3()
if err != nil {
return err.Trace()
}
var clnt client.Client
clnt, err = url2Client(targetURL)
if err != nil {
return err.Trace()
}
if expires.Seconds() < 1 {
return probe.NewError(errors.New("Too low expires, expiration cannot be less than 1 second."))
}
if expires.Seconds() > 604800 {
return probe.NewError(errors.New("Too high expires, expiration cannot be larger than 7 days."))
}
for contentCh := range clnt.List(recursive, false) {
if contentCh.Err != nil {
return contentCh.Err.Trace()
}
var newClnt client.Client
newClnt, err = url2Client(getNewTargetURL(clnt.URL(), contentCh.Content.Name))
if err != nil {
return err.Trace()
}
var sharedURL string
sharedURL, err = newClnt.ShareDownload(expires)
if err != nil {
return err.Trace()
}
shareMessage := ShareMessage{
Expiry: expires,
DownloadURL: sharedURL,
Key: newClnt.URL().String(),
}
shareMessageV3 := ShareMessageV3{
Expiry: expires,
DownloadURL: sharedURL,
Key: newClnt.URL().String(),
}
sURLs.URLs = append(sURLs.URLs, struct {
Date time.Time
Message ShareMessageV3
}{
Date: shareDate,
Message: shareMessageV3,
})
Prints("%s\n", shareMessage)
}
saveSharedURLsV3(sURLs)
return nil
}
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive, multipleArgs bool) *probe.Error {
var err *probe.Error
var parentContent *client.Content
urlStr := clnt.URL().String()
parentDir := url2Dir(urlStr)
parentClnt, err := url2Client(parentDir)
if err != nil {
return err.Trace(clnt.URL().String())
}
parentContent, err = parentClnt.Stat()
if err != nil {
return err.Trace(clnt.URL().String())
}
for contentCh := range clnt.List(recursive, false) {
if contentCh.Err != nil {
switch contentCh.Err.ToGoError().(type) {
// handle this specifically for filesystem
case client.BrokenSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
continue
case client.TooManyLevelsSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
continue
}
if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
if contentCh.Content != nil {
if contentCh.Content.Type.IsDir() {
if contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink {
errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
continue
}
errorIf(contentCh.Err.Trace(), "Unable to list folder.")
}
} else {
errorIf(contentCh.Err.Trace(), "Unable to list.")
continue
}
}
err = contentCh.Err.Trace()
break
}
if multipleArgs && parentContent.Type.IsDir() {
contentCh.Content.Name = filepath.Join(parentContent.Name, strings.TrimPrefix(contentCh.Content.Name, parentContent.Name))
}
Prints("%s\n", parseContent(contentCh.Content))
}
if err != nil {
return err.Trace()
}
return nil
}
// Create create an on disk sorted file from clnt
func (sl *sortedList) Create(clnt client.Client, id string) *probe.Error {
var e error
if err := createSortedListDir(); err != nil {
return err.Trace()
}
sortedListDir, err := getSortedListDir()
if err != nil {
return err.Trace()
}
sl.name = filepath.Join(sortedListDir, id)
sl.file, e = os.OpenFile(sl.name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
if e != nil {
return probe.NewError(e)
}
sl.enc = gob.NewEncoder(sl.file)
sl.dec = gob.NewDecoder(sl.file)
for content := range clnt.List(true) {
if content.Err != nil {
switch err := content.Err.ToGoError().(type) {
case client.BrokenSymlink:
// FIXME: send the error to caller using channel
errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken symlink ‘%s’.", err.Path))
continue
case client.TooManyLevelsSymlink:
// FIXME: send the error to caller using channel
errorIf(content.Err.Trace(), fmt.Sprintf("Skipping too many levels symlink ‘%s’.", err.Path))
continue
}
if os.IsNotExist(content.Err.ToGoError()) || os.IsPermission(content.Err.ToGoError()) {
// FIXME: abstract this at fs.go layer
if content.Content != nil {
if content.Content.Type.IsDir() && (content.Content.Type&os.ModeSymlink == os.ModeSymlink) {
errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken folder symlink ‘%s’.", content.Content.Name))
continue
}
errorIf(content.Err.Trace(), fmt.Sprintf("Skipping ‘%s’.", content.Content.Name))
continue
}
errorIf(content.Err.Trace(), "Skipping unknown file.")
continue
}
return content.Err.Trace()
}
sl.enc.Encode(*content.Content)
}
if _, err := sl.file.Seek(0, os.SEEK_SET); err != nil {
return probe.NewError(err)
}
return nil
}
// doList - list all entities inside a folder.
func doList(clnt client.Client, isRecursive, isIncomplete bool) *probe.Error {
_, parentContent, err := url2Stat(clnt.GetURL().String())
if err != nil {
return err.Trace(clnt.GetURL().String())
}
for contentCh := range clnt.List(isRecursive, isIncomplete) {
if contentCh.Err != nil {
switch contentCh.Err.ToGoError().(type) {
// handle this specifically for filesystem
case client.BrokenSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
continue
case client.TooManyLevelsSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
continue
}
if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
if contentCh.Content != nil {
if contentCh.Content.Type.IsDir() {
if contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink {
errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
continue
}
errorIf(contentCh.Err.Trace(), "Unable to list folder.")
}
} else {
errorIf(contentCh.Err.Trace(), "Unable to list.")
continue
}
}
err = contentCh.Err.Trace()
break
}
trimmedContent := trimContent(parentContent, contentCh.Content, isRecursive)
parsedContent := parseContent(trimmedContent)
printMsg(parsedContent)
}
if err != nil {
return err.Trace()
}
return nil
}
// diffFoldersRecursive diff folders for all files recursively.
//
// 4: diff(d1..., d2) -> []diff(d1/f, d2/f) -> VALID.
func diffFoldersRecursive(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) {
var scanBar scanBarFunc
if !globalQuietFlag && !globalJSONFlag { // set up progress bar
scanBar = scanBarFactory()
}
recursive := true
firstListCh := firstClnt.List(recursive, false) // Copy first list channel.
for firstContentCh := range firstListCh {
if firstContentCh.Err != nil {
outCh <- diffMessage{Error: firstContentCh.Err.Trace()}
continue
}
if firstContentCh.Content.Type.IsDir() {
// Skip directories there is no concept of directories on S3.
continue
}
firstContent := firstContentCh.Content
secondURL := secondClnt.GetURL()
secondURL.Path = filepath.Join(secondURL.Path,
strings.TrimPrefix(firstContent.URL.Path, url2Dir(firstClnt.GetURL().Path)))
_, secondContent, err := url2Stat(secondURL.String())
if err != nil {
outCh <- diffMessage{
FirstURL: firstContent.URL.String(),
SecondURL: secondURL.String(),
Diff: "only-in-first",
}
continue
}
if diffMsg := diffObjects(firstContent, secondContent); diffMsg != nil {
outCh <- *diffMsg
continue
}
if !globalQuietFlag && !globalJSONFlag { // set up progress bar
scanBar(firstContent.URL.String())
}
}
}
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive bool) *probe.Error {
var err *probe.Error
for contentCh := range clnt.List(recursive) {
if contentCh.Err != nil {
switch contentCh.Err.ToGoError().(type) {
// handle this specifically for filesystem
case client.ISBrokenSymlink:
errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
continue
}
if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
errorIf(contentCh.Err.Trace(), "Unable to list.")
continue
}
err = contentCh.Err.Trace()
break
}
console.Println(parseContent(contentCh.Content))
}
if err != nil {
return err.Trace()
}
return nil
}
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
firstTrie := patricia.NewTrie()
secondTrie := patricia.NewTrie()
wg := new(sync.WaitGroup)
type urlAttr struct {
Size int64
Type os.FileMode
}
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for firstContentCh := range firstClnt.List(true) {
if firstContentCh.Err != nil {
ch <- DiffMessage{
Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
}
return
}
firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
}
}(ch)
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for secondContentCh := range secondClnt.List(true) {
if secondContentCh.Err != nil {
ch <- DiffMessage{
Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
}
return
}
secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
}
}(ch)
doneCh := make(chan struct{})
defer close(doneCh)
go func(doneCh <-chan struct{}) {
cursorCh := cursorAnimate()
for {
select {
case <-time.Tick(100 * time.Millisecond):
if !globalQuietFlag && !globalJSONFlag {
console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
}
case <-doneCh:
return
}
}
}(doneCh)
wg.Wait()
doneCh <- struct{}{}
if !globalQuietFlag && !globalJSONFlag {
console.Printf("%c[2K\n", 27)
console.Printf("%c[A", 27)
}
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
firstTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
firstURL := firstURLDelimited + matchName
secondURL := secondURLDelimited + matchName
if !secondTrie.Match(patricia.Prefix(matchName)) {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
} else {
firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)
if firstURLAttr.Type.IsRegular() {
if !secondURLAttr.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Type.IsDir() {
if !secondURLAttr.Type.IsDir() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Size != secondURLAttr.Size {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "size",
}
}
}
}
}
func dodiffdirs(firstClnt client.Client, firstURL, secondURL string, recursive bool, ch chan diff) {
for contentCh := range firstClnt.List(recursive) {
if contentCh.Err != nil {
ch <- diff{
message: "Failed to list ‘" + firstURL + "’",
err: iodine.New(contentCh.Err, nil),
}
return
}
newFirstURL, err := urlJoinPath(firstURL, contentCh.Content.Name)
if err != nil {
ch <- diff{
message: "Unable to construct new URL from ‘" + firstURL + "’ using ‘" + contentCh.Content.Name + "’",
err: iodine.New(err, nil),
}
return
}
newSecondURL, err := urlJoinPath(secondURL, contentCh.Content.Name)
if err != nil {
ch <- diff{
message: "Unable to construct new URL from ‘" + secondURL + "’ using ‘" + contentCh.Content.Name + "’",
err: iodine.New(err, nil),
}
return
}
_, newFirstContent, errFirst := url2Stat(newFirstURL)
_, newSecondContent, errSecond := url2Stat(newSecondURL)
switch {
case errFirst != nil && errSecond == nil:
ch <- diff{
message: "‘" + newSecondURL + "’ Only in ‘" + secondURL + "’",
err: nil,
}
continue
case errFirst == nil && errSecond != nil:
ch <- diff{
message: "‘" + newFirstURL + "’ Only in ‘" + firstURL + "’",
err: nil,
}
continue
case errFirst == nil && errSecond == nil:
switch {
case newFirstContent.Type.IsDir():
if !newSecondContent.Type.IsDir() {
ch <- diff{
message: newFirstURL + " and " + newSecondURL + " differs in type.",
err: nil,
}
}
continue
case newFirstContent.Type.IsRegular():
if !newSecondContent.Type.IsRegular() {
ch <- diff{
message: newFirstURL + " and " + newSecondURL + " differs in type.",
err: nil,
}
continue
}
doDiffObjects(newFirstURL, newSecondURL, ch)
}
}
} // End of for-loop
}
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
mirrorURLsCh := make(chan mirrorURLs)
go func() {
defer close(mirrorURLsCh)
sourceTrie := patricia.NewTrie()
targetTries := make([]*patricia.Trie, len(targetClnts))
wg := new(sync.WaitGroup)
wg.Add(1)
go func() {
defer wg.Done()
for sourceContentCh := range sourceClnt.List(true) {
if sourceContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
return
}
if sourceContentCh.Content.Type.IsRegular() {
sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
}
}
}()
for i, targetClnt := range targetClnts {
wg.Add(1)
go func(i int, targetClnt client.Client) {
defer wg.Done()
targetTrie := patricia.NewTrie()
for targetContentCh := range targetClnt.List(true) {
if targetContentCh.Err != nil {
mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
return
}
if targetContentCh.Content.Type.IsRegular() {
targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
}
}
targetTries[i] = targetTrie
}(i, targetClnt)
}
wg.Wait()
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
sourceTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
sourceContent := new(client.Content)
var targetContents []*client.Content
for i, targetTrie := range targetTries {
if !targetTrie.Match(patricia.Prefix(matchName)) {
sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
string(sourceClnt.URL().Separator))+1]
newTargetURLParse := *targetClnts[i].URL()
newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
sourceContent.Name = sourceURLDelimited + matchName
targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
}
}
mirrorURLsCh <- mirrorURLs{
SourceContent: sourceContent,
TargetContents: targetContents,
}
}
}()
return mirrorURLsCh
}
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
firstURLDelimited := firstClnt.URL().String()
secondURLDelimited := secondClnt.URL().String()
if strings.HasSuffix(firstURLDelimited, "/") == false {
firstURLDelimited = firstURLDelimited + "/"
}
if strings.HasSuffix(secondURLDelimited, "/") == false {
secondURLDelimited = secondURLDelimited + "/"
}
firstClnt, err := url2Client(firstURLDelimited)
if err != nil {
ch <- DiffMessage{Error: err.Trace()}
return
}
secondClnt, err = url2Client(secondURLDelimited)
if err != nil {
ch <- DiffMessage{Error: err.Trace()}
return
}
fch := firstClnt.List(true, false)
sch := secondClnt.List(true, false)
f, fok := <-fch
s, sok := <-sch
for {
if fok == false {
break
}
if f.Err != nil {
ch <- DiffMessage{Error: f.Err.Trace()}
continue
}
if f.Content.Type.IsDir() {
// skip directories
// there is no concept of directories on S3
f, fok = <-fch
continue
}
firstURL := firstURLDelimited + f.Content.Name
secondURL := secondURLDelimited + f.Content.Name
if sok == false {
// Second list reached EOF
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
f, fok = <-fch
continue
}
if s.Err != nil {
ch <- DiffMessage{Error: s.Err.Trace()}
continue
}
if s.Content.Type.IsDir() {
// skip directories
s, sok = <-sch
continue
}
fC := f.Content
sC := s.Content
if fC.Name == sC.Name {
if fC.Type.IsRegular() {
if !sC.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
}
} else if fC.Type.IsDir() {
if !sC.Type.IsDir() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
}
} else if fC.Size != sC.Size {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "size",
}
}
f, fok = <-fch
s, sok = <-sch
}
if fC.Name < sC.Name {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
f, fok = <-fch
}
if fC.Name > sC.Name {
s, sok = <-sch
}
}
}