Example #1
0
// mainList - is a handler for mc ls command
func mainList(ctx *cli.Context) {
	// Additional command speific theme customization.
	console.SetColor("File", color.New(color.FgWhite))
	console.SetColor("Dir", color.New(color.FgCyan, color.Bold))
	console.SetColor("Size", color.New(color.FgYellow))
	console.SetColor("Time", color.New(color.FgGreen))

	// check 'ls' cli arguments
	checkListSyntax(ctx)

	args := ctx.Args()
	isIncomplete := ctx.Bool("incomplete")

	// mimic operating system tool behavior
	if globalMimicFlag && !ctx.Args().Present() {
		args = []string{"."}
	}

	targetURLs, err := args2URLs(args.Head())
	fatalIf(err.Trace(args...), "One or more unknown URL types passed.")
	for _, targetURL := range targetURLs {
		// if recursive strip off the "..."
		var clnt client.Client
		clnt, err = url2Client(stripRecursiveURL(targetURL))
		fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")

		err = doList(clnt, isURLRecursive(targetURL), isIncomplete)
		fatalIf(err.Trace(clnt.GetURL().String()), "Unable to list target ‘"+clnt.GetURL().String()+"’.")
	}
}
Example #2
0
// mainList - is a handler for mc ls command
func mainList(ctx *cli.Context) {
	// Additional command speific theme customization.
	console.SetColor("File", color.New(color.FgWhite))
	console.SetColor("Dir", color.New(color.FgCyan, color.Bold))
	console.SetColor("Size", color.New(color.FgYellow))
	console.SetColor("Time", color.New(color.FgGreen))

	// Set global flags from context.
	setGlobalsFromContext(ctx)

	// check 'ls' cli arguments.
	checkListSyntax(ctx)

	// Set command flags from context.
	isRecursive := ctx.Bool("recursive")
	isIncomplete := ctx.Bool("incomplete")

	args := ctx.Args()
	// mimic operating system tool behavior.
	if !ctx.Args().Present() {
		args = []string{"."}
	}

	for _, targetURL := range args {
		var clnt client.Client
		clnt, err := newClient(targetURL)
		fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")

		err = doList(clnt, isRecursive, isIncomplete)
		if err != nil {
			errorIf(err.Trace(clnt.GetURL().String()), "Unable to list target ‘"+clnt.GetURL().String()+"’.")
			continue
		}
	}
}
Example #3
0
// mainList - is a handler for mc ls command
func mainList(ctx *cli.Context) {
	checkListSyntax(ctx)

	args := ctx.Args()
	// Operating system tool behavior
	if globalMimicFlag && !ctx.Args().Present() {
		args = []string{"."}
	}

	console.SetCustomTheme(map[string]*color.Color{
		"File": color.New(color.FgWhite),
		"Dir":  color.New(color.FgCyan, color.Bold),
		"Size": color.New(color.FgYellow),
		"Time": color.New(color.FgGreen),
	})

	targetURLs, err := args2URLs(args)
	fatalIf(err.Trace(args...), "One or more unknown URL types passed.")

	for _, targetURL := range targetURLs {
		// if recursive strip off the "..."
		var clnt client.Client
		clnt, err = target2Client(stripRecursiveURL(targetURL))
		fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")

		err = doList(clnt, isURLRecursive(targetURL), len(targetURLs) > 1)
		fatalIf(err.Trace(clnt.URL().String()), "Unable to list target ‘"+clnt.URL().String()+"’.")
	}
}
Example #4
0
File: ls.go Project: bosky101/mc
// doList - list all entities inside a folder
func doList(clnt client.Client, recursive bool) error {
	var err error
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			switch err := iodine.ToError(contentCh.Err).(type) {
			// handle this specifically for filesystem
			case client.ISBrokenSymlink:
				console.Errors(ErrorMessage{
					Message: "Failed with",
					Error:   iodine.New(err, nil),
				})
				continue
			}
			if os.IsNotExist(iodine.ToError(contentCh.Err)) || os.IsPermission(iodine.ToError(contentCh.Err)) {
				console.Errors(ErrorMessage{
					Message: "Failed with",
					Error:   iodine.New(contentCh.Err, nil),
				})
				continue
			}
			err = contentCh.Err
			break
		}
		console.Prints(parseContent(contentCh.Content))
	}
	if err != nil {
		return iodine.New(err, map[string]string{"Target": clnt.URL().String()})
	}
	return nil
}
Example #5
0
func doUpdateAccessCmd(targetURL string, targetPERMS bucketPerms) *probe.Error {
	var clnt client.Client
	clnt, err := target2Client(targetURL)
	if err != nil {
		return err.Trace(targetURL)
	}
	if err = clnt.SetBucketACL(targetPERMS.String()); err != nil {
		return err.Trace(targetURL, targetPERMS.String())
	}
	return nil
}
Example #6
0
// Create create an on disk sorted file from clnt
func (sl *sortedList) Create(clnt client.Client, id string) *probe.Error {
	var e error
	if err := createSortedListDir(); err != nil {
		return err.Trace()
	}
	sortedListDir, err := getSortedListDir()
	if err != nil {
		return err.Trace()
	}
	sl.name = filepath.Join(sortedListDir, id)
	sl.file, e = os.OpenFile(sl.name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
	if e != nil {
		return probe.NewError(e)
	}
	sl.enc = gob.NewEncoder(sl.file)
	sl.dec = gob.NewDecoder(sl.file)
	for content := range clnt.List(true) {
		if content.Err != nil {
			switch err := content.Err.ToGoError().(type) {
			case client.BrokenSymlink:
				// FIXME: send the error to caller using channel
				errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken symlink ‘%s’.", err.Path))
				continue
			case client.TooManyLevelsSymlink:
				// FIXME: send the error to caller using channel
				errorIf(content.Err.Trace(), fmt.Sprintf("Skipping too many levels symlink ‘%s’.", err.Path))
				continue
			}
			if os.IsNotExist(content.Err.ToGoError()) || os.IsPermission(content.Err.ToGoError()) {
				// FIXME: abstract this at fs.go layer
				if content.Content != nil {
					if content.Content.Type.IsDir() && (content.Content.Type&os.ModeSymlink == os.ModeSymlink) {
						errorIf(content.Err.Trace(), fmt.Sprintf("Skipping broken folder symlink ‘%s’.", content.Content.Name))
						continue
					}
					errorIf(content.Err.Trace(), fmt.Sprintf("Skipping ‘%s’.", content.Content.Name))
					continue
				}
				errorIf(content.Err.Trace(), "Skipping unknown file.")
				continue
			}
			return content.Err.Trace()
		}
		sl.enc.Encode(*content.Content)
	}
	if _, err := sl.file.Seek(0, os.SEEK_SET); err != nil {
		return probe.NewError(err)
	}
	return nil
}
Example #7
0
// doShareURL share files from target
func doShareURL(targetURL string, recursive bool, expires time.Duration) *probe.Error {
	shareDate := time.Now().UTC()
	sURLs, err := loadSharedURLsV2()
	if err != nil {
		return err.Trace()
	}

	var clnt client.Client
	clnt, err = target2Client(targetURL)
	if err != nil {
		return err.Trace()
	}
	if expires.Seconds() < 1 {
		return probe.NewError(errors.New("Too low expires, expiration cannot be less than 1 second."))
	}
	if expires.Seconds() > 604800 {
		return probe.NewError(errors.New("Too high expires, expiration cannot be larger than 7 days."))
	}
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			return contentCh.Err.Trace()
		}
		var newClnt client.Client
		newClnt, err = url2Client(getNewTargetURL(clnt.URL(), contentCh.Content.Name))
		if err != nil {
			return err.Trace()
		}
		var sharedURL string
		sharedURL, err = newClnt.Share(expires)
		if err != nil {
			return err.Trace()
		}
		shareMessage := ShareMessage{
			Expiry: expires,
			URL:    sharedURL,
			Key:    newClnt.URL().String(),
		}
		sURLs.URLs = append(sURLs.URLs, struct {
			Date    time.Time
			Message ShareMessage
		}{
			Date:    shareDate,
			Message: shareMessage,
		})
		Prints("%s\n", shareMessage)
	}
	saveSharedURLsV2(sURLs)
	return nil
}
Example #8
0
File: ls.go Project: henrylee2cn/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive, multipleArgs bool) *probe.Error {
	var err *probe.Error
	var parentContent *client.Content
	urlStr := clnt.URL().String()
	parentDir := url2Dir(urlStr)
	parentClnt, err := url2Client(parentDir)
	if err != nil {
		return err.Trace(clnt.URL().String())
	}
	parentContent, err = parentClnt.Stat()
	if err != nil {
		return err.Trace(clnt.URL().String())
	}
	for contentCh := range clnt.List(recursive, false) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.BrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				if contentCh.Content != nil {
					if contentCh.Content.Type.IsDir() {
						if contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink {
							errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
							continue
						}
						errorIf(contentCh.Err.Trace(), "Unable to list folder.")
					}
				} else {
					errorIf(contentCh.Err.Trace(), "Unable to list.")
					continue
				}
			}
			err = contentCh.Err.Trace()
			break
		}
		if multipleArgs && parentContent.Type.IsDir() {
			contentCh.Content.Name = filepath.Join(parentContent.Name, strings.TrimPrefix(contentCh.Content.Name, parentContent.Name))
		}
		Prints("%s\n", parseContent(contentCh.Content))
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #9
0
// doShareURL share files from target
func doShareDownloadURL(targetURL string, recursive bool, expires time.Duration) *probe.Error {
	shareDate := time.Now().UTC()
	sURLs, err := loadSharedURLsV3()
	if err != nil {
		return err.Trace()
	}
	var clnt client.Client
	clnt, err = url2Client(targetURL)
	if err != nil {
		return err.Trace()
	}
	if expires.Seconds() < 1 {
		return probe.NewError(errors.New("Too low expires, expiration cannot be less than 1 second."))
	}
	if expires.Seconds() > 604800 {
		return probe.NewError(errors.New("Too high expires, expiration cannot be larger than 7 days."))
	}
	for contentCh := range clnt.List(recursive, false) {
		if contentCh.Err != nil {
			return contentCh.Err.Trace()
		}
		var newClnt client.Client
		newClnt, err = url2Client(contentCh.Content.URL.String())
		if err != nil {
			return err.Trace()
		}
		var sharedURL string
		sharedURL, err = newClnt.ShareDownload(expires)
		if err != nil {
			return err.Trace()
		}
		var shareMsg interface{}
		shareMsg = shareMessage{
			Expiry:      expires,
			DownloadURL: sharedURL,
			Key:         newClnt.GetURL().String(),
		}
		shareMsgV3 := shareMessageV3(shareMsg.(shareMessage))
		sURLs.URLs = append(sURLs.URLs, struct {
			Date    time.Time
			Message shareMessageV3
		}{
			Date:    shareDate,
			Message: shareMsgV3,
		})
		printMsg(shareMsg.(shareMessage))
	}
	saveSharedURLsV3(sURLs)
	return nil
}
Example #10
0
// diffFolders - diff of contents of two folders only top level content.
//
// 3: diff(d1, d2) -> []diff(d1/f, d2/f) -> VALID
func diffFolders(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) {
	recursive := false
	// Range on the List to consume incoming content
	for contentCh := range firstClnt.List(recursive, false) {
		if contentCh.Err != nil {
			outCh <- diffMessage{
				Error: contentCh.Err.Trace(firstClnt.GetURL().String()),
			}
			continue
		}
		// Store incoming content
		newFirstContent := contentCh.Content
		newFirstURLStr := newFirstContent.URL.String()
		// Construct the second URL.
		newSecondURL := secondClnt.GetURL()
		// Need to verify the same path from first URL, construct the second URL
		newSecondURL.Path = filepath.Join(newSecondURL.Path, filepath.Base(contentCh.Content.URL.Path))
		newSecondURLStr := newSecondURL.String()
		// Send a stat to verify
		_, newSecondContent, err := url2Stat(newSecondURLStr)
		if err != nil {
			outCh <- diffMessage{
				FirstURL:  newFirstURLStr,
				SecondURL: newSecondURLStr,
				Diff:      "only-in-first",
			}
			continue
		}
		diffMsg := diffObjects(newFirstContent, newSecondContent)
		if diffMsg != nil {
			outCh <- *diffMsg
			continue
		}
	} // Reached EOF
}
Example #11
0
// mainList - is a handler for mc ls command
func mainList(ctx *cli.Context) {
	setListPalette(ctx.GlobalString("colors"))
	checkListSyntax(ctx)

	args := ctx.Args()
	// Operating system tool behavior
	if globalMimicFlag && !ctx.Args().Present() {
		args = []string{"."}
	}

	var targetURLs []string
	var err *probe.Error
	if args.First() == "incomplete" {
		targetURLs, err = args2URLs(args.Tail())
		fatalIf(err.Trace(args...), "One or more unknown URL types passed.")
		for _, targetURL := range targetURLs {
			// if recursive strip off the "..."
			var clnt client.Client
			clnt, err = url2Client(stripRecursiveURL(targetURL))
			fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")

			err = doListIncomplete(clnt, isURLRecursive(targetURL), len(targetURLs) > 1)
			fatalIf(err.Trace(clnt.URL().String()), "Unable to list target ‘"+clnt.URL().String()+"’.")
		}
	} else {
		targetURLs, err = args2URLs(args)
		fatalIf(err.Trace(args...), "One or more unknown URL types passed.")
		for _, targetURL := range targetURLs {
			// if recursive strip off the "..."
			var clnt client.Client
			clnt, err = url2Client(stripRecursiveURL(targetURL))
			fatalIf(err.Trace(targetURL), "Unable to initialize target ‘"+targetURL+"’.")

			err = doList(clnt, isURLRecursive(targetURL), len(targetURLs) > 1)
			fatalIf(err.Trace(clnt.URL().String()), "Unable to list target ‘"+clnt.URL().String()+"’.")
		}
	}

}
Example #12
0
func getTargetContent(srcContent *client.Content, targetContent *client.Content, targetCh <-chan client.ContentOnChannel, targetClnt client.Client) (c *client.Content) {
	if srcContent == nil {
		// nothing to do for empty source content.
		return
	}

	if targetContent == nil {
		c = getContent(targetCh)
	} else {
		c = targetContent
	}

	for ; c != nil; c = getContent(targetCh) {
		// Remove prefix so that we can properly validate.
		targetURL := strings.TrimPrefix(c.URL.Path, targetClnt.GetURL().Path)
		sourceURL := strings.TrimPrefix(srcContent.URL.Path, string(srcContent.URL.Separator))
		if sourceURL <= targetURL {
			break
		}
	}
	return
}
Example #13
0
func doUpdateAccess(clnt client.Client, targetACL bucketACL) (string, error) {
	err := clnt.SetBucketACL(targetACL.String())
	if err != nil {
		msg := fmt.Sprintf("Failed to add bucket access policy for URL ‘%s’", clnt.URL().String())
		return msg, iodine.New(err, nil)
	}
	return "Bucket access policy updated successfully : " + clnt.URL().String(), nil
}
Example #14
0
// doMakeBucket - wrapper around MakeBucket() API
func doMakeBucket(clnt client.Client) (string, error) {
	err := clnt.MakeBucket()
	if err != nil {
		msg := fmt.Sprintf("Failed to create bucket for URL ‘%s’", clnt.URL().String())
		return msg, iodine.New(err, nil)
	}
	return "Bucket created successfully : " + clnt.URL().String(), nil
}
Example #15
0
File: ls.go Project: axcoto-lab/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive bool) *probe.Error {
	var err *probe.Error
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.ISBrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				errorIf(contentCh.Err.Trace(), "Unable to list.")
				continue
			}
			err = contentCh.Err.Trace()
			break
		}
		console.Println(parseContent(contentCh.Content))
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #16
0
func dodiff(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	for contentCh := range firstClnt.List(false, false) {
		if contentCh.Err != nil {
			ch <- DiffMessage{
				Error: contentCh.Err.Trace(firstClnt.URL().String()),
			}
			return
		}
		newFirstURL := urlJoinPath(firstClnt.URL().String(), contentCh.Content.Name)
		newSecondURL := urlJoinPath(secondClnt.URL().String(), contentCh.Content.Name)
		_, newFirstContent, errFirst := url2Stat(newFirstURL)
		_, newSecondContent, errSecond := url2Stat(newSecondURL)
		switch {
		case errFirst == nil && errSecond != nil:
			ch <- DiffMessage{
				FirstURL:  newFirstURL,
				SecondURL: newSecondURL,
				Diff:      "only-in-first",
			}
			continue
		case errFirst == nil && errSecond == nil:
			switch {
			case newFirstContent.Type.IsDir():
				if !newSecondContent.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  newFirstURL,
						SecondURL: newSecondURL,
						Diff:      "type",
					}
				}
				continue
			case newFirstContent.Type.IsRegular():
				if !newSecondContent.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  newFirstURL,
						SecondURL: newSecondURL,
						Diff:      "type",
					}
					continue
				}
				doDiffObjects(newFirstURL, newSecondURL, ch)
			}
		}
	} // End of for-loop
}
Example #17
0
File: ls.go Project: koolhead17/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, isRecursive, isIncomplete bool) *probe.Error {
	_, parentContent, err := url2Stat(clnt.GetURL().String())
	if err != nil {
		return err.Trace(clnt.GetURL().String())
	}

	for contentCh := range clnt.List(isRecursive, isIncomplete) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.BrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				if contentCh.Content != nil {
					if contentCh.Content.Type.IsDir() {
						if contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink {
							errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
							continue
						}
						errorIf(contentCh.Err.Trace(), "Unable to list folder.")
					}
				} else {
					errorIf(contentCh.Err.Trace(), "Unable to list.")
					continue
				}
			}
			err = contentCh.Err.Trace()
			break
		}
		trimmedContent := trimContent(parentContent, contentCh.Content, isRecursive)
		parsedContent := parseContent(trimmedContent)
		printMsg(parsedContent)
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #18
0
// diffFoldersRecursive diff folders for all files recursively.
//
// 4: diff(d1..., d2) -> []diff(d1/f, d2/f) -> VALID.
func diffFoldersRecursive(firstClnt, secondClnt client.Client, outCh chan<- diffMessage) {
	var scanBar scanBarFunc
	if !globalQuietFlag && !globalJSONFlag { // set up progress bar
		scanBar = scanBarFactory()
	}
	recursive := true
	firstListCh := firstClnt.List(recursive, false) // Copy first list channel.
	for firstContentCh := range firstListCh {
		if firstContentCh.Err != nil {
			outCh <- diffMessage{Error: firstContentCh.Err.Trace()}
			continue
		}
		if firstContentCh.Content.Type.IsDir() {
			// Skip directories there is no concept of directories on S3.
			continue
		}
		firstContent := firstContentCh.Content
		secondURL := secondClnt.GetURL()
		secondURL.Path = filepath.Join(secondURL.Path,
			strings.TrimPrefix(firstContent.URL.Path, url2Dir(firstClnt.GetURL().Path)))
		_, secondContent, err := url2Stat(secondURL.String())
		if err != nil {
			outCh <- diffMessage{
				FirstURL:  firstContent.URL.String(),
				SecondURL: secondURL.String(),
				Diff:      "only-in-first",
			}
			continue
		}
		if diffMsg := diffObjects(firstContent, secondContent); diffMsg != nil {
			outCh <- *diffMsg
			continue
		}
		if !globalQuietFlag && !globalJSONFlag { // set up progress bar
			scanBar(firstContent.URL.String())
		}
	}
}
Example #19
0
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	firstTrie := patricia.NewTrie()
	secondTrie := patricia.NewTrie()
	wg := new(sync.WaitGroup)

	type urlAttr struct {
		Size int64
		Type os.FileMode
	}

	wg.Add(1)
	go func(ch chan<- DiffMessage) {
		defer wg.Done()
		for firstContentCh := range firstClnt.List(true) {
			if firstContentCh.Err != nil {
				ch <- DiffMessage{
					Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
				}
				return
			}
			firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
		}
	}(ch)
	wg.Add(1)
	go func(ch chan<- DiffMessage) {
		defer wg.Done()
		for secondContentCh := range secondClnt.List(true) {
			if secondContentCh.Err != nil {
				ch <- DiffMessage{
					Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
				}
				return
			}
			secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
		}
	}(ch)

	doneCh := make(chan struct{})
	defer close(doneCh)
	go func(doneCh <-chan struct{}) {
		cursorCh := cursorAnimate()
		for {
			select {
			case <-time.Tick(100 * time.Millisecond):
				if !globalQuietFlag && !globalJSONFlag {
					console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
				}
			case <-doneCh:
				return
			}
		}
	}(doneCh)
	wg.Wait()
	doneCh <- struct{}{}
	if !globalQuietFlag && !globalJSONFlag {
		console.Printf("%c[2K\n", 27)
		console.Printf("%c[A", 27)
	}

	matchNameCh := make(chan string, 10000)
	go func(matchNameCh chan<- string) {
		itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
			matchNameCh <- string(prefix)
			return nil
		}
		firstTrie.Visit(itemFunc)
		defer close(matchNameCh)
	}(matchNameCh)
	for matchName := range matchNameCh {
		firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
		secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
		firstURL := firstURLDelimited + matchName
		secondURL := secondURLDelimited + matchName
		if !secondTrie.Match(patricia.Prefix(matchName)) {
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
		} else {
			firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
			secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)

			if firstURLAttr.Type.IsRegular() {
				if !secondURLAttr.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
					continue
				}
			}

			if firstURLAttr.Type.IsDir() {
				if !secondURLAttr.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
					continue
				}
			}

			if firstURLAttr.Size != secondURLAttr.Size {
				ch <- DiffMessage{
					FirstURL:  firstURL,
					SecondURL: secondURL,
					Diff:      "size",
				}
			}
		}
	}
}
Example #20
0
File: ls.go Project: fwessels/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, isRecursive, isIncomplete bool) *probe.Error {
	prefixPath := clnt.GetURL().Path
	separator := string(clnt.GetURL().Separator)
	if !strings.HasSuffix(prefixPath, separator) {
		prefixPath = prefixPath[:strings.LastIndex(prefixPath, separator)+1]
	}
	for content := range clnt.List(isRecursive, isIncomplete) {
		if content.Err != nil {
			switch content.Err.ToGoError().(type) {
			// handle this specifically for filesystem related errors.
			case client.BrokenSymlink:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list too many levels link.")
				continue
			case client.PathNotFound:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
				continue
			case client.PathInsufficientPermission:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
				continue
			}
			errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
			continue
		}
		contentURL := content.URL.Path
		contentURL = strings.TrimPrefix(contentURL, prefixPath)
		content.URL.Path = contentURL
		parsedContent := parseContent(content)
		// print colorized or jsonized content info.
		printMsg(parsedContent)
	}
	return nil
}
Example #21
0
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
	firstURLDelimited := firstClnt.URL().String()
	secondURLDelimited := secondClnt.URL().String()
	if strings.HasSuffix(firstURLDelimited, "/") == false {
		firstURLDelimited = firstURLDelimited + "/"
	}
	if strings.HasSuffix(secondURLDelimited, "/") == false {
		secondURLDelimited = secondURLDelimited + "/"
	}
	firstClnt, err := url2Client(firstURLDelimited)
	if err != nil {
		ch <- DiffMessage{Error: err.Trace()}
		return
	}
	secondClnt, err = url2Client(secondURLDelimited)
	if err != nil {
		ch <- DiffMessage{Error: err.Trace()}
		return
	}

	fch := firstClnt.List(true, false)
	sch := secondClnt.List(true, false)
	f, fok := <-fch
	s, sok := <-sch
	for {
		if fok == false {
			break
		}
		if f.Err != nil {
			ch <- DiffMessage{Error: f.Err.Trace()}
			continue
		}
		if f.Content.Type.IsDir() {
			// skip directories
			// there is no concept of directories on S3
			f, fok = <-fch
			continue
		}
		firstURL := firstURLDelimited + f.Content.Name
		secondURL := secondURLDelimited + f.Content.Name
		if sok == false {
			// Second list reached EOF
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
			f, fok = <-fch
			continue
		}
		if s.Err != nil {
			ch <- DiffMessage{Error: s.Err.Trace()}
			continue
		}
		if s.Content.Type.IsDir() {
			// skip directories
			s, sok = <-sch
			continue
		}
		fC := f.Content
		sC := s.Content
		if fC.Name == sC.Name {
			if fC.Type.IsRegular() {
				if !sC.Type.IsRegular() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
				}
			} else if fC.Type.IsDir() {
				if !sC.Type.IsDir() {
					ch <- DiffMessage{
						FirstURL:  firstURL,
						SecondURL: secondURL,
						Diff:      "type",
					}
				}
			} else if fC.Size != sC.Size {
				ch <- DiffMessage{
					FirstURL:  firstURL,
					SecondURL: secondURL,
					Diff:      "size",
				}
			}
			f, fok = <-fch
			s, sok = <-sch
		}
		if fC.Name < sC.Name {
			ch <- DiffMessage{
				FirstURL:  firstURL,
				SecondURL: secondURL,
				Diff:      "only-in-first",
			}
			f, fok = <-fch
		}
		if fC.Name > sC.Name {
			s, sok = <-sch
		}
	}
}
Example #22
0
File: diff.go Project: bosky101/mc
func dodiffdirs(firstClnt client.Client, firstURL, secondURL string, recursive bool, ch chan diff) {
	for contentCh := range firstClnt.List(recursive) {
		if contentCh.Err != nil {
			ch <- diff{
				message: "Failed to list ‘" + firstURL + "’",
				err:     iodine.New(contentCh.Err, nil),
			}
			return
		}
		newFirstURL, err := urlJoinPath(firstURL, contentCh.Content.Name)
		if err != nil {
			ch <- diff{
				message: "Unable to construct new URL from ‘" + firstURL + "’ using ‘" + contentCh.Content.Name + "’",
				err:     iodine.New(err, nil),
			}
			return
		}
		newSecondURL, err := urlJoinPath(secondURL, contentCh.Content.Name)
		if err != nil {
			ch <- diff{
				message: "Unable to construct new URL from ‘" + secondURL + "’ using ‘" + contentCh.Content.Name + "’",
				err:     iodine.New(err, nil),
			}
			return
		}
		_, newFirstContent, errFirst := url2Stat(newFirstURL)
		_, newSecondContent, errSecond := url2Stat(newSecondURL)
		switch {
		case errFirst != nil && errSecond == nil:
			ch <- diff{
				message: "‘" + newSecondURL + "’ Only in ‘" + secondURL + "’",
				err:     nil,
			}
			continue
		case errFirst == nil && errSecond != nil:
			ch <- diff{
				message: "‘" + newFirstURL + "’ Only in ‘" + firstURL + "’",
				err:     nil,
			}
			continue
		case errFirst == nil && errSecond == nil:
			switch {
			case newFirstContent.Type.IsDir():
				if !newSecondContent.Type.IsDir() {
					ch <- diff{
						message: newFirstURL + " and " + newSecondURL + " differs in type.",
						err:     nil,
					}
				}
				continue
			case newFirstContent.Type.IsRegular():
				if !newSecondContent.Type.IsRegular() {
					ch <- diff{
						message: newFirstURL + " and " + newSecondURL + " differs in type.",
						err:     nil,
					}
					continue
				}
				doDiffObjects(newFirstURL, newSecondURL, ch)
			}
		}
	} // End of for-loop
}
Example #23
0
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
	mirrorURLsCh := make(chan mirrorURLs)
	go func() {
		defer close(mirrorURLsCh)
		sourceTrie := patricia.NewTrie()
		targetTries := make([]*patricia.Trie, len(targetClnts))
		wg := new(sync.WaitGroup)

		wg.Add(1)
		go func() {
			defer wg.Done()
			for sourceContentCh := range sourceClnt.List(true) {
				if sourceContentCh.Err != nil {
					mirrorURLsCh <- mirrorURLs{Error: sourceContentCh.Err.Trace()}
					return
				}
				if sourceContentCh.Content.Type.IsRegular() {
					sourceTrie.Insert(patricia.Prefix(sourceContentCh.Content.Name), sourceContentCh.Content.Size)
				}
			}
		}()

		for i, targetClnt := range targetClnts {
			wg.Add(1)
			go func(i int, targetClnt client.Client) {
				defer wg.Done()
				targetTrie := patricia.NewTrie()
				for targetContentCh := range targetClnt.List(true) {
					if targetContentCh.Err != nil {
						mirrorURLsCh <- mirrorURLs{Error: targetContentCh.Err.Trace()}
						return
					}
					if targetContentCh.Content.Type.IsRegular() {
						targetTrie.Insert(patricia.Prefix(targetContentCh.Content.Name), struct{}{})
					}
				}
				targetTries[i] = targetTrie
			}(i, targetClnt)
		}
		wg.Wait()

		matchNameCh := make(chan string, 10000)
		go func(matchNameCh chan<- string) {
			itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
				matchNameCh <- string(prefix)
				return nil
			}
			sourceTrie.Visit(itemFunc)
			defer close(matchNameCh)
		}(matchNameCh)
		for matchName := range matchNameCh {
			sourceContent := new(client.Content)
			var targetContents []*client.Content
			for i, targetTrie := range targetTries {
				if !targetTrie.Match(patricia.Prefix(matchName)) {
					sourceURLDelimited := sourceClnt.URL().String()[:strings.LastIndex(sourceClnt.URL().String(),
						string(sourceClnt.URL().Separator))+1]
					newTargetURLParse := *targetClnts[i].URL()
					newTargetURLParse.Path = filepath.Join(newTargetURLParse.Path, matchName)
					sourceContent.Size = sourceTrie.Get(patricia.Prefix(matchName)).(int64)
					sourceContent.Name = sourceURLDelimited + matchName
					targetContents = append(targetContents, &client.Content{Name: newTargetURLParse.String()})
				}
			}
			mirrorURLsCh <- mirrorURLs{
				SourceContent:  sourceContent,
				TargetContents: targetContents,
			}
		}
	}()
	return mirrorURLsCh
}
Example #24
0
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
	mirrorURLsCh := make(chan mirrorURLs)

	go func() {
		defer close(mirrorURLsCh)
		id := newRandomID(8)

		doneCh := make(chan bool)
		defer close(doneCh)
		go func(doneCh <-chan bool) {
			cursorCh := cursorAnimate()
			for {
				select {
				case <-time.Tick(100 * time.Millisecond):
					if !globalQuietFlag && !globalJSONFlag {
						console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
					}
				case <-doneCh:
					return
				}
			}
		}(doneCh)

		sourceSortedList := sortedList{}
		targetSortedList := make([]*sortedList, len(targetClnts))

		surldelimited := sourceClnt.URL().String()
		err := sourceSortedList.Create(sourceClnt, id+".src")
		if err != nil {
			mirrorURLsCh <- mirrorURLs{
				Error: err.Trace(),
			}
			return
		}

		turldelimited := make([]string, len(targetClnts))
		for i := range targetClnts {
			turldelimited[i] = targetClnts[i].URL().String()
			targetSortedList[i] = &sortedList{}
			err := targetSortedList[i].Create(targetClnts[i], id+"."+strconv.Itoa(i))
			if err != nil {
				// FIXME: do cleanup by calling Delete()
				mirrorURLsCh <- mirrorURLs{
					Error: err.Trace(),
				}
				return
			}
		}
		for source := range sourceSortedList.List(true) {
			if source.Content.Type.IsDir() {
				continue
			}
			targetContents := make([]*client.Content, 0, len(targetClnts))
			for i, t := range targetSortedList {
				match, err := t.Match(source.Content)
				if err != nil || match {
					// continue on io.EOF or if the keys matches
					// FIXME: handle other errors and ignore this target for future calls
					continue
				}
				targetContents = append(targetContents, &client.Content{Name: turldelimited[i] + source.Content.Name})
			}
			source.Content.Name = surldelimited + source.Content.Name
			if len(targetContents) > 0 {
				mirrorURLsCh <- mirrorURLs{
					SourceContent:  source.Content,
					TargetContents: targetContents,
				}
			}
		}
		if err := sourceSortedList.Delete(); err != nil {
			mirrorURLsCh <- mirrorURLs{
				Error: err.Trace(),
			}
		}
		for _, t := range targetSortedList {
			if err := t.Delete(); err != nil {
				mirrorURLsCh <- mirrorURLs{
					Error: err.Trace(),
				}
			}
		}
		doneCh <- true
		if !globalQuietFlag && !globalJSONFlag {
			console.Eraseline()
		}
	}()
	return mirrorURLsCh
}