// scanBarFactory returns a progress bar function to report URL scanning.
func scanBarFactory() scanBarFunc {
prevLineSize := 0
prevSource := ""
fileCount := 0
termSize, err := ts.GetSize()
if err != nil {
fatalIf(probe.NewError(err), "Unable to get terminal size. Please use --quiet option.")
}
termWidth := termSize.Col()
cursorCh := cursorAnimate()
return func(source string) {
scanPrefix := fmt.Sprintf("[%s] %s ", humanize.Comma(int64(fileCount)), string(<-cursorCh))
cmnPrefix := commonPrefix(source, prevSource)
eraseLen := prevLineSize - len([]rune(scanPrefix+cmnPrefix))
if eraseLen < 1 {
eraseLen = 0
}
if prevLineSize != 0 { // erase previous line
console.PrintC("\r" + scanPrefix + cmnPrefix + strings.Repeat(" ", eraseLen))
}
source = fixateScanBar(source, termWidth-len([]rune(scanPrefix))-1)
barText := scanPrefix + source
console.PrintC("\r" + barText)
prevSource = source
prevLineSize = len([]rune(barText))
fileCount++
}
}
// scanBarFactory returns a progress bar function to report URL scanning.
func scanBarFactory(prefix string) scanBarFunc {
prevLineSize := 0
fileCount := 0
termSize, err := ts.GetSize()
if err != nil {
fatalIf(probe.NewError(err), "Unable to get terminal size.")
}
termWidth := termSize.Col()
cursorCh := cursorAnimate()
return func(source string) {
scanPrefix := fmt.Sprintf("[%s] %s ", humanize.Comma(int64(fileCount)), string(<-cursorCh))
if prefix != "" {
scanPrefix = fmt.Sprintf("Scanning %s [%s] %s ", prefix, humanize.Comma(int64(fileCount)), string(<-cursorCh))
}
if prevLineSize != 0 { // erase previous line
console.PrintC("\r" + scanPrefix + strings.Repeat(" ", prevLineSize-len([]rune(scanPrefix))))
}
source = fixateScanBar(source, termWidth-len([]rune(scanPrefix))-1)
barText := "\r" + scanPrefix + source
console.PrintC(barText)
prevLineSize = len([]rune(barText))
fileCount++
}
}
// doCast - Cast an object to multiple destination. castURLs status contains a copy of sURLs and error if any.
func doCast(sURLs castURLs, bar *barSend, castQueueCh <-chan bool, wg *sync.WaitGroup, statusCh chan<- castURLs) {
defer wg.Done() // Notify that this copy routine is done.
defer func() {
<-castQueueCh
}()
if sURLs.Error != nil { // Errorneous sURLs passed.
sURLs.Error = iodine.New(sURLs.Error, nil)
statusCh <- sURLs
return
}
if !globalQuietFlag || !globalJSONFlag {
bar.SetCaption(sURLs.SourceContent.Name + ": ")
}
reader, length, err := getSource(sURLs.SourceContent.Name)
if err != nil {
if !globalQuietFlag || !globalJSONFlag {
bar.ErrorGet(int64(length))
}
sURLs.Error = iodine.New(err, nil)
statusCh <- sURLs
return
}
var targetURLs []string
for _, targetContent := range sURLs.TargetContents {
targetURLs = append(targetURLs, targetContent.Name)
}
var newReader io.ReadCloser
if globalQuietFlag || globalJSONFlag {
console.PrintC(CastMessage{
Source: sURLs.SourceContent.Name,
Targets: targetURLs,
})
newReader = reader
} else {
// set up progress
newReader = bar.NewProxyReader(reader)
}
defer newReader.Close()
err = putTargets(targetURLs, length, newReader)
if err != nil {
if !globalQuietFlag || !globalJSONFlag {
bar.ErrorPut(int64(length))
}
sURLs.Error = iodine.New(err, nil)
statusCh <- sURLs
return
}
sURLs.Error = nil // just for safety
statusCh <- sURLs
}
// doCopy - Copy a singe file from source to destination
func doCopy(cpURLs copyURLs, bar *barSend, cpQueue <-chan bool, wg *sync.WaitGroup, statusCh chan<- copyURLs) {
defer wg.Done() // Notify that this copy routine is done.
defer func() {
<-cpQueue
}()
if cpURLs.Error != nil {
cpURLs.Error = iodine.New(cpURLs.Error, nil)
statusCh <- cpURLs
return
}
if !globalQuietFlag || !globalJSONFlag {
bar.SetCaption(cpURLs.SourceContent.Name + ": ")
}
reader, length, err := getSource(cpURLs.SourceContent.Name)
if err != nil {
if !globalQuietFlag || !globalJSONFlag {
bar.ErrorGet(length)
}
cpURLs.Error = NewIodine(iodine.New(err, map[string]string{"URL": cpURLs.SourceContent.Name}))
statusCh <- cpURLs
return
}
var newReader io.ReadCloser
if globalQuietFlag || globalJSONFlag {
console.PrintC(CopyMessage{
Source: cpURLs.SourceContent.Name,
Target: cpURLs.TargetContent.Name,
Length: cpURLs.SourceContent.Size,
})
newReader = reader
} else {
// set up progress
newReader = bar.NewProxyReader(reader)
}
defer newReader.Close()
err = putTarget(cpURLs.TargetContent.Name, length, newReader)
if err != nil {
if !globalQuietFlag || !globalJSONFlag {
bar.ErrorPut(length)
}
cpURLs.Error = NewIodine(iodine.New(err, map[string]string{"URL": cpURLs.SourceContent.Name}))
statusCh <- cpURLs
return
}
cpURLs.Error = nil // just for safety
statusCh <- cpURLs
}
// scanBarFactory returns a progress bar function to report URL scanning.
func scanBarFactory() scanBarFunc {
fileCount := 0
termWidth, e := pb.GetTerminalWidth()
fatalIf(probe.NewError(e), "Unable to get terminal size. Please use --quiet option.")
// Cursor animate channel.
cursorCh := cursorAnimate()
return func(source string) {
scanPrefix := fmt.Sprintf("[%s] %s ", humanize.Comma(int64(fileCount)), string(<-cursorCh))
source = fixateScanBar(source, termWidth-len([]rune(scanPrefix)))
barText := scanPrefix + source
console.PrintC("\r" + barText + "\r")
fileCount++
}
}
// scanBarFactory returns a progress bar function to report URL scanning.
func scanBarFactory() scanBarFunc {
fileCount := 0
termWidth, err := pb.GetTerminalWidth()
if err != nil {
termWidth = 80
}
// Cursor animate channel.
cursorCh := cursorAnimate()
return func(source string) {
scanPrefix := fmt.Sprintf("[%s] %s ", humanize.Comma(int64(fileCount)), string(<-cursorCh))
source = fixateScanBar(source, termWidth-len([]rune(scanPrefix)))
barText := scanPrefix + source
console.PrintC("\r" + barText + "\r")
fileCount++
}
}
func dodiffRecursive(firstClnt, secondClnt client.Client, ch chan DiffMessage) {
firstTrie := patricia.NewTrie()
secondTrie := patricia.NewTrie()
wg := new(sync.WaitGroup)
type urlAttr struct {
Size int64
Type os.FileMode
}
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for firstContentCh := range firstClnt.List(true) {
if firstContentCh.Err != nil {
ch <- DiffMessage{
Error: firstContentCh.Err.Trace(firstClnt.URL().String()),
}
return
}
firstTrie.Insert(patricia.Prefix(firstContentCh.Content.Name), urlAttr{firstContentCh.Content.Size, firstContentCh.Content.Type})
}
}(ch)
wg.Add(1)
go func(ch chan<- DiffMessage) {
defer wg.Done()
for secondContentCh := range secondClnt.List(true) {
if secondContentCh.Err != nil {
ch <- DiffMessage{
Error: secondContentCh.Err.Trace(secondClnt.URL().String()),
}
return
}
secondTrie.Insert(patricia.Prefix(secondContentCh.Content.Name), urlAttr{secondContentCh.Content.Size, secondContentCh.Content.Type})
}
}(ch)
doneCh := make(chan struct{})
defer close(doneCh)
go func(doneCh <-chan struct{}) {
cursorCh := cursorAnimate()
for {
select {
case <-time.Tick(100 * time.Millisecond):
if !globalQuietFlag && !globalJSONFlag {
console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
}
case <-doneCh:
return
}
}
}(doneCh)
wg.Wait()
doneCh <- struct{}{}
if !globalQuietFlag && !globalJSONFlag {
console.Printf("%c[2K\n", 27)
console.Printf("%c[A", 27)
}
matchNameCh := make(chan string, 10000)
go func(matchNameCh chan<- string) {
itemFunc := func(prefix patricia.Prefix, item patricia.Item) error {
matchNameCh <- string(prefix)
return nil
}
firstTrie.Visit(itemFunc)
defer close(matchNameCh)
}(matchNameCh)
for matchName := range matchNameCh {
firstURLDelimited := firstClnt.URL().String()[:strings.LastIndex(firstClnt.URL().String(), string(firstClnt.URL().Separator))+1]
secondURLDelimited := secondClnt.URL().String()[:strings.LastIndex(secondClnt.URL().String(), string(secondClnt.URL().Separator))+1]
firstURL := firstURLDelimited + matchName
secondURL := secondURLDelimited + matchName
if !secondTrie.Match(patricia.Prefix(matchName)) {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "only-in-first",
}
} else {
firstURLAttr := firstTrie.Get(patricia.Prefix(matchName)).(urlAttr)
secondURLAttr := secondTrie.Get(patricia.Prefix(matchName)).(urlAttr)
if firstURLAttr.Type.IsRegular() {
if !secondURLAttr.Type.IsRegular() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Type.IsDir() {
if !secondURLAttr.Type.IsDir() {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "type",
}
continue
}
}
if firstURLAttr.Size != secondURLAttr.Size {
ch <- DiffMessage{
FirstURL: firstURL,
SecondURL: secondURL,
Diff: "size",
}
}
}
}
}
func deltaSourceTargets(sourceClnt client.Client, targetClnts []client.Client) <-chan mirrorURLs {
mirrorURLsCh := make(chan mirrorURLs)
go func() {
defer close(mirrorURLsCh)
id := newRandomID(8)
doneCh := make(chan bool)
defer close(doneCh)
go func(doneCh <-chan bool) {
cursorCh := cursorAnimate()
for {
select {
case <-time.Tick(100 * time.Millisecond):
if !globalQuietFlag && !globalJSONFlag {
console.PrintC("\r" + "Scanning.. " + string(<-cursorCh))
}
case <-doneCh:
return
}
}
}(doneCh)
sourceSortedList := sortedList{}
targetSortedList := make([]*sortedList, len(targetClnts))
surldelimited := sourceClnt.URL().String()
err := sourceSortedList.Create(sourceClnt, id+".src")
if err != nil {
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
return
}
turldelimited := make([]string, len(targetClnts))
for i := range targetClnts {
turldelimited[i] = targetClnts[i].URL().String()
targetSortedList[i] = &sortedList{}
err := targetSortedList[i].Create(targetClnts[i], id+"."+strconv.Itoa(i))
if err != nil {
// FIXME: do cleanup by calling Delete()
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
return
}
}
for source := range sourceSortedList.List(true) {
if source.Content.Type.IsDir() {
continue
}
targetContents := make([]*client.Content, 0, len(targetClnts))
for i, t := range targetSortedList {
match, err := t.Match(source.Content)
if err != nil || match {
// continue on io.EOF or if the keys matches
// FIXME: handle other errors and ignore this target for future calls
continue
}
targetContents = append(targetContents, &client.Content{Name: turldelimited[i] + source.Content.Name})
}
source.Content.Name = surldelimited + source.Content.Name
if len(targetContents) > 0 {
mirrorURLsCh <- mirrorURLs{
SourceContent: source.Content,
TargetContents: targetContents,
}
}
}
if err := sourceSortedList.Delete(); err != nil {
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
}
for _, t := range targetSortedList {
if err := t.Delete(); err != nil {
mirrorURLsCh <- mirrorURLs{
Error: err.Trace(),
}
}
}
doneCh <- true
if !globalQuietFlag && !globalJSONFlag {
console.Eraseline()
}
}()
return mirrorURLsCh
}