Example #1
0
func main() {
	start := time.Now()

	flag.Parse()
	worker.MaxJobs = maxJobs

	validateEnvironment()

	if isVerbose {
		cmd := exec.Command(pathToLessc, "-v")
		out, _ := cmd.CombinedOutput()

		fmt.Printf("less-tree v%s: %s\n", version, strings.TrimSpace(string(out)))
	}

	cssQueue := worker.NewWorker()
	cssQueue.On(worker.JobFinished, func(pk *worker.Package, args ...interface{}) {
		job := pk.Job().(*cssJob)

		if job.exitCode == 0 {
			pk.SetStatus(worker.Finished)
		} else {
			pk.SetStatus(worker.Errored)
		}
	})

	args := flag.Args()
	for _, v := range args {
		parseDirectory(v, cssQueue)
	}

	cssQueue.RunUntilDone()

	finish := time.Now()

	if len(args) > 0 {
		stats := cssQueue.Stats()

		successRate := float64(0)
		if stats.Total > 0 {
			successRate = float64(100*stats.Finished) / float64(stats.Total)
		}

		if isVerbose {
			fmt.Println("--------------------------------------")
		}
		fmt.Printf("Compiled %d LESS files in %s\n%d ok, %d errored (%.1f%% success rate)\n",
			stats.Total,
			finish.Sub(start).String(),
			stats.Finished,
			stats.Errored,
			successRate,
		)
	}
}
Example #2
0
func parseDirectory(dir string, cssQueue *worker.Worker) {
	analyzeQueue := worker.NewWorker()
	lessFileCh := make(chan *lessFile, 100)
	errCh := make(chan error, 100)
	stopCh := make(chan bool)
	files := []*lessFile{}

	crawler, err := newDirectoryCrawler(dir, func(crawler *directoryCrawler, less_dir, css_dir *os.File, less_file os.FileInfo) {
		name, _ := filepath.Rel(crawler.rootLESS.Name(), filepath.Join(less_dir.Name(), less_file.Name()))
		job := newFindImportsJob(name, less_dir, css_dir, less_file, lessFileCh, errCh)
		analyzeQueue.Add(job)
	})
	if err != nil {
		fmt.Printf("error crawling directory %s: %s\n", dir, err)
	}

	cm := newLessTreeCache(crawler.rootCSS)
	err = cm.Load()

	go func(less_file_ch chan *lessFile, error_ch chan error, stop_ch chan bool) {
		for {
			select {
			case l := <-less_file_ch:
				files = append(files, l)

			case err := <-error_ch:
				fmt.Printf("err: %s\n", err)

			case _ = <-stop_ch:
				break
			}
		}
	}(lessFileCh, errCh, stopCh)

	crawler.Parse()

	if isVerbose {
		fmt.Println("finished building queue")
	}

	analyzeQueue.RunUntilDone()
	stopCh <- true

	for _, file := range files {
		job := newCSSJob(file.Name, file.Dir, file.CSSDir, file.File, lesscArgs.out)
		isCached := cm.Test(file)
		outputFilesExist := job.OutputFilesExist()

		if !isCached || !outputFilesExist || force {
			cssQueue.Add(job)
		}
	}

	cm.Save()
}
Example #3
0
// BuildRoleImages triggers the building of the role docker images in parallel
func (r *RoleImageBuilder) BuildRoleImages(roles model.Roles, repository, baseImageName string, force, noBuild bool, workerCount int) error {
	if workerCount < 1 {
		return fmt.Errorf("Invalid worker count %d", workerCount)
	}

	dockerManager, err := newDockerImageBuilder()
	if err != nil {
		return fmt.Errorf("Error connecting to docker: %s", err.Error())
	}

	workerLib.MaxJobs = workerCount
	worker := workerLib.NewWorker()

	resultsCh := make(chan error)
	abort := make(chan struct{})
	for _, role := range roles {
		worker.Add(roleBuildJob{
			role:          role,
			builder:       r,
			ui:            r.ui,
			force:         force,
			noBuild:       noBuild,
			dockerManager: dockerManager,
			resultsCh:     resultsCh,
			abort:         abort,
			repository:    repository,
			baseImageName: baseImageName,
		})
	}

	go worker.RunUntilDone()

	aborted := false
	for i := 0; i < len(roles); i++ {
		result := <-resultsCh
		if result != nil {
			if !aborted {
				close(abort)
				aborted = true
			}
			err = result
		}
	}

	return err
}
Example #4
0
// Compile concurrency works like this:
// 1 routine producing (todoCh<-)                                  <=> Compile() itself
// n workers consuming (<-todoCh)                                  <=> compileJob.Run()'s
// 1 synchronizer consuming EXACTLY 1 <-doneCh for every <-todoCh  <=> Compile() again.
//
// Dependencies:
// - Packages with the least dependencies are queued first.
// - Workers wait for their dependencies by waiting on a map of
//   broadcasting channels that are closed by the synchronizer when
//   something is done compiling successfully
//   ==> c.signalDependencies [<fingerprint>]
//
// In the event of an error:
// - workers will try to bail out of waiting on <-todo or
//   <-c.signalDependencies[<fingerprint>] early if it finds the killCh has been
//   activated. There is a "race" here to see if the synchronizer will
//   drain <-todoCh or if they will select on <-killCh before
//   <-todoCh. In the worst case, extra packages will be compiled by
//   each active worker. See (**), (xx)
//
//   Note that jobs without dependencies ignore the kill signal. See (xx).
//
// - synchronizer will greedily drain the <-todoCh to starve the
//   workers out and won't wait for the <-doneCh for the N packages it
//   drained.
func (c *Compilator) Compile(workerCount int, releases []*model.Release, roleManifest *model.RoleManifest) error {
	packages, err := c.removeCompiledPackages(c.gatherPackages(releases, roleManifest))

	if err != nil {
		return fmt.Errorf("failed to remove compiled packages: %v", err)
	}
	if 0 == len(packages) {
		c.ui.Println("No package needed to be built")
		return nil
	}
	sort.Sort(packages)

	// Setup the queuing system ...
	doneCh := make(chan compileResult)
	killCh := make(chan struct{})

	workerLib.MaxJobs = workerCount

	worker := workerLib.NewWorker()
	buckets := createDepBuckets(packages)

	// ... load it with the jobs to run ...
	for _, pkg := range buckets {
		worker.Add(compileJob{
			pkg:        pkg,
			compilator: c,
			killCh:     killCh,
			doneCh:     doneCh,
		})
	}

	// ... and start everything
	go func() {
		worker.RunUntilDone()
		close(doneCh)
	}()

	// (**) All jobs push their results into the single doneCh.
	// The code below is a synchronizer which pulls the results
	// from the channel as fast as it can and reports them to the
	// user. In case of an error it signals this back to all jobs
	// by closing killCh. This will cause the remaining jobs to
	// abort when the queing system invokes them.  Note however,
	// that the synchronizer is in a race with the dependency
	// checker in func (j compileJob) Run() (see below), some jobs
	// may still run to regular completion.

	killed := false
	for result := range doneCh {
		if result.err == nil {
			close(c.signalDependencies[result.pkg.Fingerprint])
			c.ui.Printf("%s   > success: %s/%s\n",
				color.YellowString("result"),
				color.GreenString(result.pkg.Release.Name),
				color.GreenString(result.pkg.Name))
			continue
		}

		c.ui.Printf(
			"%s   > failure: %s/%s - %s\n",
			color.YellowString("result"),
			color.RedString(result.pkg.Release.Name),
			color.RedString(result.pkg.Name),
			color.RedString(result.err.Error()),
		)

		err = result.err
		if !killed {
			close(killCh)
			killed = true
		}
	}

	return err
}
Example #5
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	l := &sync.Mutex{}
	err := ui.Init()
	if err != nil {
		panic(err)
	}
	defer ui.Close()

	ui.UseTheme("helloworld")

	loadConfErr := LoadConfig()
	loadProxyErr := LoadProxies()

	LoadedProxies = len(D_PROXY_LIST)
	ProxiesLeft = LoadedProxies
	aboutUi := ui.NewPar(
		"Please report bugs @skype: newjolt\nOptimized to use " + strconv.Itoa(runtime.NumCPU()) + " cores.\nPress 'CTRL+C' to quit, 'CTRL+SPACE' to start task, 'CTRL+R' to reload settings.\nJoin us revive KryptoDEV.com")
	aboutUi.Height = 5
	aboutUi.TextFgColor = ui.ColorBlack
	aboutUi.Border.FgColor = ui.ColorMagenta
	aboutUi.TextBgColor = ui.ColorWhite
	aboutUi.BgColor = ui.ColorWhite
	aboutUi.Border.Label = "TOGNINJA: a topofgames.com auto voter v0.1 - kryptoDEV.com"
	aboutUi.Border.LabelFgColor = ui.AttrBold

	successUi := ui.NewPar("")
	successUi.Height = 3
	successUi.TextFgColor = ui.ColorWhite
	successUi.Border.Label = "Last Succeeded Vote"

	failedUi := ui.NewPar("")
	failedUi.Height = 3
	failedUi.TextFgColor = ui.ColorWhite
	failedUi.Border.Label = "Last Failed Vote"

	configUi := ui.NewPar(fmt.Sprintf("@Proxies Loaded: %d\n@Target ID: %d\n@Worker Count: %d", LoadedProxies, ConfigStruct.TargetID, ConfigStruct.WorkerCount))
	configUi.Height = 5
	configUi.TextFgColor = ui.ColorWhite
	configUi.Border.FgColor = ui.ColorWhite
	configUi.Border.Label = "Loaded Config."

	progressUi := ui.NewGauge()
	progressUi.Percent = 0
	progressUi.Height = 3
	progressUi.Border.Label = "Progress Report"
	progressUi.BarColor = ui.ColorRed
	progressUi.Border.FgColor = ui.ColorWhite
	progressUi.Border.LabelFgColor = ui.ColorCyan

	errorUi := ui.NewPar("")
	if loadConfErr != nil {
		errorUi.Text = loadConfErr.Error()
	} else if ConfigStruct.DBCUsername == "" {
		errorUi.Text = "Deathbycaptcha username not provided."
	} else if ConfigStruct.DBCPassword == "" {
		errorUi.Text = "Deathbycaptcha password not provided."
	} else if loadProxyErr != nil {
		errorUi.Text = loadProxyErr.Error()
	}
	errorUi.Height = 3
	errorUi.TextFgColor = ui.ColorRed
	errorUi.Border.FgColor = ui.ColorWhite
	errorUi.Border.Label = "Last Error"

	statsUi := ui.NewPar("")
	statsUi.Height = 3
	statsUi.TextFgColor = ui.ColorWhite
	statsUi.Border.FgColor = ui.ColorGreen
	statsUi.Border.Label = "Stats"

	// build layout
	ui.Body.AddRows(
		ui.NewRow(
			ui.NewCol(12, 0, aboutUi)),
		ui.NewRow(
			ui.NewCol(6, 0, configUi), ui.NewCol(6, 0, errorUi)),
		ui.NewRow(
			ui.NewCol(6, 0, successUi), ui.NewCol(6, 0, failedUi)),
		ui.NewRow(
			ui.NewCol(12, 0, progressUi)),
		ui.NewRow(
			ui.NewCol(12, 0, statsUi)))

	// calculate layout
	ui.Body.Align()

	done := make(chan bool)
	redraw := make(chan bool)

	evt := ui.EventCh()
	ui.Render(ui.Body)
	worker.MaxJobs = ConfigStruct.WorkerCount
	w := worker.NewWorker()

	w.On(worker.JobFinished, func(args ...interface{}) {
		l.Lock()
		ProxiesLeft--
		l.Unlock()
		pk := args[0].(*worker.Package)
		job := pk.Job().(*VoteJob)
		if job.HasError != "" {
			l.Lock()
			errorUi.Text = job.HasError
			ErrorsEncountered++
			l.Unlock()
		}
		if job.HasSuccessVoted {
			l.Lock()
			SuccessfulVotes++
			VotesAttempted++
			l.Unlock()
			if job.SuccessVoteText != "" {
				l.Lock()
				successUi.Text = job.SuccessVoteText
				l.Unlock()
			}
		}
		if job.HasFailVoted {
			l.Lock()
			FailedVotes++
			VotesAttempted++
			l.Unlock()
			if job.FailVoteText != "" {
				l.Lock()
				failedUi.Text = job.FailVoteText
				l.Unlock()
			}
		}
		if job.IsLast {
			l.Lock()
			statsUi.Text = "Bot: Finished | Proxies left: 0 | Successful Votes: " + strconv.Itoa(SuccessfulVotes) + " | Failed Votes: " + strconv.Itoa(FailedVotes) + " | Errors Encountered: " + strconv.Itoa(ErrorsEncountered)
			progressUi.Percent = 100
			progressUi.BarColor = ui.ColorGreen
			ui.Render(ui.Body)
			l.Unlock()
			return
		}
		l.Lock()
		progressUi.Percent = int(float32(100 - (100 * ProxiesLeft / LoadedProxies)))
		statsUi.Text = "Bot: Running | Proxies left: " + strconv.Itoa(ProxiesLeft) + " | Successful Votes: " + strconv.Itoa(SuccessfulVotes) + " | Failed Votes: " + strconv.Itoa(FailedVotes) + " | Errors Encountered: " + strconv.Itoa(ErrorsEncountered)
		ui.Render(ui.Body)
		l.Unlock()
	})

	for {
		select {
		case e := <-evt:
			if e.Type == ui.EventKey && e.Key == ui.KeyCtrlC {
				return
			}
			if e.Type == ui.EventKey && e.Key == ui.KeyCtrlR {
				//FetchCaptcha()
			}
			if e.Type == ui.EventKey && e.Key == ui.KeyCtrlSpace {
				for a := 0; a < LoadedProxies; a++ {
					if a == LoadedProxies {
						j := VoteJob{Name: D_PROXY_LIST[a], Proxy: D_PROXY_LIST[a], HasSuccessVoted: false, HasFailVoted: false, IsLast: true}
						w.Add(&j)
					} else {
						j := VoteJob{Name: D_PROXY_LIST[a], Proxy: D_PROXY_LIST[a], HasSuccessVoted: false, HasFailVoted: false, IsLast: false}
						w.Add(&j)
					}
				}
				w.RunUntilDone()
			}
			if e.Type == ui.EventResize {
				ui.Body.Width = ui.TermWidth()
				ui.Body.Align()
				go func() { redraw <- true }()
			}
		case <-done:
			return
		case <-redraw:
			ui.Render(ui.Body)
		}
	}
}