func main() {
start := time.Now()
flag.Parse()
worker.MaxJobs = maxJobs
validateEnvironment()
if isVerbose {
cmd := exec.Command(pathToLessc, "-v")
out, _ := cmd.CombinedOutput()
fmt.Printf("less-tree v%s: %s\n", version, strings.TrimSpace(string(out)))
}
cssQueue := worker.NewWorker()
cssQueue.On(worker.JobFinished, func(pk *worker.Package, args ...interface{}) {
job := pk.Job().(*cssJob)
if job.exitCode == 0 {
pk.SetStatus(worker.Finished)
} else {
pk.SetStatus(worker.Errored)
}
})
args := flag.Args()
for _, v := range args {
parseDirectory(v, cssQueue)
}
cssQueue.RunUntilDone()
finish := time.Now()
if len(args) > 0 {
stats := cssQueue.Stats()
successRate := float64(0)
if stats.Total > 0 {
successRate = float64(100*stats.Finished) / float64(stats.Total)
}
if isVerbose {
fmt.Println("--------------------------------------")
}
fmt.Printf("Compiled %d LESS files in %s\n%d ok, %d errored (%.1f%% success rate)\n",
stats.Total,
finish.Sub(start).String(),
stats.Finished,
stats.Errored,
successRate,
)
}
}
func parseDirectory(dir string, cssQueue *worker.Worker) {
analyzeQueue := worker.NewWorker()
lessFileCh := make(chan *lessFile, 100)
errCh := make(chan error, 100)
stopCh := make(chan bool)
files := []*lessFile{}
crawler, err := newDirectoryCrawler(dir, func(crawler *directoryCrawler, less_dir, css_dir *os.File, less_file os.FileInfo) {
name, _ := filepath.Rel(crawler.rootLESS.Name(), filepath.Join(less_dir.Name(), less_file.Name()))
job := newFindImportsJob(name, less_dir, css_dir, less_file, lessFileCh, errCh)
analyzeQueue.Add(job)
})
if err != nil {
fmt.Printf("error crawling directory %s: %s\n", dir, err)
}
cm := newLessTreeCache(crawler.rootCSS)
err = cm.Load()
go func(less_file_ch chan *lessFile, error_ch chan error, stop_ch chan bool) {
for {
select {
case l := <-less_file_ch:
files = append(files, l)
case err := <-error_ch:
fmt.Printf("err: %s\n", err)
case _ = <-stop_ch:
break
}
}
}(lessFileCh, errCh, stopCh)
crawler.Parse()
if isVerbose {
fmt.Println("finished building queue")
}
analyzeQueue.RunUntilDone()
stopCh <- true
for _, file := range files {
job := newCSSJob(file.Name, file.Dir, file.CSSDir, file.File, lesscArgs.out)
isCached := cm.Test(file)
outputFilesExist := job.OutputFilesExist()
if !isCached || !outputFilesExist || force {
cssQueue.Add(job)
}
}
cm.Save()
}
// BuildRoleImages triggers the building of the role docker images in parallel
func (r *RoleImageBuilder) BuildRoleImages(roles model.Roles, repository, baseImageName string, force, noBuild bool, workerCount int) error {
if workerCount < 1 {
return fmt.Errorf("Invalid worker count %d", workerCount)
}
dockerManager, err := newDockerImageBuilder()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
workerLib.MaxJobs = workerCount
worker := workerLib.NewWorker()
resultsCh := make(chan error)
abort := make(chan struct{})
for _, role := range roles {
worker.Add(roleBuildJob{
role: role,
builder: r,
ui: r.ui,
force: force,
noBuild: noBuild,
dockerManager: dockerManager,
resultsCh: resultsCh,
abort: abort,
repository: repository,
baseImageName: baseImageName,
})
}
go worker.RunUntilDone()
aborted := false
for i := 0; i < len(roles); i++ {
result := <-resultsCh
if result != nil {
if !aborted {
close(abort)
aborted = true
}
err = result
}
}
return err
}
// Compile concurrency works like this:
// 1 routine producing (todoCh<-) <=> Compile() itself
// n workers consuming (<-todoCh) <=> compileJob.Run()'s
// 1 synchronizer consuming EXACTLY 1 <-doneCh for every <-todoCh <=> Compile() again.
//
// Dependencies:
// - Packages with the least dependencies are queued first.
// - Workers wait for their dependencies by waiting on a map of
// broadcasting channels that are closed by the synchronizer when
// something is done compiling successfully
// ==> c.signalDependencies [<fingerprint>]
//
// In the event of an error:
// - workers will try to bail out of waiting on <-todo or
// <-c.signalDependencies[<fingerprint>] early if it finds the killCh has been
// activated. There is a "race" here to see if the synchronizer will
// drain <-todoCh or if they will select on <-killCh before
// <-todoCh. In the worst case, extra packages will be compiled by
// each active worker. See (**), (xx)
//
// Note that jobs without dependencies ignore the kill signal. See (xx).
//
// - synchronizer will greedily drain the <-todoCh to starve the
// workers out and won't wait for the <-doneCh for the N packages it
// drained.
func (c *Compilator) Compile(workerCount int, releases []*model.Release, roleManifest *model.RoleManifest) error {
packages, err := c.removeCompiledPackages(c.gatherPackages(releases, roleManifest))
if err != nil {
return fmt.Errorf("failed to remove compiled packages: %v", err)
}
if 0 == len(packages) {
c.ui.Println("No package needed to be built")
return nil
}
sort.Sort(packages)
// Setup the queuing system ...
doneCh := make(chan compileResult)
killCh := make(chan struct{})
workerLib.MaxJobs = workerCount
worker := workerLib.NewWorker()
buckets := createDepBuckets(packages)
// ... load it with the jobs to run ...
for _, pkg := range buckets {
worker.Add(compileJob{
pkg: pkg,
compilator: c,
killCh: killCh,
doneCh: doneCh,
})
}
// ... and start everything
go func() {
worker.RunUntilDone()
close(doneCh)
}()
// (**) All jobs push their results into the single doneCh.
// The code below is a synchronizer which pulls the results
// from the channel as fast as it can and reports them to the
// user. In case of an error it signals this back to all jobs
// by closing killCh. This will cause the remaining jobs to
// abort when the queing system invokes them. Note however,
// that the synchronizer is in a race with the dependency
// checker in func (j compileJob) Run() (see below), some jobs
// may still run to regular completion.
killed := false
for result := range doneCh {
if result.err == nil {
close(c.signalDependencies[result.pkg.Fingerprint])
c.ui.Printf("%s > success: %s/%s\n",
color.YellowString("result"),
color.GreenString(result.pkg.Release.Name),
color.GreenString(result.pkg.Name))
continue
}
c.ui.Printf(
"%s > failure: %s/%s - %s\n",
color.YellowString("result"),
color.RedString(result.pkg.Release.Name),
color.RedString(result.pkg.Name),
color.RedString(result.err.Error()),
)
err = result.err
if !killed {
close(killCh)
killed = true
}
}
return err
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
l := &sync.Mutex{}
err := ui.Init()
if err != nil {
panic(err)
}
defer ui.Close()
ui.UseTheme("helloworld")
loadConfErr := LoadConfig()
loadProxyErr := LoadProxies()
LoadedProxies = len(D_PROXY_LIST)
ProxiesLeft = LoadedProxies
aboutUi := ui.NewPar(
"Please report bugs @skype: newjolt\nOptimized to use " + strconv.Itoa(runtime.NumCPU()) + " cores.\nPress 'CTRL+C' to quit, 'CTRL+SPACE' to start task, 'CTRL+R' to reload settings.\nJoin us revive KryptoDEV.com")
aboutUi.Height = 5
aboutUi.TextFgColor = ui.ColorBlack
aboutUi.Border.FgColor = ui.ColorMagenta
aboutUi.TextBgColor = ui.ColorWhite
aboutUi.BgColor = ui.ColorWhite
aboutUi.Border.Label = "TOGNINJA: a topofgames.com auto voter v0.1 - kryptoDEV.com"
aboutUi.Border.LabelFgColor = ui.AttrBold
successUi := ui.NewPar("")
successUi.Height = 3
successUi.TextFgColor = ui.ColorWhite
successUi.Border.Label = "Last Succeeded Vote"
failedUi := ui.NewPar("")
failedUi.Height = 3
failedUi.TextFgColor = ui.ColorWhite
failedUi.Border.Label = "Last Failed Vote"
configUi := ui.NewPar(fmt.Sprintf("@Proxies Loaded: %d\n@Target ID: %d\n@Worker Count: %d", LoadedProxies, ConfigStruct.TargetID, ConfigStruct.WorkerCount))
configUi.Height = 5
configUi.TextFgColor = ui.ColorWhite
configUi.Border.FgColor = ui.ColorWhite
configUi.Border.Label = "Loaded Config."
progressUi := ui.NewGauge()
progressUi.Percent = 0
progressUi.Height = 3
progressUi.Border.Label = "Progress Report"
progressUi.BarColor = ui.ColorRed
progressUi.Border.FgColor = ui.ColorWhite
progressUi.Border.LabelFgColor = ui.ColorCyan
errorUi := ui.NewPar("")
if loadConfErr != nil {
errorUi.Text = loadConfErr.Error()
} else if ConfigStruct.DBCUsername == "" {
errorUi.Text = "Deathbycaptcha username not provided."
} else if ConfigStruct.DBCPassword == "" {
errorUi.Text = "Deathbycaptcha password not provided."
} else if loadProxyErr != nil {
errorUi.Text = loadProxyErr.Error()
}
errorUi.Height = 3
errorUi.TextFgColor = ui.ColorRed
errorUi.Border.FgColor = ui.ColorWhite
errorUi.Border.Label = "Last Error"
statsUi := ui.NewPar("")
statsUi.Height = 3
statsUi.TextFgColor = ui.ColorWhite
statsUi.Border.FgColor = ui.ColorGreen
statsUi.Border.Label = "Stats"
// build layout
ui.Body.AddRows(
ui.NewRow(
ui.NewCol(12, 0, aboutUi)),
ui.NewRow(
ui.NewCol(6, 0, configUi), ui.NewCol(6, 0, errorUi)),
ui.NewRow(
ui.NewCol(6, 0, successUi), ui.NewCol(6, 0, failedUi)),
ui.NewRow(
ui.NewCol(12, 0, progressUi)),
ui.NewRow(
ui.NewCol(12, 0, statsUi)))
// calculate layout
ui.Body.Align()
done := make(chan bool)
redraw := make(chan bool)
evt := ui.EventCh()
ui.Render(ui.Body)
worker.MaxJobs = ConfigStruct.WorkerCount
w := worker.NewWorker()
w.On(worker.JobFinished, func(args ...interface{}) {
l.Lock()
ProxiesLeft--
l.Unlock()
pk := args[0].(*worker.Package)
job := pk.Job().(*VoteJob)
if job.HasError != "" {
l.Lock()
errorUi.Text = job.HasError
ErrorsEncountered++
l.Unlock()
}
if job.HasSuccessVoted {
l.Lock()
SuccessfulVotes++
VotesAttempted++
l.Unlock()
if job.SuccessVoteText != "" {
l.Lock()
successUi.Text = job.SuccessVoteText
l.Unlock()
}
}
if job.HasFailVoted {
l.Lock()
FailedVotes++
VotesAttempted++
l.Unlock()
if job.FailVoteText != "" {
l.Lock()
failedUi.Text = job.FailVoteText
l.Unlock()
}
}
if job.IsLast {
l.Lock()
statsUi.Text = "Bot: Finished | Proxies left: 0 | Successful Votes: " + strconv.Itoa(SuccessfulVotes) + " | Failed Votes: " + strconv.Itoa(FailedVotes) + " | Errors Encountered: " + strconv.Itoa(ErrorsEncountered)
progressUi.Percent = 100
progressUi.BarColor = ui.ColorGreen
ui.Render(ui.Body)
l.Unlock()
return
}
l.Lock()
progressUi.Percent = int(float32(100 - (100 * ProxiesLeft / LoadedProxies)))
statsUi.Text = "Bot: Running | Proxies left: " + strconv.Itoa(ProxiesLeft) + " | Successful Votes: " + strconv.Itoa(SuccessfulVotes) + " | Failed Votes: " + strconv.Itoa(FailedVotes) + " | Errors Encountered: " + strconv.Itoa(ErrorsEncountered)
ui.Render(ui.Body)
l.Unlock()
})
for {
select {
case e := <-evt:
if e.Type == ui.EventKey && e.Key == ui.KeyCtrlC {
return
}
if e.Type == ui.EventKey && e.Key == ui.KeyCtrlR {
//FetchCaptcha()
}
if e.Type == ui.EventKey && e.Key == ui.KeyCtrlSpace {
for a := 0; a < LoadedProxies; a++ {
if a == LoadedProxies {
j := VoteJob{Name: D_PROXY_LIST[a], Proxy: D_PROXY_LIST[a], HasSuccessVoted: false, HasFailVoted: false, IsLast: true}
w.Add(&j)
} else {
j := VoteJob{Name: D_PROXY_LIST[a], Proxy: D_PROXY_LIST[a], HasSuccessVoted: false, HasFailVoted: false, IsLast: false}
w.Add(&j)
}
}
w.RunUntilDone()
}
if e.Type == ui.EventResize {
ui.Body.Width = ui.TermWidth()
ui.Body.Align()
go func() { redraw <- true }()
}
case <-done:
return
case <-redraw:
ui.Render(ui.Body)
}
}
}