// actionRunner handles running an action which may take a while to complete // providing progress bars and signal handling. func actionRunner(cmd *cli.Cmd, action action) func() { cmd.Spec = "[--silent] [--no-progress] " + cmd.Spec silent := cmd.BoolOpt("silent", false, "Set to true to disable all non-error output") noProgress := cmd.BoolOpt("no-progress", false, "Set to true to disable the progress bar") return func() { var infoWriter io.Writer = os.Stderr var ticker <-chan time.Time if err := action.init(); err != nil { fail("Initialization failed: %v", err) } done, err := action.start(infoWriter) if err != nil { fail("Startup failed: %v", err) } var bar *pb.ProgressBar if !*silent && !*noProgress { ticker = time.Tick(statsFrequency) bar = action.newProgressBar() if bar != nil { bar.Output = os.Stderr bar.ShowSpeed = true bar.ManualUpdate = true bar.SetMaxWidth(78) bar.Start() bar.Update() } } if *silent { infoWriter = ioutil.Discard } sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT) LOOP: for { select { case <-ticker: action.updateProgress(bar) bar.Update() case <-sigchan: bar.Finish() fmt.Fprintf(os.Stderr, "\nAborting..") action.abort() <-done fmt.Fprintf(os.Stderr, "Aborted.\n") break LOOP case err := <-done: if err != nil { fail("Processing failed: %v", err) } break LOOP } } if bar != nil { bar.Finish() } if !*silent { action.printFinalStats(infoWriter) } } }
func customizeBar(bar *pb.ProgressBar) { bar.ShowCounters = true bar.ShowTimeLeft = false bar.ShowSpeed = true bar.SetMaxWidth(80) bar.SetUnits(pb.U_BYTES) }
func HostMerge(List []Host, ShowBar bool) []string { count := 0 filterList := []string{""} length := len(List) var bar *pb.ProgressBar if ShowBar == true { bar = pb.StartNew(length) bar.SetMaxWidth(80) } for _, Host := range List { length = len(filterList[count]) if length == 0 { filterList[count] = Host.Hostname } else if length+Host.length() <= 255 && length != 0 { filterList[count] += "|" filterList[count] += Host.Hostname } else { count++ filterList = append(filterList, Host.Hostname) // filterList[count] = Ref.Referrer } if ShowBar == true { bar.Increment() time.Sleep(time.Millisecond * 50) } } if ShowBar == true { bar.Finish() } return filterList }
// processSystemArchives processes archives for given system func (h *Harvester) processSystemArchives(s *system.System, archives []string) error { var bar *pb.ProgressBar nb := len(archives) // extract archives if !s.Options.Quiet { fmt.Printf("[%s] Extracting %v archive(s)\n", s.Infos.Name, nb) if !s.Options.Debug { bar = pb.StartNew(nb) bar.ShowCounters = true bar.ShowPercent = false bar.ShowTimeLeft = true bar.SetMaxWidth(80) } } for _, archive := range archives { if !s.Options.Quiet && !s.Options.Debug { bar.Increment() } if err := s.ProcessArchive(archive, h.Options.Output); err != nil { return err } } if !s.Options.Quiet && !s.Options.Debug { bar.Finish() fmt.Printf("[%s] Processed %v files (skipped: %v)\n", s.Infos.Name, s.Processed, s.Skipped) } fmt.Printf("[%s] Selected %v games\n", s.Infos.Name, len(s.Games)) return nil }
func main() { kingpin.CommandLine.HelpFlag.Short('h') kingpin.Parse() if *downloadFlag == true { *downloadFlag = false } else { *downloadFlag = true } w := new(tabwriter.Writer) var output io.Writer if *confirmFlag == false { var err error // log.SetFlags(0) LogFileLocation := flag.String("log", "BotSifter.log", "Specifies path of the log file") output, err = os.OpenFile(*LogFileLocation, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { log.Println(err) return } } else { log.SetFlags(0) } if output == nil { output = os.Stdout } w.Init(output, 4, 4, 2, ' ', 0) log.SetOutput(w) //Read config file var GooFig GoogleConfig data, err := ioutil.ReadFile(*configFile) if err != nil { fmt.Println("Unable to open configuration file: " + *configFile) return } //Load config data from file into struct err = yaml.Unmarshal(data, &GooFig) if err != nil { log.Println(err) return } fmt.Println("\nConfig File: \t\t[" + *configFile + "]") fmt.Println("Include Refferers File: [" + GooFig.RefWhite + "]") fmt.Println("Exclude Refferers File: [" + GooFig.RefBlack + "]") fmt.Println("Include UA File: \t[" + GooFig.UAWhite + "]") fmt.Println("Exclude UA File: \t[" + GooFig.UABlack + "]") fmt.Println("Exclude Hostname File: \t[" + GooFig.HostInc + "]") //Loading variables from config struct WebPropertyId := GooFig.UACode AccountID := WebPropertyId[3:11] //Authentication settings conf := &oauth2.Config{ ClientID: GooFig.ClientID, ClientSecret: GooFig.ClientSecret, RedirectURL: GooFig.RedirectURL, Scopes: []string{ "https://www.googleapis.com/auth/analytics", "https://www.googleapis.com/auth/analytics.edit", "https://www.googleapis.com/auth/analytics.manage.users", }, Endpoint: google.Endpoint, } //If the config data doesn't contain Auth Token we need to get one fmt.Println("") if GooFig.Token == nil { fmt.Print("Authenticating user...") GooFig.Token = auth(conf, GooFig.Port) fmt.Println("\t\t\t\t\tCompleted") } //Load new client and service to talk with Google api fmt.Print("Setting up Google client...") client := conf.Client(oauth2.NoContext, GooFig.Token) service, err := analytics.New(client) if err != nil { log.Fatalln(err) // GooFig.Token = nil } fmt.Println("\t\t\t\t\tCompleted") //Retrieve BotSifter list from server if the cleanFlag is false var resp Response var respDisplay Response //Load csv files into appropriate structs fmt.Print("Loading includes, excludes and hostname lists...") uainc := ReadReferrerList(GooFig.UAWhite) uaexc := ReadReferrerList(GooFig.UABlack) refs := ReadReferrerList(GooFig.RefWhite) excs := ReadReferrerList(GooFig.RefBlack) hosts := ReadHostList(GooFig.HostInc) fmt.Println("\t\tCompleted") if *cleanFlag == false { if *downloadFlag == true { fmt.Print("Downloading BotSifter Referrer List...") resp = retreiveList(GooFig.Person) respDisplay = resp fmt.Println("\t\t\t\tCompleted") } if resp.Status == "Unauthorized" { fmt.Println("Download failed: Invalid username/password") return } //Append contents from includeList.csv onto the ReferrerList struct and remove duplicate entries fmt.Print("Merging local include data with BotSifter data...") resp.ReferrerList = append(resp.ReferrerList, refs...) resp.UserAgentList = append(resp.UserAgentList, uainc...) RemoveDuplicates(&resp.UserAgentList) RemoveDuplicates(&resp.ReferrerList) fmt.Println("\t\tCompleted") //Remove contents from ReferrerList which were found on the excludeList.csv fmt.Print("Removing local exclude data from BotSifter data...") resultsRef := []Referrer{} for _, compFilter := range resp.ReferrerList { found := false for _, exc := range excs { if compFilter.Referrer == exc.Referrer { found = true } } if !found { resultsRef = append(resultsRef, compFilter) } } resp.ReferrerList = resultsRef resultsUA := []Referrer{} for _, compFilter := range resp.UserAgentList { found := false for _, exc := range uaexc { if compFilter.Referrer == exc.Referrer { found = true } } if !found { resultsUA = append(resultsUA, compFilter) } } resp.UserAgentList = resultsUA fmt.Println("\t\tCompleted") } fmt.Print("Download current BotSifter filters to build comparison lists...") //List current Botsifter filters in GA account filters, err := service.Management.Filters.List(AccountID).Do() if err != nil { log.Fatalln(err) } var oldFilterListUA []Referrer var oldFilterListRef []Referrer var oldFilterListHost []Host for _, oldFilter := range filters.Items { if strings.Contains(oldFilter.Name, "BotSifter UA") == true { if filterExpression := oldFilter.ExcludeDetails; filterExpression != nil { filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\.", ".", -1)) filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\+", "+", -1)) for _, ref := range strings.Split(filterExpression.ExpressionValue, "|") { oldFilterListUA = append(oldFilterListUA, Referrer{ref, "", ""}) } } } if strings.Contains(oldFilter.Name, "BotSifter Ref") == true { if filterExpression := oldFilter.ExcludeDetails; filterExpression != nil { filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\.", ".", -1)) filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\+", "+", -1)) for _, ref := range strings.Split(filterExpression.ExpressionValue, "|") { oldFilterListRef = append(oldFilterListRef, Referrer{ref, "", ""}) } } } if strings.Contains(oldFilter.Name, "BotSifter Hostname") == true { if filterExpression := oldFilter.IncludeDetails; filterExpression != nil { filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\.", ".", -1)) filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\+", "+", -1)) for _, ref := range strings.Split(filterExpression.ExpressionValue, "|") { oldFilterListHost = append(oldFilterListHost, Host{ref}) } } } } onlyInNewListRefs := Referrers(resp.ReferrerList).findEntriesOnlyInLeftSide(oldFilterListRef) onlyInOldListRefs := Referrers(oldFilterListRef).findEntriesOnlyInLeftSide(resp.ReferrerList) inBothListsRefs := Referrers(resp.ReferrerList).findInBoth(oldFilterListRef) onlyInNewListUAs := Referrers(resp.UserAgentList).findEntriesOnlyInLeftSide(oldFilterListUA) onlyInOldListUAs := Referrers(oldFilterListUA).findEntriesOnlyInLeftSide(resp.UserAgentList) inBothListsUAs := Referrers(resp.UserAgentList).findInBoth(oldFilterListUA) onlyInNewListHosts := Hosts(hosts).findEntriesOnlyInLeftSide(oldFilterListHost) onlyInOldListHosts := Hosts(oldFilterListHost).findEntriesOnlyInLeftSide(hosts) inBothListsHosts := Hosts(hosts).findInBoth(oldFilterListHost) var Ref Referrer resultsRef := []Referrer{} for _, Ref = range resp.ReferrerList { Ref.Referrer = (strings.Replace(Ref.Referrer, ".", "\\.", -1)) Ref.Referrer = (strings.Replace(Ref.Referrer, "+", "\\+", -1)) resultsRef = append(resultsRef, Ref) } resp.ReferrerList = resultsRef resultsUA := []Referrer{} for _, Ref = range resp.UserAgentList { Ref.Referrer = (strings.Replace(Ref.Referrer, ".", "\\.", -1)) Ref.Referrer = (strings.Replace(Ref.Referrer, "+", "\\+", -1)) resultsUA = append(resultsUA, Ref) } resp.UserAgentList = resultsUA resultsHost := []Host{} for _, h := range hosts { h.Hostname = (strings.Replace(h.Hostname, ".", "\\.", -1)) h.Hostname = (strings.Replace(h.Hostname, "+", "\\+", -1)) resultsHost = append(resultsHost, h) } hosts = resultsHost fmt.Println("\tCompleted") fmt.Println("") log.Println("Current Botsifter Bots:") log.Println("\n#################### CURRENT BotSifter BOTS ####################") log.Println("Referrers:\n") log.Println("\tRANK\tNAME\tSCORE") for _, Ref = range respDisplay.ReferrerList { log.Println(Ref) } log.Println("") log.Println("User Agents:\n") log.Println("\tRANK\tNAME\tSCORE") for _, Ref = range respDisplay.UserAgentList { log.Println(Ref) } log.Println("") log.Println("\nBotSifter will make the following changes to your GA Account[" + GooFig.UACode + "]:") log.Println("\n#################### HOST CHANGES ####################") log.Println("Added Hosts:\n") if onlyInNewListHosts != nil { sort.Sort(onlyInNewListHosts) for _, h := range onlyInNewListHosts { log.Println(h) } } else { log.Println("\tNONE") } log.Println("") log.Println("Removed Hosts:\n") if onlyInOldListHosts != nil { sort.Sort(onlyInOldListUAs) for _, h := range onlyInOldListHosts { log.Println(h) } } else { log.Println("\tNONE") } // log.Println(strings.Trim(fmt.Sprint(onlyInOldListRefs), "[]")) log.Println("") log.Println("Hosts unchange:\n") if inBothListsHosts != nil { sort.Sort(inBothListsUAs) for _, h := range inBothListsHosts { log.Println(h) } } else { log.Println("\tNONE") } log.Println("\n#################### REFERRER CHANGES ####################") log.Println("Added Referrers:\n") if onlyInNewListRefs != nil { log.Println("\tRANK\tNAME\tSCORE") sort.Sort(onlyInNewListRefs) for _, Ref = range onlyInNewListRefs { log.Println(Ref) } } else { log.Println("\tNONE") } log.Println("") log.Println("Removed Referrers:\n") if onlyInOldListRefs != nil { log.Println("\tRANK\tNAME\tSCORE") sort.Sort(onlyInOldListRefs) for _, Ref = range onlyInOldListRefs { log.Println(Ref) } } else { log.Println("\tNONE") } // log.Println(strings.Trim(fmt.Sprint(onlyInOldListRefs), "[]")) log.Println("") log.Println("Referrers unchange:\n") if inBothListsRefs != nil { log.Println("\tRANK\tNAME\tSCORE") sort.Sort(inBothListsRefs) for _, Ref = range inBothListsRefs { log.Println(Ref) } } else { log.Println("\tNONE") } log.Println("\n#################### USER AGENTS CHANGES ####################") log.Println("Added User Agents:\n") if onlyInNewListUAs != nil { log.Println("\tRANK\tNAME\tSCORE") sort.Sort(onlyInNewListUAs) for _, Ref = range onlyInNewListUAs { log.Println(Ref) } } else { log.Println("\tNONE") } log.Println("") log.Println("Removed User Agents:\n") if onlyInOldListUAs != nil { log.Println("\tRANK\tNAME\tSCORE") sort.Sort(onlyInOldListUAs) for _, Ref = range onlyInOldListUAs { log.Println(Ref) } } else { log.Println("\tNONE") } // log.Println(strings.Trim(fmt.Sprint(onlyInOldListRefs), "[]")) log.Println("") log.Println("User Agents unchange:\n") if inBothListsUAs != nil { log.Println("\tRANK\tNAME\tSCORE") sort.Sort(inBothListsUAs) for _, Ref = range inBothListsUAs { log.Println(Ref) } } else { log.Println("\tNONE") } w.Flush() // log.Println(strings.Trim(fmt.Sprint(inBothListsRefs), "[]")) log.Println("") if *confirmFlag == false { length := len(filters.Items) var bar *pb.ProgressBar if length != 0 { bar = pb.StartNew(length) bar.SetMaxWidth(80) fmt.Println("Deleting old BotSifter filters ") for _, eachFilter := range filters.Items { if strings.Contains(eachFilter.Name, "BotSifter") == true { service.Management.Filters.Delete(AccountID, eachFilter.Id).Do() } bar.Increment() time.Sleep(time.Millisecond * 250) } bar.Finish() } else { fmt.Println("No filters to delete") } //If cleanFlag entered then end program here if *cleanFlag == true { return } //If view is not defined in config file then ask user which one to apply filters too if GooFig.View == "" { //List all views profiles, err := service.Management.Profiles.List(AccountID, WebPropertyId).Do() if err != nil { log.Println(err) } for i, profile := range profiles.Items { fmt.Printf("%d. %s\n", i, profile.Name) } reader := bufio.NewReader(os.Stdin) fmt.Printf("Please select a profile to apply filters too: ") index := 0 for { selectedProfileIndex, _ := reader.ReadString('\n') index, err = strconv.Atoi(strings.TrimSuffix(selectedProfileIndex, "\n")) if err == nil && index < len(profiles.Items) { break } else { fmt.Println("Invalid input", index, err) } } GooFig.View = profiles.Items[index].Id } //Prepare filters fmt.Println("Preparing Filter - combining multiple Referrers") var filterList []string filterList = RefMerge(resp.ReferrerList, true) //Build new filters from ReferrerList struct fmt.Println("Creating referral filters") var FilterIds []string length = len(filterList) bar = pb.StartNew(length) bar.SetMaxWidth(80) for i, newFilter := range filterList { counter := strconv.Itoa(i + 1) filter := &analytics.Filter{ Name: "BotSifter Ref Spam" + counter, Type: "EXCLUDE", ExcludeDetails: &analytics.FilterExpression{ Field: "REFERRAL", ExpressionValue: newFilter, CaseSensitive: false, }, } filter, err = service.Management.Filters.Insert(AccountID, filter).Do() if err != nil { fmt.Print("\n") fmt.Println(err) return } //Save filter Ids for later FilterIds = append(FilterIds, filter.Id) bar.Increment() time.Sleep(time.Millisecond * 250) } bar.Finish() //Prepare filters fmt.Println("Preparing Filter - combining multiple User Agents") var filterListua []string filterListua = RefMerge(resp.UserAgentList, true) //Build new filters from ReferrerList struct fmt.Println("Creating User Agent filters") length = len(filterListua) bar = pb.StartNew(length) bar.SetMaxWidth(80) for i, newFilter := range filterListua { counter := strconv.Itoa(i + 1) filter := &analytics.Filter{ Name: "BotSifter UA Spam" + counter, Type: "EXCLUDE", ExcludeDetails: &analytics.FilterExpression{ Field: "USER_DEFINED_VALUE", ExpressionValue: newFilter, CaseSensitive: false, }, } filter, err = service.Management.Filters.Insert(AccountID, filter).Do() if err != nil { fmt.Print("\n") fmt.Println(err) return } //Save filter Ids for later FilterIds = append(FilterIds, filter.Id) bar.Increment() time.Sleep(time.Millisecond * 250) } bar.Finish() if len(hosts) != 0 { var hostList []string hostList = HostMerge(hosts, false) //If there's hosts build "include Hostname" rule(s) fmt.Println("Creating Hostname filter(s)") length = len(hostList) bar = pb.StartNew(length) bar.SetMaxWidth(80) for i, newHost := range hostList { counter := strconv.Itoa(i) filter := &analytics.Filter{ Name: "BotSifter Hostname Spam" + counter, Type: "INCLUDE", IncludeDetails: &analytics.FilterExpression{ Field: "PAGE_HOSTNAME", ExpressionValue: newHost, CaseSensitive: false, }, } filter, err = service.Management.Filters.Insert(AccountID, filter).Do() if err != nil { log.Println(err) return } //Save filter Ids for later FilterIds = append(FilterIds, filter.Id) bar.Increment() time.Sleep(time.Millisecond * 250) } bar.Finish() } //connecting built filters to profile user selected fmt.Println("Connecting filters to profile") length = len(FilterIds) bar = pb.StartNew(length) bar.SetMaxWidth(80) for _, newLink := range FilterIds { profilefilterlink := &analytics.ProfileFilterLink{ FilterRef: &analytics.FilterRef{Id: newLink}, } _, err := service.Management.ProfileFilterLinks.Insert(AccountID, WebPropertyId, GooFig.View, profilefilterlink).Do() if err != nil { log.Println("Error Connecting Filter to View\n") } bar.Increment() time.Sleep(time.Millisecond * 250) } bar.Finish() } fmt.Println("Saving configuration data to " + *configFile) //Marshal data to save into config file data, err = yaml.Marshal(&GooFig) if err != nil { log.Println(err) return } //Write config file err = ioutil.WriteFile(*configFile, data, 0644) if err != nil { log.Println(err) return } fmt.Println("Completed") }