// actionRunner handles running an action which may take a while to complete
// providing progress bars and signal handling.
func actionRunner(cmd *cli.Cmd, action action) func() {
cmd.Spec = "[--silent] [--no-progress] " + cmd.Spec
silent := cmd.BoolOpt("silent", false, "Set to true to disable all non-error output")
noProgress := cmd.BoolOpt("no-progress", false, "Set to true to disable the progress bar")
return func() {
var infoWriter io.Writer = os.Stderr
var ticker <-chan time.Time
if err := action.init(); err != nil {
fail("Initialization failed: %v", err)
}
done, err := action.start(infoWriter)
if err != nil {
fail("Startup failed: %v", err)
}
var bar *pb.ProgressBar
if !*silent && !*noProgress {
ticker = time.Tick(statsFrequency)
bar = action.newProgressBar()
if bar != nil {
bar.Output = os.Stderr
bar.ShowSpeed = true
bar.ManualUpdate = true
bar.SetMaxWidth(78)
bar.Start()
bar.Update()
}
}
if *silent {
infoWriter = ioutil.Discard
}
sigchan := make(chan os.Signal, 1)
signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)
LOOP:
for {
select {
case <-ticker:
action.updateProgress(bar)
bar.Update()
case <-sigchan:
bar.Finish()
fmt.Fprintf(os.Stderr, "\nAborting..")
action.abort()
<-done
fmt.Fprintf(os.Stderr, "Aborted.\n")
break LOOP
case err := <-done:
if err != nil {
fail("Processing failed: %v", err)
}
break LOOP
}
}
if bar != nil {
bar.Finish()
}
if !*silent {
action.printFinalStats(infoWriter)
}
}
}
func customizeBar(bar *pb.ProgressBar) {
bar.ShowCounters = true
bar.ShowTimeLeft = false
bar.ShowSpeed = true
bar.SetMaxWidth(80)
bar.SetUnits(pb.U_BYTES)
}
func HostMerge(List []Host, ShowBar bool) []string {
count := 0
filterList := []string{""}
length := len(List)
var bar *pb.ProgressBar
if ShowBar == true {
bar = pb.StartNew(length)
bar.SetMaxWidth(80)
}
for _, Host := range List {
length = len(filterList[count])
if length == 0 {
filterList[count] = Host.Hostname
} else if length+Host.length() <= 255 && length != 0 {
filterList[count] += "|"
filterList[count] += Host.Hostname
} else {
count++
filterList = append(filterList, Host.Hostname)
// filterList[count] = Ref.Referrer
}
if ShowBar == true {
bar.Increment()
time.Sleep(time.Millisecond * 50)
}
}
if ShowBar == true {
bar.Finish()
}
return filterList
}
// processSystemArchives processes archives for given system
func (h *Harvester) processSystemArchives(s *system.System, archives []string) error {
var bar *pb.ProgressBar
nb := len(archives)
// extract archives
if !s.Options.Quiet {
fmt.Printf("[%s] Extracting %v archive(s)\n", s.Infos.Name, nb)
if !s.Options.Debug {
bar = pb.StartNew(nb)
bar.ShowCounters = true
bar.ShowPercent = false
bar.ShowTimeLeft = true
bar.SetMaxWidth(80)
}
}
for _, archive := range archives {
if !s.Options.Quiet && !s.Options.Debug {
bar.Increment()
}
if err := s.ProcessArchive(archive, h.Options.Output); err != nil {
return err
}
}
if !s.Options.Quiet && !s.Options.Debug {
bar.Finish()
fmt.Printf("[%s] Processed %v files (skipped: %v)\n", s.Infos.Name, s.Processed, s.Skipped)
}
fmt.Printf("[%s] Selected %v games\n", s.Infos.Name, len(s.Games))
return nil
}
func main() {
kingpin.CommandLine.HelpFlag.Short('h')
kingpin.Parse()
if *downloadFlag == true {
*downloadFlag = false
} else {
*downloadFlag = true
}
w := new(tabwriter.Writer)
var output io.Writer
if *confirmFlag == false {
var err error
// log.SetFlags(0)
LogFileLocation := flag.String("log", "BotSifter.log", "Specifies path of the log file")
output, err = os.OpenFile(*LogFileLocation, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Println(err)
return
}
} else {
log.SetFlags(0)
}
if output == nil {
output = os.Stdout
}
w.Init(output, 4, 4, 2, ' ', 0)
log.SetOutput(w)
//Read config file
var GooFig GoogleConfig
data, err := ioutil.ReadFile(*configFile)
if err != nil {
fmt.Println("Unable to open configuration file: " + *configFile)
return
}
//Load config data from file into struct
err = yaml.Unmarshal(data, &GooFig)
if err != nil {
log.Println(err)
return
}
fmt.Println("\nConfig File: \t\t[" + *configFile + "]")
fmt.Println("Include Refferers File: [" + GooFig.RefWhite + "]")
fmt.Println("Exclude Refferers File: [" + GooFig.RefBlack + "]")
fmt.Println("Include UA File: \t[" + GooFig.UAWhite + "]")
fmt.Println("Exclude UA File: \t[" + GooFig.UABlack + "]")
fmt.Println("Exclude Hostname File: \t[" + GooFig.HostInc + "]")
//Loading variables from config struct
WebPropertyId := GooFig.UACode
AccountID := WebPropertyId[3:11]
//Authentication settings
conf := &oauth2.Config{
ClientID: GooFig.ClientID,
ClientSecret: GooFig.ClientSecret,
RedirectURL: GooFig.RedirectURL,
Scopes: []string{
"https://www.googleapis.com/auth/analytics",
"https://www.googleapis.com/auth/analytics.edit",
"https://www.googleapis.com/auth/analytics.manage.users",
},
Endpoint: google.Endpoint,
}
//If the config data doesn't contain Auth Token we need to get one
fmt.Println("")
if GooFig.Token == nil {
fmt.Print("Authenticating user...")
GooFig.Token = auth(conf, GooFig.Port)
fmt.Println("\t\t\t\t\tCompleted")
}
//Load new client and service to talk with Google api
fmt.Print("Setting up Google client...")
client := conf.Client(oauth2.NoContext, GooFig.Token)
service, err := analytics.New(client)
if err != nil {
log.Fatalln(err)
// GooFig.Token = nil
}
fmt.Println("\t\t\t\t\tCompleted")
//Retrieve BotSifter list from server if the cleanFlag is false
var resp Response
var respDisplay Response
//Load csv files into appropriate structs
fmt.Print("Loading includes, excludes and hostname lists...")
uainc := ReadReferrerList(GooFig.UAWhite)
uaexc := ReadReferrerList(GooFig.UABlack)
refs := ReadReferrerList(GooFig.RefWhite)
excs := ReadReferrerList(GooFig.RefBlack)
hosts := ReadHostList(GooFig.HostInc)
fmt.Println("\t\tCompleted")
if *cleanFlag == false {
if *downloadFlag == true {
fmt.Print("Downloading BotSifter Referrer List...")
resp = retreiveList(GooFig.Person)
respDisplay = resp
fmt.Println("\t\t\t\tCompleted")
}
if resp.Status == "Unauthorized" {
fmt.Println("Download failed: Invalid username/password")
return
}
//Append contents from includeList.csv onto the ReferrerList struct and remove duplicate entries
fmt.Print("Merging local include data with BotSifter data...")
resp.ReferrerList = append(resp.ReferrerList, refs...)
resp.UserAgentList = append(resp.UserAgentList, uainc...)
RemoveDuplicates(&resp.UserAgentList)
RemoveDuplicates(&resp.ReferrerList)
fmt.Println("\t\tCompleted")
//Remove contents from ReferrerList which were found on the excludeList.csv
fmt.Print("Removing local exclude data from BotSifter data...")
resultsRef := []Referrer{}
for _, compFilter := range resp.ReferrerList {
found := false
for _, exc := range excs {
if compFilter.Referrer == exc.Referrer {
found = true
}
}
if !found {
resultsRef = append(resultsRef, compFilter)
}
}
resp.ReferrerList = resultsRef
resultsUA := []Referrer{}
for _, compFilter := range resp.UserAgentList {
found := false
for _, exc := range uaexc {
if compFilter.Referrer == exc.Referrer {
found = true
}
}
if !found {
resultsUA = append(resultsUA, compFilter)
}
}
resp.UserAgentList = resultsUA
fmt.Println("\t\tCompleted")
}
fmt.Print("Download current BotSifter filters to build comparison lists...")
//List current Botsifter filters in GA account
filters, err := service.Management.Filters.List(AccountID).Do()
if err != nil {
log.Fatalln(err)
}
var oldFilterListUA []Referrer
var oldFilterListRef []Referrer
var oldFilterListHost []Host
for _, oldFilter := range filters.Items {
if strings.Contains(oldFilter.Name, "BotSifter UA") == true {
if filterExpression := oldFilter.ExcludeDetails; filterExpression != nil {
filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\.", ".", -1))
filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\+", "+", -1))
for _, ref := range strings.Split(filterExpression.ExpressionValue, "|") {
oldFilterListUA = append(oldFilterListUA, Referrer{ref, "", ""})
}
}
}
if strings.Contains(oldFilter.Name, "BotSifter Ref") == true {
if filterExpression := oldFilter.ExcludeDetails; filterExpression != nil {
filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\.", ".", -1))
filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\+", "+", -1))
for _, ref := range strings.Split(filterExpression.ExpressionValue, "|") {
oldFilterListRef = append(oldFilterListRef, Referrer{ref, "", ""})
}
}
}
if strings.Contains(oldFilter.Name, "BotSifter Hostname") == true {
if filterExpression := oldFilter.IncludeDetails; filterExpression != nil {
filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\.", ".", -1))
filterExpression.ExpressionValue = (strings.Replace(filterExpression.ExpressionValue, "\\+", "+", -1))
for _, ref := range strings.Split(filterExpression.ExpressionValue, "|") {
oldFilterListHost = append(oldFilterListHost, Host{ref})
}
}
}
}
onlyInNewListRefs := Referrers(resp.ReferrerList).findEntriesOnlyInLeftSide(oldFilterListRef)
onlyInOldListRefs := Referrers(oldFilterListRef).findEntriesOnlyInLeftSide(resp.ReferrerList)
inBothListsRefs := Referrers(resp.ReferrerList).findInBoth(oldFilterListRef)
onlyInNewListUAs := Referrers(resp.UserAgentList).findEntriesOnlyInLeftSide(oldFilterListUA)
onlyInOldListUAs := Referrers(oldFilterListUA).findEntriesOnlyInLeftSide(resp.UserAgentList)
inBothListsUAs := Referrers(resp.UserAgentList).findInBoth(oldFilterListUA)
onlyInNewListHosts := Hosts(hosts).findEntriesOnlyInLeftSide(oldFilterListHost)
onlyInOldListHosts := Hosts(oldFilterListHost).findEntriesOnlyInLeftSide(hosts)
inBothListsHosts := Hosts(hosts).findInBoth(oldFilterListHost)
var Ref Referrer
resultsRef := []Referrer{}
for _, Ref = range resp.ReferrerList {
Ref.Referrer = (strings.Replace(Ref.Referrer, ".", "\\.", -1))
Ref.Referrer = (strings.Replace(Ref.Referrer, "+", "\\+", -1))
resultsRef = append(resultsRef, Ref)
}
resp.ReferrerList = resultsRef
resultsUA := []Referrer{}
for _, Ref = range resp.UserAgentList {
Ref.Referrer = (strings.Replace(Ref.Referrer, ".", "\\.", -1))
Ref.Referrer = (strings.Replace(Ref.Referrer, "+", "\\+", -1))
resultsUA = append(resultsUA, Ref)
}
resp.UserAgentList = resultsUA
resultsHost := []Host{}
for _, h := range hosts {
h.Hostname = (strings.Replace(h.Hostname, ".", "\\.", -1))
h.Hostname = (strings.Replace(h.Hostname, "+", "\\+", -1))
resultsHost = append(resultsHost, h)
}
hosts = resultsHost
fmt.Println("\tCompleted")
fmt.Println("")
log.Println("Current Botsifter Bots:")
log.Println("\n#################### CURRENT BotSifter BOTS ####################")
log.Println("Referrers:\n")
log.Println("\tRANK\tNAME\tSCORE")
for _, Ref = range respDisplay.ReferrerList {
log.Println(Ref)
}
log.Println("")
log.Println("User Agents:\n")
log.Println("\tRANK\tNAME\tSCORE")
for _, Ref = range respDisplay.UserAgentList {
log.Println(Ref)
}
log.Println("")
log.Println("\nBotSifter will make the following changes to your GA Account[" + GooFig.UACode + "]:")
log.Println("\n#################### HOST CHANGES ####################")
log.Println("Added Hosts:\n")
if onlyInNewListHosts != nil {
sort.Sort(onlyInNewListHosts)
for _, h := range onlyInNewListHosts {
log.Println(h)
}
} else {
log.Println("\tNONE")
}
log.Println("")
log.Println("Removed Hosts:\n")
if onlyInOldListHosts != nil {
sort.Sort(onlyInOldListUAs)
for _, h := range onlyInOldListHosts {
log.Println(h)
}
} else {
log.Println("\tNONE")
}
// log.Println(strings.Trim(fmt.Sprint(onlyInOldListRefs), "[]"))
log.Println("")
log.Println("Hosts unchange:\n")
if inBothListsHosts != nil {
sort.Sort(inBothListsUAs)
for _, h := range inBothListsHosts {
log.Println(h)
}
} else {
log.Println("\tNONE")
}
log.Println("\n#################### REFERRER CHANGES ####################")
log.Println("Added Referrers:\n")
if onlyInNewListRefs != nil {
log.Println("\tRANK\tNAME\tSCORE")
sort.Sort(onlyInNewListRefs)
for _, Ref = range onlyInNewListRefs {
log.Println(Ref)
}
} else {
log.Println("\tNONE")
}
log.Println("")
log.Println("Removed Referrers:\n")
if onlyInOldListRefs != nil {
log.Println("\tRANK\tNAME\tSCORE")
sort.Sort(onlyInOldListRefs)
for _, Ref = range onlyInOldListRefs {
log.Println(Ref)
}
} else {
log.Println("\tNONE")
}
// log.Println(strings.Trim(fmt.Sprint(onlyInOldListRefs), "[]"))
log.Println("")
log.Println("Referrers unchange:\n")
if inBothListsRefs != nil {
log.Println("\tRANK\tNAME\tSCORE")
sort.Sort(inBothListsRefs)
for _, Ref = range inBothListsRefs {
log.Println(Ref)
}
} else {
log.Println("\tNONE")
}
log.Println("\n#################### USER AGENTS CHANGES ####################")
log.Println("Added User Agents:\n")
if onlyInNewListUAs != nil {
log.Println("\tRANK\tNAME\tSCORE")
sort.Sort(onlyInNewListUAs)
for _, Ref = range onlyInNewListUAs {
log.Println(Ref)
}
} else {
log.Println("\tNONE")
}
log.Println("")
log.Println("Removed User Agents:\n")
if onlyInOldListUAs != nil {
log.Println("\tRANK\tNAME\tSCORE")
sort.Sort(onlyInOldListUAs)
for _, Ref = range onlyInOldListUAs {
log.Println(Ref)
}
} else {
log.Println("\tNONE")
}
// log.Println(strings.Trim(fmt.Sprint(onlyInOldListRefs), "[]"))
log.Println("")
log.Println("User Agents unchange:\n")
if inBothListsUAs != nil {
log.Println("\tRANK\tNAME\tSCORE")
sort.Sort(inBothListsUAs)
for _, Ref = range inBothListsUAs {
log.Println(Ref)
}
} else {
log.Println("\tNONE")
}
w.Flush()
// log.Println(strings.Trim(fmt.Sprint(inBothListsRefs), "[]"))
log.Println("")
if *confirmFlag == false {
length := len(filters.Items)
var bar *pb.ProgressBar
if length != 0 {
bar = pb.StartNew(length)
bar.SetMaxWidth(80)
fmt.Println("Deleting old BotSifter filters ")
for _, eachFilter := range filters.Items {
if strings.Contains(eachFilter.Name, "BotSifter") == true {
service.Management.Filters.Delete(AccountID, eachFilter.Id).Do()
}
bar.Increment()
time.Sleep(time.Millisecond * 250)
}
bar.Finish()
} else {
fmt.Println("No filters to delete")
}
//If cleanFlag entered then end program here
if *cleanFlag == true {
return
}
//If view is not defined in config file then ask user which one to apply filters too
if GooFig.View == "" {
//List all views
profiles, err := service.Management.Profiles.List(AccountID, WebPropertyId).Do()
if err != nil {
log.Println(err)
}
for i, profile := range profiles.Items {
fmt.Printf("%d. %s\n", i, profile.Name)
}
reader := bufio.NewReader(os.Stdin)
fmt.Printf("Please select a profile to apply filters too: ")
index := 0
for {
selectedProfileIndex, _ := reader.ReadString('\n')
index, err = strconv.Atoi(strings.TrimSuffix(selectedProfileIndex, "\n"))
if err == nil && index < len(profiles.Items) {
break
} else {
fmt.Println("Invalid input", index, err)
}
}
GooFig.View = profiles.Items[index].Id
}
//Prepare filters
fmt.Println("Preparing Filter - combining multiple Referrers")
var filterList []string
filterList = RefMerge(resp.ReferrerList, true)
//Build new filters from ReferrerList struct
fmt.Println("Creating referral filters")
var FilterIds []string
length = len(filterList)
bar = pb.StartNew(length)
bar.SetMaxWidth(80)
for i, newFilter := range filterList {
counter := strconv.Itoa(i + 1)
filter := &analytics.Filter{
Name: "BotSifter Ref Spam" + counter,
Type: "EXCLUDE",
ExcludeDetails: &analytics.FilterExpression{
Field: "REFERRAL",
ExpressionValue: newFilter,
CaseSensitive: false,
},
}
filter, err = service.Management.Filters.Insert(AccountID, filter).Do()
if err != nil {
fmt.Print("\n")
fmt.Println(err)
return
}
//Save filter Ids for later
FilterIds = append(FilterIds, filter.Id)
bar.Increment()
time.Sleep(time.Millisecond * 250)
}
bar.Finish()
//Prepare filters
fmt.Println("Preparing Filter - combining multiple User Agents")
var filterListua []string
filterListua = RefMerge(resp.UserAgentList, true)
//Build new filters from ReferrerList struct
fmt.Println("Creating User Agent filters")
length = len(filterListua)
bar = pb.StartNew(length)
bar.SetMaxWidth(80)
for i, newFilter := range filterListua {
counter := strconv.Itoa(i + 1)
filter := &analytics.Filter{
Name: "BotSifter UA Spam" + counter,
Type: "EXCLUDE",
ExcludeDetails: &analytics.FilterExpression{
Field: "USER_DEFINED_VALUE",
ExpressionValue: newFilter,
CaseSensitive: false,
},
}
filter, err = service.Management.Filters.Insert(AccountID, filter).Do()
if err != nil {
fmt.Print("\n")
fmt.Println(err)
return
}
//Save filter Ids for later
FilterIds = append(FilterIds, filter.Id)
bar.Increment()
time.Sleep(time.Millisecond * 250)
}
bar.Finish()
if len(hosts) != 0 {
var hostList []string
hostList = HostMerge(hosts, false)
//If there's hosts build "include Hostname" rule(s)
fmt.Println("Creating Hostname filter(s)")
length = len(hostList)
bar = pb.StartNew(length)
bar.SetMaxWidth(80)
for i, newHost := range hostList {
counter := strconv.Itoa(i)
filter := &analytics.Filter{
Name: "BotSifter Hostname Spam" + counter,
Type: "INCLUDE",
IncludeDetails: &analytics.FilterExpression{
Field: "PAGE_HOSTNAME",
ExpressionValue: newHost,
CaseSensitive: false,
},
}
filter, err = service.Management.Filters.Insert(AccountID, filter).Do()
if err != nil {
log.Println(err)
return
}
//Save filter Ids for later
FilterIds = append(FilterIds, filter.Id)
bar.Increment()
time.Sleep(time.Millisecond * 250)
}
bar.Finish()
}
//connecting built filters to profile user selected
fmt.Println("Connecting filters to profile")
length = len(FilterIds)
bar = pb.StartNew(length)
bar.SetMaxWidth(80)
for _, newLink := range FilterIds {
profilefilterlink := &analytics.ProfileFilterLink{
FilterRef: &analytics.FilterRef{Id: newLink},
}
_, err := service.Management.ProfileFilterLinks.Insert(AccountID, WebPropertyId, GooFig.View, profilefilterlink).Do()
if err != nil {
log.Println("Error Connecting Filter to View\n")
}
bar.Increment()
time.Sleep(time.Millisecond * 250)
}
bar.Finish()
}
fmt.Println("Saving configuration data to " + *configFile)
//Marshal data to save into config file
data, err = yaml.Marshal(&GooFig)
if err != nil {
log.Println(err)
return
}
//Write config file
err = ioutil.WriteFile(*configFile, data, 0644)
if err != nil {
log.Println(err)
return
}
fmt.Println("Completed")
}