func getReader(file io.Reader) (reader *gonx.Reader, err error) {
if *format != "" {
reader = gonx.NewReader(file, *format)
} else {
if *nginxConfig == "" || *nginxFormat == "" {
badUsage()
}
cfg, err := os.Open(*nginxConfig)
if err != nil {
return nil, err
}
defer cfg.Close()
reader, err = gonx.NewNginxReader(file, cfg, *nginxFormat)
}
return
}
func main() {
addr := flag.String("addr", "", "hostport of SOLR to query, e.g. 10.1.1.7:8085 or https://10.1.1.8:1234")
run := flag.Bool("run", false, "actually run the requests")
ignore := flag.Bool("ignore", false, "ignore errors")
w := flag.Int("w", 1, "number of requests to run in parallel")
version := flag.Bool("v", false, "show version and exit")
flag.Parse()
if *version {
fmt.Println(Version)
os.Exit(0)
}
reader := gonx.NewReader(os.Stdin, format)
queue := make(chan string)
out := make(chan string)
done := make(chan bool)
go writer(out, done)
opts := options{addr: *addr, ignore: *ignore}
var wg sync.WaitGroup
for i := 0; i < *w; i++ {
wg.Add(1)
go worker(queue, out, opts, &wg)
}
for {
rec, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
f, err := rec.Field("request")
if err != nil {
log.Fatal(err)
}
if !strings.HasPrefix(f, "GET") {
continue
}
// we only want the path
p := replacer.Replace(f)
if !strings.HasPrefix(p, "/solr/biblio/select") {
continue
}
link := path.Join(*addr, p)
if !strings.HasPrefix(link, "http") {
link = "http://" + link
}
if *run {
queue <- link
} else {
log.Println(link)
}
}
close(queue)
wg.Wait()
close(out)
<-done
}
func main() {
file, err := os.Open("/home/juno/workspace/SearchGoogleLogParser/logscollector/rawallaccess.log")
if err != nil {
panic(err)
}
defer file.Close()
format := `$remote_addr $remote_user [$time_local] $host "$request" "$request_time" "$upstream_response_time" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$uid_got" "$uid_set" "$geoip_country_code"`
reader := gonx.NewReader(file, format)
var entryArr []domains.Entry
newHitsEntry := entryHandler.NewEmptyHitsEntry()
bing_newHitsEntry := entryHandler.NewEmptyHitsEntry()
for {
rec, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
panic(err)
}
remote_addr, _ := rec.Field("remote_addr")
time_local, _ := rec.Field("time_local")
host, _ := rec.Field("host")
request, _ := rec.Field("request")
status, _ := rec.Field("status")
body_bytes_sent, _ := rec.Field("body_bytes_sent")
http_referer, _ := rec.Field("http_referer")
http_user_agent, _ := rec.Field("http_user_agent")
geoip_country_code, _ := rec.Field("geoip_country_code")
uid_set, _ := rec.Field("uid_set")
uid_got, _ := rec.Field("uid_got")
entry := domains.Entry{
Remote_addr: remote_addr,
Time_local: time_local,
Host: host,
Request: request,
Status: status,
Body_bytes_sent: body_bytes_sent,
Http_referer: http_referer,
Http_user_agent: http_user_agent,
Geoip_country_code: geoip_country_code,
Uid_set: uid_set,
Uid_got: uid_got,
}
entryArr = append(entryArr, entry)
}
allsearchengines := make(map[string]int)
for _, entry := range entryArr {
searchengine := (*newHitsEntry).Handle(entry, "www.google")
(*bing_newHitsEntry).Handle(entry, "www.bing")
if searchengine != "" {
_, ok := allsearchengines[searchengine]
if ok {
i := allsearchengines[searchengine]
allsearchengines[searchengine] = i + 1
} else {
allsearchengines[searchengine] = 0
}
}
}
db, err := sql.Open("sqlite3", "/home/juno/git/goFastCgi/goFastCgi/singo.db")
if err != nil {
log.Fatal(err)
}
defer db.Close()
alloldphrases := getallphrases.GetAll(*db, "fi_FI", "porno")
alloldkeywords := getallkeywords.GetAll(*db, "fi_FI", "porno")
var phrasestoUpdate []string
var phrasestoInsert []string
var keywodstoUpdate []domains.KeywordObj
var keywodstoInsert []domains.KeywordObj
for key := range entryHandler.NewQueryEntry.QueryEntry {
_, ok := alloldphrases[key]
if ok {
phrasestoUpdate = append(phrasestoUpdate, key)
} else {
phrasestoInsert = append(phrasestoInsert, key)
}
}
for key := range entryHandler.NewKewordsEntry.KeywordsEntry {
keywordObj := examkeyword.ExamKeyword(key)
if keywordObj.Keyword != "" && len(keywordObj.Keyword) > 2 {
_, ok := alloldkeywords[keywordObj.Keyword]
if ok {
keywodstoUpdate = append(keywodstoUpdate, keywordObj)
} else {
keywodstoInsert = append(keywodstoInsert, keywordObj)
}
}
}
updateallphrases.UpdateAll(*db, "fi_FI", "porno", phrasestoUpdate)
insertnewphrases.InsertAll(*db, "fi_FI", "porno", phrasestoInsert)
updateallkeywords.UpdateAll(*db, "fi_FI", "porno", keywodstoUpdate)
insertnewkeywords.InsertAll(*db, "fi_FI", "porno", keywodstoInsert)
var g_total int = 0
var b_total int = 0
var s_total int = 0
var buffer bytes.Buffer
currenttime := time.Now().Local()
buffer.WriteString("\n" + currenttime.Format("2006-01-02 15:04:05 +0300") + "\n\n")
buffer.WriteString("google -------------------------\n\n")
google_records := sort_map_by_value.SortByValue(newHitsEntry.Domhits)
for _, record := range google_records {
buffer.WriteString(record.Keyval + " " + strconv.Itoa(record.Valval) + "\n")
g_total = g_total + record.Valval
}
buffer.WriteString("-------------------------\n")
buffer.WriteString("g_total " + strconv.Itoa(len(newHitsEntry.Domhits)) + " " + strconv.Itoa(g_total) + "\n\n")
buffer.WriteString("bing -------------------------\n\n")
bing_records := sort_map_by_value.SortByValue(bing_newHitsEntry.Domhits)
for _, record := range bing_records {
buffer.WriteString(record.Keyval + " " + strconv.Itoa(record.Valval) + "\n")
b_total = b_total + record.Valval
}
buffer.WriteString("-------------------------\n")
buffer.WriteString("b_total " + strconv.Itoa(len(bing_newHitsEntry.Domhits)) + " " + strconv.Itoa(b_total) + "\n\n")
searchengines_record := sort_map_by_value.SortByValue(allsearchengines)
buffer.WriteString("\nSearchengines-----------------------\n\n")
for _, record := range searchengines_record {
buffer.WriteString(record.Keyval + " " + strconv.Itoa(record.Valval) + "\n")
s_total = s_total + record.Valval
}
buffer.WriteString("\n-------------------------\n")
buffer.WriteString("s_total " + strconv.Itoa(len(allsearchengines)) + " " + strconv.Itoa(s_total) + "\n\n")
file, err = os.OpenFile("stat.txt", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
if err != nil {
panic(err)
}
defer file.Close()
if _, err = file.WriteString(buffer.String()); err != nil {
panic(err)
}
}