func main() { geoip.Quiet = true file6 := "./GeoIPv6-20120730.dat" gi6 := geoip.Open(file6) if gi6 == nil { fmt.Printf("Could not open GeoIPv6 database\n") } gi := geoip.Open() if gi == nil { fmt.Printf("Could not open GeoIP database\n") } if gi != nil { test4(*gi, "207.171.7.51") test4(*gi, "127.0.0.1") } if gi6 != nil { ip := "2607:f238:2::5" country := gi6.GetCountry_v6(ip) fmt.Println(ip, ": ") display(country) } }
func main() { file6 := "./GeoIPv6.dat" gi6, err := geoip.Open(file6) if err != nil { fmt.Printf("Could not open GeoIPv6 database: %s\n", err) } gi, err := geoip.Open() if err != nil { fmt.Printf("Could not open GeoIP database: %s\n", err) } giasn, err := geoip.Open("./GeoIPASNum.dat") if err != nil { fmt.Printf("Could not open GeoIPASN database: %s\n", err) } giasn6, err := geoip.Open("./GeoIPASNumv6.dat") if err != nil { fmt.Printf("Could not open GeoIPASN database: %s\n", err) } gcity, err := geoip.Open("./GeoLiteCity.dat") if err != nil { fmt.Printf("Could not open GeoIPASN database: %s\n", err) } if giasn != nil { ip := "207.171.7.51" asn, netmask := giasn.GetName(ip) fmt.Printf("%s: %s (netmask /%d)\n", ip, asn, netmask) } if gcity != nil { ip := "10.47.173.157" country, _ := gcity.GetRegion(ip) fmt.Printf("%s: %s\n", ip, country) } if gi != nil { test4(*gi, "207.171.7.51") test4(*gi, "127.0.0.1") } if gi6 != nil { ip := "2607:f238:2::5" country, netmask := gi6.GetCountry_v6(ip) var asn string var asn_netmask int if giasn6 != nil { asn, asn_netmask = giasn6.GetNameV6(ip) } fmt.Printf("%s: %s/%d %s/%d\n", ip, country, netmask, asn, asn_netmask) } }
func init() { var err error g4, err = geoip.Open("/usr/share/GeoIP/GeoIP.dat") if err != nil { fmt.Println("can't open GeoIP.dat") } g6, err = geoip.Open("/usr/share/GeoIP/GeoIPv6.dat") if err != nil { fmt.Println("can't open GeoIPv6.dat") } rand.Seed(time.Now().UnixNano() * int64(os.Getpid())) }
func NewTaskServer(s store.Store, serverUrl, templatesPath, geoipDatabase string) *measurementsServerState { queries := make(chan *store.Query) go s.WriteQueries(queries) measurementIds := generateMeasurementIds() go s.ScheduleTaskFunctions() taskRequests := make(chan *store.TaskRequest) go s.Tasks(taskRequests) countResultsRequests := make(chan store.CountResultsRequest) go s.CountResultsForReferrer(countResultsRequests) geolocator, err := geoip.Open(geoipDatabase) if err != nil { log.Fatalf("error opening geoip database: %v", err) } return &measurementsServerState{ Store: s, Templates: template.Must(template.ParseGlob(filepath.Join(templatesPath, "[a-zA-Z]*"))), Queries: queries, MeasurementIds: measurementIds, TaskRequests: taskRequests, CountResultsRequests: countResultsRequests, ServerUrl: serverUrl, Geolocator: geolocator, } }
func setupGeoIP() *geoip.GeoIP { gi := geoip.Open() if gi == nil { log.Printf("Could not open GeoIP database\n") } return gi }
func loadGeoIpDb(dbName string) *geoip.GeoIP { // Open the GeoIP database geo, geoErr := geoip.Open(dbName) if geoErr != nil { fmt.Printf("Warning, could not open GeoIP database: %s\n", geoErr) } return geo }
func geoipLookup(configuration *Configuration, actor string) *geoip.GeoIPRecord { gi, err := geoip.Open(configuration.GeoIPDatabase) if err != nil { fmt.Printf("GeoIP: Could not open %s\n", configuration.GeoIPDatabase) return &geoip.GeoIPRecord{} } return gi.GetRecord(actor) }
func main() { gi, err := geoip.Open("GeoIP.dat") if err != nil { fmt.Printf("Could not open GeoIP database: %s\n", err) } if gi != nil { test4(*gi, "207.171.7.51") test4(*gi, "10.47.173.157") test4(*gi, "8.8.8.8") } rc, _ := gi.GetName("202.13.34.13") fmt.Println(rc) }
func NewPostgresDatastore() (Datastore, error) { db, err := sql.Open("postgres", "") if err != nil { return nil, fmt.Errorf("Error connecting to Postgres database: %s", err) } geolocator, err := geoip.Open(geoipDatabase) if err != nil { return nil, err } return PostgresDatastore{db, geolocator}, nil }
func ParseRequest(args handler.RegexpArgs, r *http.Request) Event { // move it into init gi, err := geoip.Open(GEOIP_DATA) if err != nil { log.Fatal("Could not open GEOIP") } remote_ip := strings.Split(r.RemoteAddr, IP_SEP)[0] // FIXME scopes? // apparently I have no clue in understanding difference between = and := country_code := "???" //tz := "???" if strings.HasPrefix(remote_ip, "192.168.") || strings.HasPrefix(remote_ip, "10.") || strings.HasPrefix(remote_ip, "127.") { //server side country_code = "GB" //tz = "Europe/London" } else { //client side country_code = "???" // return self.geoip.country_code_by_addr(self.request.remote_ip) //tz = "???" //return self.geoip.time_zone_by_addr(self.request.remote_ip) } country, geo_err := gi.GetCountry(remote_ip) if geo_err != 0 { log.Fatal("Could not resolve country") } log.Printf(country) user_agent := r.Header.Get("User-Agent") uainfo := uaparser.Parse(user_agent) event := Event{ recipient_identifier: args["recipient_id"], job_identifier: args["job_id"], blakey_identifier: args["blakey_id"], link_id: args["link_id"], link: args["link"], remote_ip: remote_ip, user_agent: user_agent, os: strings.Trim(uainfo.OS.Name+" "+uainfo.OS.Version, " "), client: uainfo.Browser.Name, client_version: uainfo.Browser.Version, country_code: country_code, } return event }
func Parser3(lines []string, session map[string]Client) string { file := "/usr/share/GeoIP/GeoIP.dat" gi, err := geoip.Open(file) if err != nil { fmt.Printf("Could not open GeoIP database please install in /usr/share/GeoIP/\n") } var update string for i := range lines { var c Client tmp := strings.Split(lines[i], "\t") if i == 1 { update = tmp[1] } if i > 2 && len(tmp) == 9 { c.Name = tmp[1] c.Vpn_ip = tmp[3] tmp1 := strings.Split(tmp[2], ":") c.Real_ip = tmp1[0] c.Real_port = tmp1[1] upload, _ := strconv.ParseInt(tmp[5], 10, 32) download, _ := strconv.ParseInt(tmp[4], 10, 32) c.Upload = fmt.Sprint(upload/1000, " Kb") c.Download = fmt.Sprint(download/1000, " Kb") if gi != nil { country, _ := gi.GetCountry(c.Real_ip) if len(country) < 2 { c.Country = "Lan" } else { c.Country = country } } if len(tmp) > 6 { c.Connected = tmp[6] } session[tmp[1]] = c } } return update }
func main() { var geoipDatabase, logfile string flag.StringVar(&geoipDatabase, "geoip_database", "/usr/share/GeoIP/GeoIP.dat", "Path of GeoIP database") flag.StringVar(&logfile, "logfile", "", "Write logs to this file instead of stdout") flag.Parse() if logfile != "" { f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) if err != nil { log.Fatalf("error opening logfile: %v", err) } defer f.Close() log.SetOutput(f) } log.Printf("starting") s := store.Open() defer s.Close() geolocator, err := geoip.Open(geoipDatabase) if err != nil { panic(err) } queries := s.UnparsedQueries() parsedQueries := parseQueries(queries, geolocator) s.WriteParsedQueries(parsedQueries) results := s.UnparsedResults() parsedResults := parseResults(results, geolocator) s.WriteParsedResults(parsedResults) if err := s.ComputeResultsTables(); err != nil { panic(err) } log.Printf("done") }
func (ld *GeoIpDecoder) Init(config interface{}) (err error) { conf := config.(*GeoIpDecoderConfig) if string(conf.SourceIpField) == "" { return errors.New("`source_ip_field` must be specified") } if conf.TargetField == "" { return errors.New("`target_field` must be specified") } ld.TargetField = conf.TargetField ld.SourceIpField = conf.SourceIpField if ld.gi == nil { ld.gi, err = geoip.Open(conf.DatabaseFile) } if err != nil { return fmt.Errorf("Could not open GeoIP database: %s\n") } return }
func main() { if len(os.Args) < 2 { fmt.Println("\nUsage: parselog [filepath] [csv|json]\n") os.Exit(0) } filePath := os.Args[1] fmt.Println("\n-------------------------------------------") fmt.Println("Reading file:", filePath, "\n") f, err := os.Open(filePath) if err != nil { fmt.Printf("Error! Could not open file: %v\n", err) fmt.Println("") os.Exit(0) } ext := "" if os.Args[2] == "json" { ext = ".json" } else { ext = ".csv" } // Attempt to update the GeoIP DB //fmt.Println("Downloading "+geoipdb_base+geoipdb_city) // If not updated in a month, then attempt to update the GeoIP DB files if tools.FileExists(geoipdb_city) == false { if tools.Download(geoipdb_base + geoipdb_city) { fmt.Println("Update to " + geoipdb_city + " successful") } else { fmt.Println("Could not update " + geoipdb_city) } } // Now open the GeoIP database //var geo *geoip.GeoIP //if tools.FileExists(geoipdb_city) == true { geo, geoErr := geoip.Open(geoipdb_city) if geoErr != nil { fmt.Printf("Warning, could not open GeoIP database: %s\n", err) } //} else { // fmt.Println("File doesn't exist!") //} r := bufio.NewReader(f) s, e := Readln(r) lineNum := 0 mapIP := make(map[string]int, 10) mapURI := make(map[string]int, 10) mapUA := make(map[string]int, 10) for e == nil { // Read a line from the file s, e = Readln(r) // Attempt to extrac the IP and store it inn a map matches := regexp.MustCompile(`^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(s) if len(matches) >= 2 { mapIP[matches[1]] = mapIP[matches[1]] + 1 } /***********************************************************/ // Attempt to extract and store the URI in a map matches = regexp.MustCompile(`"(GET|POST)[^\\"]*(\\"[^\\"]*)*"`).FindStringSubmatch(s) if len(matches) >= 1 { key := regexp.MustCompile("(GET|POST|\"|HTTP/1.[0-1])").ReplaceAllString(matches[0], "") mapURI[key] = mapURI[key] + 1 } /***********************************************************/ // Attempt to extract and store the Useragent in a map matches = regexp.MustCompile(`"[^\\"]*(\\"[^\\"]*)*"$`).FindStringSubmatch(s) if len(matches) >= 1 { mapUA[regexp.MustCompile("\"").ReplaceAllString(matches[0], "")] = mapUA[matches[0]] + 1 } /***********************************************************/ lineNum++ matches = nil } /*********** Now sort the maps by value descending using merge sort ******/ sMapIP := sortMapByValue(mapIP) mapIP = nil sMapURI := sortMapByValue(mapURI) mapURI = nil sMapUA := sortMapByValue(mapUA) mapUA = nil /************* Now write the data to a csv file ***************/ ts := int(time.Now().Unix()) y, m, d := time.Now().Date() fileList := [3]FileElement{ {Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_ip", Ext: ext, Size: 0}, {Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_uri", Ext: ext, Size: 0}, {Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_ua", Ext: ext, Size: 0}, } for nf := 0; nf < 3; nf++ { fh, err := os.Create(fileList[nf].FullName()) check(err) fileList[nf].Handle = fh defer fileList[nf].Handle.Close() } // Update each file element property if ext == "json" { fileList[0].Size = mapPairToJson("IP"+csvd+"Hits"+csvd+"City"+csvd+"Country", sMapIP, fileList[0].Handle) fileList[1].Size = mapPairToJson("URI"+csvd+"Hits\n", sMapURI, fileList[1].Handle) fileList[2].Size = mapPairToJson("UA"+csvd+"Hits\n", sMapUA, fileList[2].Handle) } else { fileList[0].Size = mapPairToCSV("IP"+csvd+"Hits"+csvd+"City"+csvd+"Country", sMapIP, fileList[0].Handle, geo) fileList[1].Size = mapPairToCSV("URI"+csvd+"Hits", sMapURI, fileList[1].Handle, geo) fileList[2].Size = mapPairToCSV("UA"+csvd+"Hits", sMapUA, fileList[2].Handle, geo) } numLines := (lineNum - 1) numIp := len(sMapIP) numUri := len(sMapURI) numUa := len(sMapUA) fmt.Println("Processing details:\n-----------------------") fmt.Println("Total lines in log file: ", numLines) fmt.Println("Total unique IPs: ", numIp) fmt.Println("Total unique User-Agents: ", numUa) fmt.Println("Total unique URIs: ", numUri, "\n") fmt.Println("Top 5 IPs\n----------------------------") for i := (numIp - 1); i > 0; i-- { record := geo.GetRecord(sMapIP[i].Key) fmt.Println("IP:", sMapIP[i].Key, "Hits:", sMapIP[i].Value, "Location:", record.City+", "+record.CountryName) if i == (numIp - 5) { break } } fmt.Println("") fmt.Println("Top 5 URIs\n----------------------------") for i := (numUri - 1); i > 0; i-- { fmt.Println("URI:", sMapURI[i].Key, "\nHits:", sMapURI[i].Value, "\n") if i == (numUri - 5) { break } } fmt.Println("Top 5 UAs\n----------------------------") for i := (numUa - 1); i > 0; i-- { fmt.Println("UA:", sMapUA[i].Key, "\nHits:", sMapUA[i].Value, "\n") if i == (numUa - 5) { break } } fmt.Println("") fmt.Println("View the following files for full report:", "\n") totalBytes := 0 for i := 0; i < 3; i++ { fmt.Println("\t" + fileList[i].FullName() + " (" + strconv.Itoa(fileList[i].Size) + " bytes)") totalBytes += fileList[i].Size } fmt.Println("\nTotal bytes written:", totalBytes) fmt.Println("\n----------------------------------\n") }
func (worker *Worker) Load() error { worker.Config = &Config{} var err error worker.GeoDB, err = geoip.Open("GeoLiteCity.dat") if err != nil { return err } consulConfig := &consul.Config{ Address: "127.0.0.1:8500", Scheme: "http", HttpClient: http.DefaultClient, } client, err := consul.NewClient(consulConfig) if err != nil { return err } err = worker.Config.fillFromConsul(client, "silvia") if err != nil { return err } ringSize, err := strconv.Atoi(worker.Config.RingSize) if err != nil { return err } worker.Stats = &Stats{ AdjustSuccessRing: &AdjustRing{Size: ringSize}, AdjustFailRing: &AdjustRing{Size: ringSize}, SnowplowSuccessRing: &SnowplowRing{Size: ringSize}, SnowplowFailRing: &SnowplowRing{Size: ringSize}, } worker.Stats.StartTime = time.Now() worker.AdjustRequestBus = make(chan []byte) worker.SnowplowRequestBus = make(chan []byte) worker.AdjustEventBus = make(chan *AdjustEvent) worker.SnowplowEventBus = make(chan *SnowplowEvent) port, err := strconv.Atoi(worker.Config.Port) if err != nil { return nil } worker.ConsulServiceID = uuid.NewV4().String() checks := consul.AgentServiceChecks{ &consul.AgentServiceCheck{ TTL: "10s", }, &consul.AgentServiceCheck{ HTTP: "http://localhost:" + worker.Config.Port + "/v1/status", Interval: "10s", Timeout: "1s", }, } service := &consul.AgentServiceRegistration{ ID: worker.ConsulServiceID, Name: "silvia", Port: port, Checks: checks, } worker.ConsulAgent = client.Agent() err = worker.ConsulAgent.ServiceRegister(service) if err != nil { return err } go func() { for { <-time.After(8 * time.Second) err = worker.ConsulAgent.PassTTL("service:"+worker.ConsulServiceID+":1", "Internal TTL ping") if err != nil { log.Println(err) } } }() log.Println("Service registred with ID:", worker.ConsulServiceID) return nil }