func main() {
geoip.Quiet = true
file6 := "./GeoIPv6-20120730.dat"
gi6 := geoip.Open(file6)
if gi6 == nil {
fmt.Printf("Could not open GeoIPv6 database\n")
}
gi := geoip.Open()
if gi == nil {
fmt.Printf("Could not open GeoIP database\n")
}
if gi != nil {
test4(*gi, "207.171.7.51")
test4(*gi, "127.0.0.1")
}
if gi6 != nil {
ip := "2607:f238:2::5"
country := gi6.GetCountry_v6(ip)
fmt.Println(ip, ": ")
display(country)
}
}
func main() {
file6 := "./GeoIPv6.dat"
gi6, err := geoip.Open(file6)
if err != nil {
fmt.Printf("Could not open GeoIPv6 database: %s\n", err)
}
gi, err := geoip.Open()
if err != nil {
fmt.Printf("Could not open GeoIP database: %s\n", err)
}
giasn, err := geoip.Open("./GeoIPASNum.dat")
if err != nil {
fmt.Printf("Could not open GeoIPASN database: %s\n", err)
}
giasn6, err := geoip.Open("./GeoIPASNumv6.dat")
if err != nil {
fmt.Printf("Could not open GeoIPASN database: %s\n", err)
}
gcity, err := geoip.Open("./GeoLiteCity.dat")
if err != nil {
fmt.Printf("Could not open GeoIPASN database: %s\n", err)
}
if giasn != nil {
ip := "207.171.7.51"
asn, netmask := giasn.GetName(ip)
fmt.Printf("%s: %s (netmask /%d)\n", ip, asn, netmask)
}
if gcity != nil {
ip := "10.47.173.157"
country, _ := gcity.GetRegion(ip)
fmt.Printf("%s: %s\n", ip, country)
}
if gi != nil {
test4(*gi, "207.171.7.51")
test4(*gi, "127.0.0.1")
}
if gi6 != nil {
ip := "2607:f238:2::5"
country, netmask := gi6.GetCountry_v6(ip)
var asn string
var asn_netmask int
if giasn6 != nil {
asn, asn_netmask = giasn6.GetNameV6(ip)
}
fmt.Printf("%s: %s/%d %s/%d\n", ip, country, netmask, asn, asn_netmask)
}
}
func init() {
var err error
g4, err = geoip.Open("/usr/share/GeoIP/GeoIP.dat")
if err != nil {
fmt.Println("can't open GeoIP.dat")
}
g6, err = geoip.Open("/usr/share/GeoIP/GeoIPv6.dat")
if err != nil {
fmt.Println("can't open GeoIPv6.dat")
}
rand.Seed(time.Now().UnixNano() * int64(os.Getpid()))
}
func NewTaskServer(s store.Store, serverUrl, templatesPath, geoipDatabase string) *measurementsServerState {
queries := make(chan *store.Query)
go s.WriteQueries(queries)
measurementIds := generateMeasurementIds()
go s.ScheduleTaskFunctions()
taskRequests := make(chan *store.TaskRequest)
go s.Tasks(taskRequests)
countResultsRequests := make(chan store.CountResultsRequest)
go s.CountResultsForReferrer(countResultsRequests)
geolocator, err := geoip.Open(geoipDatabase)
if err != nil {
log.Fatalf("error opening geoip database: %v", err)
}
return &measurementsServerState{
Store: s,
Templates: template.Must(template.ParseGlob(filepath.Join(templatesPath, "[a-zA-Z]*"))),
Queries: queries,
MeasurementIds: measurementIds,
TaskRequests: taskRequests,
CountResultsRequests: countResultsRequests,
ServerUrl: serverUrl,
Geolocator: geolocator,
}
}
func setupGeoIP() *geoip.GeoIP {
gi := geoip.Open()
if gi == nil {
log.Printf("Could not open GeoIP database\n")
}
return gi
}
func loadGeoIpDb(dbName string) *geoip.GeoIP {
// Open the GeoIP database
geo, geoErr := geoip.Open(dbName)
if geoErr != nil {
fmt.Printf("Warning, could not open GeoIP database: %s\n", geoErr)
}
return geo
}
func geoipLookup(configuration *Configuration, actor string) *geoip.GeoIPRecord {
gi, err := geoip.Open(configuration.GeoIPDatabase)
if err != nil {
fmt.Printf("GeoIP: Could not open %s\n", configuration.GeoIPDatabase)
return &geoip.GeoIPRecord{}
}
return gi.GetRecord(actor)
}
func main() {
gi, err := geoip.Open("GeoIP.dat")
if err != nil {
fmt.Printf("Could not open GeoIP database: %s\n", err)
}
if gi != nil {
test4(*gi, "207.171.7.51")
test4(*gi, "10.47.173.157")
test4(*gi, "8.8.8.8")
}
rc, _ := gi.GetName("202.13.34.13")
fmt.Println(rc)
}
func NewPostgresDatastore() (Datastore, error) {
db, err := sql.Open("postgres", "")
if err != nil {
return nil, fmt.Errorf("Error connecting to Postgres database: %s", err)
}
geolocator, err := geoip.Open(geoipDatabase)
if err != nil {
return nil, err
}
return PostgresDatastore{db, geolocator}, nil
}
func ParseRequest(args handler.RegexpArgs, r *http.Request) Event {
// move it into init
gi, err := geoip.Open(GEOIP_DATA)
if err != nil {
log.Fatal("Could not open GEOIP")
}
remote_ip := strings.Split(r.RemoteAddr, IP_SEP)[0]
// FIXME scopes?
// apparently I have no clue in understanding difference between = and :=
country_code := "???"
//tz := "???"
if strings.HasPrefix(remote_ip, "192.168.") || strings.HasPrefix(remote_ip, "10.") || strings.HasPrefix(remote_ip, "127.") {
//server side
country_code = "GB"
//tz = "Europe/London"
} else {
//client side
country_code = "???"
// return self.geoip.country_code_by_addr(self.request.remote_ip)
//tz = "???"
//return self.geoip.time_zone_by_addr(self.request.remote_ip)
}
country, geo_err := gi.GetCountry(remote_ip)
if geo_err != 0 {
log.Fatal("Could not resolve country")
}
log.Printf(country)
user_agent := r.Header.Get("User-Agent")
uainfo := uaparser.Parse(user_agent)
event := Event{
recipient_identifier: args["recipient_id"],
job_identifier: args["job_id"],
blakey_identifier: args["blakey_id"],
link_id: args["link_id"],
link: args["link"],
remote_ip: remote_ip,
user_agent: user_agent,
os: strings.Trim(uainfo.OS.Name+" "+uainfo.OS.Version, " "),
client: uainfo.Browser.Name,
client_version: uainfo.Browser.Version,
country_code: country_code,
}
return event
}
func Parser3(lines []string, session map[string]Client) string {
file := "/usr/share/GeoIP/GeoIP.dat"
gi, err := geoip.Open(file)
if err != nil {
fmt.Printf("Could not open GeoIP database please install in /usr/share/GeoIP/\n")
}
var update string
for i := range lines {
var c Client
tmp := strings.Split(lines[i], "\t")
if i == 1 {
update = tmp[1]
}
if i > 2 && len(tmp) == 9 {
c.Name = tmp[1]
c.Vpn_ip = tmp[3]
tmp1 := strings.Split(tmp[2], ":")
c.Real_ip = tmp1[0]
c.Real_port = tmp1[1]
upload, _ := strconv.ParseInt(tmp[5], 10, 32)
download, _ := strconv.ParseInt(tmp[4], 10, 32)
c.Upload = fmt.Sprint(upload/1000, " Kb")
c.Download = fmt.Sprint(download/1000, " Kb")
if gi != nil {
country, _ := gi.GetCountry(c.Real_ip)
if len(country) < 2 {
c.Country = "Lan"
} else {
c.Country = country
}
}
if len(tmp) > 6 {
c.Connected = tmp[6]
}
session[tmp[1]] = c
}
}
return update
}
func main() {
var geoipDatabase, logfile string
flag.StringVar(&geoipDatabase, "geoip_database", "/usr/share/GeoIP/GeoIP.dat", "Path of GeoIP database")
flag.StringVar(&logfile, "logfile", "", "Write logs to this file instead of stdout")
flag.Parse()
if logfile != "" {
f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
log.Fatalf("error opening logfile: %v", err)
}
defer f.Close()
log.SetOutput(f)
}
log.Printf("starting")
s := store.Open()
defer s.Close()
geolocator, err := geoip.Open(geoipDatabase)
if err != nil {
panic(err)
}
queries := s.UnparsedQueries()
parsedQueries := parseQueries(queries, geolocator)
s.WriteParsedQueries(parsedQueries)
results := s.UnparsedResults()
parsedResults := parseResults(results, geolocator)
s.WriteParsedResults(parsedResults)
if err := s.ComputeResultsTables(); err != nil {
panic(err)
}
log.Printf("done")
}
func (ld *GeoIpDecoder) Init(config interface{}) (err error) {
conf := config.(*GeoIpDecoderConfig)
if string(conf.SourceIpField) == "" {
return errors.New("`source_ip_field` must be specified")
}
if conf.TargetField == "" {
return errors.New("`target_field` must be specified")
}
ld.TargetField = conf.TargetField
ld.SourceIpField = conf.SourceIpField
if ld.gi == nil {
ld.gi, err = geoip.Open(conf.DatabaseFile)
}
if err != nil {
return fmt.Errorf("Could not open GeoIP database: %s\n")
}
return
}
func main() {
if len(os.Args) < 2 {
fmt.Println("\nUsage: parselog [filepath] [csv|json]\n")
os.Exit(0)
}
filePath := os.Args[1]
fmt.Println("\n-------------------------------------------")
fmt.Println("Reading file:", filePath, "\n")
f, err := os.Open(filePath)
if err != nil {
fmt.Printf("Error! Could not open file: %v\n", err)
fmt.Println("")
os.Exit(0)
}
ext := ""
if os.Args[2] == "json" {
ext = ".json"
} else {
ext = ".csv"
}
// Attempt to update the GeoIP DB
//fmt.Println("Downloading "+geoipdb_base+geoipdb_city)
// If not updated in a month, then attempt to update the GeoIP DB files
if tools.FileExists(geoipdb_city) == false {
if tools.Download(geoipdb_base + geoipdb_city) {
fmt.Println("Update to " + geoipdb_city + " successful")
} else {
fmt.Println("Could not update " + geoipdb_city)
}
}
// Now open the GeoIP database
//var geo *geoip.GeoIP
//if tools.FileExists(geoipdb_city) == true {
geo, geoErr := geoip.Open(geoipdb_city)
if geoErr != nil {
fmt.Printf("Warning, could not open GeoIP database: %s\n", err)
}
//} else {
// fmt.Println("File doesn't exist!")
//}
r := bufio.NewReader(f)
s, e := Readln(r)
lineNum := 0
mapIP := make(map[string]int, 10)
mapURI := make(map[string]int, 10)
mapUA := make(map[string]int, 10)
for e == nil {
// Read a line from the file
s, e = Readln(r)
// Attempt to extrac the IP and store it inn a map
matches := regexp.MustCompile(`^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(s)
if len(matches) >= 2 {
mapIP[matches[1]] = mapIP[matches[1]] + 1
}
/***********************************************************/
// Attempt to extract and store the URI in a map
matches = regexp.MustCompile(`"(GET|POST)[^\\"]*(\\"[^\\"]*)*"`).FindStringSubmatch(s)
if len(matches) >= 1 {
key := regexp.MustCompile("(GET|POST|\"|HTTP/1.[0-1])").ReplaceAllString(matches[0], "")
mapURI[key] = mapURI[key] + 1
}
/***********************************************************/
// Attempt to extract and store the Useragent in a map
matches = regexp.MustCompile(`"[^\\"]*(\\"[^\\"]*)*"$`).FindStringSubmatch(s)
if len(matches) >= 1 {
mapUA[regexp.MustCompile("\"").ReplaceAllString(matches[0], "")] = mapUA[matches[0]] + 1
}
/***********************************************************/
lineNum++
matches = nil
}
/*********** Now sort the maps by value descending using merge sort ******/
sMapIP := sortMapByValue(mapIP)
mapIP = nil
sMapURI := sortMapByValue(mapURI)
mapURI = nil
sMapUA := sortMapByValue(mapUA)
mapUA = nil
/************* Now write the data to a csv file ***************/
ts := int(time.Now().Unix())
y, m, d := time.Now().Date()
fileList := [3]FileElement{
{Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_ip", Ext: ext, Size: 0},
{Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_uri", Ext: ext, Size: 0},
{Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_ua", Ext: ext, Size: 0},
}
for nf := 0; nf < 3; nf++ {
fh, err := os.Create(fileList[nf].FullName())
check(err)
fileList[nf].Handle = fh
defer fileList[nf].Handle.Close()
}
// Update each file element property
if ext == "json" {
fileList[0].Size = mapPairToJson("IP"+csvd+"Hits"+csvd+"City"+csvd+"Country", sMapIP, fileList[0].Handle)
fileList[1].Size = mapPairToJson("URI"+csvd+"Hits\n", sMapURI, fileList[1].Handle)
fileList[2].Size = mapPairToJson("UA"+csvd+"Hits\n", sMapUA, fileList[2].Handle)
} else {
fileList[0].Size = mapPairToCSV("IP"+csvd+"Hits"+csvd+"City"+csvd+"Country", sMapIP, fileList[0].Handle, geo)
fileList[1].Size = mapPairToCSV("URI"+csvd+"Hits", sMapURI, fileList[1].Handle, geo)
fileList[2].Size = mapPairToCSV("UA"+csvd+"Hits", sMapUA, fileList[2].Handle, geo)
}
numLines := (lineNum - 1)
numIp := len(sMapIP)
numUri := len(sMapURI)
numUa := len(sMapUA)
fmt.Println("Processing details:\n-----------------------")
fmt.Println("Total lines in log file: ", numLines)
fmt.Println("Total unique IPs: ", numIp)
fmt.Println("Total unique User-Agents: ", numUa)
fmt.Println("Total unique URIs: ", numUri, "\n")
fmt.Println("Top 5 IPs\n----------------------------")
for i := (numIp - 1); i > 0; i-- {
record := geo.GetRecord(sMapIP[i].Key)
fmt.Println("IP:", sMapIP[i].Key, "Hits:", sMapIP[i].Value, "Location:", record.City+", "+record.CountryName)
if i == (numIp - 5) {
break
}
}
fmt.Println("")
fmt.Println("Top 5 URIs\n----------------------------")
for i := (numUri - 1); i > 0; i-- {
fmt.Println("URI:", sMapURI[i].Key, "\nHits:", sMapURI[i].Value, "\n")
if i == (numUri - 5) {
break
}
}
fmt.Println("Top 5 UAs\n----------------------------")
for i := (numUa - 1); i > 0; i-- {
fmt.Println("UA:", sMapUA[i].Key, "\nHits:", sMapUA[i].Value, "\n")
if i == (numUa - 5) {
break
}
}
fmt.Println("")
fmt.Println("View the following files for full report:", "\n")
totalBytes := 0
for i := 0; i < 3; i++ {
fmt.Println("\t" + fileList[i].FullName() + " (" + strconv.Itoa(fileList[i].Size) + " bytes)")
totalBytes += fileList[i].Size
}
fmt.Println("\nTotal bytes written:", totalBytes)
fmt.Println("\n----------------------------------\n")
}
func (worker *Worker) Load() error {
worker.Config = &Config{}
var err error
worker.GeoDB, err = geoip.Open("GeoLiteCity.dat")
if err != nil {
return err
}
consulConfig := &consul.Config{
Address: "127.0.0.1:8500",
Scheme: "http",
HttpClient: http.DefaultClient,
}
client, err := consul.NewClient(consulConfig)
if err != nil {
return err
}
err = worker.Config.fillFromConsul(client, "silvia")
if err != nil {
return err
}
ringSize, err := strconv.Atoi(worker.Config.RingSize)
if err != nil {
return err
}
worker.Stats = &Stats{
AdjustSuccessRing: &AdjustRing{Size: ringSize},
AdjustFailRing: &AdjustRing{Size: ringSize},
SnowplowSuccessRing: &SnowplowRing{Size: ringSize},
SnowplowFailRing: &SnowplowRing{Size: ringSize},
}
worker.Stats.StartTime = time.Now()
worker.AdjustRequestBus = make(chan []byte)
worker.SnowplowRequestBus = make(chan []byte)
worker.AdjustEventBus = make(chan *AdjustEvent)
worker.SnowplowEventBus = make(chan *SnowplowEvent)
port, err := strconv.Atoi(worker.Config.Port)
if err != nil {
return nil
}
worker.ConsulServiceID = uuid.NewV4().String()
checks := consul.AgentServiceChecks{
&consul.AgentServiceCheck{
TTL: "10s",
},
&consul.AgentServiceCheck{
HTTP: "http://localhost:" + worker.Config.Port + "/v1/status",
Interval: "10s",
Timeout: "1s",
},
}
service := &consul.AgentServiceRegistration{
ID: worker.ConsulServiceID,
Name: "silvia",
Port: port,
Checks: checks,
}
worker.ConsulAgent = client.Agent()
err = worker.ConsulAgent.ServiceRegister(service)
if err != nil {
return err
}
go func() {
for {
<-time.After(8 * time.Second)
err = worker.ConsulAgent.PassTTL("service:"+worker.ConsulServiceID+":1", "Internal TTL ping")
if err != nil {
log.Println(err)
}
}
}()
log.Println("Service registred with ID:", worker.ConsulServiceID)
return nil
}
