Пример #1
0
func main() {

	geoip.Quiet = true

	file6 := "./GeoIPv6-20120730.dat"

	gi6 := geoip.Open(file6)
	if gi6 == nil {
		fmt.Printf("Could not open GeoIPv6 database\n")
	}

	gi := geoip.Open()
	if gi == nil {
		fmt.Printf("Could not open GeoIP database\n")
	}

	if gi != nil {
		test4(*gi, "207.171.7.51")
		test4(*gi, "127.0.0.1")
	}
	if gi6 != nil {
		ip := "2607:f238:2::5"
		country := gi6.GetCountry_v6(ip)
		fmt.Println(ip, ": ")
		display(country)
	}

}
Пример #2
0
func main() {

	file6 := "./GeoIPv6.dat"

	gi6, err := geoip.Open(file6)
	if err != nil {
		fmt.Printf("Could not open GeoIPv6 database: %s\n", err)
	}

	gi, err := geoip.Open()
	if err != nil {
		fmt.Printf("Could not open GeoIP database: %s\n", err)
	}

	giasn, err := geoip.Open("./GeoIPASNum.dat")
	if err != nil {
		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
	}

	giasn6, err := geoip.Open("./GeoIPASNumv6.dat")
	if err != nil {
		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
	}

	gcity, err := geoip.Open("./GeoLiteCity.dat")
	if err != nil {
		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
	}

	if giasn != nil {
		ip := "207.171.7.51"
		asn, netmask := giasn.GetName(ip)
		fmt.Printf("%s: %s (netmask /%d)\n", ip, asn, netmask)

	}

	if gcity != nil {
		ip := "10.47.173.157"
		country, _ := gcity.GetRegion(ip)
		fmt.Printf("%s: %s\n", ip, country)
	}

	if gi != nil {
		test4(*gi, "207.171.7.51")
		test4(*gi, "127.0.0.1")
	}
	if gi6 != nil {
		ip := "2607:f238:2::5"
		country, netmask := gi6.GetCountry_v6(ip)
		var asn string
		var asn_netmask int
		if giasn6 != nil {
			asn, asn_netmask = giasn6.GetNameV6(ip)
		}
		fmt.Printf("%s: %s/%d %s/%d\n", ip, country, netmask, asn, asn_netmask)

	}

}
Пример #3
0
func init() {
	var err error

	g4, err = geoip.Open("/usr/share/GeoIP/GeoIP.dat")
	if err != nil {
		fmt.Println("can't open GeoIP.dat")
	}

	g6, err = geoip.Open("/usr/share/GeoIP/GeoIPv6.dat")
	if err != nil {
		fmt.Println("can't open GeoIPv6.dat")
	}

	rand.Seed(time.Now().UnixNano() * int64(os.Getpid()))
}
Пример #4
0
func NewTaskServer(s store.Store, serverUrl, templatesPath, geoipDatabase string) *measurementsServerState {
	queries := make(chan *store.Query)
	go s.WriteQueries(queries)

	measurementIds := generateMeasurementIds()

	go s.ScheduleTaskFunctions()

	taskRequests := make(chan *store.TaskRequest)
	go s.Tasks(taskRequests)

	countResultsRequests := make(chan store.CountResultsRequest)
	go s.CountResultsForReferrer(countResultsRequests)

	geolocator, err := geoip.Open(geoipDatabase)
	if err != nil {
		log.Fatalf("error opening geoip database: %v", err)
	}

	return &measurementsServerState{
		Store:                s,
		Templates:            template.Must(template.ParseGlob(filepath.Join(templatesPath, "[a-zA-Z]*"))),
		Queries:              queries,
		MeasurementIds:       measurementIds,
		TaskRequests:         taskRequests,
		CountResultsRequests: countResultsRequests,
		ServerUrl:            serverUrl,
		Geolocator:           geolocator,
	}
}
Пример #5
0
func setupGeoIP() *geoip.GeoIP {

	gi := geoip.Open()
	if gi == nil {
		log.Printf("Could not open GeoIP database\n")
	}
	return gi
}
Пример #6
0
func loadGeoIpDb(dbName string) *geoip.GeoIP {

	// Open the GeoIP database
	geo, geoErr := geoip.Open(dbName)
	if geoErr != nil {
		fmt.Printf("Warning, could not open GeoIP database: %s\n", geoErr)
	}
	return geo
}
Пример #7
0
func geoipLookup(configuration *Configuration, actor string) *geoip.GeoIPRecord {
	gi, err := geoip.Open(configuration.GeoIPDatabase)
	if err != nil {
		fmt.Printf("GeoIP: Could not open %s\n", configuration.GeoIPDatabase)
		return &geoip.GeoIPRecord{}
	}

	return gi.GetRecord(actor)
}
Пример #8
0
func main() {
	gi, err := geoip.Open("GeoIP.dat")
	if err != nil {
		fmt.Printf("Could not open GeoIP database: %s\n", err)
	}
	if gi != nil {
		test4(*gi, "207.171.7.51")
		test4(*gi, "10.47.173.157")
		test4(*gi, "8.8.8.8")
	}
	rc, _ := gi.GetName("202.13.34.13")
	fmt.Println(rc)
}
Пример #9
0
func NewPostgresDatastore() (Datastore, error) {
	db, err := sql.Open("postgres", "")
	if err != nil {
		return nil, fmt.Errorf("Error connecting to Postgres database: %s", err)
	}

	geolocator, err := geoip.Open(geoipDatabase)
	if err != nil {
		return nil, err
	}

	return PostgresDatastore{db, geolocator}, nil
}
Пример #10
0
func ParseRequest(args handler.RegexpArgs, r *http.Request) Event {
	// move it into init
	gi, err := geoip.Open(GEOIP_DATA)
	if err != nil {
		log.Fatal("Could not open GEOIP")
	}
	remote_ip := strings.Split(r.RemoteAddr, IP_SEP)[0]

	// FIXME scopes?
	// apparently I have no clue in understanding difference between = and :=
	country_code := "???"
	//tz := "???"
	if strings.HasPrefix(remote_ip, "192.168.") || strings.HasPrefix(remote_ip, "10.") || strings.HasPrefix(remote_ip, "127.") {
		//server side
		country_code = "GB"
		//tz = "Europe/London"
	} else {
		//client side
		country_code = "???"
		// return self.geoip.country_code_by_addr(self.request.remote_ip)
		//tz = "???"
		//return self.geoip.time_zone_by_addr(self.request.remote_ip)
	}

	country, geo_err := gi.GetCountry(remote_ip)
	if geo_err != 0 {
		log.Fatal("Could not resolve country")
	}

	log.Printf(country)

	user_agent := r.Header.Get("User-Agent")
	uainfo := uaparser.Parse(user_agent)

	event := Event{
		recipient_identifier: args["recipient_id"],
		job_identifier:       args["job_id"],
		blakey_identifier:    args["blakey_id"],
		link_id:              args["link_id"],
		link:                 args["link"],
		remote_ip:            remote_ip,
		user_agent:           user_agent,
		os:                   strings.Trim(uainfo.OS.Name+" "+uainfo.OS.Version, " "),
		client:               uainfo.Browser.Name,
		client_version:       uainfo.Browser.Version,
		country_code:         country_code,
	}
	return event
}
func Parser3(lines []string, session map[string]Client) string {
	file := "/usr/share/GeoIP/GeoIP.dat"
	gi, err := geoip.Open(file)
	if err != nil {
		fmt.Printf("Could not open GeoIP database please install in /usr/share/GeoIP/\n")
	}

	var update string
	for i := range lines {
		var c Client
		tmp := strings.Split(lines[i], "\t")
		if i == 1 {
			update = tmp[1]
		}
		if i > 2 && len(tmp) == 9 {
			c.Name = tmp[1]
			c.Vpn_ip = tmp[3]
			tmp1 := strings.Split(tmp[2], ":")
			c.Real_ip = tmp1[0]
			c.Real_port = tmp1[1]
			upload, _ := strconv.ParseInt(tmp[5], 10, 32)
			download, _ := strconv.ParseInt(tmp[4], 10, 32)
			c.Upload = fmt.Sprint(upload/1000, " Kb")
			c.Download = fmt.Sprint(download/1000, " Kb")
			if gi != nil {
				country, _ := gi.GetCountry(c.Real_ip)
				if len(country) < 2 {
					c.Country = "Lan"
				} else {
					c.Country = country
				}
			}
			if len(tmp) > 6 {
				c.Connected = tmp[6]
			}
			session[tmp[1]] = c
		}
	}
	return update
}
Пример #12
0
func main() {
	var geoipDatabase, logfile string
	flag.StringVar(&geoipDatabase, "geoip_database", "/usr/share/GeoIP/GeoIP.dat", "Path of GeoIP database")
	flag.StringVar(&logfile, "logfile", "", "Write logs to this file instead of stdout")
	flag.Parse()

	if logfile != "" {
		f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			log.Fatalf("error opening logfile: %v", err)
		}
		defer f.Close()
		log.SetOutput(f)
	}

	log.Printf("starting")

	s := store.Open()
	defer s.Close()

	geolocator, err := geoip.Open(geoipDatabase)
	if err != nil {
		panic(err)
	}

	queries := s.UnparsedQueries()
	parsedQueries := parseQueries(queries, geolocator)
	s.WriteParsedQueries(parsedQueries)

	results := s.UnparsedResults()
	parsedResults := parseResults(results, geolocator)
	s.WriteParsedResults(parsedResults)

	if err := s.ComputeResultsTables(); err != nil {
		panic(err)
	}

	log.Printf("done")
}
Пример #13
0
func (ld *GeoIpDecoder) Init(config interface{}) (err error) {
	conf := config.(*GeoIpDecoderConfig)

	if string(conf.SourceIpField) == "" {
		return errors.New("`source_ip_field` must be specified")
	}

	if conf.TargetField == "" {
		return errors.New("`target_field` must be specified")
	}

	ld.TargetField = conf.TargetField
	ld.SourceIpField = conf.SourceIpField

	if ld.gi == nil {
		ld.gi, err = geoip.Open(conf.DatabaseFile)
	}
	if err != nil {
		return fmt.Errorf("Could not open GeoIP database: %s\n")
	}

	return
}
Пример #14
0
func main() {

	if len(os.Args) < 2 {
		fmt.Println("\nUsage: parselog [filepath] [csv|json]\n")
		os.Exit(0)
	}

	filePath := os.Args[1]
	fmt.Println("\n-------------------------------------------")
	fmt.Println("Reading file:", filePath, "\n")

	f, err := os.Open(filePath)
	if err != nil {
		fmt.Printf("Error! Could not open file: %v\n", err)
		fmt.Println("")
		os.Exit(0)
	}

	ext := ""
	if os.Args[2] == "json" {
		ext = ".json"
	} else {
		ext = ".csv"
	}

	// Attempt to update the GeoIP DB
	//fmt.Println("Downloading "+geoipdb_base+geoipdb_city)

	// If not updated in a month, then attempt to update the GeoIP DB files
	if tools.FileExists(geoipdb_city) == false {

		if tools.Download(geoipdb_base + geoipdb_city) {
			fmt.Println("Update to " + geoipdb_city + " successful")
		} else {
			fmt.Println("Could not update " + geoipdb_city)
		}

	}

	// Now open the GeoIP database
	//var geo *geoip.GeoIP
	//if tools.FileExists(geoipdb_city) == true {
	geo, geoErr := geoip.Open(geoipdb_city)
	if geoErr != nil {
		fmt.Printf("Warning, could not open GeoIP database: %s\n", err)
	}
	//} else {
	//  fmt.Println("File doesn't exist!")
	//}

	r := bufio.NewReader(f)
	s, e := Readln(r)

	lineNum := 0
	mapIP := make(map[string]int, 10)
	mapURI := make(map[string]int, 10)
	mapUA := make(map[string]int, 10)

	for e == nil {

		// Read a line from the file
		s, e = Readln(r)

		// Attempt to extrac the IP and store it inn a map
		matches := regexp.MustCompile(`^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(s)
		if len(matches) >= 2 {
			mapIP[matches[1]] = mapIP[matches[1]] + 1
		}
		/***********************************************************/

		// Attempt to extract and store the URI in a map
		matches = regexp.MustCompile(`"(GET|POST)[^\\"]*(\\"[^\\"]*)*"`).FindStringSubmatch(s)
		if len(matches) >= 1 {
			key := regexp.MustCompile("(GET|POST|\"|HTTP/1.[0-1])").ReplaceAllString(matches[0], "")
			mapURI[key] = mapURI[key] + 1
		}

		/***********************************************************/

		// Attempt to extract and store the Useragent in a map
		matches = regexp.MustCompile(`"[^\\"]*(\\"[^\\"]*)*"$`).FindStringSubmatch(s)
		if len(matches) >= 1 {
			mapUA[regexp.MustCompile("\"").ReplaceAllString(matches[0], "")] = mapUA[matches[0]] + 1
		}

		/***********************************************************/

		lineNum++
		matches = nil
	}

	/*********** Now sort the maps by value descending using merge sort ******/

	sMapIP := sortMapByValue(mapIP)
	mapIP = nil

	sMapURI := sortMapByValue(mapURI)
	mapURI = nil

	sMapUA := sortMapByValue(mapUA)
	mapUA = nil

	/************* Now write the data to a csv file ***************/

	ts := int(time.Now().Unix())
	y, m, d := time.Now().Date()

	fileList := [3]FileElement{
		{Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_ip", Ext: ext, Size: 0},
		{Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_uri", Ext: ext, Size: 0},
		{Name: "logfile_results_" + strconv.Itoa(y) + "" + m.String() + "" + strconv.Itoa(d) + "_" + strconv.Itoa(ts) + "_ua", Ext: ext, Size: 0},
	}

	for nf := 0; nf < 3; nf++ {
		fh, err := os.Create(fileList[nf].FullName())
		check(err)
		fileList[nf].Handle = fh
		defer fileList[nf].Handle.Close()
	}

	// Update each file element property
	if ext == "json" {
		fileList[0].Size = mapPairToJson("IP"+csvd+"Hits"+csvd+"City"+csvd+"Country", sMapIP, fileList[0].Handle)
		fileList[1].Size = mapPairToJson("URI"+csvd+"Hits\n", sMapURI, fileList[1].Handle)
		fileList[2].Size = mapPairToJson("UA"+csvd+"Hits\n", sMapUA, fileList[2].Handle)
	} else {
		fileList[0].Size = mapPairToCSV("IP"+csvd+"Hits"+csvd+"City"+csvd+"Country", sMapIP, fileList[0].Handle, geo)
		fileList[1].Size = mapPairToCSV("URI"+csvd+"Hits", sMapURI, fileList[1].Handle, geo)
		fileList[2].Size = mapPairToCSV("UA"+csvd+"Hits", sMapUA, fileList[2].Handle, geo)
	}

	numLines := (lineNum - 1)
	numIp := len(sMapIP)
	numUri := len(sMapURI)
	numUa := len(sMapUA)

	fmt.Println("Processing details:\n-----------------------")
	fmt.Println("Total lines in log file: ", numLines)
	fmt.Println("Total unique IPs: ", numIp)
	fmt.Println("Total unique User-Agents: ", numUa)
	fmt.Println("Total unique URIs: ", numUri, "\n")

	fmt.Println("Top 5 IPs\n----------------------------")
	for i := (numIp - 1); i > 0; i-- {
		record := geo.GetRecord(sMapIP[i].Key)
		fmt.Println("IP:", sMapIP[i].Key, "Hits:", sMapIP[i].Value, "Location:", record.City+", "+record.CountryName)
		if i == (numIp - 5) {
			break
		}
	}
	fmt.Println("")

	fmt.Println("Top 5 URIs\n----------------------------")
	for i := (numUri - 1); i > 0; i-- {
		fmt.Println("URI:", sMapURI[i].Key, "\nHits:", sMapURI[i].Value, "\n")
		if i == (numUri - 5) {
			break
		}
	}

	fmt.Println("Top 5 UAs\n----------------------------")
	for i := (numUa - 1); i > 0; i-- {
		fmt.Println("UA:", sMapUA[i].Key, "\nHits:", sMapUA[i].Value, "\n")
		if i == (numUa - 5) {
			break
		}
	}
	fmt.Println("")

	fmt.Println("View the following files for full report:", "\n")
	totalBytes := 0
	for i := 0; i < 3; i++ {
		fmt.Println("\t" + fileList[i].FullName() + " (" + strconv.Itoa(fileList[i].Size) + " bytes)")
		totalBytes += fileList[i].Size
	}
	fmt.Println("\nTotal bytes written:", totalBytes)
	fmt.Println("\n----------------------------------\n")

}
Пример #15
0
func (worker *Worker) Load() error {
	worker.Config = &Config{}

	var err error
	worker.GeoDB, err = geoip.Open("GeoLiteCity.dat")
	if err != nil {
		return err
	}

	consulConfig := &consul.Config{
		Address:    "127.0.0.1:8500",
		Scheme:     "http",
		HttpClient: http.DefaultClient,
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		return err
	}

	err = worker.Config.fillFromConsul(client, "silvia")
	if err != nil {
		return err
	}

	ringSize, err := strconv.Atoi(worker.Config.RingSize)
	if err != nil {
		return err
	}

	worker.Stats = &Stats{
		AdjustSuccessRing:   &AdjustRing{Size: ringSize},
		AdjustFailRing:      &AdjustRing{Size: ringSize},
		SnowplowSuccessRing: &SnowplowRing{Size: ringSize},
		SnowplowFailRing:    &SnowplowRing{Size: ringSize},
	}

	worker.Stats.StartTime = time.Now()

	worker.AdjustRequestBus = make(chan []byte)
	worker.SnowplowRequestBus = make(chan []byte)
	worker.AdjustEventBus = make(chan *AdjustEvent)
	worker.SnowplowEventBus = make(chan *SnowplowEvent)

	port, err := strconv.Atoi(worker.Config.Port)
	if err != nil {
		return nil
	}

	worker.ConsulServiceID = uuid.NewV4().String()

	checks := consul.AgentServiceChecks{
		&consul.AgentServiceCheck{
			TTL: "10s",
		},
		&consul.AgentServiceCheck{
			HTTP:     "http://localhost:" + worker.Config.Port + "/v1/status",
			Interval: "10s",
			Timeout:  "1s",
		},
	}

	service := &consul.AgentServiceRegistration{
		ID:     worker.ConsulServiceID,
		Name:   "silvia",
		Port:   port,
		Checks: checks,
	}

	worker.ConsulAgent = client.Agent()
	err = worker.ConsulAgent.ServiceRegister(service)
	if err != nil {
		return err
	}

	go func() {
		for {
			<-time.After(8 * time.Second)
			err = worker.ConsulAgent.PassTTL("service:"+worker.ConsulServiceID+":1", "Internal TTL ping")
			if err != nil {
				log.Println(err)
			}
		}
	}()

	log.Println("Service registred with ID:", worker.ConsulServiceID)

	return nil
}