Пример #1
0
func streamUsers(conf *Config) chan models.User {
	config := consumergroup.NewConfig()
	config.Offsets.Initial = sarama.OffsetOldest
	config.Offsets.CommitInterval = 100 * time.Millisecond

	consumer, err := consumergroup.JoinConsumerGroup(
		"indexer",
		[]string{conf.Topic},
		conf.Zookeepers,
		config)
	if err != nil {
		log.Fatalf("Can't create consumer. Err: %v", err)
	}

	var received, errors int

	// Trap SIGINT to trigger a graceful shutdown.
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	out := make(chan models.User, 1024)
	go func() {
		for {
			select {
			case msg := <-consumer.Messages():
				received++

				var user models.User
				if err := json.Unmarshal(msg.Value, &user); err != nil {
					log.Fatalf("Can't unmarshal data from queue! Err: %v", err)
				}

				if *user.Dob == "0000-00-00" {
					user.Dob = nil
				}

				out <- user
				consumer.CommitUpto(msg)
			case err := <-consumer.Errors():
				errors++
				log.Printf("Error reading from topic! Err: %v", err)
			case <-signals:
				log.Printf("Start consumer closing")
				consumer.Close()
				log.Printf("Consumer closed!")
				close(out)
				log.Printf("Successfully consumed: %d; errors: %d", received, errors)
				return
			}
		}
	}()

	return out
}
Пример #2
0
func streamDbUsers(conf *Config) chan models.User {
	db, err := sql.Open("postgres", conf.DbString)
	if err != nil {
		log.Fatal("can't open database conn:", err)
	}

	log.Infof("Start reading users from DB!")

	out := make(chan models.User, 1024)
	go func() {
		rows, err := db.Query("select pnum, email, dob, weight, height, nickname, country, city, caption, longitude, latitude, gender from aminno_member limit 1000000")
		if err != nil {
			log.Fatal(err)
		}
		defer rows.Close()

		noOfUsers := 0
		for rows.Next() {
			u := models.User{}
			u.Location = &models.Location{}
			if err := rows.Scan(&u.Pnum, &u.Email, &u.Dob, &u.Weight, &u.Height, &u.Nickname, &u.Country, &u.City, &u.Caption, &u.Location.Longitude, &u.Location.Latitude, &u.Gender); err != nil {
				log.Fatal(err)
			}

			out <- u

			if noOfUsers%100 == 0 {
				log.Printf("Total no of read users: %v", noOfUsers)
			}

			noOfUsers++
		}

		err = rows.Err()
		if err != nil {
			log.Fatal(err)
		}

		log.Infof("All users sent")
		close(out)
	}()

	return out
}
Пример #3
0
func streamCsvUsers(conf *Config) chan models.User {
	f, err := os.Open(conf.CsvPath)
	if err != nil {
		log.Fatal("can't file with users:", err)
	}
	defer f.Close()

	log.Infof("Start reading CSV file!")

	r := csv.NewReader(bufio.NewReader(f))
	r.Comma = '|'
	r.LazyQuotes = true
	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	log.Infof("Read %d lines!", len(records))

	out := make(chan models.User, 1024)
	go func() {
		for i, line := range records {
			if len(line) != 12 {
				log.Fatal(fmt.Errorf("Wrong number of parsed fields: %d. Index %d", len(line), i))
			}
			u := models.User{}

			var err error
			u.Pnum, err = strconv.ParseInt(line[0], 10, 64)
			if err != nil {
				log.Fatalf("Can't deserialize pnum. Val: %s", line[0])
			}

			long, err := strconv.ParseFloat(line[1], 64)
			if err != nil {
				log.Fatalf("Can't deserialize longitude. Val: %s. Line: %d", line[1], i)
			}

			lat, err := strconv.ParseFloat(line[2], 64)
			if err != nil {
				log.Fatalf("Can't deserialize latitude. Val: %s. Line: %d", line[2], i)
			}

			u.Location = &models.Location{long, lat}
			if long == 0 || lat == 0 {
				log.Warningf("At least one value of location could be wrong. Vals long, %d, lat: %d", long, lat)
			}

			u.Email = line[3]
			weight, err := strconv.Atoi(line[4])
			if err != nil {
				log.Fatalf("Can't deserialize weight. Val: %s", line[4])
			}
			u.Weight = &weight

			height, err := strconv.Atoi(line[5])
			if err != nil {
				log.Fatalf("Can't deserialize height. Val: %s", line[5])
			}
			u.Height = &height

			u.Nickname = &line[6]
			u.Country, err = strconv.Atoi(line[7])
			if err != nil {
				log.Fatalf("Can't deserialize country: Val: %s", line[7])
			}
			u.City = &line[8]
			u.Caption = &line[9]
			gender, err := strconv.Atoi(line[10])
			if err != nil {
				log.Fatalf("Can't deserialize gender. Val: %s", line[10])
			}
			u.Gender = &gender
			u.Dob = &line[11]

			out <- u
		}

		log.Infof("All users sent. Closing channel")
		close(out)
	}()

	return out
}
Пример #4
0
func (r *RestApi) users(w http.ResponseWriter, req *http.Request) {
	field := req.URL.Query().Get("field")
	query := req.URL.Query().Get("query")
	distance := req.URL.Query().Get("distance")
	latStr := req.URL.Query().Get("lat")
	lonStr := req.URL.Query().Get("lon")
	size := req.URL.Query().Get("l")
	skip := req.URL.Query().Get("s")
	wildcard := req.URL.Query().Get("w")

	sizeInt, err := strconv.Atoi(size)
	if err != nil {
		sizeInt = 100
	}

	skipInt, err := strconv.Atoi(skip)
	if err != nil {
		skipInt = 0
	}

	if field == "" {
		field = "_all"
	}

	if query == "" && distance == "" {
		fmt.Fprintf(w, "Query and Distance can't be empty!")
		return
	}

	var elasticQuery elastic.Query
	var long, lat float64
	if distance != "" {
		long, err = strconv.ParseFloat(lonStr, 64)
		if err != nil {
			fmt.Fprintf(w, "Longitude can't be empty!")
			return
		}

		lat, err = strconv.ParseFloat(latStr, 64)
		if err != nil {
			fmt.Fprintf(w, "Latitude can't be empty!")
			return
		}

		log.Infof("Start searching for users in distance: %s from (lat, lng): (%f, %f)", distance, lat, long)

		// Search with a geo query
		geoQuery := elastic.NewGeoDistanceFilter("location").Distance(distance + "km").Lat(lat).Lon(long)
		elasticQuery = elastic.NewFilteredQuery(elastic.NewMatchAllQuery()).Filter(geoQuery)
	} else {
		log.Infof("Start searching for users: %s", query)

		if wildcard == "true" {
			// Search for
			elasticQuery = elastic.NewSimpleQueryStringQuery(query).Field(field)
		} else {
			// Search with a term query
			elasticQuery = elastic.NewMatchQuery(field, query)
		}

	}

	searchResult, err := r.e.Search().
		Index("users").
		Type("user").
		Query(elasticQuery).
		From(skipInt).Size(sizeInt).
		Do()
	if err != nil {
		fmt.Fprintf(w, "Can't search for users. Err: %v", err)
		return
	}

	response := UsersResponse{}

	// Here's how you iterate through results with full control over each step.
	if searchResult.Hits != nil {
		log.Printf("Found a total of %d users", searchResult.Hits.TotalHits)

		response.Total = searchResult.Hits.TotalHits

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			var u models.User
			err := json.Unmarshal(*hit.Source, &u)
			if err != nil {
				fmt.Fprintf(w, "Can't deserialize user. Err: %v", err)
				return
			}

			u.Score = hit.Score

			response.Users = append(response.Users, u)
			log.Infof("Found user %v", u.Email)
		}
	} else {
		fmt.Fprintf(w, "Found no users")
		return
	}

	json, err := json.Marshal(response)
	if err != nil {
		fmt.Fprintf(w, "Can't serialize users. Err: %v", err)
		return
	}

	w.Write(json)
}