func streamUsers(conf *Config) chan models.User {
config := consumergroup.NewConfig()
config.Offsets.Initial = sarama.OffsetOldest
config.Offsets.CommitInterval = 100 * time.Millisecond
consumer, err := consumergroup.JoinConsumerGroup(
"indexer",
[]string{conf.Topic},
conf.Zookeepers,
config)
if err != nil {
log.Fatalf("Can't create consumer. Err: %v", err)
}
var received, errors int
// Trap SIGINT to trigger a graceful shutdown.
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
out := make(chan models.User, 1024)
go func() {
for {
select {
case msg := <-consumer.Messages():
received++
var user models.User
if err := json.Unmarshal(msg.Value, &user); err != nil {
log.Fatalf("Can't unmarshal data from queue! Err: %v", err)
}
if *user.Dob == "0000-00-00" {
user.Dob = nil
}
out <- user
consumer.CommitUpto(msg)
case err := <-consumer.Errors():
errors++
log.Printf("Error reading from topic! Err: %v", err)
case <-signals:
log.Printf("Start consumer closing")
consumer.Close()
log.Printf("Consumer closed!")
close(out)
log.Printf("Successfully consumed: %d; errors: %d", received, errors)
return
}
}
}()
return out
}
func streamCsvUsers(conf *Config) chan models.User {
f, err := os.Open(conf.CsvPath)
if err != nil {
log.Fatal("can't file with users:", err)
}
defer f.Close()
log.Infof("Start reading CSV file!")
r := csv.NewReader(bufio.NewReader(f))
r.Comma = '|'
r.LazyQuotes = true
records, err := r.ReadAll()
if err != nil {
log.Fatal(err)
}
log.Infof("Read %d lines!", len(records))
out := make(chan models.User, 1024)
go func() {
for i, line := range records {
if len(line) != 12 {
log.Fatal(fmt.Errorf("Wrong number of parsed fields: %d. Index %d", len(line), i))
}
u := models.User{}
var err error
u.Pnum, err = strconv.ParseInt(line[0], 10, 64)
if err != nil {
log.Fatalf("Can't deserialize pnum. Val: %s", line[0])
}
long, err := strconv.ParseFloat(line[1], 64)
if err != nil {
log.Fatalf("Can't deserialize longitude. Val: %s. Line: %d", line[1], i)
}
lat, err := strconv.ParseFloat(line[2], 64)
if err != nil {
log.Fatalf("Can't deserialize latitude. Val: %s. Line: %d", line[2], i)
}
u.Location = &models.Location{long, lat}
if long == 0 || lat == 0 {
log.Warningf("At least one value of location could be wrong. Vals long, %d, lat: %d", long, lat)
}
u.Email = line[3]
weight, err := strconv.Atoi(line[4])
if err != nil {
log.Fatalf("Can't deserialize weight. Val: %s", line[4])
}
u.Weight = &weight
height, err := strconv.Atoi(line[5])
if err != nil {
log.Fatalf("Can't deserialize height. Val: %s", line[5])
}
u.Height = &height
u.Nickname = &line[6]
u.Country, err = strconv.Atoi(line[7])
if err != nil {
log.Fatalf("Can't deserialize country: Val: %s", line[7])
}
u.City = &line[8]
u.Caption = &line[9]
gender, err := strconv.Atoi(line[10])
if err != nil {
log.Fatalf("Can't deserialize gender. Val: %s", line[10])
}
u.Gender = &gender
u.Dob = &line[11]
out <- u
}
log.Infof("All users sent. Closing channel")
close(out)
}()
return out
}