func NewMetricMessage(wireLine string) (*MetricMessage, error) {
var err error
var mM *MetricMessage
s := strings.Split(wireLine, "|")
dateTime, err := time.Parse(time.RFC3339Nano, s[1])
if err != nil {
mM = &MetricMessage{
Uuid: uuid.NewV1().String(),
}
err = ErrUnmarshalingPayload
} else {
mM = &MetricMessage{
Uuid: uuid.NewV1().String(),
ProtoVer: s[0],
DateTime: dateTime,
RubyVersion: s[2],
RubyPlatform: s[3],
CapistranoVersion: s[4],
AnonProjectHash: s[5],
}
}
return mM, err
}
//Addlisting function adding listings data to db
func (r Listing) Add(config *config.Conf) error {
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
client := s3.New(auth, aws.USWest2)
bucket := client.Bucket("yellowpagesng")
p := rand.New(rand.NewSource(time.Now().UnixNano()))
str := strconv.Itoa(p.Intn(10))
r.Date = time.Now()
if r.Image != "" {
tm, _ := strconv.Atoi(r.Duration)
t := r.Date.AddDate(tm, 1, 0)
r.Expiry = t
byt, er := base64.StdEncoding.DecodeString(strings.Split(r.Image, "base64,")[1])
if er != nil {
log.Println(er)
}
meta := strings.Split(r.Image, "base64,")[0]
newmeta := strings.Replace(strings.Replace(meta, "data:", "", -1), ";", "", -1)
imagename := "listings/" + uuid.NewV1().String()
err = bucket.Put(imagename, byt, newmeta, s3.PublicReadWrite, s3.Options{})
if err != nil {
log.Println(err)
}
log.Println(bucket.URL(imagename))
r.Image = bucket.URL(imagename)
var images []string
for _, v := range r.Images {
var byt []byte
byt, err = base64.StdEncoding.DecodeString(strings.Split(v, "base64,")[1])
if err != nil {
log.Println(err)
}
meta := strings.Split(v, "base64,")[0]
newmeta := strings.Replace(strings.Replace(meta, "data:", "", -1), ";", "", -1)
imagename = "listings/" + uuid.NewV1().String()
err = bucket.Put(imagename, byt, newmeta, s3.PublicReadWrite, s3.Options{})
if err != nil {
log.Println(err)
}
images = append(images, bucket.URL(imagename))
}
r.Images = images
} else {
r.Plus = "false"
}
r.Slug = strings.Replace(r.CompanyName, " ", "-", -1) + str
r.Slug = strings.Replace(r.Slug, "&", "-", -1) + str
r.Slug = strings.Replace(r.Slug, "/", "-", -1) + str
r.Slug = strings.Replace(r.Slug, ",", "-", -1) + str
index := mgo.Index{
Key: []string{"$text:specialisation", "$text:companyname"},
Unique: true,
DropDups: true,
Background: true,
}
collection := config.Database.C("Listings").With(config.Database.Session.Copy())
collection.EnsureIndex(index)
collection.Insert(r)
return err
}
func main() {
fmt.Println(("Starting Producer"))
runtime.GOMAXPROCS(runtime.NumCPU())
numMessage := 0
brokerConnect, topic, sleepTime, graphiteConnect, graphiteFlushInterval, flushMsgCount, flushFrequency, producerCount, maxMessagesPerReq := resolveConfig()
_ = graphiteConnect
_ = graphiteFlushInterval
startMetrics(graphiteConnect, graphiteFlushInterval)
produceRate := metrics.NewRegisteredMeter("ProduceRate", metrics.DefaultRegistry)
//kafkaClient.CreateMultiplePartitionsTopic(zkConnect, topic, numPartitions)
//p := producer.NewKafkaProducer(topic, []string{brokerConnect})
//defer producer.Close()
//defer p.Close()
saramaError := make(chan *sarama.ProducerError)
saramaSuccess := make(chan *sarama.ProducerMessage)
clientConfig := sarama.NewConfig()
clientConfig.ClientID = uuid.NewV1().String()
clientConfig.Producer.Flush.Messages = flushMsgCount
clientConfig.Producer.Flush.Frequency = flushFrequency
clientConfig.Producer.Flush.MaxMessages = maxMessagesPerReq
clientConfig.Producer.Return.Successes = true
clientConfig.Producer.RequiredAcks = sarama.NoResponse //WaitForAll
clientConfig.Producer.Timeout = 1000 * time.Millisecond
client, err := sarama.NewClient([]string{brokerConnect}, clientConfig)
for i := 0; i < producerCount; i++ {
if err != nil {
panic(err)
}
// config.Compression = 2
producer, err := sarama.NewAsyncProducerFromClient(client)
go func() {
if err != nil {
panic(err)
}
for {
message := &sarama.ProducerMessage{Topic: topic, Key: sarama.StringEncoder(fmt.Sprintf("%d", numMessage)), Value: sarama.StringEncoder(fmt.Sprintf("message %d!", numMessage))}
numMessage++
producer.Input() <- message
time.Sleep(sleepTime)
}
}()
go func() {
for {
select {
case error := <-producer.Errors():
saramaError <- error
case success := <-producer.Successes():
saramaSuccess <- success
}
}
}()
}
ctrlc := make(chan os.Signal, 1)
signal.Notify(ctrlc, os.Interrupt)
go func() {
start := time.Now()
count := 0
for {
select {
case err := <-saramaError:
fmt.Println(err)
case <-saramaSuccess:
produceRate.Mark(1)
count++
elapsed := time.Since(start)
if elapsed.Seconds() >= 1 {
fmt.Println(fmt.Sprintf("Per Second %d", count))
count = 0
start = time.Now()
}
}
}
}()
<-ctrlc
}
func NewUUIDHex() (newuuid UUIDHex) {
newuuid = UUIDHex(uuid.NewV1().String())
return
}