Example #1
0
func NewMetricMessage(wireLine string) (*MetricMessage, error) {
	var err error
	var mM *MetricMessage
	s := strings.Split(wireLine, "|")
	dateTime, err := time.Parse(time.RFC3339Nano, s[1])
	if err != nil {
		mM = &MetricMessage{
			Uuid: uuid.NewV1().String(),
		}
		err = ErrUnmarshalingPayload
	} else {
		mM = &MetricMessage{
			Uuid:              uuid.NewV1().String(),
			ProtoVer:          s[0],
			DateTime:          dateTime,
			RubyVersion:       s[2],
			RubyPlatform:      s[3],
			CapistranoVersion: s[4],
			AnonProjectHash:   s[5],
		}
	}
	return mM, err
}
Example #2
0
//Addlisting function adding listings data to db
func (r Listing) Add(config *config.Conf) error {

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Fatal(err)
	}
	client := s3.New(auth, aws.USWest2)
	bucket := client.Bucket("yellowpagesng")

	p := rand.New(rand.NewSource(time.Now().UnixNano()))
	str := strconv.Itoa(p.Intn(10))
	r.Date = time.Now()
	if r.Image != "" {
		tm, _ := strconv.Atoi(r.Duration)
		t := r.Date.AddDate(tm, 1, 0)
		r.Expiry = t
		byt, er := base64.StdEncoding.DecodeString(strings.Split(r.Image, "base64,")[1])
		if er != nil {
			log.Println(er)
		}

		meta := strings.Split(r.Image, "base64,")[0]
		newmeta := strings.Replace(strings.Replace(meta, "data:", "", -1), ";", "", -1)
		imagename := "listings/" + uuid.NewV1().String()

		err = bucket.Put(imagename, byt, newmeta, s3.PublicReadWrite, s3.Options{})
		if err != nil {
			log.Println(err)
		}

		log.Println(bucket.URL(imagename))

		r.Image = bucket.URL(imagename)

		var images []string
		for _, v := range r.Images {
			var byt []byte
			byt, err = base64.StdEncoding.DecodeString(strings.Split(v, "base64,")[1])
			if err != nil {
				log.Println(err)
			}

			meta := strings.Split(v, "base64,")[0]

			newmeta := strings.Replace(strings.Replace(meta, "data:", "", -1), ";", "", -1)

			imagename = "listings/" + uuid.NewV1().String()

			err = bucket.Put(imagename, byt, newmeta, s3.PublicReadWrite, s3.Options{})
			if err != nil {
				log.Println(err)
			}
			images = append(images, bucket.URL(imagename))
		}

		r.Images = images

	} else {
		r.Plus = "false"
	}

	r.Slug = strings.Replace(r.CompanyName, " ", "-", -1) + str
	r.Slug = strings.Replace(r.Slug, "&", "-", -1) + str
	r.Slug = strings.Replace(r.Slug, "/", "-", -1) + str
	r.Slug = strings.Replace(r.Slug, ",", "-", -1) + str
	index := mgo.Index{
		Key:        []string{"$text:specialisation", "$text:companyname"},
		Unique:     true,
		DropDups:   true,
		Background: true,
	}
	collection := config.Database.C("Listings").With(config.Database.Session.Copy())
	collection.EnsureIndex(index)
	collection.Insert(r)
	return err
}
Example #3
0
func main() {
	fmt.Println(("Starting Producer"))
	runtime.GOMAXPROCS(runtime.NumCPU())
	numMessage := 0

	brokerConnect, topic, sleepTime, graphiteConnect, graphiteFlushInterval, flushMsgCount, flushFrequency, producerCount, maxMessagesPerReq := resolveConfig()

	_ = graphiteConnect
	_ = graphiteFlushInterval
	startMetrics(graphiteConnect, graphiteFlushInterval)
	produceRate := metrics.NewRegisteredMeter("ProduceRate", metrics.DefaultRegistry)

	//kafkaClient.CreateMultiplePartitionsTopic(zkConnect, topic, numPartitions)

	//p := producer.NewKafkaProducer(topic, []string{brokerConnect})

	//defer producer.Close()
	//defer p.Close()

	saramaError := make(chan *sarama.ProducerError)
	saramaSuccess := make(chan *sarama.ProducerMessage)

	clientConfig := sarama.NewConfig()
	clientConfig.ClientID = uuid.NewV1().String()
	clientConfig.Producer.Flush.Messages = flushMsgCount
	clientConfig.Producer.Flush.Frequency = flushFrequency
	clientConfig.Producer.Flush.MaxMessages = maxMessagesPerReq
	clientConfig.Producer.Return.Successes = true
	clientConfig.Producer.RequiredAcks = sarama.NoResponse //WaitForAll
	clientConfig.Producer.Timeout = 1000 * time.Millisecond
	client, err := sarama.NewClient([]string{brokerConnect}, clientConfig)
	for i := 0; i < producerCount; i++ {
		if err != nil {
			panic(err)
		}

		//		config.Compression = 2
		producer, err := sarama.NewAsyncProducerFromClient(client)
		go func() {
			if err != nil {
				panic(err)
			}
			for {
				message := &sarama.ProducerMessage{Topic: topic, Key: sarama.StringEncoder(fmt.Sprintf("%d", numMessage)), Value: sarama.StringEncoder(fmt.Sprintf("message %d!", numMessage))}
				numMessage++
				producer.Input() <- message
				time.Sleep(sleepTime)
			}
		}()

		go func() {
			for {
				select {
				case error := <-producer.Errors():
					saramaError <- error
				case success := <-producer.Successes():
					saramaSuccess <- success
				}
			}
		}()
	}

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)
	go func() {
		start := time.Now()
		count := 0
		for {
			select {
			case err := <-saramaError:
				fmt.Println(err)
			case <-saramaSuccess:
				produceRate.Mark(1)
				count++
				elapsed := time.Since(start)
				if elapsed.Seconds() >= 1 {
					fmt.Println(fmt.Sprintf("Per Second %d", count))
					count = 0
					start = time.Now()
				}
			}
		}
	}()
	<-ctrlc
}
Example #4
0
func NewUUIDHex() (newuuid UUIDHex) {
	newuuid = UUIDHex(uuid.NewV1().String())
	return
}