Example #1
0
func NewEventPublisher() (*EventPublisher, error) {
	config := sarama.NewConfig()
	config.ClientID = ipresolver.GetLocalAddr()
	config.Producer.RequiredAcks = sarama.WaitForLocal
	config.Producer.Compression = sarama.CompressionNone
	config.Producer.Return.Successes = false
	config.Producer.Return.Errors = false
	config.Producer.Partitioner = sarama.NewHashPartitioner
	asyncProducer, err := sarama.NewAsyncProducer(eatonconfig.KafkaServers, config)

	if err != nil {
		return nil, err
	}
	if config.Producer.Return.Successes {
		go func() {
			for msg := range asyncProducer.Successes() {
				log.Println("Sent Message to logs: ", msg.Key)
			}
		}()
	}
	if config.Producer.Return.Errors {
		go func() {
			for err := range asyncProducer.Errors() {
				log.Println("failed to send message to logs: ", err.Error())
			}
		}()
	}
	return &EventPublisher{
		producer: asyncProducer,
	}, nil
}
Example #2
0
func NewKafkaProducer() (*IndeedKafkaProducer, error) {
	config := sarama.NewConfig()
	config.ClientID = ipresolver.GetLocalAddr()
	config.Producer.RequiredAcks = sarama.WaitForLocal
	config.Producer.Compression = sarama.CompressionNone
	config.Producer.Return.Successes = true
	config.Producer.Return.Errors = true
	config.Producer.Partitioner = sarama.NewHashPartitioner
	asyncProducer, err := sarama.NewAsyncProducer(eatonconfig.KafkaServers, config)
	if err != nil {
		return nil, err
	}
	go func() {
		for msg := range asyncProducer.Successes() {
			eatonevents.Info(fmt.Sprintf("Successfully sent message to topic %s with key %s", msg.Topic, msg.Key))
		}
	}()
	go func() {
		for err := range asyncProducer.Errors() {
			eatonevents.Error("Failed to send message due to error: ", err)
		}
	}()
	return &IndeedKafkaProducer{
		producer: asyncProducer,
	}, nil
}
Example #3
0
func (e *EventPublisher) doPublish(skip int, msg, level string) error {
	msg = fmt.Sprintf("%s - %s", strings.ToUpper(level), msg)
	log.Println(msg)
	//0 would be the function calling the Caller.
	//1 would be the function calling doPublish.
	//2 is the function calling the function calling doPublish
	_, path, line, ok := runtime.Caller(skip)
	a := &mapping.ApplicationEvent{
		Level:   level,
		Message: msg,
		Date:    time.Now(),
		Logger:  GetCallingFunctionName(path, line, ok),
		Address: ipresolver.GetLocalAddr(),
	}
	if ok {
		a.Path = path
		a.Line = line
	}
	err := e.PublishEvent(a)
	if err != nil {
		log.Println("ERROR - failed to publish msg to kafka servers: ", err)
		return err
	}
	return nil
}
Example #4
0
func NewSaramaConsumers(servers []string, topic, offsetType string) (*sarama.Consumer, *[]sarama.PartitionConsumer, error) {
	config := sarama.NewConfig()
	config.ClientID = ipresolver.GetLocalAddr()
	config.Consumer.Return.Errors = true
	consumer, err := sarama.NewConsumer(servers, config)
	if err != nil {
		return nil, nil, err
	}
	partitions, err := consumer.Partitions(topic)
	if err != nil {
		return nil, nil, err
	}
	if eatonconfig.IsDebug() {
		log.Println("Returned Partitions for topic: ", topic, partitions)
	}
	if len(partitions) == 0 {
		return nil, nil, errors.New("no partitions returned to consume!")
	}
	partitionConsumers := make([]sarama.PartitionConsumer, len(partitions), len(partitions))
	chosenOffset := sarama.OffsetOldest
	switch offsetType {
	case "oldest":
		chosenOffset = sarama.OffsetOldest
		break
	case "newest":
		chosenOffset = sarama.OffsetNewest
		break
	default:
		log.Fatal("unknown offsetType provided: ", offsetType)
	}
	for index, partition := range partitions {
		if eatonconfig.IsDebug() {
			log.Println("Creating partition consumer for partition: ", partition, " with offset: ", chosenOffset)
		}
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, chosenOffset)
		if eatonconfig.IsDebug() {
			log.Println("Created partition consumer: ", consumer)
		}
		if err != nil {
			return nil, nil, err
		}
		if partitionConsumer == nil {
			return nil, nil, errors.New("nil consumer returned!")
		}
		partitionConsumers[index] = partitionConsumer
	}
	return &consumer, &partitionConsumers, nil
}
Example #5
0
	"time"
)

var (
	indeedConstants = &IndeedDefaults{
		IndeedUrl:            "http://api.indeed.com/ads/apisearch?",
		PublisherId:          os.Getenv("INDEED_PUBLISHER_ID"),
		IndeedResponseFormat: "xml",
		Version:              "2",
		FromAge:              0,
		HighLight:            false,
		Filter:               true,
		LatLong:              true,
		Country:              "us",
		Channel:              "",
		UserIP:               ipresolver.GetLocalAddr(),
		UserAgent:            "golang indeed client",
		Location:             "22033",
	}
	constantValues, queryError = query.Values(indeedConstants)
	encodedConstants           = constantValues.Encode()
	maxLimit                   = 25
	interval                   = -1
)

type IndeedDefaults struct {
	IndeedUrl            string `url:"-"`
	PublisherId          string `url:"publisher"`
	IndeedResponseFormat string `url:"format"`
	Version              string `url:"v"`
	FromAge              int    `url:"fromage,omitempty"`