func NewEventPublisher() (*EventPublisher, error) {
config := sarama.NewConfig()
config.ClientID = ipresolver.GetLocalAddr()
config.Producer.RequiredAcks = sarama.WaitForLocal
config.Producer.Compression = sarama.CompressionNone
config.Producer.Return.Successes = false
config.Producer.Return.Errors = false
config.Producer.Partitioner = sarama.NewHashPartitioner
asyncProducer, err := sarama.NewAsyncProducer(eatonconfig.KafkaServers, config)
if err != nil {
return nil, err
}
if config.Producer.Return.Successes {
go func() {
for msg := range asyncProducer.Successes() {
log.Println("Sent Message to logs: ", msg.Key)
}
}()
}
if config.Producer.Return.Errors {
go func() {
for err := range asyncProducer.Errors() {
log.Println("failed to send message to logs: ", err.Error())
}
}()
}
return &EventPublisher{
producer: asyncProducer,
}, nil
}
func NewKafkaProducer() (*IndeedKafkaProducer, error) {
config := sarama.NewConfig()
config.ClientID = ipresolver.GetLocalAddr()
config.Producer.RequiredAcks = sarama.WaitForLocal
config.Producer.Compression = sarama.CompressionNone
config.Producer.Return.Successes = true
config.Producer.Return.Errors = true
config.Producer.Partitioner = sarama.NewHashPartitioner
asyncProducer, err := sarama.NewAsyncProducer(eatonconfig.KafkaServers, config)
if err != nil {
return nil, err
}
go func() {
for msg := range asyncProducer.Successes() {
eatonevents.Info(fmt.Sprintf("Successfully sent message to topic %s with key %s", msg.Topic, msg.Key))
}
}()
go func() {
for err := range asyncProducer.Errors() {
eatonevents.Error("Failed to send message due to error: ", err)
}
}()
return &IndeedKafkaProducer{
producer: asyncProducer,
}, nil
}
func (e *EventPublisher) doPublish(skip int, msg, level string) error {
msg = fmt.Sprintf("%s - %s", strings.ToUpper(level), msg)
log.Println(msg)
//0 would be the function calling the Caller.
//1 would be the function calling doPublish.
//2 is the function calling the function calling doPublish
_, path, line, ok := runtime.Caller(skip)
a := &mapping.ApplicationEvent{
Level: level,
Message: msg,
Date: time.Now(),
Logger: GetCallingFunctionName(path, line, ok),
Address: ipresolver.GetLocalAddr(),
}
if ok {
a.Path = path
a.Line = line
}
err := e.PublishEvent(a)
if err != nil {
log.Println("ERROR - failed to publish msg to kafka servers: ", err)
return err
}
return nil
}
func NewSaramaConsumers(servers []string, topic, offsetType string) (*sarama.Consumer, *[]sarama.PartitionConsumer, error) {
config := sarama.NewConfig()
config.ClientID = ipresolver.GetLocalAddr()
config.Consumer.Return.Errors = true
consumer, err := sarama.NewConsumer(servers, config)
if err != nil {
return nil, nil, err
}
partitions, err := consumer.Partitions(topic)
if err != nil {
return nil, nil, err
}
if eatonconfig.IsDebug() {
log.Println("Returned Partitions for topic: ", topic, partitions)
}
if len(partitions) == 0 {
return nil, nil, errors.New("no partitions returned to consume!")
}
partitionConsumers := make([]sarama.PartitionConsumer, len(partitions), len(partitions))
chosenOffset := sarama.OffsetOldest
switch offsetType {
case "oldest":
chosenOffset = sarama.OffsetOldest
break
case "newest":
chosenOffset = sarama.OffsetNewest
break
default:
log.Fatal("unknown offsetType provided: ", offsetType)
}
for index, partition := range partitions {
if eatonconfig.IsDebug() {
log.Println("Creating partition consumer for partition: ", partition, " with offset: ", chosenOffset)
}
partitionConsumer, err := consumer.ConsumePartition(topic, partition, chosenOffset)
if eatonconfig.IsDebug() {
log.Println("Created partition consumer: ", consumer)
}
if err != nil {
return nil, nil, err
}
if partitionConsumer == nil {
return nil, nil, errors.New("nil consumer returned!")
}
partitionConsumers[index] = partitionConsumer
}
return &consumer, &partitionConsumers, nil
}
"time"
)
var (
indeedConstants = &IndeedDefaults{
IndeedUrl: "http://api.indeed.com/ads/apisearch?",
PublisherId: os.Getenv("INDEED_PUBLISHER_ID"),
IndeedResponseFormat: "xml",
Version: "2",
FromAge: 0,
HighLight: false,
Filter: true,
LatLong: true,
Country: "us",
Channel: "",
UserIP: ipresolver.GetLocalAddr(),
UserAgent: "golang indeed client",
Location: "22033",
}
constantValues, queryError = query.Values(indeedConstants)
encodedConstants = constantValues.Encode()
maxLimit = 25
interval = -1
)
type IndeedDefaults struct {
IndeedUrl string `url:"-"`
PublisherId string `url:"publisher"`
IndeedResponseFormat string `url:"format"`
Version string `url:"v"`
FromAge int `url:"fromage,omitempty"`