func NewEventPublisher() (*EventPublisher, error) { config := sarama.NewConfig() config.ClientID = ipresolver.GetLocalAddr() config.Producer.RequiredAcks = sarama.WaitForLocal config.Producer.Compression = sarama.CompressionNone config.Producer.Return.Successes = false config.Producer.Return.Errors = false config.Producer.Partitioner = sarama.NewHashPartitioner asyncProducer, err := sarama.NewAsyncProducer(eatonconfig.KafkaServers, config) if err != nil { return nil, err } if config.Producer.Return.Successes { go func() { for msg := range asyncProducer.Successes() { log.Println("Sent Message to logs: ", msg.Key) } }() } if config.Producer.Return.Errors { go func() { for err := range asyncProducer.Errors() { log.Println("failed to send message to logs: ", err.Error()) } }() } return &EventPublisher{ producer: asyncProducer, }, nil }
func NewKafkaProducer() (*IndeedKafkaProducer, error) { config := sarama.NewConfig() config.ClientID = ipresolver.GetLocalAddr() config.Producer.RequiredAcks = sarama.WaitForLocal config.Producer.Compression = sarama.CompressionNone config.Producer.Return.Successes = true config.Producer.Return.Errors = true config.Producer.Partitioner = sarama.NewHashPartitioner asyncProducer, err := sarama.NewAsyncProducer(eatonconfig.KafkaServers, config) if err != nil { return nil, err } go func() { for msg := range asyncProducer.Successes() { eatonevents.Info(fmt.Sprintf("Successfully sent message to topic %s with key %s", msg.Topic, msg.Key)) } }() go func() { for err := range asyncProducer.Errors() { eatonevents.Error("Failed to send message due to error: ", err) } }() return &IndeedKafkaProducer{ producer: asyncProducer, }, nil }
func (e *EventPublisher) doPublish(skip int, msg, level string) error { msg = fmt.Sprintf("%s - %s", strings.ToUpper(level), msg) log.Println(msg) //0 would be the function calling the Caller. //1 would be the function calling doPublish. //2 is the function calling the function calling doPublish _, path, line, ok := runtime.Caller(skip) a := &mapping.ApplicationEvent{ Level: level, Message: msg, Date: time.Now(), Logger: GetCallingFunctionName(path, line, ok), Address: ipresolver.GetLocalAddr(), } if ok { a.Path = path a.Line = line } err := e.PublishEvent(a) if err != nil { log.Println("ERROR - failed to publish msg to kafka servers: ", err) return err } return nil }
func NewSaramaConsumers(servers []string, topic, offsetType string) (*sarama.Consumer, *[]sarama.PartitionConsumer, error) { config := sarama.NewConfig() config.ClientID = ipresolver.GetLocalAddr() config.Consumer.Return.Errors = true consumer, err := sarama.NewConsumer(servers, config) if err != nil { return nil, nil, err } partitions, err := consumer.Partitions(topic) if err != nil { return nil, nil, err } if eatonconfig.IsDebug() { log.Println("Returned Partitions for topic: ", topic, partitions) } if len(partitions) == 0 { return nil, nil, errors.New("no partitions returned to consume!") } partitionConsumers := make([]sarama.PartitionConsumer, len(partitions), len(partitions)) chosenOffset := sarama.OffsetOldest switch offsetType { case "oldest": chosenOffset = sarama.OffsetOldest break case "newest": chosenOffset = sarama.OffsetNewest break default: log.Fatal("unknown offsetType provided: ", offsetType) } for index, partition := range partitions { if eatonconfig.IsDebug() { log.Println("Creating partition consumer for partition: ", partition, " with offset: ", chosenOffset) } partitionConsumer, err := consumer.ConsumePartition(topic, partition, chosenOffset) if eatonconfig.IsDebug() { log.Println("Created partition consumer: ", consumer) } if err != nil { return nil, nil, err } if partitionConsumer == nil { return nil, nil, errors.New("nil consumer returned!") } partitionConsumers[index] = partitionConsumer } return &consumer, &partitionConsumers, nil }
"time" ) var ( indeedConstants = &IndeedDefaults{ IndeedUrl: "http://api.indeed.com/ads/apisearch?", PublisherId: os.Getenv("INDEED_PUBLISHER_ID"), IndeedResponseFormat: "xml", Version: "2", FromAge: 0, HighLight: false, Filter: true, LatLong: true, Country: "us", Channel: "", UserIP: ipresolver.GetLocalAddr(), UserAgent: "golang indeed client", Location: "22033", } constantValues, queryError = query.Values(indeedConstants) encodedConstants = constantValues.Encode() maxLimit = 25 interval = -1 ) type IndeedDefaults struct { IndeedUrl string `url:"-"` PublisherId string `url:"publisher"` IndeedResponseFormat string `url:"format"` Version string `url:"v"` FromAge int `url:"fromage,omitempty"`