func pullLoop(ctx context.Context, sub string, result chan<- int) {
for {
msgs, err := pubsub.PullWait(ctx, sub, *size)
if err != nil {
log.Printf("PullWait failed, %v\n", err)
time.Sleep(5 * time.Second)
continue
}
if len(msgs) == 0 {
log.Println("Received no messages")
continue
}
if *reportMPS {
result <- len(msgs)
}
ackIDs := make([]string, len(msgs))
for i, msg := range msgs {
if !*reportMPS {
fmt.Printf("Got a message: %s\n", msg.Data)
}
ackIDs[i] = msg.AckID
}
go ack(ctx, sub, ackIDs...)
}
}
func (this *SalesforceOpportunityEvents) WaitForEvents(ctx context.Context) ([]*SalesforceEvent, error) {
ctx = this.context(ctx)
emptyEventList := []*SalesforceEvent{}
logger.Info("Calling pubsub.PullWait...")
msgs, err := pubsub.PullWait(ctx, this.Subscription, 10)
if err != nil {
logger.Errorf("Error polling for new SalesforceOpportunity: %s", err.Error())
return emptyEventList, err
}
if msgs == nil || len(msgs) == 0 {
return emptyEventList, nil
}
logger.Infof("Received %d SalesforceOpportunity messages", len(msgs))
events := make([]*SalesforceEvent, 0, len(msgs))
for _, msg := range msgs {
// expecting message to be base64 encoded
// logger.Debugf("Raw string of received msg.Data: %s", string(msg.Data))
// decoded, err := base64.StdEncoding.DecodeString(string(msg.Data))
// if err != nil {
// logger.Errorf("Error trying to base64 decode msg.Data from PubSub: %s", err.Error())
// this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying
// continue // try processing the next msg
// }
fmt.Printf("\nmsg.Data:\n%#v\n", string(msg.Data))
evt := &SalesforceEvent{
AckID: msg.AckID,
MsgID: msg.ID,
}
if strings.Contains(string(msg.Data), "Lead") {
var data *SalesforceLead
if err := json.Unmarshal(msg.Data, &data); err != nil {
logger.Errorf("Error trying to Unmarshal SalesforceLead from PubSub: %s", err.Error())
this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying
continue // try processing the next msg
} else {
evt.Data = data
}
} else {
var data *SalesforceOpportunity
if err := json.Unmarshal(msg.Data, &data); err != nil {
logger.Errorf("Error trying to Unmarshal SalesforceOpportunity from PubSub: %s", err.Error())
this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying
continue // try processing the next msg
} else {
evt.Data = data
}
}
events = append(events, evt)
logger.Infof("Received SalesforceOpportunity %s", msg.ID)
}
return events, nil
}
// Subscribe prepares the peer to consume messages.
func (c *Peer) Subscribe() error {
// Subscription names must start with a lowercase letter, end with a
// lowercase letter or number, and contain only lowercase letters, numbers,
// dashes, underscores or periods.
c.subscription = strings.ToLower(fmt.Sprintf("x%sx", broker.GenerateName()))
exists, err := pubsub.SubExists(c.context, c.subscription)
if err != nil {
return err
}
if exists {
return fmt.Errorf("Subscription %s already exists", c.subscription)
}
if err := pubsub.CreateSub(c.context, c.subscription, topic, 0, ""); err != nil {
return err
}
go c.ack()
go func() {
// TODO: Can we avoid using atomic flag?
for atomic.LoadInt32(&c.stopped) != stopped {
messages, err := pubsub.PullWait(c.context, c.subscription, bufferSize)
if err != nil {
// Timed out.
continue
}
ids := make([]string, len(messages))
for i, message := range messages {
ids[i] = message.AckID
c.messages <- message.Data
}
c.acks <- ids
}
}()
return nil
}
func subscribe() {
for {
// Pull up to 10 messages (maybe fewer) from the subscription.
// Blocks for an indeterminate amount of time.
msgs, err := pubsub.PullWait(pubSubCtx, subName, 10)
if err != nil {
log.Fatalf("could not pull: %v", err)
}
for _, m := range msgs {
msg := m
var id int64
if err := json.Unmarshal(msg.Data, &id); err != nil {
log.Printf("could not decode message data: %#v", msg)
go pubsub.Ack(pubSubCtx, subName, msg.AckID)
continue
}
log.Printf("[ID %d] Processing.", id)
go func() {
if err := update(id); err != nil {
log.Printf("[ID %d] could not update: %v", id, err)
return
}
countMu.Lock()
count++
countMu.Unlock()
pubsub.Ack(pubSubCtx, subName, msg.AckID)
log.Printf("[ID %d] ACK", id)
}()
}
}
}
func consumePubSubMsgs(projectId string, topicName string) <-chan []byte {
msgC := make(chan []byte, 100)
subName := topicName + "-2loggly"
ackDeadline := time.Second * 20
client, err := google.DefaultClient(context.Background(), pubsub.ScopePubSub)
if err != nil {
logFatal(LogData{
"event": "pubsub.error.auth",
"error": err.Error(),
})
}
ctx := cloud.NewContext(projectId, client)
isSub, err := pubsub.SubExists(ctx, subName)
if err != nil {
logFatal(LogData{
"event": "pubsub.error.subexists",
"error": err.Error(),
})
}
if !isSub {
err = pubsub.CreateSub(ctx, subName, topicName, ackDeadline, "")
if err != nil {
logFatal(LogData{
"event": "pubsub.error.createsub",
"error": err.Error(),
})
}
}
go func() {
var lastAckErr error
var lastDecodeErr error
for {
msgs, err := pubsub.PullWait(ctx, subName, 20)
if err != nil {
logInfo(LogData{
"event": "pubsub.error.pull",
"error": err.Error(),
})
time.Sleep(time.Second)
continue
}
lastAckErr = nil
lastDecodeErr = nil
for _, m := range msgs {
jsonBytes, err := msgToLogJson(m.Data)
if err != nil {
lastDecodeErr = err
} else {
msgC <- jsonBytes
}
err = pubsub.Ack(ctx, subName, m.AckID)
if err != nil {
lastAckErr = err
}
}
// protect against flood of logs from this application
// when acks/decodes fail but adding rollup & sleep delay
if lastAckErr != nil {
logInfo(LogData{
"event": "pubsub.error.ack",
"error": lastAckErr.Error(),
})
time.Sleep(time.Second)
}
if lastDecodeErr != nil {
logInfo(LogData{
"event": "pubsub.error.decode",
"error": lastDecodeErr.Error(),
})
time.Sleep(time.Second)
}
}
}()
return msgC
}