func pullLoop(ctx context.Context, sub string, result chan<- int) { for { msgs, err := pubsub.PullWait(ctx, sub, *size) if err != nil { log.Printf("PullWait failed, %v\n", err) time.Sleep(5 * time.Second) continue } if len(msgs) == 0 { log.Println("Received no messages") continue } if *reportMPS { result <- len(msgs) } ackIDs := make([]string, len(msgs)) for i, msg := range msgs { if !*reportMPS { fmt.Printf("Got a message: %s\n", msg.Data) } ackIDs[i] = msg.AckID } go ack(ctx, sub, ackIDs...) } }
func (this *SalesforceOpportunityEvents) WaitForEvents(ctx context.Context) ([]*SalesforceEvent, error) { ctx = this.context(ctx) emptyEventList := []*SalesforceEvent{} logger.Info("Calling pubsub.PullWait...") msgs, err := pubsub.PullWait(ctx, this.Subscription, 10) if err != nil { logger.Errorf("Error polling for new SalesforceOpportunity: %s", err.Error()) return emptyEventList, err } if msgs == nil || len(msgs) == 0 { return emptyEventList, nil } logger.Infof("Received %d SalesforceOpportunity messages", len(msgs)) events := make([]*SalesforceEvent, 0, len(msgs)) for _, msg := range msgs { // expecting message to be base64 encoded // logger.Debugf("Raw string of received msg.Data: %s", string(msg.Data)) // decoded, err := base64.StdEncoding.DecodeString(string(msg.Data)) // if err != nil { // logger.Errorf("Error trying to base64 decode msg.Data from PubSub: %s", err.Error()) // this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying // continue // try processing the next msg // } fmt.Printf("\nmsg.Data:\n%#v\n", string(msg.Data)) evt := &SalesforceEvent{ AckID: msg.AckID, MsgID: msg.ID, } if strings.Contains(string(msg.Data), "Lead") { var data *SalesforceLead if err := json.Unmarshal(msg.Data, &data); err != nil { logger.Errorf("Error trying to Unmarshal SalesforceLead from PubSub: %s", err.Error()) this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying continue // try processing the next msg } else { evt.Data = data } } else { var data *SalesforceOpportunity if err := json.Unmarshal(msg.Data, &data); err != nil { logger.Errorf("Error trying to Unmarshal SalesforceOpportunity from PubSub: %s", err.Error()) this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying continue // try processing the next msg } else { evt.Data = data } } events = append(events, evt) logger.Infof("Received SalesforceOpportunity %s", msg.ID) } return events, nil }
// Subscribe prepares the peer to consume messages. func (c *Peer) Subscribe() error { // Subscription names must start with a lowercase letter, end with a // lowercase letter or number, and contain only lowercase letters, numbers, // dashes, underscores or periods. c.subscription = strings.ToLower(fmt.Sprintf("x%sx", broker.GenerateName())) exists, err := pubsub.SubExists(c.context, c.subscription) if err != nil { return err } if exists { return fmt.Errorf("Subscription %s already exists", c.subscription) } if err := pubsub.CreateSub(c.context, c.subscription, topic, 0, ""); err != nil { return err } go c.ack() go func() { // TODO: Can we avoid using atomic flag? for atomic.LoadInt32(&c.stopped) != stopped { messages, err := pubsub.PullWait(c.context, c.subscription, bufferSize) if err != nil { // Timed out. continue } ids := make([]string, len(messages)) for i, message := range messages { ids[i] = message.AckID c.messages <- message.Data } c.acks <- ids } }() return nil }
func subscribe() { for { // Pull up to 10 messages (maybe fewer) from the subscription. // Blocks for an indeterminate amount of time. msgs, err := pubsub.PullWait(pubSubCtx, subName, 10) if err != nil { log.Fatalf("could not pull: %v", err) } for _, m := range msgs { msg := m var id int64 if err := json.Unmarshal(msg.Data, &id); err != nil { log.Printf("could not decode message data: %#v", msg) go pubsub.Ack(pubSubCtx, subName, msg.AckID) continue } log.Printf("[ID %d] Processing.", id) go func() { if err := update(id); err != nil { log.Printf("[ID %d] could not update: %v", id, err) return } countMu.Lock() count++ countMu.Unlock() pubsub.Ack(pubSubCtx, subName, msg.AckID) log.Printf("[ID %d] ACK", id) }() } } }
func consumePubSubMsgs(projectId string, topicName string) <-chan []byte { msgC := make(chan []byte, 100) subName := topicName + "-2loggly" ackDeadline := time.Second * 20 client, err := google.DefaultClient(context.Background(), pubsub.ScopePubSub) if err != nil { logFatal(LogData{ "event": "pubsub.error.auth", "error": err.Error(), }) } ctx := cloud.NewContext(projectId, client) isSub, err := pubsub.SubExists(ctx, subName) if err != nil { logFatal(LogData{ "event": "pubsub.error.subexists", "error": err.Error(), }) } if !isSub { err = pubsub.CreateSub(ctx, subName, topicName, ackDeadline, "") if err != nil { logFatal(LogData{ "event": "pubsub.error.createsub", "error": err.Error(), }) } } go func() { var lastAckErr error var lastDecodeErr error for { msgs, err := pubsub.PullWait(ctx, subName, 20) if err != nil { logInfo(LogData{ "event": "pubsub.error.pull", "error": err.Error(), }) time.Sleep(time.Second) continue } lastAckErr = nil lastDecodeErr = nil for _, m := range msgs { jsonBytes, err := msgToLogJson(m.Data) if err != nil { lastDecodeErr = err } else { msgC <- jsonBytes } err = pubsub.Ack(ctx, subName, m.AckID) if err != nil { lastAckErr = err } } // protect against flood of logs from this application // when acks/decodes fail but adding rollup & sleep delay if lastAckErr != nil { logInfo(LogData{ "event": "pubsub.error.ack", "error": lastAckErr.Error(), }) time.Sleep(time.Second) } if lastDecodeErr != nil { logInfo(LogData{ "event": "pubsub.error.decode", "error": lastDecodeErr.Error(), }) time.Sleep(time.Second) } } }() return msgC }