Ejemplo n.º 1
0
func pullLoop(ctx context.Context, sub string, result chan<- int) {
	for {
		msgs, err := pubsub.PullWait(ctx, sub, *size)
		if err != nil {
			log.Printf("PullWait failed, %v\n", err)
			time.Sleep(5 * time.Second)
			continue
		}
		if len(msgs) == 0 {
			log.Println("Received no messages")
			continue
		}
		if *reportMPS {
			result <- len(msgs)
		}
		ackIDs := make([]string, len(msgs))
		for i, msg := range msgs {
			if !*reportMPS {
				fmt.Printf("Got a message: %s\n", msg.Data)
			}
			ackIDs[i] = msg.AckID
		}
		go ack(ctx, sub, ackIDs...)
	}
}
func (this *SalesforceOpportunityEvents) WaitForEvents(ctx context.Context) ([]*SalesforceEvent, error) {
	ctx = this.context(ctx)
	emptyEventList := []*SalesforceEvent{}

	logger.Info("Calling pubsub.PullWait...")
	msgs, err := pubsub.PullWait(ctx, this.Subscription, 10)
	if err != nil {
		logger.Errorf("Error polling for new SalesforceOpportunity: %s", err.Error())
		return emptyEventList, err
	}
	if msgs == nil || len(msgs) == 0 {
		return emptyEventList, nil
	}
	logger.Infof("Received %d SalesforceOpportunity messages", len(msgs))

	events := make([]*SalesforceEvent, 0, len(msgs))
	for _, msg := range msgs {

		// expecting message to be base64 encoded
		// logger.Debugf("Raw string of received msg.Data: %s", string(msg.Data))
		// decoded, err := base64.StdEncoding.DecodeString(string(msg.Data))
		// if err != nil {
		// 	logger.Errorf("Error trying to base64 decode msg.Data from PubSub: %s", err.Error())
		// 	this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying
		// 	continue                 // try processing the next msg
		// }

		fmt.Printf("\nmsg.Data:\n%#v\n", string(msg.Data))

		evt := &SalesforceEvent{
			AckID: msg.AckID,
			MsgID: msg.ID,
		}
		if strings.Contains(string(msg.Data), "Lead") {
			var data *SalesforceLead
			if err := json.Unmarshal(msg.Data, &data); err != nil {
				logger.Errorf("Error trying to Unmarshal SalesforceLead from PubSub: %s", err.Error())
				this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying
				continue                 // try processing the next msg
			} else {
				evt.Data = data
			}
		} else {
			var data *SalesforceOpportunity
			if err := json.Unmarshal(msg.Data, &data); err != nil {
				logger.Errorf("Error trying to Unmarshal SalesforceOpportunity from PubSub: %s", err.Error())
				this.Ack(ctx, msg.AckID) // clear it out of subscription queue so we don't keep retrying
				continue                 // try processing the next msg
			} else {
				evt.Data = data
			}
		}

		events = append(events, evt)
		logger.Infof("Received SalesforceOpportunity %s", msg.ID)
	}

	return events, nil
}
Ejemplo n.º 3
0
// Subscribe prepares the peer to consume messages.
func (c *Peer) Subscribe() error {
	// Subscription names must start with a lowercase letter, end with a
	// lowercase letter or number, and contain only lowercase letters, numbers,
	// dashes, underscores or periods.
	c.subscription = strings.ToLower(fmt.Sprintf("x%sx", broker.GenerateName()))
	exists, err := pubsub.SubExists(c.context, c.subscription)
	if err != nil {
		return err
	}

	if exists {
		return fmt.Errorf("Subscription %s already exists", c.subscription)
	}

	if err := pubsub.CreateSub(c.context, c.subscription, topic, 0, ""); err != nil {
		return err
	}

	go c.ack()

	go func() {
		// TODO: Can we avoid using atomic flag?
		for atomic.LoadInt32(&c.stopped) != stopped {
			messages, err := pubsub.PullWait(c.context, c.subscription, bufferSize)
			if err != nil {
				// Timed out.
				continue
			}

			ids := make([]string, len(messages))
			for i, message := range messages {
				ids[i] = message.AckID
				c.messages <- message.Data
			}
			c.acks <- ids
		}
	}()
	return nil
}
Ejemplo n.º 4
0
func subscribe() {
	for {
		// Pull up to 10 messages (maybe fewer) from the subscription.
		// Blocks for an indeterminate amount of time.
		msgs, err := pubsub.PullWait(pubSubCtx, subName, 10)
		if err != nil {
			log.Fatalf("could not pull: %v", err)
		}

		for _, m := range msgs {
			msg := m

			var id int64
			if err := json.Unmarshal(msg.Data, &id); err != nil {
				log.Printf("could not decode message data: %#v", msg)
				go pubsub.Ack(pubSubCtx, subName, msg.AckID)
				continue
			}

			log.Printf("[ID %d] Processing.", id)
			go func() {
				if err := update(id); err != nil {
					log.Printf("[ID %d] could not update: %v", id, err)
					return
				}

				countMu.Lock()
				count++
				countMu.Unlock()

				pubsub.Ack(pubSubCtx, subName, msg.AckID)
				log.Printf("[ID %d] ACK", id)
			}()
		}
	}
}
Ejemplo n.º 5
0
func consumePubSubMsgs(projectId string, topicName string) <-chan []byte {
	msgC := make(chan []byte, 100)
	subName := topicName + "-2loggly"
	ackDeadline := time.Second * 20

	client, err := google.DefaultClient(context.Background(), pubsub.ScopePubSub)
	if err != nil {
		logFatal(LogData{
			"event": "pubsub.error.auth",
			"error": err.Error(),
		})
	}
	ctx := cloud.NewContext(projectId, client)

	isSub, err := pubsub.SubExists(ctx, subName)
	if err != nil {
		logFatal(LogData{
			"event": "pubsub.error.subexists",
			"error": err.Error(),
		})
	}
	if !isSub {
		err = pubsub.CreateSub(ctx, subName, topicName, ackDeadline, "")
		if err != nil {
			logFatal(LogData{
				"event": "pubsub.error.createsub",
				"error": err.Error(),
			})
		}
	}

	go func() {
		var lastAckErr error
		var lastDecodeErr error

		for {
			msgs, err := pubsub.PullWait(ctx, subName, 20)
			if err != nil {
				logInfo(LogData{
					"event": "pubsub.error.pull",
					"error": err.Error(),
				})
				time.Sleep(time.Second)
				continue
			}

			lastAckErr = nil
			lastDecodeErr = nil

			for _, m := range msgs {
				jsonBytes, err := msgToLogJson(m.Data)
				if err != nil {
					lastDecodeErr = err
				} else {
					msgC <- jsonBytes
				}

				err = pubsub.Ack(ctx, subName, m.AckID)
				if err != nil {
					lastAckErr = err
				}
			}

			// protect against flood of logs from this application
			// when acks/decodes fail but adding rollup & sleep delay
			if lastAckErr != nil {
				logInfo(LogData{
					"event": "pubsub.error.ack",
					"error": lastAckErr.Error(),
				})
				time.Sleep(time.Second)
			}
			if lastDecodeErr != nil {
				logInfo(LogData{
					"event": "pubsub.error.decode",
					"error": lastDecodeErr.Error(),
				})
				time.Sleep(time.Second)
			}
		}
	}()

	return msgC
}