コード例 #1
0
ファイル: main.go プロジェクト: keelerh/chrome_infra
// run executes the application.
func (a *application) run(ctx context.Context) error {
	// Setup common context parameters.
	ctx, cancelFunc := context.WithCancel(ctx)
	ctx = retry.WithTransientOnly(retry.Use(ctx, exponentialBackoff))

	// Monitor our shutdown singal. Cancel our context if it is closed.
	go func() {
		<-a.shutdownC

		log.Infof(ctx, "Shutdown signal received; cancelling context.")
		cancelFunc()
	}()

	wg := sync.WaitGroup{}
	for i := 0; i < a.config.numWorkers; i++ {
		i := i
		wg.Add(1)
		go func(id int) {
			defer wg.Done()

			a.process(log.SetField(ctx, "worker", i))
		}(i)
	}

	wg.Wait()
	return nil
}
コード例 #2
0
ファイル: main_test.go プロジェクト: keelerh/chrome_infra
func TestMain(t *testing.T) {
	t.Parallel()

	// Do not retry.
	ctx := retry.Use(context.Background(), func(context.Context) retry.Iterator {
		return &retry.Limited{}
	})

	Convey(`An application using testing stubs`, t, func() {
		config := config{
			pubsub: pubsubConfig{
				project:      "test-project",
				topic:        "test-topic",
				subscription: "test-subscription",
				create:       false,
				batchSize:    64,
			},
			endpoint: endpointConfig{
				url: "fake-protocol://test.endpoint",
			},
			numWorkers: 10,
		}

		pubsubMock := &testPubSubService{
			infinitePull: true,
		}
		endpointMock := &endpointServiceMock{
			msgC: make(chan []byte),
		}

		pubsubMock.MockCall("SubExists", "test-subscription").WithResult(true, nil)
		pubsubClient, err := newPubSubClient(ctx, config.pubsub, pubsubMock)
		So(err, ShouldBeNil)

		app := newApplication(config)
		app.pubsub = pubsubClient
		app.endpoint = endpointMock

		// Run the application. Shut down after each test is complete.
		finishedC := make(chan struct{})
		go func() {
			app.run(ctx)
			close(finishedC)
		}()
		defer func() {
			app.shutdown()
			<-finishedC

			So(pubsubMock, mock.ShouldHaveNoErrors)
			So(endpointMock, mock.ShouldHaveNoErrors)
		}()

		Convey(`Will consume messages from Pub/Sub.`, func() {
			buf := make([]byte, binary.MaxVarintLen64)
			missing := make(map[int]bool)
			for i := 0; i < 1024; i++ {
				missing[i] = true
				count := binary.PutUvarint(buf, uint64(i))

				msg := &pubsub.Message{
					ID:    fmt.Sprintf("msg-%d", i),
					AckID: fmt.Sprintf("ack-%d", i),
					Data:  make([]byte, count),
				}
				copy(msg.Data, buf)
				pubsubMock.MockCall("Pull", "test-subscription", 64).WithResult([]*pubsub.Message{msg}, nil)
				pubsubMock.MockCall("Ack", "test-subscription", []string{msg.AckID}).WithResult(nil)
				endpointMock.MockCall("send", mock.Ignore, msg.Data).WithResult(nil)
			}

			missingCount := len(missing)
			for i := 0; i < 1024; i++ {
				msg := <-endpointMock.msgC
				index, _ := binary.Uvarint(msg)

				v := missing[int(index)]
				So(v, ShouldBeTrue)

				missing[int(index)] = false
				missingCount--
			}
			So(missingCount, ShouldEqual, 0)
		})

		Convey(`Will refuse to process a message that is too large, and will ACK it.`, func() {
			msgs := []*pubsub.Message{
				{
					ID:    "msg-big",
					AckID: "ack-big",
					Data:  bytes.Repeat([]byte{0xAA}, maxMessageSize+1),
				},
				{
					ID:    "msg-legit",
					AckID: "ack-legit",
					Data:  bytes.Repeat([]byte{0x55}, maxMessageSize),
				},
			}

			pubsubMock.MockCall("Pull", "test-subscription", 64).WithResult(msgs, nil)
			pubsubMock.MockCall("Ack", "test-subscription", []string{"ack-big", "ack-legit"}).WithResult(nil)
			endpointMock.MockCall("send", mock.Ignore, msgs[1].Data).WithResult(nil)

			So(<-endpointMock.msgC, ShouldResemble, msgs[1].Data)
		})
	})
}
コード例 #3
0
ファイル: logging_test.go プロジェクト: keelerh/chrome_infra
func TestCloudLogging(t *testing.T) {
	Convey(`A cloud logging instance using a test HTTP client/server`, t, func() {
		ctx, _ := testclock.UseTime(context.Background(), time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC))

		// Do not retry.
		ctx = retry.Use(ctx, func(context.Context) retry.Iterator {
			return &retry.Limited{}
		})

		h := &testCloudLoggingHandler{
			logC: make(chan *cloudLogBundle, 1),
		}
		srv := httptest.NewServer(h)
		defer srv.Close()

		tr := &http.Transport{
			DialTLS: func(network, addr string) (net.Conn, error) {
				u, err := url.Parse(srv.URL)
				if err != nil {
					return nil, err
				}
				return net.Dial(network, u.Host)
			},
		}
		client := &http.Client{
			Transport: tr,
		}

		config := loggerConfig{
			projectName:  "test-project",
			resourceType: "test-resource",
			logsID:       "test-logs-id",
		}

		service, err := cloudlog.New(client)
		So(err, ShouldBeNil)

		cl := newCloudLogging(ctx, &config, service)
		defer cl.finish()

		Convey(`A bound cloud logging instance`, func() {
			l := cl.bind(ctx)

			Convey(`Can publish logging data.`, func() {
				l.Infof("Message at %s", "INFO")

				bundle := <-h.logC
				So(len(bundle.Entries), ShouldEqual, 1)
				So(bundle.Entries[0], shouldMatchLog, &cloudlog.LogEntry{
					InsertId: "-0-0",
					Metadata: &cloudlog.LogEntryMetadata{
						ProjectId: "test-project",
						Severity:  "INFO",
						Timestamp: "2015-01-01T00:00:00Z",
					},
					TextPayload: "Message at INFO",
				})
			})

			Convey(`Will batch logging data.`, func() {
				cl.testLogAckC = make(chan []*logEntry, 1)

				// The first message will be read immediately.
				l.Infof("Initial unbatched message.")
				<-cl.testLogAckC

				// The next set of messages will be batched, since we're not release our
				// HTTP server yet.
				for i := 0; i < cloudLoggingBatchSize; i++ {
					l.Infof("Batch message #%d", i)
				}
				<-cl.testLogAckC

				// Read the first bundle.
				bundle := <-h.logC
				So(len(bundle.Entries), ShouldEqual, 1)
				So(bundle.Entries[0], shouldMatchLog, &cloudlog.LogEntry{
					InsertId: "-0-0",
					Metadata: &cloudlog.LogEntryMetadata{
						ProjectId: "test-project",
						Severity:  "INFO",
						Timestamp: "2015-01-01T00:00:00Z",
					},
					TextPayload: "Initial unbatched message.",
				})

				// Read the second bundle.
				bundle = <-h.logC
				So(len(bundle.Entries), ShouldEqual, cloudLoggingBatchSize)
				for i, entry := range bundle.Entries {
					So(entry, shouldMatchLog, &cloudlog.LogEntry{
						InsertId: fmt.Sprintf("-1-%d", i),
						Metadata: &cloudlog.LogEntryMetadata{
							ProjectId: "test-project",
							Severity:  "INFO",
							Timestamp: "2015-01-01T00:00:00Z",
						},
						TextPayload: fmt.Sprintf("Batch message #%d", i),
					})
				}
			})
		})
	})
}
コード例 #4
0
ファイル: pubsub_test.go プロジェクト: keelerh/chrome_infra
func TestPubSub(t *testing.T) {
	t.Parallel()

	Convey(`Using a testing Pub/Sub config`, t, func() {
		// Do not retry.
		ctx := retry.Use(context.Background(), func(context.Context) retry.Iterator {
			return &retry.Limited{}
		})

		config := pubsubConfig{
			project:      "test-project",
			topic:        "test-topic",
			subscription: "test-subscription",
			create:       true,
			batchSize:    64,
		}

		svc := &testPubSubService{}
		defer So(svc, mock.ShouldHaveNoErrors)

		Convey(`When the subscription does not exist`, func() {
			svc.MockCall("SubExists", "test-subscription").WithResult(false, nil)

			Convey(`And the topic does not exist, will create a new topic and subscription.`, func() {
				svc.MockCall("TopicExists", "test-topic").WithResult(false, nil)
				svc.MockCall("CreateTopic", "test-topic").WithResult(nil)
				svc.MockCall("CreatePullSub", "test-subscription", "test-topic").WithResult(nil)

				_, err := newPubSubClient(ctx, config, svc)
				So(err, ShouldBeNil)
			})

			Convey(`And the topic exists, will create a new subscription.`, func() {
				svc.MockCall("TopicExists", "test-topic").WithResult(true, nil)
				svc.MockCall("CreatePullSub", "test-subscription", "test-topic").WithResult(nil)

				_, err := newPubSubClient(ctx, config, svc)
				So(err, ShouldBeNil)
			})

			Convey(`Will fail to create a new client when "create" is false.`, func() {
				config.create = false

				_, err := newPubSubClient(ctx, config, svc)
				So(err, ShouldNotBeNil)
			})
		})

		Convey(`Will create a new client.`, func() {
			svc.MockCall("SubExists", "test-subscription").WithResult(true, nil)

			client, err := newPubSubClient(ctx, config, svc)
			So(err, ShouldBeNil)

			Convey(`When executing pull/ack with no messages`, func() {
				svc.MockCall("Pull", "test-subscription", 64).WithResult(nil, nil)

				Convey(`Returns errNoMessages.`, func() {
					err := client.pullAckMessages(ctx, func([]*pubsub.Message) {})
					So(err, ShouldEqual, errNoMessages)
				})
			})

			Convey(`When executing pull/ack with one message`, func() {
				msgs := []*pubsub.Message{
					{
						ID:    "id0",
						AckID: "ack0",
						Data:  []byte{0xd0, 0x65},
					},
				}
				svc.MockCall("Pull", "test-subscription", 64).WithResult(msgs, nil)

				Convey(`Returns and ACKs that message.`, func() {
					svc.MockCall("Ack", "test-subscription", []string{"ack0"}).WithResult(nil)

					var pullMsg []*pubsub.Message
					err := client.pullAckMessages(ctx, func(msg []*pubsub.Message) {
						pullMsg = msg
					})
					So(err, ShouldBeNil)

					So(pullMsg, ShouldResemble, msgs)
				})

				Convey(`ACKs the message even if the handler panics.`, func() {
					svc.MockCall("Ack", "test-subscription", []string{"ack0"}).WithResult(nil)

					So(func() {
						client.pullAckMessages(ctx, func(msg []*pubsub.Message) {
							panic("Handler failure!")
						})
					}, ShouldPanic)
				})

				Convey(`Does not ACK the message if the handler clears it.`, func() {
					err := client.pullAckMessages(ctx, func(msg []*pubsub.Message) {
						for i := range msg {
							msg[i] = nil
						}
					})
					So(err, ShouldBeNil)
				})
			})

			Convey(`When executing pull/ack with an error`, func() {
				e := errors.New("TEST ERROR")
				svc.MockCall("Pull", "test-subscription", 64).WithResult(nil, e)

				Convey(`Returns the error as transient.`, func() {
					So(client.pullAckMessages(ctx, func([]*pubsub.Message) {}), ShouldResemble, luciErrors.Transient{Err: e})
				})
			})
		})
	})
}
コード例 #5
0
ファイル: endpoint_test.go プロジェクト: keelerh/chrome_infra
// TestEndpoint tests the endpoint implementation and API.
func TestEndpointService(t *testing.T) {
	t.Parallel()

	Convey(`An endpoint service connected to a testing HTTP server`, t, func() {
		ctx, tc := testclock.UseTime(context.Background(), time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC))

		// Retry up to ten times without delaying.
		ctx = retry.Use(ctx, func(context.Context) retry.Iterator {
			return &retry.Limited{Retries: 10}
		})

		h := &testEndpointServiceHandler{}
		srv := httptest.NewTLSServer(h)
		defer srv.Close()

		tr := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
		client := &http.Client{
			Transport: tr,
		}

		c := &endpointServiceImpl{
			endpointConfig: endpointConfig{url: srv.URL},
			client:         client,
		}

		msg := bytes.Repeat([]byte{0x60, 0x0d, 0xd0, 0x65}, 32)
		Convey(`Successfully posts a message.`, func() {
			So(c.send(ctx, msg), ShouldBeNil)

			So(h.connections, ShouldEqual, 1)
			So(h.errors, ShouldEqual, 0)
			So(h.messages, ShouldResemble, [][]byte{msg})
		})

		Convey(`Retries sending when an error is encountered.`, func() {
			tc.SetTimerCallback(func(t clock.Timer) {
				tc.Add(time.Second)
			})
			h.failures = 4
			So(c.send(ctx, msg), ShouldBeNil)

			So(h.connections, ShouldEqual, 5)
			So(h.errors, ShouldEqual, 4)
			So(h.messages, ShouldResemble, [][]byte{msg})
		})

		Convey(`Returns a transient error when a send completely fails.`, func() {
			h.failures = 11
			err := c.send(ctx, msg)
			So(err, ShouldNotBeNil)
			So(err, ShouldHaveSameTypeAs, luciErrors.Transient{})

			So(h.connections, ShouldEqual, 11)
			So(h.errors, ShouldEqual, 11)
			So(h.messages, ShouldResemble, [][]byte(nil))
		})
	})
}