func (s *endpointServiceImpl) send(ctx context.Context, data []byte) error {
ctx = log.SetField(ctx, "endpointURL", s.url)
return retryCall(ctx, "endpoint.send", func() error {
startTime := clock.Now(ctx)
log.Debugf(ctx, "Pushing message to endpoint.")
req, err := http.NewRequest("POST", s.url, bytes.NewReader(data))
if err != nil {
log.Errorf(log.SetError(ctx, err), "Failed to create HTTP request.")
return err
}
req.Header.Add("content-type", protobufContentType)
req.Header.Add("user-agent", monitoringEndpointUserAgent)
resp, err := s.client.Do(req)
if err != nil {
// Treat a client error as transient.
log.Warningf(log.SetError(ctx, err), "Failed proxy client request.")
return errors.WrapTransient(err)
}
defer resp.Body.Close()
// Read the full response body. This will enable us to re-use the
// connection.
bodyData, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Errorf(log.SetError(ctx, err), "Error during endpoint connection.")
return errors.WrapTransient(err)
}
log.Fields{
"status": resp.Status,
"statusCode": resp.StatusCode,
"headers": resp.Header,
"contentLength": resp.ContentLength,
"body": string(bodyData),
"duration": clock.Now(ctx).Sub(startTime),
}.Debugf(ctx, "Received HTTP response from endpoint.")
if http.StatusOK <= resp.StatusCode && resp.StatusCode < http.StatusMultipleChoices {
log.Debugf(ctx, "Message pushed successfully.")
return nil
}
err = fmt.Errorf("http: server error (%d)", resp.StatusCode)
if resp.StatusCode >= http.StatusInternalServerError {
err = errors.WrapTransient(err)
}
log.Fields{
log.ErrorKey: err,
"status": resp.Status,
"statusCode": resp.StatusCode,
}.Warningf(ctx, "Proxy error.")
return err
})
}
// pullAckMessages pulls a set of messages from the configured Subscription.
// If no messages are available, errNoMessages will be returned.
//
// handler is a method that returns true if there was a transient failure,
// indicating that the messages shouldn't be ACK'd.
func (p *pubsubClient) pullAckMessages(ctx context.Context, handler func([]*pubsub.Message)) error {
var err error
var msgs []*pubsub.Message
ackCount := 0
// Report the duration of a Pull/ACK cycle.
startTime := clock.Now(ctx)
defer func() {
duration := clock.Now(ctx).Sub(startTime)
log.Fields{
"count": len(msgs),
"ackCount": ackCount,
"duration": duration,
}.Infof(ctx, "Pull/ACK cycle complete.")
}()
err = retryCall(ctx, "Pull()", func() error {
var err error
msgs, err = p.service.Pull(p.subscription, p.batchSize)
return p.wrapTransient(err)
})
log.Fields{
log.ErrorKey: err,
"duration": clock.Now(ctx).Sub(startTime),
"count": len(msgs),
}.Debugf(ctx, "Pull() complete.")
if err != nil {
return err
}
if len(msgs) == 0 {
return errNoMessages
}
defer func() {
ackCount, err = p.ackMessages(ctx, msgs)
if err != nil {
log.Warningf(log.SetError(ctx, err), "Failed to ACK messages!")
}
}()
handler(msgs)
return nil
}
func (d *dsCache) GetMulti(keys []ds.Key, metas ds.MultiMetaGetter, cb ds.GetMultiCB) error {
lockItems, nonce := d.mkRandLockItems(keys, metas)
if len(lockItems) == 0 {
return d.RawInterface.GetMulti(keys, metas, cb)
}
if err := d.mc.AddMulti(lockItems); err != nil {
(log.Fields{log.ErrorKey: err}).Warningf(
d.c, "dscache: GetMulti: memcache.AddMulti")
}
if err := d.mc.GetMulti(lockItems); err != nil {
(log.Fields{log.ErrorKey: err}).Warningf(
d.c, "dscache: GetMulti: memcache.GetMulti")
}
p := makeFetchPlan(d.c, d.aid, d.ns, &facts{keys, metas, lockItems, nonce})
if !p.empty() {
// looks like we have something to pull from datastore, and maybe some work
// to save stuff back to memcache.
toCas := []memcache.Item{}
j := 0
err := d.RawInterface.GetMulti(p.toGet, p.toGetMeta, func(pm ds.PropertyMap, err error) {
i := p.idxMap[j]
toSave := p.toSave[j]
j++
data := []byte(nil)
// true: save entity to memcache
// false: lock entity in memcache forever
shouldSave := true
if err == nil {
p.decoded[i] = pm
if toSave != nil {
data = encodeItemValue(pm)
if len(data) > internalValueSizeLimit {
shouldSave = false
log.Warningf(
d.c, "dscache: encoded entity too big (%d/%d)!",
len(data), internalValueSizeLimit)
}
}
} else {
p.lme.Assign(i, err)
if err != ds.ErrNoSuchEntity {
return // aka continue to the next entry
}
}
if toSave != nil {
if shouldSave { // save
expSecs := metas.GetMetaDefault(i, CacheExpirationMeta, CacheTimeSeconds).(int64)
toSave.SetFlags(uint32(ItemHasData))
toSave.SetExpiration(time.Duration(expSecs) * time.Second)
toSave.SetValue(data)
} else {
// Set a lock with an infinite timeout. No one else should try to
// serialize this item to memcache until something Put/Delete's it.
toSave.SetFlags(uint32(ItemHasLock))
toSave.SetExpiration(0)
toSave.SetValue(nil)
}
toCas = append(toCas, toSave)
}
})
if err != nil {
return err
}
if len(toCas) > 0 {
// we have entries to save back to memcache.
if err := d.mc.CompareAndSwapMulti(toCas); err != nil {
(log.Fields{log.ErrorKey: err}).Warningf(
d.c, "dscache: GetMulti: memcache.CompareAndSwapMulti")
}
}
}
// finally, run the callback for all of the decoded items and the errors,
// if any.
for i, dec := range p.decoded {
cb(dec, p.lme.GetOne(i))
}
return nil
}
// setupSubscription asserts that the configured subscription exists. In doing
// so, it also asserts that the client credentials are valid with respect to the
// configured project/subscription.
//
// If the subscription doesn't exist, this method can create the subscription
// and (if missing) its topic, if the "create" flag is set.
func (p *pubsubClient) setupSubscription(ctx context.Context) error {
exists := false
log.Fields{
"subscription": p.topic,
}.Infof(ctx, "Checking for subscription existence.")
err := retryCall(ctx, "SubExists()", func() error {
var err error
exists, err = p.service.SubExists(p.subscription)
return p.wrapTransient(err)
})
if err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to test for subscription; assuming it doesn't exist.")
}
if exists {
return nil
}
if !p.create {
return errors.New("pubsub: subscription doesn't exist, not configured to create")
}
// Create the subscription if it doesn't exist.
if p.topic == "" {
log.Errorf(ctx, "Cannot create subscription; no topic was specified.")
return errors.New("pubsub: cannot create subscription")
}
// Test if the topic exists...
log.Fields{
"topic": p.topic,
}.Infof(ctx, "Checking for topic existence.")
err = retryCall(ctx, "TopicExists()", func() error {
var err error
exists, err = p.service.TopicExists(p.topic)
return p.wrapTransient(err)
})
if err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to test for topic; assuming it doesn't exist.")
}
if !exists {
log.Fields{
"topic": p.topic,
}.Infof(ctx, "Creating topic.")
err := retryCall(ctx, "CreateTopic()", func() error {
return p.service.CreateTopic(p.topic)
})
if err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to create topic.")
return errors.New("pubsub: cannot create topic")
}
}
log.Fields{
"topic": p.topic,
"subscription": p.subscription,
}.Infof(ctx, "Creating pull subscription for topic.")
if err := retryCall(ctx, "CreateSub()", func() error {
return p.service.CreatePullSub(p.subscription, p.topic)
}); err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to test for subscription; assuming it doesn't exist.")
return errors.New("pubsub: failed to create subscription")
}
return nil
}
// mainImpl is the main execution function.
func mainImpl(args []string) int {
// Use all of teh corez.
if os.Getenv("GOMAXPROCS") == "" {
runtime.GOMAXPROCS(runtime.NumCPU())
}
// Install a console logger by default.
ctx := context.Background()
ctx = gologger.Use(ctx)
loggerConfig := newLoggerConfig() // Internal logging config (cloud logging).
logConfig := log.Config{Level: log.Debug}
config := config{}
fs := flag.CommandLine
config.addFlags(fs)
loggerConfig.addFlags(fs)
logConfig.AddFlags(fs)
fs.Parse(args)
// TODO(dnj): Fix this once LUCI logging CL lands.
ctx = log.SetLevel(ctx, logConfig.Level)
ctx = logConfig.Set(ctx)
// Load authenticated client.
client, err := config.createAuthenticatedClient(ctx)
if err != nil {
log.Errorf(log.SetError(ctx, err), "Failed to create authenticated service client.")
return 1
}
// Setup local logging configuration.
ctx, logFlushFunc, err := loggerConfig.use(ctx, client)
if err == nil {
defer logFlushFunc()
} else {
log.Warningf(log.SetError(ctx, err), "Failed to setup cloud logging.")
}
app := newApplication(config)
if err := app.loadServices(ctx, client); err != nil {
log.Errorf(log.SetError(ctx, err), "Failed to initialize services.")
return 1
}
// Set up interrupt handler.
signalC := make(chan os.Signal, 1)
go func() {
triggered := false
for sig := range signalC {
if !triggered {
triggered = true
log.Infof(log.SetField(ctx, "signal", sig),
"Received signal; starting shutdown.")
app.shutdown()
} else {
// Triggered multiple times; immediately shut down.
os.Exit(1)
}
}
}()
signal.Notify(signalC, os.Interrupt, os.Kill)
defer func() {
signal.Stop(signalC)
close(signalC)
}()
log.Infof(ctx, "Starting application execution...")
if err := app.run(ctx); err != nil {
log.Errorf(log.SetError(ctx, err), "Error during application execution.")
return 1
}
return 0
}