func (this *Client) AddJob(payload []byte, delay string, opt PubOption) (jobId string, err error) {
buf := mpool.BytesBufferGet()
defer mpool.BytesBufferPut(buf)
buf.Reset()
buf.Write(payload)
var req *http.Request
var u url.URL
u.Scheme = this.cf.Pub.Scheme
u.Host = this.cf.Pub.Endpoint
u.Path = fmt.Sprintf("/v1/jobs/%s/%s", opt.Topic, opt.Ver)
q := u.Query()
q.Set("delay", delay)
u.RawQuery = q.Encode()
req, err = http.NewRequest("POST", u.String(), buf)
if err != nil {
return
}
req.Header.Set("AppId", this.cf.AppId)
req.Header.Set("Pubkey", this.cf.Secret)
var response *http.Response
response, err = this.pubConn.Do(req)
if err != nil {
return
}
// TODO if 201, needn't read body
var b []byte
b, err = ioutil.ReadAll(response.Body)
if err != nil {
return
}
// reuse the connection
response.Body.Close()
if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusAccepted {
return "", errors.New(string(b))
}
jobId = response.Header.Get(gateway.HttpHeaderJobId)
if this.cf.Debug {
log.Printf("--> [%s]", response.Status)
log.Printf("JobId:%s", response.Header.Get(gateway.HttpHeaderJobId))
}
return
}
func (this *dummyStore) KafkaTopic(appid string, topic string, ver string) (r string) {
b := mpool.BytesBufferGet()
b.Reset()
b.WriteString(appid)
b.WriteString(".")
b.WriteString(topic)
b.WriteString(".")
b.WriteString(ver)
r = b.String()
mpool.BytesBufferPut(b)
return
}
func BenchmarkManualCreateJson(b *testing.B) {
for i := 0; i < b.N; i++ {
buffer := mpool.BytesBufferGet()
buffer.Reset()
buffer.WriteString(`{"partition":`)
buffer.WriteString(strconv.Itoa(int(6)))
buffer.WriteString(`,"offset":`)
buffer.WriteString(strconv.Itoa(int(7)))
buffer.WriteString(`}`)
mpool.BytesBufferPut(buffer)
}
}
func (this *WebhookExecutor) pushToEndpoint(msg *sarama.ConsumerMessage, uri string) (ok bool) {
log.Debug("%s sending[%s] %s", this.topic, uri, string(msg.Value))
if this.circuits[uri].Open() {
log.Warn("%s %s circuit open", this.topic, uri)
return false
}
body := mpool.BytesBufferGet()
defer mpool.BytesBufferPut(body)
body.Reset()
body.Write(msg.Value)
// TODO user defined post body schema, e,g. ElasticSearch
req, err := http.NewRequest("POST", uri, body)
if err != nil {
this.circuits[uri].Fail()
return false
}
req.Header.Set(gateway.HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
req.Header.Set(gateway.HttpHeaderPartition, strconv.FormatInt(int64(msg.Partition), 10))
req.Header.Set("User-Agent", this.userAgent)
req.Header.Set("X-App-Signature", this.appSignature)
response, err := this.httpClient.Do(req)
if err != nil {
log.Error("%s %s %s", this.topic, uri, err)
this.circuits[uri].Fail()
return false
}
io.Copy(ioutil.Discard, response.Body)
response.Body.Close()
if response.StatusCode >= 300 {
this.circuits[uri].Fail()
log.Error("%s %s response: %s", this.topic, uri, http.StatusText(response.StatusCode))
return
}
// audit
log.Info("pushed %s/%d %d", this.topic, msg.Partition, msg.Offset)
return true
}
func (this *mysqlStore) KafkaTopic(appid string, topic string, ver string) (r string) {
b := mpool.BytesBufferGet()
b.Reset()
b.WriteString(appid)
b.WriteByte('.')
b.WriteString(topic)
b.WriteByte('.')
b.WriteString(ver)
if len(ver) > 2 {
// ver starts with 'v1', from 'v10' on, will use obfuscation
b.WriteByte('.')
// can't use app secret as part of cookie: what if user changes his secret?
// FIXME user can guess the cookie if they know the algorithm in advance
cookie := adler32.Checksum([]byte(appid + topic))
b.WriteString(strconv.Itoa(int(cookie % 1000)))
}
r = b.String()
mpool.BytesBufferPut(b)
return
}
func (this *Agent) Callback(endpoint string, msg *sarama.ConsumerMessage) error {
buf := mpool.BytesBufferGet()
defer mpool.BytesBufferPut(buf)
buf.Reset()
buf.Write(msg.Value)
req, err := http.NewRequest("POST", endpoint, buf)
if err != nil {
return err
}
// setup http headers
req.Header.Set("User-Agent", UserAgent)
// invoke the callback over the wire
response, err := this.callbackConn.Do(req)
if err != nil {
return err
}
b, err := ioutil.ReadAll(response.Body)
if err != nil {
return err
}
// reuse the connection
response.Body.Close()
if response.StatusCode != http.StatusCreated {
return errors.New(string(b))
}
if this.debug {
log.Debug("%s -> %s", endpoint, string(msg.Value))
}
return nil
}
// Pub publish a keyed message to specified versioned topic.
func (this *Client) Pub(key string, msg []byte, opt PubOption) (err error) {
buf := mpool.BytesBufferGet()
defer mpool.BytesBufferPut(buf)
buf.Reset()
buf.Write(msg)
var req *http.Request
var u url.URL
u.Scheme = this.cf.Pub.Scheme
u.Host = this.cf.Pub.Endpoint
u.Path = fmt.Sprintf("/v1/msgs/%s/%s", opt.Topic, opt.Ver)
q := u.Query()
q.Set("key", key)
if opt.AckAll {
q.Set("ack", "all")
}
if opt.Async {
q.Set("async", "1")
}
u.RawQuery = q.Encode()
req, err = http.NewRequest("POST", u.String(), buf)
if err != nil {
return
}
req.Header.Set(gateway.HttpHeaderAppid, this.cf.AppId)
req.Header.Set(gateway.HttpHeaderPubkey, this.cf.Secret)
if opt.Tag != "" {
req.Header.Set(gateway.HttpHeaderMsgTag, opt.Tag)
}
var response *http.Response
response, err = this.pubConn.Do(req)
// when you get a redirection failure both response and err will be non-nil
if response != nil {
// reuse the connection
defer response.Body.Close()
}
if err != nil {
return
}
// TODO if 201, needn't read body
var b []byte
b, err = ioutil.ReadAll(response.Body)
if err != nil {
return
}
if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusAccepted {
return errors.New(string(b))
}
if this.cf.Debug {
log.Printf("--> [%s]", response.Status)
log.Printf("Partition:%s Offset:%s",
response.Header.Get(gateway.HttpHeaderPartition),
response.Header.Get(gateway.HttpHeaderOffset))
}
return nil
}