// Grabs the data stored in requestURI.
// The requestURI is resolved using the `STORAGE_BASE_URL` as the base.
//
// Retries transient errors `retries` number of times.
//
// Usage:
//
// requestURI := "1/2/3?X-Amz-Algorithm=...&..."
// reader, err := storage.Get(requestURI, 0)
//
func Get(requestURI string, offset int64) (rd io.ReadCloser, err error) {
for i := retries; i > 0; i-- {
rd, err = get(requestURI, offset)
if err == nil {
util.Count("storage.get.success")
return rd, nil
}
if err != Err5xx {
util.Count("storage.get.error")
return rd, err
}
// Close the body immediately to prevent
// file descriptor leaks.
if rd != nil {
rd.Close()
}
util.Count("storage.get.retry")
}
// We've ran out of retries
util.Count("storage.get.maxretries")
return rd, err
}
func (s *Server) createStream(w http.ResponseWriter, r *http.Request) {
registrar := broker.NewRedisRegistrar()
if err := registrar.Register(key(r)); err != nil {
http.Error(w, "Unable to create stream. Please try again.", http.StatusServiceUnavailable)
rollbar.Error(rollbar.ERR, fmt.Errorf("unable to register stream: %#v", err))
util.CountWithData("put.create.fail", 1, "error=%s", err)
return
}
util.Count("put.create.success")
w.WriteHeader(http.StatusCreated)
}
// Stores the given reader onto the underlying blob storage
// with the given requestURI. The requestURI is resolved
// using the `STORAGE_BASE_URL` as the base.
//
// Retries transient errors `retries` number of times.
//
// Usage:
//
// reader := strings.NewReader("hello")
// requestURI := "1/2/3?X-Amz-Algorithm=...&..."
// err := storage.Put(requestURI, reader)
//
func Put(requestURI string, reader io.Reader) (err error) {
for i := retries; i > 0; i-- {
err = put(requestURI, reader)
// Break if we get nil / any error other than Err5xx
if err == nil {
util.Count("storage.put.success")
return nil
}
if err != Err5xx {
util.Count("storage.put.error")
return err
}
// Log the put retry
util.Count("storage.put.retry")
}
// We've ran out of retries
util.Count("storage.put.maxretries")
return err
}
func (r *reader) read(msg redis.PMessage, p []byte) (n int, err error) {
var buf []byte
if msg.Channel == r.channel.killID() || msg.Channel == r.channel.id() {
buf, err = r.fetch(len(p))
if n = len(buf); n > 0 {
copy(p, buf)
r.offset += int64(n)
}
}
if msg.Channel == r.channel.killID() || err == io.EOF {
util.Count("RedisBroker.redisSubscribe.Channel.kill")
r.Close()
err = io.EOF
}
return n, err
}
func mkstream(w http.ResponseWriter, _ *http.Request) {
registrar := broker.NewRedisRegistrar()
uuid, err := util.NewUUID()
if err != nil {
http.Error(w, "Unable to create stream. Please try again.", http.StatusServiceUnavailable)
rollbar.Error(rollbar.ERR, fmt.Errorf("unable to create new uuid for stream: %#v", err))
util.CountWithData("mkstream.create.fail", 1, "error=%s", err)
return
}
if err := registrar.Register(uuid); err != nil {
http.Error(w, "Unable to create stream. Please try again.", http.StatusServiceUnavailable)
rollbar.Error(rollbar.ERR, fmt.Errorf("unable to register stream: %#v", err))
util.CountWithData("mkstream.create.fail", 1, "error=%s", err)
return
}
util.Count("mkstream.create.success")
io.WriteString(w, string(uuid))
}
func (r *reader) replay(p []byte) (n int, err error) {
var buf []byte
if !r.replayed {
buf, err = r.fetch(len(p))
n = len(buf)
if n > 0 {
r.offset += int64(n)
copy(p, buf)
}
// Only mark as fully replayed if
// we're no longer in buffered state.
r.replayed = (!r.buffered || err == io.EOF)
if err == io.EOF {
util.Count("RedisBroker.replay.channelDone")
}
}
return n, err
}
