// MultiUploader accepts a list of filepaths which are synced to a remote server
func (c *Client) MultiUploader(filepaths []string) (files Resources, err error) {
files, err = c.multiStat(filepaths)
if err != nil {
return nil, err
}
toUpload := make([]string, 0, len(filepaths))
for _, res := range files {
if res.Created {
toUpload = append(toUpload, res.Path)
} else {
log.Printf("%s exists - skipping", res.Path)
}
}
if len(toUpload) == 0 {
return files, nil
}
log.Printf("upload %d %v", len(toUpload), toUpload)
err = c.multiUpload(files, toUpload)
return files, err
}
// LogHandler adds logging to http requests
func LogHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
log.Printf("%s %s", r.Method, r.URL.Path)
h.ServeHTTP(w, r)
//log.Println(r.Method, r.URL, r.Form, time.Since(start))
log.Printf("%s %s in %v\n", r.Method, r.URL.Path, time.Since(start))
})
}
func ListenAndServe(laddr string) error {
l, err := net.Listen("tcp", laddr)
if err != nil {
return err
}
log.Printf("Listen on %s", l.Addr())
sig.TrapCloser(l)
err = serve(l)
log.Printf("Shutting down ..")
return err
}
func parseRequest(buf []byte) (resp []byte, err error) {
switch {
case bytes.Equal(protocolPolicy, buf):
log.Printf("Policy request")
resp = protocolPolicyResponse
case bytes.Equal(protocolPing, buf):
log.Printf("Ping request")
resp = protocolPingResponse
default:
err = fmt.Errorf("Invalid protocol request")
}
return
}
func blacklistGet(t *testing.T, count, offset int, out []string) {
r, err := http.Get(fmt.Sprintf("http://%s/v1/profanity/blacklist/?lang=%s&count=%d&offset=%d", serverAddr, "en_US", count, offset))
if err != nil {
t.Fatalf("error getting: %s", err)
return
}
var res blacklistResponse
err = json.NewDecoder(r.Body).Decode(&res)
if err != nil {
t.Fatal(err)
}
log.Printf("res: %v", res)
expLen := math.IntMin(math.IntMax(res.Total-offset, 0), count)
if len(res.Blacklist) != expLen {
t.Fatalf("%d != %d", len(res.Blacklist), expLen)
}
if out != nil {
for i := 0; i < len(res.Blacklist); i++ {
if res.Blacklist[i] != out[i] {
t.Fatalf("%s != %s", res.Blacklist[i], out[i])
}
}
}
}
func handle(conn net.Conn) {
defer conn.Close()
err := conn.SetDeadline(time.Now().Add(Timeout))
if err != nil {
log.Errorf("Error setting deadline on conn")
return
}
buf := getBuf(bufSize)
defer putBuf(buf)
n, err := conn.Read(buf)
if err != nil {
log.Error("Error reading from conn")
return
}
log.Printf("Got %+q", buf[:n])
resp, err := parseRequest(buf[:n])
if err != nil {
log.Error(err)
return
}
n, err = conn.Write(resp)
if err != nil || n != len(resp) {
log.Errorf("Error writing to conn")
return
}
}
func ListenAndServe(laddr string, storage blobserver.Storage) error {
if err := setupServer(storage); err != nil {
return err
}
l, err := net.Listen("tcp", laddr)
if err != nil {
return err
}
log.Printf("Listen on %s", l.Addr())
sig.TrapCloser(l)
err = http.Serve(l, nil)
log.Printf("Shutting down ..")
return err
}
func runServer() {
router := mux.NewRouter()
router.HandleFunc("/", indexHandler).Methods("GET").Name("index")
router.HandleFunc("/", creatorHandler).Methods("POST").Name("key-create")
router.HandleFunc("/{key:[0-9A-Za-z-]+}/", deleteHandler).Methods("DELETE").Name("key-delete")
router.HandleFunc("/select-region/", selectRegionHandler).Methods("POST").Name("select-region")
log.Printf("Listen on %s\n", config.Laddr)
http.ListenAndServe(config.Laddr, router)
}
func ListenAndServe(laddr string, imgBackend backend.ImageBackend) error {
if err := setupServer(imgBackend); err != nil {
return err
}
l, err := net.Listen("tcp", laddr)
if err != nil {
return err
}
log.Printf("Listen on %s", l.Addr())
sigTrapCloser(l)
err = http.Serve(l, nil)
log.Printf("Shutting down ..")
return err
}
func BenchmarkPolicy(b *testing.B) {
b.StopTimer()
var wg sync.WaitGroup
l, err := net.Listen("tcp", ":9001")
defer l.Close()
if err != nil {
b.Fatal(err)
}
log.Printf("Listen on %v", l.Addr())
wg.Add(1)
go func() {
serve(l)
wg.Done()
}()
b.StartTimer()
k := 8 << 2
for i := 0; i < b.N; i += k {
var wg2 sync.WaitGroup
for j := 0; j < k; j++ {
wg2.Add(1)
go func() {
conn, err := net.Dial("tcp", ":9001")
if err != nil {
b.Errorf("error dialing: %v", err)
}
n, err := conn.Write(protocolPolicy)
if err != nil || n != len(protocolPolicy) {
b.Errorf("error sending: %v, %d == %d", err, n, len(protocolPolicy))
}
buf := make([]byte, testBufSize)
n, err = conn.Read(buf)
if err != nil || n != len(protocolPolicyResponse) {
b.Errorf("error reading: %v, %d == %d", err, n, len(protocolPolicyResponse))
}
if !bytes.Equal(protocolPolicyResponse, buf[:n]) {
b.Errorf("unexpected response value, exp: %+q, got %+q", protocolPolicyResponse, buf[:n])
}
wg2.Done()
}()
}
wg2.Wait()
}
b.StopTimer()
l.Close()
wg.Wait()
b.StartTimer()
}
func handleStat(req *http.Request, storage blobserver.BlobStatter, blobs []string) (interface{}, error) {
res := new(protocol.StatResponse)
needStat := map[blob.Ref]bool{}
n := 0
for _, value := range blobs {
n++
if value == "" {
n--
break
}
if n > maxStatBlobs {
return nil, newRateLimitError(maxStatBlobs)
}
ref, ok := blob.Parse(value)
if !ok {
return nil, newHTTPError("Bogus blobref for value", http.StatusBadRequest)
}
needStat[ref] = true
}
toStat := make([]blob.Ref, 0, len(needStat))
for br := range needStat {
toStat = append(toStat, br)
}
log.Printf("Need to stat blob cnt: %d, got %d", len(needStat), len(blobs))
blobch := make(chan blob.SizedInfoRef)
errch := make(chan error, 1)
go func() {
err := storage.StatBlobs(blobch, toStat)
close(blobch)
errch <- err
}()
for sb := range blobch {
res.Stat = append(res.Stat, protocol.RefInfo{
Ref: sb.Ref,
Size: uint32(sb.Size),
MD5: sb.MD5,
})
delete(needStat, sb.Ref)
}
err := <-errch
if err != nil {
log.Errorf("Stat error: %v", err)
return nil, newHTTPError("Server Error", http.StatusInternalServerError)
}
return res, nil
}
func main() {
flag.Usage = usage
flag.Parse()
log.Println("start blobserver service …")
if *version {
fmt.Fprintln(os.Stdout, Version)
return
}
if *help {
flag.Usage()
os.Exit(1)
}
conf, err := config.ReadFile(*configFilename)
if err != nil {
log.Fatal(err)
}
if conf.Listen == "" && *laddr == "" {
log.Fatal("Listen address required")
} else if conf.Listen == "" {
conf.Listen = *laddr
}
runtime.GOMAXPROCS(runtime.NumCPU())
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
storage, err := blobserver.CreateStorage(conf)
if err != nil {
log.Fatalf("error instantiating storage for type %s: %v",
conf.StorageType(), err)
}
log.Printf("Using `%s` storage", conf.StorageType())
err = server.ListenAndServe(*laddr, storage)
if err != nil {
log.Errorln(err)
}
monitoring.MeasuringPointsPrintAll()
}
func sigTrapCloser(l net.Listener) {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)
go func() {
for _ = range c {
// Once we close the listener the main loop will exit
l.Close()
log.Printf("Closed listener %s", l.Addr())
}
}()
}
// ServeJSONCodeError sends a JSON error response to rw for the provided
// error value.
func ServeJSONCodeError(rw http.ResponseWriter, err interface{}, code int) {
if i, ok := err.(httpCoder); ok {
code = i.HTTPCode()
}
msg := fmt.Sprint(err)
log.Printf("Sending error %v to client for: %v", code, msg)
ReturnJSONCode(rw, code, map[string]interface{}{
"Error": map[string]string{
"Message": msg,
"Type": http.StatusText(code),
},
})
}
func TrapCloser(cl io.Closer, cleanups ...Cleanup) {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)
go func() {
for {
rawSig := <-c
sig, ok := rawSig.(syscall.Signal)
if !ok {
log.Fatal("not a unix signal")
}
switch {
case sig == syscall.SIGHUP:
log.Print("SIGHUP: restart process")
err := RestartProcess()
if err != nil {
log.Fatal("failed to restart: " + err.Error())
}
case sig == syscall.SIGINT || sig == syscall.SIGKILL || sig == syscall.SIGTERM:
log.Print("shutting down")
donec := make(chan bool)
go func() {
for _, cb := range cleanups {
if err := cb(); err != nil {
log.Error(err)
}
}
if err := cl.Close(); err != nil {
log.Fatalf("Error shutting down: %v", err)
}
donec <- true
}()
select {
case <-donec:
log.Printf("shutdown")
os.Exit(0)
case <-time.After(5 * time.Second):
log.Fatal("Timeout shutting down. Exiting uncleanly.")
}
default:
log.Fatal("Received another signal, should not happen.")
}
}
}()
}
func ListenAndServe(conf *config.Config) error {
setupServer(conf)
l, err := net.Listen("tcp", conf.Listen)
if err != nil {
return err
}
log.Printf("Listen on %s", l.Addr())
sig.TrapCloser(l)
err = http.Serve(l, nil)
log.Print("Shutting down ..")
return err
}
func TestPolicy(t *testing.T) {
ast := assert.NewAssert(t)
var wg sync.WaitGroup
l, err := net.Listen("tcp", ":9001")
defer l.Close()
ast.Nil(err)
log.Printf("Listen on %v", l.Addr())
wg.Add(1)
go func() {
serve(l)
wg.Done()
}()
for _, p := range policyTests {
conn, err := net.Dial("tcp", ":9001")
ast.Nil(err)
n, err := conn.Write(p.send)
if !p.failWrite {
ast.Nil(err)
ast.Equal(len(p.send), n)
} else {
ast.NotNil(err)
continue
}
buf := make([]byte, testBufSize)
n, err = conn.Read(buf)
if !p.failRead {
ast.Nil(err)
ast.Equal(len(p.exp), n)
ast.True(bytes.Equal(p.exp, buf[:n]))
} else {
ast.NotNil(err)
continue
}
}
l.Close()
wg.Wait()
}
func imageHandle(w http.ResponseWriter, r *http.Request, f ImageFilter) {
start := time.Now()
m := mux.Vars(r)
log.Println(m["fileinfo"])
fi, err := f.SizeParser(m["fileinfo"])
if err != nil {
writeError(w, err.Error(), 400)
return
}
log.Println(fi)
data, err := imageBackend.ReadFile(fi.filepath)
if err != nil {
writeError(w, err.Error(), 400)
return
}
mimeType := http.DetectContentType(data)
if err := validContentType(mimeType); err != nil {
writeError(w, err.Error(), 400)
return
}
thumb, err := f.Filter(data, fi)
if err != nil {
writeError(w, err.Error(), 400)
return
}
newMimeType := http.DetectContentType(thumb)
w.Header().Set("Content-Type", newMimeType)
w.Header().Set("Content-Length", strconv.Itoa(len(thumb)))
w.Write(thumb)
log.Printf("Image Handle OK %v", time.Since(start))
}
// checkInit will create all containers for shards
func (s *swiftStorage) checkInit() error {
if !s.shard {
return nil
}
w := runtime.NumCPU() << 3
cch := make(chan string)
sch := make(chan string)
ech := make(chan error)
qch := make(chan string, 32)
defer close(ech)
defer close(cch)
defer close(sch)
defer close(qch)
log.Printf("swift: check init containers. worker cnt %d\n", w)
for i := 0; i < w; i++ {
go createCont(ech, cch, sch, s)
go statCont(ech, sch, qch, s)
}
for _, shard := range shards {
go func(ch chan string, shard string) {
ch <- fmt.Sprintf("%s-%s", s.containerName, shard)
}(cch, shard)
}
for i := len(shards); i > 0; i-- {
select {
case <-qch:
case err := <-ech:
return err
}
}
return nil
}
func handleMultiPartUpload(req *http.Request, blobReceiver blobserver.Storage) (interface{}, error) {
res := new(protocol.UploadResponse)
receivedBlobs := make([]blob.SizedRef, 0, 4)
multipart, err := req.MultipartReader()
if err != nil {
return nil, newHTTPError(fmt.Sprintf("Expected multipart/form-data POST request; %v", err), http.StatusBadRequest)
}
useFilename := false
req.ParseForm()
if req.FormValue("use-filename") != "" {
useFilename = true
}
for {
mimePart, err := multipart.NextPart()
if err == io.EOF {
break
}
if err != nil {
return nil, newHTTPError(fmt.Sprintf("Error reading multipart section: %v", err), http.StatusBadRequest)
}
contentDisposition, _, err := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition"))
if err != nil {
return nil, newHTTPError("Invalid Content-Disposition", http.StatusBadRequest)
}
if contentDisposition != "form-data" {
return nil, newHTTPError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition), http.StatusBadRequest)
}
var ref blob.Ref
var tooBig int64 = blobserver.MaxBlobSize + 1
var readBytes int64
filename := mimePart.FileName()
log.Println("filename:", filename)
if useFilename {
log.Println("Use filename")
ref = blob.NewRefFilename(filename)
} else {
ref = blob.NewRef(filename)
}
blobGot, err := blobReceiver.ReceiveBlob(ref, &readerutil.CountingReader{
Reader: io.LimitReader(mimePart, tooBig),
N: &readBytes,
})
if readBytes == tooBig {
err = fmt.Errorf("blob over the limit of %d bytes", blobserver.MaxBlobSize)
}
if err != nil {
var errmsg string
if log.Severity >= log.LevelInfo {
errmsg = fmt.Sprintf("Error receiving blob (read bytes: %d) %v: %v\n", readBytes, ref, err)
} else {
errmsg = fmt.Sprintf("Error receiving blob: %v\n", err)
}
return nil, newHTTPError(errmsg, http.StatusInternalServerError)
}
log.Printf("Received blob %v\n", blobGot)
receivedBlobs = append(receivedBlobs, blobGot)
}
for _, got := range receivedBlobs {
rv := protocol.RefInfo{
Ref: got.Ref,
Size: uint32(got.Size),
}
if h := got.Hash(); h != nil {
rv.MD5 = hex.EncodeToString(h.Sum(nil))
}
res.Received = append(res.Received, rv)
}
return res, nil
}
func writeError(w http.ResponseWriter, err string, statusCode int) {
log.Printf("err: %v", err)
w.WriteHeader(statusCode)
w.Write([]byte(err))
}
func ForbiddenError(conn http.ResponseWriter, errorMessage string, args ...interface{}) {
conn.WriteHeader(http.StatusForbidden)
log.Printf("Forbidden: %s", fmt.Sprintf(errorMessage, args...))
fmt.Fprintf(conn, "<h1>Forbidden</h1>")
}
func BadRequestError(conn http.ResponseWriter, errorMessage string, args ...interface{}) {
conn.WriteHeader(http.StatusBadRequest)
log.Printf("Bad request: %s", fmt.Sprintf(errorMessage, args...))
fmt.Fprintf(conn, "<h1>Bad Request</h1>")
}
func ErrorRouting(conn http.ResponseWriter, req *http.Request) {
http.Error(conn, "Handlers wired up wrong; this path shouldn't be hit", 500)
log.Printf("Internal routing error on %q", req.URL.Path)
}