Example #1
0
// MultiUploader accepts a list of filepaths which are synced to a remote server
func (c *Client) MultiUploader(filepaths []string) (files Resources, err error) {
	files, err = c.multiStat(filepaths)

	if err != nil {
		return nil, err
	}

	toUpload := make([]string, 0, len(filepaths))

	for _, res := range files {
		if res.Created {
			toUpload = append(toUpload, res.Path)
		} else {
			log.Printf("%s exists - skipping", res.Path)
		}
	}

	if len(toUpload) == 0 {
		return files, nil
	}

	log.Printf("upload %d %v", len(toUpload), toUpload)
	err = c.multiUpload(files, toUpload)
	return files, err
}
Example #2
0
// LogHandler adds logging to http requests
func LogHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		log.Printf("%s %s", r.Method, r.URL.Path)
		h.ServeHTTP(w, r)
		//log.Println(r.Method, r.URL, r.Form, time.Since(start))
		log.Printf("%s %s in %v\n", r.Method, r.URL.Path, time.Since(start))
	})
}
Example #3
0
func ListenAndServe(laddr string) error {
	l, err := net.Listen("tcp", laddr)

	if err != nil {
		return err
	}

	log.Printf("Listen on %s", l.Addr())
	sig.TrapCloser(l)
	err = serve(l)
	log.Printf("Shutting down ..")
	return err
}
Example #4
0
func parseRequest(buf []byte) (resp []byte, err error) {
	switch {
	case bytes.Equal(protocolPolicy, buf):
		log.Printf("Policy request")
		resp = protocolPolicyResponse
	case bytes.Equal(protocolPing, buf):
		log.Printf("Ping request")
		resp = protocolPingResponse
	default:
		err = fmt.Errorf("Invalid protocol request")
	}

	return
}
Example #5
0
func blacklistGet(t *testing.T, count, offset int, out []string) {
	r, err := http.Get(fmt.Sprintf("http://%s/v1/profanity/blacklist/?lang=%s&count=%d&offset=%d", serverAddr, "en_US", count, offset))

	if err != nil {
		t.Fatalf("error getting: %s", err)
		return
	}

	var res blacklistResponse
	err = json.NewDecoder(r.Body).Decode(&res)

	if err != nil {
		t.Fatal(err)
	}

	log.Printf("res: %v", res)
	expLen := math.IntMin(math.IntMax(res.Total-offset, 0), count)

	if len(res.Blacklist) != expLen {
		t.Fatalf("%d != %d", len(res.Blacklist), expLen)
	}

	if out != nil {
		for i := 0; i < len(res.Blacklist); i++ {
			if res.Blacklist[i] != out[i] {
				t.Fatalf("%s != %s", res.Blacklist[i], out[i])
			}
		}
	}
}
Example #6
0
func handle(conn net.Conn) {
	defer conn.Close()

	err := conn.SetDeadline(time.Now().Add(Timeout))

	if err != nil {
		log.Errorf("Error setting deadline on conn")
		return
	}

	buf := getBuf(bufSize)
	defer putBuf(buf)

	n, err := conn.Read(buf)

	if err != nil {
		log.Error("Error reading from conn")
		return
	}

	log.Printf("Got %+q", buf[:n])
	resp, err := parseRequest(buf[:n])

	if err != nil {
		log.Error(err)
		return
	}

	n, err = conn.Write(resp)

	if err != nil || n != len(resp) {
		log.Errorf("Error writing to conn")
		return
	}
}
Example #7
0
func ListenAndServe(laddr string, storage blobserver.Storage) error {
	if err := setupServer(storage); err != nil {
		return err
	}

	l, err := net.Listen("tcp", laddr)

	if err != nil {
		return err
	}

	log.Printf("Listen on %s", l.Addr())

	sig.TrapCloser(l)
	err = http.Serve(l, nil)
	log.Printf("Shutting down ..")
	return err
}
Example #8
0
func runServer() {
	router := mux.NewRouter()
	router.HandleFunc("/", indexHandler).Methods("GET").Name("index")
	router.HandleFunc("/", creatorHandler).Methods("POST").Name("key-create")
	router.HandleFunc("/{key:[0-9A-Za-z-]+}/", deleteHandler).Methods("DELETE").Name("key-delete")
	router.HandleFunc("/select-region/", selectRegionHandler).Methods("POST").Name("select-region")
	log.Printf("Listen on %s\n", config.Laddr)
	http.ListenAndServe(config.Laddr, router)
}
Example #9
0
func ListenAndServe(laddr string, imgBackend backend.ImageBackend) error {
	if err := setupServer(imgBackend); err != nil {
		return err
	}

	l, err := net.Listen("tcp", laddr)

	if err != nil {
		return err
	}

	log.Printf("Listen on %s", l.Addr())

	sigTrapCloser(l)
	err = http.Serve(l, nil)
	log.Printf("Shutting down ..")
	return err
}
Example #10
0
func BenchmarkPolicy(b *testing.B) {
	b.StopTimer()
	var wg sync.WaitGroup
	l, err := net.Listen("tcp", ":9001")
	defer l.Close()

	if err != nil {
		b.Fatal(err)
	}

	log.Printf("Listen on %v", l.Addr())

	wg.Add(1)
	go func() {
		serve(l)
		wg.Done()
	}()

	b.StartTimer()
	k := 8 << 2
	for i := 0; i < b.N; i += k {
		var wg2 sync.WaitGroup
		for j := 0; j < k; j++ {
			wg2.Add(1)
			go func() {
				conn, err := net.Dial("tcp", ":9001")

				if err != nil {
					b.Errorf("error dialing: %v", err)
				}

				n, err := conn.Write(protocolPolicy)

				if err != nil || n != len(protocolPolicy) {
					b.Errorf("error sending: %v, %d == %d", err, n, len(protocolPolicy))
				}

				buf := make([]byte, testBufSize)
				n, err = conn.Read(buf)

				if err != nil || n != len(protocolPolicyResponse) {
					b.Errorf("error reading: %v, %d == %d", err, n, len(protocolPolicyResponse))
				}

				if !bytes.Equal(protocolPolicyResponse, buf[:n]) {
					b.Errorf("unexpected response value, exp: %+q, got %+q", protocolPolicyResponse, buf[:n])
				}
				wg2.Done()
			}()
		}
		wg2.Wait()
	}
	b.StopTimer()
	l.Close()
	wg.Wait()
	b.StartTimer()
}
Example #11
0
func handleStat(req *http.Request, storage blobserver.BlobStatter, blobs []string) (interface{}, error) {
	res := new(protocol.StatResponse)
	needStat := map[blob.Ref]bool{}
	n := 0

	for _, value := range blobs {
		n++
		if value == "" {
			n--
			break
		}
		if n > maxStatBlobs {
			return nil, newRateLimitError(maxStatBlobs)
		}
		ref, ok := blob.Parse(value)
		if !ok {
			return nil, newHTTPError("Bogus blobref for value", http.StatusBadRequest)
		}
		needStat[ref] = true
	}

	toStat := make([]blob.Ref, 0, len(needStat))

	for br := range needStat {
		toStat = append(toStat, br)
	}

	log.Printf("Need to stat blob cnt: %d, got %d", len(needStat), len(blobs))
	blobch := make(chan blob.SizedInfoRef)
	errch := make(chan error, 1)

	go func() {
		err := storage.StatBlobs(blobch, toStat)
		close(blobch)
		errch <- err
	}()

	for sb := range blobch {
		res.Stat = append(res.Stat, protocol.RefInfo{
			Ref:  sb.Ref,
			Size: uint32(sb.Size),
			MD5:  sb.MD5,
		})
		delete(needStat, sb.Ref)
	}

	err := <-errch

	if err != nil {
		log.Errorf("Stat error: %v", err)
		return nil, newHTTPError("Server Error", http.StatusInternalServerError)
	}

	return res, nil
}
Example #12
0
func main() {
	flag.Usage = usage
	flag.Parse()
	log.Println("start blobserver service …")

	if *version {
		fmt.Fprintln(os.Stdout, Version)
		return
	}

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	conf, err := config.ReadFile(*configFilename)

	if err != nil {
		log.Fatal(err)
	}

	if conf.Listen == "" && *laddr == "" {
		log.Fatal("Listen address required")
	} else if conf.Listen == "" {
		conf.Listen = *laddr
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	storage, err := blobserver.CreateStorage(conf)

	if err != nil {
		log.Fatalf("error instantiating storage for type %s: %v",
			conf.StorageType(), err)
	}

	log.Printf("Using `%s` storage", conf.StorageType())
	err = server.ListenAndServe(*laddr, storage)

	if err != nil {
		log.Errorln(err)
	}

	monitoring.MeasuringPointsPrintAll()
}
Example #13
0
func sigTrapCloser(l net.Listener) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)

	go func() {
		for _ = range c {
			// Once we close the listener the main loop will exit
			l.Close()
			log.Printf("Closed listener %s", l.Addr())
		}
	}()
}
Example #14
0
// ServeJSONCodeError sends a JSON error response to rw for the provided
// error value.
func ServeJSONCodeError(rw http.ResponseWriter, err interface{}, code int) {
	if i, ok := err.(httpCoder); ok {
		code = i.HTTPCode()
	}
	msg := fmt.Sprint(err)
	log.Printf("Sending error %v to client for: %v", code, msg)
	ReturnJSONCode(rw, code, map[string]interface{}{
		"Error": map[string]string{
			"Message": msg,
			"Type":    http.StatusText(code),
		},
	})
}
Example #15
0
func TrapCloser(cl io.Closer, cleanups ...Cleanup) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)

	go func() {
		for {
			rawSig := <-c
			sig, ok := rawSig.(syscall.Signal)

			if !ok {
				log.Fatal("not a unix signal")
			}

			switch {
			case sig == syscall.SIGHUP:
				log.Print("SIGHUP: restart process")
				err := RestartProcess()

				if err != nil {
					log.Fatal("failed to restart: " + err.Error())
				}
			case sig == syscall.SIGINT || sig == syscall.SIGKILL || sig == syscall.SIGTERM:
				log.Print("shutting down")
				donec := make(chan bool)
				go func() {
					for _, cb := range cleanups {
						if err := cb(); err != nil {
							log.Error(err)
						}
					}

					if err := cl.Close(); err != nil {
						log.Fatalf("Error shutting down: %v", err)
					}

					donec <- true
				}()
				select {
				case <-donec:
					log.Printf("shutdown")
					os.Exit(0)
				case <-time.After(5 * time.Second):
					log.Fatal("Timeout shutting down. Exiting uncleanly.")
				}
			default:
				log.Fatal("Received another signal, should not happen.")
			}
		}
	}()
}
Example #16
0
func ListenAndServe(conf *config.Config) error {
	setupServer(conf)

	l, err := net.Listen("tcp", conf.Listen)

	if err != nil {
		return err
	}

	log.Printf("Listen on %s", l.Addr())

	sig.TrapCloser(l)
	err = http.Serve(l, nil)
	log.Print("Shutting down ..")
	return err
}
Example #17
0
func TestPolicy(t *testing.T) {
	ast := assert.NewAssert(t)

	var wg sync.WaitGroup
	l, err := net.Listen("tcp", ":9001")
	defer l.Close()

	ast.Nil(err)
	log.Printf("Listen on %v", l.Addr())

	wg.Add(1)
	go func() {
		serve(l)
		wg.Done()
	}()

	for _, p := range policyTests {
		conn, err := net.Dial("tcp", ":9001")
		ast.Nil(err)

		n, err := conn.Write(p.send)

		if !p.failWrite {
			ast.Nil(err)
			ast.Equal(len(p.send), n)
		} else {
			ast.NotNil(err)
			continue
		}

		buf := make([]byte, testBufSize)
		n, err = conn.Read(buf)

		if !p.failRead {
			ast.Nil(err)
			ast.Equal(len(p.exp), n)
			ast.True(bytes.Equal(p.exp, buf[:n]))
		} else {
			ast.NotNil(err)
			continue
		}
	}
	l.Close()
	wg.Wait()
}
Example #18
0
func imageHandle(w http.ResponseWriter, r *http.Request, f ImageFilter) {
	start := time.Now()
	m := mux.Vars(r)
	log.Println(m["fileinfo"])

	fi, err := f.SizeParser(m["fileinfo"])

	if err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	log.Println(fi)

	data, err := imageBackend.ReadFile(fi.filepath)

	if err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	mimeType := http.DetectContentType(data)

	if err := validContentType(mimeType); err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	thumb, err := f.Filter(data, fi)

	if err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	newMimeType := http.DetectContentType(thumb)
	w.Header().Set("Content-Type", newMimeType)
	w.Header().Set("Content-Length", strconv.Itoa(len(thumb)))
	w.Write(thumb)
	log.Printf("Image Handle OK %v", time.Since(start))
}
Example #19
0
// checkInit will create all containers for shards
func (s *swiftStorage) checkInit() error {
	if !s.shard {
		return nil
	}

	w := runtime.NumCPU() << 3

	cch := make(chan string)
	sch := make(chan string)
	ech := make(chan error)
	qch := make(chan string, 32)

	defer close(ech)
	defer close(cch)
	defer close(sch)
	defer close(qch)

	log.Printf("swift: check init containers. worker cnt %d\n", w)

	for i := 0; i < w; i++ {
		go createCont(ech, cch, sch, s)
		go statCont(ech, sch, qch, s)
	}

	for _, shard := range shards {
		go func(ch chan string, shard string) {
			ch <- fmt.Sprintf("%s-%s", s.containerName, shard)
		}(cch, shard)
	}

	for i := len(shards); i > 0; i-- {
		select {
		case <-qch:
		case err := <-ech:
			return err
		}
	}

	return nil
}
Example #20
0
func handleMultiPartUpload(req *http.Request, blobReceiver blobserver.Storage) (interface{}, error) {
	res := new(protocol.UploadResponse)
	receivedBlobs := make([]blob.SizedRef, 0, 4)
	multipart, err := req.MultipartReader()

	if err != nil {
		return nil, newHTTPError(fmt.Sprintf("Expected multipart/form-data POST request; %v", err), http.StatusBadRequest)
	}

	useFilename := false
	req.ParseForm()

	if req.FormValue("use-filename") != "" {
		useFilename = true
	}

	for {
		mimePart, err := multipart.NextPart()

		if err == io.EOF {
			break
		}

		if err != nil {
			return nil, newHTTPError(fmt.Sprintf("Error reading multipart section: %v", err), http.StatusBadRequest)
		}

		contentDisposition, _, err := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition"))

		if err != nil {
			return nil, newHTTPError("Invalid Content-Disposition", http.StatusBadRequest)
		}

		if contentDisposition != "form-data" {
			return nil, newHTTPError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition), http.StatusBadRequest)
		}

		var ref blob.Ref
		var tooBig int64 = blobserver.MaxBlobSize + 1
		var readBytes int64

		filename := mimePart.FileName()
		log.Println("filename:", filename)

		if useFilename {
			log.Println("Use filename")
			ref = blob.NewRefFilename(filename)
		} else {
			ref = blob.NewRef(filename)
		}

		blobGot, err := blobReceiver.ReceiveBlob(ref, &readerutil.CountingReader{
			Reader: io.LimitReader(mimePart, tooBig),
			N:      &readBytes,
		})

		if readBytes == tooBig {
			err = fmt.Errorf("blob over the limit of %d bytes", blobserver.MaxBlobSize)
		}

		if err != nil {
			var errmsg string

			if log.Severity >= log.LevelInfo {
				errmsg = fmt.Sprintf("Error receiving blob (read bytes: %d) %v: %v\n", readBytes, ref, err)
			} else {
				errmsg = fmt.Sprintf("Error receiving blob: %v\n", err)
			}

			return nil, newHTTPError(errmsg, http.StatusInternalServerError)
		}

		log.Printf("Received blob %v\n", blobGot)
		receivedBlobs = append(receivedBlobs, blobGot)
	}

	for _, got := range receivedBlobs {
		rv := protocol.RefInfo{
			Ref:  got.Ref,
			Size: uint32(got.Size),
		}
		if h := got.Hash(); h != nil {
			rv.MD5 = hex.EncodeToString(h.Sum(nil))
		}
		res.Received = append(res.Received, rv)
	}
	return res, nil
}
Example #21
0
func writeError(w http.ResponseWriter, err string, statusCode int) {
	log.Printf("err: %v", err)
	w.WriteHeader(statusCode)
	w.Write([]byte(err))
}
Example #22
0
func ForbiddenError(conn http.ResponseWriter, errorMessage string, args ...interface{}) {
	conn.WriteHeader(http.StatusForbidden)
	log.Printf("Forbidden: %s", fmt.Sprintf(errorMessage, args...))
	fmt.Fprintf(conn, "<h1>Forbidden</h1>")
}
Example #23
0
func BadRequestError(conn http.ResponseWriter, errorMessage string, args ...interface{}) {
	conn.WriteHeader(http.StatusBadRequest)
	log.Printf("Bad request: %s", fmt.Sprintf(errorMessage, args...))
	fmt.Fprintf(conn, "<h1>Bad Request</h1>")
}
Example #24
0
func ErrorRouting(conn http.ResponseWriter, req *http.Request) {
	http.Error(conn, "Handlers wired up wrong; this path shouldn't be hit", 500)
	log.Printf("Internal routing error on %q", req.URL.Path)
}