Beispiel #1
0
func (sto *swiftStorage) StatBlobs(dest chan<- blob.SizedInfoRef, blobs []blob.Ref) error {
	var wg syncutil.Group

	for _, br := range blobs {
		br := sto.createPathRef(br)
		statGate.Start()
		wg.Go(func() error {
			defer statGate.Done()
			ref, cont := sto.refContainer(br)
			log.Println("REF:", ref, cont)
			info, _, err := sto.conn.Object(cont, ref)
			log.Println("Stat:", info, err, ref, br.Path)

			if err == nil {
				dest <- blob.SizedInfoRef{
					Ref:  br,
					Size: uint32(info.Bytes),
					MD5:  info.Hash,
				}
				return nil
			}
			if err == swift.ObjectNotFound {
				return nil
			}
			return fmt.Errorf("error statting %v: %v", br, err)
		})
	}
	return wg.Err()
}
Beispiel #2
0
func (sto *swiftStorage) ReceiveBlob(b blob.Ref, source io.Reader) (sr blob.SizedRef, err error) {
	slurper := newSwiftSlurper(b)
	defer slurper.Cleanup()

	size, err := io.Copy(slurper, source)

	if err != nil {
		return sr, err
	}

	hash := hex.EncodeToString(slurper.md5.Sum(nil))
	name, cont := sto.refContainer(b)
	retries := 1
retry:
	_, err = sto.conn.ObjectPut(cont, name, slurper, false, hash, "", nil)

	if err != nil {
		// assume both of these mean container not found in this context. Create the container first
		if retries > 0 && (err == swift.ObjectNotFound || err == swift.ContainerNotFound) {
			retries--

			if err = sto.createContainer(cont); err != nil {
				return sr, err
			}

			slurper.Seek(0, 0)
			goto retry
		}
		return sr, err
	}
	ref := sto.createPathRef(b)
	ref.SetHash(slurper.md5)
	log.Println("Create: ", ref)
	return blob.SizedRef{Ref: ref, Size: uint32(size)}, nil
}
Beispiel #3
0
func main() {
	flag.Parse()
	log.Println("start uploader")

	if flag.NArg() == 0 {
		log.Fatal("Expected file argument missing")
	}

	c, err := client.New(*serverAddr)

	if err != nil {
		log.Fatal(err)
	}

	res, _ := c.MultiUploader(flag.Args())
	var enc client.ResourceEncoder

	switch *outputType {
	case "json":
		enc = client.NewJSONEncoder(os.Stdout)
	default:
		os.Exit(0)
	}

	err = enc.Encode(res)

	if err != nil {
		log.Fatal(err)
	}
}
Beispiel #4
0
func main() {
	flag.Usage = usage
	flag.Parse()
	log.Println("start policy service …")

	if *version {
		fmt.Fprintln(os.Stdout, Version)
		return
	}

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	err := policy.ListenAndServe(*laddr)

	if err != nil {
		log.Errorln(err)
	}

	monitoring.MeasuringPointsPrintAll()
}
Beispiel #5
0
func main() {
	flag.Usage = usage
	flag.Parse()
	log.Println("start uploader")

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	ctx, err := newCtx(*serverAddr, flag.Args())

	if err != nil {
		log.Fatal(err)
	}

	ctx.search()
	err = ctx.upload()

	if err != nil {
		log.Fatal(err)
	}

	ctx.replace()
	ctx.WriteTo(os.Stdout)
}
Beispiel #6
0
func main() {
	flag.Usage = usage
	flag.Parse()
	log.Println("start blobserver service …")

	if *version {
		fmt.Fprintln(os.Stdout, Version)
		return
	}

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	conf, err := config.ReadFile(*configFilename)

	if err != nil {
		log.Fatal(err)
	}

	if conf.Listen == "" && *laddr == "" {
		log.Fatal("Listen address required")
	} else if conf.Listen == "" {
		conf.Listen = *laddr
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	storage, err := blobserver.CreateStorage(conf)

	if err != nil {
		log.Fatalf("error instantiating storage for type %s: %v",
			conf.StorageType(), err)
	}

	log.Printf("Using `%s` storage", conf.StorageType())
	err = server.ListenAndServe(*laddr, storage)

	if err != nil {
		log.Errorln(err)
	}

	monitoring.MeasuringPointsPrintAll()
}
Beispiel #7
0
func imageHandle(w http.ResponseWriter, r *http.Request, f ImageFilter) {
	start := time.Now()
	m := mux.Vars(r)
	log.Println(m["fileinfo"])

	fi, err := f.SizeParser(m["fileinfo"])

	if err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	log.Println(fi)

	data, err := imageBackend.ReadFile(fi.filepath)

	if err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	mimeType := http.DetectContentType(data)

	if err := validContentType(mimeType); err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	thumb, err := f.Filter(data, fi)

	if err != nil {
		writeError(w, err.Error(), 400)
		return
	}

	newMimeType := http.DetectContentType(thumb)
	w.Header().Set("Content-Type", newMimeType)
	w.Header().Set("Content-Length", strconv.Itoa(len(thumb)))
	w.Write(thumb)
	log.Printf("Image Handle OK %v", time.Since(start))
}
Beispiel #8
0
func (a *authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := a.loadSession(r); err != nil {
		log.Println("handler error: ", err)

		if a.mustAuth {
			log.Println("Status Unauthorized")
			w.WriteHeader(http.StatusUnauthorized)
			return
		}
	}
	a.handler.ServeHTTP(w, r)
	//rec := httptest.NewRecorder()
	//a.handler.ServeHTTP(rec, r)

	//for k, v := range rec.Header() {
	//	w.Header()[k] = v
	//}

	//w.Header().Set("X-Authenticated-By", "kogama")
	//w.WriteHeader(rec.Code)
	//rec.Body.WriteTo(w)
}
Beispiel #9
0
func (sto *swiftStorage) Fetch(br blob.Ref) (file io.ReadCloser, size uint32, err error) {
	ref, cont := sto.refContainer(br)
	log.Println("Fetch: ", ref, cont)
	f, h, err := sto.conn.ObjectOpen(cont, ref, true, nil)
	if err != nil {
		return
	}
	n, err := getInt64FromHeader(h, "Content-Length")
	if err != nil {
		return
	}
	return f, uint32(n), err
}
Beispiel #10
0
func main() {
	flag.Parse()
	log.Println("start blobperf")

	queue := make(chan int)

	for i := 0; i < *workers; i++ {
		go uploader(queue)
	}

	lastPrint = time.Now()
	for {
		queue <- 1
	}
}
Beispiel #11
0
func (sto *swiftStorage) RemoveBlobs(blobs []blob.Ref) error {
	var wg syncutil.Group

	for _, br := range blobs {
		br := br
		removeGate.Start()
		wg.Go(func() error {
			defer removeGate.Done()
			ref, cont := sto.refContainer(br)
			log.Println("Remove: ", cont, ref)
			return sto.conn.ObjectDelete(cont, ref)
		})
	}
	return wg.Err()
}
Beispiel #12
0
func main() {
	flag.Usage = usage
	flag.Parse()

	if *version {
		fmt.Fprintln(os.Stdout, Version)
		return
	}

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	if *laddr == "" {
		fmt.Fprintln(os.Stderr, "listen address required")
		os.Exit(1)
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var imgBackend backend.ImageBackend

	if *fsBaseDir != "" {
		imgBackend = backend.Dir(*fsBaseDir)
	} else if *awsAccessKeyId != "" && *awsSecretAccessKey != "" && *awsRegion != "" && *awsBucket != "" {
		imgBackend = backend.NewS3(*awsAccessKeyId, *awsSecretAccessKey, *awsRegion, *awsBucket)
	} else {
		log.Errorln("Expected either aws-* or fs-* arguments")
		os.Exit(1)
	}

	err := server.ListenAndServe(*laddr, imgBackend)

	if err != nil {
		log.Println(err)
	}
}
Beispiel #13
0
func currentRegion(r *http.Request) *Region {
	cookie, err := r.Cookie("region")

	if err != nil {
		log.Println(err)
	} else {
		codename := cookie.Value

		for k, v := range config.Regions {
			if codename == k {
				b := *v
				return &b
			}
		}
	}

	for _, v := range config.Regions {
		b := *v
		return &b
	}

	panic("no regions")
}
Beispiel #14
0
func (c *Client) multiUpload(results Resources, toUpload []string) error {
	req, err := c.multiMultipartRequest("/blob/upload/", toUpload)

	if err != nil {
		return err
	}

	res, err := http.DefaultClient.Do(req)

	if err != nil {
		return err
	}

	if res.StatusCode != 201 {
		return fmt.Errorf("Unexpected status code %d", res.StatusCode)
	}

	ur := new(protocol.UploadResponse)
	parseResponse(res, ur)

	if len(ur.Received) != len(toUpload) {
		return fmt.Errorf("Expected %d received got %d", len(toUpload), len(ur.Received))
	}

	for _, rec := range ur.Received {
		log.Println("got:", rec.Path)
		cur := results.findByPath(rec.Path)

		if cur == nil {
			return fmt.Errorf("upload error %s", rec.Path)
		}

		cur.URL = c.CDNBaseURL + rec.Path
	}

	return nil
}
Beispiel #15
0
func handleMultiPartUpload(req *http.Request, blobReceiver blobserver.Storage) (interface{}, error) {
	res := new(protocol.UploadResponse)
	receivedBlobs := make([]blob.SizedRef, 0, 4)
	multipart, err := req.MultipartReader()

	if err != nil {
		return nil, newHTTPError(fmt.Sprintf("Expected multipart/form-data POST request; %v", err), http.StatusBadRequest)
	}

	useFilename := false
	req.ParseForm()

	if req.FormValue("use-filename") != "" {
		useFilename = true
	}

	for {
		mimePart, err := multipart.NextPart()

		if err == io.EOF {
			break
		}

		if err != nil {
			return nil, newHTTPError(fmt.Sprintf("Error reading multipart section: %v", err), http.StatusBadRequest)
		}

		contentDisposition, _, err := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition"))

		if err != nil {
			return nil, newHTTPError("Invalid Content-Disposition", http.StatusBadRequest)
		}

		if contentDisposition != "form-data" {
			return nil, newHTTPError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition), http.StatusBadRequest)
		}

		var ref blob.Ref
		var tooBig int64 = blobserver.MaxBlobSize + 1
		var readBytes int64

		filename := mimePart.FileName()
		log.Println("filename:", filename)

		if useFilename {
			log.Println("Use filename")
			ref = blob.NewRefFilename(filename)
		} else {
			ref = blob.NewRef(filename)
		}

		blobGot, err := blobReceiver.ReceiveBlob(ref, &readerutil.CountingReader{
			Reader: io.LimitReader(mimePart, tooBig),
			N:      &readBytes,
		})

		if readBytes == tooBig {
			err = fmt.Errorf("blob over the limit of %d bytes", blobserver.MaxBlobSize)
		}

		if err != nil {
			var errmsg string

			if log.Severity >= log.LevelInfo {
				errmsg = fmt.Sprintf("Error receiving blob (read bytes: %d) %v: %v\n", readBytes, ref, err)
			} else {
				errmsg = fmt.Sprintf("Error receiving blob: %v\n", err)
			}

			return nil, newHTTPError(errmsg, http.StatusInternalServerError)
		}

		log.Printf("Received blob %v\n", blobGot)
		receivedBlobs = append(receivedBlobs, blobGot)
	}

	for _, got := range receivedBlobs {
		rv := protocol.RefInfo{
			Ref:  got.Ref,
			Size: uint32(got.Size),
		}
		if h := got.Hash(); h != nil {
			rv.MD5 = hex.EncodeToString(h.Sum(nil))
		}
		res.Received = append(res.Received, rv)
	}
	return res, nil
}
Beispiel #16
0
func handleConfig(w http.ResponseWriter, req *http.Request, storage blobserver.StorageConfiger) {
	res := new(protocol.ConfigResponse)
	res.Data = storage.Config()
	log.Println("config:", res)
	httputil.ReturnJSON(w, res)
}
Beispiel #17
0
// MeasureHandler adds measuring to http requests
func MeasureHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("MeasureHandler is deprecated")
		h.ServeHTTP(w, r)
	})
}