// errorIf synonymous with fatalIf but doesn't exit on error != nil
func errorIf(err *probe.Error, msg string) {
if err == nil {
return
}
if globalJSONFlag {
errorMessage := ErrorMessage{
Message: msg,
Type: "error",
Cause: err.ToGoError(),
SysInfo: err.SysInfo,
}
if globalDebugFlag {
errorMessage.CallTrace = err.CallTrace
}
json, err := json.Marshal(struct {
Error ErrorMessage `json:"error"`
}{
Error: errorMessage,
})
if err != nil {
console.Fatalln(probe.NewError(err))
}
console.Println(string(json))
return
}
if !globalDebugFlag {
console.Errorln(fmt.Sprintf("%s %s", msg, err.ToGoError()))
return
}
console.Errorln(fmt.Sprintf("%s %s", msg, err))
}
// fatalIf wrapper function which takes error and selectively prints stack frames if available on debug
func fatalIf(err *probe.Error, msg string) {
if err == nil {
return
}
if globalJSON {
errorMsg := errorMessage{
Message: msg,
Type: "fatal",
Cause: causeMessage{
Message: err.ToGoError().Error(),
Error: err.ToGoError(),
},
SysInfo: err.SysInfo,
}
if globalDebug {
errorMsg.CallTrace = err.CallTrace
}
json, err := json.Marshal(struct {
Status string `json:"status"`
Error errorMessage `json:"error"`
}{
Status: "error",
Error: errorMsg,
})
if err != nil {
console.Fatalln(probe.NewError(err))
}
console.Println(string(json))
console.Fatalln()
}
if !globalDebug {
console.Fatalln(fmt.Sprintf("%s %s", msg, err.ToGoError()))
}
console.Fatalln(fmt.Sprintf("%s %s", msg, err))
}
// New instantiate a new donut
func New(rootPath string, minFreeDisk int64) (Filesystem, *probe.Error) {
setFSBucketsMetadataPath(filepath.Join(rootPath, "$buckets.json"))
setFSMultipartsMetadataPath(filepath.Join(rootPath, "$multiparts-session.json"))
var err *probe.Error
// load multiparts session from disk
var multiparts *Multiparts
multiparts, err = loadMultipartsSession()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
multiparts = &Multiparts{
Version: "1",
ActiveSession: make(map[string]*MultipartSession),
}
if err := saveMultipartsSession(*multiparts); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
var buckets *Buckets
buckets, err = loadBucketsMetadata()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
buckets = &Buckets{
Version: "1",
Metadata: make(map[string]*BucketMetadata),
}
if err := saveBucketsMetadata(*buckets); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
fs := Filesystem{
rwLock: &sync.RWMutex{},
}
fs.path = rootPath
fs.multiparts = multiparts
fs.buckets = buckets
/// Defaults
// minium free disk required for i/o operations to succeed.
fs.minFreeDisk = minFreeDisk
// Start list goroutine.
if err = fs.listObjectsService(); err != nil {
return Filesystem{}, err.Trace(rootPath)
}
// Return here.
return fs, nil
}
func fatalIf(err *probe.Error, msg string, fields map[string]interface{}) {
if err == nil {
return
}
if fields == nil {
fields = make(map[string]interface{})
}
fields["error"] = err.ToGoError()
if jsonErr, e := json.Marshal(err); e == nil {
fields["probe"] = string(jsonErr)
}
log.WithFields(fields).Fatal(msg)
}
func (s rpcSignatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var signature *rpcSignature
if isRequestSignatureRPC(r) {
// Init signature V4 verification
var err *probe.Error
signature, err = initSignatureRPC(r)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, r, AuthorizationHeaderMalformed, r.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
}
buffer := new(bytes.Buffer)
if _, err := io.Copy(buffer, r.Body); err != nil {
errorIf(probe.NewError(err), "Unable to read payload from request body.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
value := sha256.Sum256(buffer.Bytes())
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(value[:]))
if err != nil {
errorIf(err.Trace(), "Unable to verify signature.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
if !ok {
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
// Copy the buffer back into request body to be read by the RPC service callers
r.Body = ioutil.NopCloser(buffer)
s.handler.ServeHTTP(w, r)
} else {
writeErrorResponse(w, r, AccessDenied, r.URL.Path)
}
}
// New instantiate a new donut
func New(rootPath string) (Filesystem, *probe.Error) {
setFSBucketsConfigPath(filepath.Join(rootPath, "$buckets.json"))
setFSMultipartsConfigPath(filepath.Join(rootPath, "$multiparts-session.json"))
var err *probe.Error
// load multiparts session from disk
var multiparts *Multiparts
multiparts, err = loadMultipartsSession()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
multiparts = &Multiparts{
Version: "1",
ActiveSession: make(map[string]*MultipartSession),
}
if err := saveMultipartsSession(multiparts); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
var buckets *Buckets
buckets, err = loadBucketsMetadata()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
buckets = &Buckets{
Version: "1",
Metadata: make(map[string]*BucketMetadata),
}
if err := saveBucketsMetadata(buckets); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
a := Filesystem{lock: new(sync.Mutex)}
a.path = rootPath
a.multiparts = multiparts
a.buckets = buckets
return a, nil
}
// New instantiate a new donut
func New() (Filesystem, *probe.Error) {
var err *probe.Error
// load multiparts session from disk
var multiparts *Multiparts
multiparts, err = loadMultipartsSession()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
multiparts = &Multiparts{
Version: "1",
ActiveSession: make(map[string]*MultipartSession),
}
if err := SaveMultipartsSession(multiparts); err != nil {
return Filesystem{}, err.Trace()
}
} else {
return Filesystem{}, err.Trace()
}
}
a := Filesystem{lock: new(sync.Mutex)}
a.multiparts = multiparts
return a, nil
}
// PutBucketHandler - PUT Bucket
// ----------
// This implementation of the PUT operation creates a new bucket for authenticated request
func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
bucket := vars["bucket"]
if !api.Anonymous {
if isRequestRequiresACLCheck(req) {
writeErrorResponse(w, req, AccessDenied, req.URL.Path)
return
}
}
// read from 'x-amz-acl'
aclType := getACLType(req)
if aclType == unsupportedACLType {
writeErrorResponse(w, req, NotImplemented, req.URL.Path)
return
}
var signature *fs.Signature
if !api.Anonymous {
// Init signature V4 verification
if isRequestSignatureV4(req) {
var err *probe.Error
signature, err = initSignatureV4(req)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, req, InternalError, req.URL.Path)
return
}
}
}
}
// if body of request is non-nil then check for validity of Content-Length
if req.Body != nil {
/// if Content-Length is unknown/missing, deny the request
if req.ContentLength == -1 {
writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
return
}
if signature != nil {
locationBytes, err := ioutil.ReadAll(req.Body)
if err != nil {
sh := sha256.New()
sh.Write(locationBytes)
ok, perr := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
if perr != nil {
errorIf(perr.Trace(), "MakeBucket failed.", nil)
writeErrorResponse(w, req, InternalError, req.URL.Path)
return
}
if !ok {
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
return
}
}
}
}
err := api.Filesystem.MakeBucket(bucket, getACLTypeString(aclType))
if err != nil {
errorIf(err.Trace(), "MakeBucket failed.", nil)
switch err.ToGoError().(type) {
case fs.BucketNameInvalid:
writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
case fs.BucketExists:
writeErrorResponse(w, req, BucketAlreadyExists, req.URL.Path)
default:
writeErrorResponse(w, req, InternalError, req.URL.Path)
}
return
}
// Make sure to add Location information here only for bucket
w.Header().Set("Location", "/"+bucket)
writeSuccessResponse(w, nil)
}
// CompleteMultipartUploadHandler - Complete multipart upload
func (api CloudStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
if !api.Anonymous {
if isRequestRequiresACLCheck(req) {
writeErrorResponse(w, req, AccessDenied, req.URL.Path)
return
}
}
objectResourcesMetadata := getObjectResources(req.URL.Query())
var signature *fs.Signature
if !api.Anonymous {
if isRequestSignatureV4(req) {
// Init signature V4 verification
var err *probe.Error
signature, err = initSignatureV4(req)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, req, InternalError, req.URL.Path)
return
}
}
}
}
metadata, err := api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, req.Body, signature)
if err != nil {
errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil)
switch err.ToGoError().(type) {
case fs.BucketNameInvalid:
writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
case fs.BucketNotFound:
writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
case fs.ObjectNotFound:
writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
case fs.ObjectNameInvalid:
writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
case fs.InvalidUploadID:
writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
case fs.InvalidPart:
writeErrorResponse(w, req, InvalidPart, req.URL.Path)
case fs.InvalidPartOrder:
writeErrorResponse(w, req, InvalidPartOrder, req.URL.Path)
case fs.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
case fs.MalformedXML:
writeErrorResponse(w, req, MalformedXML, req.URL.Path)
default:
writeErrorResponse(w, req, InternalError, req.URL.Path)
}
return
}
response := generateCompleteMultpartUploadResponse(bucket, object, req.URL.String(), metadata.Md5)
encodedSuccessResponse := encodeSuccessResponse(response)
// write headers
setCommonHeaders(w)
// write success response.
writeSuccessResponse(w, encodedSuccessResponse)
}
// PutObjectPartHandler - Upload part
func (api CloudStorageAPI) PutObjectPartHandler(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
if !api.Anonymous {
if isRequestRequiresACLCheck(req) {
writeErrorResponse(w, req, AccessDenied, req.URL.Path)
return
}
}
// get Content-MD5 sent by client and verify if valid
md5 := req.Header.Get("Content-MD5")
if !isValidMD5(md5) {
writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
return
}
/// if Content-Length is unknown/missing, throw away
size := req.ContentLength
if size == -1 {
writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
return
}
/// maximum Upload size for multipart objects in a single operation
if isMaxObjectSize(size) {
writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
return
}
uploadID := req.URL.Query().Get("uploadId")
partIDString := req.URL.Query().Get("partNumber")
var partID int
{
var err error
partID, err = strconv.Atoi(partIDString)
if err != nil {
writeErrorResponse(w, req, InvalidPart, req.URL.Path)
return
}
}
var signature *fs.Signature
if !api.Anonymous {
if isRequestSignatureV4(req) {
// Init signature V4 verification
var err *probe.Error
signature, err = initSignatureV4(req)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, req, InternalError, req.URL.Path)
return
}
}
}
}
calculatedMD5, err := api.Filesystem.CreateObjectPart(bucket, object, uploadID, md5, partID, size, req.Body, signature)
if err != nil {
errorIf(err.Trace(), "CreateObjectPart failed.", nil)
switch err.ToGoError().(type) {
case fs.RootPathFull:
writeErrorResponse(w, req, RootPathFull, req.URL.Path)
case fs.InvalidUploadID:
writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
case fs.BadDigest:
writeErrorResponse(w, req, BadDigest, req.URL.Path)
case fs.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
case fs.EntityTooLarge:
writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
case fs.InvalidDigest:
writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
default:
writeErrorResponse(w, req, InternalError, req.URL.Path)
}
return
}
w.Header().Set("ETag", "\""+calculatedMD5+"\"")
writeSuccessResponse(w, nil)
}
// PutObjectHandler - PUT Object
// ----------
// This implementation of the PUT operation adds an object to a bucket.
func (api CloudStorageAPI) PutObjectHandler(w http.ResponseWriter, req *http.Request) {
var object, bucket string
vars := mux.Vars(req)
bucket = vars["bucket"]
object = vars["object"]
if !api.Anonymous {
if isRequestRequiresACLCheck(req) {
if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
writeErrorResponse(w, req, AccessDenied, req.URL.Path)
return
}
}
}
// get Content-MD5 sent by client and verify if valid
md5 := req.Header.Get("Content-MD5")
if !isValidMD5(md5) {
writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
return
}
/// if Content-Length is unknown/missing, deny the request
size := req.ContentLength
if size == -1 {
writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
return
}
/// maximum Upload size for objects in a single operation
if isMaxObjectSize(size) {
writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
return
}
var signature *fs.Signature
if !api.Anonymous {
if isRequestSignatureV4(req) {
// Init signature V4 verification
var err *probe.Error
signature, err = initSignatureV4(req)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, req, InternalError, req.URL.Path)
return
}
}
}
}
metadata, err := api.Filesystem.CreateObject(bucket, object, md5, size, req.Body, signature)
if err != nil {
errorIf(err.Trace(), "CreateObject failed.", nil)
switch err.ToGoError().(type) {
case fs.RootPathFull:
writeErrorResponse(w, req, RootPathFull, req.URL.Path)
case fs.BucketNotFound:
writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
case fs.BucketNameInvalid:
writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
case fs.BadDigest:
writeErrorResponse(w, req, BadDigest, req.URL.Path)
case fs.MissingDateHeader:
writeErrorResponse(w, req, RequestTimeTooSkewed, req.URL.Path)
case fs.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
case fs.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
case fs.EntityTooLarge:
writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
case fs.InvalidDigest:
writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
default:
writeErrorResponse(w, req, InternalError, req.URL.Path)
}
return
}
w.Header().Set("ETag", "\""+metadata.Md5+"\"")
writeSuccessResponse(w, nil)
}
func (s signatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if isRequestPostPolicySignatureV4(r) && r.Method == "POST" {
s.handler.ServeHTTP(w, r)
return
}
var signature *signv4.Signature
if isRequestSignatureV4(r) {
// For PUT and POST requests with payload, send the call upwards for verification.
// Or PUT and POST requests without payload, verify here.
if (r.Body == nil && (r.Method == "PUT" || r.Method == "POST")) || (r.Method != "PUT" && r.Method != "POST") {
// Init signature V4 verification
var err *probe.Error
signature, err = initSignatureV4(r)
if err != nil {
switch err.ToGoError() {
case errInvalidRegion:
errorIf(err.Trace(), "Unknown region in authorization header.", nil)
writeErrorResponse(w, r, AuthorizationHeaderMalformed, r.URL.Path)
return
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id.", nil)
writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
}
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256.Sum256([]byte(""))))
if err != nil {
errorIf(err.Trace(), "Unable to verify signature.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
if !ok {
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
}
s.handler.ServeHTTP(w, r)
return
}
if isRequestPresignedSignatureV4(r) {
var err *probe.Error
signature, err = initPresignedSignatureV4(r)
if err != nil {
switch err.ToGoError() {
case errAccessKeyIDInvalid:
errorIf(err.Trace(), "Invalid access key id requested.", nil)
writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
return
default:
errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
}
ok, err := signature.DoesPresignedSignatureMatch()
if err != nil {
errorIf(err.Trace(), "Unable to verify signature.", nil)
writeErrorResponse(w, r, InternalError, r.URL.Path)
return
}
if !ok {
writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
return
}
s.handler.ServeHTTP(w, r)
return
}
writeErrorResponse(w, r, AccessDenied, r.URL.Path)
}