Example #1
0
// mainCopy is bound to sub-command
func mainCopy(ctx *cli.Context) {
	checkCopySyntax(ctx)

	console.SetCustomTheme(map[string]*color.Color{
		"Copy": color.New(color.FgGreen, color.Bold),
	})

	session := newSessionV2()

	var e error
	session.Header.CommandType = "cp"
	session.Header.RootPath, e = os.Getwd()
	if e != nil {
		session.Delete()
		fatalIf(probe.NewError(e), "Unable to get current working folder.")
	}

	// extract URLs.
	var err *probe.Error
	session.Header.CommandArgs, err = args2URLs(ctx.Args())
	if err != nil {
		session.Delete()
		fatalIf(err.Trace(), "One or more unknown URL types passed.")
	}

	doCopyCmdSession(session)
	session.Delete()
}
Example #2
0
func mainMirror(ctx *cli.Context) {
	checkMirrorSyntax(ctx)

	console.SetCustomTheme(map[string]*color.Color{
		"Mirror": color.New(color.FgGreen, color.Bold),
	})

	var e error
	session := newSessionV2()
	session.Header.CommandType = "mirror"
	session.Header.RootPath, e = os.Getwd()
	if e != nil {
		session.Close()
		session.Delete()
		fatalIf(probe.NewError(e), "Unable to get current working folder.")
	}

	// extract URLs.
	var err *probe.Error
	session.Header.CommandArgs, err = args2URLs(ctx.Args())
	if err != nil {
		session.Close()
		session.Delete()
		fatalIf(err.Trace(ctx.Args()...), fmt.Sprintf("One or more unknown argument types found in ā€˜%sā€™.", ctx.Args()))
	}

	doMirrorCmdSession(session)
	session.Close()
	session.Delete()
}
Example #3
0
// errorIf synonymous with fatalIf but doesn't exit on error != nil
func errorIf(err *probe.Error, msg string) {
	if err == nil {
		return
	}
	if globalJSONFlag {
		errorMessage := ErrorMessage{
			Message: msg,
			Type:    "error",
			Cause:   err.ToGoError(),
			SysInfo: err.SysInfo,
		}
		if globalDebugFlag {
			errorMessage.CallTrace = err.CallTrace
		}
		json, err := json.Marshal(struct {
			Error ErrorMessage `json:"error"`
		}{
			Error: errorMessage,
		})
		if err != nil {
			console.Fatalln(probe.NewError(err))
		}
		console.Println(string(json))
		return
	}
	if !globalDebugFlag {
		console.Errorln(fmt.Sprintf("%s %s", msg, err.ToGoError()))
		return
	}
	console.Errorln(fmt.Sprintf("%s %s", msg, err))
}
Example #4
0
// mainCopy is bound to sub-command
func mainCopy(ctx *cli.Context) {
	checkCopySyntax(ctx)

	setCopyPalette(ctx.GlobalString("colors"))

	session := newSessionV2()

	var e error
	session.Header.CommandType = "cp"
	session.Header.RootPath, e = os.Getwd()
	if e != nil {
		session.Delete()
		fatalIf(probe.NewError(e), "Unable to get current working folder.")
	}

	// extract URLs.
	var err *probe.Error
	session.Header.CommandArgs, err = args2URLs(ctx.Args())
	if err != nil {
		session.Delete()
		fatalIf(err.Trace(), "One or more unknown URL types passed.")
	}

	doCopySession(session)
	session.Delete()
}
Example #5
0
// getBucketMetadataReaders - readers are returned in map rather than slice
func (xl API) getBucketMetadataReaders() (map[int]io.ReadCloser, *probe.Error) {
	readers := make(map[int]io.ReadCloser)
	disks := make(map[int]block.Block)
	var err *probe.Error
	for _, node := range xl.nodes {
		nDisks := make(map[int]block.Block)
		nDisks, err = node.ListDisks()
		if err != nil {
			return nil, err.Trace()
		}
		for k, v := range nDisks {
			disks[k] = v
		}
	}
	var bucketMetaDataReader io.ReadCloser
	for order, disk := range disks {
		bucketMetaDataReader, err = disk.Open(filepath.Join(xl.config.XLName, bucketMetadataConfig))
		if err != nil {
			continue
		}
		readers[order] = bucketMetaDataReader
	}
	if err != nil {
		return nil, err.Trace()
	}
	return readers, nil
}
Example #6
0
// fatalIf wrapper function which takes error and selectively prints stack frames if available on debug
func fatalIf(err *probe.Error, msg string) {
	if err == nil {
		return
	}
	if globalJSON {
		errorMsg := errorMessage{
			Message: msg,
			Type:    "fatal",
			Cause: causeMessage{
				Message: err.ToGoError().Error(),
				Error:   err.ToGoError(),
			},
			SysInfo: err.SysInfo,
		}
		if globalDebug {
			errorMsg.CallTrace = err.CallTrace
		}
		json, e := json.Marshal(struct {
			Status string       `json:"status"`
			Error  errorMessage `json:"error"`
		}{
			Status: "error",
			Error:  errorMsg,
		})
		if e != nil {
			console.Fatalln(probe.NewError(e))
		}
		console.Println(string(json))
		console.Fatalln()
	}
	if !globalDebug {
		console.Fatalln(fmt.Sprintf("%s %s", msg, err.ToGoError()))
	}
	console.Fatalln(fmt.Sprintf("%s %s", msg, err))
}
Example #7
0
func mainMirror(ctx *cli.Context) {
	checkMirrorSyntax(ctx)

	setMirrorPalette(ctx.GlobalString("colors"))

	var e error
	session := newSessionV2()
	session.Header.CommandType = "mirror"
	session.Header.RootPath, e = os.Getwd()
	if e != nil {
		session.Delete()
		fatalIf(probe.NewError(e), "Unable to get current working folder.")
	}

	// extract URLs.
	var err *probe.Error
	session.Header.CommandArgs, err = args2URLs(ctx.Args())
	if err != nil {
		session.Delete()
		fatalIf(err.Trace(ctx.Args()...), fmt.Sprintf("One or more unknown argument types found in ā€˜%sā€™.", ctx.Args()))
	}

	doMirrorSession(session)
	session.Delete()
}
Example #8
0
// getObjectReaders -
func (b bucket) getObjectReaders(objectName, objectMeta string) (map[int]io.ReadCloser, *probe.Error) {
	readers := make(map[int]io.ReadCloser)
	var disks map[int]disk.Disk
	var err *probe.Error
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err = node.ListDisks()
		if err != nil {
			return nil, err.Trace()
		}
		for order, disk := range disks {
			var objectSlice io.ReadCloser
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, order)
			objectPath := filepath.Join(b.donutName, bucketSlice, objectName, objectMeta)
			objectSlice, err = disk.Open(objectPath)
			if err == nil {
				readers[order] = objectSlice
			}
		}
		nodeSlice = nodeSlice + 1
	}
	if err != nil {
		return nil, err.Trace()
	}
	return readers, nil
}
// CompleteMultipartUploadHandler - Complete multipart upload
func (api API) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
	// Ticket master block
	{
		op := APIOperation{}
		op.ProceedCh = make(chan struct{})
		api.OP <- op
		// block until Ticket master gives us a go
		<-op.ProceedCh
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	objectResourcesMetadata := getObjectResources(req.URL.Query())

	var signature *signv4.Signature
	if !api.Anonymous {
		if _, ok := req.Header["Authorization"]; ok {
			// Init signature V4 verification
			var err *probe.Error
			signature, err = initSignatureV4(req)
			if err != nil {
				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
				writeErrorResponse(w, req, InternalError, req.URL.Path)
				return
			}
		}
	}

	metadata, err := api.Donut.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, req.Body, signature)
	if err != nil {
		errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil)
		switch err.ToGoError().(type) {
		case donut.InvalidUploadID:
			writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
		case donut.InvalidPart:
			writeErrorResponse(w, req, InvalidPart, req.URL.Path)
		case donut.InvalidPartOrder:
			writeErrorResponse(w, req, InvalidPartOrder, req.URL.Path)
		case signv4.MissingDateHeader:
			writeErrorResponse(w, req, RequestTimeTooSkewed, req.URL.Path)
		case signv4.DoesNotMatch:
			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
		case donut.IncompleteBody:
			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
		case donut.MalformedXML:
			writeErrorResponse(w, req, MalformedXML, req.URL.Path)
		default:
			writeErrorResponse(w, req, InternalError, req.URL.Path)
		}
		return
	}
	response := generateCompleteMultpartUploadResponse(bucket, object, "", metadata.MD5Sum)
	encodedSuccessResponse := encodeSuccessResponse(response)
	// write headers
	setCommonHeaders(w, len(encodedSuccessResponse))
	// write body
	w.Write(encodedSuccessResponse)
}
// NewMultipartUploadHandler - New multipart upload
func (api MinioAPI) NewMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
	// Ticket master block
	{
		op := APIOperation{}
		op.ProceedCh = make(chan struct{})
		api.OP <- op
		// block until Ticket master gives us a go
		<-op.ProceedCh
	}

	acceptsContentType := getContentType(req)
	if !api.isValidOp(w, req, acceptsContentType) {
		return
	}

	if !isRequestUploads(req.URL.Query()) {
		writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path)
		return
	}

	var object, bucket string
	vars := mux.Vars(req)
	bucket = vars["bucket"]
	object = vars["object"]

	var signature *donut.Signature
	if _, ok := req.Header["Authorization"]; ok {
		// Init signature V4 verification
		var err *probe.Error
		signature, err = initSignatureV4(req)
		if err != nil {
			errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
			writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
			return
		}
	}

	uploadID, err := api.Donut.NewMultipartUpload(bucket, object, req.Header.Get("Content-Type"), signature)
	if err != nil {
		errorIf(err.Trace(), "NewMultipartUpload failed.", nil)
		switch err.ToGoError().(type) {
		case donut.SignatureDoesNotMatch:
			writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
		case donut.ObjectExists:
			writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path)
		default:
			writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
		}
		return
	}

	response := generateInitiateMultipartUploadResponse(bucket, object, uploadID)
	encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
	// write headers
	setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
	// write body
	w.Write(encodedSuccessResponse)
}
// HeadObjectHandler - HEAD Object
// -----------
// The HEAD operation retrieves metadata from an object without returning the object itself.
func (api MinioAPI) HeadObjectHandler(w http.ResponseWriter, req *http.Request) {
	// ticket master block
	{
		op := APIOperation{}
		op.ProceedCh = make(chan struct{})
		api.OP <- op
		// block until Ticket master gives us a go
		<-op.ProceedCh
	}

	acceptsContentType := getContentType(req)
	if !api.isValidOp(w, req, acceptsContentType) {
		return
	}

	var object, bucket string
	vars := mux.Vars(req)
	bucket = vars["bucket"]
	object = vars["object"]

	var signature *donut.Signature
	if _, ok := req.Header["Authorization"]; ok {
		// Init signature V4 verification
		var err *probe.Error
		signature, err = initSignatureV4(req)
		if err != nil {
			errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
			writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
			return
		}
	}

	metadata, err := api.Donut.GetObjectMetadata(bucket, object, signature)
	if err != nil {
		errorIf(err.Trace(), "GetObjectMetadata failed.", nil)
		switch err.ToGoError().(type) {
		case donut.SignatureDoesNotMatch:
			writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
		case donut.BucketNameInvalid:
			writeErrorResponse(w, req, InvalidBucketName, acceptsContentType, req.URL.Path)
		case donut.BucketNotFound:
			writeErrorResponse(w, req, NoSuchBucket, acceptsContentType, req.URL.Path)
		case donut.ObjectNotFound:
			writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path)
		case donut.ObjectNameInvalid:
			writeErrorResponse(w, req, NoSuchKey, acceptsContentType, req.URL.Path)
		default:
			writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
		}
		return
	}
	setObjectHeaders(w, metadata, nil)
	w.WriteHeader(http.StatusOK)
}
Example #12
0
func fatalIf(err *probe.Error, msg string, fields map[string]interface{}) {
	if err == nil {
		return
	}
	if fields == nil {
		fields = make(map[string]interface{})
	}

	fields["error"] = err.ToGoError()
	if jsonErr, e := json.Marshal(err); e == nil {
		fields["probe"] = string(jsonErr)
	}
	log.WithFields(fields).Fatal(msg)
}
// AbortMultipartUploadHandler - Abort multipart upload
func (api MinioAPI) AbortMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
	// Ticket master block
	{
		op := APIOperation{}
		op.ProceedCh = make(chan struct{})
		api.OP <- op
		// block until Ticket master gives us a go
		<-op.ProceedCh
	}

	acceptsContentType := getContentType(req)
	if !api.isValidOp(w, req, acceptsContentType) {
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	objectResourcesMetadata := getObjectResources(req.URL.Query())

	var signature *donut.Signature
	if _, ok := req.Header["Authorization"]; ok {
		// Init signature V4 verification
		var err *probe.Error
		signature, err = initSignatureV4(req)
		if err != nil {
			errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
			writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
			return
		}
	}

	err := api.Donut.AbortMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, signature)
	if err != nil {
		errorIf(err.Trace(), "AbortMutlipartUpload failed.", nil)
		switch err.ToGoError().(type) {
		case donut.SignatureDoesNotMatch:
			writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
		case donut.InvalidUploadID:
			writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path)
		default:
			writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
		}
		return
	}
	setCommonHeaders(w, getContentTypeString(acceptsContentType), 0)
	w.WriteHeader(http.StatusNoContent)
}
Example #14
0
// ListObjects - GET bucket (list objects)
func (fs API) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) {
	if !IsValidBucket(bucket) {
		return nil, resources, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	if resources.Prefix != "" && IsValidObjectName(resources.Prefix) == false {
		return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix})
	}

	p := bucketDir{}
	rootPrefix := filepath.Join(fs.path, bucket)
	// check bucket exists
	if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
		return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
	}

	p.root = rootPrefix
	err := filepath.Walk(rootPrefix, p.getAllFiles)
	if err != nil {
		return nil, resources, probe.NewError(err)
	}

	var metadataList []ObjectMetadata
	var metadata ObjectMetadata

	// Populate filtering mode
	for _, content := range p.files {
		if len(metadataList) == resources.Maxkeys {
			resources.IsTruncated = true
			if resources.IsTruncated && resources.Delimiter != "" {
				resources.NextMarker = metadataList[len(metadataList)-1].Object
			}
			break
		}

		if content.Prefix > resources.Marker {
			var err *probe.Error
			metadata, resources, err = fs.filterObjects(bucket, content, resources)
			if err != nil {
				return nil, resources, err.Trace()
			}
			if metadata.Bucket != "" {
				metadataList = append(metadataList, metadata)
			}
		}
	}
	return metadataList, resources, nil
}
Example #15
0
// listXLBuckets -
func (xl API) listXLBuckets() *probe.Error {
	var disks map[int]block.Block
	var err *probe.Error
	for _, node := range xl.nodes {
		disks, err = node.ListDisks()
		if err != nil {
			return err.Trace()
		}
	}
	var dirs []os.FileInfo
	for _, disk := range disks {
		dirs, err = disk.ListDir(xl.config.XLName)
		if err == nil {
			break
		}
	}
	// if all disks are missing then return error
	if err != nil {
		return err.Trace()
	}
	for _, dir := range dirs {
		splitDir := strings.Split(dir.Name(), "$")
		if len(splitDir) < 3 {
			return probe.NewError(CorruptedBackend{Backend: dir.Name()})
		}
		bucketName := splitDir[0]
		// we dont need this once we cache from makeXLBucket()
		bkt, _, err := newBucket(bucketName, "private", xl.config.XLName, xl.nodes)
		if err != nil {
			return err.Trace()
		}
		xl.buckets[bucketName] = bkt
	}
	return nil
}
Example #16
0
// New instantiate a new donut
func New(path string) (CloudStorage, *probe.Error) {
	var err *probe.Error
	// load multiparts session from disk
	var multiparts *Multiparts
	multiparts, err = loadMultipartsSession()
	if err != nil {
		if os.IsNotExist(err.ToGoError()) {
			multiparts = &Multiparts{
				Version:       "1",
				ActiveSession: make(map[string]*MultipartSession),
			}
			if err := SaveMultipartsSession(multiparts); err != nil {
				return nil, err.Trace()
			}
		} else {
			return nil, err.Trace()
		}
	}
	a := API{
		path: path,
		lock: new(sync.Mutex),
	}
	a.multiparts = multiparts
	return a, nil
}
Example #17
0
// getNewClient gives a new client interface
func getNewClient(urlStr string, auth hostConfig) (client.Client, *probe.Error) {
	url := client.NewURL(urlStr)
	switch url.Type {
	case client.Object: // Minio and S3 compatible cloud storage
		s3Config := new(client.Config)
		s3Config.AccessKeyID = func() string {
			if auth.AccessKeyID == globalAccessKeyID {
				return ""
			}
			return auth.AccessKeyID
		}()
		s3Config.SecretAccessKey = func() string {
			if auth.SecretAccessKey == globalSecretAccessKey {
				return ""
			}
			return auth.SecretAccessKey
		}()
		s3Config.AppName = "Minio"
		s3Config.AppVersion = globalMCVersion
		s3Config.AppComments = []string{os.Args[0], runtime.GOOS, runtime.GOARCH}
		s3Config.HostURL = urlStr
		s3Config.Debug = globalDebugFlag

		var s3Client client.Client
		var err *probe.Error
		if auth.API == "S3v2" {
			s3Client, err = s3v2.New(s3Config)
		} else {
			s3Client, err = s3v4.New(s3Config)
		}
		if err != nil {
			return nil, err.Trace()
		}
		return s3Client, nil
	case client.Filesystem:
		fsClient, err := fs.New(urlStr)
		if err != nil {
			return nil, err.Trace()
		}
		return fsClient, nil
	}
	return nil, errInitClient(urlStr).Trace()
}
Example #18
0
File: ls.go Project: dudymas/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive, multipleArgs bool) *probe.Error {
	var err *probe.Error
	var parentContent *client.Content
	parentContent, err = clnt.Stat()
	if err != nil {
		return err.Trace(clnt.URL().String())
	}
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.BrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			case client.TooManyLevelsSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list too many levels link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				if contentCh.Content != nil {
					if contentCh.Content.Type.IsDir() && (contentCh.Content.Type&os.ModeSymlink == os.ModeSymlink) {
						errorIf(contentCh.Err.Trace(), "Unable to list broken folder link.")
						continue
					}
				}
				errorIf(contentCh.Err.Trace(), "Unable to list.")
				continue
			}
			err = contentCh.Err.Trace()
			break
		}
		if multipleArgs && parentContent.Type.IsDir() {
			contentCh.Content.Name = filepath.Join(parentContent.Name, strings.TrimPrefix(contentCh.Content.Name, parentContent.Name))
		}
		Prints("%s\n", parseContent(contentCh.Content))
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #19
0
// getBucketMetadata -
func (b bucket) getBucketMetadata() (*AllBuckets, *probe.Error) {
	metadata := new(AllBuckets)
	var readers map[int]io.ReadCloser
	{
		var err *probe.Error
		readers, err = b.getBucketMetadataReaders()
		if err != nil {
			return nil, err.Trace()
		}
	}
	for _, reader := range readers {
		defer reader.Close()
	}
	var err error
	for _, reader := range readers {
		jenc := json.NewDecoder(reader)
		if err = jenc.Decode(metadata); err == nil {
			return metadata, nil
		}
	}
	return nil, probe.NewError(err)
}
Example #20
0
// getBucketMetadataReaders -
func (b bucket) getBucketMetadataReaders() (map[int]io.ReadCloser, *probe.Error) {
	readers := make(map[int]io.ReadCloser)
	var disks map[int]disk.Disk
	var err *probe.Error
	for _, node := range b.nodes {
		disks, err = node.ListDisks()
		if err != nil {
			return nil, err.Trace()
		}
	}
	var bucketMetaDataReader io.ReadCloser
	for order, disk := range disks {
		bucketMetaDataReader, err = disk.Open(filepath.Join(b.donutName, bucketMetadataConfig))
		if err != nil {
			continue
		}
		readers[order] = bucketMetaDataReader
	}
	if err != nil {
		return nil, err.Trace()
	}
	return readers, nil
}
Example #21
0
// catURL displays contents of a URL to stdout.
func catURL(sourceURL string) *probe.Error {
	var reader io.Reader
	size := int64(-1)
	switch sourceURL {
	case "-":
		reader = os.Stdin
	default:
		var err *probe.Error
		client, content, err := url2Stat(sourceURL)
		if err != nil {
			return err.Trace(sourceURL)
		}
		// Ignore size for filesystem objects since os.Stat() would not
		// return proper size all the time, for example with /proc files.
		if client.GetURL().Type == objectStorage {
			size = content.Size
		}
		if reader, err = getSourceStream(sourceURL); err != nil {
			return err.Trace(sourceURL)
		}
	}
	return catOut(reader, size).Trace(sourceURL)
}
Example #22
0
File: ls.go Project: axcoto-lab/mc
// doList - list all entities inside a folder.
func doList(clnt client.Client, recursive bool) *probe.Error {
	var err *probe.Error
	for contentCh := range clnt.List(recursive) {
		if contentCh.Err != nil {
			switch contentCh.Err.ToGoError().(type) {
			// handle this specifically for filesystem
			case client.ISBrokenSymlink:
				errorIf(contentCh.Err.Trace(), "Unable to list broken link.")
				continue
			}
			if os.IsNotExist(contentCh.Err.ToGoError()) || os.IsPermission(contentCh.Err.ToGoError()) {
				errorIf(contentCh.Err.Trace(), "Unable to list.")
				continue
			}
			err = contentCh.Err.Trace()
			break
		}
		console.Println(parseContent(contentCh.Content))
	}
	if err != nil {
		return err.Trace()
	}
	return nil
}
Example #23
0
// getInheritedListeners - look for LISTEN_FDS in environment variables and populate listeners accordingly
func (n *minNet) getInheritedListeners() *probe.Error {
	var retErr *probe.Error
	n.inheritOnce.Do(func() {
		n.mutex.Lock()
		defer n.mutex.Unlock()
		countStr := os.Getenv(envCountKey)
		if countStr == "" {
			return
		}
		count, err := strconv.Atoi(countStr)
		if err != nil {
			retErr = probe.NewError(fmt.Errorf("found invalid count value: %s=%s", envCountKey, countStr))
			return
		}

		fdStart := 3
		for i := fdStart; i < fdStart+count; i++ {
			file := os.NewFile(uintptr(i), "listener")
			l, err := net.FileListener(file)
			if err != nil {
				file.Close()
				retErr = probe.NewError(err)
				return
			}
			if err := file.Close(); err != nil {
				retErr = probe.NewError(err)
				return
			}
			n.inheritedListeners = append(n.inheritedListeners, l)
		}
	})
	if retErr != nil {
		return retErr.Trace()
	}
	return nil
}
Example #24
0
// New instantiate a new donut
func New() (Interface, *probe.Error) {
	var conf *Config
	var err *probe.Error
	conf, err = LoadConfig()
	if err != nil {
		conf = &Config{
			Version:     "0.0.1",
			MaxSize:     512000000,
			NodeDiskMap: nil,
			DonutName:   "",
		}
		if err := quick.CheckData(conf); err != nil {
			return nil, err.Trace()
		}
	}
	a := API{config: conf}
	a.storedBuckets = metadata.NewCache()
	a.nodes = make(map[string]node)
	a.buckets = make(map[string]bucket)
	a.objects = data.NewCache(a.config.MaxSize)
	a.multiPartObjects = make(map[string]*data.Cache)
	a.objects.OnEvicted = a.evictedObject
	a.lock = new(sync.Mutex)

	if len(a.config.NodeDiskMap) > 0 {
		for k, v := range a.config.NodeDiskMap {
			if len(v) == 0 {
				return nil, probe.NewError(InvalidDisksArgument{})
			}
			err := a.AttachNode(k, v)
			if err != nil {
				return nil, err.Trace()
			}
		}
		/// Initialization, populate all buckets into memory
		buckets, err := a.listBuckets()
		if err != nil {
			return nil, err.Trace()
		}
		for k, v := range buckets {
			var newBucket = storedBucket{}
			newBucket.bucketMetadata = v
			newBucket.objectMetadata = make(map[string]ObjectMetadata)
			newBucket.multiPartSession = make(map[string]MultiPartSession)
			newBucket.partMetadata = make(map[string]map[int]PartMetadata)
			a.storedBuckets.Set(k, newBucket)
		}
		a.Heal()
	}
	return a, nil
}
Example #25
0
func isErrIgnored(err *probe.Error) (ignored bool) {
	// For all non critical errors we can continue for the remaining files.
	switch err.ToGoError().(type) {
	// Handle these specifically for filesystem related errors.
	case BrokenSymlink, TooManyLevelsSymlink, PathNotFound, PathInsufficientPermission:
		ignored = true
	// Handle these specifically for object storage related errors.
	case BucketNameEmpty, ObjectMissing, ObjectAlreadyExists:
		ignored = true
	case ObjectAlreadyExistsAsDirectory, BucketDoesNotExist, BucketInvalid, ObjectOnGlacier:
		ignored = true
	default:
		ignored = false
	}
	return ignored
}
Example #26
0
// Parses signature version '4' header of the following form.
//
//    Authorization: algorithm Credential=access key ID/credential scope, \
//            SignedHeaders=SignedHeaders, Signature=signature
//
func parseSignV4(v4Auth string) (signValues, *probe.Error) {
	// Replace all spaced strings, some clients can send spaced
	// parameters and some won't. So we pro-actively remove any spaces
	// to make parsing easier.
	v4Auth = strings.Replace(v4Auth, " ", "", -1)
	if v4Auth == "" {
		return signValues{}, ErrAuthHeaderEmpty("Auth header empty.").Trace(v4Auth)
	}

	// Verify if the header algorithm is supported or not.
	if !strings.HasPrefix(v4Auth, signV4Algorithm) {
		return signValues{}, ErrUnsuppSignAlgo("Unsupported algorithm in authorization header.", v4Auth).Trace(v4Auth)
	}

	// Strip off the Algorithm prefix.
	v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm)
	authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
	if len(authFields) != 3 {
		return signValues{}, ErrMissingFields("Missing fields in authorization header.", v4Auth).Trace(v4Auth)
	}

	// Initialize signature version '4' structured header.
	signV4Values := signValues{}

	var err *probe.Error
	// Save credentail values.
	signV4Values.Creds, err = parseCredential(authFields[0])
	if err != nil {
		return signValues{}, err.Trace(v4Auth)
	}

	// Save signed headers.
	signV4Values.SignedHeaders, err = parseSignedHeaders(authFields[1])
	if err != nil {
		return signValues{}, err.Trace(v4Auth)
	}

	// Save signature.
	signV4Values.Signature, err = parseSignature(authFields[2])
	if err != nil {
		return signValues{}, err.Trace(v4Auth)
	}

	// Return the structure here.
	return signV4Values, nil
}
Example #27
0
// Parses signature version '4' query string of the following form.
//
//   querystring = X-Amz-Algorithm=algorithm
//   querystring += &X-Amz-Credential= urlencode(access_key_ID + '/' + credential_scope)
//   querystring += &X-Amz-Date=date
//   querystring += &X-Amz-Expires=timeout interval
//   querystring += &X-Amz-SignedHeaders=signed_headers
//   querystring += &X-Amz-Signature=signature
//
func parsePreSignV4(query url.Values) (preSignValues, *probe.Error) {
	// Verify if the query algorithm is supported or not.
	if query.Get("X-Amz-Algorithm") != signV4Algorithm {
		return preSignValues{}, ErrUnsuppSignAlgo("Unsupported algorithm in query string.", query.Get("X-Amz-Algorithm"))
	}

	// Initialize signature version '4' structured header.
	preSignV4Values := preSignValues{}

	var err *probe.Error
	// Save credentail values.
	preSignV4Values.Creds, err = parseCredential("Credential=" + query.Get("X-Amz-Credential"))
	if err != nil {
		return preSignValues{}, err.Trace(query.Get("X-Amz-Credential"))
	}

	var e error
	// Save date in native time.Time.
	preSignV4Values.Date, e = time.Parse(iso8601Format, query.Get("X-Amz-Date"))
	if e != nil {
		return preSignValues{}, ErrMalformedDate("Malformed date string.", query.Get("X-Amz-Date")).Trace(query.Get("X-Amz-Date"))
	}

	// Save expires in native time.Duration.
	preSignV4Values.Expires, e = time.ParseDuration(query.Get("X-Amz-Expires") + "s")
	if e != nil {
		return preSignValues{}, ErrMalformedExpires("Malformed expires string.", query.Get("X-Amz-Expires")).Trace(query.Get("X-Amz-Expires"))
	}

	// Save signed headers.
	preSignV4Values.SignedHeaders, err = parseSignedHeaders("SignedHeaders=" + query.Get("X-Amz-SignedHeaders"))
	if err != nil {
		return preSignValues{}, err.Trace(query.Get("X-Amz-SignedHeaders"))
	}

	// Save signature.
	preSignV4Values.Signature, err = parseSignature("Signature=" + query.Get("X-Amz-Signature"))
	if err != nil {
		return preSignValues{}, err.Trace(query.Get("X-Amz-Signature"))
	}

	// Return structed form of signature query string.
	return preSignV4Values, nil
}
// PutObjectHandler - PUT Object
// ----------
// This implementation of the PUT operation adds an object to a bucket.
func (api API) PutObjectHandler(w http.ResponseWriter, req *http.Request) {
	// Ticket master block
	{
		op := APIOperation{}
		op.ProceedCh = make(chan struct{})
		api.OP <- op
		// block until Ticket master gives us a go
		<-op.ProceedCh
	}

	if !api.isValidOp(w, req) {
		return
	}

	var object, bucket string
	vars := mux.Vars(req)
	bucket = vars["bucket"]
	object = vars["object"]

	// get Content-MD5 sent by client and verify if valid
	md5 := req.Header.Get("Content-MD5")
	if !isValidMD5(md5) {
		writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
		return
	}
	/// if Content-Length missing, deny the request
	size := req.Header.Get("Content-Length")
	if size == "" {
		writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
		return
	}
	/// maximum Upload size for objects in a single operation
	if isMaxObjectSize(size) {
		writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
		return
	}
	/// minimum Upload size for objects in a single operation
	//
	// Surprisingly while Amazon in their document states that S3 objects have 1byte
	// as the minimum limit, they do not seem to enforce it one can successfully
	// create a 0byte file using a regular putObject() operation
	//
	// if isMinObjectSize(size) {
	//      writeErrorResponse(w, req, EntityTooSmall,  req.URL.Path)
	//	return
	// }
	var sizeInt64 int64
	{
		var err error
		sizeInt64, err = strconv.ParseInt(size, 10, 64)
		if err != nil {
			writeErrorResponse(w, req, InvalidRequest, req.URL.Path)
			return
		}
	}

	var signature *signv4.Signature
	if !api.Anonymous {
		if _, ok := req.Header["Authorization"]; ok {
			// Init signature V4 verification
			var err *probe.Error
			signature, err = initSignatureV4(req)
			if err != nil {
				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
				writeErrorResponse(w, req, InternalError, req.URL.Path)
				return
			}
		}
	}

	metadata, err := api.Donut.CreateObject(bucket, object, md5, sizeInt64, req.Body, nil, signature)
	if err != nil {
		errorIf(err.Trace(), "CreateObject failed.", nil)
		switch err.ToGoError().(type) {
		case donut.BucketNotFound:
			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
		case donut.BucketNameInvalid:
			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
		case donut.ObjectExists:
			writeErrorResponse(w, req, MethodNotAllowed, req.URL.Path)
		case donut.BadDigest:
			writeErrorResponse(w, req, BadDigest, req.URL.Path)
		case signv4.MissingDateHeader:
			writeErrorResponse(w, req, RequestTimeTooSkewed, req.URL.Path)
		case signv4.DoesNotMatch:
			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
		case donut.IncompleteBody:
			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
		case donut.EntityTooLarge:
			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
		case donut.InvalidDigest:
			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
		default:
			writeErrorResponse(w, req, InternalError, req.URL.Path)
		}
		return
	}
	w.Header().Set("ETag", metadata.MD5Sum)
	writeSuccessResponse(w)
}
func (s rpcSignatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var signature *rpcSignature
	if isRequestSignatureRPC(r) {
		// Init signature V4 verification
		var err *probe.Error
		signature, err = initSignatureRPC(r)
		if err != nil {
			switch err.ToGoError() {
			case errInvalidRegion:
				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
				writeErrorResponse(w, r, AuthorizationHeaderMalformed, r.URL.Path)
				return
			case errAccessKeyIDInvalid:
				errorIf(err.Trace(), "Invalid access key id.", nil)
				writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
				return
			default:
				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
				writeErrorResponse(w, r, InternalError, r.URL.Path)
				return
			}
		}
		buffer := new(bytes.Buffer)
		if _, err := io.Copy(buffer, r.Body); err != nil {
			errorIf(probe.NewError(err), "Unable to read payload from request body.", nil)
			writeErrorResponse(w, r, InternalError, r.URL.Path)
			return
		}
		value := sha256.Sum256(buffer.Bytes())
		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(value[:]))
		if err != nil {
			errorIf(err.Trace(), "Unable to verify signature.", nil)
			writeErrorResponse(w, r, InternalError, r.URL.Path)
			return
		}
		if !ok {
			writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
			return
		}
		s.handler.ServeHTTP(w, r)
		return
	}
	writeErrorResponse(w, r, AccessDenied, r.URL.Path)
}
// PutObjectPartHandler - Upload part
func (api API) PutObjectPartHandler(w http.ResponseWriter, req *http.Request) {
	// Ticket master block
	{
		op := APIOperation{}
		op.ProceedCh = make(chan struct{})
		api.OP <- op
		// block until Ticket master gives us a go
		<-op.ProceedCh
	}

	if !api.isValidOp(w, req) {
		return
	}

	// get Content-MD5 sent by client and verify if valid
	md5 := req.Header.Get("Content-MD5")
	if !isValidMD5(md5) {
		writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
		return
	}

	/// if Content-Length missing, throw away
	size := req.Header.Get("Content-Length")
	if size == "" {
		writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
		return
	}

	/// maximum Upload size for multipart objects in a single operation
	if isMaxObjectSize(size) {
		writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
		return
	}

	var sizeInt64 int64
	{
		var err error
		sizeInt64, err = strconv.ParseInt(size, 10, 64)
		if err != nil {
			writeErrorResponse(w, req, InvalidRequest, req.URL.Path)
			return
		}
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	uploadID := req.URL.Query().Get("uploadId")
	partIDString := req.URL.Query().Get("partNumber")

	var partID int
	{
		var err error
		partID, err = strconv.Atoi(partIDString)
		if err != nil {
			writeErrorResponse(w, req, InvalidPart, req.URL.Path)
			return
		}
	}

	var signature *signv4.Signature
	if !api.Anonymous {
		if _, ok := req.Header["Authorization"]; ok {
			// Init signature V4 verification
			var err *probe.Error
			signature, err = initSignatureV4(req)
			if err != nil {
				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
				writeErrorResponse(w, req, InternalError, req.URL.Path)
				return
			}
		}
	}

	calculatedMD5, err := api.Donut.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body, signature)
	if err != nil {
		errorIf(err.Trace(), "CreateObjectPart failed.", nil)
		switch err.ToGoError().(type) {
		case donut.InvalidUploadID:
			writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
		case donut.ObjectExists:
			writeErrorResponse(w, req, MethodNotAllowed, req.URL.Path)
		case donut.BadDigest:
			writeErrorResponse(w, req, BadDigest, req.URL.Path)
		case signv4.DoesNotMatch:
			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
		case donut.IncompleteBody:
			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
		case donut.EntityTooLarge:
			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
		case donut.InvalidDigest:
			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
		default:
			writeErrorResponse(w, req, InternalError, req.URL.Path)
		}
		return
	}
	w.Header().Set("ETag", calculatedMD5)
	writeSuccessResponse(w)
}