예제 #1
0
파일: s3.go 프로젝트: russross/propolis
func (p *Propolis) SignRequest(req *http.Request) {
	// gather the string to be signed

	// method
	msg := req.Method + "\n"

	// md5sum
	msg += req.Header.Get("Content-MD5") + "\n"

	// content-type
	msg += req.Header.Get("Content-Type") + "\n"

	// date
	msg += req.Header.Get("Date") + "\n"

	// add headers
	for _, key := range AWS_HEADERS {
		if value := req.Header.Get(key); value != "" {
			msg += strings.ToLower(key) + ":" + value + "\n"
		}
	}

	// resource: the path components should be URL-encoded, but not the slashes
	u := new(url.URL)
	u.Path = "/" + p.Bucket + req.URL.Path
	msg += u.String()

	// create the signature
	hmac := hmac.NewSHA1([]byte(p.Secret))
	hmac.Write([]byte(msg))

	// get a base64 encoding of the signature
	var encoded bytes.Buffer
	encoder := base64.NewEncoder(base64.StdEncoding, &encoded)
	encoder.Write(hmac.Sum())
	encoder.Close()
	signature := encoded.String()

	req.Header.Set("Authorization", "AWS "+p.Key+":"+signature)
}
예제 #2
0
파일: s3.go 프로젝트: russross/propolis
func (p *Propolis) SendRequest(method string, reduced bool, src string, target *url.URL, body io.ReadCloser, hash string, info *os.FileInfo) (resp *http.Response, err os.Error) {
	defer func() {
		// if anything goes wrong, close the body reader
		// if it ends normally, this will be closed already and set to nil
		if body != nil {
			body.Close()
		}
	}()

	var req *http.Request
	if req, err = http.NewRequest(method, target.String(), body); err != nil {
		return
	}

	// set upload file info if applicable
	if info != nil && body != nil {
		// TODO: 0-length files fail because the Content-Length field is missing
		// a fix is in the works in the Go library
		req.ContentLength = info.Size
	}

	if info != nil {
		p.SetRequestMetaData(req, info)
	}

	// reduced redundancy?
	if reduced {
		req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
	}

	// are we uploading a file with a content hash?
	if hash != "" {
		req.Header.Set("Content-MD5", hash)
	}

	// is this a copy/metadata update?
	if src != "" {
		// note: src should already be a full bucket + path name
		u := new(url.URL)
		u.Path = src
		req.Header.Set("X-Amz-Copy-Source", u.String())
		req.Header.Set("X-Amz-Metadata-Directive", "REPLACE")
	}

	// sign and execute the request
	// note: 2nd argument is temporary hack to set Content-Length: 0 when needed
	if resp, err = p.SignAndExecute(req, method == "PUT" && body == nil || (info != nil && info.Size == 0)); err != nil {
		return
	}

	// body was closed when the request was written out,
	// so nullify the deferred close
	body = nil

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		err = os.NewError(resp.Status)
		return
	}

	return
}
예제 #3
0
파일: main.go 프로젝트: russross/propolis
func Setup() (p *Propolis, push bool) {
	var refresh, watch, delete, paranoid, reset, practice, public, secure, reduced, directories bool
	var delay, concurrent int
	flag.BoolVar(&refresh, "refresh", true,
		"Scan online bucket to update cache at startup\n"+
			"\tLonger startup time, but catches changes made while offline")
	flag.BoolVar(&watch, "watch", false,
		"Go into daemon mode and watch the local file system\n"+
			"\tfor changes after initial sync (false means sync then quit)")
	flag.BoolVar(&delete, "delete", true,
		"Delete files when syncing as well as copying changed files")
	flag.BoolVar(&paranoid, "paranoid", false,
		"Always verify md5 hash of file contents,\n"+
			"\teven when all metadata is an exact match (slower)")
	flag.BoolVar(&reset, "reset", false,
		"Reset the cache (implies -refresh=true)")
	flag.BoolVar(&practice, "practice", false,
		"Do a practice run without changing any files\n"+
			"\tShows what would be changed (implies -watch=false)")
	flag.BoolVar(&public, "public", true,
		"Make world-readable local files publicly readable\n"+
			"\tin the online bucket (downloadable via the web)")
	flag.BoolVar(&secure, "secure", false,
		"Use secure connections to Amazon S3\n"+
			"\tA bit slower, but data is encrypted when being transferred")
	flag.BoolVar(&reduced, "reduced", false,
		"Use reduced redundancy storage when uploading\n"+
			"\tCheaper, but higher chance of loosing data")
	flag.BoolVar(&directories, "directories", false,
		"Track directories using special zero-length files\n"+
			"\tMostly useful for greater compatibility with s3fslite")
	flag.IntVar(&delay, "delay", 5,
		"Wait this number of seconds from the last change to a file\n"+
			"\tbefore syncing it with the server")
	flag.IntVar(&concurrent, "concurrent", 25,
		"Maximum number of server transactions that are\n"+
			"\tallowed to run concurrently")

	var accesskeyid, secretaccesskey, cache_location string
	flag.StringVar(&accesskeyid, "accesskeyid", "",
		"Amazon AWS Access Key ID")
	flag.StringVar(&secretaccesskey, "secretaccesskey", "",
		"Amazon AWS Secret Access Key")
	flag.StringVar(&cache_location, "cache", default_cache_location,
		"Metadata cache location\n"+
			"\tA sqlite3 database file that caches online metadata")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			"Propolis:\n"+
				"  Amazon S3 <--> local file system synchronizer\n"+
				"  Synchronizes a local directory with an S3 bucket, then\n"+
				"  watches the local directory for changes and automatically\n"+
				"  propogates them to the bucket.\n\n"+
				"  See http://github.com/russross/propolis for details\n\n"+
				"  Copyright 2011 by Russ Ross <*****@*****.**>\n\n"+
				"  Propolis comes with ABSOLUTELY NO WARRANTY.  This is free software, and you\n"+
				"  are welcome to redistribute it under certain conditions.  See the GNU\n"+
				"  General Public Licence for details.\n\n"+
				"Usage:\n"+
				"  To start by syncing remote bucket to match local file system:\n"+
				"      %s [flags] local/dir s3:bucket[:remote/dir]\n"+
				"  To start by syncing local file system to match remote bucket:\n"+
				"      %s [flags] s3:bucket[:remote/dir] local/dir\n\n"+
				"  Amazon Access Key ID and Secret Access Key can be specified in\n"+
				"  one of three ways, listed in decreasing order of precedence.\n"+
				"  Note: both values must be supplied using a single method:\n\n"+
				"      1. On the command line\n"+
				"      2. In the environment variables %s and %s\n"+
				"      3. In the file %s as key:secret on a single line\n\n"+
				"Options:\n",
			os.Args[0], os.Args[0],
			s3_access_key_id_variable, s3_secret_access_key_variable, s3_password_file)
		flag.PrintDefaults()
	}
	flag.Parse()

	// enforce certain option combinations
	if reset {
		refresh = true
	}
	if practice {
		watch = false
	}

	// make sure we get access keys
	if accesskeyid == "" || secretaccesskey == "" {
		accesskeyid, secretaccesskey = getKeys()
	}
	if accesskeyid == "" || secretaccesskey == "" {
		fmt.Fprintln(os.Stderr, "Error: Amazon AWS Access Key ID and/or Secret Access Key undefined\n")
		flag.Usage()
		os.Exit(-1)
	}

	// check command-line arguments
	args := flag.Args()
	if len(args) != 2 {
		flag.Usage()
		os.Exit(-1)
	}

	// figure out the direction of sync, parse the bucket and directory info
	var bucketname, bucketprefix, localdir string

	switch {
	case !strings.HasPrefix(args[0], "s3:") && strings.HasPrefix(args[1], "s3:"):
		push = true
		localdir = parseLocalDir(args[0])
		bucketname, bucketprefix = parseBucket(args[1])
	case strings.HasPrefix(args[0], "s3:") && !strings.HasPrefix(args[1], "s3:"):
		push = false
		bucketname, bucketprefix = parseBucket(args[0])
		localdir = parseLocalDir(args[1])
	default:
		flag.Usage()
		os.Exit(-1)
	}

	// make sure the root directory exists
	if info, err := os.Lstat(localdir); err != nil || !info.IsDirectory() {
		fmt.Fprintf(os.Stderr, "%s is not a valid directory\n", localdir)
	}

	// open the database
	var err os.Error
	var cache Cache
	if cache, err = Connect(path.Join(cache_location, bucketname+".sqlite")); err != nil {
		fmt.Println("Error connecting to database:", err)
		os.Exit(-1)
	}

	// create the Propolis object
	url := new(url.URL)
	url.Scheme = "http"
	if secure {
		url.Scheme = "https"
	}
	url.Host = bucketname + ".s3.amazonaws.com"
	url.Path = "/"

	p = &Propolis{
		Bucket:            bucketname,
		Url:               url,
		Secure:            secure,
		ReducedRedundancy: reduced,
		Key:               accesskeyid,
		Secret:            secretaccesskey,

		BucketRoot: bucketprefix,
		LocalRoot:  localdir,

		Refresh:     refresh,
		Paranoid:    paranoid,
		Reset:       reset,
		Directories: directories,
		Practice:    practice,
		Watch:       watch,
		Delay:       delay,
		Concurrent:  concurrent,

		Db: cache,
	}
	return
}