func newConn(url *url.URL) (*http.ClientConn, os.Error) {
addr := url.Host
//just set the default scheme to http
if url.Scheme == "" {
url.Scheme = "http"
}
if !hasPort(addr) {
addr += ":" + url.Scheme
}
var conn net.Conn
var err os.Error
if url.Scheme == "http" {
conn, err = net.Dial("tcp", addr)
if err != nil {
return nil, err
}
} else { // https
conn, err = tls.Dial("tcp", addr, nil)
if err != nil {
return nil, err
}
h := url.Host
if hasPort(h) {
h = h[0:strings.LastIndex(h, ":")]
}
if err := conn.(*tls.Conn).VerifyHostname(h); err != nil {
return nil, err
}
}
return http.NewClientConn(conn, nil), nil
}
func Setup() (p *Propolis, push bool) {
var refresh, watch, delete, paranoid, reset, practice, public, secure, reduced, directories bool
var delay, concurrent int
flag.BoolVar(&refresh, "refresh", true,
"Scan online bucket to update cache at startup\n"+
"\tLonger startup time, but catches changes made while offline")
flag.BoolVar(&watch, "watch", false,
"Go into daemon mode and watch the local file system\n"+
"\tfor changes after initial sync (false means sync then quit)")
flag.BoolVar(&delete, "delete", true,
"Delete files when syncing as well as copying changed files")
flag.BoolVar(¶noid, "paranoid", false,
"Always verify md5 hash of file contents,\n"+
"\teven when all metadata is an exact match (slower)")
flag.BoolVar(&reset, "reset", false,
"Reset the cache (implies -refresh=true)")
flag.BoolVar(&practice, "practice", false,
"Do a practice run without changing any files\n"+
"\tShows what would be changed (implies -watch=false)")
flag.BoolVar(&public, "public", true,
"Make world-readable local files publicly readable\n"+
"\tin the online bucket (downloadable via the web)")
flag.BoolVar(&secure, "secure", false,
"Use secure connections to Amazon S3\n"+
"\tA bit slower, but data is encrypted when being transferred")
flag.BoolVar(&reduced, "reduced", false,
"Use reduced redundancy storage when uploading\n"+
"\tCheaper, but higher chance of loosing data")
flag.BoolVar(&directories, "directories", false,
"Track directories using special zero-length files\n"+
"\tMostly useful for greater compatibility with s3fslite")
flag.IntVar(&delay, "delay", 5,
"Wait this number of seconds from the last change to a file\n"+
"\tbefore syncing it with the server")
flag.IntVar(&concurrent, "concurrent", 25,
"Maximum number of server transactions that are\n"+
"\tallowed to run concurrently")
var accesskeyid, secretaccesskey, cache_location string
flag.StringVar(&accesskeyid, "accesskeyid", "",
"Amazon AWS Access Key ID")
flag.StringVar(&secretaccesskey, "secretaccesskey", "",
"Amazon AWS Secret Access Key")
flag.StringVar(&cache_location, "cache", default_cache_location,
"Metadata cache location\n"+
"\tA sqlite3 database file that caches online metadata")
flag.Usage = func() {
fmt.Fprintf(os.Stderr,
"Propolis:\n"+
" Amazon S3 <--> local file system synchronizer\n"+
" Synchronizes a local directory with an S3 bucket, then\n"+
" watches the local directory for changes and automatically\n"+
" propogates them to the bucket.\n\n"+
" See http://github.com/russross/propolis for details\n\n"+
" Copyright 2011 by Russ Ross <*****@*****.**>\n\n"+
" Propolis comes with ABSOLUTELY NO WARRANTY. This is free software, and you\n"+
" are welcome to redistribute it under certain conditions. See the GNU\n"+
" General Public Licence for details.\n\n"+
"Usage:\n"+
" To start by syncing remote bucket to match local file system:\n"+
" %s [flags] local/dir s3:bucket[:remote/dir]\n"+
" To start by syncing local file system to match remote bucket:\n"+
" %s [flags] s3:bucket[:remote/dir] local/dir\n\n"+
" Amazon Access Key ID and Secret Access Key can be specified in\n"+
" one of three ways, listed in decreasing order of precedence.\n"+
" Note: both values must be supplied using a single method:\n\n"+
" 1. On the command line\n"+
" 2. In the environment variables %s and %s\n"+
" 3. In the file %s as key:secret on a single line\n\n"+
"Options:\n",
os.Args[0], os.Args[0],
s3_access_key_id_variable, s3_secret_access_key_variable, s3_password_file)
flag.PrintDefaults()
}
flag.Parse()
// enforce certain option combinations
if reset {
refresh = true
}
if practice {
watch = false
}
// make sure we get access keys
if accesskeyid == "" || secretaccesskey == "" {
accesskeyid, secretaccesskey = getKeys()
}
if accesskeyid == "" || secretaccesskey == "" {
fmt.Fprintln(os.Stderr, "Error: Amazon AWS Access Key ID and/or Secret Access Key undefined\n")
flag.Usage()
os.Exit(-1)
}
// check command-line arguments
args := flag.Args()
if len(args) != 2 {
flag.Usage()
os.Exit(-1)
}
// figure out the direction of sync, parse the bucket and directory info
var bucketname, bucketprefix, localdir string
switch {
case !strings.HasPrefix(args[0], "s3:") && strings.HasPrefix(args[1], "s3:"):
push = true
localdir = parseLocalDir(args[0])
bucketname, bucketprefix = parseBucket(args[1])
case strings.HasPrefix(args[0], "s3:") && !strings.HasPrefix(args[1], "s3:"):
push = false
bucketname, bucketprefix = parseBucket(args[0])
localdir = parseLocalDir(args[1])
default:
flag.Usage()
os.Exit(-1)
}
// make sure the root directory exists
if info, err := os.Lstat(localdir); err != nil || !info.IsDirectory() {
fmt.Fprintf(os.Stderr, "%s is not a valid directory\n", localdir)
}
// open the database
var err os.Error
var cache Cache
if cache, err = Connect(path.Join(cache_location, bucketname+".sqlite")); err != nil {
fmt.Println("Error connecting to database:", err)
os.Exit(-1)
}
// create the Propolis object
url := new(url.URL)
url.Scheme = "http"
if secure {
url.Scheme = "https"
}
url.Host = bucketname + ".s3.amazonaws.com"
url.Path = "/"
p = &Propolis{
Bucket: bucketname,
Url: url,
Secure: secure,
ReducedRedundancy: reduced,
Key: accesskeyid,
Secret: secretaccesskey,
BucketRoot: bucketprefix,
LocalRoot: localdir,
Refresh: refresh,
Paranoid: paranoid,
Reset: reset,
Directories: directories,
Practice: practice,
Watch: watch,
Delay: delay,
Concurrent: concurrent,
Db: cache,
}
return
}
func getNewUrl(url *url.URL) *url.URL {
urlBackend, _ := url.Parse("http://127.0.0.1")
url.Scheme = urlBackend.Scheme
url.Host = urlBackend.Host
return url
}