func (s *S3) init() error {
if s.Bucket == "" {
return errors.New("Please Specify an S3 Bucket")
}
if s.Region == "" {
return errors.New("Please Specify an S3 Region")
}
if s.Root == "" {
return errors.New("Please Specify an S3 Root Path")
}
if s.BufferDir == "" {
return errors.New("Please Specify a Buffer Directory to use for Uploads")
}
var ok bool
if s.region, ok = aws.Regions[s.Region]; !ok {
return errors.New("Invalid Region: " + s.Region)
}
err := s.getAuth()
if err != nil {
return err
}
s.s3 = s3.New(s.auth, s.region)
s.bucket = s.s3.Bucket(s.Bucket)
if err := os.MkdirAll(s.BufferDir, 0755); err != nil && !os.IsExist(err) {
// there was an error and it wasn't that the directory already exists
return err
}
s.bufferDir = &BufferDir{Mutex: sync.Mutex{}, root: s.BufferDir}
s.root = strings.TrimPrefix(s.Root, "/")
go s.updateAuthLoop()
return nil
}
func (app *AppContext) setupS3Logger() (err error) {
auth, err := aws.GetAuth("", "", "", time.Now())
if err != nil {
log.Fatalln("Failed to find AWS credentials in env")
}
awsConnection := s3.New(
auth,
getAWSRegion(app.config.aws_region),
)
bucket := awsConnection.Bucket(app.config.bucket)
instanceInfo := keygen.BuildInstanceInfo(
&keygen.EnvInstanceFetcher{},
serviceName,
app.config.logging_dir,
)
rotateCoordinator := gologging.NewRotateCoordinator(
app.config.max_log_lines,
app.config.max_log_age,
)
metricsLogger := MetricsLogger{app.metrics}
app.s3log, err = gologging.StartS3Logger(
rotateCoordinator,
instanceInfo,
&metricsLogger,
&uploader.S3UploaderBuilder{
Bucket: bucket,
KeyNameGenerator: &KeyNameGenerator{
Info: instanceInfo,
Prefix: app.config.key_prefix,
},
},
&metricsLogger,
app.config.num_workers,
)
if err != nil {
return
}
// Make sure logger is flushed when shutdown signal is received
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
go func() {
<-sigc
log.Println("interrupted, closing logger...")
app.s3log.Close()
os.Exit(0)
}()
return nil
}
// Deploys a site to S3.
func (s *Site) Deploy(user, pass, url string) error {
auth := aws.Auth{AccessKey: user, SecretKey: pass}
b := s3.New(auth, aws.USEast).Bucket(url)
// walks _site directory and uploads file to S3
walker := func(fn string, fi os.FileInfo, err error) error {
if fi.IsDir() {
return nil
}
rel, _ := filepath.Rel(s.Dest, fn)
typ := mime.TypeByExtension(filepath.Ext(rel))
content, err := ioutil.ReadFile(fn)
logf(MsgUploadFile, rel)
if err != nil {
return err
}
// try to upload the file ... sometimes this fails due to amazon
// issues. If so, we'll re-try
if err := b.Put(rel, content, typ, s3.PublicRead, s3.Options{}); err != nil {
time.Sleep(100 * time.Millisecond) // sleep so that we don't immediately retry
return b.Put(rel, content, typ, s3.PublicRead, s3.Options{})
}
// file upload was a success, return nil
return nil
}
return filepath.Walk(s.Dest, walker)
}
// Communicate with all endpoints to see if they are alive.
func (s *ClientTests) TestRegions(c *check.C) {
errs := make(chan error, len(aws.Regions))
for _, region := range aws.Regions {
go func(r aws.Region) {
s := s3.New(s.s3.Auth, r)
b := s.Bucket("goamz-" + s.Auth.AccessKey)
_, err := b.Get("non-existent")
errs <- err
}(region)
}
for _ = range aws.Regions {
err := <-errs
if err != nil {
s3_err, ok := err.(*s3.Error)
if ok {
c.Check(s3_err.Code, check.Matches, "NoSuchBucket")
} else if _, ok = err.(*net.DNSError); ok {
// Okay as well.
} else {
c.Errorf("Non-S3 error: %s", err)
}
} else {
c.Errorf("Test should have errored but it seems to have succeeded")
}
}
}
func s3Setup(bucketName string, path string, opts sequinsOptions) *sequins {
auth, err := aws.GetAuth(*s3AccessKey, *s3SecretKey, "", time.Time{})
if err != nil {
log.Fatal(err)
}
regionName := *s3Region
if regionName == "" {
regionName = aws.InstanceRegion()
if regionName == "" {
log.Fatal("Unspecified --s3-region, and no instance region found.")
}
}
region, exists := aws.Regions[regionName]
if !exists {
log.Fatalf("Invalid AWS region: %s", regionName)
}
bucket := s3.New(auth, region).Bucket(bucketName)
backend := backend.NewS3Backend(bucket, path)
if opts.LocalPath == "" {
tmpDir, err := ioutil.TempDir("", "sequins-")
if err != nil {
log.Fatal(err)
}
opts.LocalPath = tmpDir
}
return newSequins(backend, opts)
}
func setupS3() *backend.S3Backend {
fakeS3, _ := s3test.NewServer(&s3test.Config{})
// cargo-culted from s3test
fakeRegion := aws.Region{
Name: "faux-region-1",
S3Endpoint: fakeS3.URL(),
S3LocationConstraint: true,
}
auth, _ := aws.GetAuth("foo", "bar", "", time.Time{})
bucket := s3.New(auth, fakeRegion).Bucket("sequinstest")
bucket.PutBucket("")
putFile(bucket, "test_data/0/part-00000")
putFile(bucket, "test_data/0/part-00001")
putFile(bucket, "test_data/0/_SUCCESS")
putFile(bucket, "test_data/1/part-00000")
putFile(bucket, "test_data/1/part-00001")
bucket.Put("test_data/foo", []byte("nothing"), "", "", s3.Options{})
return backend.NewS3Backend(bucket, "test_data")
}
func s3Put(config *BackupConfig, local, remote string, public bool) error {
localf, err := os.Open(local)
if err != nil {
return err
}
defer localf.Close()
localfi, err := localf.Stat()
if err != nil {
return err
}
auth := aws.Auth{AccessKey: config.AwsAccess, SecretKey: config.AwsSecret}
b := s3.New(auth, aws.USEast).Bucket(config.Bucket)
acl := s3.Private
if public {
acl = s3.PublicRead
}
contType := mime.TypeByExtension(path.Ext(local))
if contType == "" {
contType = "binary/octet-stream"
}
err = b.PutBucket(acl)
if err != nil {
return err
}
opts := s3.Options{}
return b.PutReader(remote, localf, localfi.Size(), contType, acl, opts)
}
func SyncFiles(filesToSync []FileToSync) ([]FileProcessed, error) {
filesProcessedChan := make(chan FileProcessed, len(filesToSync))
// From IAM or env
auth, err := GetAWSAuth()
if err != nil {
return []FileProcessed{}, err
}
region := aws.USWest
s3_conn := s3.New(auth, region)
for _, file := range filesToSync {
go func(file FileToSync) {
updated := syncFileFromS3(s3_conn, file)
fileProcessed := FileProcessed{updated, file}
filesProcessedChan <- fileProcessed
}(file)
}
var filesProcessed []FileProcessed
for i := 0; i < len(filesToSync); i++ {
fileProcessed := <-filesProcessedChan
filesProcessed = append(filesProcessed, fileProcessed)
}
return filesProcessed, nil
}
func (s *LocalServerSuite) SetUpSuite(c *gocheck.C) {
s.srv.SetUp(c)
s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)
// TODO Sadly the fake server ignores auth completely right now. :-(
s.clientTests.authIsBroken = true
s.clientTests.Cleanup()
}
func (s *AmazonClientSuite) SetUpSuite(c *check.C) {
if !testutil.Amazon {
c.Skip("live tests against AWS disabled (no -amazon)")
}
s.srv.SetUp(c)
s.s3 = s3.New(s.srv.auth, s.Region)
// In case tests were interrupted in the middle before.
s.ClientTests.Cleanup()
}
func GetBucket() *s3.Bucket {
auth, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
s := s3.New(auth, aws.USEast)
return s.Bucket("filmapp-development")
}
func (s *AmazonDomainClientSuite) SetUpSuite(c *check.C) {
if !testutil.Amazon {
c.Skip("live tests against AWS disabled (no -amazon)")
}
s.srv.SetUp(c)
region := s.Region
region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com"
s.s3 = s3.New(s.srv.auth, region)
s.ClientTests.Cleanup()
}
func (ss *S3Storage) Connect() error {
if ss.BucketName == "" {
return errors.New("S3 bucket name mandatory to establish a connection")
}
s3Conn := s3.New(ss.AwsAuth, ss.Region)
ss.connexion = s3Conn.Bucket(ss.BucketName)
return nil
}
func main() {
flag.Parse()
auth, err := aws.EnvAuth()
if err != nil {
log.Fatalln("Failed to recieve auth from env")
}
awsConnection := s3.New(
auth,
aws.USWest2,
)
bucket := awsConnection.Bucket(targetBucket)
info := gen.BuildInstanceInfo(&localInstanceFetcher{}, "basic_example", ".")
rotateCoordinator := gologging.NewRotateCoordinator(adjustedMaxLines, time.Hour*1)
logger, err := gologging.StartS3Logger(
rotateCoordinator,
info,
&stdoutNotifier{},
&uploader.S3UploaderBuilder{
Bucket: bucket,
KeyNameGenerator: &gen.EdgeKeyNameGenerator{info},
},
&stderrNotifier{},
5,
)
if err != nil {
log.Fatalf("Error building uploader: %s\n ", err)
}
i := 0
now := time.Now()
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
go func() {
<-sigc
logger.Close()
fmt.Printf("Produced %f rps\n", float64(i)/(float64(time.Now().Sub(now))/float64(time.Second)))
os.Exit(0)
}()
x := int(time.Second) / *rps
for ; i < MAX_LINES_PER_LOG*4; i++ {
// throttle - there is a better solution to this
defer func() {
if x := recover(); x != nil { // means we cuagh a signal
time.Sleep(120 * time.Second)
}
}()
time.Sleep(time.Duration(int(0.8 * float64(x))))
logger.Log("MOAR! %d", i)
}
logger.Close()
}
// create a new s3 client from the url
func newS3Client(config config.Config) (*s3.S3, error) {
auth, err := aws.GetAuth(config.AWS.AccessKeyID, config.AWS.SecretAccessKey, "", time.Now())
if err != nil {
return &s3.S3{}, err
}
if config.AWS.Region == "" {
return nil, errors.New("Region not set for S3 client lib (missing SetS3URL?)")
}
return s3.New(auth, aws.Regions[config.AWS.Region]), nil
}
func getS3(u *url.URL) (io.ReadCloser, error) {
if AWSAuth.AccessKey == "" || AWSRegion.Name == "" {
return nil, fmt.Errorf("Invalid AWS Auth or Region. Please check env AWS_CONFIG_FILE.")
}
client := s3.New(AWSAuth, AWSRegion)
bucket := client.Bucket(u.Host)
rc, err := bucket.GetReader(u.Path)
if err != nil {
return nil, err
}
return rc, nil
}
func GetBucket(bucketPrefix string) (*s3.Bucket, error) {
auth, err := aws.GetAuth("", "", "", time.Time{})
if err != nil {
return nil, err
}
s := s3.New(auth, aws.USWest2)
s.ConnectTimeout = time.Second * 30
s.ReadTimeout = time.Second * 30
bucketName := strings.TrimPrefix(bucketPrefix, "s3://") + "-" + environment.GetCloudEnv()
return s.Bucket(bucketName), nil
}
func (so *s3Options) Execute(args []string) error {
if err := app.CheckArity(1, 1, args); err != nil {
return err
}
repo := args[0]
s3 := s3.New(awsAuth, awsRegion)
buk := s3.Bucket(defBucket)
data, err := buk.Get("/binary/repos/repositories")
if err != nil {
return err
}
ts := &env.TagStore{}
err = json.Unmarshal(data, &ts)
id, err := ts.Lookup(repo)
if err != nil {
return err
}
dts, err := env.DefaultTagStore()
if err != nil {
return err
}
i := &Importer{tags: ts, sysTags: dts}
if !so.Force {
if i.alreadyExists(id) {
return fmt.Errorf("Already have %s, skipping download\n", utils.TruncateID(id))
}
}
fmt.Printf("Downloading %s (%s)\n", repo, utils.TruncateID(id))
err = i.download(buk, id)
dts.Flush()
return err
}
func (d *S3Downstream) Init() error {
u, err := url.Parse(d.downstreamURI)
if err != nil || u.Scheme != "s3" || u.User == nil {
log.Panic("Bad URL scheme ", d.downstreamURI)
}
username := u.User.Username()
password, _ := u.User.Password()
auth := aws.Auth{AccessKey: username, SecretKey: password}
log.Println("Init s3 connection using key ", auth.AccessKey, u.Host)
connection := s3.New(auth, aws.APSoutheast)
d.bucket = connection.Bucket(u.Host)
return nil
}
// create a new s3 client from the url
func newS3Client(config config.Config) (*s3.S3, error) {
auth, err := aws.GetAuth(config.AWS.AccessKeyID, config.AWS.SecretAccessKey, "", time.Now())
if err != nil {
return &s3.S3{}, err
}
var regionName string
regQuery := config.AWS.S3URL.Query()["region"]
if len(regQuery) > 0 && regQuery[0] != "" {
regionName = regQuery[0]
} else {
regionName = S3DefaultRegion
}
region := aws.Regions[regionName]
return s3.New(auth, region), nil
}
func listBackups() {
bucketName := *config.S3BackupBucket
dir := sanitizeDirForList(*config.S3BackupDir, bucketDelim)
auth := aws.Auth{AccessKey: *config.AwsAccess, SecretKey: *config.AwsSecret}
b := s3.New(auth, aws.USEast).Bucket(bucketName)
fmt.Printf("Listing files in %s\n", fullUrl(bucketName))
rsp, err := b.List(dir, bucketDelim, "", 1000)
if err != nil {
log.Fatalf("Invalid s3 backup: bucket.List failed %s\n", err)
}
//fmt.Printf("rsp: %v\n", rsp)
if 0 == len(rsp.Contents) {
fmt.Printf("There are no files in %s\n", fullUrl(*config.S3BackupBucket))
return
}
//fmt.Printf("Backup files in %s:\n", fullUrl(*config.S3BackupBucket))
for _, key := range rsp.Contents {
fmt.Printf(" %s %d\n", key.Key, key.Size)
}
}
func main() {
auth := aws.Auth{
AccessKey: "abc",
SecretKey: "123",
}
fakeRegion := aws.Region{
Name: "fakes3",
S3Endpoint: fmt.Sprintf("http://%s:%s", fakes3host, fakes3port),
}
s := s3.New(auth, fakeRegion)
bucket := s.Bucket(bucketname)
err := bucket.PutBucket(s3.BucketOwnerFull)
if err != nil {
panic(err.Error())
}
_, err = bucket.List("", "/", "", 20)
if err != nil {
panic(err.Error())
}
}
func listBlobFiles(config *BackupConfig, dir string) ([]s3.Key, error) {
auth := aws.Auth{AccessKey: config.AwsAccess, SecretKey: config.AwsSecret}
b := s3.New(auth, aws.USEast).Bucket(config.Bucket)
ret := make([]s3.Key, 0)
dir = sanitizeDirForList(dir, bucketDelim)
marker := ""
for {
// note: according to my tests, 1000 is max
if res, err := b.List(dir, "", marker, 1000); err != nil {
return nil, err
} else {
for _, k := range res.Contents {
ret = append(ret, k)
}
if !res.IsTruncated {
break
}
marker = res.Contents[len(res.Contents)-1].Key
}
}
return ret, nil
}
func (u *S3Uploader) Setup(destination string) error {
u.Destination = destination
// Setup the AWS authentication
auth, err := aws.EnvAuth()
if err != nil {
return errors.New("Error loading AWS credentials: " + err.Error())
}
// Decide what region to use
// https://github.com/crowdmob/goamz/blob/master/aws/regions.go
// I think S3 defaults to us-east-1
regionName := "us-east-1"
if os.Getenv("AWS_DEFAULT_REGION") != "" {
regionName = os.Getenv("AWS_DEFAULT_REGION")
}
// Check to make sure the region exists
region, ok := aws.Regions[regionName]
if ok == false {
return errors.New("Unknown AWS Region `" + regionName + "`")
}
// Find the bucket
s3 := s3.New(auth, region)
bucket := s3.Bucket(u.bucketName())
// If the list doesn't return an error, then we've got our
// bucket
_, err = bucket.List("", "", "", 0)
if err != nil {
return errors.New("Could not find bucket `" + u.bucketName() + " in region `" + region.Name + "` (" + err.Error() + ")")
}
u.Bucket = bucket
return nil
}
func (s *S) SetUpSuite(c *C) {
testServer.Start()
auth, _ := aws.GetAuth("abc", "123", "", time.Time{})
client := s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL})
tempDir, err := ioutil.TempDir("", "dogestry-test")
if err != nil {
c.Fatalf("couldn't get tempdir. Error: %s", err)
}
s.TempDir = tempDir
baseConfig, err := config.NewConfig(false)
if err != nil {
c.Fatalf("couldn't initialize config. Error: %s", err)
}
s.remote = &S3Remote{
config: baseConfig,
BucketName: "bucket",
client: client,
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
if AWS_ACCESS_KEY_ID == "" || AWS_SECRET_ACCESS_KEY == "" {
log.Fatal("AWS Credentials Required")
}
os.Setenv("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID)
os.Setenv("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY)
// Since we're not messing with cacheAge, it's just easier to treat is as a string.
if cacheAge == "" {
cacheAge = "0"
}
if maxObjs != 0 || stopMarker != "" {
// Set the conditional bit to check to stop
doStop = true
}
log.Println("Starting Cache Alterations:")
// Connect to AWS using goamz
auth, err := aws.EnvAuth()
if err != nil {
log.Panic(err.Error())
}
// Instantiate S3 Object
s := s3.New(auth, aws.USEast)
// Set the Bucket
Bucket := s.Bucket(bucketName)
// Initial Request - Outside Loop
Response, err := Bucket.List("", "", lastMarker, 1000)
if err != nil {
log.Panic(err.Error())
}
// Set up the header for iterating.
opts := s3.CopyOptions{}
opts.CacheControl = "max-age=" + cacheAge
opts.MetadataDirective = "REPLACE"
log.Println("-> 0 START")
// Loop Results
for _, v := range Response.Contents {
fmt.Printf(".") // Indicator that something is happening
_, err := Bucket.PutCopy(v.Key, s3.PublicRead, opts, bucketName+"/"+v.Key)
if err != nil {
log.Panic(err.Error())
}
// We generate our own lastMarker. This allows us to perform our own resume.
lastMarker = v.Key
results++
if doStop == true {
if results == maxObjs || lastMarker == stopMarker {
break // End here.
}
}
}
fmt.Printf("\n")
log.Println("->", results, " ", lastMarker)
// Did Amazon say there was more? If so, keep going.
if Response.IsTruncated == true {
for {
// Issue List Command
Response, err := Bucket.List("", "", lastMarker, 1000)
if err != nil {
panic(err.Error())
}
// Loop through Response and dump it to the console.
for _, v := range Response.Contents {
fmt.Printf(".") // Indicator that something is happening
_, err := Bucket.PutCopy(v.Key, s3.PublicRead, opts, bucketName+"/"+v.Key)
if err != nil {
log.Panic(err.Error())
}
lastMarker = v.Key
results++
if doStop == true {
if results == maxObjs || lastMarker == stopMarker {
break // End here.
}
}
}
if Response.IsTruncated == false {
break // End loop
} else {
fmt.Printf("\n")
log.Println("->", results, " ", lastMarker)
}
}
}
log.Println("Wrote to", results, " S3 Objects. Last object was:", lastMarker)
}
func (s *S) SetUpSuite(c *check.C) {
testServer.Start()
auth := aws.Auth{AccessKey: "abc", SecretKey: "123"}
s.s3 = s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL})
}
func main() {
flag.Parse()
stats, err := initStatsd(*stats_prefix, os.Getenv("STATSD_HOSTPORT"))
if err != nil {
log.Fatalf("Statsd configuration error: %v", err)
}
auth, err := aws.GetAuth("", "", "", time.Now())
if err != nil {
log.Fatalln("Failed to recieve auth from env")
}
awsConnection := s3.New(
auth,
aws.USWest2,
)
auditBucket := awsConnection.Bucket(auditBucketName + "-" + CLOUD_ENV)
auditBucket.PutBucket(s3.BucketOwnerFull)
eventBucket := awsConnection.Bucket(eventBucketName + "-" + CLOUD_ENV)
eventBucket.PutBucket(s3.BucketOwnerFull)
auditInfo := gen.BuildInstanceInfo(&gen.EnvInstanceFetcher{}, "spade_edge_audit", *logging_dir)
loggingInfo := gen.BuildInstanceInfo(&gen.EnvInstanceFetcher{}, "spade_edge", *logging_dir)
auditRotateCoordinator := gologging.NewRotateCoordinator(auditMaxLogLines, auditMaxLogAge)
loggingRotateCoordinator := gologging.NewRotateCoordinator(maxLogLines, maxLogAge)
auditLogger, err := gologging.StartS3Logger(
auditRotateCoordinator,
auditInfo,
&DummyNotifierHarness{},
&uploader.S3UploaderBuilder{
Bucket: auditBucket,
KeyNameGenerator: &gen.EdgeKeyNameGenerator{Info: auditInfo},
},
BuildSQSErrorHarness(),
2,
)
if err != nil {
log.Fatalf("Got Error while building audit: %s\n", err)
}
spadeEventLogger, err := gologging.StartS3Logger(
loggingRotateCoordinator,
loggingInfo,
BuildSQSNotifierHarness(),
&uploader.S3UploaderBuilder{
Bucket: eventBucket,
KeyNameGenerator: &gen.EdgeKeyNameGenerator{Info: loggingInfo},
},
BuildSQSErrorHarness(),
2,
)
if err != nil {
log.Fatalf("Got Error while building logger: %s\n", err)
}
// Initialize Loggers.
// AuditLogger writes to the audit log, for analysis of system success rate.
// SpadeLogger writes requests to a file for processing by the spade processor.
// K(afka)Logger writes produces messages for kafka, currently in dark launch.
// We allow the klogger to be null incase we boot up with an unresponsive kafka cluster.
var logger *request_handler.EventLoggers
brokerList := ParseBrokerList(*brokers)
klogger, err := kafka_logger.NewKafkaLogger(*clientId, brokerList)
if err == nil {
klogger.(*kafka_logger.KafkaLogger).Init()
logger = &request_handler.EventLoggers{
AuditLogger: auditLogger,
SpadeLogger: spadeEventLogger,
KLogger: klogger,
}
} else {
log.Printf("Got Error while building logger: %s + %v\nUsing Nop Logger\n", err, brokerList)
logger = &request_handler.EventLoggers{
AuditLogger: auditLogger,
SpadeLogger: spadeEventLogger,
KLogger: &request_handler.NoopLogger{},
}
}
// Trigger close on receipt of SIGINT
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGINT)
go func() {
<-sigc
// Cause flush
logger.Close()
os.Exit(0)
}()
hystrixStreamHandler := hystrix.NewStreamHandler()
hystrixStreamHandler.Start()
go http.ListenAndServe(net.JoinHostPort("", "81"), hystrixStreamHandler)
// setup server and listen
server := &http.Server{
Addr: *listen_port,
Handler: &request_handler.SpadeHandler{
StatLogger: stats,
EdgeLogger: logger,
Assigner: request_handler.Assigner,
},
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
MaxHeaderBytes: 1 << 20, // 0.5MB
}
if err := server.ListenAndServe(); err != nil {
log.Fatalln(err)
}
}
func pushToS3(bundlesPath string) error {
if _, err := os.Stat(bundlesPath); os.IsNotExist(err) {
return fmt.Errorf("This is awkward, the bundles path DNE: %s", bundlesPath)
}
// use env variables to connect to s3
auth, err := aws.EnvAuth()
if err != nil {
return fmt.Errorf("AWS Auth failed: %v", err)
}
// connect to s3 bucket
s := s3.New(auth, aws.GetRegion(region))
bucketname, bucketpath := bucketParts(bucket)
bucket := s.Bucket(bucketname)
//walk the bundles directory
var html string
walkFn := func(fpath string, info os.FileInfo, err error) error {
stat, err := os.Stat(fpath)
if err != nil {
return err
}
relFilePath, err := filepath.Rel(bundlesPath, fpath)
if err != nil || (fpath == bundlesPath && stat.IsDir()) {
// Error getting relative path OR we are looking
// at the root path. Skip in both situations.
return nil
}
if stat.IsDir() {
return nil
}
if err = uploadFileToS3(bucket, fpath, path.Join(bucketpath, relFilePath)); err != nil {
log.Warnf("Uploading %s to s3 failed: %v", fpath, err)
return err
}
// add to html
image := "default"
if strings.HasSuffix(relFilePath, ".sha256") || strings.HasSuffix(relFilePath, ".md5") {
image = "text"
}
html += fmt.Sprintf(`<tr>
<td valign="top"><a href="%s"><img src="/static/%s.png" alt="[ICO]"/></a></td>
<td><a href="%s">%s</a></td>
<td>%s</td>
<td>%s</td>
</tr>`, relFilePath, image, relFilePath, relFilePath, humanSize(stat.Size()), stat.ModTime().Format(time.RFC3339))
return nil
}
// walk the filepath
if err := filepath.Walk(bundlesPath, walkFn); err != nil {
return err
}
// add html to template
if err := createIndexFile(bucket, bucketpath, html); err != nil {
return err
}
return nil
}
func (s *S3) Run(env *tachyon.CommandEnv) (*tachyon.Result, error) {
auth, err := aws.GetAuth("", "", "", time.Time{})
if err != nil {
return nil, err
}
c := s3.New(auth, aws.USWest2)
b := c.Bucket(s.Bucket)
res := tachyon.NewResult(true)
res.Add("bucket", s.Bucket)
res.Add("remote", s.At)
if s.PutFile != "" {
path := env.Paths.File(s.PutFile)
f, err := os.Open(path)
if err != nil {
return nil, err
}
if f == nil {
return nil, fmt.Errorf("Unknown local file %s", s.PutFile)
}
defer f.Close()
var perm s3.ACL
if s.Public {
if s.Writable {
perm = s3.PublicReadWrite
} else {
perm = s3.PublicRead
}
} else {
perm = s3.Private
}
ct := s.ContentType
if ct == "" {
ct = "application/octet-stream"
}
fi, err := f.Stat()
if err != nil {
return nil, err
}
var (
input io.Reader
opts s3.Options
size int64
)
h := md5.New()
if s.GZip {
var buf bytes.Buffer
z := gzip.NewWriter(io.MultiWriter(h, &buf))
_, err = io.Copy(z, f)
if err != nil {
return nil, err
}
z.Close()
opts.ContentEncoding = "gzip"
input = &buf
size = int64(buf.Len())
} else {
input = io.TeeReader(f, h)
size = fi.Size()
}
err = b.PutReader(s.At, input, size, ct, perm, opts)
rep, err := b.Head(s.At, nil)
if err != nil {
return nil, err
}
localMD5 := hex.EncodeToString(h.Sum(nil))
res.Add("wrote", size)
res.Add("local", s.PutFile)
res.Add("md5", localMD5)
etag := rep.Header.Get("ETag")
if etag != "" {
etag = etag[1 : len(etag)-1]
if localMD5 != etag {
return nil, fmt.Errorf("corruption uploading file detected")
}
}
} else if s.GetFile != "" {
f, err := os.OpenFile(s.GetFile, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, err
}
defer f.Close()
i, err := b.GetReader(s.At)
if err != nil {
return nil, err
}
defer i.Close()
n, err := io.Copy(f, i)
if err != nil {
return nil, err
}
res.Add("read", n)
res.Add("local", s.GetFile)
} else {
return nil, fmt.Errorf("Specify put_file or get_file")
}
return res, nil
}