func uploadFileToS3(bucket *s3.Bucket, fpath, s3path string) error {
// try to get the mime type
mimetype := ""
err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Debugf("Magic meme failed for: %v", err)
} else {
mimetype, err = magicmime.TypeByFile(fpath)
if err != nil {
log.Debugf("Mime type detection for %s failed: %v", fpath, err)
}
}
contents, err := ioutil.ReadFile(fpath)
if err != nil {
log.Warnf("Reading %q failed: %v", fpath, err)
}
// push the file to s3
log.Debugf("Pushing %s to s3", s3path)
if err := bucket.Put(s3path, contents, mimetype, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
return err
}
log.Infof("Sucessfully pushed %s to s3", s3path)
return nil
}
func createIndexFile(bucket *s3.Bucket, bucketpath, html string) error {
p := path.Join(bucketpath, "index.html")
contents := strings.Replace(index, "{{ . }}", html, -1)
// push the file to s3
log.Debugf("Pushing %s to s3", p)
if err := bucket.Put(p, []byte(contents), "text/html", "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
return err
}
log.Infof("Sucessfully pushed %s to s3", p)
return nil
}
func CreateManifestInBucket(batch *metadata.LoadBatch, bucket *s3.Bucket) (string, error) {
manifest, err := makeManifestJson(batch)
if err != nil {
return "", err
}
url := manifestUrl(bucket.Name, batch.UUID)
err = bucket.Put(batch.UUID+".json", manifest, "application/json", s3.BucketOwnerRead, s3.Options{})
if err != nil {
return "", err
}
return url, err
}
func uploadFileToS3(bucket *s3.Bucket, fpath, s3path, contentType string) error {
contents, err := ioutil.ReadFile(fpath)
if err != nil {
return fmt.Errorf("Reading %q failed: %v", fpath, err)
}
// push the file to s3
logrus.Debugf("Pushing %s to s3", s3path)
if err := bucket.Put(s3path, contents, contentType, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
return err
}
logrus.Infof("Sucessfully pushed %s to s3", s3path)
return nil
}
func (chunkBuffer *ChunkBuffer) StoreToS3AndRelease(s3bucket *s3.Bucket) (bool, error) {
var s3path string
var err error
if debug {
fmt.Printf("Closing bufferfile: %s\n", chunkBuffer.File.Name())
}
chunkBuffer.File.Close()
contents, err := ioutil.ReadFile(chunkBuffer.File.Name())
if err != nil {
return false, err
}
if len(contents) <= 0 {
if debug {
fmt.Printf("Nothing to store to s3 for bufferfile: %s\n", chunkBuffer.File.Name())
}
} else { // Write to s3 in a new filename
alreadyExists := true
for alreadyExists {
writeTime := time.Now()
s3path = fmt.Sprintf("%s%s%d", S3TopicPartitionPrefix(chunkBuffer.Topic, chunkBuffer.Partition), S3DatePrefix(&writeTime), writeTime.UnixNano())
alreadyExists, err = s3bucket.Exists(s3path)
if err != nil {
panic(err)
return false, err
}
}
fmt.Printf("S3 Put Object: { Bucket: %s, Key: %s, MimeType:%s }\n", s3bucket.Name, s3path, mime.TypeByExtension(filepath.Ext(chunkBuffer.File.Name())))
err = s3bucket.Put(s3path, contents, mime.TypeByExtension(filepath.Ext(chunkBuffer.File.Name())), s3.Private, s3.Options{})
if err != nil {
panic(err)
}
}
if !keepBufferFiles {
if debug {
fmt.Printf("Deleting bufferfile: %s\n", chunkBuffer.File.Name())
}
err = os.Remove(chunkBuffer.File.Name())
if err != nil {
fmt.Errorf("Error deleting bufferfile %s: %#v", chunkBuffer.File.Name(), err)
}
}
return true, nil
}
func putLog(t transfer, bucket *s3.Bucket, dry bool) {
data, err := ioutil.ReadFile(t.Src)
if err != nil {
// Error reading log
log.Printf("Error reading source file %s:\n", t.Src)
log.Fatal(err)
}
contType := "binary/octet-stream"
perm := s3.ACL("private")
if dry {
log.Printf("Starting sync of %s to bucket path %s...\n", t.Src, t.Dest)
} else {
log.Printf("Starting sync of %s to s3://%s/%s...\n", t.Src, bucket.Name, t.Dest)
err = bucket.Put(t.Dest, data, contType, perm, s3.Options{})
if err != nil {
// Error uploading log to s3
log.Printf("Sync of %s to s3://%s/%s failed:\n", t.Src, bucket.Name, t.Dest)
log.Fatal(err)
}
}
}
func putFile(bucket *s3.Bucket, src string) {
bytes, _ := ioutil.ReadFile(src)
bucket.Put(src, bytes, "", "", s3.Options{})
}