// Put puts a serialize.Msgpack object into bucket b.
// by default uploads checksum of the files
func Put(b *s3.Bucket, data []byte, path string, checksum string) error {
// data is a log of msgpack file kind
// aws api wants []byte
options := s3.Options{}
options.ContentMD5 = checksum
err := b.Put(path, data, "", s3.Private, options)
if err != nil {
return err
}
return nil
}
func processDir(dirName string, s3KeyPrefix string, bucket *s3.Bucket) {
if verbose {
log.Printf("Processing directory %s", dirName)
}
fileInfos, err := ioutil.ReadDir(dirName)
if err != nil {
log.Fatal(err)
}
for _, fileInfo := range fileInfos {
if time.Now().After(stopTime) {
log.Fatal("Timeout limit reached")
}
filePath := path.Join(dirName, fileInfo.Name())
// Ignore symlinks for now.
// TODO: add option to follow symlinks
if (fileInfo.Mode() & os.ModeSymlink) != 0 {
continue
}
if fileInfo.IsDir() {
if shouldRecurseInto(fileInfo.Name()) {
subDirName := path.Join(dirName, fileInfo.Name())
processDir(subDirName, s3KeyPrefix+fileInfo.Name()+"/", bucket)
}
continue
}
if ignoreNames[fileInfo.Name()] != "" {
continue
}
s3Key := s3KeyPrefix + fileInfo.Name()
putRequired := false
var data []byte
s3ETag := s3Objects[s3Key]
if s3ETag == "" {
if verbose {
log.Printf("Not found in S3 bucket: %s", s3Key)
}
putRequired = true
}
data, err := ioutil.ReadFile(filePath)
if err != nil {
log.Fatal(err)
}
// if the object exists, then we check the MD5 of the file to determine whether
// the file needs to be uploaded
if !putRequired {
digest := md5.Sum(data)
// note the need to convert digest to a slice because it is a byte array ([16]byte)
fileETag := "\"" + hex.EncodeToString(digest[:]) + "\""
if fileETag != s3ETag {
if verbose {
log.Printf("Need to upload %s: expected ETag = %s, actual = %s", filePath, fileETag, s3ETag)
}
putRequired = true
}
}
if putRequired {
// TODO: this should be configurable, but for now if the mime-type cannot
// be determined, do not upload
contentType := mime.TypeByExtension(path.Ext(strings.ToLower(fileInfo.Name())))
if contentType == "" && includeUnknownMimeTypes {
contentType = "application/octet-stream"
}
if contentType != "" {
err = bucket.Put(s3Key, data, contentType, s3.Private, s3.Options{})
if err != nil {
log.Fatal(err)
}
log.Printf("Uploaded %s\n", s3Key)
}
} else {
if verbose {
log.Printf("Identical file, no upload required: %s", filePath)
}
}
}
}