func uploadFileToS3(bucket *s3.Bucket, fpath, s3path string) error {
// try to get the mime type
mimetype := ""
err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Debugf("Magic meme failed for: %v", err)
} else {
mimetype, err = magicmime.TypeByFile(fpath)
if err != nil {
log.Debugf("Mime type detection for %s failed: %v", fpath, err)
}
}
contents, err := ioutil.ReadFile(fpath)
if err != nil {
log.Warnf("Reading %q failed: %v", fpath, err)
}
// push the file to s3
log.Debugf("Pushing %s to s3", s3path)
if err := bucket.Put(s3path, contents, mimetype, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
return err
}
log.Infof("Sucessfully pushed %s to s3", s3path)
return nil
}
func walkFn(path string, fi os.FileInfo, err error) error {
if fi.IsDir() {
if recursive {
return nil
} else {
if topLevel {
topLevel = false
return nil
} else {
return filepath.SkipDir
}
}
}
mimetype, err := magicmime.TypeByFile(path)
if err != nil {
warning.Println("mimetype error (skipping "+path+"):", err)
return nil
}
if strings.Contains(mimetype, mimetypePattern) {
info.Println("Adding " + path + " (" + mimetype + ")")
wg.Add(1)
c <- path
return nil
} else {
info.Println("Skipping " + path + " (" + mimetype + ")")
return nil
}
}
func Example_typeByFile() {
if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
log.Fatal(err)
}
defer magicmime.Close()
mimetype, err := magicmime.TypeByFile("/path/to/file")
if err != nil {
log.Fatalf("error occured during type lookup: %v", err)
}
log.Printf("mime-type: %v", mimetype)
}
// Go-routine which serves magicmime requests because libmagic is not thread
// safe.
func magicMimeWorker() {
err := magicmime.Open(magicmime.MAGIC_MIME_TYPE |
magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Fatalln("libmagic initialization failure", err.Error())
}
defer magicmime.Close()
// Listen
for filePath := range mimeQueryCh {
mimetype, err := magicmime.TypeByFile(filePath)
mimeResponseCh <- mimeResponse{mimetype, err}
}
}
func GuessMimeType(path string) (mimeType string, err error) {
// Get the mime type of the file. In some cases, MagicMime
// returns an empty string, and in rare cases (about 1 in 10000),
// it returns unprintable characters. These are not valid mime
// types and cause ingest to fail. So we default to the safe
// text/plain and then set the MimeType only if
// MagicMime returned something that looks legit.
// Open the Mime Magic DB only once.
mimeType = "text/plain"
mutex.Lock()
guessedType, _ := magicmime.TypeByFile(path)
mutex.Unlock()
if guessedType != "" && validMimeType.MatchString(guessedType) {
mimeType = guessedType
}
return mimeType, nil
}
func walkFn(path string, fi os.FileInfo, err error) error {
if fi.IsDir() {
if options.Recursive {
return nil
} else {
if topLevel {
topLevel = false
return nil
} else {
return filepath.SkipDir
}
}
}
if resume {
_, already_processed := processed[path]
if already_processed {
wg.Add(1)
info.Printf("Skipping sample %s, because it was already uploaded successfully\n", path)
logC <- path + "\t200\n"
return nil
}
}
mimetype, err := magicmime.TypeByFile(path)
if err != nil {
warning.Println("mimetype error (skipping "+path+"):", err)
return nil
}
if strings.Contains(mimetype, options.MimetypePattern) {
info.Println("Adding " + path + " (" + mimetype + ")")
wg.Add(1)
c <- path
return nil
} else {
info.Println("Skipping " + path + " (" + mimetype + ")")
return nil
}
}
func Upload(path string, maxMemory int64, w http.ResponseWriter, r *http.Request) (Image, error) {
image := Image{}
errRes := ersp.New(w, r)
r.Body = http.MaxBytesReader(w, r.Body, maxMemory)
file, handler, err := r.FormFile("image")
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
defer file.Close()
randName, err := str.Rand(32)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
fileName := fmt.Sprintf("%s_%s", randName, handler.Filename)
filePath := fmt.Sprintf("%s/%s", path, fileName)
f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
defer f.Close()
_, err = io.Copy(f, file)
if err != nil {
return image, err
}
err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
mimetype, err := magicmime.TypeByFile(filePath)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
if ok := imageMime.MatchString(mimetype); !ok {
errRes.Field("image", "must be valid image")
errRes.Send(http.StatusBadRequest)
os.Remove(filePath)
return image, err
}
stat, err := f.Stat()
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
image.Name = fileName
image.OriginalName = handler.Filename
image.Stat = stat
return image, nil
}
/**
* Upload a file to ./public/ - with a short-name.
*
* Each file will also have a ".meta" file created, to contain
* some content.
*/
func UploadHandler(res http.ResponseWriter, req *http.Request) {
var (
status int
err error
)
defer func() {
if nil != err {
http.Error(res, err.Error(), status)
}
}()
/**
* Get the authentication-header. If missing we abort.
*/
auth := string(req.Header.Get("X-Auth"))
if len(auth) < 1 {
status = 401
err = errors.New("Missing X-Auth header")
return
}
auth = strings.TrimSpace(auth)
/**
* Load the secret.
*/
state, err := LoadState()
if err != nil {
status = 500
err = errors.New("Loading state failed")
return
}
/**
* Test the token.
*/
otpc := &dgoogauth.OTPConfig{
Secret: state.Secret,
WindowSize: 3,
HotpCounter: 0,
}
val, err := otpc.Authenticate(auth)
if err != nil {
status = 401
err = errors.New("Failed to use X-Auth header")
return
}
/**
* If it failed then we're done.
*/
if !val {
status = 401
err = errors.New("Invalid X-Auth header")
return
}
/**
** At ths point we know we have an authorized submitter.
**/
/**
* Parse the incoming request
*/
const _24K = (1 << 20) * 24
if err = req.ParseMultipartForm(_24K); nil != err {
status = http.StatusInternalServerError
return
}
/**
* Get the short-ID
*/
sn := NextShortID()
for _, fheaders := range req.MultipartForm.File {
for _, hdr := range fheaders {
// open uploaded
var infile multipart.File
if infile, err = hdr.Open(); nil != err {
status = http.StatusInternalServerError
return
}
// open destination
var outfile *os.File
if outfile, err = os.Create("./public/" + sn); nil != err {
status = http.StatusInternalServerError
return
}
// 32K buffer copy
if _, err = io.Copy(outfile, infile); nil != err {
status = http.StatusInternalServerError
return
}
// Get the MIME-type of the uploaded file.
err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
status = http.StatusInternalServerError
return
}
defer magicmime.Close()
mimetype, _ := magicmime.TypeByFile("./public/" + sn)
//
// Write out the meta-data - which is a structure
// containing the following members.
//
md := &UploadMetaData{MIME: mimetype, IP: getRemoteIP(req), AT: time.Now().Format(time.RFC850)}
data_json, _ := json.Marshal(md)
var meta *os.File
defer meta.Close()
if meta, err = os.Create("./public/" + sn + ".meta"); nil != err {
status = http.StatusInternalServerError
return
}
meta.WriteString(string(data_json)) //mimetype)
//
// Write out the redirection - using the host
// scheme, and the end-point of the new upload.
//
hostname := req.Host
scheme := "http"
if strings.HasPrefix(req.Proto, "HTTPS") {
scheme = "https"
}
if req.Header.Get("X-Forwarded-Proto") == "https" {
scheme = "https"
}
res.Write([]byte(scheme + "://" + hostname + "/get/" + sn + "\n"))
}
}
}