func uploadFileToS3(bucket *s3.Bucket, fpath, s3path string) error {
// try to get the mime type
mimetype := ""
err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Debugf("Magic meme failed for: %v", err)
} else {
mimetype, err = magicmime.TypeByFile(fpath)
if err != nil {
log.Debugf("Mime type detection for %s failed: %v", fpath, err)
}
}
contents, err := ioutil.ReadFile(fpath)
if err != nil {
log.Warnf("Reading %q failed: %v", fpath, err)
}
// push the file to s3
log.Debugf("Pushing %s to s3", s3path)
if err := bucket.Put(s3path, contents, mimetype, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
return err
}
log.Infof("Sucessfully pushed %s to s3", s3path)
return nil
}
// getMimeFromMagic accepts a sample and counter and then tries to determin the
// mime type of the file. If a panic occures in an external library the function
// will recover and try to get the mime type up to three times before returning
// "N/A" as mime type.
func getMimeFromMagic(fileBytes []byte, try int) (mimeType string, err error) {
defer func() {
magicmime.Close()
mimeLock.Unlock()
if err := recover(); err != nil {
warning.Println("magicMime paniced")
time.Sleep(time.Second)
mimeType, err = getMimeFromMagic(fileBytes, try+1)
}
}()
// if we tried to get the mimeType 3 times but paniced we'll return a
// static string
if try >= 3 {
err = nil
mimeType = "N/A"
return
}
mimeLock.Lock()
err = magicmime.Open(magicmime.MAGIC_ERROR)
if err != nil {
err = errors.New("ExtendedMime is activated but libmagic is not installed!")
return
}
mimeType, err = magicmime.TypeByBuffer(fileBytes)
return
}
func main_object() {
info.Println("Uploading objects...")
c = make(chan string)
for i := 0; i < numWorkers; i++ {
debug.Printf("Starting worker #%d\n", i)
go worker()
}
if options.FPath != "" {
file, err := os.Open(options.FPath)
if err != nil {
warning.Println("Couln't open file containing sample list!", err.Error())
return
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
// line by line
for scanner.Scan() {
sample := scanner.Text()
wg.Add(1)
if resume {
_, already_processed := processed[sample]
if already_processed {
info.Printf("Skipping sample %s, because it was already uploaded successfully\n", sample)
logC <- sample + "\t200\n"
continue
}
}
c <- sample
//go copySample(scanner.Text())
}
}
if options.Directory != "" {
magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
defer magicmime.Close()
fullPath, err := filepath.Abs(options.Directory)
if err != nil {
warning.Println("path error:", err)
return
}
topLevel = true
err = filepath.Walk(fullPath, walkFn)
if err != nil {
warning.Println("walk error:", err)
return
}
}
wg.Wait()
}
// GetMimetype gets the mimetype of a data []byte
func GetMimetype(data []byte) (string, error) {
if err := magicmime.Open(magicmime.MAGIC_NO_CHECK_COMPRESS); err != nil {
return "", err
}
defer magicmime.Close()
mimetype, err := magicmime.TypeByBuffer(data)
if err != nil {
return "", err
}
return mimetype, nil
}
func Example_typeByFile() {
if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
log.Fatal(err)
}
defer magicmime.Close()
mimetype, err := magicmime.TypeByFile("/path/to/file")
if err != nil {
log.Fatalf("error occured during type lookup: %v", err)
}
log.Printf("mime-type: %v", mimetype)
}
// Go-routine which serves magicmime requests because libmagic is not thread
// safe.
func magicMimeWorker() {
err := magicmime.Open(magicmime.MAGIC_MIME_TYPE |
magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Fatalln("libmagic initialization failure", err.Error())
}
defer magicmime.Close()
// Listen
for filePath := range mimeQueryCh {
mimetype, err := magicmime.TypeByFile(filePath)
mimeResponseCh <- mimeResponse{mimetype, err}
}
}
func main_object() {
info.Println("Uploading objects...")
c = make(chan string)
for i := 0; i < numWorkers; i++ {
debug.Printf("Starting worker #%d\n", i)
go worker()
}
if fPath != "" {
file, err := os.Open(fPath)
if err != nil {
warning.Println("Couln't open file containing sample list!", err.Error())
return
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
// line by line
for scanner.Scan() {
wg.Add(1)
c <- scanner.Text()
//go copySample(scanner.Text())
}
}
if directory != "" {
magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
defer magicmime.Close()
fullPath, err := filepath.Abs(directory)
if err != nil {
warning.Println("path error:", err)
return
}
topLevel = true
err = filepath.Walk(fullPath, walkFn)
if err != nil {
warning.Println("walk error:", err)
return
}
}
wg.Wait()
}
func init() {
// add missing extensions
for k, v := range mimeRdfExt {
mime.AddExtensionType(k, v)
}
for _, syntax := range crdf.ParserSyntax {
switch syntax.MimeType {
case "", "text/html":
continue
}
mimeParser[syntax.MimeType] = syntax.Name
}
mimeParser["text/n3"] = mimeParser["text/turtle"]
for name, syntax := range crdf.SerializerSyntax {
switch name {
case "json-triples":
// only activate: json
continue
case "rdfxml-xmp", "rdfxml":
// only activate: rdfxml-abbrev
continue
}
mimeSerializer[syntax.MimeType] = syntax.Name
}
for mime := range mimeSerializer {
switch mime {
case "application/xhtml+xml":
continue
}
serializerMimes = append(serializerMimes, mime)
}
magicmime.Open(magicmime.MAGIC_MIME_TYPE)
}
func Upload(path string, maxMemory int64, w http.ResponseWriter, r *http.Request) (Image, error) {
image := Image{}
errRes := ersp.New(w, r)
r.Body = http.MaxBytesReader(w, r.Body, maxMemory)
file, handler, err := r.FormFile("image")
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
defer file.Close()
randName, err := str.Rand(32)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
fileName := fmt.Sprintf("%s_%s", randName, handler.Filename)
filePath := fmt.Sprintf("%s/%s", path, fileName)
f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
defer f.Close()
_, err = io.Copy(f, file)
if err != nil {
return image, err
}
err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
mimetype, err := magicmime.TypeByFile(filePath)
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
if ok := imageMime.MatchString(mimetype); !ok {
errRes.Field("image", "must be valid image")
errRes.Send(http.StatusBadRequest)
os.Remove(filePath)
return image, err
}
stat, err := f.Stat()
if err != nil {
log.Println(err)
w.WriteHeader(http.StatusInternalServerError)
return image, err
}
image.Name = fileName
image.OriginalName = handler.Filename
image.Stat = stat
return image, nil
}
/**
* Upload a file to ./public/ - with a short-name.
*
* Each file will also have a ".meta" file created, to contain
* some content.
*/
func UploadHandler(res http.ResponseWriter, req *http.Request) {
var (
status int
err error
)
defer func() {
if nil != err {
http.Error(res, err.Error(), status)
}
}()
/**
* Get the authentication-header. If missing we abort.
*/
auth := string(req.Header.Get("X-Auth"))
if len(auth) < 1 {
status = 401
err = errors.New("Missing X-Auth header")
return
}
auth = strings.TrimSpace(auth)
/**
* Load the secret.
*/
state, err := LoadState()
if err != nil {
status = 500
err = errors.New("Loading state failed")
return
}
/**
* Test the token.
*/
otpc := &dgoogauth.OTPConfig{
Secret: state.Secret,
WindowSize: 3,
HotpCounter: 0,
}
val, err := otpc.Authenticate(auth)
if err != nil {
status = 401
err = errors.New("Failed to use X-Auth header")
return
}
/**
* If it failed then we're done.
*/
if !val {
status = 401
err = errors.New("Invalid X-Auth header")
return
}
/**
** At ths point we know we have an authorized submitter.
**/
/**
* Parse the incoming request
*/
const _24K = (1 << 20) * 24
if err = req.ParseMultipartForm(_24K); nil != err {
status = http.StatusInternalServerError
return
}
/**
* Get the short-ID
*/
sn := NextShortID()
for _, fheaders := range req.MultipartForm.File {
for _, hdr := range fheaders {
// open uploaded
var infile multipart.File
if infile, err = hdr.Open(); nil != err {
status = http.StatusInternalServerError
return
}
// open destination
var outfile *os.File
if outfile, err = os.Create("./public/" + sn); nil != err {
status = http.StatusInternalServerError
return
}
// 32K buffer copy
if _, err = io.Copy(outfile, infile); nil != err {
status = http.StatusInternalServerError
return
}
// Get the MIME-type of the uploaded file.
err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
if err != nil {
status = http.StatusInternalServerError
return
}
defer magicmime.Close()
mimetype, _ := magicmime.TypeByFile("./public/" + sn)
//
// Write out the meta-data - which is a structure
// containing the following members.
//
md := &UploadMetaData{MIME: mimetype, IP: getRemoteIP(req), AT: time.Now().Format(time.RFC850)}
data_json, _ := json.Marshal(md)
var meta *os.File
defer meta.Close()
if meta, err = os.Create("./public/" + sn + ".meta"); nil != err {
status = http.StatusInternalServerError
return
}
meta.WriteString(string(data_json)) //mimetype)
//
// Write out the redirection - using the host
// scheme, and the end-point of the new upload.
//
hostname := req.Host
scheme := "http"
if strings.HasPrefix(req.Proto, "HTTPS") {
scheme = "https"
}
if req.Header.Get("X-Forwarded-Proto") == "https" {
scheme = "https"
}
res.Write([]byte(scheme + "://" + hostname + "/get/" + sn + "\n"))
}
}
}