Exemplo n.º 1
0
func uploadFileToS3(bucket *s3.Bucket, fpath, s3path string) error {
	// try to get the mime type
	mimetype := ""
	err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Debugf("Magic meme failed for: %v", err)
	} else {
		mimetype, err = magicmime.TypeByFile(fpath)
		if err != nil {
			log.Debugf("Mime type detection for %s failed: %v", fpath, err)
		}
	}

	contents, err := ioutil.ReadFile(fpath)
	if err != nil {
		log.Warnf("Reading %q failed: %v", fpath, err)
	}

	// push the file to s3
	log.Debugf("Pushing %s to s3", s3path)
	if err := bucket.Put(s3path, contents, mimetype, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
		return err
	}
	log.Infof("Sucessfully pushed %s to s3", s3path)
	return nil
}
Exemplo n.º 2
0
func walkFn(path string, fi os.FileInfo, err error) error {
	if fi.IsDir() {
		if recursive {
			return nil
		} else {
			if topLevel {
				topLevel = false
				return nil
			} else {
				return filepath.SkipDir
			}
		}
	}
	mimetype, err := magicmime.TypeByFile(path)
	if err != nil {
		warning.Println("mimetype error (skipping "+path+"):", err)
		return nil
	}
	if strings.Contains(mimetype, mimetypePattern) {
		info.Println("Adding " + path + " (" + mimetype + ")")
		wg.Add(1)
		c <- path
		return nil
	} else {
		info.Println("Skipping " + path + " (" + mimetype + ")")
		return nil
	}
}
Exemplo n.º 3
0
func Example_typeByFile() {
	if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
		log.Fatal(err)
	}
	defer magicmime.Close()

	mimetype, err := magicmime.TypeByFile("/path/to/file")
	if err != nil {
		log.Fatalf("error occured during type lookup: %v", err)
	}

	log.Printf("mime-type: %v", mimetype)
}
Exemplo n.º 4
0
// Go-routine which serves magicmime requests because libmagic is not thread
// safe.
func magicMimeWorker() {
	err := magicmime.Open(magicmime.MAGIC_MIME_TYPE |
		magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Fatalln("libmagic initialization failure", err.Error())
	}
	defer magicmime.Close()

	// Listen
	for filePath := range mimeQueryCh {
		mimetype, err := magicmime.TypeByFile(filePath)
		mimeResponseCh <- mimeResponse{mimetype, err}
	}
}
Exemplo n.º 5
0
func GuessMimeType(path string) (mimeType string, err error) {
	// Get the mime type of the file. In some cases, MagicMime
	// returns an empty string, and in rare cases (about 1 in 10000),
	// it returns unprintable characters. These are not valid mime
	// types and cause ingest to fail. So we default to the safe
	// text/plain and then set the MimeType only if
	// MagicMime returned something that looks legit.
	// Open the Mime Magic DB only once.
	mimeType = "text/plain"
	mutex.Lock()
	guessedType, _ := magicmime.TypeByFile(path)
	mutex.Unlock()
	if guessedType != "" && validMimeType.MatchString(guessedType) {
		mimeType = guessedType
	}
	return mimeType, nil
}
func walkFn(path string, fi os.FileInfo, err error) error {
	if fi.IsDir() {
		if options.Recursive {
			return nil
		} else {
			if topLevel {
				topLevel = false
				return nil
			} else {
				return filepath.SkipDir
			}
		}
	}
	if resume {
		_, already_processed := processed[path]
		if already_processed {
			wg.Add(1)
			info.Printf("Skipping sample %s, because it was already uploaded successfully\n", path)
			logC <- path + "\t200\n"
			return nil
		}
	}

	mimetype, err := magicmime.TypeByFile(path)
	if err != nil {
		warning.Println("mimetype error (skipping "+path+"):", err)
		return nil
	}
	if strings.Contains(mimetype, options.MimetypePattern) {
		info.Println("Adding " + path + " (" + mimetype + ")")
		wg.Add(1)
		c <- path
		return nil
	} else {
		info.Println("Skipping " + path + " (" + mimetype + ")")
		return nil
	}
}
Exemplo n.º 7
0
func Upload(path string, maxMemory int64, w http.ResponseWriter, r *http.Request) (Image, error) {
	image := Image{}
	errRes := ersp.New(w, r)
	r.Body = http.MaxBytesReader(w, r.Body, maxMemory)

	file, handler, err := r.FormFile("image")
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}
	defer file.Close()

	randName, err := str.Rand(32)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	fileName := fmt.Sprintf("%s_%s", randName, handler.Filename)
	filePath := fmt.Sprintf("%s/%s", path, fileName)

	f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}
	defer f.Close()

	_, err = io.Copy(f, file)
	if err != nil {
		return image, err
	}

	err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	mimetype, err := magicmime.TypeByFile(filePath)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	if ok := imageMime.MatchString(mimetype); !ok {
		errRes.Field("image", "must be valid image")
		errRes.Send(http.StatusBadRequest)
		os.Remove(filePath)
		return image, err
	}

	stat, err := f.Stat()
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	image.Name = fileName
	image.OriginalName = handler.Filename
	image.Stat = stat

	return image, nil
}
Exemplo n.º 8
0
/**
 * Upload a file to ./public/ - with a short-name.
 *
 * Each file will also have a ".meta" file created, to contain
 * some content.
 */
func UploadHandler(res http.ResponseWriter, req *http.Request) {
	var (
		status int
		err    error
	)
	defer func() {
		if nil != err {
			http.Error(res, err.Error(), status)
		}
	}()

	/**
	 * Get the authentication-header.  If missing we abort.
	 */
	auth := string(req.Header.Get("X-Auth"))
	if len(auth) < 1 {
		status = 401
		err = errors.New("Missing X-Auth header")
		return
	}
	auth = strings.TrimSpace(auth)

	/**
	 * Load the secret.
	 */
	state, err := LoadState()
	if err != nil {
		status = 500
		err = errors.New("Loading state failed")
		return
	}

	/**
	 * Test the token.
	 */
	otpc := &dgoogauth.OTPConfig{
		Secret:      state.Secret,
		WindowSize:  3,
		HotpCounter: 0,
	}

	val, err := otpc.Authenticate(auth)
	if err != nil {
		status = 401
		err = errors.New("Failed to use X-Auth header")
		return
	}

	/**
	 * If it failed then we're done.
	 */
	if !val {
		status = 401
		err = errors.New("Invalid X-Auth header")
		return
	}

	/**
	 ** At ths point we know we have an authorized submitter.
	 **/

	/**
	 * Parse the incoming request
	 */
	const _24K = (1 << 20) * 24
	if err = req.ParseMultipartForm(_24K); nil != err {
		status = http.StatusInternalServerError
		return
	}

	/**
	 * Get the short-ID
	 */
	sn := NextShortID()

	for _, fheaders := range req.MultipartForm.File {
		for _, hdr := range fheaders {
			// open uploaded
			var infile multipart.File
			if infile, err = hdr.Open(); nil != err {
				status = http.StatusInternalServerError
				return
			}
			// open destination
			var outfile *os.File
			if outfile, err = os.Create("./public/" + sn); nil != err {
				status = http.StatusInternalServerError
				return
			}
			// 32K buffer copy
			if _, err = io.Copy(outfile, infile); nil != err {
				status = http.StatusInternalServerError
				return
			}

			// Get the MIME-type of the uploaded file.
			err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
			if err != nil {
				status = http.StatusInternalServerError
				return
			}
			defer magicmime.Close()
			mimetype, _ := magicmime.TypeByFile("./public/" + sn)

			//
			// Write out the meta-data - which is a structure
			// containing the following members.
			//
			md := &UploadMetaData{MIME: mimetype, IP: getRemoteIP(req), AT: time.Now().Format(time.RFC850)}
			data_json, _ := json.Marshal(md)

			var meta *os.File
			defer meta.Close()
			if meta, err = os.Create("./public/" + sn + ".meta"); nil != err {
				status = http.StatusInternalServerError
				return
			}
			meta.WriteString(string(data_json)) //mimetype)

			//
			// Write out the redirection - using the host
			// scheme, and the end-point of the new upload.
			//
			hostname := req.Host
			scheme := "http"

			if strings.HasPrefix(req.Proto, "HTTPS") {
				scheme = "https"
			}
			if req.Header.Get("X-Forwarded-Proto") == "https" {
				scheme = "https"
			}

			res.Write([]byte(scheme + "://" + hostname + "/get/" + sn + "\n"))
		}
	}
}