Exemplo n.º 1
0
func uploadFileToS3(bucket *s3.Bucket, fpath, s3path string) error {
	// try to get the mime type
	mimetype := ""
	err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Debugf("Magic meme failed for: %v", err)
	} else {
		mimetype, err = magicmime.TypeByFile(fpath)
		if err != nil {
			log.Debugf("Mime type detection for %s failed: %v", fpath, err)
		}
	}

	contents, err := ioutil.ReadFile(fpath)
	if err != nil {
		log.Warnf("Reading %q failed: %v", fpath, err)
	}

	// push the file to s3
	log.Debugf("Pushing %s to s3", s3path)
	if err := bucket.Put(s3path, contents, mimetype, "public-read", s3.Options{CacheControl: "no-cache"}); err != nil {
		return err
	}
	log.Infof("Sucessfully pushed %s to s3", s3path)
	return nil
}
Exemplo n.º 2
0
// getMimeFromMagic accepts a sample and counter and then tries to determin the
// mime type of the file. If a panic occures in an external library the function
// will recover and try to get the mime type up to three times before returning
// "N/A" as mime type.
func getMimeFromMagic(fileBytes []byte, try int) (mimeType string, err error) {
	defer func() {
		magicmime.Close()
		mimeLock.Unlock()

		if err := recover(); err != nil {
			warning.Println("magicMime paniced")
			time.Sleep(time.Second)
			mimeType, err = getMimeFromMagic(fileBytes, try+1)
		}
	}()

	// if we tried to get the mimeType 3 times but paniced we'll return a
	// static string
	if try >= 3 {
		err = nil
		mimeType = "N/A"
		return
	}

	mimeLock.Lock()

	err = magicmime.Open(magicmime.MAGIC_ERROR)

	if err != nil {
		err = errors.New("ExtendedMime is activated but libmagic is not installed!")
		return
	}

	mimeType, err = magicmime.TypeByBuffer(fileBytes)
	return
}
func main_object() {
	info.Println("Uploading objects...")

	c = make(chan string)
	for i := 0; i < numWorkers; i++ {
		debug.Printf("Starting worker #%d\n", i)
		go worker()
	}

	if options.FPath != "" {
		file, err := os.Open(options.FPath)
		if err != nil {
			warning.Println("Couln't open file containing sample list!", err.Error())
			return
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		scanner.Split(bufio.ScanLines)
		// line by line
		for scanner.Scan() {
			sample := scanner.Text()
			wg.Add(1)
			if resume {
				_, already_processed := processed[sample]
				if already_processed {
					info.Printf("Skipping sample %s, because it was already uploaded successfully\n", sample)
					logC <- sample + "\t200\n"
					continue
				}
			}
			c <- sample
			//go copySample(scanner.Text())
		}
	}

	if options.Directory != "" {
		magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
		defer magicmime.Close()

		fullPath, err := filepath.Abs(options.Directory)

		if err != nil {
			warning.Println("path error:", err)
			return
		}
		topLevel = true
		err = filepath.Walk(fullPath, walkFn)
		if err != nil {
			warning.Println("walk error:", err)
			return
		}
	}

	wg.Wait()
}
Exemplo n.º 4
0
// GetMimetype gets the mimetype of a data []byte
func GetMimetype(data []byte) (string, error) {
	if err := magicmime.Open(magicmime.MAGIC_NO_CHECK_COMPRESS); err != nil {
		return "", err
	}
	defer magicmime.Close()

	mimetype, err := magicmime.TypeByBuffer(data)
	if err != nil {
		return "", err
	}
	return mimetype, nil
}
Exemplo n.º 5
0
func Example_typeByFile() {
	if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
		log.Fatal(err)
	}
	defer magicmime.Close()

	mimetype, err := magicmime.TypeByFile("/path/to/file")
	if err != nil {
		log.Fatalf("error occured during type lookup: %v", err)
	}

	log.Printf("mime-type: %v", mimetype)
}
Exemplo n.º 6
0
// Go-routine which serves magicmime requests because libmagic is not thread
// safe.
func magicMimeWorker() {
	err := magicmime.Open(magicmime.MAGIC_MIME_TYPE |
		magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Fatalln("libmagic initialization failure", err.Error())
	}
	defer magicmime.Close()

	// Listen
	for filePath := range mimeQueryCh {
		mimetype, err := magicmime.TypeByFile(filePath)
		mimeResponseCh <- mimeResponse{mimetype, err}
	}
}
Exemplo n.º 7
0
func main_object() {
	info.Println("Uploading objects...")

	c = make(chan string)
	for i := 0; i < numWorkers; i++ {
		debug.Printf("Starting worker #%d\n", i)
		go worker()
	}

	if fPath != "" {
		file, err := os.Open(fPath)
		if err != nil {
			warning.Println("Couln't open file containing sample list!", err.Error())
			return
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		scanner.Split(bufio.ScanLines)
		// line by line
		for scanner.Scan() {
			wg.Add(1)
			c <- scanner.Text()
			//go copySample(scanner.Text())
		}
	}

	if directory != "" {
		magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
		defer magicmime.Close()

		fullPath, err := filepath.Abs(directory)

		if err != nil {
			warning.Println("path error:", err)
			return
		}
		topLevel = true
		err = filepath.Walk(fullPath, walkFn)
		if err != nil {
			warning.Println("walk error:", err)
			return
		}
	}

	wg.Wait()
}
Exemplo n.º 8
0
func init() {
	// add missing extensions
	for k, v := range mimeRdfExt {
		mime.AddExtensionType(k, v)
	}

	for _, syntax := range crdf.ParserSyntax {
		switch syntax.MimeType {
		case "", "text/html":
			continue
		}
		mimeParser[syntax.MimeType] = syntax.Name
	}
	mimeParser["text/n3"] = mimeParser["text/turtle"]

	for name, syntax := range crdf.SerializerSyntax {
		switch name {
		case "json-triples":
			// only activate: json
			continue
		case "rdfxml-xmp", "rdfxml":
			// only activate: rdfxml-abbrev
			continue
		}
		mimeSerializer[syntax.MimeType] = syntax.Name
	}

	for mime := range mimeSerializer {
		switch mime {
		case "application/xhtml+xml":
			continue
		}
		serializerMimes = append(serializerMimes, mime)
	}

	magicmime.Open(magicmime.MAGIC_MIME_TYPE)
}
Exemplo n.º 9
0
func Upload(path string, maxMemory int64, w http.ResponseWriter, r *http.Request) (Image, error) {
	image := Image{}
	errRes := ersp.New(w, r)
	r.Body = http.MaxBytesReader(w, r.Body, maxMemory)

	file, handler, err := r.FormFile("image")
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}
	defer file.Close()

	randName, err := str.Rand(32)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	fileName := fmt.Sprintf("%s_%s", randName, handler.Filename)
	filePath := fmt.Sprintf("%s/%s", path, fileName)

	f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}
	defer f.Close()

	_, err = io.Copy(f, file)
	if err != nil {
		return image, err
	}

	err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	mimetype, err := magicmime.TypeByFile(filePath)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	if ok := imageMime.MatchString(mimetype); !ok {
		errRes.Field("image", "must be valid image")
		errRes.Send(http.StatusBadRequest)
		os.Remove(filePath)
		return image, err
	}

	stat, err := f.Stat()
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return image, err
	}

	image.Name = fileName
	image.OriginalName = handler.Filename
	image.Stat = stat

	return image, nil
}
Exemplo n.º 10
0
/**
 * Upload a file to ./public/ - with a short-name.
 *
 * Each file will also have a ".meta" file created, to contain
 * some content.
 */
func UploadHandler(res http.ResponseWriter, req *http.Request) {
	var (
		status int
		err    error
	)
	defer func() {
		if nil != err {
			http.Error(res, err.Error(), status)
		}
	}()

	/**
	 * Get the authentication-header.  If missing we abort.
	 */
	auth := string(req.Header.Get("X-Auth"))
	if len(auth) < 1 {
		status = 401
		err = errors.New("Missing X-Auth header")
		return
	}
	auth = strings.TrimSpace(auth)

	/**
	 * Load the secret.
	 */
	state, err := LoadState()
	if err != nil {
		status = 500
		err = errors.New("Loading state failed")
		return
	}

	/**
	 * Test the token.
	 */
	otpc := &dgoogauth.OTPConfig{
		Secret:      state.Secret,
		WindowSize:  3,
		HotpCounter: 0,
	}

	val, err := otpc.Authenticate(auth)
	if err != nil {
		status = 401
		err = errors.New("Failed to use X-Auth header")
		return
	}

	/**
	 * If it failed then we're done.
	 */
	if !val {
		status = 401
		err = errors.New("Invalid X-Auth header")
		return
	}

	/**
	 ** At ths point we know we have an authorized submitter.
	 **/

	/**
	 * Parse the incoming request
	 */
	const _24K = (1 << 20) * 24
	if err = req.ParseMultipartForm(_24K); nil != err {
		status = http.StatusInternalServerError
		return
	}

	/**
	 * Get the short-ID
	 */
	sn := NextShortID()

	for _, fheaders := range req.MultipartForm.File {
		for _, hdr := range fheaders {
			// open uploaded
			var infile multipart.File
			if infile, err = hdr.Open(); nil != err {
				status = http.StatusInternalServerError
				return
			}
			// open destination
			var outfile *os.File
			if outfile, err = os.Create("./public/" + sn); nil != err {
				status = http.StatusInternalServerError
				return
			}
			// 32K buffer copy
			if _, err = io.Copy(outfile, infile); nil != err {
				status = http.StatusInternalServerError
				return
			}

			// Get the MIME-type of the uploaded file.
			err = magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR)
			if err != nil {
				status = http.StatusInternalServerError
				return
			}
			defer magicmime.Close()
			mimetype, _ := magicmime.TypeByFile("./public/" + sn)

			//
			// Write out the meta-data - which is a structure
			// containing the following members.
			//
			md := &UploadMetaData{MIME: mimetype, IP: getRemoteIP(req), AT: time.Now().Format(time.RFC850)}
			data_json, _ := json.Marshal(md)

			var meta *os.File
			defer meta.Close()
			if meta, err = os.Create("./public/" + sn + ".meta"); nil != err {
				status = http.StatusInternalServerError
				return
			}
			meta.WriteString(string(data_json)) //mimetype)

			//
			// Write out the redirection - using the host
			// scheme, and the end-point of the new upload.
			//
			hostname := req.Host
			scheme := "http"

			if strings.HasPrefix(req.Proto, "HTTPS") {
				scheme = "https"
			}
			if req.Header.Get("X-Forwarded-Proto") == "https" {
				scheme = "https"
			}

			res.Write([]byte(scheme + "://" + hostname + "/get/" + sn + "\n"))
		}
	}
}