Example #1
0
File: fs.go Project: MG-RAST/Shock
func (node *Node) SetFileFromParts(allowEmpty bool) (err error) {
	outf := fmt.Sprintf("%s/%s.data", node.Path(), node.Id)
	outh, oerr := os.Create(outf)
	if oerr != nil {
		return oerr
	}
	defer outh.Close()

	pReader, pWriter := io.Pipe()
	defer pReader.Close()

	cError := make(chan error)
	cKill := make(chan bool)

	// goroutine to add file readers to pipe while gzip reads pipe
	// allows us to only open one file at a time but stream the data to gzip as if it was one reader
	go func() {
		var ferr error
		killed := false
		for i := 1; i <= node.Parts.Count; i++ {
			select {
			case <-cKill:
				killed = true
				break
			default:
			}
			filename := fmt.Sprintf("%s/parts/%d", node.Path(), i)
			// skip this portion unless either
			// 1. file exists, or
			// 2. file does not exist and allowEmpty == false
			if _, errf := os.Stat(filename); errf == nil || (errf != nil && allowEmpty == false) {
				part, err := os.Open(filename)
				if err != nil {
					ferr = err
					break
				}
				_, err = io.Copy(pWriter, part)
				part.Close()
				if err != nil {
					ferr = err
					break
				}
			}
		}
		pWriter.Close()
		select {
		case <-cKill:
			killed = true
		default:
		}
		if killed {
			return
		}
		cError <- ferr
	}()

	md5h := md5.New()
	dst := io.MultiWriter(outh, md5h)

	// write from pipe to outfile / md5
	// handle optional compression
	ucReader, ucErr := archive.UncompressReader(node.Parts.Compression, pReader)
	if ucErr != nil {
		close(cKill)
		os.Remove(outf)
		return ucErr
	}
	_, cerr := io.Copy(dst, ucReader)

	// get any errors from channel / finish copy
	if eerr := <-cError; eerr != nil {
		os.Remove(outf)
		return eerr
	}
	if cerr != nil {
		os.Remove(outf)
		return cerr
	}

	// get file info and update node
	fileStat, ferr := os.Stat(outf)
	if ferr != nil {
		return ferr
	}
	if node.File.Name == "" {
		node.File.Name = node.Id
	}
	node.File.Size = fileStat.Size()
	node.File.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
	node.File.CreatedOn = fileStat.ModTime()

	//fill size index info
	totalunits := node.File.Size / conf.CHUNK_SIZE
	m := node.File.Size % conf.CHUNK_SIZE
	if m != 0 {
		totalunits += 1
	}
	node.Indexes["size"] = IdxInfo{
		Type:        "size",
		TotalUnits:  totalunits,
		AvgUnitSize: conf.CHUNK_SIZE,
		Format:      "dynamic",
		CreatedOn:   time.Now(),
	}

	err = node.Save()
	return
}
Example #2
0
// helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files node.FormFiles, err error) {
	params = make(map[string]string)
	files = make(node.FormFiles)
	reader, err := r.MultipartReader()
	if err != nil {
		return
	}

	tmpPath := ""
	for {
		if part, err := reader.NextPart(); err == nil {
			// params don't have a FileName()
			// files must have FormName() of either "upload", "gzip", "bzip2", "attributes", "subset_indices", or an integer
			if part.FileName() == "" {
				if !util.IsValidParamName(part.FormName()) {
					return nil, files, errors.New("invalid param: " + part.FormName())
				}
				buffer := make([]byte, 32*1024)
				n, err := part.Read(buffer)
				if n == 0 || err != nil {
					break
				}
				formValue := fmt.Sprintf("%s", buffer[0:n])
				if part.FormName() == "upload_url" {
					tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
					files[part.FormName()] = node.FormFile{Name: "", Path: tmpPath, Checksum: make(map[string]string)}
					// download from url
					if tmpFile, err := os.Create(tmpPath); err == nil {
						defer tmpFile.Close()
						var tmpform = files[part.FormName()]
						md5h := md5.New()
						dst := io.MultiWriter(tmpFile, md5h)
						fileName, body, err := fetchFileStream(formValue)
						if err != nil {
							return nil, files, errors.New("unable to stream url: " + err.Error())
						}
						defer body.Close()
						if _, err = io.Copy(dst, body); err != nil {
							return nil, files, err
						}
						tmpform.Name = fileName
						tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
						files[part.FormName()] = tmpform
					} else {
						return nil, files, err
					}
				} else {
					// regular form field
					params[part.FormName()] = formValue
				}
			} else {
				// determine file type
				isSubsetFile := false
				if part.FormName() == "subset_indices" {
					isSubsetFile = true
				}
				isPartsFile := false
				if _, er := strconv.Atoi(part.FormName()); er == nil {
					isPartsFile = true
				}
				if !isPartsFile && !util.IsValidFileName(part.FormName()) {
					return nil, files, errors.New("invalid file param: " + part.FormName())
				}
				// download it
				tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
				files[part.FormName()] = node.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
				if tmpFile, err := os.Create(tmpPath); err == nil {
					defer tmpFile.Close()
					if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
						// handle upload or parts files
						var tmpform = files[part.FormName()]
						md5h := md5.New()
						dst := io.MultiWriter(tmpFile, md5h)
						ucReader, ucErr := archive.UncompressReader(part.FormName(), part)
						if ucErr != nil {
							return nil, files, ucErr
						}
						if _, err = io.Copy(dst, ucReader); err != nil {
							return nil, files, err
						}
						if archive.IsValidUncompress(part.FormName()) {
							tmpform.Name = util.StripSuffix(part.FileName())
						}
						tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
						files[part.FormName()] = tmpform
					} else {
						// handle file where md5 not needed
						if _, err = io.Copy(tmpFile, part); err != nil {
							return nil, files, err
						}
					}
				} else {
					return nil, files, err
				}
			}
		} else if err.Error() != "EOF" {
			return nil, files, err
		} else {
			break
		}
	}
	return
}