func (node *Node) SetFileFromParts(allowEmpty bool) (err error) {
outf := fmt.Sprintf("%s/%s.data", node.Path(), node.Id)
outh, oerr := os.Create(outf)
if oerr != nil {
return oerr
}
defer outh.Close()
pReader, pWriter := io.Pipe()
defer pReader.Close()
cError := make(chan error)
cKill := make(chan bool)
// goroutine to add file readers to pipe while gzip reads pipe
// allows us to only open one file at a time but stream the data to gzip as if it was one reader
go func() {
var ferr error
killed := false
for i := 1; i <= node.Parts.Count; i++ {
select {
case <-cKill:
killed = true
break
default:
}
filename := fmt.Sprintf("%s/parts/%d", node.Path(), i)
// skip this portion unless either
// 1. file exists, or
// 2. file does not exist and allowEmpty == false
if _, errf := os.Stat(filename); errf == nil || (errf != nil && allowEmpty == false) {
part, err := os.Open(filename)
if err != nil {
ferr = err
break
}
_, err = io.Copy(pWriter, part)
part.Close()
if err != nil {
ferr = err
break
}
}
}
pWriter.Close()
select {
case <-cKill:
killed = true
default:
}
if killed {
return
}
cError <- ferr
}()
md5h := md5.New()
dst := io.MultiWriter(outh, md5h)
// write from pipe to outfile / md5
// handle optional compression
ucReader, ucErr := archive.UncompressReader(node.Parts.Compression, pReader)
if ucErr != nil {
close(cKill)
os.Remove(outf)
return ucErr
}
_, cerr := io.Copy(dst, ucReader)
// get any errors from channel / finish copy
if eerr := <-cError; eerr != nil {
os.Remove(outf)
return eerr
}
if cerr != nil {
os.Remove(outf)
return cerr
}
// get file info and update node
fileStat, ferr := os.Stat(outf)
if ferr != nil {
return ferr
}
if node.File.Name == "" {
node.File.Name = node.Id
}
node.File.Size = fileStat.Size()
node.File.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
node.File.CreatedOn = fileStat.ModTime()
//fill size index info
totalunits := node.File.Size / conf.CHUNK_SIZE
m := node.File.Size % conf.CHUNK_SIZE
if m != 0 {
totalunits += 1
}
node.Indexes["size"] = IdxInfo{
Type: "size",
TotalUnits: totalunits,
AvgUnitSize: conf.CHUNK_SIZE,
Format: "dynamic",
CreatedOn: time.Now(),
}
err = node.Save()
return
}
// helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files node.FormFiles, err error) {
params = make(map[string]string)
files = make(node.FormFiles)
reader, err := r.MultipartReader()
if err != nil {
return
}
tmpPath := ""
for {
if part, err := reader.NextPart(); err == nil {
// params don't have a FileName()
// files must have FormName() of either "upload", "gzip", "bzip2", "attributes", "subset_indices", or an integer
if part.FileName() == "" {
if !util.IsValidParamName(part.FormName()) {
return nil, files, errors.New("invalid param: " + part.FormName())
}
buffer := make([]byte, 32*1024)
n, err := part.Read(buffer)
if n == 0 || err != nil {
break
}
formValue := fmt.Sprintf("%s", buffer[0:n])
if part.FormName() == "upload_url" {
tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
files[part.FormName()] = node.FormFile{Name: "", Path: tmpPath, Checksum: make(map[string]string)}
// download from url
if tmpFile, err := os.Create(tmpPath); err == nil {
defer tmpFile.Close()
var tmpform = files[part.FormName()]
md5h := md5.New()
dst := io.MultiWriter(tmpFile, md5h)
fileName, body, err := fetchFileStream(formValue)
if err != nil {
return nil, files, errors.New("unable to stream url: " + err.Error())
}
defer body.Close()
if _, err = io.Copy(dst, body); err != nil {
return nil, files, err
}
tmpform.Name = fileName
tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
files[part.FormName()] = tmpform
} else {
return nil, files, err
}
} else {
// regular form field
params[part.FormName()] = formValue
}
} else {
// determine file type
isSubsetFile := false
if part.FormName() == "subset_indices" {
isSubsetFile = true
}
isPartsFile := false
if _, er := strconv.Atoi(part.FormName()); er == nil {
isPartsFile = true
}
if !isPartsFile && !util.IsValidFileName(part.FormName()) {
return nil, files, errors.New("invalid file param: " + part.FormName())
}
// download it
tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
files[part.FormName()] = node.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
if tmpFile, err := os.Create(tmpPath); err == nil {
defer tmpFile.Close()
if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
// handle upload or parts files
var tmpform = files[part.FormName()]
md5h := md5.New()
dst := io.MultiWriter(tmpFile, md5h)
ucReader, ucErr := archive.UncompressReader(part.FormName(), part)
if ucErr != nil {
return nil, files, ucErr
}
if _, err = io.Copy(dst, ucReader); err != nil {
return nil, files, err
}
if archive.IsValidUncompress(part.FormName()) {
tmpform.Name = util.StripSuffix(part.FileName())
}
tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
files[part.FormName()] = tmpform
} else {
// handle file where md5 not needed
if _, err = io.Copy(tmpFile, part); err != nil {
return nil, files, err
}
}
} else {
return nil, files, err
}
}
} else if err.Error() != "EOF" {
return nil, files, err
} else {
break
}
}
return
}