func (t *TmpFiles) GetByField(field string) (f *temp.File, err error) {
f = t.fields[field]
if f == nil {
aelog.Debugf("Find file by field: %s", field)
url := t.r.FormValue(field + "_url")
f = temp.NewFile(t.tmpDir)
if url != "" {
aelog.Debugf("Url found[%s]: %s", field, url)
f.OrigName = filepath.Base(url)
if err = f.LoadFromUrl(url); err != nil {
return
}
} else {
mf, mh, e := t.r.FormFile(field)
if e != nil {
err = e
return
}
if err = f.LoadFromForm(mf); err != nil {
return
}
f.OrigName = mh.Filename
}
t.fields[field] = f
}
return
}
func (u *Uploader) TryRequest(r *http.Request, w http.ResponseWriter) (status bool, err error, errCode int) {
if !u.conf.UploaderEnable {
return
}
token := r.URL.Query().Get(u.conf.UploaderParamName)
if token == "" {
return
}
aelog.Debugf("Uploader: token found: %s\n", token)
status = true
write := func(token string, params interface{}, err error) {
var resp *http.Response
if err != nil {
resp, err = u.ctrl.SetStatusErr(token, err)
} else {
resp, err = u.ctrl.SetStatus(token, params)
}
if err != nil {
errCode = 500
return
}
w.Header().Set("Content-Type", "application/x-javascript")
w.WriteHeader(resp.StatusCode)
io.Copy(w, resp.Body)
err = nil
}
files, err := u.ctrl.GetParams(token)
if err != nil {
if err == ErrTokenNotFound {
aelog.Debugln("Uploader: ctrl return not found")
errCode = 404
} else {
errCode = 500
aelog.Debugf("Uploader: ctrl return error: %v\n", err)
write(token, nil, err)
}
return
}
err = files.Upload(u.conf, u.stor, r, w)
if err != nil {
aelog.Debugf("Uploader: files.Upload has err: %v\n", err)
errCode = 500
write(token, nil, err)
return
}
write(token, files, nil)
aelog.Debugln("Uploader: files.Upload done")
return
}
func (c *Container) Dump() (err error) {
c.m.Lock()
defer c.m.Unlock()
if !c.ch {
return
}
st := time.Now()
var i int64
c.FDump = make(map[int64]*File, c.FileCount)
c.HDump = make(map[int64]*Hole, c.holeIndex.Count)
var last Space = c.last
for last != nil && c.last != nil {
if last.IsFree() {
c.HDump[i] = last.(*Hole)
} else {
c.FDump[i] = last.(*File)
}
last = last.Prev()
i++
}
pr := time.Since(st)
n, err := dump.DumpTo(c.indexName(), c)
aelog.Debugf("Dump container %d, writed %s for a %v (prep: %v)", c.Id, utils.HumanBytes(n), time.Since(st), pr)
c.ch = false
return
}
func (s *Server) Download(name string, w http.ResponseWriter, r *http.Request) {
_, ok := s.stor.Get(name)
if ok {
// File exists
s.Err(409, r, w)
return
}
url := s.downloadUrl(r)
tf := temp.NewFile(s.conf.TmpDir)
aelog.Debugf("Start download from %s\n", url)
err := tf.LoadFromUrl(url)
defer tf.Close()
if err != nil {
aelog.Infof("Can't download : %s, err: %v\n", url, err)
s.Err(500, r, w)
return
}
// again check for exists
_, ok = s.stor.Get(name)
if ok {
s.Err(409, r, w)
return
}
s.save(name, tf.Size, tf.File, r, w)
}
func (fs *Files) Upload(conf *config.Config, stor *storage.Storage, r *http.Request, w http.ResponseWriter) (err error) {
// make temp files object
tmpfs := &TmpFiles{r: r, fields: make(map[string]*temp.File), result: make(map[string]*temp.File), tmpDir: conf.TmpDir}
defer tmpfs.Close()
// upload
for _, f := range *fs {
err = f.Upload(tmpfs)
if err != nil {
return
}
}
result := tmpfs.Result()
// add to storage and set stae
for name, tf := range result {
aelog.Debugf("Uploader add file: %s (%s, %s)", name, utils.HumanBytes(tf.Size), tf.MimeType)
file, _ := stor.Add(name, tf.File, tf.Size)
for _, f := range *fs {
if f.Name == name {
f.SetState(file)
}
}
}
return
}
func (s *Storage) Open() (err error) {
aelog.Debugf("Try open %s..", s.Conf.DataPath)
dir, err := os.Open(s.Conf.DataPath)
if err != nil {
return
}
info, err := dir.Stat()
if err != nil {
return
}
if !info.IsDir() {
return errors.New("Data path must be dir")
}
wg := &sync.WaitGroup{}
files, err := dir.Readdir(-1)
for _, file := range files {
if filepath.Ext(file.Name()) == ".index" {
wg.Add(1)
go func(name string) {
name = s.Conf.DataPath + name
e := s.restoreContainer(name)
if e != nil {
err = e
}
wg.Done()
}(file.Name())
}
}
wg.Wait()
if err != nil {
return
}
if len(s.Containers) == 0 {
aelog.Info("Create first container")
if _, err = s.createContainer(); err != nil {
return
}
}
go func() {
if s.Conf.DumpTime > 0 {
for {
time.Sleep(s.Conf.DumpTime)
s.Dump()
}
}
}()
return
}
func (u unZip) OnSave(file *storage.File, w http.ResponseWriter, r *http.Request, s *storage.Storage) (e error) {
mode := u.needUnZip(r)
if mode == UNZIP_NO {
return
}
aelog.Debugf("UnZip: start unzip(%d) files to: %s (%s)", mode, file.Name, utils.HumanBytes(file.FSize))
if mode == UNZIP_FORCE {
s.DeleteChilds(file.Name)
}
filesCount, filesSize, err := u.unZipTo(file.Name, file, s)
if err != nil {
aelog.Debugf("UnZip: error: %v", err)
w.Header().Add("X-Ae-Unzip-Error", err.Error())
return
}
aelog.Debugf("UnZip: unziped: %s (%d files, %s files size)", file.Name, filesCount, utils.HumanBytes(filesSize))
w.Header().Add("X-Ae-Unzip-Count", fmt.Sprint(filesCount))
w.Header().Add("X-Ae-Unzip-Size", utils.HumanBytes(filesSize))
return
}
func StartRpcServer(s *storage.Storage) {
r := &Storage{
s: s,
}
rpc.Register(r)
rpc.HandleHTTP()
l, e := net.Listen("tcp", s.Conf.RpcAddr)
if e != nil {
panic("Rpc listen error:" + e.Error())
}
aelog.Debugf("Start rpc server on %s\n", s.Conf.RpcAddr)
go http.Serve(l, nil)
}
func (s *Storage) restoreContainer(path string) (err error) {
aelog.Debugf("Restore container from %s..", path)
container := &Container{}
if err, _ = dump.LoadData(path, container); err != nil {
return err
}
if container.Created {
if err = container.Init(s); err != nil {
return
}
s.m.Lock()
if container.Id > s.LastContainerId {
s.LastContainerId = container.Id
}
s.Containers[container.Id] = container
s.m.Unlock()
aelog.Debugf("Container %d restored. %d (%d) files (holes) found", container.Id, container.FileCount, container.holeIndex.Count)
//container.holeIndex.Print()
//container.Print()
} else {
return fmt.Errorf("Can't restore container from %s", path)
}
return
}