// Write write needles, if key exists append to super block, then update
// needle cache offset to new offset.
func (v *Volume) Write(ns *needle.Needles) (err error) {
var (
i int
ok bool
nc int64
ncs []int64
offset, ooffset uint32
n *needle.Needle
now = time.Now().UnixNano()
)
v.lock.Lock()
offset = v.Block.Offset
if err = v.Block.Write(ns.Buffer()); err == nil {
for i = 0; i < ns.Num; i++ {
n = ns.Needle(i)
if err = v.Indexer.Add(n.Key, offset, n.TotalSize); err != nil {
break
}
if nc, ok = v.needles[n.Key]; ok {
ncs = append(ncs, nc)
}
v.needles[n.Key] = needle.NewCache(offset, n.TotalSize)
v.addCheck(n.Key, n.Cookie)
offset += n.IncrOffset
if log.V(1) {
log.Infof("add needle, offset: %d, size: %d", offset, n.TotalSize)
log.Info(n)
}
}
}
v.lock.Unlock()
if err != nil {
return
}
for _, nc = range ncs {
ooffset, _ = needle.Cache(nc)
err = v.asyncDel(ooffset)
log.Warningf("same key: %d, old offset: %d, new offset: %d", n.Key, ooffset, offset)
}
atomic.AddUint64(&v.Stats.TotalWriteProcessed, 1)
atomic.AddUint64(&v.Stats.TotalWriteBytes, uint64(n.TotalSize))
atomic.AddUint64(&v.Stats.TotalWriteDelay, uint64(time.Now().UnixNano()-now))
return
}
// FreeNeedles free the needles to pool.
func (s *Store) FreeNeedles(i int, n *needle.Needles) {
n.TotalSize = 0
s.nsp[i].Put(n)
}
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
i, nn int
ok bool
err error
vid int64
key int64
cookie int64
size int64
str string
keys []string
cookies []string
sr sizer
fr *os.File
fi os.FileInfo
v *Volume
n *needle.Needle
ns *needle.Needles
uerr errors.Error
file multipart.File
fh *multipart.FileHeader
fhs []*multipart.FileHeader
res = map[string]interface{}{"ret": errors.RetOK}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
res["ret"] = errors.RetInternalErr
return
}
if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
res["ret"] = errors.RetNeedleTooLarge
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
res["ret"] = errors.RetParamErr
return
}
keys = r.MultipartForm.Value["keys"]
cookies = r.MultipartForm.Value["cookies"]
if len(keys) != len(cookies) {
log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
res["ret"] = errors.RetParamErr
return
}
fhs = r.MultipartForm.File["file"]
nn = len(fhs)
if len(keys) != nn {
log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
res["ret"] = errors.RetParamErr
return
}
ns = h.s.Needles(nn)
for i, fh = range fhs {
if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
err = errors.ErrParam
break
}
if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
err = errors.ErrParam
break
}
if file, err = fh.Open(); err != nil {
log.Errorf("fh.Open() error(%v)", err)
break
}
// check size
if sr, ok = file.(sizer); ok {
size = sr.Size()
} else if fr, ok = file.(*os.File); ok {
if fi, err = fr.Stat(); err != nil {
goto failed
}
size = fi.Size()
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
goto failed
}
n = &ns.Items[i]
n.InitSize(key, int32(cookie), int32(size))
if err = ns.WriteFrom(n, file); err != nil {
goto failed
}
continue
failed:
file.Close()
break
}
if err == nil {
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Write(ns)
} else {
err = errors.ErrVolumeNotExist
}
}
h.s.FreeNeedles(nn, ns)
if err != nil {
if uerr, ok = err.(errors.Error); ok {
res["ret"] = int(uerr)
} else {
res["ret"] = errors.RetInternalErr
}
}
return
}
// FreeNeedles free the needles to pool.
func (s *Store) FreeNeedles(i int, ns *needle.Needles) {
ns.Reset()
s.nsp[i].Put(ns)
}
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
i, nn int
err error
vid int64
key int64
cookie int64
size int64
str string
keys []string
cookies []string
v *volume.Volume
ns *needle.Needles
file multipart.File
fh *multipart.FileHeader
fhs []*multipart.FileHeader
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
err = errors.ErrInternal
return
}
if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
err = errors.ErrInternal
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
keys = r.MultipartForm.Value["keys"]
cookies = r.MultipartForm.Value["cookies"]
if len(keys) != len(cookies) {
log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
err = errors.ErrParam
return
}
fhs = r.MultipartForm.File["file"]
nn = len(fhs)
if len(keys) != nn {
log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
err = errors.ErrParam
return
}
ns = h.s.Needles(nn)
for i, fh = range fhs {
if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
err = errors.ErrParam
break
}
if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
err = errors.ErrParam
break
}
if file, err = fh.Open(); err != nil {
log.Errorf("fh.Open() error(%v)", err)
break
}
defer file.Close()
// check size
if size, err = fileSize(file); err != nil {
break
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
break
}
if err = ns.WriteFrom(key, int32(cookie), int32(size), file); err != nil {
break
}
}
if err == nil {
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Write(ns)
} else {
err = errors.ErrVolumeNotExist
}
}
h.s.FreeNeedles(nn, ns)
return
}