func compareTestNeedle(t *testing.T, key int64, cookie int32, flag byte, n *needle.Needle, data, buf []byte) (err error) {
if err = n.Parse(); err != nil {
t.Error(err)
return
}
if !bytes.Equal(n.Data, data) {
err = fmt.Errorf("data: %s not match", n.Data)
t.Error(err)
return
}
if n.Cookie != cookie {
err = fmt.Errorf("cookie: %d not match", n.Cookie)
t.Error(err)
return
}
if n.Key != key {
err = fmt.Errorf("key: %d not match", n.Key)
t.Error(err)
return
}
if n.Flag != flag {
err = fmt.Errorf("flag: %d not match", n.Flag)
t.Error(err)
return
}
if n.Size != int32(len(data)) {
err = fmt.Errorf("size: %d not match", n.Size)
t.Error(err)
return
}
return
}
func (h httpGetHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
now = time.Now()
v *Volume
n *needle.Needle
err error
vid, key, cookie int64
ret = http.StatusOK
params = r.URL.Query()
)
if r.Method != "GET" && r.Method != "HEAD" {
ret = http.StatusMethodNotAllowed
http.Error(wr, "method not allowed", ret)
return
}
defer HttpGetWriter(r, wr, now, &err, &ret)
if vid, err = strconv.ParseInt(params.Get("vid"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("vid"), err)
ret = http.StatusBadRequest
return
}
if key, err = strconv.ParseInt(params.Get("key"), 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("key"), err)
ret = http.StatusBadRequest
return
}
if cookie, err = strconv.ParseInt(params.Get("cookie"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("cookie"), err)
ret = http.StatusBadRequest
return
}
n = h.s.Needle()
n.Key = key
n.Cookie = int32(cookie)
if v = h.s.Volumes[int32(vid)]; v != nil {
if err = v.Get(n); err != nil {
if err == errors.ErrNeedleDeleted || err == errors.ErrNeedleNotExist {
ret = http.StatusNotFound
} else {
ret = http.StatusInternalServerError
}
}
} else {
ret = http.StatusNotFound
err = errors.ErrVolumeNotExist
}
if err == nil {
if r.Method == "GET" {
if _, err = wr.Write(n.Data); err != nil {
log.Errorf("wr.Write() error(%v)", err)
ret = http.StatusInternalServerError
}
}
if log.V(1) {
log.Infof("get a needle: %v", n)
}
}
h.s.FreeNeedle(n)
return
}
// Get get a needle by key and cookie.
func (v *Volume) Get(n *needle.Needle) (err error) {
var (
ok bool
nc int64
size int32
offset uint32
key = n.Key
cookie = n.Cookie
now = time.Now().UnixNano()
)
// WARN pread syscall is atomic, so use rlock
v.lock.RLock()
if nc, ok = v.needles[n.Key]; ok {
offset, size = needle.Cache(nc)
if offset != needle.CacheDelOffset {
err = v.Block.Get(offset, n.Buffer[:size])
} else {
err = errors.ErrNeedleDeleted
}
} else {
err = errors.ErrNeedleNotExist
}
v.lock.RUnlock()
if err != nil {
return
}
if log.V(1) {
log.Infof("get needle key: %d, cookie: %d, offset: %d, size: %d", n.Key, n.Cookie, offset, size)
}
if err = n.Parse(); err != nil {
return
}
if log.V(1) {
log.Infof("%v\n", n)
}
if n.Key != key {
err = errors.ErrNeedleKey
return
}
if n.Cookie != cookie {
err = errors.ErrNeedleCookie
return
}
// needles map may be out-dated, recheck
if n.Flag == needle.FlagDel {
v.lock.Lock()
v.needles[key] = needle.NewCache(needle.CacheDelOffset, size)
v.lock.Unlock()
err = errors.ErrNeedleDeleted
} else {
atomic.AddUint64(&v.Stats.TotalGetProcessed, 1)
atomic.AddUint64(&v.Stats.TotalReadBytes, uint64(size))
atomic.AddUint64(&v.Stats.TotalGetDelay, uint64(time.Now().UnixNano()-now))
}
return
}
// Write start add needles to the block, must called after start a transaction.
func (b *SuperBlock) Write(n *needle.Needle) (err error) {
if b.LastErr != nil {
return b.LastErr
}
var incrOffset = needle.NeedleOffset(int64(n.TotalSize))
if err = b.available(incrOffset); err != nil {
return
}
if err = n.Write(b.bw); err != nil {
b.LastErr = err
return
}
b.Offset += incrOffset
return
}
func BenchmarkVolumeWrite(b *testing.B) {
var (
v *Volume
err error
file = "./test/testb2"
ifile = "./test/testb2.idx"
data = make([]byte, _16kb) // 16kb
)
os.Remove(file)
os.Remove(ifile)
defer os.Remove(file)
defer os.Remove(ifile)
if _, err = rand.Read(data); err != nil {
b.Errorf("rand.Read() error(%v)", err)
b.FailNow()
}
if v, err = NewVolume(1, file, ifile, testConf); err != nil {
b.Errorf("NewVolume() error(%v)", err)
b.FailNow()
}
defer v.Close()
b.SetParallelism(8)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
var (
i, j int
ts int32
t int64
err1 error
n *needle.Needle
ns = make([]needle.Needle, 9)
buf = make([]byte, 163840) // 16kb
)
for i = 0; i < 9; i++ {
t = mrand.Int63()
n = &ns[i]
n.Init(t, 1, data)
n.Write(buf[ts:])
ts += n.TotalSize
}
for pb.Next() {
for j = 0; j < 9; j++ {
t = mrand.Int63()
n = &ns[j]
n.Key = t
binary.BigEndian.PutInt64(buf[n.TotalSize+needle.KeyOffset:], n.Key)
}
if err1 = v.Write(ns, buf[:ts]); err1 != nil {
b.Errorf("Add() error(%v)", err1)
v.Unlock()
b.FailNow()
}
b.SetBytes(int64(ts))
}
})
os.Remove(file)
os.Remove(ifile)
}
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
i, rn, tn, nn, nb int
ok bool
buf []byte
err error
vid int64
key int64
cookie int64
size int64
str string
keys []string
cookies []string
sr sizer
fr *os.File
fi os.FileInfo
v *Volume
n *needle.Needle
ns []needle.Needle
uerr errors.Error
file multipart.File
fh *multipart.FileHeader
fhs []*multipart.FileHeader
res = map[string]interface{}{"ret": errors.RetOK}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
res["ret"] = errors.RetInternalErr
return
}
if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
res["ret"] = errors.RetNeedleTooLarge
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
res["ret"] = errors.RetParamErr
return
}
keys = r.MultipartForm.Value["keys"]
cookies = r.MultipartForm.Value["cookies"]
if len(keys) != len(cookies) {
log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
res["ret"] = errors.RetParamErr
return
}
fhs = r.MultipartForm.File["file"]
nn = len(fhs)
if len(keys) != nn {
log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
res["ret"] = errors.RetParamErr
return
}
nb = int(size-1)/(h.c.NeedleMaxSize) + 1
buf = h.s.Buffer(nb)
ns = h.s.Needle(nn)
for i, fh = range fhs {
if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
err = errors.ErrParam
break
}
if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
err = errors.ErrParam
break
}
file, err = fh.Open()
file.Close()
if err != nil {
log.Errorf("fh.Open() error(%v)", err)
break
}
// check size
if sr, ok = file.(sizer); ok {
size = sr.Size()
} else if fr, ok = file.(*os.File); ok {
if fi, err = fr.Stat(); err != nil {
break
}
size = fi.Size()
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
break
}
if rn, err = file.Read(buf[tn:]); err != nil {
log.Errorf("file.Read() error(%v)", err)
break
}
n = &(ns[i])
n.Parse(key, int32(cookie), buf[tn:tn+rn])
tn += rn
}
if err == nil {
h.s.RLockVolume()
if v = h.s.Volumes[int32(vid)]; v != nil {
v.Lock()
for i = 0; i < nn; i++ {
n = &(ns[i])
if err = v.Write(n); err != nil {
break
}
}
if err == nil {
err = v.Flush()
}
v.Unlock()
} else {
err = errors.ErrVolumeNotExist
}
h.s.RUnlockVolume()
}
h.s.FreeBuffer(nb, buf)
h.s.FreeNeedle(nn, ns)
if err != nil {
if uerr, ok = err.(errors.Error); ok {
res["ret"] = int(uerr)
} else {
res["ret"] = errors.RetInternalErr
}
}
return
}
func (h httpUploadHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
ok bool
rn int
vid int64
key int64
cookie int64
size int64
err error
str string
buf []byte
v *Volume
n *needle.Needle
ns []needle.Needle
file multipart.File
sr sizer
fr *os.File
fi os.FileInfo
uerr errors.Error
res = map[string]interface{}{"ret": errors.RetOK}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
res["ret"] = errors.RetInternalErr
return
}
if size > int64(h.c.NeedleMaxSize) {
res["ret"] = errors.RetNeedleTooLarge
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
res["ret"] = errors.RetParamErr
return
}
str = r.FormValue("key")
if key, err = strconv.ParseInt(str, 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
res["ret"] = errors.RetParamErr
return
}
str = r.FormValue("cookie")
if cookie, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
res["ret"] = errors.RetParamErr
return
}
if file, _, err = r.FormFile("file"); err != nil {
res["ret"] = errors.RetInternalErr
return
}
if sr, ok = file.(sizer); ok {
size = sr.Size()
} else if fr, ok = file.(*os.File); ok {
if fi, err = fr.Stat(); err != nil {
res["ret"] = errors.RetInternalErr
return
}
size = fi.Size()
}
if size > int64(h.c.NeedleMaxSize) {
res["ret"] = errors.RetNeedleTooLarge
return
}
ns = h.s.Needle(1)
n = &(ns[0])
buf = h.s.Buffer(1)
rn, err = file.Read(buf)
file.Close()
if err != nil {
res["ret"] = errors.RetInternalErr
return
}
n.Parse(key, int32(cookie), buf[:rn])
h.s.RLockVolume()
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Add(n)
} else {
err = errors.ErrVolumeNotExist
}
h.s.RUnlockVolume()
h.s.FreeBuffer(1, buf)
h.s.FreeNeedle(1, ns)
if err != nil {
if uerr, ok = err.(errors.Error); ok {
res["ret"] = int(uerr)
} else {
res["ret"] = errors.RetInternalErr
}
}
return
}
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
i, nn int
ok bool
err error
vid int64
key int64
cookie int64
size int64
str string
keys []string
cookies []string
sr sizer
fr *os.File
fi os.FileInfo
v *Volume
n *needle.Needle
ns *needle.Needles
uerr errors.Error
file multipart.File
fh *multipart.FileHeader
fhs []*multipart.FileHeader
res = map[string]interface{}{"ret": errors.RetOK}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
res["ret"] = errors.RetInternalErr
return
}
if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
res["ret"] = errors.RetNeedleTooLarge
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
res["ret"] = errors.RetParamErr
return
}
keys = r.MultipartForm.Value["keys"]
cookies = r.MultipartForm.Value["cookies"]
if len(keys) != len(cookies) {
log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
res["ret"] = errors.RetParamErr
return
}
fhs = r.MultipartForm.File["file"]
nn = len(fhs)
if len(keys) != nn {
log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
res["ret"] = errors.RetParamErr
return
}
ns = h.s.Needles(nn)
for i, fh = range fhs {
if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
err = errors.ErrParam
break
}
if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
err = errors.ErrParam
break
}
if file, err = fh.Open(); err != nil {
log.Errorf("fh.Open() error(%v)", err)
break
}
// check size
if sr, ok = file.(sizer); ok {
size = sr.Size()
} else if fr, ok = file.(*os.File); ok {
if fi, err = fr.Stat(); err != nil {
goto failed
}
size = fi.Size()
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
goto failed
}
n = &ns.Items[i]
n.InitSize(key, int32(cookie), int32(size))
if err = ns.WriteFrom(n, file); err != nil {
goto failed
}
continue
failed:
file.Close()
break
}
if err == nil {
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Write(ns)
} else {
err = errors.ErrVolumeNotExist
}
}
h.s.FreeNeedles(nn, ns)
if err != nil {
if uerr, ok = err.(errors.Error); ok {
res["ret"] = int(uerr)
} else {
res["ret"] = errors.RetInternalErr
}
}
return
}
func (h httpUploadHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
vid int64
key int64
cookie int64
size int64
err error
str string
v *volume.Volume
n *needle.Needle
file multipart.File
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
err = errors.ErrInternal
return
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.FormValue("key")
if key, err = strconv.ParseInt(str, 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.FormValue("cookie")
if cookie, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
if file, _, err = r.FormFile("file"); err != nil {
log.Errorf("r.FormFile() error(%v)", err)
err = errors.ErrInternal
return
}
defer file.Close()
if size, err = fileSize(file); err != nil {
log.Errorf("fileSize() error(%v)", err)
err = errors.ErrInternal
return
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
return
}
n = h.s.Needle()
if err = n.WriteFrom(key, int32(cookie), int32(size), file); err == nil {
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Add(n)
} else {
err = errors.ErrVolumeNotExist
}
}
h.s.FreeNeedle(n)
return
}