func (s *Server) probe(wr http.ResponseWriter, r *http.Request) {
var (
v *volume.Volume
err error
vid int64
ret = http.StatusOK
params = r.URL.Query()
now = time.Now()
)
if r.Method != "HEAD" {
ret = http.StatusMethodNotAllowed
http.Error(wr, "method not allowed", ret)
return
}
defer HttpGetWriter(r, wr, now, &err, &ret)
if vid, err = strconv.ParseInt(params.Get("vid"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("vid"), err)
ret = http.StatusBadRequest
return
}
if v = s.store.Volumes[int32(vid)]; v != nil {
if err = v.Probe(); err != nil {
if err == errors.ErrNeedleDeleted || err == errors.ErrNeedleNotExist {
ret = http.StatusNotFound
} else {
ret = http.StatusInternalServerError
}
}
} else {
ret = http.StatusNotFound
err = errors.ErrVolumeNotExist
}
return
}
// AddFreeVolume add free volumes.
func (s *Store) AddFreeVolume(n int, bdir, idir string) (sn int, err error) {
var (
i int
bfile, ifile string
v *volume.Volume
)
s.flock.Lock()
for i = 0; i < n; i++ {
s.FreeId++
bfile, ifile = s.freeFile(s.FreeId, bdir, idir)
if myos.Exist(bfile) || myos.Exist(ifile) {
continue
}
if v, err = newVolume(volumeFreeId, bfile, ifile, s.conf); err != nil {
// if no free space, delete the file
os.Remove(bfile)
os.Remove(ifile)
break
}
v.Close()
s.FreeVolumes = append(s.FreeVolumes, v)
sn++
}
err = s.saveFreeVolumeIndex()
s.flock.Unlock()
return
}
// parseFreeVolumeIndex parse free index from local.
func (s *Store) parseFreeVolumeIndex() (err error) {
var (
i int
id int32
bfile string
ifile string
v *volume.Volume
data []byte
ids []int32
lines []string
bfs []string
ifs []string
)
if data, err = ioutil.ReadAll(s.fvf); err != nil {
log.Errorf("ioutil.ReadAll() error(%v)", err)
return
}
lines = strings.Split(string(data), "\n")
if _, ids, bfs, ifs, err = s.parseIndex(lines); err != nil {
return
}
for i = 0; i < len(bfs); i++ {
id, bfile, ifile = ids[i], bfs[i], ifs[i]
if v, err = newVolume(id, bfile, ifile, s.conf); err != nil {
return
}
v.Close()
s.FreeVolumes = append(s.FreeVolumes, v)
if id = s.fileFreeId(bfile); id > s.FreeId {
s.FreeId = id
}
}
log.V(1).Infof("current max free volume id: %d", s.FreeId)
return
}
// DelVolume del the volume by volume id.
func (s *Store) DelVolume(id int32) (err error) {
var v *volume.Volume
s.vlock.Lock()
if v = s.Volumes[id]; v != nil {
if !v.Compact {
s.delVolume(id)
if err = s.saveVolumeIndex(); err == nil {
err = s.zk.DelVolume(id)
}
if err != nil {
log.Errorf("del volume: %d error(%v), local index or zookeeper index may save failed", id, err)
}
} else {
err = errors.ErrVolumeInCompact
}
} else {
err = errors.ErrVolumeNotExist
}
s.vlock.Unlock()
// if succced or not meta data saved error, close volume
if err == nil || (err != errors.ErrVolumeInCompact &&
err != errors.ErrVolumeNotExist) {
v.Destroy()
}
return
}
func (s *Server) get(wr http.ResponseWriter, r *http.Request) {
var (
v *volume.Volume
n *needle.Needle
err error
vid, key, cookie int64
ret = http.StatusOK
params = r.URL.Query()
now = time.Now()
)
if r.Method != "GET" && r.Method != "HEAD" {
ret = http.StatusMethodNotAllowed
http.Error(wr, "method not allowed", ret)
return
}
defer HttpGetWriter(r, wr, now, &err, &ret)
if vid, err = strconv.ParseInt(params.Get("vid"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("vid"), err)
ret = http.StatusBadRequest
return
}
if key, err = strconv.ParseInt(params.Get("key"), 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("key"), err)
ret = http.StatusBadRequest
return
}
if cookie, err = strconv.ParseInt(params.Get("cookie"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("cookie"), err)
ret = http.StatusBadRequest
return
}
n = s.store.Needle()
n.Key = key
n.Cookie = int32(cookie)
if v = s.store.Volumes[int32(vid)]; v != nil {
if err = v.Get(n); err != nil {
if err == errors.ErrNeedleDeleted || err == errors.ErrNeedleNotExist {
ret = http.StatusNotFound
} else {
ret = http.StatusInternalServerError
}
}
} else {
ret = http.StatusNotFound
err = errors.ErrVolumeNotExist
}
if err == nil {
if r.Method == "GET" {
if _, err = wr.Write(n.Data); err != nil {
log.Errorf("wr.Write() error(%v)", err)
ret = http.StatusInternalServerError
}
}
if log.V(1) {
log.Infof("get a needle: %v", n)
}
}
s.store.FreeNeedle(n)
return
}
// saveVolumeIndex save volumes index info to disk.
func (s *Store) saveVolumeIndex() (err error) {
var (
tn, n int
v *volume.Volume
)
if _, err = s.vf.Seek(0, os.SEEK_SET); err != nil {
log.Errorf("vf.Seek() error(%v)", err)
return
}
for _, v = range s.Volumes {
if n, err = s.vf.WriteString(fmt.Sprintf("%s\n", string(v.Meta()))); err != nil {
log.Errorf("vf.WriteString() error(%v)", err)
return
}
tn += n
}
if err = s.vf.Sync(); err != nil {
log.Errorf("vf.Sync() error(%v)", err)
return
}
if err = os.Truncate(s.conf.Store.VolumeIndex, int64(tn)); err != nil {
log.Errorf("os.Truncate() error(%v)", err)
}
return
}
func (s *Server) del(wr http.ResponseWriter, r *http.Request) {
var (
err error
key, vid int64
str string
v *volume.Volume
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
str = r.PostFormValue("key")
if key, err = strconv.ParseInt(str, 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.PostFormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
if v = s.store.Volumes[int32(vid)]; v != nil {
err = v.Delete(key)
} else {
err = errors.ErrVolumeNotExist
}
return
}
func (s *Server) upload(wr http.ResponseWriter, r *http.Request) {
var (
vid int64
key int64
cookie int64
size int64
err error
str string
v *volume.Volume
n *needle.Needle
file multipart.File
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
if err = checkContentLength(r, s.conf.NeedleMaxSize); err != nil {
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.FormValue("key")
if key, err = strconv.ParseInt(str, 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.FormValue("cookie")
if cookie, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
if file, _, err = r.FormFile("file"); err != nil {
log.Errorf("r.FormFile() error(%v)", err)
err = errors.ErrInternal
return
}
if size, err = checkFileSize(file, s.conf.NeedleMaxSize); err == nil {
n = s.store.Needle()
if err = n.WriteFrom(key, int32(cookie), int32(size), file); err == nil {
if v = s.store.Volumes[int32(vid)]; v != nil {
err = v.Write(n)
} else {
err = errors.ErrVolumeNotExist
}
}
s.store.FreeNeedle(n)
}
file.Close()
return
}
// Close close the store.
// WARN the global variable store must first set nil and reject any other
// requests then safty close.
func (s *Store) Close() {
var v *volume.Volume
if s.vf != nil {
s.vf.Close()
}
if s.fvf != nil {
s.fvf.Close()
}
if s.Volumes != nil {
for _, v = range s.Volumes {
v.Close()
}
}
if s.zk != nil {
s.zk.Close()
}
return
}
// Close close the store.
// WARN the global variable store must first set nil and reject any other
// requests then safty close.
func (s *Store) Close() {
log.Info("store close")
var v *volume.Volume
if s.vf != nil {
s.vf.Close()
}
if s.fvf != nil {
s.fvf.Close()
}
for _, v = range s.Volumes {
log.Infof("volume[%d] close", v.Id)
v.Close()
}
if s.zk != nil {
s.zk.Close()
}
return
}
// BulkVolume copy a super block from another store server add to this server.
func (s *Store) BulkVolume(id int32, bfile, ifile string) (err error) {
var v, nv *volume.Volume
// recovery new block
if nv, err = newVolume(id, bfile, ifile, s.conf); err != nil {
return
}
s.vlock.Lock()
if v = s.Volumes[id]; v == nil {
s.addVolume(id, nv)
if err = s.saveVolumeIndex(); err == nil {
err = s.zk.AddVolume(id, nv.Meta())
}
if err != nil {
log.Errorf("bulk volume: %d error(%v), local index or zookeeper index may save failed", id, err)
}
} else {
err = errors.ErrVolumeExist
}
s.vlock.Unlock()
return
}
func TestStore(t *testing.T) {
var (
s *Store
z *zk.Zookeeper
v *volume.Volume
err error
n = needle.NewBufferNeedle(40)
buf = &bytes.Buffer{}
)
os.Remove(testConf.Store.VolumeIndex)
os.Remove(testConf.Store.FreeVolumeIndex)
os.Remove("./test/_free_block_1")
os.Remove("./test/_free_block_1.idx")
os.Remove("./test/_free_block_2")
os.Remove("./test/_free_block_2.idx")
os.Remove("./test/_free_block_3")
os.Remove("./test/_free_block_3.idx")
os.Remove("./test/1_0")
os.Remove("./test/1_0.idx")
os.Remove("./test/1_1")
os.Remove("./test/1_1.idx")
os.Remove("./test/block_store_1")
os.Remove("./test/block_store_1.idx")
defer os.Remove(testConf.Store.VolumeIndex)
defer os.Remove(testConf.Store.FreeVolumeIndex)
defer os.Remove("./test/_free_block_1")
defer os.Remove("./test/_free_block_1.idx")
defer os.Remove("./test/_free_block_2")
defer os.Remove("./test/_free_block_2.idx")
defer os.Remove("./test/_free_block_3")
defer os.Remove("./test/_free_block_3.idx")
defer os.Remove("./test/1_0")
defer os.Remove("./test/1_0.idx")
defer os.Remove("./test/1_1")
defer os.Remove("./test/1_1.idx")
defer os.Remove("./test/block_store_1")
defer os.Remove("./test/block_store_1.idx")
if z, err = zk.NewZookeeper(testConf); err != nil {
t.Errorf("NewZookeeper() error(%v)", err)
t.FailNow()
}
defer z.Close()
z.DelVolume(1)
z.DelVolume(2)
z.DelVolume(3)
defer z.DelVolume(1)
defer z.DelVolume(2)
defer z.DelVolume(3)
if s, err = NewStore(testConf); err != nil {
t.Errorf("NewStore() error(%v)", err)
t.FailNow()
}
defer s.Close()
if _, err = s.AddFreeVolume(2, "./test", "./test"); err != nil {
t.Errorf("s.AddFreeVolume() error(%v)", err)
t.FailNow()
}
if v, err = s.AddVolume(1); err != nil {
t.Errorf("AddVolume() error(%v)", err)
t.FailNow()
}
if v = s.Volumes[1]; v == nil {
t.Error("Volume(1) not exist")
t.FailNow()
}
buf.WriteString("test")
n.WriteFrom(1, 1, 4, buf)
if err = v.Write(n); err != nil {
t.Errorf("v.Add(1) error(%v)", err)
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
if err = s.BulkVolume(2, "./test/block_store_1", "./test/block_store_1.idx"); err != nil {
t.Errorf("Bulk(1) error(%v)", err)
t.FailNow()
}
if v = s.Volumes[2]; v == nil {
t.Error("Volume(2) not exist")
t.FailNow()
}
if err = v.Write(n); err != nil {
t.Errorf("v.Add() error(%v)", err)
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
if err = s.CompactVolume(1); err != nil {
t.Errorf("Compress(1) error(%v)", err)
t.FailNow()
}
if v = s.Volumes[1]; v == nil {
t.Error("Volume(1) not exist")
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
s.DelVolume(1)
if v = s.Volumes[1]; v != nil {
t.Error(err)
t.FailNow()
}
}
// CompactVolume compact a super block to another file.
func (s *Store) CompactVolume(id int32) (err error) {
var (
v, nv *volume.Volume
bdir, idir string
)
// try check volume
if v = s.Volumes[id]; v != nil {
if v.Compact {
return errors.ErrVolumeInCompact
}
} else {
return errors.ErrVolumeExist
}
// find a free volume
if nv, err = s.freeVolume(id); err != nil {
return
}
log.Infof("compact volume: (%d) %s to %s", id, v.Block.File, nv.Block.File)
// no lock here, Compact is no side-effect
if err = v.StartCompact(nv); err != nil {
nv.Destroy()
v.StopCompact(nil)
return
}
s.vlock.Lock()
if v = s.Volumes[id]; v != nil {
if err = v.StopCompact(nv); err == nil {
// WARN no need update volumes map, use same object, only update
// zookeeper the local index cause the block and index file changed.
if err = s.saveVolumeIndex(); err == nil {
err = s.zk.SetVolume(id, v.Meta())
}
if err != nil {
log.Errorf("compact volume: %d error(%v), local index or zookeeper index may save failed", id, err)
}
}
} else {
// never happen
err = errors.ErrVolumeExist
log.Errorf("compact volume: %d not exist(may bug)", id)
}
s.vlock.Unlock()
// WARN if failed, nv is free volume, if succeed nv replace with v.
nv.Destroy()
if err == nil {
bdir, idir = filepath.Dir(nv.Block.File), filepath.Dir(nv.Indexer.File)
_, err = s.AddFreeVolume(1, bdir, idir)
}
return
}
// parseVolumeIndex parse index from local config and zookeeper.
func (s *Store) parseVolumeIndex() (err error) {
var (
i int
ok bool
id int32
bfile string
ifile string
v *volume.Volume
data []byte
lids, zids []int32
lines []string
lbfs, lifs []string
zbfs, zifs []string
lim, zim map[int32]struct{}
)
if data, err = ioutil.ReadAll(s.vf); err != nil {
log.Errorf("ioutil.ReadAll() error(%v)", err)
return
}
lines = strings.Split(string(data), "\n")
if lim, lids, lbfs, lifs, err = s.parseIndex(lines); err != nil {
return
}
if lines, err = s.zk.Volumes(); err != nil {
return
}
if zim, zids, zbfs, zifs, err = s.parseIndex(lines); err != nil {
return
}
// local index
for i = 0; i < len(lbfs); i++ {
id, bfile, ifile = lids[i], lbfs[i], lifs[i]
if _, ok = s.Volumes[id]; ok {
continue
}
if v, err = newVolume(id, bfile, ifile, s.conf); err != nil {
return
}
s.Volumes[id] = v
if _, ok = zim[id]; !ok {
if err = s.zk.AddVolume(id, v.Meta()); err != nil {
return
}
} else {
if err = s.zk.SetVolume(id, v.Meta()); err != nil {
return
}
}
}
// zk index
for i = 0; i < len(zbfs); i++ {
id, bfile, ifile = zids[i], zbfs[i], zifs[i]
if _, ok = s.Volumes[id]; ok {
continue
}
// if not exists in local
if _, ok = lim[id]; !ok {
if v, err = newVolume(id, bfile, ifile, s.conf); err != nil {
return
}
s.Volumes[id] = v
}
}
err = s.saveVolumeIndex()
return
}
func (s *Server) uploads(wr http.ResponseWriter, r *http.Request) {
var (
i, nn int
err error
vid int64
key int64
cookie int64
size int64
str string
keys []string
cookies []string
v *volume.Volume
ns *needle.Needles
file multipart.File
fh *multipart.FileHeader
fhs []*multipart.FileHeader
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
if err = checkContentLength(r, s.conf.NeedleMaxSize*s.conf.BatchMaxNum); err != nil {
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
keys = r.MultipartForm.Value["keys"]
cookies = r.MultipartForm.Value["cookies"]
if len(keys) != len(cookies) {
log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
err = errors.ErrParam
return
}
fhs = r.MultipartForm.File["file"]
nn = len(fhs)
if len(keys) != nn {
log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
err = errors.ErrParam
return
}
ns = s.store.Needles(nn)
for i, fh = range fhs {
if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
err = errors.ErrParam
break
}
if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
err = errors.ErrParam
break
}
if file, err = fh.Open(); err != nil {
log.Errorf("fh.Open() error(%v)", err)
break
}
if size, err = checkFileSize(file, s.conf.NeedleMaxSize); err == nil {
err = ns.WriteFrom(key, int32(cookie), int32(size), file)
}
file.Close()
if err != nil {
break
}
}
if err == nil {
if v = s.store.Volumes[int32(vid)]; v != nil {
err = v.Writes(ns)
} else {
err = errors.ErrVolumeNotExist
}
}
s.store.FreeNeedles(nn, ns)
return
}