// AddVolume add a volume data in zk.
func (z *Zookeeper) AddVolume(v *volume.Volume) (err error) {
var vpath = z.volumePath(v.Id)
if _, err = z.c.Create(vpath, v.Meta(), 0, zk.WorldACL(zk.PermAll)); err != nil {
log.Errorf("zk.Create(\"%s\") error(%v)", vpath, err)
}
return
}
// parseFreeVolumeIndex parse free index from local.
func (s *Store) parseFreeVolumeIndex() (err error) {
var (
i int
id int32
bfile string
ifile string
v *volume.Volume
data []byte
ids []int32
lines []string
bfs []string
ifs []string
)
if data, err = ioutil.ReadAll(s.fvf); err != nil {
log.Errorf("ioutil.ReadAll() error(%v)", err)
return
}
lines = strings.Split(string(data), "\n")
if _, ids, bfs, ifs, err = s.parseIndex(lines); err != nil {
return
}
for i = 0; i < len(bfs); i++ {
id, bfile, ifile = ids[i], bfs[i], ifs[i]
if v, err = newVolume(id, bfile, ifile, s.conf); err != nil {
return
}
v.Close()
s.FreeVolumes = append(s.FreeVolumes, v)
if id = s.fileFreeId(bfile); id > s.FreeId {
s.FreeId = id
}
}
log.V(1).Infof("current max free volume id: %d", s.FreeId)
return
}
// AddFreeVolume add free volumes.
func (s *Store) AddFreeVolume(n int, bdir, idir string) (sn int, err error) {
var (
i int
bfile, ifile string
v *volume.Volume
)
s.flock.Lock()
for i = 0; i < n; i++ {
s.FreeId++
bfile, ifile = s.freeFile(s.FreeId, bdir, idir)
if myos.Exist(bfile) || myos.Exist(ifile) {
continue
}
if v, err = newVolume(volumeFreeId, bfile, ifile, s.conf); err != nil {
// if no free space, delete the file
os.Remove(bfile)
os.Remove(ifile)
break
}
v.Close()
s.FreeVolumes = append(s.FreeVolumes, v)
sn++
}
err = s.saveFreeVolumeIndex()
s.flock.Unlock()
return
}
func (h httpGetHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
v *volume.Volume
n *needle.Needle
err error
vid, key, cookie int64
ret = http.StatusOK
params = r.URL.Query()
now = time.Now()
)
if r.Method != "GET" && r.Method != "HEAD" {
ret = http.StatusMethodNotAllowed
http.Error(wr, "method not allowed", ret)
return
}
defer HttpGetWriter(r, wr, now, &err, &ret)
if vid, err = strconv.ParseInt(params.Get("vid"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("vid"), err)
ret = http.StatusBadRequest
return
}
if key, err = strconv.ParseInt(params.Get("key"), 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("key"), err)
ret = http.StatusBadRequest
return
}
if cookie, err = strconv.ParseInt(params.Get("cookie"), 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("cookie"), err)
ret = http.StatusBadRequest
return
}
n = h.s.Needle()
n.Key = key
n.Cookie = int32(cookie)
if v = h.s.Volumes[int32(vid)]; v != nil {
if err = v.Get(n); err != nil {
if err == errors.ErrNeedleDeleted || err == errors.ErrNeedleNotExist {
ret = http.StatusNotFound
} else {
ret = http.StatusInternalServerError
}
}
} else {
ret = http.StatusNotFound
err = errors.ErrVolumeNotExist
}
if err == nil {
if r.Method == "GET" {
if _, err = wr.Write(n.Data); err != nil {
log.Errorf("wr.Write() error(%v)", err)
ret = http.StatusInternalServerError
}
}
if log.V(1) {
log.Infof("get a needle: %v", n)
}
}
h.s.FreeNeedle(n)
return
}
func (h httpDelHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
v *volume.Volume
err error
key, vid int64
str string
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
str = r.PostFormValue("key")
if key, err = strconv.ParseInt(str, 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.PostFormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Del(key)
} else {
err = errors.ErrVolumeNotExist
}
return
}
// SetVolume set the data into fpath.
func (z *Zookeeper) SetVolume(v *volume.Volume) (err error) {
var (
stat *zk.Stat
vpath = z.volumePath(v.Id)
)
if _, stat, err = z.c.Get(vpath); err != nil {
log.Errorf("zk.Get(\"%s\") error(%v)", vpath, err)
return
}
if _, err = z.c.Set(vpath, v.Meta(), stat.Version); err != nil {
log.Errorf("zk.Set(\"%s\") error(%v)", vpath, err)
}
return
}
// Close close the store.
// WARN the global variable store must first set nil and reject any other
// requests then safty close.
func (s *Store) Close() {
var v *volume.Volume
if s.vf != nil {
s.vf.Close()
}
if s.fvf != nil {
s.fvf.Close()
}
if s.Volumes != nil {
for _, v = range s.Volumes {
v.Close()
}
}
if s.zk != nil {
s.zk.Close()
}
return
}
func TestZookeeper(t *testing.T) {
var (
v *volume.Volume
zk *Zookeeper
err error
lines []string
root = "/rack"
rack = "rack-a"
serverId = "store-a"
bfile = "./test/hijohn_1"
ifile = "./test/hijohn_1.idx"
)
os.Remove(bfile)
os.Remove(ifile)
defer os.Remove(bfile)
defer os.Remove(ifile)
if zk, err = NewZookeeper([]string{"localhost:2181"}, time.Second, root, rack, serverId); err != nil {
t.Errorf("Newzookeeper() error(%v)", err)
t.FailNow()
}
zk.DelVolume(1)
zk.DelVolume(2)
if v, err = newVolume(1, bfile, ifile, testConf); err != nil {
t.Errorf("NewVolume() error(%v)", err)
t.FailNow()
}
v.Close()
if err = zk.AddVolume(v); err != nil {
t.Errorf("zk.AddVolume() error(%v)", err)
t.FailNow()
}
v.Id = 2
if err = zk.AddVolume(v); err != nil {
t.Errorf("zk.AddVolume() error(%v)", err)
t.FailNow()
}
if lines, err = zk.Volumes(); err != nil {
t.Errorf("zk.Volumes() error(%v)", err)
t.FailNow()
}
if len(lines) != 2 || lines[0] != fmt.Sprintf("%s,%s,%d", bfile, ifile, 1) || lines[1] != fmt.Sprintf("%s,%s,%d", bfile, ifile, 2) {
t.FailNow()
}
}
// saveVolumeIndex save volumes index info to disk.
func (s *Store) saveVolumeIndex() (err error) {
var (
tn, n int
v *volume.Volume
)
if _, err = s.vf.Seek(0, os.SEEK_SET); err != nil {
return
}
for _, v = range s.Volumes {
if n, err = s.vf.WriteString(fmt.Sprintf("%s\n", string(v.Meta()))); err != nil {
return
}
tn += n
}
if err = s.vf.Sync(); err != nil {
return
}
err = os.Truncate(s.conf.VolumeIndex, int64(tn))
return
}
// DelVolume del the volume by volume id.
func (s *Store) DelVolume(id int32) (err error) {
var v *volume.Volume
s.vlock.Lock()
if v = s.Volumes[id]; v != nil {
if !v.Compact {
s.delVolume(id)
if err = s.saveVolumeIndex(); err == nil {
err = s.zk.DelVolume(id)
}
} else {
err = errors.ErrVolumeInCompact
}
} else {
err = errors.ErrVolumeNotExist
}
s.vlock.Unlock()
if err == nil {
v.Close()
}
return
}
// CompactVolume compact a super block to another file.
func (s *Store) CompactVolume(id int32) (err error) {
var (
v, nv *volume.Volume
bdir, idir string
)
// try check volume
if v = s.Volumes[id]; v != nil {
if v.Compact {
return errors.ErrVolumeInCompact
}
} else {
return errors.ErrVolumeExist
}
// find a free volume
if nv, err = s.freeVolume(id); err != nil {
return
}
// no lock here, Compact is no side-effect
if err = v.StartCompact(nv); err != nil {
v.StopCompact(nil)
return
}
s.vlock.Lock()
if v = s.Volumes[id]; v != nil {
if err = v.StopCompact(nv); err == nil {
// WARN no need update volumes map, use same object, only update
// zookeeper the local index cause the block and index file changed.
if err = s.saveVolumeIndex(); err == nil {
err = s.zk.SetVolume(v)
}
}
} else {
err = errors.ErrVolumeExist
}
s.vlock.Unlock()
if err == nil {
// nv now has old block/index
nv.Close()
nv.Destroy()
bdir, idir = filepath.Dir(nv.Block.File), filepath.Dir(nv.Indexer.File)
_, err = s.AddFreeVolume(1, bdir, idir)
}
return
}
func TestStore(t *testing.T) {
var (
s *Store
z *Zookeeper
v *volume.Volume
err error
data = []byte("test")
n = &needle.Needle{}
bfile = "./test/_free_block_1"
ifile = "./test/_free_block_1.idx"
vbfile = "./test/1_0"
vifile = "./test/1_0.idx"
b2file = "./test/_free_block_2"
i2file = "./test/_free_block_2.idx"
vb2file = "./test/2_0"
vi2file = "./test/2_0.idx"
)
os.Remove(testConf.VolumeIndex)
os.Remove(testConf.FreeVolumeIndex)
os.Remove(bfile)
os.Remove(ifile)
os.Remove(b2file)
os.Remove(i2file)
os.Remove(vbfile)
os.Remove(vifile)
os.Remove(vb2file)
os.Remove(vi2file)
os.Remove("./test/1_1")
os.Remove("./test/1_1.idx")
os.Remove("./test/1_2")
os.Remove("./test/1_2.idx")
defer os.Remove(testConf.VolumeIndex)
defer os.Remove(testConf.FreeVolumeIndex)
defer os.Remove(bfile)
defer os.Remove(ifile)
defer os.Remove(b2file)
defer os.Remove(i2file)
defer os.Remove(vbfile)
defer os.Remove(vifile)
defer os.Remove(vb2file)
defer os.Remove(vi2file)
defer os.Remove("./test/1_1")
defer os.Remove("./test/1_1.idx")
defer os.Remove("./test/1_2")
defer os.Remove("./test/1_2.idx")
if z, err = NewZookeeper([]string{"localhost:2181"}, time.Second*1, "/rack", "", "test"); err != nil {
t.Errorf("NewZookeeper() error(%v)", err)
t.FailNow()
}
z.DelVolume(1)
z.DelVolume(2)
if s, err = NewStore(z, testConf); err != nil {
t.Errorf("NewStore() error(%v)", err)
t.FailNow()
}
defer s.Close()
if _, err = s.AddFreeVolume(2, "./test", "./test"); err != nil {
t.Errorf("s.AddFreeVolume() error(%v)", err)
t.FailNow()
}
if v, err = s.AddVolume(1); err != nil {
t.Errorf("AddVolume() error(%v)", err)
t.FailNow()
}
if v = s.Volumes[1]; v == nil {
t.Error("Volume(1) not exist")
t.FailNow()
}
n.Buffer = make([]byte, testConf.NeedleMaxSize)
n.Init(1, 1, data)
n.Write()
if err = v.Add(n); err != nil {
t.Errorf("v.Add(1) error(%v)", err)
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
if err = s.BulkVolume(2, b2file, i2file); err != nil {
t.Errorf("Bulk(1) error(%v)", err)
t.FailNow()
}
if v = s.Volumes[2]; v == nil {
t.Error("Volume(2) not exist")
t.FailNow()
}
if err = v.Add(n); err != nil {
t.Errorf("v.Add() error(%v)", err)
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
if err = s.CompactVolume(1); err != nil {
t.Errorf("Compress(1) error(%v)", err)
t.FailNow()
}
if v = s.Volumes[1]; v == nil {
t.Error("Volume(1) not exist")
t.FailNow()
}
n.Key = 1
n.Cookie = 1
if err = v.Get(n); err != nil {
t.Errorf("v.Get(1) error(%v)", err)
t.FailNow()
}
s.DelVolume(1)
if v = s.Volumes[1]; v != nil {
t.Error(err)
t.FailNow()
}
}
func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
i, nn int
err error
vid int64
key int64
cookie int64
size int64
str string
keys []string
cookies []string
v *volume.Volume
ns *needle.Needles
file multipart.File
fh *multipart.FileHeader
fhs []*multipart.FileHeader
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
err = errors.ErrInternal
return
}
if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
err = errors.ErrInternal
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
keys = r.MultipartForm.Value["keys"]
cookies = r.MultipartForm.Value["cookies"]
if len(keys) != len(cookies) {
log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
err = errors.ErrParam
return
}
fhs = r.MultipartForm.File["file"]
nn = len(fhs)
if len(keys) != nn {
log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
err = errors.ErrParam
return
}
ns = h.s.Needles(nn)
for i, fh = range fhs {
if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
err = errors.ErrParam
break
}
if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
err = errors.ErrParam
break
}
if file, err = fh.Open(); err != nil {
log.Errorf("fh.Open() error(%v)", err)
break
}
defer file.Close()
// check size
if size, err = fileSize(file); err != nil {
break
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
break
}
if err = ns.WriteFrom(key, int32(cookie), int32(size), file); err != nil {
break
}
}
if err == nil {
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Write(ns)
} else {
err = errors.ErrVolumeNotExist
}
}
h.s.FreeNeedles(nn, ns)
return
}
func (h httpUploadHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
var (
vid int64
key int64
cookie int64
size int64
err error
str string
v *volume.Volume
n *needle.Needle
file multipart.File
res = map[string]interface{}{}
)
if r.Method != "POST" {
http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
return
}
defer HttpPostWriter(r, wr, time.Now(), &err, res)
// check total content-length
if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
err = errors.ErrInternal
return
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
return
}
str = r.FormValue("vid")
if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.FormValue("key")
if key, err = strconv.ParseInt(str, 10, 64); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
str = r.FormValue("cookie")
if cookie, err = strconv.ParseInt(str, 10, 32); err != nil {
log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
err = errors.ErrParam
return
}
if file, _, err = r.FormFile("file"); err != nil {
log.Errorf("r.FormFile() error(%v)", err)
err = errors.ErrInternal
return
}
defer file.Close()
if size, err = fileSize(file); err != nil {
log.Errorf("fileSize() error(%v)", err)
err = errors.ErrInternal
return
}
if size > int64(h.c.NeedleMaxSize) {
err = errors.ErrNeedleTooLarge
return
}
n = h.s.Needle()
if err = n.WriteFrom(key, int32(cookie), int32(size), file); err == nil {
if v = h.s.Volumes[int32(vid)]; v != nil {
err = v.Add(n)
} else {
err = errors.ErrVolumeNotExist
}
}
h.s.FreeNeedle(n)
return
}