// Init initializes ledis session provider.
// configs: data_dir=./app.db,db=0
func (p *LedisProvider) Init(expire int64, configs string) error {
p.expire = expire
cfg, err := ini.Load([]byte(strings.Replace(configs, ",", "\n", -1)))
if err != nil {
return err
}
db := 0
opt := new(config.Config)
for k, v := range cfg.Section("").KeysHash() {
switch k {
case "data_dir":
opt.DataDir = v
case "db":
db = com.StrTo(v).MustInt()
default:
return fmt.Errorf("session/ledis: unsupported option '%s'", k)
}
}
l, err := ledis.Open(opt)
if err != nil {
return fmt.Errorf("session/ledis: error opening db: %v", err)
}
p.c, err = l.Select(db)
return err
}
func TestBinLog(t *testing.T) {
cfg := new(config.Config)
cfg.BinLog.MaxFileNum = 1
cfg.BinLog.MaxFileSize = 1024
cfg.DataDir = "/tmp/ledis_binlog"
os.RemoveAll(cfg.DataDir)
b, err := NewBinLog(cfg)
if err != nil {
t.Fatal(err)
}
if err := b.Log(make([]byte, 1024)); err != nil {
t.Fatal(err)
}
if err := b.Log(make([]byte, 1024)); err != nil {
t.Fatal(err)
}
if fs, err := ioutil.ReadDir(b.LogPath()); err != nil {
t.Fatal(err)
} else if len(fs) != 2 {
t.Fatal(len(fs))
}
}
// StartAndGC starts GC routine based on config string settings.
// AdapterConfig: data_dir=./app.db,db=0
func (c *LedisCacher) StartAndGC(opts cache.Options) error {
c.interval = opts.Interval
cfg, err := ini.Load([]byte(strings.Replace(opts.AdapterConfig, ",", "\n", -1)))
if err != nil {
return err
}
db := 0
opt := new(config.Config)
for k, v := range cfg.Section("").KeysHash() {
switch k {
case "data_dir":
opt.DataDir = v
case "db":
db = com.StrTo(v).MustInt()
default:
return fmt.Errorf("session/ledis: unsupported option '%s'", k)
}
}
l, err := ledis.Open(opt)
if err != nil {
return fmt.Errorf("session/ledis: error opening db: %v", err)
}
c.c, err = l.Select(db)
if err != nil {
return err
}
go c.startGC()
return nil
}
func TestScan(t *testing.T) {
cfg := new(config.Config)
cfg.DataDir = "/tmp/test_scan"
cfg.Addr = "127.0.0.1:11185"
os.RemoveAll(cfg.DataDir)
s, err := NewApp(cfg)
if err != nil {
t.Fatal(err)
}
go s.Run()
defer s.Close()
cc := new(ledis.Config)
cc.Addr = cfg.Addr
cc.MaxIdleConns = 1
c := ledis.NewClient(cc)
defer c.Close()
testKVScan(t, c)
testHashScan(t, c)
testListScan(t, c)
testZSetScan(t, c)
testSetScan(t, c)
testBitScan(t, c)
}
func Open(cfg *config.Config) (*Ledis, error) {
if len(cfg.DataDir) == 0 {
cfg.DataDir = config.DefaultDataDir
}
if cfg.Databases == 0 {
cfg.Databases = 16
} else if cfg.Databases > MaxDatabases {
cfg.Databases = MaxDatabases
}
os.MkdirAll(cfg.DataDir, 0755)
var err error
l := new(Ledis)
l.cfg = cfg
if l.lock, err = filelock.Lock(path.Join(cfg.DataDir, "LOCK")); err != nil {
return nil, err
}
l.quit = make(chan struct{})
if l.ldb, err = store.Open(cfg); err != nil {
return nil, err
}
if cfg.UseReplication {
if l.r, err = rpl.NewReplication(cfg); err != nil {
return nil, err
}
l.rc = make(chan struct{}, 1)
l.rbatch = l.ldb.NewWriteBatch()
l.rDoneCh = make(chan struct{}, 1)
l.wg.Add(1)
go l.onReplication()
//first we must try wait all replication ok
//maybe some logs are not committed
l.WaitReplication()
} else {
l.r = nil
}
l.dbs = make(map[int]*DB, 16)
l.checkTTL()
return l, nil
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
var cfg *config.Config
var err error
if len(*configFile) == 0 {
println("no config set, using default config")
cfg = config.NewConfigDefault()
} else {
cfg, err = config.NewConfigWithFile(*configFile)
}
if err != nil {
println(err.Error())
return
}
if len(*dbName) > 0 {
cfg.DBName = *dbName
}
var app *server.App
app, err = server.NewApp(cfg)
if err != nil {
println(err.Error())
return
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
go func() {
<-sc
app.Close()
}()
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()
app.Run()
}
func newTestRocksDB() *DB {
cfg := new(config.Config)
cfg.DBName = "rocksdb"
cfg.DataDir = "/tmp/testdb"
os.RemoveAll(getStorePath(cfg))
db, err := Open(cfg)
if err != nil {
println(err.Error())
panic(err)
}
return db
}
func Open(cfg *config.Config) (*Ledis, error) {
if len(cfg.DataDir) == 0 {
cfg.DataDir = config.DefaultDataDir
}
ldb, err := store.Open(cfg)
if err != nil {
return nil, err
}
l := new(Ledis)
l.quit = make(chan struct{})
l.jobs = new(sync.WaitGroup)
l.ldb = ldb
if cfg.BinLog.MaxFileNum > 0 && cfg.BinLog.MaxFileSize > 0 {
l.binlog, err = NewBinLog(cfg)
if err != nil {
return nil, err
}
} else {
l.binlog = nil
}
for i := uint8(0); i < MaxDBNumber; i++ {
l.dbs[i] = l.newDB(i)
}
l.activeExpireCycle()
return l, nil
}
func newTestLMDB() *DB {
cfg := new(config.Config)
cfg.DBName = "lmdb"
cfg.DataDir = "/tmp/testdb"
cfg.LMDB.MapSize = 10 * 1024 * 1024
os.RemoveAll(getStorePath(cfg))
db, err := Open(cfg)
if err != nil {
println(err.Error())
panic(err)
}
return db
}
func TestDump(t *testing.T) {
cfgM := new(config.Config)
cfgM.DataDir = "/tmp/test_ledis_master"
os.RemoveAll(cfgM.DataDir)
master, err := Open(cfgM)
if err != nil {
t.Fatal(err)
}
cfgS := new(config.Config)
cfgS.DataDir = "/tmp/test_ledis_slave"
os.RemoveAll(cfgM.DataDir)
var slave *Ledis
if slave, err = Open(cfgS); err != nil {
t.Fatal(err)
}
db, _ := master.Select(0)
db.Set([]byte("a"), []byte("1"))
db.Set([]byte("b"), []byte("2"))
db.Set([]byte("c"), []byte("3"))
if err := master.DumpFile("/tmp/testdb.dump"); err != nil {
t.Fatal(err)
}
if _, err := slave.LoadDumpFile("/tmp/testdb.dump"); err != nil {
t.Fatal(err)
}
it := master.ldb.RangeLimitIterator(nil, nil, store.RangeClose, 0, -1)
for ; it.Valid(); it.Next() {
key := it.Key()
value := it.Value()
if v, err := slave.ldb.Get(key); err != nil {
t.Fatal(err)
} else if !bytes.Equal(v, value) {
t.Fatal("load dump error")
}
}
}
func InitDb() {
initLedisFunc := func() {
cfg := new(config.Config)
cfg.DataDir = beego.AppConfig.String("ledisdb::DataDir")
var err error
nowLedis, err = ledis.Open(cfg)
if err != nil {
println(err.Error())
panic(err)
}
}
ledisOnce.Do(initLedisFunc)
db, _ := beego.AppConfig.Int("ledisdb::DB")
LedisDB, _ = nowLedis.Select(db)
}
func GetStore(cfg *config.Config) (Store, error) {
if len(cfg.DBName) == 0 {
cfg.DBName = config.DefaultDBName
}
s, ok := dbs[cfg.DBName]
if !ok {
return nil, fmt.Errorf("store %s is not registered", cfg.DBName)
}
return s, nil
}
func getTestDB() *DB {
f := func() {
cfg := new(config.Config)
cfg.DataDir = "/tmp/test_ledis"
cfg.BinLog.MaxFileSize = 1073741824
cfg.BinLog.MaxFileNum = 3
os.RemoveAll(cfg.DataDir)
var err error
testLedis, err = Open(cfg)
if err != nil {
println(err.Error())
panic(err)
}
}
testLedisOnce.Do(f)
db, _ := testLedis.Select(0)
return db
}
func NewApp(cfg *config.Config) (*App, error) {
if len(cfg.DataDir) == 0 {
println("use default datadir %s", config.DefaultDataDir)
cfg.DataDir = config.DefaultDataDir
}
app := new(App)
app.quit = make(chan struct{})
app.closed = false
app.cfg = cfg
var err error
if app.info, err = newInfo(app); err != nil {
return nil, err
}
if app.listener, err = net.Listen(netType(cfg.Addr), cfg.Addr); err != nil {
return nil, err
}
if len(cfg.HttpAddr) > 0 {
if app.httpListener, err = net.Listen(netType(cfg.HttpAddr), cfg.HttpAddr); err != nil {
return nil, err
}
}
if len(cfg.AccessLog) > 0 {
if path.Dir(cfg.AccessLog) == "." {
app.access, err = newAcessLog(path.Join(cfg.DataDir, cfg.AccessLog))
} else {
app.access, err = newAcessLog(cfg.AccessLog)
}
if err != nil {
return nil, err
}
}
if app.ldb, err = ledis.Open(cfg); err != nil {
return nil, err
}
app.m = newMaster(app)
app.openScript()
return app, nil
}
func testTx(t *testing.T, name string) {
cfg := new(config.Config)
cfg.DataDir = "/tmp/ledis_test_tx"
cfg.DBName = name
cfg.LMDB.MapSize = 10 * 1024 * 1024
os.RemoveAll(cfg.DataDir)
l, err := Open(cfg)
if err != nil {
t.Fatal(err)
}
defer l.Close()
db, _ := l.Select(0)
testTxRollback(t, db)
testTxCommit(t, db)
testTxSelect(t, db)
}
// init ledis session
// savepath like ledis server saveDataPath,pool size
// e.g. 127.0.0.1:6379,100,astaxie
func (lp *LedisProvider) SessionInit(maxlifetime int64, savePath string) error {
var err error
lp.maxlifetime = maxlifetime
configs := strings.Split(savePath, ",")
if len(configs) == 1 {
lp.savePath = configs[0]
} else if len(configs) == 2 {
lp.savePath = configs[0]
lp.db, err = strconv.Atoi(configs[1])
if err != nil {
return err
}
}
cfg := new(config.Config)
cfg.DataDir = lp.savePath
nowLedis, err := ledis.Open(cfg)
c, err = nowLedis.Select(lp.db)
if err != nil {
println(err)
return nil
}
return nil
}
func TestStore(t *testing.T) {
cfg := new(config.Config)
cfg.DataDir = "/tmp/testdb"
cfg.LMDB.MapSize = 10 * 1024 * 1024
ns := driver.ListStores()
for _, s := range ns {
cfg.DBName = s
os.RemoveAll(getStorePath(cfg))
db, err := Open(cfg)
if err != nil {
t.Fatal(err)
}
testStore(db, t)
testClear(db, t)
testTx(db, t)
db.Close()
}
}
func startTestApp() {
f := func() {
newTestLedisClient()
cfg := new(config.Config)
cfg.DataDir = "/tmp/testdb"
os.RemoveAll(cfg.DataDir)
cfg.Addr = "127.0.0.1:16380"
os.RemoveAll("/tmp/testdb")
var err error
testApp, err = NewApp(cfg)
if err != nil {
println(err.Error())
panic(err)
}
go testApp.Run()
}
testAppOnce.Do(f)
}
func Open(cfg *config.Config) (*Ledis, error) {
if len(cfg.DataDir) == 0 {
fmt.Printf("no datadir set, use default %s\n", config.DefaultDataDir)
cfg.DataDir = config.DefaultDataDir
}
ldb, err := store.Open(cfg)
if err != nil {
return nil, err
}
l := new(Ledis)
l.quit = make(chan struct{})
l.jobs = new(sync.WaitGroup)
l.ldb = ldb
if cfg.BinLog.MaxFileNum > 0 && cfg.BinLog.MaxFileSize > 0 {
println("binlog will be refactored later, use your own risk!!!")
l.binlog, err = NewBinLog(cfg)
if err != nil {
return nil, err
}
} else {
l.binlog = nil
}
for i := uint8(0); i < MaxDBNumber; i++ {
l.dbs[i] = newDB(l, i)
}
l.activeExpireCycle()
return l, nil
}
func NewApp(cfg *config.Config) (*App, error) {
if len(cfg.DataDir) == 0 {
println("use default datadir %s", config.DefaultDataDir)
cfg.DataDir = config.DefaultDataDir
}
app := new(App)
app.quit = make(chan struct{})
app.closed = false
app.cfg = cfg
app.slaves = make(map[string]*client)
app.slaveSyncAck = make(chan uint64)
app.rcs = make(map[*respClient]struct{})
app.migrateClients = make(map[string]*goredis.Client)
app.newMigrateKeyLockers()
var err error
if app.info, err = newInfo(app); err != nil {
return nil, err
}
if app.listener, err = net.Listen(netType(cfg.Addr), cfg.Addr); err != nil {
return nil, err
}
if len(cfg.HttpAddr) > 0 {
if app.httpListener, err = net.Listen(netType(cfg.HttpAddr), cfg.HttpAddr); err != nil {
return nil, err
}
}
if len(cfg.AccessLog) > 0 {
if path.Dir(cfg.AccessLog) == "." {
app.access, err = newAcessLog(path.Join(cfg.DataDir, cfg.AccessLog))
} else {
app.access, err = newAcessLog(cfg.AccessLog)
}
if err != nil {
return nil, err
}
}
if app.snap, err = newSnapshotStore(cfg); err != nil {
return nil, err
}
if len(app.cfg.SlaveOf) > 0 {
//slave must readonly
app.cfg.Readonly = true
}
if app.ldb, err = ledis.Open(cfg); err != nil {
return nil, err
}
app.m = newMaster(app)
app.openScript()
app.ldb.AddNewLogEventHandler(app.publishNewLog)
return app, nil
}
func TestReplication(t *testing.T) {
var master *Ledis
var slave *Ledis
var err error
cfgM := new(config.Config)
cfgM.DataDir = "/tmp/test_repl/master"
cfgM.BinLog.MaxFileNum = 10
cfgM.BinLog.MaxFileSize = 50
os.RemoveAll(cfgM.DataDir)
master, err = Open(cfgM)
if err != nil {
t.Fatal(err)
}
cfgS := new(config.Config)
cfgS.DataDir = "/tmp/test_repl/slave"
os.RemoveAll(cfgS.DataDir)
slave, err = Open(cfgS)
if err != nil {
t.Fatal(err)
}
db, _ := master.Select(0)
db.Set([]byte("a"), []byte("value"))
db.Set([]byte("b"), []byte("value"))
db.Set([]byte("c"), []byte("value"))
if tx, err := db.Begin(); err == nil {
tx.HSet([]byte("a"), []byte("1"), []byte("value"))
tx.HSet([]byte("b"), []byte("2"), []byte("value"))
tx.HSet([]byte("c"), []byte("3"), []byte("value"))
tx.Commit()
} else {
db.HSet([]byte("a"), []byte("1"), []byte("value"))
db.HSet([]byte("b"), []byte("2"), []byte("value"))
db.HSet([]byte("c"), []byte("3"), []byte("value"))
}
m, _ := db.Multi()
m.Set([]byte("a1"), []byte("value"))
m.Set([]byte("b1"), []byte("value"))
m.Set([]byte("c1"), []byte("value"))
m.Close()
for _, name := range master.binlog.LogNames() {
p := path.Join(master.binlog.LogPath(), name)
err = slave.ReplicateFromBinLog(p)
if err != nil {
t.Fatal(err)
}
}
if err = checkLedisEqual(master, slave); err != nil {
t.Fatal(err)
}
slave.FlushAll()
db.Set([]byte("a1"), []byte("value"))
db.Set([]byte("b1"), []byte("value"))
db.Set([]byte("c1"), []byte("value"))
db.HSet([]byte("a1"), []byte("1"), []byte("value"))
db.HSet([]byte("b1"), []byte("2"), []byte("value"))
db.HSet([]byte("c1"), []byte("3"), []byte("value"))
if tx, err := db.Begin(); err == nil {
tx.HSet([]byte("a1"), []byte("1"), []byte("value1"))
tx.HSet([]byte("b1"), []byte("2"), []byte("value1"))
tx.HSet([]byte("c1"), []byte("3"), []byte("value1"))
tx.Rollback()
}
info := new(BinLogAnchor)
info.LogFileIndex = 1
info.LogPos = 0
var buf bytes.Buffer
var n int
for {
buf.Reset()
n, err = master.ReadEventsTo(info, &buf)
if err != nil {
t.Fatal(err)
} else if info.LogFileIndex == -1 {
t.Fatal("invalid log file index -1")
} else if info.LogFileIndex == 0 {
t.Fatal("invalid log file index 0")
} else {
if err = slave.ReplicateFromReader(&buf); err != nil {
t.Fatal(err)
}
if n == 0 {
break
}
}
}
if err = checkLedisEqual(master, slave); err != nil {
t.Fatal(err)
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
var cfg *config.Config
var err error
if len(*configFile) == 0 {
println("no config set, using default config")
cfg = config.NewConfigDefault()
} else {
cfg, err = config.NewConfigWithFile(*configFile)
}
if err != nil {
println(err.Error())
return
}
if len(*addr) > 0 {
cfg.Addr = *addr
}
if len(*dataDir) > 0 {
cfg.DataDir = *dataDir
}
if len(*dbName) > 0 {
cfg.DBName = *dbName
}
if *databases > 0 {
cfg.Databases = *databases
}
// check bool flag, use it.
for _, arg := range os.Args {
arg := strings.ToLower(arg)
switch arg {
case "-rpl", "-rpl=true", "-rpl=false":
cfg.UseReplication = *rpl
case "-readonly", "-readonly=true", "-readonly=false":
cfg.Readonly = *readonly
case "-rpl_sync", "-rpl_sync=true", "-rpl_sync=false":
cfg.Replication.Sync = *rplSync
}
}
if len(*slaveof) > 0 {
cfg.SlaveOf = *slaveof
cfg.Readonly = true
cfg.UseReplication = true
}
if *ttlCheck > 0 {
cfg.TTLCheckInterval = *ttlCheck
}
var app *server.App
app, err = server.NewApp(cfg)
if err != nil {
println(err.Error())
return
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
os.Kill,
os.Interrupt,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
if *usePprof {
go func() {
log.Println(http.ListenAndServe(fmt.Sprintf(":%d", *pprofPort), nil))
}()
}
go app.Run()
<-sc
println("ledis-server is closing")
app.Close()
println("ledis-server is closed")
}
func TestReplication(t *testing.T) {
data_dir := "/tmp/test_replication"
os.RemoveAll(data_dir)
masterCfg := new(config.Config)
masterCfg.DataDir = fmt.Sprintf("%s/master", data_dir)
masterCfg.Addr = "127.0.0.1:11182"
masterCfg.BinLog.MaxFileSize = 1 * 1024 * 1024
masterCfg.BinLog.MaxFileNum = 10
var master *App
var slave *App
var err error
master, err = NewApp(masterCfg)
if err != nil {
t.Fatal(err)
}
defer master.Close()
slaveCfg := new(config.Config)
slaveCfg.DataDir = fmt.Sprintf("%s/slave", data_dir)
slaveCfg.Addr = "127.0.0.1:11183"
slaveCfg.SlaveOf = masterCfg.Addr
slave, err = NewApp(slaveCfg)
if err != nil {
t.Fatal(err)
}
defer slave.Close()
go master.Run()
db, _ := master.ldb.Select(0)
value := make([]byte, 10)
db.Set([]byte("a"), value)
db.Set([]byte("b"), value)
db.HSet([]byte("a"), []byte("1"), value)
db.HSet([]byte("b"), []byte("2"), value)
go slave.Run()
time.Sleep(1 * time.Second)
if err = checkDataEqual(master, slave); err != nil {
t.Fatal(err)
}
db.Set([]byte("a1"), value)
db.Set([]byte("b1"), value)
db.HSet([]byte("a1"), []byte("1"), value)
db.HSet([]byte("b1"), []byte("2"), value)
time.Sleep(1 * time.Second)
if err = checkDataEqual(master, slave); err != nil {
t.Fatal(err)
}
slave.slaveof("")
db.Set([]byte("a2"), value)
db.Set([]byte("b2"), value)
db.HSet([]byte("a2"), []byte("1"), value)
db.HSet([]byte("b2"), []byte("2"), value)
db.Set([]byte("a3"), value)
db.Set([]byte("b3"), value)
db.HSet([]byte("a3"), []byte("1"), value)
db.HSet([]byte("b3"), []byte("2"), value)
if err = checkDataEqual(master, slave); err == nil {
t.Fatal("must error")
}
slave.slaveof(masterCfg.Addr)
time.Sleep(1 * time.Second)
if err = checkDataEqual(master, slave); err != nil {
t.Fatal(err)
}
}
func TestLuaCall(t *testing.T) {
cfg := new(config.Config)
cfg.Addr = ":11188"
cfg.DataDir = "/tmp/testscript"
cfg.DBName = "memory"
app, e := NewApp(cfg)
if e != nil {
t.Fatal(e)
}
go app.Run()
defer app.Close()
db, _ := app.ldb.Select(0)
m, _ := db.Multi()
defer m.Close()
luaClient := app.s.c
luaClient.db = m.DB
luaClient.script = m
l := app.s.l
err := app.s.l.DoString(testScript1)
if err != nil {
t.Fatal(err)
}
v := luaReplyToLedisReply(l)
if vv, ok := v.([]interface{}); ok {
if len(vv) != 3 {
t.Fatal(len(vv))
}
} else {
t.Fatal(fmt.Sprintf("%v %T", v, v))
}
err = app.s.l.DoString(testScript2)
if err != nil {
t.Fatal(err)
}
v = luaReplyToLedisReply(l)
if vv := v.(string); vv != "PONG" {
t.Fatal(fmt.Sprintf("%v %T", v, v))
}
err = app.s.l.DoString(testScript3)
if err != nil {
t.Fatal(err)
}
if v, err := db.Get([]byte("1")); err != nil {
t.Fatal(err)
} else if string(v) != "a" {
t.Fatal(string(v))
}
err = app.s.l.DoString(testScript4)
if err != nil {
t.Fatal(err)
}
if luaClient.db.Index() != 2 {
t.Fatal(luaClient.db.Index())
}
db2, _ := app.ldb.Select(2)
if v, err := db2.Get([]byte("2")); err != nil {
t.Fatal(err)
} else if string(v) != "b" {
t.Fatal(string(v))
}
luaClient.db = nil
}