func (tbl *Table) Sync(req *PkgArgs) ([]byte, bool) { var in proto.PkgMultiOp _, err := in.Decode(req.Pkg) if err == nil { // Sync full DB, including DB 0 var wb = tbl.db.NewWriteBatch() for i := 0; i < len(in.Kvs); i++ { tbl.setSyncKV(wb, in.DbId, &in.Kvs[i]) } tbl.rwMtx.RLock() err = tbl.db.Commit(wb) tbl.rwMtx.RUnlock() wb.Destroy() if err == nil { return nil, true } else { in.SetErrCode(table.EcWriteFail) log.Printf("setRawKV failed: %s\n", err) } } else { in.SetErrCode(table.EcDecodeFail) } return replyHandle(&in), table.EcOk == in.ErrCode }
func myMIncr(in proto.PkgMultiOp, au Authorize, wa *WriteAccess, expected bool, t *testing.T) proto.PkgMultiOp { var pkg = make([]byte, in.Length()) _, err := in.Encode(pkg) if err != nil { t.Fatalf("Encode failed: ", err) } pkg, ok := testTbl.MIncr(&PkgArgs{in.Cmd, in.DbId, in.Seq, pkg}, au, wa) if ok != expected { if expected { t.Fatalf("MIncr failed") } else { t.Fatalf("MIncr should fail") } } var out proto.PkgMultiOp _, err = out.Decode(pkg) if err != nil { t.Fatalf("Decode failed: ", err) } if expected { if out.ErrCode != 0 { t.Fatalf("Failed with ErrCode %d", out.ErrCode) } } if out.DbId != in.DbId || out.Seq != in.Seq { t.Fatalf("DbId/Seq mismatch") } return out }
func myMGet(in proto.PkgMultiOp, au Authorize, wa *WriteAccess, t *testing.T) proto.PkgMultiOp { var pkg = make([]byte, in.Length()) _, err := in.Encode(pkg) if err != nil { t.Fatalf("Encode failed: ", err) } pkg = testTbl.MGet(&PkgArgs{in.Cmd, in.DbId, in.Seq, pkg}, au, wa) var out proto.PkgMultiOp _, err = out.Decode(pkg) if err != nil { t.Fatalf("Decode failed: ", err) } if out.ErrCode != 0 { t.Fatalf("Failed with ErrCode %d", out.ErrCode) } if out.DbId != in.DbId || out.Seq != in.Seq { t.Fatalf("DbId/Seq mismatch") } return out }
func (srv *Server) replyMultiOp(req *Request, errCode int8) { var out proto.PkgMultiOp out.Cmd = req.Cmd out.DbId = req.DbId out.Seq = req.Seq out.ErrCode = errCode var pkg = make([]byte, out.Length()) _, err := out.Encode(pkg) if err != nil { log.Fatalf("Encode failed: %s\n", err) } srv.sendResp(false, req, pkg) }
func replyMulti(out *proto.PkgMultiOp) []byte { var pkgLen = out.Length() if pkgLen > proto.MaxPkgLen { out.Kvs = nil out.SetErrCode(table.EcInvPkgLen) pkgLen = out.Length() } var pkg = make([]byte, pkgLen) _, err := out.Encode(pkg) if err != nil { log.Fatalf("Encode failed: %s\n", err) } return pkg }
func SeekAndCopySyncPkg(it *Iterator, p *proto.PkgMultiOp, migration bool, migSlotId uint16) bool { p.PkgFlag &^= 0xFF p.ErrCode = 0 p.Kvs = nil var size = 0 for i := 0; i < 10 && size < 400 && it.Valid(); i++ { var kv proto.KeyValue dbId, slotId, ok := seekAndCopySyncKV(it, &kv) if !ok { return false } if migration && migSlotId != slotId { if migSlotId < slotId { return false } else if migSlotId > slotId { seekToSlot(it, migSlotId, 0, 0) if !it.Valid() { return false } dbId, slotId, ok = seekAndCopySyncKV(it, &kv) if !ok || migSlotId != slotId { return false } } } if len(p.Kvs) == 0 { p.DbId = dbId } else if p.DbId != dbId { return true } p.Kvs = append(p.Kvs, kv) size += 13 + len(kv.RowKey) + len(kv.ColKey) + len(kv.Value) it.Next() } return true }
func TestTableMDel(t *testing.T) { var in proto.PkgMultiOp in.Cmd = proto.CmdMSet in.DbId = 2 in.Seq = 20 in.Kvs = append(in.Kvs, getTestKV(2, []byte("row1"), []byte("col0"), nil, 0, 0)) in.Kvs = append(in.Kvs, getTestKV(2, []byte("row1"), []byte("col1"), nil, 0, 0)) in.Kvs = append(in.Kvs, getTestKV(2, []byte("row1"), []byte("col2"), nil, 0, 0)) out := myMDel(in, testAuth, getTestWA(), true, t) if len(out.Kvs) != 3 { t.Fatalf("Invalid KV number: %d", len(out.Kvs)) } for i := 0; i < len(out.Kvs); i++ { if out.Kvs[i].ErrCode != 0 { t.Fatalf("ErrCode is 0") } if out.Kvs[i].TableId != 2 { t.Fatalf("TableId mismatch") } if bytes.Compare(out.Kvs[i].RowKey, []byte("row1")) != 0 { t.Fatalf("RowKey mismatch") } if bytes.Compare(out.Kvs[i].ColKey, []byte(fmt.Sprintf("col%d", i))) != 0 { t.Fatalf("ColKey mismatch") } } }
// MGet, MSet, MDel, MIncr, ZMGet, ZMSet, ZMDel, ZMIncr func (c *Context) goMultiOp(zop bool, args multiArgs, cmd uint8, done chan *Call) (*Call, error) { call := c.cli.newCall(cmd, done) if call.err != nil { return call, call.err } var p proto.PkgMultiOp p.Seq = call.seq p.DbId = c.dbId p.Cmd = call.cmd // ZMGet, ZMSet, ZMDel, ZMIncr if zop { p.PkgFlag |= proto.FlagZop } p.Kvs = make([]proto.KeyValue, args.length()) args.toKV(p.Kvs) var pkgLen = p.Length() if pkgLen > proto.MaxPkgLen { c.cli.errCall(call, ErrInvPkgLen) return call, call.err } call.pkg = make([]byte, pkgLen) _, err := p.Encode(call.pkg) if err != nil { c.cli.errCall(call, err) return call, err } c.cli.sending <- call return call, nil }
func checkMultiOp(in *proto.PkgMultiOp, req *PkgArgs, au Authorize) bool { n, err := in.Decode(req.Pkg) if err != nil || n != len(req.Pkg) { in.ErrCode = table.EcDecodeFail } if in.ErrCode == 0 && in.DbId == proto.AdminDbId { in.ErrCode = table.EcInvDbId } if in.ErrCode == 0 && !au.IsAuth(in.DbId) { in.ErrCode = table.EcNoPrivilege } if in.ErrCode != 0 { in.Kvs = nil } return in.ErrCode == 0 }
func (ms *master) fullSync(tbl *store.Table) (uint64, error) { var lastSeq uint64 if ms.lastSeq > 0 { lastSeq = ms.lastSeq if ms.migration { log.Printf("Migration lastSeq is not 0, close now!\n") return lastSeq, errors.New("migration lastSeq is not 0") } else { log.Printf("Already full synced to %s\n", ms.slaveAddr) return lastSeq, nil } } // Stop write globally rwMtx := tbl.GetRWMutex() rwMtx.Lock() var chanLen int for lastSeq, chanLen = ms.bin.GetLogSeqChanLen(); chanLen != 0; { log.Println("Stop write globally for 1ms") time.Sleep(time.Millisecond) lastSeq, chanLen = ms.bin.GetLogSeqChanLen() } var it = tbl.NewIterator(false) rwMtx.Unlock() defer it.Destroy() // Open BinLog reader to keep log files from deleting var err = ms.openReader(lastSeq) if err != nil { return lastSeq, err } // Full sync var p proto.PkgMultiOp p.Cmd = proto.CmdSync for it.SeekToFirst(); it.Valid(); { ok := store.SeekAndCopySyncPkg(it, &p, ms.migration, ms.slotId) if ms.cli.IsClosed() { return lastSeq, nil } if len(p.Kvs) > 0 { p.Seq = 0 var pkg = make([]byte, p.Length()) p.Encode(pkg) ms.cli.AddResp(pkg) } if !ok { break } } // Tell slave full sync finished if ms.migration { ms.syncStatus(store.KeyFullSyncEnd, 0) log.Printf("Full migration to %s slotId %d finished\n", ms.slaveAddr, ms.slotId) } else { ms.syncStatus(store.KeyFullSyncEnd, lastSeq) log.Printf("Full sync to %s finished\n", ms.slaveAddr) } return lastSeq, nil }
func TestTableZScan(t *testing.T) { // MZSET { var in proto.PkgMultiOp in.Cmd = proto.CmdMSet in.DbId = 2 in.Seq = 20 in.PkgFlag |= proto.FlagZop in.Kvs = append(in.Kvs, getTestKV(2, []byte("row2"), []byte("col0"), []byte("v0"), 40, 0)) in.Kvs = append(in.Kvs, getTestKV(2, []byte("row2"), []byte("col1"), []byte("v1"), 30, 0)) in.Kvs = append(in.Kvs, getTestKV(2, []byte("row2"), []byte("col2"), []byte("v2"), 20, 0)) in.Kvs = append(in.Kvs, getTestKV(2, []byte("row2"), []byte("col3"), []byte("v3"), 10, 0)) in.Kvs = append(in.Kvs, getTestKV(2, []byte("row2"), []byte("col4"), []byte("v4"), 0, 0)) myMSet(in, testAuth, getTestWA(), true, t) } // ZSCAN ASC order by SCORE var in proto.PkgScanReq in.Cmd = proto.CmdScan in.DbId = 2 in.Seq = 20 in.Num = 10 in.TableId = 2 in.RowKey = []byte("row2") in.SetScore(-1) in.ColKey = []byte("") in.PkgFlag |= proto.FlagScanAsc in.SetColSpace(proto.ColSpaceScore1) // order by SCORE out := myScan(in, testAuth, t) if len(out.Kvs) != 5 { t.Fatalf("Invalid KV number: %d", len(out.Kvs)) } if out.PkgFlag&proto.FlagScanEnd == 0 { t.Fatalf("Scan should end") } for i := 0; i < len(out.Kvs); i++ { idx := 4 - i if out.Kvs[i].ErrCode != 0 { t.Fatalf("ErrCode is 0") } if out.Kvs[i].TableId != 2 { t.Fatalf("TableId mismatch") } if bytes.Compare(out.Kvs[i].RowKey, []byte("row2")) != 0 { t.Fatalf("RowKey mismatch") } if bytes.Compare(out.Kvs[i].ColKey, []byte(fmt.Sprintf("col%d", idx))) != 0 { t.Fatalf("ColKey mismatch") } if out.Kvs[i].Score != int64(i*10) { t.Fatalf("Score mismatch") } if bytes.Compare(out.Kvs[i].Value, []byte(fmt.Sprintf("v%d", idx))) != 0 { t.Fatalf("Value mismatch") } } // ZSCAN DESC order by SCORE in.PkgFlag &^= 0xFF in.PkgFlag |= proto.FlagScanKeyStart out = myScan(in, testAuth, t) if len(out.Kvs) != 5 { t.Fatalf("Invalid KV number: %d", len(out.Kvs)) } if out.PkgFlag&proto.FlagScanEnd == 0 { t.Fatalf("Scan should end") } for i := 0; i < len(out.Kvs); i++ { idx := 4 - i if out.Kvs[i].ErrCode != 0 { t.Fatalf("ErrCode is 0") } if out.Kvs[i].TableId != 2 { t.Fatalf("TableId mismatch") } if bytes.Compare(out.Kvs[i].RowKey, []byte("row2")) != 0 { t.Fatalf("RowKey mismatch") } if bytes.Compare(out.Kvs[i].ColKey, []byte(fmt.Sprintf("col%d", i))) != 0 { t.Fatalf("ColKey mismatch") } if out.Kvs[i].Score != int64(idx*10) { t.Fatalf("Score mismatch") } if bytes.Compare(out.Kvs[i].Value, []byte(fmt.Sprintf("v%d", i))) != 0 { t.Fatalf("Value mismatch") } } }
// Get call reply. The real reply types are: // Auth/Ping/(Z)Set/(Z)Del: nil; // (Z)Get: GetReply; // (Z)Incr: IncrReply; // (Z)MGet: []GetReply; // (Z)MSet: []SetReply; // (Z)MDel: []DelReply; // (Z)MIncr: []IncrReply; // (Z)Scan: ScanReply; // Dump: DumpReply; func (call *Call) Reply() (interface{}, error) { if call.err != nil { return nil, call.err } if !call.ready { return nil, ErrCallNotReady } if proto.CmdAuth == call.cmd || proto.CmdPing == call.cmd || proto.CmdIncr == call.cmd || proto.CmdDel == call.cmd || proto.CmdSet == call.cmd || proto.CmdGet == call.cmd { var p proto.PkgOneOp _, err := p.Decode(call.pkg) if err != nil { call.err = err return nil, call.err } if p.ErrCode < 0 { return nil, getErr(p.ErrCode) } switch call.cmd { case proto.CmdAuth: return nil, nil case proto.CmdPing: return nil, nil case proto.CmdIncr: return IncrReply{p.ErrCode, p.TableId, copyBytes(p.RowKey), copyBytes(p.ColKey), copyBytes(p.Value), p.Score}, nil case proto.CmdDel: return nil, nil case proto.CmdSet: return nil, nil case proto.CmdGet: return GetReply{p.ErrCode, p.TableId, copyBytes(p.RowKey), copyBytes(p.ColKey), copyBytes(p.Value), p.Score, p.Cas}, nil } } if proto.CmdMIncr == call.cmd || proto.CmdMDel == call.cmd || proto.CmdMSet == call.cmd || proto.CmdMGet == call.cmd { var p proto.PkgMultiOp _, err := p.Decode(call.pkg) if err != nil { call.err = err return nil, call.err } if p.ErrCode < 0 { return nil, getErr(p.ErrCode) } switch call.cmd { case proto.CmdMIncr: var r = make([]IncrReply, len(p.Kvs)) for i := 0; i < len(r); i++ { r[i] = IncrReply{p.Kvs[i].ErrCode, p.Kvs[i].TableId, copyBytes(p.Kvs[i].RowKey), copyBytes(p.Kvs[i].ColKey), copyBytes(p.Kvs[i].Value), p.Kvs[i].Score} } return r, nil case proto.CmdMDel: var r = make([]DelReply, len(p.Kvs)) for i := 0; i < len(r); i++ { r[i] = DelReply{p.Kvs[i].ErrCode, p.Kvs[i].TableId, copyBytes(p.Kvs[i].RowKey), copyBytes(p.Kvs[i].ColKey)} } return r, nil case proto.CmdMSet: var r = make([]SetReply, len(p.Kvs)) for i := 0; i < len(r); i++ { r[i] = SetReply{p.Kvs[i].ErrCode, p.Kvs[i].TableId, copyBytes(p.Kvs[i].RowKey), copyBytes(p.Kvs[i].ColKey)} } return r, nil case proto.CmdMGet: var r = make([]GetReply, len(p.Kvs)) for i := 0; i < len(r); i++ { r[i] = GetReply{p.Kvs[i].ErrCode, p.Kvs[i].TableId, copyBytes(p.Kvs[i].RowKey), copyBytes(p.Kvs[i].ColKey), copyBytes(p.Kvs[i].Value), p.Kvs[i].Score, p.Kvs[i].Cas} } return r, nil } } switch call.cmd { case proto.CmdScan: var p proto.PkgScanResp _, err := p.Decode(call.pkg) if err != nil { call.err = err return nil, call.err } if p.ErrCode < 0 { return nil, getErr(p.ErrCode) } var r ScanReply r.ctx = call.ctx.(scanContext) r.TableId = r.ctx.tableId r.RowKey = r.ctx.rowKey r.End = (p.PkgFlag&proto.FlagScanEnd != 0) r.Kvs = make([]ScanKV, len(p.Kvs)) for i := 0; i < len(p.Kvs); i++ { r.Kvs[i] = ScanKV{copyBytes(p.Kvs[i].ColKey), copyBytes(p.Kvs[i].Value), p.Kvs[i].Score} } return r, nil case proto.CmdDump: var p proto.PkgDumpResp _, err := p.Decode(call.pkg) if err != nil { call.err = err return nil, call.err } if p.ErrCode < 0 { return nil, getErr(p.ErrCode) } var r DumpReply r.ctx = call.ctx.(dumpContext) r.ctx.lastSlotId = p.LastSlotId r.ctx.slotStart = (p.PkgFlag&proto.FlagDumpSlotStart != 0) r.End = (p.PkgFlag&proto.FlagDumpEnd != 0) r.Kvs = make([]DumpKV, len(p.Kvs)) for i := 0; i < len(p.Kvs); i++ { r.Kvs[i] = DumpKV{p.Kvs[i].TableId, p.Kvs[i].ColSpace, copyBytes(p.Kvs[i].RowKey), copyBytes(p.Kvs[i].ColKey), copyBytes(p.Kvs[i].Value), p.Kvs[i].Score} } return r, nil } switch call.cmd { case proto.CmdSlaveOf: return call.replyInnerCtrl(&ctrl.PkgSlaveOf{}) case proto.CmdMigrate: return call.replyInnerCtrl(&ctrl.PkgMigrate{}) case proto.CmdSlaveSt: return call.replyInnerCtrl(&ctrl.PkgSlaveStatus{}) case proto.CmdDelSlot: return call.replyInnerCtrl(&ctrl.PkgDelSlot{}) } return nil, ErrUnknownCmd }