func TestFileStoreUpdate(t *testing.T) {
cleanSnapshot("./snapshot/")
fs := NewKiteFileStore(".", 5000000, 1*time.Second)
fs.Start()
for i := 0; i < 100; i++ {
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String(fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(true),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
// log.Printf("------------%s", entity.Header)
succ := fs.Save(entity)
if !succ {
t.Fail()
}
}
//commit and check
for i := 0; i < 100; i++ {
id := fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"
//创建消息
msg := &store.MessageEntity{
MessageId: id,
DeliverCount: 1,
SuccGroups: []string{},
FailGroups: []string{"s-mts-test"}}
succ := fs.UpdateEntity(msg)
if !succ {
t.Fail()
}
//check entity
entity := fs.Query(id)
// log.Printf("++++++++++++++|%s|%s", entity.Header, string(entity.GetBody().([]byte)))
if nil == entity {
t.Fail()
} else if !entity.Commit && entity.DeliverCount != 1 &&
entity.FailGroups[0] != "s-mts-test" {
t.Fail()
// log.Println(entity)
}
}
fs.Stop()
// cleanSnapshot("./snapshot/")
}
func TestConvert2Entity(t *testing.T) {
options := MysqlOptions{
Addr: "localhost:3306",
Username: "******",
Password: "",
ShardNum: 4,
BatchUpSize: 1000,
BatchDelSize: 1000,
FlushPeriod: 1 * time.Minute,
MaxIdleConn: 2,
MaxOpenConn: 4}
hs := newDbShard(options)
c := convertor{}
sqlwrapper := newSqlwrapper("kite_msg", hs, store.MessageEntity{})
sqlwrapper.initSQL()
c.columns = sqlwrapper.columns
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String("26c03f00665462591f696a980b5a6c4"),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(false),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
entity.SuccGroups = []string{"go-kite-test"}
hn, _ := os.Hostname()
entity.KiteServer = hn
params := c.Convert2Params(entity)
t.Logf("TestConvert2Entity|Convert2Params|%s\n", params)
econ := &store.MessageEntity{}
c.Convert2Entity(params, econ, func(colname string) bool {
return false
})
t.Logf("TestConvert2Entity|Convert2Entity|%s\n", econ)
if econ.MessageId != entity.MessageId {
t.Fail()
}
if econ.ExpiredTime != entity.ExpiredTime {
t.Fail()
}
}
func (self *AcceptHandler) Process(ctx *pipe.DefaultPipelineContext, event pipe.IEvent) error {
// log.Debug("AcceptHandler|Process|%s|%t\n", self.GetName(), event)
ae, ok := self.cast(event)
if !ok {
return pipe.ERROR_INVALID_EVENT_TYPE
}
//这里处理一下ae,做一下校验
var msg *store.MessageEntity
switch ae.msgType {
case protocol.CMD_DELIVER_ACK:
//收到投递结果直接attach响应
// log.DebugLog("kite_handler", "AcceptHandler|DELIVER_ACK|%s|%t", ae.opaque, ae.msg)
ae.remoteClient.Attach(ae.opaque, ae.msg)
return nil
case protocol.CMD_HEARTBEAT:
hb := ae.msg.(*protocol.HeartBeat)
event = pipe.NewHeartbeatEvent(ae.remoteClient, ae.opaque, hb.GetVersion())
ctx.SendForward(event)
return nil
case protocol.CMD_BYTES_MESSAGE:
msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.BytesMessage)))
case protocol.CMD_STRING_MESSAGE:
msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.StringMessage)))
default:
//这只是一个bug不支持的数据类型能给你
log.WarnLog("kite_handler", "AcceptHandler|Process|%s|%t", INVALID_MSG_TYPE_ERROR, ae.msg)
}
if nil != msg {
msg.PublishTime = time.Now().Unix()
msg.KiteServer = self.kiteserver
deliver := newPersistentEvent(msg, ae.remoteClient, ae.opaque)
ctx.SendForward(deliver)
return nil
}
return INVALID_MSG_TYPE_ERROR
}
func TestFileStoreDelete(t *testing.T) {
cleanSnapshot("./snapshot/")
fs := NewKiteFileStore(".", 5000000, 1*time.Second)
fs.Start()
for i := 0; i < 100; i++ {
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String(fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(true),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
succ := fs.Save(entity)
if !succ {
t.Fail()
}
}
//commit and check
for i := 0; i < 100; i++ {
id := fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"
//delete
fs.Delete(id)
//check entity
entity := fs.Query(id)
if nil != entity {
t.Fail()
}
}
fs.Stop()
cleanSnapshot("./snapshot/")
}
func TestRecoverManager(t *testing.T) {
pipeline := NewDefaultPipeline()
kitedb := memory.NewKiteMemoryStore(100, 100)
messageid := store.MessageId()
t.Logf("messageid:%s\b", messageid)
entity := store.NewMessageEntity(protocol.NewQMessage(buildStringMessage(messageid)))
kitedb.Save(entity)
go func() {
for {
log.Println(kitedb.Monitor())
time.Sleep(1 * time.Second)
}
}()
fs := stat.NewFlowStat("recover")
fs.Start()
ch := make(chan bool, 1)
// 临时在这里创建的BindExchanger
exchanger := binding.NewBindExchanger("localhost:2181", "127.0.0.1:13800")
pipeline.RegisteHandler("deliverpre", handler.NewDeliverPreHandler("deliverpre", kitedb, exchanger, fs, 100))
pipeline.RegisteHandler("deliver", newmockDeliverHandler("deliver", ch))
hostname, _ := os.Hostname()
rm := NewRecoverManager(hostname, 16*time.Second, pipeline, kitedb)
rm.Start()
select {
case succ := <-ch:
log.Printf("--------------recover %s\n", succ)
case <-time.After(5 * time.Second):
t.Fail()
log.Println("waite recover deliver timeout\n")
}
}
func innerT(kiteMysql *KiteMysqlStore, msg interface{}, msgid string, t *testing.T) {
qm := protocol.NewQMessage(msg)
entity := store.NewMessageEntity(qm)
entity.SuccGroups = []string{"go-kite-test"}
hn := "localhost"
entity.KiteServer = hn
entity.PublishTime = time.Now().Unix()
succ := kiteMysql.Save(entity)
if !succ {
t.Fail()
} else {
t.Logf("SAVE|SUCC|%s\n", entity)
}
ret := kiteMysql.Query(msgid)
t.Logf("Query|%s|%s\n", msgid, ret)
if nil == ret {
t.Fail()
return
}
bb, ok := qm.GetBody().([]byte)
if ok {
rb, _ := ret.GetBody().([]byte)
if string(rb) != string(bb) {
t.Fail()
} else {
t.Logf("Query|SUCC|%s\n", ret)
}
} else {
bs, _ := qm.GetBody().(string)
rs, _ := ret.GetBody().(string)
if bs != rs {
t.Fail()
} else {
t.Logf("Query|SUCC|%s\n", ret)
}
}
t.Logf("Commint BEGIN")
commit := kiteMysql.Commit(msgid)
if !commit {
t.Logf("Commint FAIL")
t.Fail()
}
t.Logf("Commint END")
time.Sleep(200 * time.Millisecond)
ret = kiteMysql.Query(msgid)
t.Logf("PageQueryEntity|COMMIT RESULT|%s\n", ret)
if !ret.Commit {
t.Logf("Commit|FAIL|%s\n", ret)
t.Fail()
}
hasNext, entities := kiteMysql.PageQueryEntity(msgid, hn, time.Now().Unix(), 0, 10)
t.Logf("PageQueryEntity|%s\n", entities)
if hasNext {
t.Logf("PageQueryEntity|FAIL|HasNext|%s\n", entities)
t.Fail()
} else {
if len(entities) != 1 {
t.Logf("PageQueryEntity|FAIL|%s\n", entities)
t.Fail()
} else {
if entities[0].Header.GetMessageId() != qm.GetHeader().GetMessageId() {
t.Fail()
}
}
}
}
//增加迁移
func TestExpiredDLQ(t *testing.T) {
options := MysqlOptions{
Addr: "localhost:3306",
DB: "kite",
Username: "******",
Password: "",
ShardNum: 4,
BatchUpSize: 100,
BatchDelSize: 100,
FlushPeriod: 10 * time.Millisecond,
MaxIdleConn: 10,
MaxOpenConn: 10}
kiteMysql := NewKiteMysql(options, "localhost")
truncate(kiteMysql)
messageIds := make([]string, 0, 10)
for i := 0; i < 256; i++ {
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String("26c03f00665862591f696a980b5a6" + fmt.Sprintf("%x", i)),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Second).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(false),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
qm := protocol.NewQMessage(msg)
entity := store.NewMessageEntity(qm)
entity.SuccGroups = []string{"go-kite-test"}
hn := "localhost"
entity.KiteServer = hn
entity.PublishTime = time.Now().Unix()
entity.DeliverCount = 101
succ := kiteMysql.Save(entity)
if !succ {
t.Fail()
} else {
// fmt.Printf("SAVE|SUCC|%s\n", entity)
}
messageIds = append(messageIds, msg.GetHeader().GetMessageId())
}
//开始清理
kiteMysql.MoveExpired()
for _, messageId := range messageIds {
entity := kiteMysql.Query(messageId)
if nil != entity {
t.Fail()
fmt.Println("MoveExpired|FAIL|" + messageId)
}
}
total := 0
for i := 0; i < 4; i++ {
db := kiteMysql.dbshard.FindSlave(fmt.Sprintf("%x", i))
rows, _ := db.Query("select count(*) from kite_msg_dlq")
if rows.Next() {
var count int
rows.Scan(&count)
t.Logf("TestExpiredDLQ|COUNT|%s|%d\n", fmt.Sprintf("%x", i), total)
total += count
}
}
if total != 256 {
t.Fail()
t.Logf("TestExpiredDLQ|TOTAL NOT EQUAL|%d\n", total)
}
truncate(kiteMysql)
kiteMysql.Stop()
}
func TestPageQuery(t *testing.T) {
options := MysqlOptions{
Addr: "localhost:3306",
DB: "kite",
Username: "******",
Password: "",
ShardNum: 4,
BatchUpSize: 100,
BatchDelSize: 100,
FlushPeriod: 10 * time.Millisecond,
MaxIdleConn: 10,
MaxOpenConn: 10}
kiteMysql := NewKiteMysql(options, "localhost")
truncate(kiteMysql)
hn := "localhost"
for i := 0; i < 10; i++ {
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String(fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(false),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
entity.KiteServer = hn
entity.PublishTime = time.Now().Unix()
kiteMysql.Save(entity)
}
startIdx := 0
hasMore := true
count := 0
//开始分页查询未过期的消息实体
for hasMore {
more, entities := kiteMysql.PageQueryEntity("c", hn,
time.Now().Unix(), 0, 1)
if len(entities) <= 0 {
break
}
//开始发起重投
for _, entity := range entities {
count++
t.Logf("TestPageQuery|PageQueryEntity|%s\n", entity.MessageId)
msg := &store.MessageEntity{
MessageId: entity.MessageId,
DeliverCount: 1,
SuccGroups: []string{},
FailGroups: []string{"s-mts-test"},
//设置一下下一次投递时间
NextDeliverTime: time.Now().Add(1 * time.Minute).Unix()}
kiteMysql.AsyncUpdate(msg)
}
time.Sleep(1 * time.Second)
hasMore = more
startIdx += len(entities)
}
if count != 10 {
t.Fail()
t.Logf("TestPageQuery|IDX|FAIL|%d\n", count)
return
}
startIdx = 0
hasMore = true
//开始分页查询未过期的消息实体
for hasMore {
more, entities := kiteMysql.PageQueryEntity("6c", hn,
time.Now().Add(8*time.Minute).Unix(), startIdx, 1)
if len(entities) <= 0 {
t.Logf("TestPageQuery|CHECK|NO DATA|%s\n", entities)
break
}
//开始发起重投
for _, entity := range entities {
if entity.DeliverCount != 1 || entity.FailGroups[0] != "s-mts-test" {
t.Fail()
}
t.Logf("TestPageQuery|PageQueryEntity|CHECK|%s\n", entity.MessageId)
}
startIdx += len(entities)
hasMore = more
}
t.Logf("TestPageQuery|CHECK|%d\n", startIdx)
if startIdx != 10 {
t.Fail()
}
truncate(kiteMysql)
}
func TestBatch(t *testing.T) {
options := MysqlOptions{
Addr: "localhost:3306",
DB: "kite",
Username: "******",
Password: "",
ShardNum: 4,
BatchUpSize: 100,
BatchDelSize: 100,
FlushPeriod: 10 * time.Millisecond,
MaxIdleConn: 10,
MaxOpenConn: 10}
kiteMysql := NewKiteMysql(options, "localhost")
truncate(kiteMysql)
mids := make([]string, 0, 16)
for i := 0; i < 16; i++ {
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String("26c03f00665862591f696a980b5a6" + fmt.Sprintf("%x", i)),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(false),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
entity.SuccGroups = []string{"go-kite-test"}
hn := "localhost"
entity.KiteServer = hn
entity.PublishTime = time.Now().Unix()
kiteMysql.Save(entity)
mids = append(mids, entity.MessageId)
}
for _, v := range mids {
msg := &store.MessageEntity{
MessageId: v,
DeliverCount: 1,
SuccGroups: []string{"s-mts-test"},
FailGroups: []string{},
//设置一下下一次投递时间
NextDeliverTime: time.Now().Unix()}
kiteMysql.AsyncUpdate(msg)
}
time.Sleep(5 * time.Second)
for _, v := range mids {
e := kiteMysql.Query(v)
if nil == e || len(e.SuccGroups) < 1 {
t.Fatalf("TestBatch|Update FAIL|%s|%s\n", e, v)
t.Fail()
return
}
t.Logf("Query|%s\n", e)
}
//测试批量删除
for _, v := range mids {
kiteMysql.AsyncDelete(v)
}
time.Sleep(5 * time.Second)
for _, v := range mids {
entity := kiteMysql.Query(v)
if nil != entity {
t.Fatalf("TestBatch|AysncDelete FAIL|%s\n", entity)
t.Fail()
}
}
truncate(kiteMysql)
}
func TestFileStoreInit(t *testing.T) {
cleanSnapshot("./snapshot/")
fs := NewKiteFileStore(".", 5000000, 1*time.Second)
fs.Start()
for i := 0; i < 100; i++ {
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String(fmt.Sprint(i) + "26c03f00665862591f696a980b5ac"),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(true),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
succ := fs.Save(entity)
if !succ {
t.Fail()
}
if i < 50 {
fs.AsyncDelete(entity.MessageId)
}
}
fs.Stop()
time.Sleep(10 * time.Second)
log.Println("-------------------Query")
fs = NewKiteFileStore(".", 5000000, 1*time.Second)
fs.Start()
for _, v := range fs.oplogs {
for _, ol := range v {
ob := ol.Value.(*opBody)
log.Printf("TestFileStoreInit|Check|%d|%s", ob.Id, ob.MessageId)
}
}
log.Printf("TestFileStoreInit|Check|SUCC|\n")
//commit and check
for i := 50; i < 100; i++ {
id := fmt.Sprint(i) + "26c03f00665862591f696a980b5ac"
//check entity
entity := fs.Query(id)
if nil == entity || !entity.Commit {
log.Printf("TestFileStoreInit|Exist|FAIL|%s|%s", id, entity)
t.Fail()
return
}
log.Printf("TestFileStoreInit|Exist|SUCC|%d|%s|%s", i, id, entity)
}
log.Printf("TestFileStoreInit|Exist|\n")
//commit and check
for i := 0; i < 50; i++ {
id := fmt.Sprint(i) + "26c03f00665862591f696a980b5ac"
//check entity
entity := fs.Query(id)
if nil != entity {
log.Printf("TestFileStoreInit|Delete|FAIL|%s", id)
t.Fail()
}
}
log.Printf("TestFileStoreInit|Delete\n")
fs.Stop()
cleanSnapshot("./snapshot/")
}
func TestConvert2Params(t *testing.T) {
options := MysqlOptions{
Addr: "localhost:3306",
Username: "******",
Password: "",
ShardNum: 4,
BatchUpSize: 1000,
BatchDelSize: 1000,
FlushPeriod: 1 * time.Minute,
MaxIdleConn: 2,
MaxOpenConn: 4}
hs := newDbShard(options)
c := convertor{}
sqlwrapper := newSqlwrapper("kite_msg", hs, store.MessageEntity{})
sqlwrapper.initSQL()
c.columns = sqlwrapper.columns
//创建消息
msg := &protocol.BytesMessage{}
msg.Header = &protocol.Header{
MessageId: proto.String("26c03f00665862591f696a980b5a6c4"),
Topic: proto.String("trade"),
MessageType: proto.String("pay-succ"),
ExpiredTime: proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
DeliverLimit: proto.Int32(100),
GroupId: proto.String("go-kite-test"),
Commit: proto.Bool(false),
Fly: proto.Bool(false)}
msg.Body = []byte("hello world")
entity := store.NewMessageEntity(protocol.NewQMessage(msg))
entity.SuccGroups = []string{"go-kite-test"}
hn, _ := os.Hostname()
entity.KiteServer = hn
params := c.Convert2Params(entity)
if nil != params {
for i, col := range c.columns {
cv := params[i]
t.Logf("TestConvert2Params|FIELD|%s|%s\n", col.fieldName, cv)
if col.fieldName == "MessageId" {
rv := reflect.ValueOf(cv)
s := rv.Elem().Interface()
if s.(string) != entity.MessageId {
t.Fail()
}
} else if col.columnName == "body" {
rv := reflect.ValueOf(cv)
s := rv.Elem().Interface()
if string(s.([]byte)) != string(entity.GetBody().([]byte)) {
t.Fail()
}
}
}
} else {
t.Fail()
}
}