func BenchmarkReadDisk(b *testing.B) {
m := NewByData(NewData([]byte(utils.RandString(256))))
buf := bytes.NewBuffer(make([]byte, 0, m.Size()*b.N))
for i := 0; i < b.N; i++ {
m.WriteTo(buf)
}
path := utils.GetRoot("/test/message/tmp")
os.MkdirAll(filepath.Dir(path), 0777)
if err := ioutil.WriteFile(path, buf.Bytes(), 0666); err != nil {
b.Fatal(err)
}
b.ResetTimer()
f, err := os.Open(path)
if err != nil {
b.Fatal(err)
}
defer f.Close()
r := bufio.NewReader(f)
var header Header
for i := 0; i < b.N; i++ {
_, err := Read(&header, r, RF_RESEEK_ON_FAULT)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkUnmarshal(b *testing.B) {
m := NewByData(NewData([]byte(utils.RandString(256))))
b.ResetTimer()
for i := 0; i < b.N; i++ {
New(m.underlay)
}
}
func BenchmarkHttpPut(b *testing.B) {
var data []byte
batch := 200
for i := 0; i < batch; i++ {
msg := message.NewByData(message.NewData([]byte(utils.RandString(200))))
data = append(data, msg.Bytes()...)
}
r := bytes.NewReader(data)
client := &http.Client{}
buffer := 0
url := "http://localhost:8611/put?topic=http-test&size=200"
for i := 0; i < b.N; i++ {
if buffer < batch {
buffer++
continue
}
buffer = 0
r.Seek(0, 0)
resp, err := client.Post(url, "", r)
if err != nil {
b.Fatal(err)
}
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
}
func BenchmarkMarshal(b *testing.B) {
source := NewData([]byte(utils.RandString(256)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
NewByData(source)
}
}
func apiSyncPut(topic string, b *testing.B) {
var data []*message.Ins
batch := 200
for i := 0; i < batch; i++ {
d := message.NewData([]byte(utils.RandString(200)))
data = append(data, message.NewByData(d))
}
client, err := api.New(":12345")
if err != nil {
b.Fatal(err)
}
defer client.Close()
if b.N < batch {
_, err := client.Put(topic, data[:b.N])
if err != nil {
b.Error(err)
}
return
}
for i := 0; i < b.N; i += batch {
_, err := client.Put(topic, data)
if err != nil {
b.Error(err)
return
}
}
}
func BenchmarkWrite(b *testing.B) {
lfs, err := newIns()
if err != nil {
b.Fatal(err)
return
}
defer lfs.Close()
worker := 1
ch := make(chan string, worker)
for i := 0; i < worker; i++ {
ch <- utils.RandString(6)
}
line := int64(b.N)
var wg sync.WaitGroup
wg.Add(worker)
var m sync.Mutex
for i := 0; i < worker; i++ {
go func() {
defer wg.Done()
w, err := lfs.OpenWriter("/" + <-ch)
if err != nil {
b.Error(err)
return
}
defer w.Close()
buf := bytes.Repeat([]byte(utils.RandString(1)), 4096)
for ; atomic.LoadInt64(&line) > 0; atomic.AddInt64(&line, -1) {
if _, err = w.Write(buf); err != nil {
b.Error(err)
break
}
m.Lock()
b.SetBytes(int64(len(buf)))
m.Unlock()
}
}()
}
wg.Wait()
}
func TestConsumer(t *testing.T) {
que, ln := runServer(t)
defer closeServer(que, ln)
config := &Config{
Endpoint: ":12345",
Size: 100,
Topic: "test-consumer",
}
if a, err := New(config.Endpoint); err != nil {
logex.Fatal(err)
} else if err := a.Delete(config.Topic); err != nil && !logex.Equal(err, ErrTopicNotFound) {
logex.Fatal(err)
}
c, err := NewConsumer(config)
if err != nil {
logex.Fatal(err)
}
c.Start()
var wg sync.WaitGroup
wg.Add(config.Size)
go func() {
for reply := range c.ReplyChan {
for _ = range reply.Msgs {
wg.Done()
}
}
}()
a, err := New(config.Endpoint)
if err != nil {
logex.Fatal(err)
}
m := message.NewByData(message.NewData([]byte(utils.RandString(256))))
msgs := make([]*message.Ins, config.Size)
for i := 0; i < len(msgs); i++ {
msgs[i] = m
}
n, err := a.Put(config.Topic, msgs)
if err != nil {
logex.Fatal(err)
}
if n != len(msgs) {
logex.Fatal("not match")
}
wg.Wait()
}
func BenchmarkRead(b *testing.B) {
m := NewByData(NewData([]byte(utils.RandString(256))))
buf := bytes.NewBuffer(make([]byte, 0, m.Size()*b.N))
for i := 0; i < b.N; i++ {
m.WriteTo(buf)
}
b.ResetTimer()
var header Header
for i := 0; i < b.N; i++ {
_, err := Read(&header, buf, RF_RESEEK_ON_FAULT)
if err != nil {
b.Fatal(err)
}
}
}
func benchmarkSize(b *testing.B, size int) {
fileName := utils.GetRoot("/test/bitmap/benchmark")
os.RemoveAll(fileName)
f, err := NewFileEx(fileName, 22)
if err != nil {
b.Fatal(err)
}
defer func() {
f.Delete()
}()
b.ResetTimer()
data := []byte(strings.Repeat(utils.RandString(1), size))
w := utils.Writer{f, 0}
for i := 0; i < b.N; i++ {
w.Write(data)
b.SetBytes(int64(len(data)))
}
}
func TestFileCon(t *testing.T) {
n := 100
dirname := utils.GetRoot("/test/bitmap.file.con")
os.RemoveAll(dirname)
f, err := NewFileEx(dirname, 1)
if err != nil {
t.Fatal(err)
}
defer f.Close()
buf := []byte(utils.RandString(3))
off := int64(0)
for i := 0; i < n; i++ {
n, err := f.WriteAt(buf, off)
if err != nil {
t.Fatal(err)
}
if n != len(buf) {
t.Fatal("short write", n, len(buf))
}
off += int64(n)
}
}
func genBlock(size int) []byte {
return bytes.Repeat([]byte(utils.RandString(1)), size)
}
// Write data via N reqs and read them via M reqs
func TestNWriteMRead(t *testing.T) {
var err error
defer utils.TDefer(t, &err)
ins, err := newIns()
if err != nil {
return
}
defer ins.Close()
factor := 50
w, err := ins.OpenWriter("test1")
if err != nil {
return
}
defer w.Close()
lengths := make([]int, factor)
datas := make([][]byte, factor)
expect := make([]byte, 0)
total := 0
for i := range lengths {
l := 1 + utils.RandInt(200*cfg.blkSize)
total += l
datas[i] = bytes.Repeat([]byte(utils.RandString(1)), l)
expect = append(expect, datas[i]...)
}
for _, data := range datas {
if err = safeWrite(w, data); err != nil {
return
}
}
r, err := ins.OpenReader("test1")
if err != nil {
return
}
defer r.Close()
maxSize := 1 << 12
pStart := 0
pEnd := maxSize
for pEnd > pStart {
if pEnd > len(expect) {
pEnd = len(expect)
}
if err = safeReadExpect(r, expect[pStart:pEnd]); err != nil {
return
}
pStart = pEnd
if pEnd < len(expect) {
pEnd += maxSize
}
}
// expect eof
r.Offset -= 1
n, err := r.Read([]byte("padding"))
if !logex.Equal(err, io.EOF) || n != 1 {
err = logex.NewError("expect EOF, got ", n, err)
return
}
n, err = r.Read([]byte("pad"))
if err != io.EOF || n != 0 {
err = logex.NewError("expect EOF, got ", n)
return
}
err = nil
}