func monitorForever(cf *conf.Conf) {
loadConfig(cf)
loadTemplates()
if err := etclib.Dial(config.etcServers); err != nil {
panic(err)
}
go watchFaes()
go watchMaintain()
<-make(chan interface{})
}
func main() {
cf := config.NewDefaultProxy()
cf.IoTimeout = time.Hour
cf.TcpNoDelay = tcpNoDelay
prx := proxy.New(cf)
etclib.Dial([]string{zk})
go prx.StartMonitorCluster()
prx.AwaitClusterTopologyReady()
// test pool
if testPool {
testServantPool(prx)
pause("pool tested")
}
go report.run()
wg := new(sync.WaitGroup)
t1 := time.Now()
for k := c1; k <= c2; k += 10 {
Concurrency = k
cf.PoolCapacity = Concurrency
prx = proxy.New(cf)
for i := 0; i < Rounds; i++ {
for j := 0; j < k; j++ {
wg.Add(1)
go runSession(prx, wg, i+1, j)
}
wg.Wait()
}
}
elapsed := time.Since(t1)
log.Printf("Elapsed: %s, calls: {%s, %.1f/s}, sessions: {%s, %.1f/s}, errors: {conn:%d, io:%d call:%d}",
elapsed,
gofmt.Comma(report.callOk),
float64(report.callOk)/elapsed.Seconds(),
gofmt.Comma(int64(report.sessionN)),
float64(report.sessionN)/elapsed.Seconds(),
report.connErrs,
report.ioErrs,
report.callErrs)
}
func (this *Engine) UnregisterEtcd() {
if config.Engine.IsProxyOnly() {
return
}
if config.Engine.EtcdSelfAddr != "" {
if !etclib.IsConnected() {
if err := etclib.Dial(config.Engine.EtcdServers); err != nil {
log.Error("etcd[%+v]: %s", config.Engine.EtcdServers, err)
return
}
}
etclib.ShutdownService(config.Engine.EtcdSelfAddr, etclib.SERVICE_FAE)
etclib.Close()
log.Info("etcd self[%s] unregistered", config.Engine.EtcdSelfAddr)
}
}
func pingCluster(proxy *proxy.Proxy) {
if err := etclib.Dial([]string{zk}); err != nil {
fmt.Printf("zk: %s", err.Error())
return
}
go proxy.StartMonitorCluster()
proxy.AwaitClusterTopologyReady()
peers := proxy.ClusterPeers()
if len(peers) == 0 {
fmt.Println("found no fae peers")
return
}
for i := 0; i < loops; i++ {
for _, peerAddr := range peers {
client, err := proxy.ServantByAddr(peerAddr)
if err != nil {
fmt.Printf("[%6d] %21s: %s\n", i+1, peerAddr, err)
continue
}
ctx := rpc.NewContext()
ctx.Reason = REASON
ctx.Rid = time.Now().UnixNano() + int64(i)
pong, err := client.Ping(ctx)
if err != nil {
client.Close()
fmt.Printf("[%6d] %21s: %s\n", i+1, peerAddr, err)
} else {
fmt.Printf("[%6d] %21s: %s\n", i+1, peerAddr, pong)
}
client.Recycle()
}
}
}
// thrift internal layer
//
// Server
// Processor (compiler genereated)
// Protocol (JSON/compact/...), what is transmitted
// Transport (TCP/HTTP/...), how is transmitted
func (this *Engine) launchRpcServe() (done chan null.NullStruct) {
var (
protocolFactory thrift.TProtocolFactory
serverTransport thrift.TServerTransport
transportFactory thrift.TTransportFactory
err error
serverNetwork string
)
switch config.Engine.Rpc.Protocol {
case "binary":
protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()
case "json":
protocolFactory = thrift.NewTJSONProtocolFactory()
case "simplejson":
protocolFactory = thrift.NewTSimpleJSONProtocolFactory()
case "compact":
protocolFactory = thrift.NewTCompactProtocolFactory()
default:
panic(fmt.Sprintf("Unknown protocol: %s", config.Engine.Rpc.Protocol))
}
// client-side Thrift protocol/transport stack must match
// the server-side, otherwise you are very likely to get in trouble
switch {
case config.Engine.Rpc.Framed:
// each payload is sent over the wire with a frame header containing its size
transportFactory = thrift.NewTFramedTransportFactory(transportFactory)
default:
// there is no BufferedTransport in Java: only FramedTransport
transportFactory = thrift.NewTBufferedTransportFactory(
config.Engine.Rpc.BufferSize)
}
switch {
case strings.Contains(config.Engine.Rpc.ListenAddr, "/"):
serverNetwork = "unix"
if config.Engine.Rpc.SessionTimeout > 0 {
serverTransport, err = NewTUnixSocketTimeout(
config.Engine.Rpc.ListenAddr, config.Engine.Rpc.SessionTimeout)
} else {
serverTransport, err = NewTUnixSocket(
config.Engine.Rpc.ListenAddr)
}
default:
serverNetwork = "tcp"
if config.Engine.Rpc.SessionTimeout > 0 {
serverTransport, err = thrift.NewTServerSocketTimeout(
config.Engine.Rpc.ListenAddr, config.Engine.Rpc.SessionTimeout)
} else {
serverTransport, err = thrift.NewTServerSocket(
config.Engine.Rpc.ListenAddr)
}
}
if err != nil {
panic(err)
}
// dial zk before startup servants
// because proxy servant is dependent upon zk
if config.Engine.EtcdSelfAddr != "" {
if err := etclib.Dial(config.Engine.EtcdServers); err != nil {
panic(err)
} else {
log.Debug("etcd connected: %+v", config.Engine.EtcdServers)
}
}
// when config loaded, create the servants
this.svt = servant.NewFunServantWrapper(config.Engine.Servants)
this.rpcProcessor = rpc.NewFunServantProcessor(this.svt)
this.svt.Start()
this.rpcServer = NewTFunServer(this,
config.Engine.Rpc.PreforkMode,
this.rpcProcessor,
serverTransport, transportFactory, protocolFactory)
log.Info("RPC server ready at %s:%s", serverNetwork, config.Engine.Rpc.ListenAddr)
this.launchDashboard()
done = make(chan null.NullStruct)
go func() {
if err = this.rpcServer.Serve(); err != nil {
log.Error("RPC server: %+v", err)
}
done <- null.Null
}()
return done
}
func (this *Start) main() {
ctx.LoadFromHome()
// TODO zk session timeout
err := etclib.Dial(strings.Split(ctx.ZoneZkAddrs(this.zone), ","))
swalllow(err)
root := zkr.Root(this.zone)
ch := make(chan []string, 10)
go etclib.WatchChildren(root, ch)
var servers = BackendServers{
CpuNum: ctx.NumCPU(),
HaproxyRoot: this.root,
ForwardFor: this.forwardFor,
PubPort: this.pubPort,
SubPort: this.subPort,
ManPort: this.manPort,
}
var lastInstances []string
for {
select {
case <-this.quitCh:
time.Sleep(time.Second) // FIXME just wait log flush
return
case <-ch:
kwInstances, err := etclib.Children(root)
if err != nil {
log.Error("%s: %v", root, err)
continue
}
log.Info("kateway ids: %+v -> %+v", lastInstances, kwInstances)
lastInstances = kwInstances
servers.reset()
for _, kwId := range kwInstances {
kwNode := fmt.Sprintf("%s/%s", root, kwId)
data, err := etclib.Get(kwNode)
if err != nil {
log.Error("%s: %v", kwNode, err)
continue
}
info := make(map[string]string)
if err = json.Unmarshal([]byte(data), &info); err != nil {
log.Error("%s: %v", data, err)
continue
}
// pub
if info["pub"] != "" {
be := Backend{
Name: "p" + info["id"],
Addr: info["pub"],
Cpu: info["cpu"],
}
servers.Pub = append(servers.Pub, be)
}
// sub
if info["sub"] != "" {
be := Backend{
Name: "s" + info["id"],
Addr: info["sub"],
Cpu: info["cpu"],
}
servers.Sub = append(servers.Sub, be)
}
// man
if info["man"] != "" {
be := Backend{
Name: "m" + info["id"],
Addr: info["man"],
Cpu: info["cpu"],
}
servers.Man = append(servers.Man, be)
}
}
if err = this.createConfigFile(servers); err != nil {
log.Error(err)
continue
}
if err = this.reloadHAproxy(); err != nil {
log.Error("reloading haproxy: %v", err)
panic(err)
}
}
}
}