func TestProvider_Disconnect(t *testing.T) {
config := testProviderConfig()
p, err := NewProvider(config)
if err != nil {
t.Fatalf("err: %v", err)
}
defer p.Shutdown()
// Setup RPC client
a, b := testConn(t)
client, _ := yamux.Client(a, yamux.DefaultConfig())
server, _ := yamux.Server(b, yamux.DefaultConfig())
go p.handleSession(client, make(chan struct{}))
stream, _ := server.Open()
cc := msgpackrpc.NewCodec(false, false, stream)
// Make the connect rpc
args := &DisconnectRequest{
NoRetry: true,
Backoff: 300 * time.Second,
}
resp := &DisconnectResponse{}
err = msgpackrpc.CallWithCodec(cc, "Client.Disconnect", args, resp)
if err != nil {
t.Fatalf("err: %v", err)
}
p.backoffLock.Lock()
defer p.backoffLock.Unlock()
if p.backoff != 300*time.Second {
t.Fatalf("bad: %v", p.backoff)
}
if !p.noRetry {
t.Fatalf("bad")
}
p.sessionLock.Lock()
defer p.sessionLock.Unlock()
if p.sessionID != "" {
t.Fatalf("Bad: %v", p.sessionID)
}
if p.sessionAuth {
t.Fatalf("Bad: %v", p.sessionAuth)
}
}
func TestProvider_Connect(t *testing.T) {
config := testProviderConfig()
config.Service.Capabilities["foo"] = 1
config.Handlers["foo"] = fooCapability(t)
p, err := NewProvider(config)
if err != nil {
t.Fatalf("err: %v", err)
}
defer p.Shutdown()
// Setup RPC client
a, b := testConn(t)
client, _ := yamux.Client(a, yamux.DefaultConfig())
server, _ := yamux.Server(b, yamux.DefaultConfig())
go p.handleSession(client, make(chan struct{}))
stream, _ := server.Open()
cc := msgpackrpc.NewCodec(false, false, stream)
// Make the connect rpc
args := &ConnectRequest{
Capability: "foo",
Meta: map[string]string{
"zip": "zap",
},
}
resp := &ConnectResponse{}
err = msgpackrpc.CallWithCodec(cc, "Client.Connect", args, resp)
if err != nil {
t.Fatalf("err: %v", err)
}
// Should be successful!
if !resp.Success {
t.Fatalf("bad")
}
// At this point, we should be connected
out := make([]byte, 9)
n, err := stream.Read(out)
if err != nil {
t.Fatalf("err: %v %d", err, n)
}
if string(out) != "foobarbaz" {
t.Fatalf("bad: %s", out)
}
}
// NewClient creates a new tunnel that is established between the serverAddr
// and localAddr. It exits if it can't create a new control connection to the
// server. If localAddr is empty client will always try to proxy to a local
// port.
func NewClient(cfg *ClientConfig) (*Client, error) {
yamuxConfig := yamux.DefaultConfig()
if cfg.YamuxConfig != nil {
yamuxConfig = cfg.YamuxConfig
}
log := newLogger("tunnel-client", cfg.Debug)
if cfg.Log != nil {
log = cfg.Log
}
if err := cfg.verify(); err != nil {
return nil, err
}
forever := backoff.NewExponentialBackOff()
forever.MaxElapsedTime = 365 * 24 * time.Hour // 1 year
client := &Client{
config: cfg,
log: log,
yamuxConfig: yamuxConfig,
redialBackoff: forever,
startNotify: make(chan bool, 1),
}
return client, nil
}
// NewServer creates a new Server. The defaults are used if config is nil.
func NewServer(cfg *ServerConfig) (*Server, error) {
yamuxConfig := yamux.DefaultConfig()
if cfg.YamuxConfig != nil {
if err := yamux.VerifyConfig(cfg.YamuxConfig); err != nil {
return nil, err
}
yamuxConfig = cfg.YamuxConfig
}
log := newLogger("tunnel-server", cfg.Debug)
if cfg.Log != nil {
log = cfg.Log
}
return &Server{
pending: make(map[string]chan net.Conn),
sessions: make(map[string]*yamux.Session),
onDisconnect: make(map[string]func() error),
virtualHosts: newVirtualHosts(),
controls: newControls(),
yamuxConfig: yamuxConfig,
log: log,
}, nil
}
// NewClient creates a new tunnel that is established between the serverAddr
// and localAddr. It exits if it can't create a new control connection to the
// server. If localAddr is empty client will always try to proxy to a local
// port.
func NewClient(cfg *ClientConfig) (*Client, error) {
yamuxConfig := yamux.DefaultConfig()
if cfg.YamuxConfig != nil {
yamuxConfig = cfg.YamuxConfig
}
log := newLogger("tunnel-client", cfg.Debug)
if cfg.Log != nil {
log = cfg.Log
}
if err := cfg.verify(); err != nil {
return nil, err
}
client := &Client{
config: cfg,
log: log,
yamuxConfig: yamuxConfig,
redialBackoff: cfg.Backoff,
startNotify: make(chan bool, 1),
}
if client.redialBackoff == nil {
client.redialBackoff = newForeverBackoff()
}
return client, nil
}
func testHandshake(t *testing.T, list net.Listener, expect *HandshakeRequest) {
client, err := list.Accept()
if err != nil {
t.Fatalf("err: %v", err)
}
defer client.Close()
preamble := make([]byte, len(clientPreamble))
n, err := client.Read(preamble)
if err != nil || n != len(preamble) {
t.Fatalf("err: %v", err)
}
server, _ := yamux.Server(client, yamux.DefaultConfig())
conn, err := server.Accept()
if err != nil {
t.Fatalf("err: %v", err)
}
defer conn.Close()
rpcCodec := msgpackrpc.NewCodec(true, true, conn)
rpcSrv := rpc.NewServer()
rpcSrv.RegisterName("Session", &TestHandshake{t, expect})
err = rpcSrv.ServeRequest(rpcCodec)
if err != nil {
t.Fatalf("err: %v", err)
}
}
func startTLSServer(config *Config) (net.Conn, chan error) {
errc := make(chan error, 1)
tlsConfigServer, err := config.IncomingTLSConfig()
if err != nil {
errc <- err
return nil, errc
}
client, server := net.Pipe()
// Use yamux to buffer the reads, otherwise it's easy to deadlock
muxConf := yamux.DefaultConfig()
serverSession, _ := yamux.Server(server, muxConf)
clientSession, _ := yamux.Client(client, muxConf)
clientConn, _ := clientSession.Open()
serverConn, _ := serverSession.Accept()
go func() {
tlsServer := tls.Server(serverConn, tlsConfigServer)
if err := tlsServer.Handshake(); err != nil {
errc <- err
}
close(errc)
// Because net.Pipe() is unbuffered, if both sides
// Close() simultaneously, we will deadlock as they
// both send an alert and then block. So we make the
// server read any data from the client until error or
// EOF, which will allow the client to Close(), and
// *then* we Close() the server.
io.Copy(ioutil.Discard, tlsServer)
tlsServer.Close()
}()
return clientConn, errc
}
// getNewConn is used to return a new connection
func (p *ConnPool) getNewConn(addr string, timeout time.Duration) (*Conn, error) {
// Try to dial the conn
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", addr, p.config)
if err != nil {
return nil, err
}
// Setup the logger
conf := yamux.DefaultConfig()
conf.LogOutput = p.logOutput
// Create a multiplexed session
session, err := yamux.Client(conn, conf)
if err != nil {
conn.Close()
return nil, err
}
// Wrap the connection
c := &Conn{
addr: addr,
session: session,
lastUsed: time.Now(),
pool: p,
}
return c, nil
}
// getNewConn is used to return a new connection
func (p *ConnPool) getNewConn(region string, addr net.Addr, version int) (*Conn, error) {
// Try to dial the conn
conn, err := net.DialTimeout("tcp", addr.String(), 10*time.Second)
if err != nil {
return nil, err
}
// Cast to TCPConn
if tcp, ok := conn.(*net.TCPConn); ok {
tcp.SetKeepAlive(true)
tcp.SetNoDelay(true)
}
// Check if TLS is enabled
if p.tlsWrap != nil {
// Switch the connection into TLS mode
if _, err := conn.Write([]byte{byte(rpcTLS)}); err != nil {
conn.Close()
return nil, err
}
// Wrap the connection in a TLS client
tlsConn, err := p.tlsWrap(region, conn)
if err != nil {
conn.Close()
return nil, err
}
conn = tlsConn
}
// Write the multiplex byte to set the mode
if _, err := conn.Write([]byte{byte(rpcMultiplex)}); err != nil {
conn.Close()
return nil, err
}
// Setup the logger
conf := yamux.DefaultConfig()
conf.LogOutput = p.logOutput
// Create a multiplexed session
session, err := yamux.Client(conn, conf)
if err != nil {
conn.Close()
return nil, err
}
// Wrap the connection
c := &Conn{
refCount: 1,
addr: addr,
session: session,
clients: list.New(),
lastUsed: time.Now(),
version: version,
pool: p,
}
return c, nil
}
func TestProvider_Flash(t *testing.T) {
config := testProviderConfig()
buf := bytes.NewBuffer(nil)
config.LogOutput = buf
p, err := NewProvider(config)
if err != nil {
t.Fatalf("err: %v", err)
}
defer p.Shutdown()
// Setup RPC client
a, b := testConn(t)
client, _ := yamux.Client(a, yamux.DefaultConfig())
server, _ := yamux.Server(b, yamux.DefaultConfig())
go p.handleSession(client, make(chan struct{}))
stream, _ := server.Open()
cc := msgpackrpc.NewCodec(false, false, stream)
// Make the connect rpc
args := &FlashRequest{
Severity: "INFO",
Message: "TESTING",
}
resp := &FlashResponse{}
err = msgpackrpc.CallWithCodec(cc, "Client.Flash", args, resp)
if err != nil {
t.Fatalf("err: %v", err)
}
// Wait until we are disconnected
start := time.Now()
for time.Now().Sub(start) < time.Second {
if bytes.Contains(buf.Bytes(), []byte("TESTING")) {
break
}
time.Sleep(10 * time.Millisecond)
}
if !bytes.Contains(buf.Bytes(), []byte("TESTING")) {
t.Fatalf("missing: %s", buf)
}
}
func (y *yamuxer) handleConn(g grim.GrimReaper, conn net.Conn) {
defer g.Wait()
conf := yamux.DefaultConfig()
conf.LogOutput = y.logOutput
session, _ := yamux.Server(conn, conf)
streamCh := make(chan net.Conn)
g.SpawnFunc(processStreams(g.New(), conn, streamCh, y.dispatcher))
g.SpawnFunc(acceptStreams(y.logger, session, streamCh))
}
// NewClient creates a new tunnel that is established between the serverAddr
// and localAddr. It exits if it can't create a new control connection to the
// server. If localAddr is empty client will always try to proxy to a local
// port.
func NewClient(cfg *ClientConfig) (*Client, error) {
if err := cfg.verify(); err != nil {
return nil, err
}
yamuxConfig := yamux.DefaultConfig()
if cfg.YamuxConfig != nil {
yamuxConfig = cfg.YamuxConfig
}
var proxy = DefaultProxy
if cfg.Proxy != nil {
proxy = cfg.Proxy
}
// DEPRECATED API SUPPORT
if cfg.LocalAddr != "" || cfg.FetchLocalAddr != nil {
var f ProxyFuncs
if cfg.LocalAddr != "" {
f.HTTP = (&HTTPProxy{LocalAddr: cfg.LocalAddr}).Proxy
f.WS = (&HTTPProxy{LocalAddr: cfg.LocalAddr}).Proxy
}
if cfg.FetchLocalAddr != nil {
f.TCP = (&TCPProxy{FetchLocalAddr: cfg.FetchLocalAddr}).Proxy
}
proxy = Proxy(f)
}
var bo Backoff = newForeverBackoff()
if cfg.Backoff != nil {
bo = cfg.Backoff
}
log := newLogger("tunnel-client", cfg.Debug)
if cfg.Log != nil {
log = cfg.Log
}
client := &Client{
config: cfg,
yamuxConfig: yamuxConfig,
proxy: proxy,
startNotify: make(chan bool, 1),
redialBackoff: bo,
log: log,
}
return client, nil
}
// handleMultiplex handles a multiplexed connection.
func (r *rpcServer) handleMultiplex(conn net.Conn) {
defer conn.Close()
conf := yamux.DefaultConfig()
conf.LogOutput = r.n.config.LogOutput
server, _ := yamux.Server(conn, conf)
for {
sub, err := server.Accept()
if err != nil {
if err != io.EOF {
r.n.logger.Printf("[ERR] onecache.rpc: multiplex conn accept failed: %v", err)
}
return
}
go r.handleConn(sub)
}
}
// handleMultiplex is used to multiplex a single incoming connection
// using the Yamux multiplexer
func (s *Server) handleMultiplex(conn net.Conn) {
defer conn.Close()
conf := yamux.DefaultConfig()
conf.LogOutput = s.config.LogOutput
server, _ := yamux.Server(conn, conf)
for {
sub, err := server.Accept()
if err != nil {
if err != io.EOF {
s.logger.Printf("[ERR] nomad.rpc: multiplex conn accept failed: %v", err)
}
return
}
go s.handleNomadConn(sub)
}
}
func (li *Listener) getSession() (*yamux.Session, error) {
li.mu.Lock()
defer li.mu.Unlock()
if li.session != nil {
return li.session, nil
}
// connect to the socket master
conn, err := net.Dial("tcp", li.socketMasterAddress)
if err != nil {
return nil, err
}
// bind to a port
err = protocol.WriteHandshakeRequest(conn, protocol.HandshakeRequest{
SocketDefinition: li.socketDefinition,
})
if err != nil {
conn.Close()
return nil, err
}
// see if that worked
res, err := protocol.ReadHandshakeResponse(conn)
if err != nil {
conn.Close()
return nil, err
}
if res.Status != "OK" {
conn.Close()
return nil, fmt.Errorf("%s", res.Status)
}
// start a new session
session, err := yamux.Server(conn, yamux.DefaultConfig())
if err != nil {
conn.Close()
return nil, err
}
return session, nil
}
// NewServer creates a new Server. The defaults are used if config is nil.
func NewServer(cfg *ServerConfig) (*Server, error) {
yamuxConfig := yamux.DefaultConfig()
if cfg.YamuxConfig != nil {
if err := yamux.VerifyConfig(cfg.YamuxConfig); err != nil {
return nil, err
}
yamuxConfig = cfg.YamuxConfig
}
log := newLogger("tunnel-server", cfg.Debug)
if cfg.Log != nil {
log = cfg.Log
}
connCh := make(chan net.Conn)
opts := &vaddrOptions{
connCh: connCh,
log: log,
}
s := &Server{
pending: make(map[string]chan net.Conn),
sessions: make(map[string]*yamux.Session),
onConnectCallbacks: newCallbacks("OnConnect"),
onDisconnectCallbacks: newCallbacks("OnDisconnect"),
virtualHosts: newVirtualHosts(),
virtualAddrs: newVirtualAddrs(opts),
controls: newControls(),
states: make(map[string]ClientState),
stateCh: cfg.StateChanges,
httpDirector: cfg.Director,
yamuxConfig: yamuxConfig,
connCh: connCh,
log: log,
}
go s.serveTCP()
return s, nil
}
// getNewConn is used to return a new connection
func (p *connPool) getNewConn(addr net.Addr) (*conn, error) {
// Try to dial the conn
con, err := net.DialTimeout("tcp", addr.String(), 10*time.Second)
if err != nil {
return nil, err
}
// Cast to TCPConn
if tcp, ok := con.(*net.TCPConn); ok {
tcp.SetKeepAlive(true)
tcp.SetNoDelay(true)
}
// Write the multiplex byte to set the mode
if _, err := con.Write([]byte{byte(rpcInternal)}); err != nil {
con.Close()
return nil, err
}
// Setup the logger
conf := yamux.DefaultConfig()
conf.LogOutput = p.logOutput
// Create a multiplexed session
session, err := yamux.Client(con, conf)
if err != nil {
con.Close()
return nil, err
}
// Wrap the connection
c := &conn{
refCount: 1,
addr: addr,
session: session,
clients: list.New(),
lastUsed: time.Now(),
pool: p,
}
return c, nil
}
// initClient does the common initialization
func initClient(conn net.Conn, opts *Opts) (*Client, error) {
// Send the preamble
_, err := conn.Write([]byte(clientPreamble))
if err != nil {
return nil, fmt.Errorf("preamble write failed: %v", err)
}
// Wrap the connection in yamux for multiplexing
ymConf := yamux.DefaultConfig()
if opts.LogOutput != nil {
ymConf.LogOutput = opts.LogOutput
}
client, _ := yamux.Client(conn, ymConf)
// Create the client
c := &Client{
conn: conn,
client: client,
}
return c, nil
}
// getNewConn is used to return a new connection
func (p *ConnPool) getNewConn(dc string, addr net.Addr, version int) (*Conn, error) {
// Get a new, raw connection.
conn, _, err := p.Dial(dc, addr)
if err != nil {
return nil, err
}
// Switch the multiplexing based on version
var session muxSession
if version < 2 {
conn.Close()
return nil, fmt.Errorf("cannot make client connection, unsupported protocol version %d", version)
} else {
// Write the Consul multiplex byte to set the mode
if _, err := conn.Write([]byte{byte(rpcMultiplexV2)}); err != nil {
conn.Close()
return nil, err
}
// Setup the logger
conf := yamux.DefaultConfig()
conf.LogOutput = p.logOutput
// Create a multiplexed session
session, _ = yamux.Client(conn, conf)
}
// Wrap the connection
c := &Conn{
refCount: 1,
addr: addr,
session: session,
clients: list.New(),
lastUsed: time.Now(),
version: version,
pool: p,
}
return c, nil
}
// Listen connects to the socket master, binds a port, and accepts
// multiplexed traffic as new connections
func (client *Client) Listen(socketDefinition protocol.SocketDefinition) (net.Listener, error) {
// connect to the socket master
conn, err := net.Dial("tcp", client.socketMasterAddress)
if err != nil {
return nil, err
}
// bind to a port
err = protocol.WriteHandshakeRequest(conn, protocol.HandshakeRequest{
SocketDefinition: socketDefinition,
})
if err != nil {
conn.Close()
return nil, err
}
// see if that worked
res, err := protocol.ReadHandshakeResponse(conn)
if err != nil {
conn.Close()
return nil, err
}
if res.Status != "OK" {
conn.Close()
return nil, fmt.Errorf("%s", res.Status)
}
// start a new session
session, err := yamux.Server(conn, yamux.DefaultConfig())
if err != nil {
conn.Close()
return nil, err
}
return session, nil
}
"sync"
"time"
"github.com/hashicorp/yamux"
"github.com/ugorji/go/codec"
"github.com/vektra/errors"
"github.com/vektra/seconn"
)
const DefaultPort = 8475
var msgpack codec.MsgpackHandle
var EProtocolError = errors.New("protocol error")
var muxConfig = yamux.DefaultConfig()
type Service struct {
Address string
Registry Storage
listener net.Listener
wg sync.WaitGroup
closed bool
shutdown chan struct{}
lock sync.Mutex
}
func (s *Service) Port() int {
// getNewConn is used to return a new connection
func (p *ConnPool) getNewConn(addr net.Addr, version int) (*Conn, error) {
// Try to dial the conn
conn, err := net.DialTimeout("tcp", addr.String(), 10*time.Second)
if err != nil {
return nil, err
}
// Cast to TCPConn
if tcp, ok := conn.(*net.TCPConn); ok {
tcp.SetKeepAlive(true)
tcp.SetNoDelay(true)
}
// Check if TLS is enabled
if p.tlsConfig != nil {
// Switch the connection into TLS mode
if _, err := conn.Write([]byte{byte(rpcTLS)}); err != nil {
conn.Close()
return nil, err
}
// Wrap the connection in a TLS client
conn = tls.Client(conn, p.tlsConfig)
}
// Switch the multiplexing based on version
var session muxSession
if version < 2 {
// Write the Consul multiplex byte to set the mode
if _, err := conn.Write([]byte{byte(rpcMultiplex)}); err != nil {
conn.Close()
return nil, err
}
// Create a multiplexed session
session = &muxadoWrapper{muxado.Client(conn)}
} else {
// Write the Consul multiplex byte to set the mode
if _, err := conn.Write([]byte{byte(rpcMultiplexV2)}); err != nil {
conn.Close()
return nil, err
}
// Setup the logger
conf := yamux.DefaultConfig()
conf.LogOutput = p.logOutput
// Create a multiplexed session
session, _ = yamux.Client(conn, conf)
}
// Wrap the connection
c := &Conn{
refCount: 1,
addr: addr,
session: session,
clients: list.New(),
lastUsed: time.Now(),
version: version,
pool: p,
}
// Track this connection, handle potential race condition
p.Lock()
if existing := p.pool[addr.String()]; existing != nil {
c.Close()
p.Unlock()
return existing, nil
} else {
p.pool[addr.String()] = c
p.Unlock()
return c, nil
}
}
