// getConn is used to get a connection from the pool
func (n *NetworkTransport) getConn(target net.Addr) (*netConn, error) {
// Check for a pooled conn
if conn := n.getPooledConn(target); conn != nil {
return conn, nil
}
// Dial a new connection
conn, err := n.stream.Dial(target.String(), n.timeout)
if err != nil {
return nil, err
}
// Wrap the conn
netConn := &netConn{
target: target,
conn: conn,
r: bufio.NewReader(conn),
w: bufio.NewWriter(conn),
}
// Setup encoder/decoders
netConn.dec = codec.NewDecoder(netConn.r, &codec.MsgpackHandle{})
netConn.enc = codec.NewEncoder(netConn.w, &codec.MsgpackHandle{})
// Done
return netConn, nil
}
// Encode writes an encoded object to a new bytes buffer
func encodeMsgPack(in interface{}) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
hd := codec.MsgpackHandle{}
enc := codec.NewEncoder(buf, &hd)
err := enc.Encode(in)
return buf, err
}
// Encode writes an encoded object to a new bytes buffer
func encode(msgType messageType, in interface{}) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
buf.WriteByte(uint8(msgType))
hd := codec.MsgpackHandle{}
enc := codec.NewEncoder(buf, &hd)
err := enc.Encode(in)
return buf, err
}
func encodeFilter(f filterType, filt interface{}) ([]byte, error) {
buf := bytes.NewBuffer(nil)
buf.WriteByte(uint8(f))
handle := codec.MsgpackHandle{}
encoder := codec.NewEncoder(buf, &handle)
err := encoder.Encode(filt)
return buf.Bytes(), err
}
func encodeMessage(t messageType, msg interface{}) ([]byte, error) {
buf := bytes.NewBuffer(nil)
buf.WriteByte(uint8(t))
handle := codec.MsgpackHandle{}
encoder := codec.NewEncoder(buf, &handle)
err := encoder.Encode(msg)
return buf.Bytes(), err
}
func MockEncoder(obj interface{}) []byte {
buf := bytes.NewBuffer(nil)
encoder := codec.NewEncoder(buf, msgpackHandle)
err := encoder.Encode(obj)
if err != nil {
panic(err)
}
return buf.Bytes()
}
func (m *MockSnapshot) Persist(sink SnapshotSink) error {
hd := codec.MsgpackHandle{}
enc := codec.NewEncoder(sink, &hd)
if err := enc.Encode(m.logs[:m.maxIndex]); err != nil {
sink.Cancel()
return err
}
sink.Close()
return nil
}
// encodeTags is used to encode a tag map
func (s *Serf) encodeTags(tags map[string]string) []byte {
// Support role-only backwards compatibility
if s.ProtocolVersion() < 3 {
role := tags["role"]
return []byte(role)
}
// Use a magic byte prefix and msgpack encode the tags
var buf bytes.Buffer
buf.WriteByte(tagMagicByte)
enc := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
if err := enc.Encode(tags); err != nil {
panic(fmt.Sprintf("Failed to encode tags: %v", err))
}
return buf.Bytes()
}
// handleConn is used to handle an inbound connection for its lifespan
func (n *NetworkTransport) handleConn(conn net.Conn) {
defer conn.Close()
r := bufio.NewReader(conn)
w := bufio.NewWriter(conn)
dec := codec.NewDecoder(r, &codec.MsgpackHandle{})
enc := codec.NewEncoder(w, &codec.MsgpackHandle{})
for {
if err := n.handleCommand(r, dec, enc); err != nil {
if err != io.EOF {
n.logger.Printf("[ERR] raft-net: Failed to decode incoming command: %v", err)
}
return
}
if err := w.Flush(); err != nil {
n.logger.Printf("[ERR] raft-net: Failed to flush response: %v", err)
return
}
}
}
func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error {
defer metrics.MeasureSince([]string{"consul", "fsm", "persist"}, time.Now())
// Register the nodes
encoder := codec.NewEncoder(sink, msgpackHandle)
// Write the header
header := snapshotHeader{
LastIndex: s.state.LastIndex(),
}
if err := encoder.Encode(&header); err != nil {
sink.Cancel()
return err
}
if err := s.persistNodes(sink, encoder); err != nil {
sink.Cancel()
return err
}
if err := s.persistSessions(sink, encoder); err != nil {
sink.Cancel()
return err
}
if err := s.persistACLs(sink, encoder); err != nil {
sink.Cancel()
return err
}
if err := s.persistKV(sink, encoder); err != nil {
sink.Cancel()
return err
}
if err := s.persistTombstones(sink, encoder); err != nil {
sink.Cancel()
return err
}
return nil
}
// listen is a long running routine that listens for new clients
func (i *AgentRPC) listen() {
for {
conn, err := i.listener.Accept()
if err != nil {
if i.stop {
return
}
i.logger.Printf("[ERR] agent.rpc: Failed to accept client: %v", err)
continue
}
i.logger.Printf("[INFO] agent.rpc: Accepted client: %v", conn.RemoteAddr())
// Wrap the connection in a client
client := &rpcClient{
name: conn.RemoteAddr().String(),
conn: conn,
reader: bufio.NewReader(conn),
writer: bufio.NewWriter(conn),
}
client.dec = codec.NewDecoder(client.reader, msgpackHandle)
client.enc = codec.NewEncoder(client.writer, msgpackHandle)
if err != nil {
i.logger.Printf("[ERR] agent.rpc: Failed to create decoder: %v", err)
conn.Close()
continue
}
// Register the client
i.Lock()
if !i.stop {
i.clients[client.name] = client
go i.handleClient(client)
} else {
conn.Close()
}
i.Unlock()
}
}
// NewRPCClient is used to create a new RPC client given the address.
// This will properly dial, handshake, and start listening
func NewRPCClient(addr string) (*RPCClient, error) {
var conn net.Conn
var err error
if envAddr := os.Getenv("CONSUL_RPC_ADDR"); envAddr != "" {
addr = envAddr
}
// Try to dial to agent
mode := "tcp"
if strings.HasPrefix(addr, "/") {
mode = "unix"
}
if conn, err = net.Dial(mode, addr); err != nil {
return nil, err
}
// Create the client
client := &RPCClient{
seq: 0,
conn: conn,
reader: bufio.NewReader(conn),
writer: bufio.NewWriter(conn),
dispatch: make(map[uint64]seqHandler),
shutdownCh: make(chan struct{}),
}
client.dec = codec.NewDecoder(client.reader, msgpackHandle)
client.enc = codec.NewEncoder(client.writer, msgpackHandle)
go client.listen()
// Do the initial handshake
if err := client.handshake(); err != nil {
client.Close()
return nil, err
}
return client, err
}
// Encode is used to encode a MsgPack object with type prefix
func Encode(t MessageType, msg interface{}) ([]byte, error) {
var buf bytes.Buffer
buf.WriteByte(uint8(t))
err := codec.NewEncoder(&buf, msgpackHandle).Encode(msg)
return buf.Bytes(), err
}
// sendLocalState is invoked to send our local state over a tcp connection
func (m *Memberlist) sendLocalState(conn net.Conn, join bool) error {
// Setup a deadline
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
// Prepare the local node state
m.nodeLock.RLock()
localNodes := make([]pushNodeState, len(m.nodes))
for idx, n := range m.nodes {
localNodes[idx].Name = n.Name
localNodes[idx].Addr = n.Addr
localNodes[idx].Port = n.Port
localNodes[idx].Incarnation = n.Incarnation
localNodes[idx].State = n.State
localNodes[idx].Meta = n.Meta
localNodes[idx].Vsn = []uint8{
n.PMin, n.PMax, n.PCur,
n.DMin, n.DMax, n.DCur,
}
}
m.nodeLock.RUnlock()
// Get the delegate state
var userData []byte
if m.config.Delegate != nil {
userData = m.config.Delegate.LocalState(join)
}
// Create a bytes buffer writer
bufConn := bytes.NewBuffer(nil)
// Send our node state
header := pushPullHeader{Nodes: len(localNodes), UserStateLen: len(userData), Join: join}
hd := codec.MsgpackHandle{}
enc := codec.NewEncoder(bufConn, &hd)
// Begin state push
if _, err := bufConn.Write([]byte{byte(pushPullMsg)}); err != nil {
return err
}
if err := enc.Encode(&header); err != nil {
return err
}
for i := 0; i < header.Nodes; i++ {
if err := enc.Encode(&localNodes[i]); err != nil {
return err
}
}
// Write the user state as well
if userData != nil {
if _, err := bufConn.Write(userData); err != nil {
return err
}
}
// Get the send buffer
sendBuf := bufConn.Bytes()
// Check if compresion is enabled
if m.config.EnableCompression {
compBuf, err := compressPayload(bufConn.Bytes())
if err != nil {
m.logger.Printf("[ERROR] memberlist: Failed to compress local state: %v", err)
} else {
sendBuf = compBuf.Bytes()
}
}
// Check if encryption is enabled
if m.config.EncryptionEnabled() {
crypt, err := m.encryptLocalState(sendBuf)
if err != nil {
m.logger.Printf("[ERROR] memberlist: Failed to encrypt local state: %v", err)
return err
}
sendBuf = crypt
}
// Write out the entire send buffer
metrics.IncrCounter([]string{"memberlist", "tcp", "sent"}, float32(len(sendBuf)))
if _, err := conn.Write(sendBuf); err != nil {
return err
}
return nil
}
func TestTCPPushPull(t *testing.T) {
m := GetMemberlist(t)
defer m.Shutdown()
m.nodes = append(m.nodes, &nodeState{
Node: Node{
Name: "Test 0",
Addr: net.ParseIP(m.config.BindAddr),
Port: uint16(m.config.BindPort),
},
Incarnation: 0,
State: stateSuspect,
StateChange: time.Now().Add(-1 * time.Second),
})
addr := fmt.Sprintf("%s:%d", m.config.BindAddr, m.config.BindPort)
conn, err := net.Dial("tcp", addr)
if err != nil {
t.Fatalf("unexpected err %s", err)
}
defer conn.Close()
localNodes := make([]pushNodeState, 3)
localNodes[0].Name = "Test 0"
localNodes[0].Addr = net.ParseIP(m.config.BindAddr)
localNodes[0].Port = uint16(m.config.BindPort)
localNodes[0].Incarnation = 1
localNodes[0].State = stateAlive
localNodes[1].Name = "Test 1"
localNodes[1].Addr = net.ParseIP(m.config.BindAddr)
localNodes[1].Port = uint16(m.config.BindPort)
localNodes[1].Incarnation = 1
localNodes[1].State = stateAlive
localNodes[2].Name = "Test 2"
localNodes[2].Addr = net.ParseIP(m.config.BindAddr)
localNodes[2].Port = uint16(m.config.BindPort)
localNodes[2].Incarnation = 1
localNodes[2].State = stateAlive
// Send our node state
header := pushPullHeader{Nodes: 3}
hd := codec.MsgpackHandle{}
enc := codec.NewEncoder(conn, &hd)
// Send the push/pull indicator
conn.Write([]byte{byte(pushPullMsg)})
if err := enc.Encode(&header); err != nil {
t.Fatalf("unexpected err %s", err)
}
for i := 0; i < header.Nodes; i++ {
if err := enc.Encode(&localNodes[i]); err != nil {
t.Fatalf("unexpected err %s", err)
}
}
// Read the message type
var msgType messageType
if err := binary.Read(conn, binary.BigEndian, &msgType); err != nil {
t.Fatalf("unexpected err %s", err)
}
var bufConn io.Reader = conn
msghd := codec.MsgpackHandle{}
dec := codec.NewDecoder(bufConn, &msghd)
// Check if we have a compressed message
if msgType == compressMsg {
var c compress
if err := dec.Decode(&c); err != nil {
t.Fatalf("unexpected err %s", err)
}
decomp, err := decompressBuffer(&c)
if err != nil {
t.Fatalf("unexpected err %s", err)
}
// Reset the message type
msgType = messageType(decomp[0])
// Create a new bufConn
bufConn = bytes.NewReader(decomp[1:])
// Create a new decoder
dec = codec.NewDecoder(bufConn, &hd)
}
// Quit if not push/pull
if msgType != pushPullMsg {
t.Fatalf("bad message type")
}
if err := dec.Decode(&header); err != nil {
t.Fatalf("unexpected err %s", err)
}
// Allocate space for the transfer
remoteNodes := make([]pushNodeState, header.Nodes)
// Try to decode all the states
for i := 0; i < header.Nodes; i++ {
if err := dec.Decode(&remoteNodes[i]); err != nil {
t.Fatalf("unexpected err %s", err)
}
}
if len(remoteNodes) != 1 {
t.Fatalf("bad response")
}
n := &remoteNodes[0]
if n.Name != "Test 0" {
t.Fatalf("bad name")
}
if bytes.Compare(n.Addr, net.ParseIP(m.config.BindAddr)) != 0 {
t.Fatal("bad addr")
}
if n.Incarnation != 0 {
t.Fatal("bad incarnation")
}
if n.State != stateSuspect {
t.Fatal("bad state")
}
}
// encodeMsgPack is used to encode an object with msgpack
func encodeMsgPack(msg interface{}) ([]byte, error) {
var buf bytes.Buffer
err := codec.NewEncoder(&buf, msgpackHandle).Encode(msg)
return buf.Bytes(), err
}