// sendRPC takes an RPC call, and will send it to the correct region server. If // the correct region server is offline or otherwise unavailable, sendRPC will // continually retry until the deadline set on the RPC's context is exceeded. func (c *Client) sendRPC(rpc hrpc.Call) (proto.Message, error) { log.WithFields(log.Fields{ "Type": rpc.GetName(), "Table": string(rpc.Table()), "Key": string(rpc.Key()), }).Debug("Sending RPC") err := c.queueRPC(rpc) if err == ErrDeadline { return nil, err } else if err != nil { log.WithFields(log.Fields{ "Type": rpc.GetName(), "Table": string(rpc.Table()), "Key": string(rpc.Key()), }).Debug("We hit an error queuing the RPC. Resending.") // There was an error locating the region for the RPC, or the client // for the region encountered an error and has shut down. return c.sendRPC(rpc) } if err == nil { var res hrpc.RPCResult resch := rpc.GetResultChan() select { case res = <-resch: case <-rpc.GetContext().Done(): return nil, ErrDeadline } err := res.Error log.WithFields(log.Fields{ "Type": rpc.GetName(), "Table": string(rpc.Table()), "Key": string(rpc.Key()), "Result": res.Msg, "Error": err, }).Debug("Successfully sent RPC. Returning.") if _, ok := err.(region.RetryableError); ok { return c.sendRPC(rpc) } else if _, ok := err.(region.UnrecoverableError); ok { // Prevents dropping into the else block below, // error handling happens a few lines down } else { return res.Msg, res.Error } } // There was an issue related to the network, so we're going to mark the // region as unavailable, and generate the channel used for announcing // when it's available again region := rpc.GetRegion() log.WithFields(log.Fields{ "Type": rpc.GetName(), "Table": string(rpc.Table()), "Key": string(rpc.Key()), }).Debug("Encountered a network error. Region unavailable?") if region != nil { succ := region.MarkUnavailable() if succ { go c.reestablishRegion(region) } } log.WithFields(log.Fields{ "Type": rpc.GetName(), "Table": string(rpc.Table()), "Key": string(rpc.Key()), }).Debug("Retrying sendRPC") return c.sendRPC(rpc) }