func (c *client) findRegionForRPC(rpc hrpc.Call) (proto.Message, error) {
// The region was not in the cache, it
// must be looked up in the meta table
backoff := backoffStart
ctx := rpc.Context()
for {
// Look up the region in the meta table
reg, host, port, err := c.locateRegion(ctx, rpc.Table(), rpc.Key())
if err != nil {
if err == TableNotFound {
return nil, err
}
// There was an error with the meta table. Let's sleep for some
// backoff amount and retry.
backoff, err = sleepAndIncreaseBackoff(ctx, backoff)
if err != nil {
return nil, err
}
continue
}
// Check that the region wasn't added to
// the cache while we were looking it up.
c.regionsLock.Lock()
if existing := c.getRegionFromCache(rpc.Table(), rpc.Key()); existing != nil {
// The region was added to the cache while we were looking it
// up. Send the RPC to the region that was in the cache.
c.regionsLock.Unlock()
return c.sendRPCToRegion(rpc, existing)
}
// The region wasn't added to the cache while we were looking it
// up. Mark this one as unavailable and add it to the cache.
reg.MarkUnavailable()
removed := c.regions.put(reg)
for _, r := range removed {
c.clients.del(r)
}
c.regionsLock.Unlock()
// Start a goroutine to connect to the region
go c.establishRegion(reg, host, port)
// Wait for the new region to become
// available, and then send the RPC
return c.waitOnRegion(rpc, reg)
}
}
func (c *client) waitOnRegion(rpc hrpc.Call, reg hrpc.RegionInfo) (proto.Message, error) {
ch := reg.AvailabilityChan()
if ch == nil {
// WTF, this region is available? Maybe it was marked as such
// since waitOnRegion was called.
return c.sendRPC(rpc)
}
// The region is unavailable. Wait for it to become available,
// or for the deadline to be exceeded.
select {
case <-ch:
return c.sendRPC(rpc)
case <-rpc.Context().Done():
return nil, ErrDeadline
}
}
func (c *client) sendRPCToRegion(rpc hrpc.Call, reg hrpc.RegionInfo) (proto.Message, error) {
client := reg.Client()
// On the first sendRPC to the meta or admin regions, a goroutine must be
// manually kicked off for the meta or admin region client
if reg == c.adminRegionInfo && client == nil && !c.adminRegionInfo.IsUnavailable() ||
reg == c.metaRegionInfo && client == nil && !c.metaRegionInfo.IsUnavailable() {
if reg.MarkUnavailable() {
go c.reestablishRegion(reg)
}
}
// The region was in the cache, check
// if the region is marked as available
if reg.IsUnavailable() {
return c.waitOnRegion(rpc, reg)
}
rpc.SetRegion(reg)
// Queue the RPC to be sent to the region
var err error
if client == nil {
err = errors.New("no client for this region")
} else {
err = client.QueueRPC(rpc)
}
if err != nil {
// There was an error queueing the RPC.
// Mark the region as unavailable.
first := reg.MarkUnavailable()
// If this was the first goroutine to mark the region as
// unavailable, start a goroutine to reestablish a connection
if first {
go c.reestablishRegion(reg)
}
// Block until the region becomes available.
return c.waitOnRegion(rpc, reg)
}
// Wait for the response
var res hrpc.RPCResult
select {
case res = <-rpc.ResultChan():
case <-rpc.Context().Done():
return nil, ErrDeadline
}
// Check for errors
if _, ok := res.Error.(region.RetryableError); ok {
// There's an error specific to this region, but
// our region client is fine. Mark this region as
// unavailable (as opposed to all regions sharing
// the client), and start a goroutine to reestablish
// it.
first := reg.MarkUnavailable()
if first {
go c.reestablishRegion(reg)
}
if reg != c.metaRegionInfo && reg != c.adminRegionInfo {
// The client won't be in the cache if this is the
// meta or admin region
c.clients.del(reg)
}
return c.waitOnRegion(rpc, reg)
} else if _, ok := res.Error.(region.UnrecoverableError); ok {
// If it was an unrecoverable error, the region client is
// considered dead.
if reg == c.metaRegionInfo || reg == c.adminRegionInfo {
// If this is the admin client or the meta table, mark the
// region as unavailable and start up a goroutine to
// reconnect if it wasn't already marked as such.
first := reg.MarkUnavailable()
if first {
go c.reestablishRegion(reg)
}
} else {
// Else this is a normal region. Mark all the regions
// sharing this region's client as unavailable, and start
// a goroutine to reconnect for each of them.
downregions := c.clients.clientDown(reg)
for _, downreg := range downregions {
go c.reestablishRegion(downreg)
}
}
// Fall through to the case of the region being unavailable,
// which will result in blocking until it's available again.
return c.waitOnRegion(rpc, reg)
} else {
// RPC was successfully sent, or an unknown type of error
// occurred. In either case, return the results.
return res.Msg, res.Error
}
}