Example #1
0
func (c *client) findRegionForRPC(rpc hrpc.Call) (proto.Message, error) {
	// The region was not in the cache, it
	// must be looked up in the meta table

	backoff := backoffStart
	ctx := rpc.Context()
	for {
		// Look up the region in the meta table
		reg, host, port, err := c.locateRegion(ctx, rpc.Table(), rpc.Key())

		if err != nil {
			if err == TableNotFound {
				return nil, err
			}
			// There was an error with the meta table. Let's sleep for some
			// backoff amount and retry.
			backoff, err = sleepAndIncreaseBackoff(ctx, backoff)
			if err != nil {
				return nil, err
			}
			continue
		}

		// Check that the region wasn't added to
		// the cache while we were looking it up.
		c.regionsLock.Lock()

		if existing := c.getRegionFromCache(rpc.Table(), rpc.Key()); existing != nil {
			// The region was added to the cache while we were looking it
			// up. Send the RPC to the region that was in the cache.
			c.regionsLock.Unlock()
			return c.sendRPCToRegion(rpc, existing)
		}

		// The region wasn't added to the cache while we were looking it
		// up. Mark this one as unavailable and add it to the cache.
		reg.MarkUnavailable()
		removed := c.regions.put(reg)
		for _, r := range removed {
			c.clients.del(r)
		}
		c.regionsLock.Unlock()

		// Start a goroutine to connect to the region
		go c.establishRegion(reg, host, port)

		// Wait for the new region to become
		// available, and then send the RPC
		return c.waitOnRegion(rpc, reg)
	}
}
Example #2
0
func (c *client) waitOnRegion(rpc hrpc.Call, reg hrpc.RegionInfo) (proto.Message, error) {
	ch := reg.AvailabilityChan()
	if ch == nil {
		// WTF, this region is available? Maybe it was marked as such
		// since waitOnRegion was called.
		return c.sendRPC(rpc)
	}
	// The region is unavailable. Wait for it to become available,
	// or for the deadline to be exceeded.
	select {
	case <-ch:
		return c.sendRPC(rpc)
	case <-rpc.Context().Done():
		return nil, ErrDeadline
	}
}
Example #3
0
func (c *client) sendRPCToRegion(rpc hrpc.Call, reg hrpc.RegionInfo) (proto.Message, error) {
	client := reg.Client()
	// On the first sendRPC to the meta or admin regions, a goroutine must be
	// manually kicked off for the meta or admin region client
	if reg == c.adminRegionInfo && client == nil && !c.adminRegionInfo.IsUnavailable() ||
		reg == c.metaRegionInfo && client == nil && !c.metaRegionInfo.IsUnavailable() {
		if reg.MarkUnavailable() {
			go c.reestablishRegion(reg)
		}
	}
	// The region was in the cache, check
	// if the region is marked as available
	if reg.IsUnavailable() {
		return c.waitOnRegion(rpc, reg)
	}

	rpc.SetRegion(reg)

	// Queue the RPC to be sent to the region
	var err error
	if client == nil {
		err = errors.New("no client for this region")
	} else {
		err = client.QueueRPC(rpc)
	}

	if err != nil {
		// There was an error queueing the RPC.
		// Mark the region as unavailable.
		first := reg.MarkUnavailable()
		// If this was the first goroutine to mark the region as
		// unavailable, start a goroutine to reestablish a connection
		if first {
			go c.reestablishRegion(reg)
		}
		// Block until the region becomes available.
		return c.waitOnRegion(rpc, reg)
	}

	// Wait for the response
	var res hrpc.RPCResult
	select {
	case res = <-rpc.ResultChan():
	case <-rpc.Context().Done():
		return nil, ErrDeadline
	}

	// Check for errors
	if _, ok := res.Error.(region.RetryableError); ok {
		// There's an error specific to this region, but
		// our region client is fine. Mark this region as
		// unavailable (as opposed to all regions sharing
		// the client), and start a goroutine to reestablish
		// it.
		first := reg.MarkUnavailable()
		if first {
			go c.reestablishRegion(reg)
		}
		if reg != c.metaRegionInfo && reg != c.adminRegionInfo {
			// The client won't be in the cache if this is the
			// meta or admin region
			c.clients.del(reg)
		}
		return c.waitOnRegion(rpc, reg)
	} else if _, ok := res.Error.(region.UnrecoverableError); ok {
		// If it was an unrecoverable error, the region client is
		// considered dead.
		if reg == c.metaRegionInfo || reg == c.adminRegionInfo {
			// If this is the admin client or the meta table, mark the
			// region as unavailable and start up a goroutine to
			// reconnect if it wasn't already marked as such.
			first := reg.MarkUnavailable()
			if first {
				go c.reestablishRegion(reg)
			}
		} else {
			// Else this is a normal region. Mark all the regions
			// sharing this region's client as unavailable, and start
			// a goroutine to reconnect for each of them.
			downregions := c.clients.clientDown(reg)
			for _, downreg := range downregions {
				go c.reestablishRegion(downreg)
			}
		}

		// Fall through to the case of the region being unavailable,
		// which will result in blocking until it's available again.
		return c.waitOnRegion(rpc, reg)
	} else {
		// RPC was successfully sent, or an unknown type of error
		// occurred. In either case, return the results.
		return res.Msg, res.Error
	}
}