示例#1
0
// reserve send a reservation request rpc to the node and store
// based on the toStoreID. It returns an error if the reservation was not
// successfully booked. When unsuccessful, the store is marked as having a
// declined reservation so it will not be considered for up-replication or
// rebalancing until after the configured timeout period has passed.
// TODO(bram): consider moving the nodeID to the store pool during
// NewStorePool.
func (sp *StorePool) reserve(
	curIdent roachpb.StoreIdent,
	toStoreID roachpb.StoreID,
	rangeID roachpb.RangeID,
	rangeSize int64,
) error {
	if !sp.reservationsEnabled {
		return nil
	}
	sp.mu.Lock()
	defer sp.mu.Unlock()
	detail, ok := sp.mu.stores[toStoreID]
	if !ok {
		return errors.Errorf("store %d does not exist in the store pool", toStoreID)
	}
	addr, err := sp.resolver(detail.desc.Node.NodeID)
	if err != nil {
		return err
	}
	conn, err := sp.rpcContext.GRPCDial(addr.String())
	if err != nil {
		return errors.Wrapf(err, "failed to dial store %+v, addr %q, node %+v", toStoreID, addr, detail.desc.Node)
	}

	client := roachpb.NewInternalStoresClient(conn)
	req := &roachpb.ReservationRequest{
		StoreRequestHeader: roachpb.StoreRequestHeader{
			NodeID:  detail.desc.Node.NodeID,
			StoreID: toStoreID,
		},
		FromNodeID:  curIdent.NodeID,
		FromStoreID: curIdent.StoreID,
		RangeSize:   rangeSize,
		RangeID:     rangeID,
	}

	if log.V(2) {
		log.Infof(context.TODO(), "proposing new reservation:%+v", req)
	}

	ctxWithTimeout, cancel := context.WithTimeout(context.TODO(), sp.reserveRPCTimeout)
	defer cancel()
	resp, err := client.Reserve(ctxWithTimeout, req)

	// If a reservation is declined, be it due to an error or because it was
	// rejected, we mark the store detail as having been rejected so it won't
	// be considered as a candidate for new replicas until after the configured
	// timeout period has passed.
	if err != nil {
		detail.throttledUntil = sp.clock.Now().GoTime().Add(sp.failedReservationsTimeout)
		if log.V(2) {
			log.Infof(context.TODO(), "reservation failed, store:%s will be throttled for %s until %s",
				toStoreID, sp.failedReservationsTimeout, detail.throttledUntil)
		}
		return errors.Wrapf(err, "reservation failed:%+v", req)
	}

	if resp.RangeCount != nil {
		detail.desc.Capacity.RangeCount = *resp.RangeCount
	}
	if !resp.Reserved {
		detail.throttledUntil = sp.clock.Now().GoTime().Add(sp.declinedReservationsTimeout)
		if log.V(2) {
			log.Infof(context.TODO(), "reservation failed, store:%s will be throttled for %s until %s",
				toStoreID, sp.declinedReservationsTimeout, detail.throttledUntil)
		}
		return errors.Errorf("reservation declined:%+v", req)
	}

	if log.V(2) {
		log.Infof(context.TODO(), "reservation was approved:%+v", req)
	}
	return nil
}
示例#2
0
// waitForStoreFrozen polls the given stores until they all report having no
// unfrozen Replicas (or an error or timeout occurs).
func (s *adminServer) waitForStoreFrozen(
	stream serverpb.Admin_ClusterFreezeServer,
	stores map[roachpb.StoreID]roachpb.NodeID,
	wantFrozen bool,
) error {
	mu := struct {
		syncutil.Mutex
		oks map[roachpb.StoreID]bool
	}{
		oks: make(map[roachpb.StoreID]bool),
	}

	opts := base.DefaultRetryOptions()
	opts.Closer = s.server.stopper.ShouldQuiesce()
	opts.MaxRetries = 20
	sem := make(chan struct{}, 256)
	errChan := make(chan error, 1)
	sendErr := func(err error) {
		select {
		case errChan <- err:
		default:
		}
	}

	numWaiting := len(stores) // loop until this drops to zero
	var err error
	for r := retry.Start(opts); r.Next(); {
		mu.Lock()
		for storeID, nodeID := range stores {
			storeID, nodeID := storeID, nodeID // loop-local copies for goroutine
			var nodeDesc roachpb.NodeDescriptor
			if err := s.server.gossip.GetInfoProto(gossip.MakeNodeIDKey(nodeID), &nodeDesc); err != nil {
				sendErr(err)
				break
			}
			addr := nodeDesc.Address.String()

			if _, inflightOrSucceeded := mu.oks[storeID]; inflightOrSucceeded {
				continue
			}
			mu.oks[storeID] = false // mark as inflight
			action := func() (err error) {
				var resp *roachpb.PollFrozenResponse
				defer func() {
					message := fmt.Sprintf("node %d, store %d: ", nodeID, storeID)

					if err != nil {
						message += err.Error()
					} else {
						numMismatching := len(resp.Results)
						mu.Lock()
						if numMismatching == 0 {
							// If the Store is in the right state, mark it as such.
							// This means we won't try it again.
							message += "ready"
							mu.oks[storeID] = true
						} else {
							// Otherwise, forget that we tried the Store so that
							// the retry loop picks it up again.
							message += fmt.Sprintf("%d replicas report wrong status", numMismatching)
							if limit := 10; numMismatching > limit {
								message += " [truncated]: "
								resp.Results = resp.Results[:limit]
							} else {
								message += ": "
							}
							message += fmt.Sprintf("%+v", resp.Results)
							delete(mu.oks, storeID)
						}
						mu.Unlock()
						err = stream.Send(&serverpb.ClusterFreezeResponse{
							Message: message,
						})
					}
				}()
				conn, err := s.server.rpcContext.GRPCDial(addr)
				if err != nil {
					return err
				}
				client := roachpb.NewInternalStoresClient(conn)
				resp, err = client.PollFrozen(context.Background(),
					&roachpb.PollFrozenRequest{
						StoreRequestHeader: roachpb.StoreRequestHeader{
							NodeID:  nodeID,
							StoreID: storeID,
						},
						// If we are looking to freeze everything, we want to
						// collect thawed Replicas, and vice versa.
						CollectFrozen: !wantFrozen,
					})
				return err
			}
			// Run a limited, non-blocking task. That means the task simply
			// won't run if the semaphore is full (or the node is draining).
			// Both are handled by the surrounding retry loop.
			if err := s.server.stopper.RunLimitedAsyncTask(sem, func() {
				if err := action(); err != nil {
					sendErr(err)
				}
			}); err != nil {
				// Node draining.
				sendErr(err)
				break
			}
		}

		numWaiting = len(stores)
		for _, ok := range mu.oks {
			if ok {
				// Store has reported that it is frozen.
				numWaiting--
				continue
			}
		}
		mu.Unlock()

		select {
		case err = <-errChan:
		default:
		}

		// Keep going unless there's been an error or everyone's frozen.
		if err != nil || numWaiting == 0 {
			break
		}
		if err := stream.Send(&serverpb.ClusterFreezeResponse{
			Message: fmt.Sprintf("waiting for %d store%s to apply operation",
				numWaiting, util.Pluralize(int64(numWaiting))),
		}); err != nil {
			return err
		}
	}
	if err != nil {
		return err
	}
	if numWaiting > 0 {
		err = fmt.Errorf("timed out waiting for %d store%s to report freeze",
			numWaiting, util.Pluralize(int64(numWaiting)))
	}
	return err
}