// GossipStore broadcasts the store on the gossip network.
func (s *Store) gossipStore(rangeCount int) error {
desc := s.getDesc(rangeCount)
// Unique gossip key per store.
gossipKey := gossip.MakeStoreKey(desc.StoreID)
// Gossip store descriptor.
return s.gossip.AddInfoProto(gossipKey, &desc, 0)
}
// GossipStores queues up a list of stores to gossip and blocks until each one
// is gossiped before returning.
func (sg *StoreGossiper) GossipStores(storeDescs []*roachpb.StoreDescriptor, t *testing.T) {
storeIDs := make([]roachpb.StoreID, len(storeDescs))
for i, store := range storeDescs {
storeIDs[i] = store.StoreID
}
sg.GossipWithFunction(storeIDs, func() {
for i, storeDesc := range storeDescs {
if err := sg.g.AddInfoProto(gossip.MakeStoreKey(storeIDs[i]), storeDesc, 0); err != nil {
t.Fatal(err)
}
}
})
}
// GossipWithFunction calls gossipFn and blocks until gossip callbacks have
// fired on each of the stores specified by storeIDs.
func (sg *StoreGossiper) GossipWithFunction(storeIDs []roachpb.StoreID, gossipFn func()) {
sg.mu.Lock()
defer sg.mu.Unlock()
sg.storeKeyMap = make(map[string]struct{})
for _, storeID := range storeIDs {
storeKey := gossip.MakeStoreKey(storeID)
sg.storeKeyMap[storeKey] = struct{}{}
}
gossipFn()
// Wait for gossip callbacks to be invoked on all the stores.
for len(sg.storeKeyMap) > 0 {
sg.cond.Wait()
}
}
// RemoveTarget returns a suitable replica to remove from the provided replica
// set. It attempts to consider which of the provided replicas would be the best
// candidate for removal.
//
// TODO(mrtracy): RemoveTarget eventually needs to accept the attributes from
// the zone config associated with the provided replicas. This will allow it to
// make correct decisions in the case of ranges with heterogeneous replica
// requirements (i.e. multiple data centers).
func (a *allocator) RemoveTarget(existing []proto.Replica) (proto.Replica, error) {
if len(existing) == 0 {
return proto.Replica{}, util.Error("must supply at least one replica to allocator.RemoveTarget()")
}
a.Lock()
defer a.Unlock()
// Retrieve store descriptors for the provided replicas from gossip.
type replStore struct {
repl proto.Replica
store *proto.StoreDescriptor
}
replStores := make([]replStore, len(existing))
usedStat := stat{}
for i := range existing {
desc, err := storeDescFromGossip(gossip.MakeStoreKey(existing[i].StoreID), a.gossip)
if err != nil {
return proto.Replica{}, err
}
replStores[i] = replStore{
repl: existing[i],
store: desc,
}
usedStat.Update(desc.Capacity.FractionUsed())
}
// Based on store statistics, determine which replica is the "worst" and
// thus should be removed.
var worst replStore
for i, rs := range replStores {
if i == 0 {
worst = rs
continue
}
if usedStat.mean < minFractionUsedThreshold {
if rs.store.Capacity.RangeCount > worst.store.Capacity.RangeCount {
worst = rs
}
continue
}
if rs.store.Capacity.FractionUsed() > worst.store.Capacity.FractionUsed() {
worst = rs
}
}
return worst.repl, nil
}
func (sg *storeGossiper) gossipStores(stores []*proto.StoreDescriptor, t *testing.T) {
sg.mu.Lock()
defer sg.mu.Unlock()
sg.wg.Add(len(stores))
for _, s := range stores {
keyStoreGossip := gossip.MakeStoreKey(s.StoreID)
// Gossip store descriptor.
err := sg.g.AddInfoProto(keyStoreGossip, s, 0)
if err != nil {
t.Fatal(err)
}
}
// Wait for all gossip callbacks to be invoked.
sg.wg.Wait()
}
// GossipWithFunction is similar to GossipStores but instead of gossiping the
// store descriptors directly, call the passed in function to do so.
func (sg *StoreGossiper) GossipWithFunction(stores []roachpb.StoreID, gossiper func()) {
sg.mu.Lock()
sg.storeKeyMap = make(map[string]struct{})
sg.wg.Add(len(stores))
for _, s := range stores {
storeKey := gossip.MakeStoreKey(s)
sg.storeKeyMap[storeKey] = struct{}{}
}
// Gossip the stores via the passed in function.
gossiper()
sg.mu.Unlock()
// Wait for all gossip callbacks to be invoked.
sg.wg.Wait()
}
func gossipStores(g *gossip.Gossip, stores []*proto.StoreDescriptor, t *testing.T) {
var wg sync.WaitGroup
wg.Add(len(stores))
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ bool, _ interface{}) { wg.Done() })
for _, s := range stores {
keyStoreGossip := gossip.MakeStoreKey(s.StoreID)
// Gossip store descriptor.
err := g.AddInfo(keyStoreGossip, *s, 0)
if err != nil {
t.Fatal(err)
}
}
// Wait for all gossip callbacks to be invoked.
wg.Wait()
}
// gossipStores queues up a list of stores to gossip and blocks until each one
// is gossiped before returning.
func (sg *storeGossiper) gossipStores(stores []*proto.StoreDescriptor, t *testing.T) {
sg.mu.Lock()
sg.storeKeyMap = make(map[string]struct{})
sg.wg.Add(len(stores))
for _, s := range stores {
storeKey := gossip.MakeStoreKey(s.StoreID)
sg.storeKeyMap[storeKey] = struct{}{}
// Gossip store descriptor.
err := sg.g.AddInfoProto(storeKey, s, 0)
if err != nil {
t.Fatal(err)
}
}
sg.mu.Unlock()
// Wait for all gossip callbacks to be invoked.
sg.wg.Wait()
}
// GossipStores queues up a list of stores to gossip and blocks until each one
// is gossiped before returning.
func (sg *StoreGossiper) GossipStores(stores []*roachpb.StoreDescriptor, t *testing.T) {
sg.mu.Lock()
defer sg.mu.Unlock()
sg.storeKeyMap = make(map[string]struct{})
for _, s := range stores {
storeKey := gossip.MakeStoreKey(s.StoreID)
sg.storeKeyMap[storeKey] = struct{}{}
// Gossip store descriptor.
err := sg.g.AddInfoProto(storeKey, s, 0)
if err != nil {
t.Fatal(err)
}
}
// Wait for all gossip callbacks to be invoked.
for len(sg.storeKeyMap) > 0 {
sg.cond.Wait()
}
}
func Example_rebalancing() {
stopper := stop.NewStopper()
defer stopper.Stop()
// Model a set of stores in a cluster,
// randomly adding / removing stores and adding bytes.
g := gossip.New(nil, nil, stopper)
// Have to call g.SetNodeID before call g.AddInfo
g.SetNodeID(roachpb.NodeID(1))
sp := NewStorePool(
g,
hlc.NewClock(hlc.UnixNano),
nil,
/* reservationsEnabled */ true,
TestTimeUntilStoreDeadOff,
stopper,
)
alloc := MakeAllocator(sp, AllocatorOptions{AllowRebalance: true, Deterministic: true})
var wg sync.WaitGroup
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) { wg.Done() })
const generations = 100
const nodes = 20
// Initialize testStores.
var testStores [nodes]testStore
for i := 0; i < len(testStores); i++ {
testStores[i].StoreID = roachpb.StoreID(i)
testStores[i].Node = roachpb.NodeDescriptor{NodeID: roachpb.NodeID(i)}
testStores[i].Capacity = roachpb.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30}
}
// Initialize the cluster with a single range.
testStores[0].add(alloc.randGen.Int63n(1 << 20))
for i := 0; i < generations; i++ {
// First loop through test stores and add data.
wg.Add(len(testStores))
for j := 0; j < len(testStores); j++ {
// Add a pretend range to the testStore if there's already one.
if testStores[j].Capacity.RangeCount > 0 {
testStores[j].add(alloc.randGen.Int63n(1 << 20))
}
if err := g.AddInfoProto(gossip.MakeStoreKey(roachpb.StoreID(j)), &testStores[j].StoreDescriptor, 0); err != nil {
panic(err)
}
}
wg.Wait()
// Next loop through test stores and maybe rebalance.
for j := 0; j < len(testStores); j++ {
ts := &testStores[j]
if alloc.ShouldRebalance(ts.StoreID) {
target := alloc.RebalanceTarget(ts.StoreID, roachpb.Attributes{}, []roachpb.ReplicaDescriptor{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}})
if target != nil {
testStores[j].rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20))
}
}
}
// Output store capacities as hexidecimal 2-character values.
if i%(generations/50) == 0 {
var maxBytes int64
for j := 0; j < len(testStores); j++ {
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
if bytes > maxBytes {
maxBytes = bytes
}
}
if maxBytes > 0 {
for j := 0; j < len(testStores); j++ {
endStr := " "
if j == len(testStores)-1 {
endStr = ""
}
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr)
}
fmt.Printf("\n")
}
}
}
var totBytes int64
var totRanges int32
for i := 0; i < len(testStores); i++ {
totBytes += testStores[i].Capacity.Capacity - testStores[i].Capacity.Available
totRanges += testStores[i].Capacity.RangeCount
}
fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges)
// Output:
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 045 140 000 000 000 000 000 105 000 000
// 999 014 143 000 000 000 000 039 017 000 112 071 000 088 009 000 097 134 000 151
// 999 196 213 000 000 000 143 098 210 039 262 260 077 139 078 087 237 316 281 267
// 999 394 368 391 000 393 316 356 364 263 474 262 214 321 345 374 403 445 574 220
// 999 337 426 577 023 525 459 426 229 315 495 327 310 370 363 423 390 473 587 308
// 999 481 529 533 132 563 519 496 396 363 636 337 414 408 425 533 445 605 559 405
// 999 572 585 507 256 609 570 586 513 341 660 347 544 443 488 525 446 596 556 462
// 999 580 575 603 325 636 590 549 495 337 698 386 663 526 518 511 517 572 546 533
// 999 576 601 637 374 629 573 558 520 391 684 446 692 555 510 461 552 593 568 564
// 999 573 636 671 441 643 619 629 628 452 705 525 795 590 542 525 589 658 589 655
// 999 585 625 651 467 686 606 662 611 508 654 516 746 594 542 528 591 646 569 642
// 999 636 690 728 501 704 638 700 619 539 688 555 738 592 556 568 659 669 602 649
// 999 655 749 773 519 790 713 781 698 604 758 601 755 634 580 661 716 735 607 660
// 999 648 716 726 549 813 748 766 693 606 784 568 749 655 579 642 692 711 587 632
// 999 688 734 731 553 805 736 779 701 575 763 562 722 647 599 631 691 732 598 608
// 999 679 770 719 590 815 754 799 687 613 748 540 715 664 590 638 703 720 621 588
// 999 736 775 724 614 813 771 829 703 679 782 560 754 692 624 658 756 763 636 643
// 999 759 792 737 688 847 782 872 761 695 841 617 756 730 607 664 762 807 677 666
// 999 793 837 754 704 876 803 897 753 742 880 639 758 766 653 684 785 850 720 670
// 999 815 864 778 735 921 843 927 778 752 896 696 775 796 698 681 775 859 730 693
// 999 827 876 759 759 911 838 938 781 798 920 708 778 794 698 711 804 870 732 710
// 999 815 893 733 790 924 849 940 755 777 901 720 794 832 704 721 834 851 722 748
// 999 820 905 772 807 941 884 938 781 788 888 738 835 849 735 742 865 884 743 791
// 999 828 889 768 828 939 865 936 789 805 913 751 841 860 751 759 895 889 730 814
// 999 829 893 794 840 933 883 943 805 830 929 735 842 871 778 788 886 912 746 845
// 999 848 892 820 824 963 913 978 832 828 952 755 860 890 784 814 905 905 755 855
// 999 847 880 846 847 963 939 984 851 835 958 777 862 880 799 829 912 895 772 870
// 999 850 886 859 871 950 921 998 847 823 925 759 877 861 787 810 908 915 798 840
// 982 854 891 854 900 956 945 999 833 804 929 767 896 861 781 797 911 932 791 855
// 961 849 884 846 881 949 928 999 829 796 906 768 868 858 797 804 883 897 774 834
// 965 863 924 874 903 988 953 999 864 831 924 786 876 886 821 804 903 940 799 843
// 963 873 936 880 915 997 966 999 885 832 935 799 891 919 854 801 916 953 802 866
// 951 886 938 873 900 990 972 999 898 822 915 795 871 917 853 798 928 953 779 850
// 932 880 939 866 897 999 948 970 884 837 912 805 877 893 866 807 922 933 791 846
// 925 896 935 885 899 999 963 965 886 858 897 820 894 876 876 811 918 921 793 856
// 926 881 933 876 896 999 952 942 857 859 878 812 898 884 883 791 920 894 783 853
// 951 890 947 898 919 999 959 952 863 871 895 845 902 898 893 816 934 920 790 881
// 962 895 959 919 921 999 982 951 883 877 901 860 911 910 899 835 949 923 803 883
// 957 886 970 905 915 999 970 974 888 894 924 879 938 930 909 847 955 937 830 899
// 941 881 958 889 914 999 957 953 885 890 900 870 946 919 885 822 950 927 832 875
// 937 888 962 897 934 999 963 950 902 900 905 890 952 920 895 831 963 930 852 872
// 916 888 967 881 924 999 970 946 912 890 901 889 958 910 911 830 966 928 834 866
// 900 859 959 877 895 999 955 931 893 868 894 881 929 893 885 813 937 909 819 849
// 902 857 960 875 896 999 944 929 911 867 911 895 946 897 897 812 926 921 815 859
// 902 855 951 867 893 999 949 938 901 867 911 892 949 898 903 803 935 930 809 868
// Total bytes=909881714, ranges=1745
}
func Example_rebalancing() {
// Model a set of stores in a cluster,
// randomly adding / removing stores and adding bytes.
g := gossip.New(nil, 0, nil)
alloc := newAllocator(g)
alloc.randGen = rand.New(rand.NewSource(0))
alloc.deterministic = true
var wg sync.WaitGroup
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ []byte) { wg.Done() })
const generations = 100
const nodes = 20
// Initialize testStores.
var testStores [nodes]testStore
for i := 0; i < len(testStores); i++ {
testStores[i].StoreID = proto.StoreID(i)
testStores[i].Node = proto.NodeDescriptor{NodeID: proto.NodeID(i)}
testStores[i].Capacity = proto.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30}
}
// Initialize the cluster with a single range.
testStores[0].Add(alloc.randGen.Int63n(1 << 20))
for i := 0; i < generations; i++ {
// First loop through test stores and add data.
wg.Add(len(testStores))
for j := 0; j < len(testStores); j++ {
// Add a pretend range to the testStore if there's already one.
if testStores[j].Capacity.RangeCount > 0 {
testStores[j].Add(alloc.randGen.Int63n(1 << 20))
}
key := gossip.MakeStoreKey(proto.StoreID(j))
if err := g.AddInfoProto(key, &testStores[j].StoreDescriptor, 0); err != nil {
panic(err)
}
}
wg.Wait()
// Next loop through test stores and maybe rebalance.
for j := 0; j < len(testStores); j++ {
ts := &testStores[j]
if alloc.ShouldRebalance(&testStores[j].StoreDescriptor) {
target := alloc.RebalanceTarget(proto.Attributes{}, []proto.Replica{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}})
if target != nil {
testStores[j].Rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20))
}
}
}
// Output store capacities as hexidecimal 2-character values.
if i%(generations/50) == 0 {
var maxBytes int64
for j := 0; j < len(testStores); j++ {
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
if bytes > maxBytes {
maxBytes = bytes
}
}
if maxBytes > 0 {
for j := 0; j < len(testStores); j++ {
endStr := " "
if j == len(testStores)-1 {
endStr = ""
}
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr)
}
fmt.Printf("\n")
}
}
}
var totBytes int64
var totRanges int32
for i := 0; i < len(testStores); i++ {
totBytes += testStores[i].Capacity.Capacity - testStores[i].Capacity.Available
totRanges += testStores[i].Capacity.RangeCount
}
fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges)
// Output:
// 999 000 000 000 000 000 000 739 000 000 000 000 000 000 000 000 000 000 000 000
// 999 107 000 000 204 000 000 375 000 000 000 000 000 000 000 000 000 000 536 000
// 999 310 000 262 872 000 000 208 000 705 000 526 000 000 439 000 000 607 933 000
// 812 258 000 220 999 673 402 480 000 430 516 374 000 431 318 000 551 714 917 000
// 582 625 185 334 720 589 647 619 000 300 483 352 279 502 208 665 816 684 999 374
// 751 617 771 542 738 676 665 525 309 435 612 449 457 616 306 837 993 754 999 445
// 759 659 828 478 693 622 594 591 349 458 630 538 526 613 462 827 879 787 999 550
// 861 658 828 559 801 660 681 560 487 529 652 686 642 716 575 999 989 875 989 581
// 775 647 724 557 779 662 670 494 535 502 681 676 624 695 561 961 999 772 888 592
// 856 712 753 661 767 658 717 606 529 615 755 699 672 700 576 955 999 755 861 671
// 882 735 776 685 844 643 740 578 610 688 787 741 661 767 587 999 955 809 803 731
// 958 716 789 719 861 689 821 608 634 724 800 782 694 799 619 994 999 851 812 818
// 949 726 788 664 873 633 749 599 680 714 790 728 663 842 628 999 978 816 823 791
// 923 698 792 712 816 605 774 651 661 728 802 718 670 819 714 999 966 801 829 791
// 962 779 847 737 900 675 811 691 745 778 835 812 680 894 790 999 989 872 923 799
// 967 812 826 772 891 685 828 683 761 808 864 820 643 873 783 969 999 873 910 781
// 923 813 837 739 867 672 792 664 773 772 879 803 610 845 740 957 999 867 912 732
// 952 803 866 759 881 655 765 668 803 772 929 762 601 844 751 973 999 892 864 731
// 970 777 867 800 859 639 774 662 787 760 906 751 595 854 732 989 999 853 859 762
// 943 776 872 787 861 686 780 663 789 793 926 784 612 832 733 999 968 868 827 767
// 914 801 912 802 878 704 800 685 818 808 939 759 627 844 717 999 976 872 828 757
// 935 806 911 797 887 710 798 711 826 824 938 775 614 870 716 999 986 886 803 767
// 991 851 898 856 872 795 828 782 826 852 963 797 710 868 775 994 999 923 896 794
// 999 924 866 877 884 883 886 836 846 869 953 851 762 887 858 985 949 900 917 836
// 999 910 887 878 897 890 906 868 906 903 983 947 801 895 913 976 924 890 904 898
// 955 884 888 916 886 879 901 872 898 883 999 874 829 888 892 937 918 889 891 862
// 974 952 957 990 950 976 945 946 980 961 999 975 942 926 957 994 965 946 960 960
// 949 929 952 999 929 961 943 946 993 918 984 961 952 919 953 950 952 941 949 934
// 907 999 916 935 903 903 909 907 960 939 973 912 901 885 916 910 941 911 906 913
// 939 999 948 948 945 962 951 954 952 964 996 942 975 962 962 956 971 969 975 969
// 940 974 964 947 971 975 949 954 953 970 992 971 981 973 948 962 999 969 978 975
// 950 971 953 938 962 967 930 964 953 978 999 945 974 972 951 950 998 951 949 962
// 934 946 943 936 942 949 929 956 928 970 989 944 945 923 987 927 999 942 931 944
// 939 957 942 958 951 970 937 946 930 950 940 959 963 937 973 943 999 931 949 940
// 933 935 945 929 933 960 937 935 919 918 930 931 950 924 969 935 999 943 949 926
// 959 941 948 952 948 957 936 937 943 930 955 962 953 949 980 948 999 934 980 942
// 950 973 954 962 949 964 935 949 925 936 951 962 979 962 999 942 990 948 969 959
// 937 993 958 949 960 960 942 954 969 950 951 952 974 970 999 927 979 964 975 944
// 981 986 971 968 964 984 954 959 985 979 966 963 994 963 999 970 991 971 988 965
// 967 997 961 957 959 985 956 940 955 955 957 955 970 952 979 964 999 951 960 968
// 937 969 931 950 945 954 932 925 954 946 944 926 955 938 957 949 999 934 947 938
// 958 967 954 955 971 973 946 934 979 947 944 958 954 954 960 948 999 936 960 951
// 950 948 940 958 937 955 928 927 953 923 935 939 934 921 934 934 999 922 940 938
// 960 960 929 962 955 955 926 935 957 928 939 941 938 926 941 924 999 923 957 942
// 979 958 947 987 980 972 945 943 984 939 951 943 944 946 942 942 999 928 970 943
// 981 941 931 961 969 962 927 935 985 925 964 945 946 939 946 938 999 933 964 928
// 980 944 929 970 973 955 942 937 977 920 955 929 937 946 935 933 999 947 956 926
// 980 948 926 981 938 939 936 936 963 949 965 935 943 946 933 933 999 947 955 943
// 968 959 945 941 929 926 924 941 970 951 959 941 924 952 931 943 999 941 951 950
// 961 946 930 923 933 932 953 937 954 940 964 944 931 952 939 935 999 936 945 948
// Total bytes=996294324, ranges=1897
}
func Example_rebalancing() {
// Model a set of stores in a cluster,
// randomly adding / removing stores and adding bytes.
g := gossip.New(nil, 0, nil)
stopper := stop.NewStopper()
defer stopper.Stop()
sp := NewStorePool(g, TestTimeUntilStoreDeadOff, stopper)
alloc := makeAllocator(sp)
alloc.randGen = rand.New(rand.NewSource(0))
alloc.deterministic = true
var wg sync.WaitGroup
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ []byte) { wg.Done() })
const generations = 100
const nodes = 20
// Initialize testStores.
var testStores [nodes]testStore
for i := 0; i < len(testStores); i++ {
testStores[i].StoreID = proto.StoreID(i)
testStores[i].Node = proto.NodeDescriptor{NodeID: proto.NodeID(i)}
testStores[i].Capacity = proto.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30}
}
// Initialize the cluster with a single range.
testStores[0].add(alloc.randGen.Int63n(1 << 20))
for i := 0; i < generations; i++ {
// First loop through test stores and add data.
wg.Add(len(testStores))
for j := 0; j < len(testStores); j++ {
// Add a pretend range to the testStore if there's already one.
if testStores[j].Capacity.RangeCount > 0 {
testStores[j].add(alloc.randGen.Int63n(1 << 20))
}
key := gossip.MakeStoreKey(proto.StoreID(j))
if err := g.AddInfoProto(key, &testStores[j].StoreDescriptor, 0); err != nil {
panic(err)
}
}
wg.Wait()
// Next loop through test stores and maybe rebalance.
for j := 0; j < len(testStores); j++ {
ts := &testStores[j]
if alloc.shouldRebalance(&testStores[j].StoreDescriptor) {
target := alloc.rebalanceTarget(proto.Attributes{}, []proto.Replica{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}})
if target != nil {
testStores[j].rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20))
}
}
}
// Output store capacities as hexidecimal 2-character values.
if i%(generations/50) == 0 {
var maxBytes int64
for j := 0; j < len(testStores); j++ {
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
if bytes > maxBytes {
maxBytes = bytes
}
}
if maxBytes > 0 {
for j := 0; j < len(testStores); j++ {
endStr := " "
if j == len(testStores)-1 {
endStr = ""
}
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr)
}
fmt.Printf("\n")
}
}
}
var totBytes int64
var totRanges int32
for i := 0; i < len(testStores); i++ {
totBytes += testStores[i].Capacity.Capacity - testStores[i].Capacity.Available
totRanges += testStores[i].Capacity.RangeCount
}
fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges)
// Output:
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 739 000 000
// 999 107 000 000 000 000 000 000 000 000 177 000 000 000 204 000 000 734 000 000
// 929 288 000 168 000 057 623 000 114 272 471 000 000 565 385 000 000 999 000 284
// 683 367 133 087 000 527 381 607 379 380 502 000 188 824 490 295 420 999 000 490
// 540 443 380 319 000 438 382 534 599 579 602 000 268 859 601 374 450 999 000 532
// 412 428 539 429 170 332 424 696 505 439 503 691 327 752 427 437 451 999 076 441
// 496 583 662 586 280 431 499 714 564 578 540 661 431 784 548 516 547 999 329 589
// 502 563 646 541 430 428 576 693 633 578 537 577 455 803 573 596 528 999 402 639
// 603 641 764 638 764 521 650 764 713 683 648 652 579 860 610 731 665 999 463 749
// 615 642 779 688 813 459 650 791 728 702 743 614 526 829 600 767 760 999 497 700
// 677 677 879 787 867 518 700 852 775 801 793 666 526 820 601 843 767 999 544 772
// 723 696 866 838 853 589 730 882 800 768 782 695 567 776 656 836 832 999 613 832
// 830 764 936 879 976 673 824 974 864 825 835 761 703 874 700 909 888 999 635 957
// 832 766 949 842 995 730 839 965 870 843 790 765 693 931 706 936 936 999 683 948
// 866 787 990 851 999 780 867 968 892 847 783 787 708 912 768 963 951 954 681 942
// 844 768 995 866 999 811 836 952 931 849 764 763 716 923 758 980 907 921 705 927
// 866 780 999 897 979 844 858 975 968 897 762 772 739 939 731 984 909 912 752 931
// 895 818 999 918 991 884 916 936 942 899 790 780 775 930 779 976 963 893 752 920
// 832 782 999 865 978 854 908 902 887 886 760 785 747 903 762 924 911 830 750 876
// 864 804 999 856 987 855 939 911 905 909 735 818 808 909 757 941 956 807 761 875
// 892 839 999 901 977 871 933 930 931 910 766 812 831 908 768 969 935 861 793 867
// 908 858 982 911 994 872 947 951 960 935 798 832 835 895 786 999 951 873 790 863
// 881 863 963 908 991 845 919 941 959 939 801 815 832 876 783 999 959 877 798 822
// 907 929 962 930 942 904 901 925 955 940 871 985 921 902 873 999 991 894 908 926
// 892 923 956 939 976 961 904 933 958 942 914 979 925 907 940 999 982 906 929 922
// 913 928 940 921 930 913 913 929 925 918 920 963 923 923 940 999 928 909 923 948
// 924 950 910 947 926 918 917 933 944 920 907 917 911 920 914 999 921 909 915 939
// 908 933 920 946 940 936 935 931 957 924 914 914 915 936 929 999 929 934 924 906
// 946 943 954 968 961 966 966 935 951 940 935 959 949 950 938 996 941 981 955 999
// 940 926 938 941 930 935 934 935 958 897 941 955 931 943 944 971 949 941 972 999
// 950 952 944 957 935 953 951 938 972 932 957 988 943 950 970 961 947 968 970 999
// 966 959 957 945 956 964 973 947 991 946 966 985 966 980 986 962 967 958 959 999
// 941 964 956 979 959 983 992 937 999 948 976 961 950 996 987 944 968 949 961 968
// 982 975 932 950 960 969 955 980 977 944 956 944 920 943 999 940 947 938 937 951
// 977 970 935 953 960 965 980 957 991 943 938 956 945 971 999 936 939 952 948 944
// 993 969 987 951 988 979 999 966 983 957 948 971 971 956 990 958 967 955 979 983
// 969 935 955 952 936 961 979 943 975 954 938 957 967 959 950 977 940 999 956 964
// 964 953 975 956 923 972 978 952 974 962 939 946 952 947 952 984 944 999 947 974
// 957 966 969 951 950 977 984 935 940 954 962 956 957 953 952 965 943 999 957 975
// 974 984 982 953 947 968 999 953 940 956 972 946 973 954 948 958 953 987 970 961
// 976 980 976 948 946 955 999 963 952 962 962 941 955 946 926 944 952 951 957 944
// 988 975 951 938 957 972 999 963 973 963 968 948 958 948 945 954 963 943 945 959
// 999 977 958 930 955 957 993 954 983 955 952 945 953 947 923 963 945 939 972 953
// 999 970 928 975 945 938 983 956 954 933 939 938 940 921 923 958 935 933 962 938
// 999 970 915 961 935 923 993 932 929 926 935 925 927 931 917 947 938 929 959 921
// 999 949 953 933 929 952 984 922 935 943 931 942 932 951 934 957 935 926 947 920
// 999 957 948 937 943 951 992 926 927 938 930 934 943 953 946 945 938 935 953 941
// 999 958 962 932 934 938 989 935 930 921 927 935 932 957 926 945 935 921 949 932
// 989 962 950 942 927 941 999 943 926 936 931 930 926 959 923 943 931 928 951 945
// 999 960 952 934 933 935 995 939 928 938 924 929 971 946 927 950 932 933 943 947
// Total bytes=1001793961, ranges=1898
}
func Example_rebalancing() {
// Model a set of stores in a cluster,
// randomly adding / removing stores and adding bytes.
g := gossip.New(nil, 0, nil)
stopper := stop.NewStopper()
defer stopper.Stop()
sp := NewStorePool(g, TestTimeUntilStoreDeadOff, stopper)
alloc := MakeAllocator(sp, RebalancingOptions{AllowRebalance: true, Deterministic: true})
alloc.randGen = rand.New(rand.NewSource(0))
var wg sync.WaitGroup
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ []byte) { wg.Done() })
const generations = 100
const nodes = 20
// Initialize testStores.
var testStores [nodes]testStore
for i := 0; i < len(testStores); i++ {
testStores[i].StoreID = roachpb.StoreID(i)
testStores[i].Node = roachpb.NodeDescriptor{NodeID: roachpb.NodeID(i)}
testStores[i].Capacity = roachpb.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30}
}
// Initialize the cluster with a single range.
testStores[0].add(alloc.randGen.Int63n(1 << 20))
for i := 0; i < generations; i++ {
// First loop through test stores and add data.
wg.Add(len(testStores))
for j := 0; j < len(testStores); j++ {
// Add a pretend range to the testStore if there's already one.
if testStores[j].Capacity.RangeCount > 0 {
testStores[j].add(alloc.randGen.Int63n(1 << 20))
}
key := gossip.MakeStoreKey(roachpb.StoreID(j))
if err := g.AddInfoProto(key, &testStores[j].StoreDescriptor, 0); err != nil {
panic(err)
}
}
wg.Wait()
// Next loop through test stores and maybe rebalance.
for j := 0; j < len(testStores); j++ {
ts := &testStores[j]
if alloc.ShouldRebalance(ts.StoreID) {
target := alloc.RebalanceTarget(ts.StoreID, roachpb.Attributes{}, []roachpb.ReplicaDescriptor{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}})
if target != nil {
testStores[j].rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20))
}
}
}
// Output store capacities as hexidecimal 2-character values.
if i%(generations/50) == 0 {
var maxBytes int64
for j := 0; j < len(testStores); j++ {
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
if bytes > maxBytes {
maxBytes = bytes
}
}
if maxBytes > 0 {
for j := 0; j < len(testStores); j++ {
endStr := " "
if j == len(testStores)-1 {
endStr = ""
}
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr)
}
fmt.Printf("\n")
}
}
}
var totBytes int64
var totRanges int32
for i := 0; i < len(testStores); i++ {
totBytes += testStores[i].Capacity.Capacity - testStores[i].Capacity.Available
totRanges += testStores[i].Capacity.RangeCount
}
fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges)
// Output:
// 138 000 000 000 000 000 000 999 000 000 000 000 000 000 000 000 000 000 000 000
// 922 319 000 000 000 239 000 999 000 000 000 000 000 214 073 000 000 000 190 000
// 999 505 480 000 634 352 421 644 212 331 396 144 000 242 419 275 000 000 727 028
// 999 678 908 705 350 558 549 714 651 824 895 694 000 373 610 490 372 106 492 796
// 932 701 763 999 660 706 571 702 787 945 848 678 062 692 762 413 603 252 513 882
// 937 656 875 984 734 717 676 685 910 895 847 841 349 754 864 463 722 377 655 999
// 885 701 805 999 647 744 802 659 778 834 830 725 569 761 922 587 684 458 693 935
// 813 650 709 931 583 733 843 619 793 881 768 658 565 713 956 598 733 594 656 999
// 873 727 721 999 544 812 848 666 817 943 831 658 556 769 927 554 799 733 670 869
// 937 765 827 999 543 875 907 670 929 997 913 768 621 853 922 618 878 832 733 937
// 902 819 744 988 547 904 922 688 879 999 812 710 554 789 890 591 808 865 658 932
// 870 873 846 997 596 937 899 765 864 969 855 751 577 824 951 579 858 908 653 999
// 880 833 856 999 640 918 932 774 920 930 869 739 686 784 853 553 885 941 685 986
// 874 797 808 999 645 925 928 781 920 956 859 762 678 761 819 627 899 941 725 959
// 886 801 835 999 638 984 927 825 968 958 860 760 813 716 800 638 908 908 798 945
// 860 840 836 973 634 999 944 834 977 923 848 769 846 728 836 605 865 915 781 896
// 859 864 891 993 633 961 999 863 951 885 857 791 864 731 871 656 868 920 790 879
// 866 845 896 999 688 966 998 904 942 864 861 815 867 756 879 704 919 940 804 888
// 825 850 876 983 712 945 999 885 943 870 854 838 848 771 825 701 939 940 809 885
// 821 872 915 999 711 927 968 928 963 898 846 865 863 814 858 719 935 951 818 877
// 829 868 940 999 729 919 938 911 957 905 846 872 860 844 872 724 920 941 844 832
// 825 848 901 999 736 882 911 926 937 935 876 901 824 870 892 714 902 927 844 846
// 837 861 921 999 872 890 875 911 894 939 868 921 871 894 887 740 905 948 881 879
// 876 879 956 999 893 889 875 910 910 953 880 898 900 895 905 736 920 965 918 916
// 921 897 909 999 924 907 895 936 955 974 901 902 933 937 929 763 909 997 914 944
// 930 882 892 995 925 910 907 942 911 952 915 922 936 911 944 819 891 999 906 953
// 904 867 889 989 913 890 877 932 931 937 936 927 939 915 936 843 901 999 937 915
// 936 916 872 937 920 901 900 917 928 972 949 936 917 923 934 897 896 999 913 907
// 979 963 909 954 923 950 953 942 969 999 975 970 925 942 971 948 933 993 958 952
// 978 950 903 942 905 949 937 927 948 991 980 948 921 929 932 946 920 999 942 937
// 962 958 937 947 910 967 949 934 945 983 966 951 917 918 957 965 923 999 961 957
// 972 953 942 921 891 955 949 921 964 977 969 951 927 930 953 953 928 999 950 936
// 965 978 953 928 896 963 964 928 983 986 948 972 963 921 953 970 928 999 937 941
// 961 964 936 930 930 969 964 930 964 973 965 983 974 928 958 969 949 999 941 961
// 977 963 931 955 959 979 970 942 941 965 963 980 951 948 966 971 954 999 962 950
// 988 966 932 960 951 973 978 942 953 964 967 989 941 959 986 956 950 999 962 954
// 989 950 955 951 940 985 964 958 941 982 970 981 946 968 999 965 959 984 979 939
// 976 959 962 956 959 975 955 974 966 988 980 991 956 967 992 953 961 999 959 925
// 983 979 962 973 970 983 962 970 997 999 987 997 970 969 997 973 972 996 981 940
// 980 959 958 968 954 983 952 966 973 974 978 981 975 947 995 977 956 999 972 932
// 960 973 955 982 956 970 944 947 979 978 985 977 962 939 999 986 945 995 965 922
// 956 983 941 982 957 958 945 950 985 973 987 986 944 947 984 999 948 959 961 914
// 970 972 946 968 968 978 951 945 980 973 988 981 956 957 967 999 955 945 966 913
// 967 976 943 974 973 969 951 948 994 963 986 984 949 958 984 999 951 945 980 926
// 960 955 933 954 973 964 943 929 973 943 988 991 948 959 981 999 950 951 978 927
// 954 948 912 953 960 968 937 931 977 936 979 978 945 950 970 999 945 949 971 925
// 953 965 922 947 963 966 933 919 992 943 973 991 933 953 975 999 949 947 970 907
// 951 983 923 958 979 972 945 936 999 941 982 987 946 960 973 975 959 940 970 925
// 954 982 909 948 973 967 945 938 999 950 994 976 957 960 970 968 964 952 966 932
// 956 964 896 944 963 953 924 940 999 936 981 961 939 956 965 951 937 937 959 908
// Total bytes=1042395713, ranges=1919
}
func Example_rebalancing() {
stopper := stop.NewStopper()
defer stopper.Stop()
// Model a set of stores in a cluster,
// randomly adding / removing stores and adding bytes.
rpcContext := rpc.NewContext(&base.Context{Insecure: true}, nil, stopper)
server := rpc.NewServer(rpcContext) // never started
g := gossip.New(context.Background(), rpcContext, server, nil, stopper, metric.NewRegistry())
// Have to call g.SetNodeID before call g.AddInfo
g.SetNodeID(roachpb.NodeID(1))
sp := NewStorePool(
g,
hlc.NewClock(hlc.UnixNano),
nil,
/* reservationsEnabled */ true,
TestTimeUntilStoreDeadOff,
stopper,
)
alloc := MakeAllocator(sp, AllocatorOptions{AllowRebalance: true, Deterministic: true})
var wg sync.WaitGroup
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) { wg.Done() })
const generations = 100
const nodes = 20
// Initialize testStores.
var testStores [nodes]testStore
for i := 0; i < len(testStores); i++ {
testStores[i].StoreID = roachpb.StoreID(i)
testStores[i].Node = roachpb.NodeDescriptor{NodeID: roachpb.NodeID(i)}
testStores[i].Capacity = roachpb.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30}
}
// Initialize the cluster with a single range.
testStores[0].add(alloc.randGen.Int63n(1 << 20))
for i := 0; i < generations; i++ {
// First loop through test stores and add data.
wg.Add(len(testStores))
for j := 0; j < len(testStores); j++ {
// Add a pretend range to the testStore if there's already one.
if testStores[j].Capacity.RangeCount > 0 {
testStores[j].add(alloc.randGen.Int63n(1 << 20))
}
if err := g.AddInfoProto(gossip.MakeStoreKey(roachpb.StoreID(j)), &testStores[j].StoreDescriptor, 0); err != nil {
panic(err)
}
}
wg.Wait()
// Next loop through test stores and maybe rebalance.
for j := 0; j < len(testStores); j++ {
ts := &testStores[j]
target := alloc.RebalanceTarget(
roachpb.Attributes{},
[]roachpb.ReplicaDescriptor{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}},
-1)
if target != nil {
testStores[j].rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20))
}
}
// Output store capacities as hexadecimal 2-character values.
if i%(generations/50) == 0 {
var maxBytes int64
for j := 0; j < len(testStores); j++ {
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
if bytes > maxBytes {
maxBytes = bytes
}
}
if maxBytes > 0 {
for j := 0; j < len(testStores); j++ {
endStr := " "
if j == len(testStores)-1 {
endStr = ""
}
bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr)
}
fmt.Printf("\n")
}
}
}
var totBytes int64
var totRanges int32
for i := 0; i < len(testStores); i++ {
totBytes += testStores[i].Capacity.Capacity - testStores[i].Capacity.Available
totRanges += testStores[i].Capacity.RangeCount
}
fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges)
// Output:
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000
// 999 000 000 000 014 000 000 118 000 000 000 000 111 000 000 000 000 000 000 000
// 999 113 095 000 073 064 000 221 003 000 020 178 182 000 057 000 027 000 055 000
// 999 398 222 000 299 366 000 525 239 135 263 385 424 261 261 000 260 194 207 322
// 999 307 170 335 380 357 336 539 233 373 218 444 539 336 258 479 267 352 384 387
// 999 558 318 587 623 602 453 719 409 506 414 617 638 411 359 486 502 507 454 673
// 999 588 404 854 616 642 559 705 482 622 505 554 673 489 410 390 524 607 535 671
// 999 651 498 922 668 696 612 809 619 691 682 674 682 584 533 449 619 724 646 702
// 999 710 505 832 645 732 719 867 605 794 743 693 717 645 602 503 683 733 686 776
// 999 773 688 810 658 761 812 957 663 875 856 797 871 670 733 602 839 781 736 909
// 959 778 750 879 685 797 786 999 751 944 870 786 882 670 828 611 880 817 714 883
// 946 843 781 892 726 887 876 993 717 999 940 802 879 672 842 634 862 818 736 906
// 924 826 754 859 742 878 836 927 721 999 893 762 874 672 882 684 918 818 745 897
// 910 872 789 858 752 878 824 976 715 999 860 739 848 684 890 699 968 846 751 833
// 938 892 789 879 754 916 861 997 774 983 887 805 827 690 912 751 999 927 800 893
// 895 887 792 845 784 920 800 999 770 961 890 747 871 701 907 733 970 893 811 858
// 887 843 742 839 792 938 826 999 778 971 859 792 857 731 889 777 979 900 779 833
// 891 861 802 819 802 966 826 999 776 946 843 792 836 769 914 792 968 879 775 874
// 923 840 830 842 778 969 820 999 791 950 843 820 838 767 893 794 995 915 789 885
// 932 816 783 830 805 926 783 999 790 977 824 856 838 789 866 787 992 892 760 896
// 917 799 781 813 800 901 759 999 776 983 795 861 813 799 852 776 944 891 739 883
// 895 759 757 827 799 894 741 999 772 955 779 864 823 812 835 785 956 882 746 865
// 906 762 773 867 848 874 747 999 763 992 766 866 831 812 839 820 973 906 765 885
// 915 795 781 884 854 899 782 983 756 999 744 890 840 791 848 806 992 934 774 904
// 935 768 813 893 859 881 788 948 758 999 748 892 828 803 857 834 989 940 798 900
// 953 752 816 919 852 882 806 966 771 976 733 877 804 802 854 822 999 957 800 898
// 909 732 804 882 874 885 814 956 758 937 703 877 805 783 849 833 999 955 796 903
// 885 744 788 859 851 881 802 929 732 905 702 843 801 774 847 810 999 936 778 880
// 856 741 790 827 842 897 771 922 732 873 719 849 771 789 845 828 999 914 764 859
// 880 787 825 841 867 941 782 962 752 918 749 886 797 819 899 862 999 935 792 891
// 902 829 841 857 903 979 786 979 760 935 767 903 816 839 907 880 999 963 827 927
// 873 809 831 837 906 964 786 952 772 928 780 904 810 817 914 878 999 974 827 914
// 879 810 855 843 936 977 806 956 799 930 801 931 823 835 928 895 997 999 864 935
// 885 806 858 825 921 971 791 965 784 930 809 936 813 829 904 893 999 974 858 902
// 865 776 855 811 903 966 771 958 770 906 809 923 810 825 896 901 999 964 841 895
// 880 789 876 816 918 987 772 972 776 912 814 935 836 833 913 901 999 978 863 903
// 866 779 861 824 926 986 773 958 776 920 810 936 836 855 894 899 999 989 859 904
// 880 798 862 826 910 997 795 948 767 910 798 923 838 835 872 911 999 975 856 894
// 885 785 845 807 906 973 783 943 782 918 789 920 832 838 861 894 999 965 849 877
// 889 793 855 802 918 985 786 948 793 920 800 941 818 849 846 899 999 982 851 886
// 866 796 854 801 911 969 782 958 791 907 788 940 800 844 843 890 999 977 851 873
// 849 794 855 815 912 970 790 942 792 898 789 938 794 850 843 884 999 964 854 886
// 856 806 867 837 930 980 787 944 789 903 804 947 800 863 840 891 999 977 860 874
// 847 796 852 849 925 980 777 948 786 905 792 922 798 853 835 887 999 968 868 866
// 851 801 866 846 936 999 795 945 774 909 793 931 794 860 846 908 985 976 882 854
// 861 815 861 845 934 999 808 958 784 913 780 924 800 860 844 912 986 974 897 844
// Total bytes=941960698, ranges=1750
}