Ejemplo n.º 1
0
func gossipStores(g *gossip.Gossip, stores []*proto.StoreDescriptor, t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(len(stores))
	g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyCapacityPrefix), func(_ string, _ bool) { wg.Done() })

	for _, s := range stores {
		keyMaxCapacity := gossip.MakeCapacityKey(s.Node.NodeID, s.StoreID)
		// Gossip store descriptor.
		err := g.AddInfo(keyMaxCapacity, *s, 0)
		if err != nil {
			t.Fatal(err)
		}
	}

	// Wait for all gossip callbacks to be invoked.
	wg.Wait()
}
Ejemplo n.º 2
0
// ExampleAllocatorRebalancing models a set of stores in a cluster,
// randomly adding / removing stores and adding bytes.
func ExampleAllocatorRebalancing() {
	g := gossip.New(nil, 0, nil)
	alloc := newAllocator(g)
	alloc.randGen = rand.New(rand.NewSource(0))
	alloc.deterministic = true

	var wg sync.WaitGroup
	g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyCapacityPrefix), func(_ string, _ bool) { wg.Done() })

	const generations = 100
	const nodes = 20

	// Initialize testStores.
	var testStores [nodes]testStore
	for i := 0; i < len(testStores); i++ {
		testStores[i].StoreID = proto.StoreID(i)
		testStores[i].Node = proto.NodeDescriptor{NodeID: proto.NodeID(i)}
		testStores[i].Capacity = proto.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30}
	}
	// Initialize the cluster with a single range.
	testStores[0].Add(alloc.randGen.Int63n(1 << 20))

	for i := 0; i < generations; i++ {
		// First loop through test stores and add data.
		wg.Add(len(testStores))
		for j := 0; j < len(testStores); j++ {
			// Add a pretend range to the testStore if there's already one.
			if testStores[j].Capacity.RangeCount > 0 {
				testStores[j].Add(alloc.randGen.Int63n(1 << 20))
			}
			key := gossip.MakeCapacityKey(proto.NodeID(j), proto.StoreID(j))
			if err := g.AddInfo(key, testStores[j].StoreDescriptor, 0); err != nil {
				panic(err)
			}
		}
		wg.Wait()

		// Next loop through test stores and maybe rebalance.
		for j := 0; j < len(testStores); j++ {
			ts := &testStores[j]
			if alloc.ShouldRebalance(&testStores[j].StoreDescriptor) {
				target := alloc.RebalanceTarget(proto.Attributes{}, []proto.Replica{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}})
				if target != nil {
					testStores[j].Rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20))
				}
			}
		}

		// Output store capacities as hexidecimal 2-character values.
		if i%(generations/50) == 0 {
			var maxBytes int64
			for j := 0; j < len(testStores); j++ {
				bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
				if bytes > maxBytes {
					maxBytes = bytes
				}
			}
			if maxBytes > 0 {
				for j := 0; j < len(testStores); j++ {
					endStr := " "
					if j == len(testStores)-1 {
						endStr = ""
					}
					bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available
					fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr)
				}
				fmt.Printf("\n")
			}
		}
	}

	var totBytes int64
	var totRanges int32
	for i := 0; i < len(testStores); i++ {
		totBytes += testStores[i].Capacity.Capacity - testStores[i].Capacity.Available
		totRanges += testStores[i].Capacity.RangeCount
	}
	fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges)

	// Output:
	// 999 000 000 000 000 000 000 739 000 000 000 000 000 000 000 000 000 000 000 000
	// 999 000 000 000 204 000 000 375 000 000 107 000 000 000 000 000 000 000 000 536
	// 942 000 000 463 140 000 000 646 000 288 288 000 442 000 058 647 000 000 316 999
	// 880 000 412 630 365 745 445 565 122 407 380 570 276 000 271 709 000 718 299 999
	// 925 000 667 600 555 975 704 552 272 491 773 890 584 000 407 974 000 930 476 999
	// 990 967 793 579 493 999 698 453 616 608 777 755 709 425 455 984 483 698 267 931
	// 965 999 869 606 635 908 630 585 567 577 818 870 740 621 550 868 805 790 411 913
	// 953 995 990 624 617 947 562 609 670 658 909 952 835 851 641 958 924 999 526 987
	// 999 923 901 571 687 915 636 636 674 685 831 881 847 820 702 905 897 983 509 981
	// 999 884 809 585 691 826 640 572 748 641 754 887 758 848 643 927 865 897 541 956
	// 999 856 891 594 691 745 602 615 766 663 814 834 719 886 733 925 882 911 593 926
	// 999 890 900 653 707 759 642 697 771 732 851 858 748 869 842 953 903 928 655 923
	// 999 924 909 696 748 797 693 689 806 766 841 902 705 897 874 914 913 916 730 892
	// 999 948 892 704 740 821 685 656 859 772 893 911 690 878 824 935 928 941 741 860
	// 999 948 931 697 770 782 697 666 893 761 944 869 658 902 816 925 923 983 742 831
	// 999 878 901 736 750 737 677 647 869 731 930 825 631 880 775 947 949 930 687 810
	// 999 890 910 764 778 757 709 663 849 777 964 837 672 891 814 978 944 946 721 868
	// 985 895 968 806 791 791 720 694 883 819 999 847 652 888 790 995 950 947 692 843
	// 960 903 956 794 815 779 746 706 891 824 958 830 665 886 757 999 931 969 701 861
	// 999 928 954 805 807 822 764 734 910 829 952 827 678 927 785 980 936 962 677 836
	// 999 903 924 800 769 822 776 730 886 815 935 781 668 890 805 948 929 965 676 837
	// 999 926 935 836 782 836 809 756 897 835 937 781 690 894 804 979 951 978 667 832
	// 999 937 936 875 843 872 854 793 908 873 950 808 714 901 860 981 975 962 693 866
	// 988 957 938 898 922 912 916 886 905 912 964 867 764 915 911 992 999 985 776 896
	// 945 959 922 910 937 913 938 944 957 921 993 916 898 957 928 999 976 997 855 957
	// 980 986 944 956 963 920 966 967 999 966 991 956 981 973 955 998 990 954 994 981
	// 956 985 942 945 950 900 933 949 981 969 946 935 963 951 931 999 936 941 972 963
	// 940 999 964 949 941 974 967 937 970 975 965 951 976 968 949 993 944 949 977 964
	// 926 999 973 932 944 952 933 944 963 965 927 940 964 960 938 995 932 935 968 951
	// 907 999 919 957 941 958 934 935 930 941 940 926 966 933 920 973 937 923 938 946
	// 924 999 914 963 976 945 911 936 929 951 930 930 972 935 941 977 932 960 939 958
	// 942 999 950 961 987 942 928 945 938 941 939 936 985 937 969 985 952 958 957 948
	// 956 999 950 947 943 939 949 934 929 935 940 942 943 957 988 974 933 936 938 951
	// 967 990 950 949 964 952 951 922 943 940 954 956 962 946 982 999 945 949 940 954
	// 970 999 952 959 970 955 957 974 937 965 968 947 950 958 947 993 953 938 958 950
	// 945 964 954 963 965 959 967 961 925 978 954 944 968 937 960 999 947 947 961 960
	// 930 957 938 974 956 944 968 930 944 972 930 946 958 974 940 999 961 945 953 947
	// 966 980 954 989 979 960 969 995 961 986 954 980 980 971 968 999 968 977 979 972
	// 963 953 958 986 990 947 973 955 955 983 974 981 961 964 977 999 984 982 966 964
	// 964 968 975 993 999 955 965 958 972 995 978 981 956 966 981 987 978 976 985 966
	// 967 957 954 999 963 940 968 966 941 966 971 969 957 961 949 940 968 963 988 947
	// 951 939 952 980 937 948 964 970 941 965 979 966 941 940 952 938 973 955 999 934
	// 939 958 941 998 942 951 962 942 962 951 972 978 946 935 958 935 950 947 999 953
	// 959 952 938 999 936 957 961 950 937 954 975 971 958 930 938 930 944 939 978 950
	// 957 943 963 999 947 965 953 937 966 953 978 972 963 937 933 945 944 937 979 952
	// 945 951 956 999 926 948 958 923 947 934 951 961 955 941 949 936 945 929 960 947
	// 956 960 975 999 945 977 956 934 954 943 961 956 956 954 960 954 958 929 969 938
	// 947 966 993 999 944 963 942 939 963 935 952 957 968 947 962 946 962 947 959 942
	// 940 961 999 992 935 946 938 932 968 939 957 938 970 949 964 934 948 957 952 939
	// 944 955 999 978 940 932 937 944 957 936 957 945 958 955 947 933 956 948 947 942
	// Total bytes=1003302292, ranges=1899
}