Example #1
0
func main() {
	log.Printf("Quiver version %s (built %s, %s).\n\n", version, buildTime, runtime.Version())
	t := time.Now()

	graphite := report.Flag()
	args := readSettings()

	stats := report.NewRecorder().
		EnableGCInfoCollection().
		MaybeReportTo(graphite).
		RegisterHttp().
		SetAsDefault()

	hostname, err := os.Hostname()
	if err != nil {
		hostname = "localhost"
	}

	registrations := new(Registrations)

	if Settings.discoveryPath != "" && !Settings.downloadOnly {
		registrations.Connect()
		defer registrations.Close()
	}

	configs := getCollectionConfig(args)

	log.Println("Loading collections...")

	cs, err := hfile.LoadCollections(configs, Settings.cachePath, Settings.downloadOnly, stats)

	if err != nil {
		log.Fatal(err)
	}

	if Settings.downloadOnly {
		stats.FlushNow()
		return
	}

	if Settings.bloom > 0 {
		beforeBloom := time.Now()
		for _, c := range cs.Collections {
			log.Println("Calculating bloom filter for", c.Name)
			c.CalculateBloom(float64(Settings.bloom) / 100)
		}
		stats.TimeSince("startup.bloom", beforeBloom)
	}

	log.Printf("Serving on http://%s:%d/ \n", hostname, Settings.port)

	http.Handle("/rpc/HFileService", WrapHttpRpcHandler(cs, stats))

	admin := adminz.New()
	admin.KillfilePaths(adminz.Killfiles(Settings.port))

	admin.Servicez(func() interface{} {
		return struct {
			Collections    map[string]*hfile.Reader `json:"collections"`
			Impl           string                   `json:"implementation"`
			QuiverVersion  string                   `json:"quiver_version"`
			PackageVersion string                   `json:"package_version"`
		}{
			cs.Collections,
			"quiver",
			version,
			Settings.packageVersion,
		}
	})

	admin.OnPause(registrations.Leave)
	admin.OnResume(func() {
		if Settings.discoveryPath != "" {
			registrations.Join(hostname, Settings.discoveryPath, configs, 0)
		}
	})

	http.HandleFunc("/hfilez", admin.ServicezHandler)
	http.HandleFunc("/", admin.ServicezHandler)

	http.HandleFunc("/debug/bloom/enable", func(w http.ResponseWriter, r *http.Request) {
		for _, c := range cs.Collections {
			c.EnableBloom()
		}
	})

	http.HandleFunc("/debug/bloom/disable", func(w http.ResponseWriter, r *http.Request) {
		for _, c := range cs.Collections {
			c.DisableBloom()
		}
	})

	http.HandleFunc("/debug/bloom/calc", func(w http.ResponseWriter, r *http.Request) {
		if falsePos, err := strconv.Atoi(r.URL.Query().Get("err")); err != nil {
			http.Error(w, err.Error(), 400)
		} else if falsePos > 99 || falsePos < 1 {
			http.Error(w, "`err` param must be a false pos rate between 0 and 100", 400)
		} else {
			admin.Pause()
			defer admin.Resume()
			for _, c := range cs.Collections {
				fmt.Fprintln(w, "Recalculating bloom for", c.Name)
				c.CalculateBloom(float64(falsePos) / 100)
			}
		}
	})

	runtime.GC()
	stats.FlushNow()

	admin.Start()
	stats.TimeSince("startup.total", t)

	if Settings.rpcPort > 0 {
		s, err := NewTRpcServer(fmt.Sprintf(":%d", Settings.rpcPort), WrapProcessor(cs, stats), thrift.NewTBinaryProtocolFactory(true, true))
		if err != nil {
			log.Fatalln("Could not open RPC port", Settings.rpcPort, err)
		} else {
			if err := s.Listen(); err != nil {
				log.Fatalln("Failed to listen on RPC port", err)
			}
			go func() {
				log.Fatalln(s.Serve())
			}()
			log.Println("Listening for raw RPC on", Settings.rpcPort)
		}

	}

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
Example #2
0
func main() {
	orig := flag.String("server", "localhost:9999", "URL of hfile server")
	rawDiff := flag.String("diff", "", "URL of second hfile server to compare")
	collection := flag.String("collection", "", "name of collection")
	graphite := report.Flag()
	workers := flag.Int("workers", 8, "worker pool size")
	flag.StringVar(&zk, "zk", "", "zookeeper host")

	qps := flag.Int("qps", 100, "qps to attempt")
	maxQps := flag.Bool("max-qps", false, "Each workers sends a query as soon as the previous response is processed")

	minKeys := flag.Int("keys-min", 10, "min number of keys per request")
	maxKeys := flag.Int("keys-max", 5000, "max number of keys per request")
	spreadKeys := flag.Float64("keys-spread", 10, "coefficient for exponential distribution of key count")
	printSpread := flag.Bool("print-spread", false, "print distribution of key count (over 100000 requests)")

	sample := flag.Int64("sampleSize", 1000, "number of random keys to use")

	mixPrefix := flag.Int("mix-prefix", 10, "getPrefixes traffic mix % (un-alloc is getSingle)")
	mixIter := flag.Int("mix-iterator", 10, "getPrefixes traffic mix % (un-alloc is getSingle)")
	mixMulti := flag.Int("mix-multi", 20, "getPrefixes traffic mix % (un-alloc is getSingle)")

	flag.Parse()

	r := report.NewRecorder().
		MaybeReportTo(graphite).
		SetAsDefault()

	rttName := "rtt"
	server, name, conn := hfileUrlAndName(*orig)
	if conn != nil {
		defer conn.Close()
	}

	if collection == nil || len(*collection) < 1 {
		fmt.Println("--collection is required")
		c := GetQuiverClient(server)
		r := &gen.InfoRequest{}

		if resp, err := c.GetInfo(r); err != nil {
			fmt.Println("tried to fetch possible collections but got an error:", err)
		} else {
			fmt.Println("possible --collection options:")
			for _, v := range resp {
				fmt.Println("\t", v.GetName())
			}
		}
		os.Exit(1)
	}

	diffing := false
	diffRtt := ""
	diffName := ""
	diff := func() string { return "" }

	if rawDiff != nil && len(*rawDiff) > 0 {
		diffing = true
		diff, diffName, conn = hfileUrlAndName(*rawDiff)
		if conn != nil {
			defer conn.Close()
		}
		diffRtt = "rtt." + diffName
		rttName = "rtt." + name
	}

	l := &Load{
		collection:       *collection,
		sample:           sample,
		server:           server,
		diffing:          diffing,
		diff:             diff,
		work:             make(chan bool, (*workers)),
		dropped:          r.GetMeter("dropped"),
		queueSize:        r.GetGuage("queue"),
		rtt:              rttName,
		diffRtt:          diffRtt,
		mixPrefix:        int32(*mixPrefix),
		mixIterator:      int32(*mixPrefix + *mixIter),
		mixMulti:         int32(*mixPrefix + *mixIter + *mixMulti),
		keysPerReqMin:    float64(*minKeys),
		keysPerReqMax:    float64(*maxKeys),
		keysPerReqSpread: *spreadKeys,
	}

	if *printSpread {
		fmt.Println("Key count distribtion:")
		l.printKeySpread()
	}

	if err := l.setKeys(); err != nil {
		fmt.Println("Failed to fetch testing keys:", err)
		os.Exit(1)
	}

	if *maxQps {
		fmt.Printf("Sending max qps to %s (%s), drawing from %d random keys...\n", name, server(), len(l.keys))
	} else {
		fmt.Printf("Sending %dqps to %s (%s), drawing from %d random keys...\n", *qps, name, server(), len(l.keys))
		go l.generator(*qps)
	}
	if l.diffing {
		fmt.Printf("Diffing against %s (%s)\n", diffName, l.diff())
	}

	l.startWorkers(*workers, *maxQps)

	reader := bufio.NewReader(os.Stdin)

	for {
		fmt.Print("Press enter for stats summary.\n")
		reader.ReadString('\n')
		l.PrintSummary()
	}
}