func main() { log.Printf("Quiver version %s (built %s, %s).\n\n", version, buildTime, runtime.Version()) t := time.Now() graphite := report.Flag() args := readSettings() stats := report.NewRecorder(). EnableGCInfoCollection(). MaybeReportTo(graphite). RegisterHttp(). SetAsDefault() hostname, err := os.Hostname() if err != nil { hostname = "localhost" } registrations := new(Registrations) if Settings.discoveryPath != "" && !Settings.downloadOnly { registrations.Connect() defer registrations.Close() } configs := getCollectionConfig(args) log.Println("Loading collections...") cs, err := hfile.LoadCollections(configs, Settings.cachePath, Settings.downloadOnly, stats) if err != nil { log.Fatal(err) } if Settings.downloadOnly { stats.FlushNow() return } if Settings.bloom > 0 { beforeBloom := time.Now() for _, c := range cs.Collections { log.Println("Calculating bloom filter for", c.Name) c.CalculateBloom(float64(Settings.bloom) / 100) } stats.TimeSince("startup.bloom", beforeBloom) } log.Printf("Serving on http://%s:%d/ \n", hostname, Settings.port) http.Handle("/rpc/HFileService", WrapHttpRpcHandler(cs, stats)) admin := adminz.New() admin.KillfilePaths(adminz.Killfiles(Settings.port)) admin.Servicez(func() interface{} { return struct { Collections map[string]*hfile.Reader `json:"collections"` Impl string `json:"implementation"` QuiverVersion string `json:"quiver_version"` PackageVersion string `json:"package_version"` }{ cs.Collections, "quiver", version, Settings.packageVersion, } }) admin.OnPause(registrations.Leave) admin.OnResume(func() { if Settings.discoveryPath != "" { registrations.Join(hostname, Settings.discoveryPath, configs, 0) } }) http.HandleFunc("/hfilez", admin.ServicezHandler) http.HandleFunc("/", admin.ServicezHandler) http.HandleFunc("/debug/bloom/enable", func(w http.ResponseWriter, r *http.Request) { for _, c := range cs.Collections { c.EnableBloom() } }) http.HandleFunc("/debug/bloom/disable", func(w http.ResponseWriter, r *http.Request) { for _, c := range cs.Collections { c.DisableBloom() } }) http.HandleFunc("/debug/bloom/calc", func(w http.ResponseWriter, r *http.Request) { if falsePos, err := strconv.Atoi(r.URL.Query().Get("err")); err != nil { http.Error(w, err.Error(), 400) } else if falsePos > 99 || falsePos < 1 { http.Error(w, "`err` param must be a false pos rate between 0 and 100", 400) } else { admin.Pause() defer admin.Resume() for _, c := range cs.Collections { fmt.Fprintln(w, "Recalculating bloom for", c.Name) c.CalculateBloom(float64(falsePos) / 100) } } }) runtime.GC() stats.FlushNow() admin.Start() stats.TimeSince("startup.total", t) if Settings.rpcPort > 0 { s, err := NewTRpcServer(fmt.Sprintf(":%d", Settings.rpcPort), WrapProcessor(cs, stats), thrift.NewTBinaryProtocolFactory(true, true)) if err != nil { log.Fatalln("Could not open RPC port", Settings.rpcPort, err) } else { if err := s.Listen(); err != nil { log.Fatalln("Failed to listen on RPC port", err) } go func() { log.Fatalln(s.Serve()) }() log.Println("Listening for raw RPC on", Settings.rpcPort) } } log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil)) }
func main() { orig := flag.String("server", "localhost:9999", "URL of hfile server") rawDiff := flag.String("diff", "", "URL of second hfile server to compare") collection := flag.String("collection", "", "name of collection") graphite := report.Flag() workers := flag.Int("workers", 8, "worker pool size") flag.StringVar(&zk, "zk", "", "zookeeper host") qps := flag.Int("qps", 100, "qps to attempt") maxQps := flag.Bool("max-qps", false, "Each workers sends a query as soon as the previous response is processed") minKeys := flag.Int("keys-min", 10, "min number of keys per request") maxKeys := flag.Int("keys-max", 5000, "max number of keys per request") spreadKeys := flag.Float64("keys-spread", 10, "coefficient for exponential distribution of key count") printSpread := flag.Bool("print-spread", false, "print distribution of key count (over 100000 requests)") sample := flag.Int64("sampleSize", 1000, "number of random keys to use") mixPrefix := flag.Int("mix-prefix", 10, "getPrefixes traffic mix % (un-alloc is getSingle)") mixIter := flag.Int("mix-iterator", 10, "getPrefixes traffic mix % (un-alloc is getSingle)") mixMulti := flag.Int("mix-multi", 20, "getPrefixes traffic mix % (un-alloc is getSingle)") flag.Parse() r := report.NewRecorder(). MaybeReportTo(graphite). SetAsDefault() rttName := "rtt" server, name, conn := hfileUrlAndName(*orig) if conn != nil { defer conn.Close() } if collection == nil || len(*collection) < 1 { fmt.Println("--collection is required") c := GetQuiverClient(server) r := &gen.InfoRequest{} if resp, err := c.GetInfo(r); err != nil { fmt.Println("tried to fetch possible collections but got an error:", err) } else { fmt.Println("possible --collection options:") for _, v := range resp { fmt.Println("\t", v.GetName()) } } os.Exit(1) } diffing := false diffRtt := "" diffName := "" diff := func() string { return "" } if rawDiff != nil && len(*rawDiff) > 0 { diffing = true diff, diffName, conn = hfileUrlAndName(*rawDiff) if conn != nil { defer conn.Close() } diffRtt = "rtt." + diffName rttName = "rtt." + name } l := &Load{ collection: *collection, sample: sample, server: server, diffing: diffing, diff: diff, work: make(chan bool, (*workers)), dropped: r.GetMeter("dropped"), queueSize: r.GetGuage("queue"), rtt: rttName, diffRtt: diffRtt, mixPrefix: int32(*mixPrefix), mixIterator: int32(*mixPrefix + *mixIter), mixMulti: int32(*mixPrefix + *mixIter + *mixMulti), keysPerReqMin: float64(*minKeys), keysPerReqMax: float64(*maxKeys), keysPerReqSpread: *spreadKeys, } if *printSpread { fmt.Println("Key count distribtion:") l.printKeySpread() } if err := l.setKeys(); err != nil { fmt.Println("Failed to fetch testing keys:", err) os.Exit(1) } if *maxQps { fmt.Printf("Sending max qps to %s (%s), drawing from %d random keys...\n", name, server(), len(l.keys)) } else { fmt.Printf("Sending %dqps to %s (%s), drawing from %d random keys...\n", *qps, name, server(), len(l.keys)) go l.generator(*qps) } if l.diffing { fmt.Printf("Diffing against %s (%s)\n", diffName, l.diff()) } l.startWorkers(*workers, *maxQps) reader := bufio.NewReader(os.Stdin) for { fmt.Print("Press enter for stats summary.\n") reader.ReadString('\n') l.PrintSummary() } }