func main() {
log.Printf("Quiver version %s (built %s, %s).\n\n", version, buildTime, runtime.Version())
t := time.Now()
graphite := report.Flag()
args := readSettings()
stats := report.NewRecorder().
EnableGCInfoCollection().
MaybeReportTo(graphite).
RegisterHttp().
SetAsDefault()
hostname, err := os.Hostname()
if err != nil {
hostname = "localhost"
}
registrations := new(Registrations)
if Settings.discoveryPath != "" && !Settings.downloadOnly {
registrations.Connect()
defer registrations.Close()
}
configs := getCollectionConfig(args)
log.Println("Loading collections...")
cs, err := hfile.LoadCollections(configs, Settings.cachePath, Settings.downloadOnly, stats)
if err != nil {
log.Fatal(err)
}
if Settings.downloadOnly {
stats.FlushNow()
return
}
if Settings.bloom > 0 {
beforeBloom := time.Now()
for _, c := range cs.Collections {
log.Println("Calculating bloom filter for", c.Name)
c.CalculateBloom(float64(Settings.bloom) / 100)
}
stats.TimeSince("startup.bloom", beforeBloom)
}
log.Printf("Serving on http://%s:%d/ \n", hostname, Settings.port)
http.Handle("/rpc/HFileService", WrapHttpRpcHandler(cs, stats))
admin := adminz.New()
admin.KillfilePaths(adminz.Killfiles(Settings.port))
admin.Servicez(func() interface{} {
return struct {
Collections map[string]*hfile.Reader `json:"collections"`
Impl string `json:"implementation"`
QuiverVersion string `json:"quiver_version"`
PackageVersion string `json:"package_version"`
}{
cs.Collections,
"quiver",
version,
Settings.packageVersion,
}
})
admin.OnPause(registrations.Leave)
admin.OnResume(func() {
if Settings.discoveryPath != "" {
registrations.Join(hostname, Settings.discoveryPath, configs, 0)
}
})
http.HandleFunc("/hfilez", admin.ServicezHandler)
http.HandleFunc("/", admin.ServicezHandler)
http.HandleFunc("/debug/bloom/enable", func(w http.ResponseWriter, r *http.Request) {
for _, c := range cs.Collections {
c.EnableBloom()
}
})
http.HandleFunc("/debug/bloom/disable", func(w http.ResponseWriter, r *http.Request) {
for _, c := range cs.Collections {
c.DisableBloom()
}
})
http.HandleFunc("/debug/bloom/calc", func(w http.ResponseWriter, r *http.Request) {
if falsePos, err := strconv.Atoi(r.URL.Query().Get("err")); err != nil {
http.Error(w, err.Error(), 400)
} else if falsePos > 99 || falsePos < 1 {
http.Error(w, "`err` param must be a false pos rate between 0 and 100", 400)
} else {
admin.Pause()
defer admin.Resume()
for _, c := range cs.Collections {
fmt.Fprintln(w, "Recalculating bloom for", c.Name)
c.CalculateBloom(float64(falsePos) / 100)
}
}
})
runtime.GC()
stats.FlushNow()
admin.Start()
stats.TimeSince("startup.total", t)
if Settings.rpcPort > 0 {
s, err := NewTRpcServer(fmt.Sprintf(":%d", Settings.rpcPort), WrapProcessor(cs, stats), thrift.NewTBinaryProtocolFactory(true, true))
if err != nil {
log.Fatalln("Could not open RPC port", Settings.rpcPort, err)
} else {
if err := s.Listen(); err != nil {
log.Fatalln("Failed to listen on RPC port", err)
}
go func() {
log.Fatalln(s.Serve())
}()
log.Println("Listening for raw RPC on", Settings.rpcPort)
}
}
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
func main() {
orig := flag.String("server", "localhost:9999", "URL of hfile server")
rawDiff := flag.String("diff", "", "URL of second hfile server to compare")
collection := flag.String("collection", "", "name of collection")
graphite := report.Flag()
workers := flag.Int("workers", 8, "worker pool size")
flag.StringVar(&zk, "zk", "", "zookeeper host")
qps := flag.Int("qps", 100, "qps to attempt")
maxQps := flag.Bool("max-qps", false, "Each workers sends a query as soon as the previous response is processed")
minKeys := flag.Int("keys-min", 10, "min number of keys per request")
maxKeys := flag.Int("keys-max", 5000, "max number of keys per request")
spreadKeys := flag.Float64("keys-spread", 10, "coefficient for exponential distribution of key count")
printSpread := flag.Bool("print-spread", false, "print distribution of key count (over 100000 requests)")
sample := flag.Int64("sampleSize", 1000, "number of random keys to use")
mixPrefix := flag.Int("mix-prefix", 10, "getPrefixes traffic mix % (un-alloc is getSingle)")
mixIter := flag.Int("mix-iterator", 10, "getPrefixes traffic mix % (un-alloc is getSingle)")
mixMulti := flag.Int("mix-multi", 20, "getPrefixes traffic mix % (un-alloc is getSingle)")
flag.Parse()
r := report.NewRecorder().
MaybeReportTo(graphite).
SetAsDefault()
rttName := "rtt"
server, name, conn := hfileUrlAndName(*orig)
if conn != nil {
defer conn.Close()
}
if collection == nil || len(*collection) < 1 {
fmt.Println("--collection is required")
c := GetQuiverClient(server)
r := &gen.InfoRequest{}
if resp, err := c.GetInfo(r); err != nil {
fmt.Println("tried to fetch possible collections but got an error:", err)
} else {
fmt.Println("possible --collection options:")
for _, v := range resp {
fmt.Println("\t", v.GetName())
}
}
os.Exit(1)
}
diffing := false
diffRtt := ""
diffName := ""
diff := func() string { return "" }
if rawDiff != nil && len(*rawDiff) > 0 {
diffing = true
diff, diffName, conn = hfileUrlAndName(*rawDiff)
if conn != nil {
defer conn.Close()
}
diffRtt = "rtt." + diffName
rttName = "rtt." + name
}
l := &Load{
collection: *collection,
sample: sample,
server: server,
diffing: diffing,
diff: diff,
work: make(chan bool, (*workers)),
dropped: r.GetMeter("dropped"),
queueSize: r.GetGuage("queue"),
rtt: rttName,
diffRtt: diffRtt,
mixPrefix: int32(*mixPrefix),
mixIterator: int32(*mixPrefix + *mixIter),
mixMulti: int32(*mixPrefix + *mixIter + *mixMulti),
keysPerReqMin: float64(*minKeys),
keysPerReqMax: float64(*maxKeys),
keysPerReqSpread: *spreadKeys,
}
if *printSpread {
fmt.Println("Key count distribtion:")
l.printKeySpread()
}
if err := l.setKeys(); err != nil {
fmt.Println("Failed to fetch testing keys:", err)
os.Exit(1)
}
if *maxQps {
fmt.Printf("Sending max qps to %s (%s), drawing from %d random keys...\n", name, server(), len(l.keys))
} else {
fmt.Printf("Sending %dqps to %s (%s), drawing from %d random keys...\n", *qps, name, server(), len(l.keys))
go l.generator(*qps)
}
if l.diffing {
fmt.Printf("Diffing against %s (%s)\n", diffName, l.diff())
}
l.startWorkers(*workers, *maxQps)
reader := bufio.NewReader(os.Stdin)
for {
fmt.Print("Press enter for stats summary.\n")
reader.ReadString('\n')
l.PrintSummary()
}
}