func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
statsdHost := flag.String("statsd", "", "Statsd host to record load test metrics")
flag.Parse()
stats, err := statsd.NewClient(*statsdHost, "hystrix.loadtest.service")
if err != nil {
log.Fatalf("could not initialize statsd client: %v", err)
}
c, err := plugins.InitializeStatsdCollector(&plugins.StatsdCollectorConfig{
StatsdAddr: *statsdHost,
Prefix: "hystrix.loadtest.circuits",
})
if err != nil {
log.Fatalf("could not initialize statsd client: %v", err)
}
metricCollector.Registry.Register(c.NewStatsdCollector)
hystrix.ConfigureCommand("test", hystrix.CommandConfig{
Timeout: 50,
})
go rotateDelay()
http.HandleFunc("/", timedHandler(handle, stats))
log.Print("starting server")
log.Fatal(http.ListenAndServe(":8888", nil))
}
func TestUpstreamTimeout(t *testing.T) {
handler := func(w http.ResponseWriter, req *http.Request) {
time.Sleep(10 * time.Millisecond)
}
server := httptest.NewServer(http.HandlerFunc(handler))
defer server.Close()
upstream := fmt.Sprintf("http://%s", server.Listener.Addr())
url, _ := url.Parse(upstream)
h := NewTokenInfoProxyHandler(url, 0, 0)
w := httptest.NewRecorder()
r, _ := http.NewRequest("GET", "/oauth2/tokeninfo?access_token=foo", nil)
hystrix.ConfigureCommand("proxy", hystrix.CommandConfig{
Timeout: 1,
MaxConcurrentRequests: hystrix.DefaultMaxConcurrent,
RequestVolumeThreshold: hystrix.DefaultVolumeThreshold,
SleepWindow: hystrix.DefaultSleepWindow,
ErrorPercentThreshold: hystrix.DefaultErrorPercentThreshold,
})
h.ServeHTTP(w, r)
if w.Code != http.StatusGatewayTimeout {
t.Errorf("Response code should be 504 Gateway Timeout but was %d %s instead", w.Code, http.StatusText(w.Code))
}
}
func StartWebServer() error {
conf, err := config.GetConfig()
if err != nil {
return err
}
var hystrixTimeout time.Duration
conf.Hystrix.Timeout = strings.TrimSpace(conf.Hystrix.Timeout)
if conf.Hystrix.Timeout != "" {
hystrixTimeout, err = time.ParseDuration(conf.Hystrix.Timeout)
if err != nil || hystrixTimeout < time.Millisecond {
hystrixTimeout = time.Second
log15.Error("Use default time", "module", "hystrix", "timeout", hystrixTimeout)
}
}
hystrix.ConfigureCommand("waitFor", hystrix.CommandConfig{
Timeout: int(int64(hystrixTimeout) / int64(time.Millisecond)), // converted into Millisecond.
MaxConcurrentRequests: conf.Hystrix.MaxConcurrentRequests,
ErrorPercentThreshold: conf.Hystrix.ErrorPercentThreshold,
RequestVolumeThreshold: conf.Hystrix.RequestVolumeThreshold,
SleepWindow: conf.Hystrix.SleepWindow,
})
e := echo.New()
e.Post("/api/v1/tweet", createTweetV1)
e.Get("/api/v1/tweets/:id", getAllTweetForV1)
e.Get("/api/v1/wait/:timeout", waitFor)
e.Get("/api/v1/wait_protected/:timeout", waitForProtected)
e.Static("/", "www/static/")
logsrv := log15.New("pid", os.Getpid(), "addr", conf.Web.Address)
return listenAndServer(logsrv, conf.Web.Address, handlers.LoggingHandler(os.Stdout, handlers.CompressHandler(e.Router())))
}
func (m *Builder) Run() error {
var err error
cfg := nsq.NewConfig()
hostname, err := os.Hostname()
cfg.Set("user_agent", fmt.Sprintf("metric_processor/%s", hostname))
cfg.Set("snappy", true)
cfg.Set("max_in_flight", m.MaxInFlight)
m.consumer, err = nsq.NewConsumer(m.Topic, m.Channel, cfg)
if err != nil {
log.Println(m.Topic, err)
return err
}
hystrix.ConfigureCommand("InsetInfluxdb", hystrix.CommandConfig{
Timeout: 1000,
MaxConcurrentRequests: 1000,
ErrorPercentThreshold: 25,
})
go m.writeLoop()
m.consumer.AddConcurrentHandlers(m, m.MaxInFlight)
err = m.consumer.ConnectToNSQLookupds(m.LookupdAddresses)
if err != nil {
return err
}
return err
}
func (m *LogTask) WriteLoop(exitchan chan int) {
hystrix.ConfigureCommand("NSQWriter", hystrix.CommandConfig{
Timeout: 1000,
MaxConcurrentRequests: 1000,
ErrorPercentThreshold: 25,
})
for {
select {
case <-m.exitChan:
return
case <-exitchan:
return
case msg := <-m.msgChan:
resultChan := make(chan int, 1)
var err error
errChan := hystrix.Go("NSQWriter", func() error {
err = m.Writer.MultiPublish(msg.topic, msg.body)
if err != nil {
return err
}
resultChan <- 1
return nil
}, nil)
select {
case <-resultChan:
case err = <-errChan:
log.Println("writeNSQ Error", err)
}
msg.ResultChan <- err
}
}
}
func (m *MetricDeliver) Run() error {
hostname, err := os.Hostname()
if err != nil {
return err
}
cfg := nsq.NewConfig()
cfg.Set("user_agent", fmt.Sprintf("metric_processor-%s/%s", VERSION, hostname))
cfg.Set("snappy", true)
cfg.Set("max_in_flight", m.MaxInFlight)
m.consumer, err = nsq.NewConsumer(m.MetricTopic, m.MetricChannel, cfg)
if err != nil {
return err
}
m.consumer.AddConcurrentHandlers(m, m.MaxInFlight)
err = m.consumer.ConnectToNSQLookupds(m.LookupdAddresses)
hystrix.ConfigureCommand("InsetInfluxdb", hystrix.CommandConfig{
Timeout: 1000,
MaxConcurrentRequests: 1000,
ErrorPercentThreshold: 25,
})
if err != nil {
return err
}
for i := 0; i < m.MaxInFlight; i++ {
go m.writeLoop()
}
return err
}
func TestHystrix(t *testing.T) {
stdlog.SetOutput(ioutil.Discard)
const (
commandName = "my-endpoint"
errorPercent = 5
maxConcurrent = 1000
)
hystrix.ConfigureCommand(commandName, hystrix.CommandConfig{
ErrorPercentThreshold: errorPercent,
MaxConcurrentRequests: maxConcurrent,
})
var (
breaker = circuitbreaker.Hystrix(commandName)
primeWith = hystrix.DefaultVolumeThreshold * 2
shouldPass = func(n int) bool { return (float64(n) / float64(primeWith+n)) <= (float64(errorPercent-1) / 100.0) }
openCircuitError = hystrix.ErrCircuitOpen.Error()
)
// hystrix-go uses buffered channels to receive reports on request success/failure,
// and so is basically impossible to test deterministically. We have to make sure
// the report buffer is emptied, by injecting a sleep between each invocation.
requestDelay := 5 * time.Millisecond
testFailingEndpoint(t, breaker, primeWith, shouldPass, requestDelay, openCircuitError)
}
func (c Client) FromPlace(place string) (Location, error) {
output := make(chan Location, 1)
hystrix.ConfigureCommand("geocodeFromPlace", hystrix.CommandConfig{
Timeout: c.Timeout,
MaxConcurrentRequests: 100,
ErrorPercentThreshold: 25,
})
errs := hystrix.Go("geocodeFromPlace", func() error {
geoLoc, err := c.GCli.Get(place)
if err != nil {
return err
}
output <- geoLoc
return nil
}, nil)
select {
case out := <-output:
return out, nil
case err := <-errs:
return Location{}, err
}
}
// Creates a KafkaLogger for a given kafka cluster. We identify ourselves with clientId.
func NewKafkaLogger(clientId string, brokers []string) (request_handler.SpadeEdgeLogger, error) {
c, err := sarama.NewClient(clientId, brokers, sarama.NewClientConfig())
if err != nil {
return nil, err
}
config := sarama.NewProducerConfig()
config.Partitioner = sarama.NewRoundRobinPartitioner
config.FlushFrequency = 500 * time.Millisecond
config.FlushMsgCount = 1000
// Might want to try out compression
config.Compression = sarama.CompressionNone
config.AckSuccesses = true
p, err := NewProducer(c, GetTopic(), config)
if err != nil {
return nil, err
}
k := &KafkaLogger{
Producer: p,
}
hystrix.ConfigureCommand(hystrixCommandName, hystrix.CommandConfig{
Timeout: 1000,
MaxConcurrentRequests: hystrixConcurrencyLevel,
ErrorPercentThreshold: 10,
})
return k, nil
}
// ConfigureCommand applies settings for a circuit
func HystrixConfigureCommand(configName string, config hystrix.CommandConfig) {
hystrixMutex.Lock()
defer hystrixMutex.Unlock()
hc := NewHystrixConfig(configName)
hystrix.ConfigureCommand(hc.Name, config)
hystrixConfigs[hc.Name] = hc
}
func main() {
hystrix.ConfigureCommand("call_backend", hystrix.CommandConfig{
Timeout: 1500,
MaxConcurrentRequests: 100,
ErrorPercentThreshold: 25,
})
goji.Get("/*", ping)
goji.Serve()
}
func configureHandler(w http.ResponseWriter, r *http.Request) {
name := r.URL.Query().Get("name")
timeoutStr := r.URL.Query().Get("timeout")
maxConcurrentRequestsStr := r.URL.Query().Get("maxConcurrentRequests")
errorPercentThresholdStr := r.URL.Query().Get("errorPercentThreshold")
if (len(name) == 0) || (len(timeoutStr) == 0) || (len(maxConcurrentRequestsStr) == 0) || (len(errorPercentThresholdStr) == 0) {
w.WriteHeader(http.StatusBadRequest)
w.Write(([]byte)("Missing or empty parameter\n"))
return
}
timeout, err := strconv.Atoi(timeoutStr)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write(([]byte)("timeout parameter not ' int' "))
return
}
maxConcurrentRequests, err := strconv.Atoi(maxConcurrentRequestsStr)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write(([]byte)("maxConcurrentRequests parameter not ' int' "))
return
}
errorPercentThreshold, err := strconv.Atoi(errorPercentThresholdStr)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write(([]byte)("errorPercentThreshold parameter not ' int' "))
return
}
hystrix.ConfigureCommand(name, hystrix.CommandConfig{
Timeout: timeout,
MaxConcurrentRequests: maxConcurrentRequests,
ErrorPercentThreshold: errorPercentThreshold,
})
w.WriteHeader(http.StatusOK)
w.Write(([]byte)("Configuration done.\n"))
}
func TestHystrix(t *testing.T) {
logger := kitlog.NewLogfmtLogger(os.Stderr)
stdlog.SetOutput(kitlog.NewStdlibAdapter(logger))
const (
commandName = "my-endpoint"
errorPercent = 5
maxConcurrent = 1000
)
hystrix.ConfigureCommand(commandName, hystrix.CommandConfig{
ErrorPercentThreshold: errorPercent,
MaxConcurrentRequests: maxConcurrent,
})
var (
breaker = circuitbreaker.Hystrix(commandName)
primeWith = hystrix.DefaultVolumeThreshold * 2
shouldPass = func(n int) bool { return (float64(n) / float64(primeWith+n)) <= (float64(errorPercent-1) / 100.0) }
openCircuitError = hystrix.ErrCircuitOpen.Error()
)
testFailingEndpoint(t, breaker, primeWith, shouldPass, openCircuitError)
}
func (c Client) FromIP(ip string) (Location, error) {
output := make(chan Location, 1)
hystrix.ConfigureCommand("geocodeFromIP", hystrix.CommandConfig{
Timeout: c.Timeout,
MaxConcurrentRequests: 100,
ErrorPercentThreshold: 25,
})
errs := hystrix.Go("geocodeFromIP", func() error {
ipLoc, err := c.IPCli.Get(ip)
if err != nil {
return err
}
combined := Location{
IP: ip,
City: ipLoc.City,
Region: ipLoc.Region,
Latitude: ipLoc.Latitude,
Longitude: ipLoc.Longitude,
}
output <- combined
return nil
}, nil)
select {
case out := <-output:
return out, nil
case err := <-errs:
return Location{}, err
}
}
func HandleCrossCompile(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
log.Printf("[INFO] invalid method: %s", r.Method)
w.WriteHeader(http.StatusMethodNotAllowed)
msg := fmt.Sprintf("Invalid method: %s\n", r.Method)
w.Write([]byte(msg))
return
}
// Handle root request, show project page
if r.URL.Path == "/" {
http.Redirect(w, r, "https://github.com/tcnksm/gox-server", 301)
return
}
// Check correct request comes
repoComponent := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if len(repoComponent) != 2 {
log.Printf("[INFO] faild to parse as repository name: %s", r.URL.Path)
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Invalid request: request must be https://gox-server.herokuapp.com/USER/REPO format\n"))
return
}
// Detect platform from user agent
targetOS, targetArch := guessPlatform(r.UserAgent())
// Set hystrix configuration
hystrix.ConfigureCommand("gox", goxHystrixConfig)
// Run
resultCh := make(chan string, 1)
errCh := hystrix.Go("gox", func() error {
// Get source code from github
if err := goGet(repoComponent[0], repoComponent[1]); err != nil {
return nil
}
// Run gox and generate binary
output, err := gox(repoComponent[0], repoComponent[1], targetOS, targetArch)
if err != nil {
return nil
}
resultCh <- output
return nil
}, nil)
select {
case output := <-resultCh:
log.Printf("[INFO] cross compile is done: %s", output)
w.WriteHeader(http.StatusOK)
http.ServeFile(w, r, output)
case err := <-errCh:
log.Printf("[ERROR] failed to cross compiling: %s", err)
w.WriteHeader(http.StatusServiceUnavailable)
switch err {
case hystrix.ErrTimeout:
w.Write([]byte("Timeout: gox-server can't handle build which takes more than 120s.\n"))
case hystrix.ErrMaxConcurrency:
w.Write([]byte("Too many access: gox-server can't handle more than 100 requests at one time.\n"))
case hystrix.ErrCircuitOpen:
w.Write([]byte("Too many errors: gox-server is unavailable now because of too many errors.\n"))
default:
msg := fmt.Sprintf("Build failed: %s\n", err.Error())
w.Write([]byte(msg))
}
}
}