func logServerLoad(server *TunnelServer) { // golang runtime stats var memStats runtime.MemStats runtime.ReadMemStats(&memStats) fields := LogFields{ "event_name": "server_load", "build_rev": common.GetBuildInfo().BuildRev, "host_id": server.sshServer.support.Config.HostID, "num_goroutine": runtime.NumGoroutine(), "mem_stats": map[string]interface{}{ "alloc": memStats.Alloc, "total_alloc": memStats.TotalAlloc, "sys": memStats.Sys, "pause_total_ns": memStats.PauseTotalNs, "pause_ns": memStats.PauseNs, "num_gc": memStats.NumGC, "gc_cpu_fraction": memStats.GCCPUFraction, }, } // tunnel server stats fields["establish_tunnels"] = server.GetEstablishTunnels() for tunnelProtocol, stats := range server.GetLoadStats() { fields[tunnelProtocol] = stats } log.LogRawFieldsWithTimestamp(fields) }
// NoticeBuildInfo reports build version info. func NoticeBuildInfo() { outputNotice("BuildInfo", 0, "buildInfo", common.GetBuildInfo()) }
// RunServices initializes support functions including logging and GeoIP services; // and then starts the server components and runs them until os.Interrupt or // os.Kill signals are received. The config determines which components are run. func RunServices(configJSON []byte) error { rand.Seed(int64(time.Now().Nanosecond())) config, err := LoadConfig(configJSON) if err != nil { log.WithContextFields(LogFields{"error": err}).Error("load config failed") return common.ContextError(err) } err = InitLogging(config) if err != nil { log.WithContextFields(LogFields{"error": err}).Error("init logging failed") return common.ContextError(err) } supportServices, err := NewSupportServices(config) if err != nil { log.WithContextFields(LogFields{"error": err}).Error("init support services failed") return common.ContextError(err) } log.WithContextFields(*common.GetBuildInfo().ToMap()).Info("startup") waitGroup := new(sync.WaitGroup) shutdownBroadcast := make(chan struct{}) errors := make(chan error) tunnelServer, err := NewTunnelServer(supportServices, shutdownBroadcast) if err != nil { log.WithContextFields(LogFields{"error": err}).Error("init tunnel server failed") return common.ContextError(err) } supportServices.TunnelServer = tunnelServer if config.RunLoadMonitor() { waitGroup.Add(1) go func() { waitGroup.Done() ticker := time.NewTicker(time.Duration(config.LoadMonitorPeriodSeconds) * time.Second) defer ticker.Stop() for { select { case <-shutdownBroadcast: return case <-ticker.C: logServerLoad(tunnelServer) } } }() } if config.RunWebServer() { waitGroup.Add(1) go func() { defer waitGroup.Done() err := RunWebServer(supportServices, shutdownBroadcast) select { case errors <- err: default: } }() } // The tunnel server is always run; it launches multiple // listeners, depending on which tunnel protocols are enabled. waitGroup.Add(1) go func() { defer waitGroup.Done() err := tunnelServer.Run() select { case errors <- err: default: } }() // Shutdown doesn't wait for the outputProcessProfiles goroutine // to complete, as it may be sleeping while running a "block" or // CPU profile. signalProcessProfiles := make(chan struct{}, 1) go func() { for { select { case <-signalProcessProfiles: outputProcessProfiles(supportServices.Config) case <-shutdownBroadcast: return } } }() // An OS signal triggers an orderly shutdown systemStopSignal := make(chan os.Signal, 1) signal.Notify(systemStopSignal, os.Interrupt, os.Kill, syscall.SIGTERM) // SIGUSR1 triggers a reload of support services reloadSupportServicesSignal := make(chan os.Signal, 1) signal.Notify(reloadSupportServicesSignal, syscall.SIGUSR1) // SIGUSR2 triggers an immediate load log and optional process profile output logServerLoadSignal := make(chan os.Signal, 1) signal.Notify(logServerLoadSignal, syscall.SIGUSR2) // SIGTSTP triggers tunnelServer to stop establishing new tunnels stopEstablishingTunnelsSignal := make(chan os.Signal, 1) signal.Notify(stopEstablishingTunnelsSignal, syscall.SIGTSTP) // SIGCONT triggers tunnelServer to resume establishing new tunnels resumeEstablishingTunnelsSignal := make(chan os.Signal, 1) signal.Notify(resumeEstablishingTunnelsSignal, syscall.SIGCONT) err = nil loop: for { select { case <-stopEstablishingTunnelsSignal: tunnelServer.SetEstablishTunnels(false) case <-resumeEstablishingTunnelsSignal: tunnelServer.SetEstablishTunnels(true) case <-reloadSupportServicesSignal: supportServices.Reload() // Reset traffic rules for established clients to reflect reloaded config // TODO: only update when traffic rules config has changed tunnelServer.ResetAllClientTrafficRules() case <-logServerLoadSignal: // Signal profiles writes first to ensure some diagnostics are // available in case logServerLoad hangs (which has happened // in the past due to a deadlock bug). select { case signalProcessProfiles <- *new(struct{}): default: } logServerLoad(tunnelServer) case <-systemStopSignal: log.WithContext().Info("shutdown by system") break loop case err = <-errors: log.WithContextFields(LogFields{"error": err}).Error("service failed") break loop } } close(shutdownBroadcast) waitGroup.Wait() return err }
// getRequestLogFields makes LogFields to log the API event following // the legacy psi_web and current ELK naming conventions. func getRequestLogFields( support *SupportServices, eventName string, geoIPData GeoIPData, params requestJSONObject, expectedParams []requestParamSpec) LogFields { logFields := make(LogFields) logFields["event_name"] = eventName logFields["host_id"] = support.Config.HostID logFields["build_rev"] = common.GetBuildInfo().BuildRev // In psi_web, the space replacement was done to accommodate space // delimited logging, which is no longer required; we retain the // transformation so that stats aggregation isn't impacted. logFields["client_region"] = strings.Replace(geoIPData.Country, " ", "_", -1) logFields["client_city"] = strings.Replace(geoIPData.City, " ", "_", -1) logFields["client_isp"] = strings.Replace(geoIPData.ISP, " ", "_", -1) if params == nil { return logFields } for _, expectedParam := range expectedParams { if expectedParam.flags&requestParamNotLogged != 0 { continue } value := params[expectedParam.name] if value == nil { // Special case: older clients don't send this value, // so log a default. if expectedParam.name == "tunnel_whole_device" { value = "0" } else { // Skip omitted, optional params continue } } switch v := value.(type) { case string: strValue := v // Special cases: // - Number fields are encoded as integer types. // - For ELK performance we record these domain-or-IP // fields as one of two different values based on type; // we also omit port from host:port fields for now. switch expectedParam.name { case "client_version": intValue, _ := strconv.Atoi(strValue) logFields[expectedParam.name] = intValue case "meek_dial_address": host, _, _ := net.SplitHostPort(strValue) if isIPAddress(support, host) { logFields["meek_dial_ip_address"] = host } else { logFields["meek_dial_domain"] = host } case "meek_host_header": host, _, _ := net.SplitHostPort(strValue) logFields[expectedParam.name] = host case "upstream_proxy_type": // Submitted value could be e.g., "SOCKS5" or "socks5"; log lowercase logFields[expectedParam.name] = strings.ToLower(strValue) default: logFields[expectedParam.name] = strValue } case []interface{}: // Note: actually validated as an array of strings logFields[expectedParam.name] = v default: // This type assertion should be checked already in // validateRequestParams, so failure is unexpected. continue } } return logFields }