func main() {
var rlim syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Errorf("获取Rlimit报错 %s", err)
os.Exit(1)
}
fmt.Printf("ENV RLIMIT_NOFILE : %+v\n", rlim)
rlim.Max = 65535
rlim.Cur = 65535
//err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Errorf("设置Rlimit报错 %s", err)
os.Exit(1)
}
//fmt.Printf("ENV RLIMIT_NOFILE : %+v\n", rlim)
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Errorf("获取Rlimit报错 %s", err)
os.Exit(1)
}
fmt.Printf("ENV RLIMIT_NOFILE : %+v\n", rlim)
}
func checkAndSetUlimit() error {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error getting rlimit: %s", err)
}
ipfsFileDescNum := int64(ipfsFileDescNum)
var setting bool
if rLimit.Cur < ipfsFileDescNum {
if rLimit.Max < ipfsFileDescNum {
log.Error("adjusting max")
rLimit.Max = ipfsFileDescNum
}
fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
rLimit.Cur = ipfsFileDescNum
setting = true
}
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error setting ulimit: %s", err)
}
if setting {
fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
}
return nil
}
func TestMain(m *testing.M) {
var lim syscall.Rlimit
syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim)
lim.Cur = 1024000
lim.Max = 1024000
syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim)
}
// setOpenFilesLimit sets the open file limit in the kernel
// cur is the soft limit, max is the ceiling (or hard limit) for that limit
func (pm *ProcessManager) setOpenFilesLimit(cur, max uint64) error {
var rLimit syscall.Rlimit
// First check if the limits are already what we want
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return err
}
// If the current values are less than we want, set them
if rLimit.Cur < cur || rLimit.Max < max {
if cur > rLimit.Cur {
rLimit.Cur = cur
}
if max > rLimit.Max {
rLimit.Max = max
}
log.Infof("Setting open files limit (soft, hard) to (%v, %v)", rLimit.Cur, rLimit.Max)
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return err
}
}
return nil
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
os.Setenv("GOTRACEBACK", "crash")
lim := syscall.Rlimit{}
syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim)
if lim.Cur < _MaxOpenfile || lim.Max < _MaxOpenfile {
lim.Cur = _MaxOpenfile
lim.Max = _MaxOpenfile
syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim)
}
ln, err := net.Listen("tcp", ":1153")
if err != nil {
log.Fatal(err)
}
go TCPServer(ln, queryTypeDNS)
ln, err = net.Listen("tcp", ":1154")
if err != nil {
log.Fatal(err)
}
go TCPServer(ln, queryTypeMYIP)
http.HandleFunc("/myip", HTTPServerMYIP)
http.HandleFunc("/dns", HTTPServerDNS)
http.HandleFunc("/health", HTTPServerHealth)
log.Fatal(http.ListenAndServe(":1053", nil))
}
func initLimit() {
var rLimit syscall.Rlimit
rLimit.Max = 10000
rLimit.Cur = 10000
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("Error Setting Rlimit ", err)
}
}
func init() {
var rLimit syscall.Rlimit
rLimit.Max = 4096
rLimit.Cur = 4096
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Println("Warning: setrlimit:", err)
}
}
// 提升程序可打开的文件描述符上限
func init() {
var rlim syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
panic(err.Error())
}
rlim.Cur = 1000000
rlim.Max = 1000000
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
panic(err.Error())
}
}
func NewServer() *Server {
s := &Server{
port: common.GetConfInt("server", "port", 8080),
maxClients: common.GetConfInt("server", "max_clients", 100000),
clientNum: 0,
acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
connTimeout: common.GetConfSecond("server", "connection_timeout", 60*3),
headerLen: common.GetConfInt("server", "header_length", 10),
maxBodyLen: common.GetConfInt("server", "max_body_length", 102400),
running: false,
heartbeat: time.Now().Unix(),
workerNum: common.WorkerNum,
currentWorker: -1,
slowread: common.GetConfSecond("server", "slow_read", 0),
logger: common.NewLogger("server"),
}
if s == nil {
fmt.Println("new server failed")
return nil
}
if s.logger == nil {
fmt.Println("new server logger failed")
return nil
}
ip, err := common.Conf.String("server", "bind")
if err != nil {
ip = ""
}
s.ip = ip
max_clients := uint64(s.maxClients)
if max_clients > 1024 {
var rlim syscall.Rlimit
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Println("server get rlimit error: " + err.Error())
return nil
}
rlim.Cur = max_clients
rlim.Max = max_clients
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Println("server set rlimit error: " + err.Error())
return nil
}
s.logger.Info("set fd limit to %d", s.maxClients)
}
return s
}
func main() {
logging.SetFormatter(logging.MustStringFormatter(format))
logging.SetLevel(logging.INFO, "gogorobot")
//logging.SetLevel(logging.DEBUG, "gogorobot")
//
// Set the file descriptor limit higher if we've permission
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Info(fmt.Sprintf("Error geting rlimit: %s", err))
}
rLimit.Max = 65536
rLimit.Cur = 65536
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Info(fmt.Sprintf("Error setting rlimit: %s", err))
}
//
fetchPipeline := make(chan FetchRequest)
savePipeline := make(chan RobotResponse)
// NOTE: Important to pass via pointer
// Otherwise, a new WaitGroup is created
var fetchGroup sync.WaitGroup
var saveGroup sync.WaitGroup
saveGroup.Add(1)
go saveRobots(savePipeline, &saveGroup)
for i := 0; i < 50; i++ {
fetchGroup.Add(1)
go fetchRobot(fetchPipeline, savePipeline, &fetchGroup)
}
// Insert entries
reader := bufio.NewReader(os.Stdin)
for {
domain, err := reader.ReadString('\n')
if err != nil {
break
}
domain = strings.TrimRight(domain, "\r\n")
log.Debug(fmt.Sprintf("MAIN: Providing %s to fetch pipeline", domain))
fetchPipeline <- FetchRequest{domain, 0}
}
// TODO: Temporary fix to allow time for the pipeline to clear (see TODO in fetcher's error area)
time.Sleep(60 * time.Second)
close(fetchPipeline)
log.Notice("Fetching pipeline closed -- waiting for pending fetches to complete")
//
fetchGroup.Wait()
close(savePipeline)
saveGroup.Wait()
}
func SetRlimit() {
var rLimit syscall.Rlimit
rLimit.Max = 999999
rLimit.Cur = 999999
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
panic(fmt.Sprintf("Error Setting Rlimit '%s'", err))
}
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
panic(fmt.Sprintf("Error Getting Rlimit '%s'", err))
}
fmt.Printf("final: rlimit.Cur = %d, rlimit.Max = %d\n", rLimit.Cur, rLimit.Max)
}
func setHardRLimit(label string, rLimitId int, rLimitMax uint64) error {
var rlimit syscall.Rlimit
if err := syscall.Getrlimit(rLimitId, &rlimit); err != nil {
return fmt.Errorf("container_daemon: getting system limit %s: %s", label, err)
}
rlimit.Max = rLimitMax
if err := syscall.Setrlimit(rLimitId, &rlimit); err != nil {
return fmt.Errorf("container_daemon: setting hard system limit %s: %s", label, err)
}
return nil
}
func init() {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("Error Getting Rlimit ", err)
}
rLimit.Max = 999999
rLimit.Cur = 999999
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("Error Setting Rlimit ", err)
}
}
func MaybeBecomeChildProcess() {
lrs := os.Getenv("_RUNSIT_LAUNCH_INFO")
if lrs == "" {
return
}
defer os.Exit(2) // should never make it this far, though
lr := new(LaunchRequest)
d := gob.NewDecoder(base64.NewDecoder(base64.StdEncoding, strings.NewReader(lrs)))
err := d.Decode(lr)
if err != nil {
log.Fatalf("Failed to decode LaunchRequest in child: %v", err)
}
if lr.NumFiles != 0 {
var lim syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
log.Fatalf("failed to get NOFILE rlimit: %v", err)
}
noFile := rlim_t(lr.NumFiles)
lim.Cur = noFile
lim.Max = noFile
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
log.Fatalf("failed to set NOFILE rlimit: %v", err)
}
}
if lr.Gid != 0 {
if err := syscall.Setgid(lr.Gid); err != nil {
log.Fatalf("failed to Setgid(%d): %v", lr.Gid, err)
}
}
if len(lr.Gids) != 0 {
if err := syscall.Setgroups(lr.Gids); err != nil {
log.Printf("setgroups: %v", err)
}
}
if lr.Uid != 0 {
if err := syscall.Setuid(lr.Uid); err != nil {
log.Fatalf("failed to Setuid(%d): %v", lr.Uid, err)
}
}
if lr.Path != "" {
err = os.Chdir(lr.Dir)
if err != nil {
log.Fatalf("failed to chdir to %q: %v", lr.Dir, err)
}
}
err = syscall.Exec(lr.Path, lr.Argv, lr.Env)
log.Fatalf("failed to exec %q: %v", lr.Path, err)
}
func setRlimit() error {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("could not get Rlimit: %s", err)
}
rLimit.Max = 999999
rLimit.Cur = 999999
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("could not set Rlimit: %s", err)
}
return nil
}
func SetMaxFileDescriptors(maxFDs uint64) (uint64, error) {
var limits syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
return maxFDs, err
}
if maxFDs > limits.Max {
maxFDs = limits.Max
}
if limits.Cur == maxFDs {
return maxFDs, nil
}
limits.Cur = maxFDs
limits.Max = maxFDs
return maxFDs, syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
}
func tweakLimit() {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
jww.ERROR.Println("Unable to obtain rLimit", err)
}
if rLimit.Cur < rLimit.Max {
rLimit.Max = 999999
rLimit.Cur = 999999
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
jww.ERROR.Println("Unable to increase number of open files limit", err)
}
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
os.Setenv("GOTRACEBACK", "crash")
_BackendAddrCache.Store(make(backendAddrMap))
var lim syscall.Rlimit
syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim)
if lim.Cur < _MaxOpenfile || lim.Max < _MaxOpenfile {
lim.Cur = _MaxOpenfile
lim.Max = _MaxOpenfile
syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim)
}
_SecretPassphase = []byte(os.Getenv("SECRET"))
mhs, err := strconv.Atoi(os.Getenv("MAX_HTTP_HEADER_SIZE"))
if err == nil && mhs > _minHTTPHeaderSize {
_maxHTTPHeaderSize = mhs
}
bt, err := strconv.Atoi(os.Getenv("BACKEND_TIMEOUT"))
if err == nil && bt > 0 {
_BackendDialTimeout = bt
}
connReadTimeout, err := strconv.Atoi(os.Getenv("CONN_READ_TIMEOUT"))
if err == nil && connReadTimeout >= 0 {
_ConnReadTimeout = time.Second * time.Duration(connReadTimeout)
}
listenPort, err := strconv.Atoi(os.Getenv("LISTEN_PORT"))
if err == nil && listenPort > 0 && listenPort <= 65535 {
_DefaultPort = listenPort
}
pprofPort, err := strconv.Atoi(os.Getenv("PPROF_PORT"))
if err == nil && pprofPort > 0 && pprofPort <= 65535 {
go func() {
log.Println(http.ListenAndServe(":"+strconv.Itoa(pprofPort), nil))
}()
}
listenAndServe()
log.Println("Exiting")
}
func SetUlimit(number uint64) {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("[Error]: Getting Rlimit ", err)
}
rLimit.Max = number
rLimit.Cur = number
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("[Error]: Setting Rlimit ", err)
}
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("[Error]: Getting Rlimit ", err)
}
}
func init() {
runtime.GOMAXPROCS(runtime.NumCPU())
os.Setenv("GOTRACEBACK", "crash")
lim := syscall.Rlimit{}
syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim)
if lim.Cur < MaxOpenfile || lim.Max < MaxOpenfile {
lim.Cur = MaxOpenfile
lim.Max = MaxOpenfile
syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim)
}
log.SetOutput(io.Writer(os.Stderr))
log.SetFlags(log.Ldate | log.Lmicroseconds | log.Lshortfile)
pid2file()
}
func setrlimit(rl syscall.Rlimit) (err error) {
if rl.Cur > 0 && rl.Max > 0 {
var lim syscall.Rlimit
if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
log.Println("failed to get NOFILE rlimit: ", err)
return
}
lim.Cur = rl.Cur
lim.Max = rl.Max
if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
log.Println("failed to set NOFILE rlimit: ", err)
return
}
}
return
}
func main() {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("Error Getting Rlimit ", err)
}
fmt.Println(rLimit)
rLimit.Max = 999999
rLimit.Cur = 999999
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("Error Setting Rlimit ", err)
}
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
fmt.Println("Error Getting Rlimit ", err)
}
fmt.Println("Rlimit Final", rLimit)
}
func MaximumUlimit() {
// via https://stackoverflow.com/questions/17817204/how-to-set-ulimit-n-from-a-golang-program
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Println("Error Getting Rlimit ", err)
}
log.Println(rLimit)
rLimit.Max = 999999
rLimit.Cur = 999999
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Println("Error Setting Rlimit ", err)
}
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Println("Error Getting Rlimit ", err)
}
log.Println("Rlimit Final", rLimit)
}
func limitMemory(cur uint64, max uint64) {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_AS, &rLimit)
if err != nil {
fmt.Println("[ERROR] Failed to get resource limit : ", err)
}
// ftm.Println(rLimit)
rLimit.Max = max // 1024 * 1024 * 256 // 256MB
rLimit.Cur = cur // 1024 * 1024 * 64 // 64MB
err = syscall.Setrlimit(syscall.RLIMIT_AS, &rLimit)
if err != nil {
fmt.Println("[ERROR] Failed to set resource limit : ", err)
}
err = syscall.Getrlimit(syscall.RLIMIT_AS, &rLimit)
if err != nil {
fmt.Println("[ERROR] Failed to get resource limit(2) : ", err)
}
fmt.Println("[INFO] Memory usage limited to : ", rLimit)
}
func setMaxFileDescriptors(maxP *uint64) {
maxFDs := DefaultMaxFileDescriptors
if maxP != nil {
maxFDs = *maxP
}
var limits syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
base.LogFatal("Getrlimit failed: %v", err)
}
if maxFDs > limits.Max {
maxFDs = limits.Max
}
if limits.Cur != maxFDs {
limits.Cur = maxFDs
limits.Max = maxFDs
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
if err != nil {
base.LogFatal("Error raising MaxFileDescriptors to %d: %v", maxFDs, err)
}
base.Log("Configured MaxFileDescriptors (RLIMIT_NOFILE) to %d", maxFDs)
}
}
func SetMaxFileDescriptors(maxFDs uint64) (uint64, error) {
var limits syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
return maxFDs, err
}
if maxFDs > limits.Max {
maxFDs = limits.Max
}
if limits.Cur == maxFDs {
return maxFDs, nil
}
limits.Cur = maxFDs
limits.Max = maxFDs
err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
if err == nil {
Logf("Configured process to allow %d open file descriptors", maxFDs)
}
return maxFDs, err
}
func checkAndSetUlimit() error {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error getting rlimit: %s", err)
}
if rLimit.Cur < ipfsFileDescNum {
if rLimit.Max < ipfsFileDescNum {
rLimit.Max = ipfsFileDescNum
}
fmt.Printf("Adjusting current ulimit to %d.\n", ipfsFileDescNum)
rLimit.Cur = ipfsFileDescNum
}
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
return fmt.Errorf("error setting ulimit: %s", err)
}
return nil
}
func init() {
var rlim syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Println("get rlimit error: " + err.Error())
} else {
rlim.Cur = 10000
rlim.Max = 10000
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Println("set rlimit error: " + err.Error())
}
}
rdHost := os.Getenv("REDIS_HOST")
rdPort := os.Getenv("REDIS_PORT")
if localMultVmTest {
rdHost = "192.168.50.1"
}
redisPool = &redis.Pool{
MaxIdle: 256,
MaxActive: 256,
IdleTimeout: 300 * time.Second,
Dial: func() (redis.Conn, error) {
for {
c, err := redis.Dial("tcp", rdHost+":"+rdPort)
if err != nil {
time.Sleep(time.Duration(2) * time.Millisecond)
continue
}
return c, err
}
},
}
initCache()
}
func SuperRlimit(cur, max uint64) {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Fatalf("Error Getting Rlimit:%v\n", err)
}
if cur > rLimit.Cur {
rLimit.Cur = cur
}
if max > rLimit.Max {
rLimit.Max = max
}
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Fatalf("Error Setting Rlimit:%v\n", err)
}
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Fatalf("Error Getting Rlimit:%v\n", err)
}
log.Println("Rlimit Final", rLimit)
}
func setUlimit(cfg *Config) error {
var rLimit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
return err
}
if cfg.NoFiles == 0 {
rLimit.Max = 1000000
} else {
rLimit.Max = cfg.NoFiles
}
rLimit.Cur = rLimit.Max
return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
}