Example #1
0
// Accept accepts connections on the listener and serves requests
// for each incoming connection.  Accept blocks; the caller typically
// invokes it in a go statement.
func acceptTCP(server *Server, lis *net.TCPListener) {
	var (
		conn *net.TCPConn
		err  error
		r    int
	)
	for {
		if conn, err = lis.AcceptTCP(); err != nil {
			// if listener close then return
			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
			return
		}
		if err = conn.SetKeepAlive(Conf.TCPKeepalive); err != nil {
			log.Error("conn.SetKeepAlive() error(%v)", err)
			return
		}
		if err = conn.SetReadBuffer(Conf.TCPSndbuf); err != nil {
			log.Error("conn.SetReadBuffer() error(%v)", err)
			return
		}
		if err = conn.SetWriteBuffer(Conf.TCPRcvbuf); err != nil {
			log.Error("conn.SetWriteBuffer() error(%v)", err)
			return
		}
		go serveTCP(server, conn, r)
		if r++; r == maxInt {
			r = 0
		}
	}
}
Example #2
0
func runWebApi(listener *net.TCPListener, store *StoreChan, staticDir string) (err error) {
	var sessions *webSessionFactory
	if sessions, err = NewWebSessionFactory(600 * time.Second); err != nil { // TODO: hardcoded value
		return err
	}

	http.Handle("/api/authenticate", webHandler{store, sessions, handleWebAuthenticate})
	http.Handle("/api/add", webHandler{store, sessions, handleWebAdd})
	http.Handle("/api/remove", webHandler{store, sessions, handleWebRemove})
	http.Handle("/api/update", webHandler{store, sessions, handleWebUpdate})
	http.Handle("/api/set-admin", webHandler{store, sessions, handleWebSetAdmin})
	http.Handle("/api/list", webHandler{store, sessions, handleWebList})
	http.Handle("/api/list-full", webHandler{store, sessions, handleWebListFull})

	http.Handle("/admin/", http.StripPrefix("/admin/", http.FileServer(http.Dir(staticDir))))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		http.Redirect(w, r, "/admin/", http.StatusTemporaryRedirect)
	})

	wl.Printf("web-api: listening on '%s'", listener.Addr())
	server := &http.Server{ReadTimeout: 60 * time.Second, WriteTimeout: 60 * time.Second}
	return server.Serve(tcpKeepAliveListener{listener})
}
Example #3
0
func (s *Server) loop(listener *net.TCPListener) {
	defer s.waitGroup.Done()
	log.Printf("[DEBUG] waiting for connections...")
	for {
		select {
		case <-s.ch:
			log.Printf("[DEBUG] stopping listening on %s", listener.Addr())
			_ = listener.Close()
			return
		default:
		}
		_ = listener.SetDeadline(time.Now().Add(s.Deadline))
		conn, err := listener.AcceptTCP()
		if nil != err {
			if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
				continue
			}
			log.Printf("[ERROR] error on loop: %s", err)
		}
		log.Printf("[DEBUG] connected to %s", conn.RemoteAddr())
		c := &connection{
			conn:          conn,
			protocol:      s.protocolCtor(conn),
			waitGroup:     s.waitGroup,
			deadline:      s.Deadline,
			readBuffSize:  s.ReadBuffSize,
			writeBuffSize: s.WriteBuffSize,
		}
		s.waitGroup.Add(1)
		go c.handle(s.ch)
	}
}
Example #4
0
func (self *Service) Serve(listener *net.TCPListener) {
	defer self.wg.Done()

	log.Println("initializing...")
	var err error
	self.repo, err = repository.Initialize(self.dataDir)
	log.Println("data directory: ", self.dataDir)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case <-self.ch:
			log.Println("stopping listening on", listener.Addr())
			listener.Close()
			return
		default:
		}
		listener.SetDeadline(time.Now().Add(1e9))
		conn, err := listener.AcceptTCP()
		if nil != err {
			if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
				continue
			}
			log.Println(err)
		}
		self.wg.Add(1)
		go self.HandleConnection(conn)
	}
}
Example #5
0
// Accept connections and spawn a goroutine to serve each one.  Stop listening
// if anything is received on the service's channel.
func (self *Service) Serve(listener *net.TCPListener) {
	defer self.waitGroup.Done()
	for {
		select {
		case <-self.done:
			log.Println("Stopping listening on", listener.Addr())
			listener.Close()
			return
		default:
		}

		listener.SetDeadline(time.Now().Add(1e9))
		conn, err := listener.AcceptTCP()
		if err != nil {
			if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
				continue
			}
			log.Println(err)
		}

		log.Println(conn.RemoteAddr(), "connected")

		connection := NewConnection(conn, make(chan []byte))
		self.dataMap[connection.Id] = connection
		self.waitGroup.Add(1)
		go self.serve(connection)
	}
}
Example #6
0
func (this *Server) Start(listener *net.TCPListener) {
	log.Printf("Start listen on %v", listener.Addr())
	this.waitGroup.Add(1)
	defer func() {
		listener.Close()
		this.waitGroup.Done()
	}()

	for {
		select {
		case <-this.exitCh:
			log.Printf("Stop listen on %v", listener.Addr())
			return
		default:
		}

		listener.SetDeadline(time.Now().Add(this.acceptTimeout))
		conn, err := listener.AcceptTCP()
		if err != nil {
			if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
				// log.Printf("Accept timeout: %v", opErr)
				continue
			}
			log.Printf("Accept error: %v", err)
			continue
		}

		log.Printf("Accept: %v", conn.RemoteAddr())
		go this.handleClientConn(conn)
	}
}
Example #7
0
func New(id circuit.WorkerID, bindAddr string, host string) *Transport {

	// Bind
	var l *net.TCPListener
	if strings.Index(bindAddr, ":") < 0 {
		bindAddr = bindAddr + ":0"
	}
	l_, err := net.Listen("tcp", bindAddr)
	if err != nil {
		panic(err)
	}

	// Build transport structure
	l = l_.(*net.TCPListener)
	t := &Transport{
		listener:   l,
		addrtabl:   makeAddrTabl(),
		pipelining: DefaultPipelining,
		remote:     make(map[circuit.WorkerID]*link),
		ach:        make(chan *conn),
	}

	// Resolve self address
	laddr := l.Addr().(*net.TCPAddr)
	t.self, err = NewAddr(id, os.Getpid(), fmt.Sprintf("%s:%d", host, laddr.Port))
	if err != nil {
		panic(err)
	}

	// This LocalAddr might be useless for connect purposes (e.g. 0.0.0.0). Consider self instead.
	t.bind = t.addrtabl.Normalize(&Addr{ID: id, PID: os.Getpid(), Addr: laddr})

	go t.loop()
	return t
}
Example #8
0
// Accept accepts connections on the listener and serves requests
// for each incoming connection.  Accept blocks; the caller typically
// invokes it in a go statement.
func (server *Server) AcceptTCP(lis *net.TCPListener, i int) {
	var (
		conn *net.TCPConn
		err  error
	)
	for {
		log.Debug("server: accept round: %d", i)
		if conn, err = lis.AcceptTCP(); err != nil {
			// if listener close then return
			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
			return
		}
		if err = conn.SetKeepAlive(Conf.TCPKeepalive); err != nil {
			log.Error("conn.SetKeepAlive() error(%v)", err)
			return
		}
		if err = conn.SetReadBuffer(Conf.TCPSndbuf); err != nil {
			log.Error("conn.SetReadBuffer() error(%v)", err)
			return
		}
		if err = conn.SetWriteBuffer(Conf.TCPRcvbuf); err != nil {
			log.Error("conn.SetWriteBuffer() error(%v)", err)
			return
		}
		go server.serveConn(conn, i)
		if i++; i == maxInt {
			i = 0
		}
	}
}
Example #9
0
func newServer(s *state.State, lis *net.TCPListener, cfg ServerConfig) (_ *Server, err error) {
	logger.Infof("listening on %q", lis.Addr())
	srv := &Server{
		state:     s,
		statePool: state.NewStatePool(s),
		addr:      lis.Addr().(*net.TCPAddr), // cannot fail
		tag:       cfg.Tag,
		dataDir:   cfg.DataDir,
		logDir:    cfg.LogDir,
		limiter:   utils.NewLimiter(loginRateLimit),
		validator: cfg.Validator,
		adminApiFactories: map[int]adminApiFactory{
			0: newAdminApiV0,
			1: newAdminApiV1,
			2: newAdminApiV2,
		},
	}
	srv.authCtxt = newAuthContext(srv)
	tlsCert, err := tls.X509KeyPair(cfg.Cert, cfg.Key)
	if err != nil {
		return nil, err
	}
	// TODO(rog) check that *srvRoot is a valid type for using
	// as an RPC server.
	tlsConfig := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	}
	changeCertListener := newChangeCertListener(lis, cfg.CertChanged, tlsConfig)
	go srv.run(changeCertListener)
	return srv, nil
}
Example #10
0
// FreePorts returns maximum n tcp ports available to use.
func FreePorts(n int, matchFunc func(int) bool) (ports []int, err error) {
	if n > 50000 {
		return nil, fmt.Errorf("too many ports requested (%d)", n)
	}
	for len(ports) < n {
		var addr *net.TCPAddr
		addr, err = net.ResolveTCPAddr("tcp", "localhost:0")
		if err != nil {
			break
		}
		var l *net.TCPListener
		l, err = net.ListenTCP("tcp", addr)
		if err != nil {
			break
		}
		port := l.Addr().(*net.TCPAddr).Port
		l.Close()

		if matchFunc == nil {
			ports = append(ports, port)
			continue
		}
		if matchFunc(port) {
			ports = append(ports, port)
		}
	}
	return
}
Example #11
0
func (tcp *Tcp) serve(listeningPoint *net.TCPListener) {
	log.Info("Begin serving TCP on address " + listeningPoint.Addr().String())

	for {
		baseConn, err := listeningPoint.Accept()
		if err != nil {
			log.Severe("Failed to accept TCP conn on address " + listeningPoint.Addr().String() + "; " + err.Error())
			continue
		}

		conn := NewConn(baseConn, tcp.output)
		log.Debug("Accepted new TCP conn %p from %s on address %s", &conn, conn.baseConn.RemoteAddr(), conn.baseConn.LocalAddr())
		tcp.connTable.Notify(baseConn.RemoteAddr().String(), conn)
	}
}
Example #12
0
/* Wait for and accept a client.  wg.Done() will be called when l is closed or
an error occurs (and the goroutine terminates). */
func accept(l *net.TCPListener, wg sync.WaitGroup) {
	defer wg.Done()
	for {
		/* Get a connection */
		c, err := l.AcceptTCP()
		/* Notify listener and die on error */
		if err != nil {
			log.Printf("Ceasing to listen on %v: %v", l.Addr(),
				err)
			return
		}
		/* Handle client */
		go handle(c)
	}
}
Example #13
0
func NewTcpAcceptor(strAddr string) (*TcpAcceptor, error) {
	var err error
	var listener *net.TCPListener
	var tcpAddr *net.TCPAddr
	if tcpAddr, err = net.ResolveTCPAddr("tcp", strAddr); err == nil {
		listener, err = net.ListenTCP("tcp", tcpAddr)
	}
	if err == nil {
		a := TcpAcceptor{}
		a.listener = listener
		addr := listener.Addr().String()
		a.endPoint, _ = NewTcpEndPoint(addr)
		return &a, nil
	} else {
		return nil, err
	}
}
Example #14
0
func listenTCP(host, port string) (err error) {
	var listener *net.TCPListener
	// Automatically assign open port
	address, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port))
	if err != nil {
		log.WithField("error", err).Fatal("Unable to resolve tcp address")
		return
	}
	listener, err = net.ListenTCP("tcp", address)
	if err != nil {
		log.WithField("error", err).Fatal("Unable to bind to tcp on localhost")
		return
	}
	log.WithField("address", listener.Addr()).Info("Listening")
	defer listener.Close()
	serve(listener)
	return
}
Example #15
0
File: server.go Project: houcy/push
func (this *Server) Run(listener *net.TCPListener) {
	defer func() {
		listener.Close()
	}()

	//go this.dealSpamConn()
	log.Infof("Starting comet server on: %s", listener.Addr().String())
	if err := storage.Instance.AddComet(this.Name); err != nil {
		log.Errorf("failed to add comet to Redis: %s", err.Error())
	}

	// keep the data of this node not expired on redis
	this.startRefreshRoutine()
	this.startWorkers()

	for {
		select {
		case <-this.ctrl:
			log.Infof("ask me to quit")
			this.stopWorkers()
			this.stopRefreshRoutine()
			return
		default:
		}

		listener.SetDeadline(time.Now().Add(this.acceptTimeout * time.Second))
		conn, err := listener.AcceptTCP()
		if err != nil {
			if e, ok := err.(*net.OpError); ok && e.Timeout() {
				continue
			}
			log.Errorf("accept failed: %v\n", err)
			continue
		}
		/*
			// first packet must sent by client in specified seconds
			if err = conn.SetReadDeadline(time.Now().Add(20)); err != nil {
				glog.Errorf("conn.SetReadDeadLine() error(%v)", err)
				conn.Close()
				continue
			}*/
		go this.handleConnection(conn)
	}
}
Example #16
0
func (this *Server) Start(listener *net.TCPListener) {
	log.Printf("Start listen on %v\r\n", listener.Addr())
	this.waitGroup.Add(1)
	defer func() {
		listener.Close()
		this.waitGroup.Done()
	}()

	// 防止恶意连接
	go this.dealSpamConn()

	// report记录,定时发送邮件
	go report.Work()

	for {
		select {
		case <-this.exitCh:
			log.Printf("Stop listen on %v\r\n", listener.Addr())
			return
		default:
		}

		listener.SetDeadline(time.Now().Add(this.acceptTimeout))
		conn, err := listener.AcceptTCP()

		if err != nil {
			if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
				// log.Printf("Accept timeout: %v\r\n", opErr)
				continue
			}
			report.AddCount(report.TryConnect, 1)
			log.Printf("Accept error: %v\r\n", err)
			continue
		}

		report.AddCount(report.SuccessConnect, 1)

		// 连接后等待登陆验证
		handlers.ConnMapLoginStatus.Set(conn, time.Now())
		log.Printf("Accept: %v\r\n", conn.RemoteAddr())

		go this.handleClientConn(conn)
	}
}
Example #17
0
func (this *Server) Run(listener *net.TCPListener) {
	this.waitGroup.Add(1)
	defer func() {
		listener.Close()
		this.waitGroup.Done()
	}()

	//go this.dealSpamConn()
	log.Infof("Starting comet server on: %s\n", listener.Addr().String())
	log.Infof("Comet server settings: readtimeout [%d], accepttimeout [%d], heartbeattimeout [%d]\n",
		this.readTimeout, this.acceptTimeout, this.heartbeatTimeout)
	for {
		select {
		case <-this.exitCh:
			log.Infof("Stopping comet server")
			return
		default:
		}

		listener.SetDeadline(time.Now().Add(this.acceptTimeout))
		conn, err := listener.AcceptTCP()
		if err != nil {
			if e, ok := err.(*net.OpError); ok && e.Timeout() {
				continue
			}
			log.Errorf("accept failed: %v\n", err)
			continue
		}
		/*
			// first packet must sent by client in specified seconds
			if err = conn.SetReadDeadline(time.Now().Add(20)); err != nil {
				glog.Errorf("conn.SetReadDeadLine() error(%v)", err)
				conn.Close()
				continue
			}*/
		go this.handleConnection(conn)
	}
}
Example #18
0
func NewTcpServer(uri string) *TcpServer {
	if uri == "" {
		uri = "tcp://127.0.0.1:0"
	}
	var u *url.URL
	var err error
	if u, err = url.Parse(uri); err != nil {
		panic(err.Error())
	}
	var addr *net.TCPAddr
	if addr, err = net.ResolveTCPAddr(u.Scheme, u.Host); err != nil {
		panic(err.Error())
	}
	var listener *net.TCPListener
	if listener, err = net.ListenTCP(u.Scheme, addr); err != nil {
		panic(err.Error())
	}
	return &TcpServer{
		TcpService:  NewTcpService(),
		URL:         u.Scheme + "://" + listener.Addr().String(),
		TCPListener: listener,
	}
}
Example #19
0
func (s *Server) serve(l *net.TCPListener) {
	defer s.waitGroup.Done()
	s.waitGroup.Add(1)
	for {
		select {
		case <-s.ch:
			log.Debug("stopping listening on: ", l.Addr())
			l.Close()
			return
		default:
		}

		l.SetDeadline(time.Now().Add(1e9))
		conn, err := l.AcceptTCP()
		if err != nil {
			if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
				continue
			}
			log.Debug(err)
		}

		// handle the connection in a new goroutine. This returns to listener
		// accepting code so that multiple connections may be served concurrently.
		keeper := NewKeeper(conn, s.storeClient)

		go func() {
			defer s.waitGroup.Done()
			s.waitGroup.Add(1)
			log.Debug("client connected: ", conn.RemoteAddr())
			if err := keeper.Handle(); err != nil {
				log.Debug("client disconnected: ", conn.RemoteAddr(), " with error: ", err)
			} else {
				log.Debug("client disconnected: ", conn.RemoteAddr())
			}
		}()
	}
}
Example #20
0
// Re-exec this image without dropping the listener passed to this function.
func Relaunch(l *net.TCPListener) error {
	argv0, err := exec.LookPath(os.Args[0])
	if nil != err {
		return err
	}
	wd, err := os.Getwd()
	if nil != err {
		return err
	}
	v := reflect.ValueOf(l).Elem().FieldByName("fd").Elem()
	fd := uintptr(v.FieldByName("sysfd").Int())
	if err := os.Setenv("GOAGAIN_FD", fmt.Sprint(fd)); nil != err {
		return err
	}
	if err := os.Setenv("GOAGAIN_NAME", fmt.Sprintf("tcp:%s->", l.Addr().String())); nil != err {
		return err
	}
	if err := os.Setenv("GOAGAIN_PPID", fmt.Sprint(syscall.Getpid())); nil != err {
		return err
	}
	files := make([]*os.File, fd+1)
	files[syscall.Stdin] = os.Stdin
	files[syscall.Stdout] = os.Stdout
	files[syscall.Stderr] = os.Stderr
	files[fd] = os.NewFile(fd, string(v.FieldByName("sysfile").String()))
	p, err := os.StartProcess(argv0, os.Args, &os.ProcAttr{
		Dir:   wd,
		Env:   os.Environ(),
		Files: files,
		Sys:   &syscall.SysProcAttr{},
	})
	if nil != err {
		return err
	}
	log.Printf("spawned child %d\n", p.Pid)
	return nil
}
Example #21
0
func run() int {
	bytearray.EnableAutoGC(60, 74)

	runtime.SetBlockProfileRate(1000)
	go func() {
		log.Println(http.ListenAndServe(":6060", nil))
	}()

	//defer profile.Start(&profile.Config{CPUProfile: false, MemProfile: true, ProfilePath: ".", NoShutdownHook: true}).Stop()

	/*defer func() {
		// Panic error handling
		if r := recover(); r != nil {
			fmt.Println(r)
			return 1
		}
	}()*/

	var err error

	exeRoot, _ := osext.ExecutableFolder()

	var serverPort int64 = int64(DEFAULT_SERVER_IP_PORT)
	datDirectory = filepath.Join(exeRoot, "data")
	idxDirectory = filepath.Join(exeRoot, "index")

	cmd.Title = fmt.Sprintf("Hashbox Server %s", Version)
	cmd.IntOption("port", "", "<port>", "Server listening port", &serverPort, cmd.Standard)
	cmd.StringOption("data", "", "<path>", "Full path to dat files", &datDirectory, cmd.Standard)
	cmd.StringOption("index", "", "<path>", "Full path to idx and meta files", &idxDirectory, cmd.Standard)
	var loglvl int64 = int64(core.LogInfo)
	cmd.IntOption("loglevel", "", "<level>", "Set log level (0=errors, 1=warnings, 2=info, 3=debug, 4=trace", &loglvl, cmd.Hidden).OnChange(func() {
		core.LogLevel = int(loglvl)
	})

	// Please note that datPath has not been set until we have parsed arguments, that is ok because neither of the handlers
	// start opening files on their own
	// TODO: remove datPath global and send them into handlers on creation instead
	accountHandler = NewAccountHandler()
	defer accountHandler.Close()
	storageHandler = NewStorageHandler()
	defer storageHandler.Close()

	cmd.Command("", "", func() { // Default
		serverAddr := net.TCPAddr{nil, int(serverPort), ""}

		if lock, err := lockfile.Lock(filepath.Join(datDirectory, "hashbox.lck")); err != nil {
			panic(err)
		} else {
			defer lock.Unlock()
		}

		var listener *net.TCPListener
		if listener, err = net.ListenTCP("tcp", &serverAddr); err != nil {
			panic(errors.New(fmt.Sprintf("Error listening: %v", err.Error())))
		}
		core.Log(core.LogInfo, "%s is listening on %s", cmd.Title, listener.Addr().String())

		done = make(chan bool)
		defer close(done)

		signalchan := make(chan os.Signal, 1)
		defer close(signalchan)
		signal.Notify(signalchan, os.Interrupt)
		signal.Notify(signalchan, os.Kill)
		go func() {
			for s := range signalchan {
				core.Log(core.LogInfo, "Received OS signal: %v", s)
				listener.Close()
				// done <- true
				return
			}
		}()

		go connectionListener(listener)

		go func() {
			var lastStats string
			for { // ever
				time.Sleep(10 * time.Second)
				s := core.MemoryStats()
				if s != lastStats {
					fmt.Println(s)
					lastStats = s
				}
			}
		}()

		// blocking channel read
		select {
		case <-done:
		case <-accountHandler.signal:
		case <-storageHandler.signal:
		}

		core.Log(core.LogInfo, "Hashbox Server terminating")
	})

	// TODO: This is a temporary hack to allow creation of hashback users on the server side
	// It should be an interface to an adminsitrative tool instead
	cmd.Command("adduser", "<username> <password>", func() {
		if lock, err := lockfile.Lock(filepath.Join(datDirectory, "hashbox.lck")); err != nil {
			panic(err)
		} else {
			defer lock.Unlock()
		}

		if len(cmd.Args) < 4 {
			panic(errors.New("Missing argument to adduser command"))
		}

		if (!accountHandler.SetInfo(AccountInfo{AccountName: core.String(cmd.Args[2]), AccessKey: core.GenerateAccessKey(cmd.Args[2], cmd.Args[3])})) {
			panic(errors.New("Error creating account"))
		}
		accountNameH := core.Hash([]byte(cmd.Args[2]))
		dataEncryptionKey := core.GenerateDataEncryptionKey()
		core.Log(core.LogDebug, "DataEncryptionKey is: %x", dataEncryptionKey)
		core.EncryptDataInPlace(dataEncryptionKey[:], core.GenerateBackupKey(cmd.Args[2], cmd.Args[3]))

		var blockData bytearray.ByteArray
		blockData.Write(dataEncryptionKey[:])
		block := core.NewHashboxBlock(core.BlockDataTypeRaw, blockData, nil)
		if !storageHandler.writeBlock(block) {
			panic(errors.New("Error writing key block"))
		}
		err := accountHandler.AddDatasetState(accountNameH, core.String("\x07HASHBACK_DEK"), core.DatasetState{BlockID: block.BlockID})
		PanicOn(err)

		block.Release()
		core.Log(core.LogInfo, "User added")
	})

	var doRepair bool
	var doRebuild bool
	var skipData, skipMeta, skipIndex bool
	cmd.BoolOption("repair", "check-storage", "Repair non-fatal errors", &doRepair, cmd.Standard)
	cmd.BoolOption("rebuild", "check-storage", "Rebuild index and meta files from data", &doRebuild, cmd.Standard)
	cmd.BoolOption("skipdata", "check-storage", "Skip checking data files", &skipData, cmd.Standard)
	cmd.BoolOption("skipmeta", "check-storage", "Skip checking meta files", &skipMeta, cmd.Standard)
	cmd.BoolOption("skipindex", "check-storage", "Skip checking index files", &skipIndex, cmd.Standard)
	cmd.Command("check-storage", "", func() {
		if doRebuild {
			if len(cmd.Args) > 2 {
				panic("Start and end file arguments are not valid in combination with rebuild")
			}
			doRepair = true
		}

		startfile := int32(0)
		endfile := int32(-1)
		if len(cmd.Args) > 2 {
			i, err := strconv.ParseInt(cmd.Args[2], 0, 32)
			if err != nil {
				panic(err)
			}
			startfile = int32(i)
			core.Log(core.LogInfo, "Starting from file #%d (%04x)", startfile, startfile)
		}
		if len(cmd.Args) > 3 {
			i, err := strconv.ParseInt(cmd.Args[3], 0, 32)
			if err != nil {
				panic(err)
			}
			endfile = int32(i)
			core.Log(core.LogInfo, "Stopping after file #%d (%04x)", endfile, endfile)
		}

		if doRepair {
			if lock, err := lockfile.Lock(filepath.Join(datDirectory, "hashbox.lck")); err != nil {
				panic(err)
			} else {
				defer lock.Unlock()
			}
		}

		start := time.Now()

		if doRebuild {
			core.Log(core.LogInfo, "Removing index files")
			storageHandler.RemoveFiles(storageFileTypeIndex)
			core.Log(core.LogInfo, "Removing meta files")
			storageHandler.RemoveFiles(storageFileTypeMeta)
		}

		core.Log(core.LogInfo, "Checking all storage files")
		repaired, critical := storageHandler.CheckFiles(doRepair)
		if !skipData && (doRepair || critical == 0) {
			core.Log(core.LogInfo, "Checking data files")
			r, c := storageHandler.CheckData(doRepair, startfile, endfile)
			repaired += r
			critical += c
		}
		if !skipMeta && (doRepair || critical == 0) {
			core.Log(core.LogInfo, "Checking meta files")
			storageHandler.CheckMeta()
		}

		if !skipIndex && (doRepair || critical == 0) {
			core.Log(core.LogInfo, "Checking index files")
			storageHandler.CheckIndexes(doRepair)
		}

		if doRepair || critical == 0 {
			core.Log(core.LogInfo, "Checking dataset transactions")
			rootlist := accountHandler.RebuildAccountFiles()

			core.Log(core.LogInfo, "Checking block chain integrity")
			verified := make(map[core.Byte128]bool) // Keep track of verified blocks
			for i, r := range rootlist {
				tag := fmt.Sprintf("%s.%s.%x", r.AccountName, r.DatasetName, r.StateID[:])
				core.Log(core.LogDebug, "CheckChain on %s", tag)
				c := storageHandler.CheckChain(r.BlockID, tag, verified)
				if c > 0 {
					accountHandler.InvalidateDatasetState(r.AccountNameH, r.DatasetName, r.StateID)
				}
				critical += c

				p := int(i * 100 / len(rootlist))
				fmt.Printf("%d%%\r", p)
			}
		}

		if critical > 0 {
			core.Log(core.LogError, "Detected %d critical errors, DO NOT start the server unless everything is repaired", critical)
		}
		if repaired > 0 {
			core.Log(core.LogWarning, "Performed %d repairs, please run again to verify repairs", repaired)
		}
		if critical == 0 && repaired == 0 {
			core.Log(core.LogInfo, "All checks completed successfully in %.1f minutes", time.Since(start).Minutes())
		}
	})

	var doCompact bool
	var deadSkip int64 = 5
	var skipSweep bool
	var doForce bool
	cmd.BoolOption("compact", "gc", "Compact data files to free space", &doCompact, cmd.Standard)
	cmd.BoolOption("skipsweep", "gc", "Skip sweeping indexes", &skipSweep, cmd.Standard)
	cmd.BoolOption("force", "gc", "Ignore broken datasets and force a garbage collect", &doForce, cmd.Standard)
	cmd.IntOption("threshold", "gc", "<percentage>", "Compact minimum dead space threshold", &deadSkip, cmd.Standard)
	cmd.Command("gc", "", func() {
		if lock, err := lockfile.Lock(filepath.Join(datDirectory, "hashbox.lck")); err != nil {
			panic(err)
		} else {
			defer lock.Unlock()
		}

		start := time.Now()
		if !skipSweep {
			core.Log(core.LogInfo, "Marking index entries")
			var roots []core.Byte128
			for _, r := range accountHandler.CollectAllRootBlocks(doForce) {
				roots = append(roots, r.BlockID)
			}
			storageHandler.MarkIndexes(roots, true)
			storageHandler.SweepIndexes(true)
			core.Log(core.LogInfo, "Mark and sweep duration %.1f minutes", time.Since(start).Minutes())
			storageHandler.ShowStorageDeadSpace()
		}
		if doCompact {
			storageHandler.CompactIndexes(true)
			storageHandler.CompactAll(storageFileTypeMeta, int(deadSkip))
			storageHandler.CompactAll(storageFileTypeData, int(deadSkip))
		}
		core.Log(core.LogInfo, "Garbage collection completed in %.1f minutes", time.Since(start).Minutes())
	})

	err = cmd.Parse()
	PanicOn(err)

	fmt.Println(core.MemoryStats())

	return 0
}
Example #22
0
File: mock.go Project: avsej/gocb
// Creates and runs a new mock instance
// The path is the path to the mock jar.
// nodes is the total number of cluster nodes (and thus the number of mock threads)
// replicas is the number of replica nodes (subset of the number of nodes) for each couchbase bucket.
// vbuckets is the number of vbuckets to use for each couchbase bucket
// specs should be a list of specifications of buckets to use..
func NewMock(path string, nodes uint, replicas uint, vbuckets uint, specs ...BucketSpec) (m *Mock, err error) {
	var lsn *net.TCPListener = nil
	chAccept := make(chan bool)
	m = &Mock{}

	defer func() {
		close(chAccept)
		if lsn != nil {
			lsn.Close()
		}
		exc := recover()

		if exc == nil {
			// No errors, everything is OK
			return
		}

		// Close mock on error, destroying resources
		m.Close()
		if mExc, ok := exc.(mockError); !ok {
			panic(mExc)
		} else {
			m = nil
			err = mExc
		}
	}()

	if lsn, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0}); err != nil {
		throwMockError("Couldn't set up listening socket", err)
	}
	_, ctlPort, _ := net.SplitHostPort(lsn.Addr().String())
	log.Printf("Listening for control connection at %s\n", ctlPort)

	go func() {
		defer func() {
			chAccept <- false
		}()
		if m.conn, err = lsn.Accept(); err != nil {
			throwMockError("Couldn't accept incoming control connection from mock", err)
			return
		}
	}()

	if len(specs) == 0 {
		specs = []BucketSpec{BucketSpec{Name: "default", Type: BCouchbase}}
	}

	options := []string{
		"-jar", path, "--harakiri-monitor", "localhost:" + ctlPort, "--port", "0",
		"--replicas", strconv.Itoa(int(replicas)),
		"--vbuckets", strconv.Itoa(int(vbuckets)),
		"--nodes", strconv.Itoa(int(nodes)),
		"--buckets", m.buildSpecStrings(specs),
	}

	log.Printf("Invoking java %s", strings.Join(options, " "))
	m.cmd = exec.Command("java", options...)

	m.cmd.Stdout = os.Stdout
	m.cmd.Stderr = os.Stderr

	if err = m.cmd.Start(); err != nil {
		m.cmd = nil
		throwMockError("Couldn't start command", err)
	}

	select {
	case <-chAccept:
		break

	case <-time.After(mockInitTimeout):
		throwMockError("Timed out waiting for initalization", errors.New("timeout"))
	}

	m.rw = bufio.NewReadWriter(bufio.NewReader(m.conn), bufio.NewWriter(m.conn))

	// Read the port buffer, which is delimited by a NUL byte
	if portBytes, err := m.rw.ReadBytes(0); err != nil {
		throwMockError("Couldn't get port information", err)
	} else {
		portBytes = portBytes[:len(portBytes)-1]
		if entryPort, err := strconv.Atoi(string(portBytes)); err != nil {
			throwMockError("Incorrectly formatted port from mock", err)
		} else {
			m.EntryPort = uint16(entryPort)
		}
	}

	log.Printf("Mock HTTP port at %d\n", m.EntryPort)
	return
}
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-cf-experimental/opentsdb-firehose-nozzle/poster"
)

var telnetChan chan []byte

var _ = Describe("OpentsdbClient Tcp", func() {

	var tcpListener *net.TCPListener
	var p *poster.TelnetPoster

	BeforeEach(func() {
		telnetChan = make(chan []byte, 1)
		tcpListener = NewTCPServer()
		p = poster.NewTelnetPoster(tcpListener.Addr().String())

	})

	It("posts value metrics over tcp", func() {
		timestamp := time.Now().Unix()
		metric := poster.Metric{
			Metric:    "origin.metricName",
			Value:     5,
			Timestamp: timestamp,
			Tags: poster.Tags{
				Deployment: "deployment-name",
				Job:        "doppler",
				Index:      "SOME-GUID",
				IP:         "10.10.10.10",
			},
Example #24
0
/* Listen for a connection on *gc.Cwait, handle it with handler */
func listener(init *sync.WaitGroup) {
	/* If someone's waiting on us... */
	if *gc.Cwait {
		defer WG.Done()
	}
	/* Make sure the listen address isn't just a port. */
	if _, err := strconv.Atoi(*gc.Cport); nil == err {
		*gc.Cport = ":" + *gc.Cport
	}
	/* Get tcp4 and tcp6 addresses, maybe */
	a4, e4 := net.ResolveTCPAddr("tcp4", *gc.Cport)
	a6, e6 := net.ResolveTCPAddr("tcp6", *gc.Cport)

	/* Die if neither work */
	if e4 != nil && e6 != nil {
		log.Printf("Unable to resolve %v as an IPv4 address: %v",
			*gc.Cport, e4)
		log.Printf("Unable to resolve %v as an IPv6 address: %v",
			*gc.Cport, e6)
		os.Exit(-6)
	}

	/* Waitgroup for listeners */
	wg := sync.WaitGroup{}

	/* Start listeners */
	var l4 *net.TCPListener
	var l6 *net.TCPListener
	if nil == e4 {
		l4, e4 = net.ListenTCP("tcp4", a4)
		if nil == e4 {
			log.Printf("Command listener started on %v", l4.Addr())
			wg.Add(1)
			go accept(l4, wg)
			defer l4.Close()
		}
	} else {
		e4 = nil
	}
	if nil == e6 {
		l6, e6 = net.ListenTCP("tcp6", a6)
		if nil == e6 {
			log.Printf("Command listener started on %v", l6.Addr())
			wg.Add(1)
			go accept(l6, wg)
			defer l6.Close()
		}
	} else {
		e6 = nil
	}
	if e4 != nil && e6 != nil {
		log.Printf("Cannot start IPv4 listener on %v: %v", a4, e4)
		log.Printf("Cannot start IPv6 listener on %v: %v", a6, e6)
		os.Exit(-7)
	}

	/* Signal end of initialization */
	init.Done()

	/* Wait for accepters to die. */
	wg.Wait()

	return

}