示例#1
0
func (d *Daemon) Init() error {

	/* Setup the TLS authentication */
	certf, keyf, err := shared.ReadMyCert("client.crt", "client.key")
	if err != nil {
		return err
	}
	d.certf = certf
	d.keyf = keyf
	d.tlsConfig, err = shared.GetTLSConfig(d.certf, d.keyf)
	if err != nil {
		return err
	}
	srvlog := log.New(log.Ctx{"module": "client"})
	d.log = srvlog

	handler := log.StdoutHandler

	if quiet {
		d.log.SetHandler(log.DiscardHandler())
	} else if verbose {
		d.log.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
	} else if debug {
		d.log.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
	} else {
		d.log.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
	}

	d.Run()
	return nil
}
示例#2
0
func sockHandler(d *Daemon, w *websocket.Conn) {
	r := w.Request()
	raddr := r.RemoteAddr
	if !d.isTrustedClient(r) {
		d.log.Warn("untrusted client connected", log.Ctx{"raddr": raddr})
		return
	}

	d.log.Debug("handing over client connection", log.Ctx{"raddr": raddr})

	clog := log.New(log.Ctx{"module": "socks", "raddr": raddr})
	handler := log.StdoutHandler

	if quiet {
		clog.SetHandler(log.DiscardHandler())
	} else if verbose {
		clog.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
	} else if debug {
		clog.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
	} else {
		clog.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
	}

	client := &clientConnection{
		websocket: w,
		log:       clog,
		raddr:     raddr,
	}
	go client.run()
}
示例#3
0
func main() {
	flag.Parse()
	if helpflag {
		flag.Usage()
		return
	}

	logger = log.New()

	handler := log.StdoutHandler
	if quietflag {
		logger.SetHandler(log.DiscardHandler())
	} else if verboseflag {
		logger.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
	} else if debugflag {
		logger.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
	} else {
		logger.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
	}

	ldifstore := &ldif.LdifBackend{
		Path: "./ldif",
		Log:  logger.New(log.Ctx{"type": "backend", "backend": "ldif"}),
	}

	if err := ldifstore.Start(); err != nil {
		logger.Error("error loading backend", log.Ctx{"error": err})
		os.Exit(1)
	}

	//Create a new LDAP Server
	server := ldap.NewServer(logger)

	fallback := &debug.DebugBackend{
		Log: logger.New(log.Ctx{"type": "backend", "backend": "debug"}),
	}

	//Create routes bindings
	routes := newRouter(fallback, logger)

	// backend specific routes
	routes.Bind(ldifstore).BaseDn("dc=enterprise,dc=org").Label("Bind LDIF")
	routes.Search(ldifstore).BaseDn("dc=enterprise,dc=org").Label("Search LDIF")
	routes.Add(ldifstore).BaseDn("dc=enterprise,dc=org").Label("Add LDIF")

	//Attach routes to server
	server.Handle(routes)

	// listen on 3389 and serve
	go server.ListenAndServe(":6389")

	// When CTRL+C, SIGINT and SIGTERM signal occurs
	// Then stop server gracefully
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	<-ch
	close(ch)

	server.Stop()
}
示例#4
0
文件: log.go 项目: rockstar/lxd
// SetLogger defines the *log.Logger where log messages are sent to.
func SetLogger(syslog string, logfile string, verbose bool, debug bool) error {
	Log = log.New()

	var handlers []log.Handler

	var syshandler log.Handler

	// System specific handler
	syshandler = GetSystemHandler(syslog, debug)
	if syshandler != nil {
		handlers = append(handlers, syshandler)
	}

	// FileHandler
	if logfile != "" {
		if !PathExists(filepath.Dir(logfile)) {
			return fmt.Errorf("Log file path doesn't exist: %s", filepath.Dir(logfile))
		}

		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlInfo,
					log.Must.FileHandler(logfile, log.LogfmtFormat()),
				),
			)
		} else {
			handlers = append(handlers, log.Must.FileHandler(logfile, log.LogfmtFormat()))
		}
	}

	// StderrHandler
	if verbose || debug {
		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlInfo,
					log.StderrHandler,
				),
			)
		} else {
			handlers = append(handlers, log.StderrHandler)
		}
	} else {
		handlers = append(
			handlers,
			log.LvlFilterHandler(
				log.LvlWarn,
				log.StderrHandler,
			),
		)
	}

	Log.SetHandler(log.MultiHandler(handlers...))

	return nil
}
示例#5
0
文件: log.go 项目: RuneTM/lxd
// SetLogger defines the *log.Logger where log messages are sent to.
func SetLogger(syslog string, logfile string, verbose bool, debug bool) error {
	Log = log.New()

	var handlers []log.Handler

	// SyslogHandler
	if syslog != "" {
		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlError,
					log.Must.SyslogHandler(syslog, log.LogfmtFormat()),
				),
			)
		} else {
			handlers = append(handlers, log.Must.SyslogHandler(syslog, log.LogfmtFormat()))
		}
	}

	// FileHandler
	if logfile != "" {
		if !PathExists(filepath.Dir(logfile)) {
			return fmt.Errorf("Log file path doesn't exist: %s\n", filepath.Dir(logfile))
		}

		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlError,
					log.Must.FileHandler(logfile, log.LogfmtFormat()),
				),
			)
		} else {
			handlers = append(handlers, log.Must.FileHandler(logfile, log.LogfmtFormat()))
		}
	}

	// StderrHandler
	if verbose || debug {
		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlError,
					log.StderrHandler,
				),
			)
		} else {
			handlers = append(handlers, log.StderrHandler)
		}
	}

	Log.SetHandler(log.MultiHandler(handlers...))

	return nil
}
示例#6
0
// InitLog ...
func (logM *LogManager) InitLog() {

	var svrlog = log.New(logM.LogContext, logM.LogContext)
	svrlog.SetHandler(log.MultiHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()),
		log.LvlFilterHandler(log.LvlError, log.Must.FileHandler(logM.LogDir+logM.ErrorFile, log.JsonFormat())),
		log.LvlFilterHandler(log.LvlInfo, log.Must.FileHandler(logM.LogDir+logM.InfoFile, log.JsonFormat())),
		log.LvlFilterHandler(log.LvlCrit, log.Must.FileHandler(logM.LogDir+logM.CritFile, log.JsonFormat())),
		log.LvlFilterHandler(log.LvlDebug, log.Must.FileHandler(logM.LogDir+logM.DebugFile, log.JsonFormat()))))

	logM.Logger = svrlog

}
示例#7
0
文件: log.go 项目: joker042/lxd
// SetLogger defines the *log.Logger where log messages are sent to.
func SetLogger(syslog string, logfile string, verbose bool, debug bool) {
	Log = log.New()

	var handlers []log.Handler

	// SyslogHandler
	if syslog != "" {
		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlError,
					log.Must.SyslogHandler(syslog, log.LogfmtFormat()),
				),
			)
		} else {
			handlers = append(handlers, log.Must.SyslogHandler(syslog, log.LogfmtFormat()))
		}
	}

	// FileHandler
	if logfile != "" {
		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlError,
					log.Must.FileHandler(logfile, log.LogfmtFormat()),
				),
			)
		} else {
			handlers = append(handlers, log.Must.FileHandler(logfile, log.LogfmtFormat()))
		}
	}

	// StderrHandler
	if verbose || debug {
		if !debug {
			handlers = append(
				handlers,
				log.LvlFilterHandler(
					log.LvlError,
					log.StderrHandler,
				),
			)
		} else {
			handlers = append(handlers, log.StderrHandler)
		}
	}

	Log.SetHandler(log.MultiHandler(handlers...))
}
示例#8
0
func InitLogger(slvl string) {
	if slvl == "" {
		slvl = "debug"
	}
	lvl, err := log.LvlFromString(slvl)
	if err != nil {
		panic(err)
	}
	Log.SetHandler(log.LvlFilterHandler(lvl, log.StreamHandler(os.Stdout, log.TerminalFormat())))
	return
}
示例#9
0
文件: main.go 项目: Taik/zing-mp3
func main() {
	var (
		zingURL     = flag.String("url", "", "Zing MP3 URL to be parsed")
		downloadDir = flag.String("dir", ".", "Directory to download into")
	)
	flag.Parse()

	zing.Logger.SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StdoutHandler))

	zing.DownloadAlbum(*zingURL, *downloadDir)
}
示例#10
0
func (l *Logger) Initialize(logOut string, lvl interface{}) error {
	var (
		level log.Lvl
		err   error
	)

	if l.Context == "" {
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		randInt := r.Int()
		t := time.Now()
		l.Context = fmt.Sprintf("%s#%d", t.Format(time.RFC3339), randInt)
	}

	if str, ok := lvl.(string); ok {
		level, err = log.LvlFromString(str)
		if err != nil {
			return err
		}
	} else {
		level = lvl.(log.Lvl)
	}
	l.Logger = log.New(log.Ctx{"context": l.Context})
	l.Level = level

	if logOut == "STDOUT" {
		normalHandler := log.LvlFilterHandler(level, log.StdoutHandler)
		errorHandler := log.LvlFilterHandler(level, log.CallerStackHandler("%+v", log.StdoutHandler))
		handler := errorMultiHandler(normalHandler, errorHandler)
		l.SetHandler(handler)
	} else if logOut == "NONE" {
		l.SetHandler(log.DiscardHandler())
	} else {
		fileHandler := log.Must.FileHandler(logOut, log.LogfmtFormat())
		normalHandler := log.LvlFilterHandler(level, fileHandler)
		errorHandler := log.LvlFilterHandler(level, log.CallerStackHandler("%+v", fileHandler))
		handler := errorMultiHandler(normalHandler, errorHandler)
		l.SetHandler(handler)
	}

	return nil
}
示例#11
0
func New() *Engine {
	// package instance of the helios type
	engine := &Engine{
		HTTPEngine: gin.Default(),
		Socket:     initSocket(),
		Logger:     log.New(),
	}

	fileHandler, _ := log.FileHandler("./log.debug", log.LogfmtFormat())
	engine.SetHandler(log.MultiHandler(log.LvlFilterHandler(log.LvlWarn, fileHandler), log.StreamHandler(os.Stdout, log.TerminalFormat())))

	return engine
}
示例#12
0
文件: bench_test.go 项目: yunhor/pgx
func BenchmarkLog15Discard(b *testing.B) {
	logger := log.New()
	lvl, err := log.LvlFromString("error")
	if err != nil {
		b.Fatal(err)
	}
	logger.SetHandler(log.LvlFilterHandler(lvl, log.DiscardHandler()))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		logger.Debug("benchmark", "i", i, "b.N", b.N)
	}
}
示例#13
0
文件: main.go 项目: gadouryd/gologgen
func init() {
	// Set global logging levels by the flag, default to WARN if not defined
	var level string
	flag.StringVar(&level, "level", "WARN", "a string")
	flag.StringVar(&confPath, "conf", "config/gologgen.conf", "a string")

	flag.Parse()

	switch level {
	case "DEBUG":
		log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlDebug, log15.StdoutHandler))
	case "INFO":
		log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlInfo, log15.StdoutHandler))
	case "WARN":
		log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlWarn, log15.StdoutHandler))
	case "ERROR":
		log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.StdoutHandler))
	default:
		log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlWarn, log15.StdoutHandler))
	}

	log = log15.New("function", log15.Lazy{Fn: loghelper.Log15LazyFunctionName})
}
示例#14
0
文件: main.go 项目: jackc/tpr
func setFilterHandler(level string, logger log.Logger, handler log.Handler) error {
	if level == "none" {
		logger.SetHandler(log.DiscardHandler())
		return nil
	}

	lvl, err := log.LvlFromString(level)
	if err != nil {
		return fmt.Errorf("Bad log level: %v", err)
	}
	logger.SetHandler(log.LvlFilterHandler(lvl, handler))

	return nil
}
示例#15
0
func (d *Daemon) Init() error {

	/* Setup the TLS authentication */
	certf, keyf, err := shared.ReadMyCert("server.crt", "server.key")
	if err != nil {
		return err
	}
	d.certf = certf
	d.keyf = keyf
	d.tlsConfig, err = shared.GetTLSConfig(d.certf, d.keyf)
	if err != nil {
		return err
	}

	srvlog := log.New(log.Ctx{"module": "server"})
	d.log = srvlog

	handler := log.StdoutHandler

	if quiet {
		d.log.SetHandler(log.DiscardHandler())
	} else if verbose {
		d.log.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
	} else if debug {
		d.log.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
	} else {
		d.log.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
	}

	err = initializeDbObject(d, "server.db")
	if err != nil {
		return err
	}
	readSavedClientCAList(d)

	return d.Run()
}
示例#16
0
文件: main.go 项目: Taik/zing-mp3
func main() {
	var (
		port = flag.Int("port", 8000, "Port to listen on")
	)
	flag.Parse()
	zing.Logger.SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StdoutHandler))

	go func() {
		http.ListenAndServe("localhost:6060", nil)
	}()

	router := fasthttprouter.New()
	router.GET("/album/", zingAlbumHandler)
	fasthttp.ListenAndServe(fmt.Sprintf(":%d", *port), router.Handler)
}
示例#17
0
// getSystemHandler on Linux writes messages to syslog.
func getSystemHandler(syslog string, debug bool) log.Handler {
	// SyslogHandler
	if syslog != "" {
		if !debug {
			return log.LvlFilterHandler(
				log.LvlInfo,
				log.Must.SyslogHandler(syslog, log.LogfmtFormat()),
			)
		} else {
			return log.Must.SyslogHandler(syslog, log.LogfmtFormat())
		}
	}

	return nil
}
示例#18
0
文件: bench_test.go 项目: yunhor/pgx
func BenchmarkSelectWithLoggingErrorWithLog15(b *testing.B) {
	connConfig := *defaultConnConfig

	logger := log.New()
	lvl, err := log.LvlFromString("error")
	if err != nil {
		b.Fatal(err)
	}
	logger.SetHandler(log.LvlFilterHandler(lvl, log.DiscardHandler()))
	connConfig.Logger = logger
	connConfig.LogLevel = pgx.LogLevelError
	conn := mustConnect(b, connConfig)
	defer closeConn(b, conn)

	benchmarkSelectWithLog(b, conn)
}
示例#19
0
func ConfigureRootLogger() {
	sh := log.StreamHandler(colorable.NewColorableStdout(), log.TerminalFormat())

	logLevel := log.LvlInfo

	if showDebug == true {
		logLevel = log.LvlDebug
	} else {
		_, found := os.LookupEnv("DEBUG")
		if found == true {
			logLevel = log.LvlDebug
		}
	}

	fh := log.LvlFilterHandler(logLevel, sh)
	cfh := log.CallerFileHandler(fh)
	log.Root().SetHandler(cfh)
}
示例#20
0
文件: logger.go 项目: grengojbo/ads
// Start Logger service
func (self *Logger) Start() {
	var err error
	self.Logger = log.New()
	if self.Release {
		self.Level, err = log.LvlFromString("error")
		if err != nil {
			fmt.Println(err)
		}
	} else {
		self.Level, err = log.LvlFromString("debug")
		if err != nil {
			fmt.Println(err)
		}
	}
	self.Logger.SetHandler(log.LvlFilterHandler(self.Level, log.StdoutHandler))
	// self.SetHandler(log.LvlFilterHandler(lvl, log.DiscardHandler()))
	self.Info("starting Logger service...")
}
示例#21
0
func setSettings(level int, filename string) {
	logFilename = filename
	debug = level

	filehandler := log.Must.FileHandler(filename, log.LogfmtFormat())
	handler := log.MultiHandler(
		filehandler,
		log.StreamHandler(os.Stderr, log.TerminalFormat()))
	if debug > 1 {
		handler = log.CallerStackHandler("%+v", handler)
	} else {
		handler = log.CallerFileHandler(handler)
	}
	if debug < 1 {
		handler = log.LvlFilterHandler(log.LvlInfo, handler)
	}
	log.Root().SetHandler(handler)
	log.Info("Logging started", "level", debug, "log_file", logFilename)
}
示例#22
0
func BenchmarkLog15TextNegative(b *testing.B) {
	stream := &blackholeStream{}
	logger := log.New()
	logger.SetHandler(log.LvlFilterHandler(
		log.LvlError,
		log.StreamHandler(stream, log.LogfmtFormat())),
	)
	b.ResetTimer()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			logger.Info("The quick brown fox jumps over the lazy dog")
		}
	})

	if stream.WriteCount() != uint64(0) {
		b.Fatalf("Log write count")
	}
}
示例#23
0
文件: axiom.go 项目: postfix/axiom
func (w *Logged) HandlerFor(target, level, format string) (log.Handler, error) {
	lvl, err := log.LvlFromString(level)
	if err != nil {
		return nil, fmt.Errorf("Invalid log level: %v", err)
	}

	var logformat log.Format
	switch format {
	case "json":
		logformat = log.JsonFormat()
	case "logfmt":
		logformat = log.LogfmtFormat()
	case "terminal", "term":
		switch {
		case target == "stdout" && term.IsTty(os.Stdout.Fd()):
			logformat = log.TerminalFormat()
		case target == "stderr" && term.IsTty(os.Stderr.Fd()):
			logformat = log.TerminalFormat()
		default:
			logformat = log.LogfmtFormat()
		}
	default:
		return nil, fmt.Errorf("Invalid log format: %v", format)
	}

	var handler log.Handler
	switch target {
	case "stdout":
		handler = log.StreamHandler(colorable.NewColorableStdout(), logformat)
	case "stderr":
		handler = log.StreamHandler(colorable.NewColorableStderr(), logformat)
	case "false":
		handler = log.DiscardHandler()
	default:
		handler, err = log.FileHandler(target, logformat)
		if err != nil {
			return nil, fmt.Errorf("Failed to open log file '%s': %v", target, err)
		}
	}

	return log.LvlFilterHandler(lvl, handler), nil
}
示例#24
0
func TestErrorHandler(t *testing.T) {
	t.Parallel()

	h, r := testHandler()
	lg := log.New()
	lg.SetHandler(EscalateErrHandler(
		log.LvlFilterHandler(log.LvlError, h)))

	lg.Debug("some function result", "err", nil)
	if r.Msg != "" {
		t.Fatalf("Expected debug level message to be filtered")
	}

	lg.Debug("some function result", "err", errors.New("failed operation"))
	if r.Msg != "some function result" {
		t.Fatalf("Expected debug level message to be escalated and pass lvlfilter")
	}

	if r.Lvl != log.LvlError {
		t.Fatalf("Expected debug level message to be escalated to LvlError")
	}
}
示例#25
0
func New() *Engine {
	// package instance of the helios type
	engine := &Engine{
		HTTPEngine: gin.Default(),
		Logger:     log.New(),
	}

	fileHandler, _ := log.FileHandler("./error.log", log.LogfmtFormat())
	engine.SetHandler(log.MultiHandler(log.LvlFilterHandler(log.LvlWarn, fileHandler), log.StreamHandler(os.Stdout, log.TerminalFormat())))

	socket, err := initSocket()
	if err != nil {
		engine.Error("Failed to initialize socket server")
	}

	// Handle error cases
	socket.On("error", func(so socketio.Socket, err error) {
		engine.Error("Error on socket.io server", "error", err.Error())
	})

	engine.Socket = socket

	return engine
}
示例#26
0
func main() {
	// Command line arguments
	port := flag.String("port", server.DEFAULT_PORT, "The port on which the server will listen")
	addr := flag.String("addr", server.DEFAULT_ADDR, "The address on which the server will listen. Set to 0.0.0.0 to listen on all addresses.")
	logFile := flag.String("logfile", "", "The filename in which to save the logs. If not specified, the logs are sent to stderr.")
	logLevel := flag.String("loglevel", "info", "The minimum level of log to be written. Possible values are 'crit', 'error', 'warn', 'info' and 'debug'.")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage of ts2-sim-server:
  ts2-sim-server [options...] file

ARGUMENTS:
  file
		The JSON simulation file to load

OPTIONS:
`)
		flag.PrintDefaults()
	}

	flag.Parse()

	// Handle ctrl+c to kill on terminal
	killChan := make(chan os.Signal, 1)
	signal.Notify(killChan, os.Interrupt)

	// Setup logging system
	logger = log.New()
	var outputHandler log.Handler
	if *logFile != "" {
		outputHandler = log.Must.FileHandler(*logFile, log.LogfmtFormat())
	} else {
		outputHandler = log.StderrHandler
	}
	logLvl, err_level := log.LvlFromString(*logLevel)
	if err_level != nil {
		fmt.Fprintf(os.Stderr, "Error: Unknown loglevel\n\n")
		flag.Usage()
		os.Exit(1)
	}
	logger.SetHandler(log.LvlFilterHandler(
		logLvl,
		outputHandler,
	))
	simulation.InitializeLogger(logger)
	server.InitializeLogger(logger)

	// Load the simulation
	if len(flag.Args()) == 0 {
		fmt.Fprintf(os.Stderr, "Error: Please specify a simulation file\n\n")
		flag.Usage()
		os.Exit(1)
	}
	simFile := flag.Arg(0)
	logger.Info("Loading simulation", "file", simFile)

	data, err := ioutil.ReadFile(simFile)
	if err != nil {
		logger.Crit("Unable to read file", "file", simFile, "error", err)
		os.Exit(1)
	}

	var sim simulation.Simulation
	errload := json.Unmarshal(data, &sim)

	if errload != nil {
		logger.Error("Load Error", "file", simFile, "error", errload)
		return
	}
	logger.Info("Simulation loaded", "sim", sim.Options.Title)

	go server.Run(&sim, *addr, *port)

	// Route all messages
	for {
		select {

		case <-killChan:
			// TODO gracefully shutdown things maybe
			logger.Info("Server killed, exiting...")
			os.Exit(0)
		}
	}
}
示例#27
0
// I'm not really sure why this bit is required (and doesn't overwrite what's in main)... I may need to build my own logging library so I can grasp all the particulars
func init() {
	log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.StdoutHandler))
	log = log15.New("function", log15.Lazy{Fn: loghelper.Log15LazyFunctionName})
}
示例#28
0
func level(max lg.Lvl) lg.Logger {
	l := lg.New()
	handler := lg.StreamHandler(os.Stdout, lg.LogfmtFormat())
	l.SetHandler(lg.LvlFilterHandler(max, handler))
	return l
}
示例#29
0
文件: gergle.go 项目: icio/gergle
func main() {
	var maxDepth uint16
	var disallow []string
	var quiet bool
	var verbose bool
	var numConns int
	var zeroBothers bool
	var delay float64
	var longOutput bool

	cmd := &cobra.Command{
		Use:   "gergle URL",
		Short: "Website crawler.",
	}
	cmd.Flags().Uint16VarP(&maxDepth, "depth", "d", 100, "Maximum crawl depth.")
	cmd.Flags().StringSliceVarP(&disallow, "disallow", "i", nil, "Disallowed paths.")
	cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "No logging to stderr.")
	cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output logging.")
	cmd.Flags().IntVarP(&numConns, "connections", "c", 5, "Maximum number of open connections to the server.")
	cmd.Flags().BoolVarP(&zeroBothers, "zero", "", false, "The number of bothers to give about robots.txt. ")
	cmd.Flags().Float64VarP(&delay, "delay", "t", -1, "The number of seconds between requests to the server.")
	cmd.Flags().BoolVarP(&longOutput, "long", "", false, "List all of the links and assets from a page.")

	cmd.RunE = func(cmd *cobra.Command, args []string) error {
		// Configure logging.
		var logLevel log.Lvl
		if verbose && quiet {
			return errors.New("--verbose and --quiet are mutually exclusive options.")
		} else if verbose {
			logLevel = log.LvlDebug
		} else if quiet {
			logLevel = log.LvlCrit
		} else {
			logLevel = log.LvlInfo
		}
		logger.SetHandler(log.LvlFilterHandler(logLevel, log.StderrHandler))

		// Ensure the user provides only a single URL.
		if len(args) < 1 {
			return errors.New("URL argument required.")
		} else if len(args) > 1 {
			return errors.New("Unexpected arguments after URL.")
		}

		// Ensure the user has provided a valid URL.
		initUrl, err := url.Parse(args[0])
		if err != nil || (initUrl.Scheme != "http" && initUrl.Scheme != "https") {
			return errors.New("Expected URL of the form http[s]://...")
		}

		// Prepare the HTTP Client with a series of connections.
		client := &http.Client{Transport: &http.Transport{
			MaxIdleConnsPerHost: numConns,
		}}

		if !zeroBothers {
			// Be a good citizen: fetch the target's preferred defaults.
			robots, err := fetchRobots(client, initUrl)
			if err == nil {
				disallow = append(disallow, readDisallowRules(robots)...)
				if delay < 0 {
					delay = readCrawlDelay(robots)
				}
			} else {
				logger.Info("Failed to fetch robots.txt", "error", err)
			}
		}

		var fetcher Fetcher = &HTTPFetcher{client, &RegexPageParser{}}

		// Rate-limiting.
		if delay > 0 {
			duration := time.Duration(delay * 1e9)
			fetcher = NewRateLimitedFetcher(duration, fetcher)
			logger.Info("Using rate-limiting", "interval", duration)
		}

		// Construct our rules for following links.
		follower := UnanimousFollower{}

		logger.Info("Ignoring external links")
		follower = append(follower, &LocalFollower{})

		if maxDepth >= 0 {
			logger.Info("Ignoring deep links", "maxDepth", maxDepth)
			follower = append(follower, &ShallowFollower{maxDepth})
		}

		if len(disallow) > 0 {
			disallowFollower := NewRobotsDisallowFollower(disallow...)
			logger.Info("Ignoring paths", "disallow", disallowFollower.Rules)
			follower = append(follower, disallowFollower)
		}

		logger.Info("Ignoring previously seen paths")
		follower = append(follower, NewUnseenFollower(initUrl))

		// Crawling.
		pages := make(chan Page, 10)
		go func() {
			crawl(fetcher, initUrl, pages, follower)
			close(pages)
			if stoppable, ok := fetcher.(Stopper); ok {
				stoppable.Stop()
			}
		}()

		// Output.
		for page := range pages {
			fmt.Printf("URL: %s, Depth: %d, Links: %d, Assets: %d\n", page.URL, page.Depth, len(page.Links), len(page.Assets))
			if longOutput {
				for _, link := range page.Links {
					fmt.Printf("- %s: %s\n", link.Type, link.URL)
				}
				for _, link := range page.Assets {
					fmt.Printf("- %s: %s\n", link.Type, link.URL)
				}
			}
		}

		return nil
	}

	cmd.Execute()
}
示例#30
0
文件: main.go 项目: pgpst/tunnel
func main() {
	// Load the configuration
	m := multiconfig.NewWithPath(os.Getenv("config"))
	cfg := &dns.Config{}
	m.MustLoad(cfg)

	// Parse the log level
	lvl, err := log.LvlFromString(cfg.LogLevel)
	if err != nil {
		panic(err)
	}

	// Create a new logger
	l := log.New()
	l.SetHandler(
		log.LvlFilterHandler(lvl, log.StdoutHandler),
	)

	// Start the initialization
	il := l.New("module", "init")
	il.Info("Starting up the application")

	// Parse the Redis connection string
	url := redisurl.Parse(cfg.Redis)

	// Create an options list
	opts := []redis.DialOption{}
	if url.Database != 0 {
		opts = append(opts, redis.DialDatabase(url.Database))
	}
	if url.Password != "" {
		opts = append(opts, redis.DialPassword(url.Password))
	}

	// Verify the DNS setup
	results, err := net.LookupNS(cfg.Domain)
	if err != nil {
		il.Warn("Unable to look up the NS of domain", "domain", cfg.Domain, "error", err.Error())
	}
	if len(results) > 0 {
		found := false
		for _, record := range results {
			if record.Host == cfg.Hostname {
				found = true
			}
		}
		if !found {
			existing := []string{}
			for _, record := range results {
				existing = append(existing, record.Host)
			}

			il.Warn("Invalid NS records for the domain", "domain", cfg.Domain, "records", existing)
		}
	} else {
		il.Warn("No NS records found for domain", "domain", cfg.Domain, "error", err.Error())
	}

	// Dial the server
	rc, err := redis.Dial("tcp", url.Host+":"+strconv.Itoa(url.Port), opts...)
	if err != nil {
		il.Crit("Unable to connect to the Redis server", "error", err.Error())
		return
	}

	// Initialize the DNS server
	ds := &dns.DNS{
		Config: cfg,
		Redis:  rc,
	}
	if err := ds.ListenAndServe(); err != nil {
		il.Crit("Error while listening to the specified port", "error", err.Error())
	}
}