func (d *Daemon) Init() error {
/* Setup the TLS authentication */
certf, keyf, err := shared.ReadMyCert("client.crt", "client.key")
if err != nil {
return err
}
d.certf = certf
d.keyf = keyf
d.tlsConfig, err = shared.GetTLSConfig(d.certf, d.keyf)
if err != nil {
return err
}
srvlog := log.New(log.Ctx{"module": "client"})
d.log = srvlog
handler := log.StdoutHandler
if quiet {
d.log.SetHandler(log.DiscardHandler())
} else if verbose {
d.log.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
} else if debug {
d.log.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
} else {
d.log.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
}
d.Run()
return nil
}
func sockHandler(d *Daemon, w *websocket.Conn) {
r := w.Request()
raddr := r.RemoteAddr
if !d.isTrustedClient(r) {
d.log.Warn("untrusted client connected", log.Ctx{"raddr": raddr})
return
}
d.log.Debug("handing over client connection", log.Ctx{"raddr": raddr})
clog := log.New(log.Ctx{"module": "socks", "raddr": raddr})
handler := log.StdoutHandler
if quiet {
clog.SetHandler(log.DiscardHandler())
} else if verbose {
clog.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
} else if debug {
clog.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
} else {
clog.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
}
client := &clientConnection{
websocket: w,
log: clog,
raddr: raddr,
}
go client.run()
}
func main() {
flag.Parse()
if helpflag {
flag.Usage()
return
}
logger = log.New()
handler := log.StdoutHandler
if quietflag {
logger.SetHandler(log.DiscardHandler())
} else if verboseflag {
logger.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
} else if debugflag {
logger.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
} else {
logger.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
}
ldifstore := &ldif.LdifBackend{
Path: "./ldif",
Log: logger.New(log.Ctx{"type": "backend", "backend": "ldif"}),
}
if err := ldifstore.Start(); err != nil {
logger.Error("error loading backend", log.Ctx{"error": err})
os.Exit(1)
}
//Create a new LDAP Server
server := ldap.NewServer(logger)
fallback := &debug.DebugBackend{
Log: logger.New(log.Ctx{"type": "backend", "backend": "debug"}),
}
//Create routes bindings
routes := newRouter(fallback, logger)
// backend specific routes
routes.Bind(ldifstore).BaseDn("dc=enterprise,dc=org").Label("Bind LDIF")
routes.Search(ldifstore).BaseDn("dc=enterprise,dc=org").Label("Search LDIF")
routes.Add(ldifstore).BaseDn("dc=enterprise,dc=org").Label("Add LDIF")
//Attach routes to server
server.Handle(routes)
// listen on 3389 and serve
go server.ListenAndServe(":6389")
// When CTRL+C, SIGINT and SIGTERM signal occurs
// Then stop server gracefully
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
close(ch)
server.Stop()
}
// SetLogger defines the *log.Logger where log messages are sent to.
func SetLogger(syslog string, logfile string, verbose bool, debug bool) error {
Log = log.New()
var handlers []log.Handler
var syshandler log.Handler
// System specific handler
syshandler = GetSystemHandler(syslog, debug)
if syshandler != nil {
handlers = append(handlers, syshandler)
}
// FileHandler
if logfile != "" {
if !PathExists(filepath.Dir(logfile)) {
return fmt.Errorf("Log file path doesn't exist: %s", filepath.Dir(logfile))
}
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlInfo,
log.Must.FileHandler(logfile, log.LogfmtFormat()),
),
)
} else {
handlers = append(handlers, log.Must.FileHandler(logfile, log.LogfmtFormat()))
}
}
// StderrHandler
if verbose || debug {
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlInfo,
log.StderrHandler,
),
)
} else {
handlers = append(handlers, log.StderrHandler)
}
} else {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlWarn,
log.StderrHandler,
),
)
}
Log.SetHandler(log.MultiHandler(handlers...))
return nil
}
// SetLogger defines the *log.Logger where log messages are sent to.
func SetLogger(syslog string, logfile string, verbose bool, debug bool) error {
Log = log.New()
var handlers []log.Handler
// SyslogHandler
if syslog != "" {
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlError,
log.Must.SyslogHandler(syslog, log.LogfmtFormat()),
),
)
} else {
handlers = append(handlers, log.Must.SyslogHandler(syslog, log.LogfmtFormat()))
}
}
// FileHandler
if logfile != "" {
if !PathExists(filepath.Dir(logfile)) {
return fmt.Errorf("Log file path doesn't exist: %s\n", filepath.Dir(logfile))
}
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlError,
log.Must.FileHandler(logfile, log.LogfmtFormat()),
),
)
} else {
handlers = append(handlers, log.Must.FileHandler(logfile, log.LogfmtFormat()))
}
}
// StderrHandler
if verbose || debug {
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlError,
log.StderrHandler,
),
)
} else {
handlers = append(handlers, log.StderrHandler)
}
}
Log.SetHandler(log.MultiHandler(handlers...))
return nil
}
// InitLog ...
func (logM *LogManager) InitLog() {
var svrlog = log.New(logM.LogContext, logM.LogContext)
svrlog.SetHandler(log.MultiHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()),
log.LvlFilterHandler(log.LvlError, log.Must.FileHandler(logM.LogDir+logM.ErrorFile, log.JsonFormat())),
log.LvlFilterHandler(log.LvlInfo, log.Must.FileHandler(logM.LogDir+logM.InfoFile, log.JsonFormat())),
log.LvlFilterHandler(log.LvlCrit, log.Must.FileHandler(logM.LogDir+logM.CritFile, log.JsonFormat())),
log.LvlFilterHandler(log.LvlDebug, log.Must.FileHandler(logM.LogDir+logM.DebugFile, log.JsonFormat()))))
logM.Logger = svrlog
}
// SetLogger defines the *log.Logger where log messages are sent to.
func SetLogger(syslog string, logfile string, verbose bool, debug bool) {
Log = log.New()
var handlers []log.Handler
// SyslogHandler
if syslog != "" {
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlError,
log.Must.SyslogHandler(syslog, log.LogfmtFormat()),
),
)
} else {
handlers = append(handlers, log.Must.SyslogHandler(syslog, log.LogfmtFormat()))
}
}
// FileHandler
if logfile != "" {
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlError,
log.Must.FileHandler(logfile, log.LogfmtFormat()),
),
)
} else {
handlers = append(handlers, log.Must.FileHandler(logfile, log.LogfmtFormat()))
}
}
// StderrHandler
if verbose || debug {
if !debug {
handlers = append(
handlers,
log.LvlFilterHandler(
log.LvlError,
log.StderrHandler,
),
)
} else {
handlers = append(handlers, log.StderrHandler)
}
}
Log.SetHandler(log.MultiHandler(handlers...))
}
func InitLogger(slvl string) {
if slvl == "" {
slvl = "debug"
}
lvl, err := log.LvlFromString(slvl)
if err != nil {
panic(err)
}
Log.SetHandler(log.LvlFilterHandler(lvl, log.StreamHandler(os.Stdout, log.TerminalFormat())))
return
}
func main() {
var (
zingURL = flag.String("url", "", "Zing MP3 URL to be parsed")
downloadDir = flag.String("dir", ".", "Directory to download into")
)
flag.Parse()
zing.Logger.SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StdoutHandler))
zing.DownloadAlbum(*zingURL, *downloadDir)
}
func (l *Logger) Initialize(logOut string, lvl interface{}) error {
var (
level log.Lvl
err error
)
if l.Context == "" {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
randInt := r.Int()
t := time.Now()
l.Context = fmt.Sprintf("%s#%d", t.Format(time.RFC3339), randInt)
}
if str, ok := lvl.(string); ok {
level, err = log.LvlFromString(str)
if err != nil {
return err
}
} else {
level = lvl.(log.Lvl)
}
l.Logger = log.New(log.Ctx{"context": l.Context})
l.Level = level
if logOut == "STDOUT" {
normalHandler := log.LvlFilterHandler(level, log.StdoutHandler)
errorHandler := log.LvlFilterHandler(level, log.CallerStackHandler("%+v", log.StdoutHandler))
handler := errorMultiHandler(normalHandler, errorHandler)
l.SetHandler(handler)
} else if logOut == "NONE" {
l.SetHandler(log.DiscardHandler())
} else {
fileHandler := log.Must.FileHandler(logOut, log.LogfmtFormat())
normalHandler := log.LvlFilterHandler(level, fileHandler)
errorHandler := log.LvlFilterHandler(level, log.CallerStackHandler("%+v", fileHandler))
handler := errorMultiHandler(normalHandler, errorHandler)
l.SetHandler(handler)
}
return nil
}
func New() *Engine {
// package instance of the helios type
engine := &Engine{
HTTPEngine: gin.Default(),
Socket: initSocket(),
Logger: log.New(),
}
fileHandler, _ := log.FileHandler("./log.debug", log.LogfmtFormat())
engine.SetHandler(log.MultiHandler(log.LvlFilterHandler(log.LvlWarn, fileHandler), log.StreamHandler(os.Stdout, log.TerminalFormat())))
return engine
}
func BenchmarkLog15Discard(b *testing.B) {
logger := log.New()
lvl, err := log.LvlFromString("error")
if err != nil {
b.Fatal(err)
}
logger.SetHandler(log.LvlFilterHandler(lvl, log.DiscardHandler()))
b.ResetTimer()
for i := 0; i < b.N; i++ {
logger.Debug("benchmark", "i", i, "b.N", b.N)
}
}
func init() {
// Set global logging levels by the flag, default to WARN if not defined
var level string
flag.StringVar(&level, "level", "WARN", "a string")
flag.StringVar(&confPath, "conf", "config/gologgen.conf", "a string")
flag.Parse()
switch level {
case "DEBUG":
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlDebug, log15.StdoutHandler))
case "INFO":
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlInfo, log15.StdoutHandler))
case "WARN":
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlWarn, log15.StdoutHandler))
case "ERROR":
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.StdoutHandler))
default:
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlWarn, log15.StdoutHandler))
}
log = log15.New("function", log15.Lazy{Fn: loghelper.Log15LazyFunctionName})
}
func setFilterHandler(level string, logger log.Logger, handler log.Handler) error {
if level == "none" {
logger.SetHandler(log.DiscardHandler())
return nil
}
lvl, err := log.LvlFromString(level)
if err != nil {
return fmt.Errorf("Bad log level: %v", err)
}
logger.SetHandler(log.LvlFilterHandler(lvl, handler))
return nil
}
func (d *Daemon) Init() error {
/* Setup the TLS authentication */
certf, keyf, err := shared.ReadMyCert("server.crt", "server.key")
if err != nil {
return err
}
d.certf = certf
d.keyf = keyf
d.tlsConfig, err = shared.GetTLSConfig(d.certf, d.keyf)
if err != nil {
return err
}
srvlog := log.New(log.Ctx{"module": "server"})
d.log = srvlog
handler := log.StdoutHandler
if quiet {
d.log.SetHandler(log.DiscardHandler())
} else if verbose {
d.log.SetHandler(log.LvlFilterHandler(log.LvlInfo, handler))
} else if debug {
d.log.SetHandler(log.LvlFilterHandler(log.LvlDebug, handler))
} else {
d.log.SetHandler(log.LvlFilterHandler(log.LvlError, handler))
}
err = initializeDbObject(d, "server.db")
if err != nil {
return err
}
readSavedClientCAList(d)
return d.Run()
}
func main() {
var (
port = flag.Int("port", 8000, "Port to listen on")
)
flag.Parse()
zing.Logger.SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StdoutHandler))
go func() {
http.ListenAndServe("localhost:6060", nil)
}()
router := fasthttprouter.New()
router.GET("/album/", zingAlbumHandler)
fasthttp.ListenAndServe(fmt.Sprintf(":%d", *port), router.Handler)
}
// getSystemHandler on Linux writes messages to syslog.
func getSystemHandler(syslog string, debug bool) log.Handler {
// SyslogHandler
if syslog != "" {
if !debug {
return log.LvlFilterHandler(
log.LvlInfo,
log.Must.SyslogHandler(syslog, log.LogfmtFormat()),
)
} else {
return log.Must.SyslogHandler(syslog, log.LogfmtFormat())
}
}
return nil
}
func BenchmarkSelectWithLoggingErrorWithLog15(b *testing.B) {
connConfig := *defaultConnConfig
logger := log.New()
lvl, err := log.LvlFromString("error")
if err != nil {
b.Fatal(err)
}
logger.SetHandler(log.LvlFilterHandler(lvl, log.DiscardHandler()))
connConfig.Logger = logger
connConfig.LogLevel = pgx.LogLevelError
conn := mustConnect(b, connConfig)
defer closeConn(b, conn)
benchmarkSelectWithLog(b, conn)
}
func ConfigureRootLogger() {
sh := log.StreamHandler(colorable.NewColorableStdout(), log.TerminalFormat())
logLevel := log.LvlInfo
if showDebug == true {
logLevel = log.LvlDebug
} else {
_, found := os.LookupEnv("DEBUG")
if found == true {
logLevel = log.LvlDebug
}
}
fh := log.LvlFilterHandler(logLevel, sh)
cfh := log.CallerFileHandler(fh)
log.Root().SetHandler(cfh)
}
// Start Logger service
func (self *Logger) Start() {
var err error
self.Logger = log.New()
if self.Release {
self.Level, err = log.LvlFromString("error")
if err != nil {
fmt.Println(err)
}
} else {
self.Level, err = log.LvlFromString("debug")
if err != nil {
fmt.Println(err)
}
}
self.Logger.SetHandler(log.LvlFilterHandler(self.Level, log.StdoutHandler))
// self.SetHandler(log.LvlFilterHandler(lvl, log.DiscardHandler()))
self.Info("starting Logger service...")
}
func setSettings(level int, filename string) {
logFilename = filename
debug = level
filehandler := log.Must.FileHandler(filename, log.LogfmtFormat())
handler := log.MultiHandler(
filehandler,
log.StreamHandler(os.Stderr, log.TerminalFormat()))
if debug > 1 {
handler = log.CallerStackHandler("%+v", handler)
} else {
handler = log.CallerFileHandler(handler)
}
if debug < 1 {
handler = log.LvlFilterHandler(log.LvlInfo, handler)
}
log.Root().SetHandler(handler)
log.Info("Logging started", "level", debug, "log_file", logFilename)
}
func BenchmarkLog15TextNegative(b *testing.B) {
stream := &blackholeStream{}
logger := log.New()
logger.SetHandler(log.LvlFilterHandler(
log.LvlError,
log.StreamHandler(stream, log.LogfmtFormat())),
)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
logger.Info("The quick brown fox jumps over the lazy dog")
}
})
if stream.WriteCount() != uint64(0) {
b.Fatalf("Log write count")
}
}
func (w *Logged) HandlerFor(target, level, format string) (log.Handler, error) {
lvl, err := log.LvlFromString(level)
if err != nil {
return nil, fmt.Errorf("Invalid log level: %v", err)
}
var logformat log.Format
switch format {
case "json":
logformat = log.JsonFormat()
case "logfmt":
logformat = log.LogfmtFormat()
case "terminal", "term":
switch {
case target == "stdout" && term.IsTty(os.Stdout.Fd()):
logformat = log.TerminalFormat()
case target == "stderr" && term.IsTty(os.Stderr.Fd()):
logformat = log.TerminalFormat()
default:
logformat = log.LogfmtFormat()
}
default:
return nil, fmt.Errorf("Invalid log format: %v", format)
}
var handler log.Handler
switch target {
case "stdout":
handler = log.StreamHandler(colorable.NewColorableStdout(), logformat)
case "stderr":
handler = log.StreamHandler(colorable.NewColorableStderr(), logformat)
case "false":
handler = log.DiscardHandler()
default:
handler, err = log.FileHandler(target, logformat)
if err != nil {
return nil, fmt.Errorf("Failed to open log file '%s': %v", target, err)
}
}
return log.LvlFilterHandler(lvl, handler), nil
}
func TestErrorHandler(t *testing.T) {
t.Parallel()
h, r := testHandler()
lg := log.New()
lg.SetHandler(EscalateErrHandler(
log.LvlFilterHandler(log.LvlError, h)))
lg.Debug("some function result", "err", nil)
if r.Msg != "" {
t.Fatalf("Expected debug level message to be filtered")
}
lg.Debug("some function result", "err", errors.New("failed operation"))
if r.Msg != "some function result" {
t.Fatalf("Expected debug level message to be escalated and pass lvlfilter")
}
if r.Lvl != log.LvlError {
t.Fatalf("Expected debug level message to be escalated to LvlError")
}
}
func New() *Engine {
// package instance of the helios type
engine := &Engine{
HTTPEngine: gin.Default(),
Logger: log.New(),
}
fileHandler, _ := log.FileHandler("./error.log", log.LogfmtFormat())
engine.SetHandler(log.MultiHandler(log.LvlFilterHandler(log.LvlWarn, fileHandler), log.StreamHandler(os.Stdout, log.TerminalFormat())))
socket, err := initSocket()
if err != nil {
engine.Error("Failed to initialize socket server")
}
// Handle error cases
socket.On("error", func(so socketio.Socket, err error) {
engine.Error("Error on socket.io server", "error", err.Error())
})
engine.Socket = socket
return engine
}
func main() {
// Command line arguments
port := flag.String("port", server.DEFAULT_PORT, "The port on which the server will listen")
addr := flag.String("addr", server.DEFAULT_ADDR, "The address on which the server will listen. Set to 0.0.0.0 to listen on all addresses.")
logFile := flag.String("logfile", "", "The filename in which to save the logs. If not specified, the logs are sent to stderr.")
logLevel := flag.String("loglevel", "info", "The minimum level of log to be written. Possible values are 'crit', 'error', 'warn', 'info' and 'debug'.")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, `Usage of ts2-sim-server:
ts2-sim-server [options...] file
ARGUMENTS:
file
The JSON simulation file to load
OPTIONS:
`)
flag.PrintDefaults()
}
flag.Parse()
// Handle ctrl+c to kill on terminal
killChan := make(chan os.Signal, 1)
signal.Notify(killChan, os.Interrupt)
// Setup logging system
logger = log.New()
var outputHandler log.Handler
if *logFile != "" {
outputHandler = log.Must.FileHandler(*logFile, log.LogfmtFormat())
} else {
outputHandler = log.StderrHandler
}
logLvl, err_level := log.LvlFromString(*logLevel)
if err_level != nil {
fmt.Fprintf(os.Stderr, "Error: Unknown loglevel\n\n")
flag.Usage()
os.Exit(1)
}
logger.SetHandler(log.LvlFilterHandler(
logLvl,
outputHandler,
))
simulation.InitializeLogger(logger)
server.InitializeLogger(logger)
// Load the simulation
if len(flag.Args()) == 0 {
fmt.Fprintf(os.Stderr, "Error: Please specify a simulation file\n\n")
flag.Usage()
os.Exit(1)
}
simFile := flag.Arg(0)
logger.Info("Loading simulation", "file", simFile)
data, err := ioutil.ReadFile(simFile)
if err != nil {
logger.Crit("Unable to read file", "file", simFile, "error", err)
os.Exit(1)
}
var sim simulation.Simulation
errload := json.Unmarshal(data, &sim)
if errload != nil {
logger.Error("Load Error", "file", simFile, "error", errload)
return
}
logger.Info("Simulation loaded", "sim", sim.Options.Title)
go server.Run(&sim, *addr, *port)
// Route all messages
for {
select {
case <-killChan:
// TODO gracefully shutdown things maybe
logger.Info("Server killed, exiting...")
os.Exit(0)
}
}
}
// I'm not really sure why this bit is required (and doesn't overwrite what's in main)... I may need to build my own logging library so I can grasp all the particulars
func init() {
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.StdoutHandler))
log = log15.New("function", log15.Lazy{Fn: loghelper.Log15LazyFunctionName})
}
func level(max lg.Lvl) lg.Logger {
l := lg.New()
handler := lg.StreamHandler(os.Stdout, lg.LogfmtFormat())
l.SetHandler(lg.LvlFilterHandler(max, handler))
return l
}
func main() {
var maxDepth uint16
var disallow []string
var quiet bool
var verbose bool
var numConns int
var zeroBothers bool
var delay float64
var longOutput bool
cmd := &cobra.Command{
Use: "gergle URL",
Short: "Website crawler.",
}
cmd.Flags().Uint16VarP(&maxDepth, "depth", "d", 100, "Maximum crawl depth.")
cmd.Flags().StringSliceVarP(&disallow, "disallow", "i", nil, "Disallowed paths.")
cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "No logging to stderr.")
cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output logging.")
cmd.Flags().IntVarP(&numConns, "connections", "c", 5, "Maximum number of open connections to the server.")
cmd.Flags().BoolVarP(&zeroBothers, "zero", "", false, "The number of bothers to give about robots.txt. ")
cmd.Flags().Float64VarP(&delay, "delay", "t", -1, "The number of seconds between requests to the server.")
cmd.Flags().BoolVarP(&longOutput, "long", "", false, "List all of the links and assets from a page.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
// Configure logging.
var logLevel log.Lvl
if verbose && quiet {
return errors.New("--verbose and --quiet are mutually exclusive options.")
} else if verbose {
logLevel = log.LvlDebug
} else if quiet {
logLevel = log.LvlCrit
} else {
logLevel = log.LvlInfo
}
logger.SetHandler(log.LvlFilterHandler(logLevel, log.StderrHandler))
// Ensure the user provides only a single URL.
if len(args) < 1 {
return errors.New("URL argument required.")
} else if len(args) > 1 {
return errors.New("Unexpected arguments after URL.")
}
// Ensure the user has provided a valid URL.
initUrl, err := url.Parse(args[0])
if err != nil || (initUrl.Scheme != "http" && initUrl.Scheme != "https") {
return errors.New("Expected URL of the form http[s]://...")
}
// Prepare the HTTP Client with a series of connections.
client := &http.Client{Transport: &http.Transport{
MaxIdleConnsPerHost: numConns,
}}
if !zeroBothers {
// Be a good citizen: fetch the target's preferred defaults.
robots, err := fetchRobots(client, initUrl)
if err == nil {
disallow = append(disallow, readDisallowRules(robots)...)
if delay < 0 {
delay = readCrawlDelay(robots)
}
} else {
logger.Info("Failed to fetch robots.txt", "error", err)
}
}
var fetcher Fetcher = &HTTPFetcher{client, &RegexPageParser{}}
// Rate-limiting.
if delay > 0 {
duration := time.Duration(delay * 1e9)
fetcher = NewRateLimitedFetcher(duration, fetcher)
logger.Info("Using rate-limiting", "interval", duration)
}
// Construct our rules for following links.
follower := UnanimousFollower{}
logger.Info("Ignoring external links")
follower = append(follower, &LocalFollower{})
if maxDepth >= 0 {
logger.Info("Ignoring deep links", "maxDepth", maxDepth)
follower = append(follower, &ShallowFollower{maxDepth})
}
if len(disallow) > 0 {
disallowFollower := NewRobotsDisallowFollower(disallow...)
logger.Info("Ignoring paths", "disallow", disallowFollower.Rules)
follower = append(follower, disallowFollower)
}
logger.Info("Ignoring previously seen paths")
follower = append(follower, NewUnseenFollower(initUrl))
// Crawling.
pages := make(chan Page, 10)
go func() {
crawl(fetcher, initUrl, pages, follower)
close(pages)
if stoppable, ok := fetcher.(Stopper); ok {
stoppable.Stop()
}
}()
// Output.
for page := range pages {
fmt.Printf("URL: %s, Depth: %d, Links: %d, Assets: %d\n", page.URL, page.Depth, len(page.Links), len(page.Assets))
if longOutput {
for _, link := range page.Links {
fmt.Printf("- %s: %s\n", link.Type, link.URL)
}
for _, link := range page.Assets {
fmt.Printf("- %s: %s\n", link.Type, link.URL)
}
}
}
return nil
}
cmd.Execute()
}
func main() {
// Load the configuration
m := multiconfig.NewWithPath(os.Getenv("config"))
cfg := &dns.Config{}
m.MustLoad(cfg)
// Parse the log level
lvl, err := log.LvlFromString(cfg.LogLevel)
if err != nil {
panic(err)
}
// Create a new logger
l := log.New()
l.SetHandler(
log.LvlFilterHandler(lvl, log.StdoutHandler),
)
// Start the initialization
il := l.New("module", "init")
il.Info("Starting up the application")
// Parse the Redis connection string
url := redisurl.Parse(cfg.Redis)
// Create an options list
opts := []redis.DialOption{}
if url.Database != 0 {
opts = append(opts, redis.DialDatabase(url.Database))
}
if url.Password != "" {
opts = append(opts, redis.DialPassword(url.Password))
}
// Verify the DNS setup
results, err := net.LookupNS(cfg.Domain)
if err != nil {
il.Warn("Unable to look up the NS of domain", "domain", cfg.Domain, "error", err.Error())
}
if len(results) > 0 {
found := false
for _, record := range results {
if record.Host == cfg.Hostname {
found = true
}
}
if !found {
existing := []string{}
for _, record := range results {
existing = append(existing, record.Host)
}
il.Warn("Invalid NS records for the domain", "domain", cfg.Domain, "records", existing)
}
} else {
il.Warn("No NS records found for domain", "domain", cfg.Domain, "error", err.Error())
}
// Dial the server
rc, err := redis.Dial("tcp", url.Host+":"+strconv.Itoa(url.Port), opts...)
if err != nil {
il.Crit("Unable to connect to the Redis server", "error", err.Error())
return
}
// Initialize the DNS server
ds := &dns.DNS{
Config: cfg,
Redis: rc,
}
if err := ds.ListenAndServe(); err != nil {
il.Crit("Error while listening to the specified port", "error", err.Error())
}
}