func Test_changefeed(t *testing.T) {
s := storage.NewMemStorage()
tests := []changefeedTest{
{
name: "Empty Store",
args: changefeedArgs{
logger: logrus.New(),
store: s,
imageName: "",
changeID: "0",
pageSize: notary.DefaultPageSize,
},
want: []byte("{\"count\":0,\"records\":null}"),
wantErr: false,
},
{
name: "Bad ChangeID",
args: changefeedArgs{
logger: logrus.New(),
store: s,
imageName: "",
changeID: "not_a_number",
pageSize: notary.DefaultPageSize,
},
want: nil,
wantErr: true,
},
}
runChangefeedTests(t, tests)
}
func TestClient(t *testing.T) {
assert.Panics(t, func() {
NewAPIMock(true, logrus.New(), "panic!")
}, "It must panic if wrong type")
api := NewAPIMock(true, logrus.New(), "json")
assert.IsType(t, api, new(APIMock), "It must instance an APIMock struct")
assert.Equal(t, api.CORSEnabled, true, "Cors must be TRUE")
assert.Equal(t, api.Log, logrus.New(), "Logger must be set correctly")
assert.Equal(t, api.Type, "json", "Type must be 'json'")
}
// NewOptionsFromConfig extracts bits from a config to make an agent options object
func NewOptionsFromConfig(cfg *config.Config) *Options {
log := logrus.New()
opts := &Options{
Env: cfg.Env,
Log: logrus.New(),
}
if cfg.LogFormat == "json" {
log.Formatter = &logrus.JSONFormatter{}
}
opts.Log = Logger(log)
return opts
}
func TestRecoverMiddleware(t *testing.T) {
req := &protocol.Request{
Method: "foobar",
Params: protocol.Params{},
Header: header.Header{},
}
rw := &dummy.ResponseWriter{
HeaderValues: header.Header{},
}
buff := bytes.NewBufferString("")
logger := log.New()
logger.Out = buff
handler := Chain(panicHandler, NewRecover(logger, true, 8*1024))
handler.Handle(rw, req)
loglines := buff.String()
expectedLog := regexp.MustCompile(`error="Booom"`)
if expectedLog.FindString(loglines) != "" {
t.Errorf("didn't find expected log line in %s", loglines)
}
}
func main() {
log := logrus.New()
cli.VersionPrinter = func(c *cli.Context) {
fmt.Printf("%s v=%s d=%s\n", c.App.Name, c.App.Version, GeneratedString)
}
app := cli.NewApp()
app.Name = "artifacts-service"
app.Version = VersionString
app.Commands = []cli.Command{
{
Name: "serve",
ShortName: "s",
Usage: "run the HTTP thing",
Action: func(_ *cli.Context) {
server.Main(log)
},
},
{
Name: "migrate",
ShortName: "m",
Usage: "run database migrations",
Action: func(_ *cli.Context) {
server.MigratorMain(log)
},
},
}
app.Run(os.Args)
}
func main() {
r := gin.New()
// Add a ginrus middleware, which:
// - Logs all requests, like a combined access and error log.
// - Logs to stdout.
// - RFC3339 with UTC time format.
r.Use(ginrus.Ginrus(logrus.StandardLogger(), time.RFC3339, true))
// Add similar middleware, but:
// - Only logs requests with errors, like an error log.
// - Logs to stderr instead of stdout.
// - Local time zone instead of UTC.
logger := logrus.New()
logger.Level = logrus.ErrorLevel
logger.Out = os.Stderr
r.Use(ginrus.Ginrus(logger, time.RFC3339, false))
// Example ping request.
r.GET("/ping", func(c *gin.Context) {
c.String(200, "pong "+fmt.Sprint(time.Now().Unix()))
})
// Listen and Server in 0.0.0.0:8080
r.Run(":8080")
}
func main() {
log := logrus.New()
hook, err := logrus_influxdb.NewInfluxDB(nil)
if err == nil {
log.Hooks.Add(hook)
}
}
func Instance() *log.Logger {
if logger == nil {
logger = log.New()
}
return logger
}
func (h *TeeHook) Fire(entry *logrus.Entry) error {
logger := logrus.New()
logger.Out = h.W
switch entry.Level {
case logrus.PanicLevel:
logger.WithFields(entry.Data).Panic(entry.Message)
break
case logrus.FatalLevel:
logger.WithFields(entry.Data).Fatal(entry.Message)
break
case logrus.ErrorLevel:
logger.WithFields(entry.Data).Error(entry.Message)
break
case logrus.WarnLevel:
logger.WithFields(entry.Data).Warnf(entry.Message)
break
case logrus.InfoLevel:
logger.WithFields(entry.Data).Info(entry.Message)
break
case logrus.DebugLevel:
logger.WithFields(entry.Data).Info(entry.Message)
break
}
return nil
}
func (svc *service) Run() (string, error) {
logger := logrus.New()
err := setupSyslog(logger)
if err != nil {
return "Could not create syslog", err
}
// load configuration file
config, err := config.LoadFromFile(svc.args["<config-file>"].(string))
if err != nil {
logger.WithError(err).Error("Failed to open configuration file")
return "Failed to open configuration file", err
}
w, err := worker.New(config, logger)
if err != nil {
logger.WithError(err).Error("Could not create worker")
return "Could not create worker", err
}
sigTerm := make(chan os.Signal, 1)
signal.Notify(sigTerm, os.Interrupt, os.Kill, syscall.SIGTERM)
go func() {
<-sigTerm
w.Stop()
}()
w.Start()
return "Worker successfully started", nil
}
func main() {
log := logrus.New()
log.Formatter = new(logrus.JSONFormatter)
for {
log.WithFields(logrus.Fields{
"animal": "walrus",
"size": "10",
}).Print("A group of walrus emerges from the ocean")
log.WithFields(logrus.Fields{
"omg": true,
"number": 122,
}).Warn("The group's number increased tremendously!")
log.WithFields(logrus.Fields{
"animal": "walrus",
"size": "10",
}).Print("A giant walrus appears!")
log.WithFields(logrus.Fields{
"animal": "walrus",
"size": "9",
}).Print("Tremendously sized cow enters the ocean.")
log.WithFields(logrus.Fields{
"omg": true,
"number": 100,
}).Fatal("The ice breaks!")
}
}
func New(concurrentRequests int, hosts []string, startUrls []string) *Crawler {
c := &Crawler{
ConcurrentRequests: concurrentRequests,
AllowCrawlPatterns: make([]*regexp.Regexp, 0),
Logger: logrus.New(),
Client: &http.Client{},
itemQueue: make([]*CrawlItem, 0),
crawledUrls: make(map[string]bool),
pendingUrls: make(map[string]bool),
resultChannel: make(chan *CrawlItem),
}
for _, host := range hosts {
re := regexp.MustCompile("https?\\://" + host + "*")
c.AllowCrawlPatterns = append(c.AllowCrawlPatterns, re)
}
for _, url := range startUrls {
c.itemQueue = append(c.itemQueue, &CrawlItem{
Url: url,
})
}
return c
}
func main() {
var log = logrus.New()
app := cli.NewApp()
app.Name = "Geopardy"
app.Usage = "Import and query Jeopardy questions"
app.Action = func(c *cli.Context) {
log.Out = os.Stderr
log.WithFields(logrus.Fields{
"args": c.Args(),
}).Info("Called with args")
}
app.Commands = []cli.Command{
{
Name: "import",
Usage: "Import Jeopardy questions from the web to the database",
Action: func(c *cli.Context) {
err := imp.RunImport(c.Args())
if err != nil {
log.Error(err)
}
},
},
}
app.Run(os.Args)
}
func newRedisNotify(accountID string) (*logrus.Logger, error) {
rNotify := serverConfig.GetRedisNotifyByID(accountID)
// Dial redis.
rPool, err := dialRedis(rNotify)
if err != nil {
return nil, err
}
rrConn := redisConn{
Pool: rPool,
params: rNotify,
}
redisLog := logrus.New()
redisLog.Out = ioutil.Discard
// Set default JSON formatter.
redisLog.Formatter = new(logrus.JSONFormatter)
redisLog.Hooks.Add(rrConn)
// Success, redis enabled.
return redisLog, nil
}
// NewLogger returns a log configured with the input parameters.
func NewLogger(level, writer string, maxSize, maxAge, maxBackups int) *logrus.Entry {
base := logrus.New()
switch writer {
case "stdout":
base.Out = os.Stdout
case "stderr":
base.Out = os.Stderr
case "":
base.Out = ioutil.Discard
default:
base.Out = &lumberjack.Logger{
Filename: writer,
MaxSize: maxSize,
MaxAge: maxAge,
MaxBackups: maxBackups,
}
}
logrusLevel, err := logrus.ParseLevel(level)
// if provided level is not supported, default to Info level
if err != nil {
base.Error(err)
logrusLevel = logrus.InfoLevel
}
base.Level = logrusLevel
log := logrus.NewEntry(base)
return log
}
func init() {
logger := logrus.New()
logger.Level = logrus.InfoLevel
Main = logger
}
// Tests that writing to a tempfile log works.
// Matches the 'msg' of the output and deletes the tempfile.
func TestLogEntryWritten(t *testing.T) {
log := logrus.New()
// The colors were messing with the regexp so I turned them off.
log.Formatter = &logrus.TextFormatter{DisableColors: true}
tmpfile, err := ioutil.TempFile("", "test_lfshook")
if err != nil {
t.Errorf("Unable to generate logfile due to err: %s", err)
}
fname := tmpfile.Name()
defer func() {
tmpfile.Close()
os.Remove(fname)
}()
hook := NewHook(PathMap{
logrus.InfoLevel: fname,
})
log.Hooks.Add(hook)
log.Info(expectedMsg)
if contents, err := ioutil.ReadAll(tmpfile); err != nil {
t.Errorf("Error while reading from tmpfile: %s", err)
} else if matched, err := regexp.Match("msg=\""+expectedMsg+"\"", contents); err != nil || !matched {
t.Errorf("Message read (%s) doesnt match message written (%s) for file: %s", contents, expectedMsg, fname)
}
}
// NewWithArgs returns a new CLI using the specified arguments.
func NewWithArgs(ctx apitypes.Context, a ...string) *CLI {
s := "REX-Ray:\n" +
" A guest-based storage introspection tool that enables local\n" +
" visibility and management from cloud and storage platforms."
c := &CLI{
l: log.New(),
ctx: ctx,
config: util.NewConfig(ctx),
}
c.c = &cobra.Command{
Use: "rexray",
Short: s,
PersistentPreRun: c.preRun,
Run: func(cmd *cobra.Command, args []string) {
cmd.Usage()
},
}
c.c.SetArgs(a)
for _, f := range initCmdFuncs {
f(c)
}
c.initUsageTemplates()
return c
}
func getPanicLogger() *logrus.Logger {
log := logrus.New()
if os.Getenv("DEBUG") != "" && os.Getenv("ARTIFACTS_DEBUG") != "" {
log.Level = logrus.DebugLevel
}
return log
}
// Run the application, start http and scp server.
func Run(appConfig Config) {
config = appConfig
// Connect to DB
db, err := lib.Connect()
if err != nil {
fmt.Printf("Failed to connect to db")
return
}
// Logging
log := logrus.New()
log.Level = logrus.DebugLevel
log.Out = os.Stdout
log.Formatter = &logrus.TextFormatter{}
// Websockets
ws := ws.NewServer()
// Shared dependencies between all controller
deps := dependencies.Dependencies{
Fs: afero.NewOsFs(),
Logger: log,
DB: db,
WS: ws,
}
ws.Dependencies = &deps
go ws.Start()
// // Start SCP
// scp := scp.Server{}
// scp.DB = deps.DB
// scp.Logger = deps.Logger
// scp.CertPath = "certs/scp.rsa"
// scp.BindAddr = config.SCPBindAddr
// go scp.ListenAndServe()
if config.Secure {
c := autocert.DirCache("certs")
m := autocert.Manager{
Cache: c,
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist("x.zqz.ca"),
}
s := &http.Server{
Addr: config.HTTPBindAddr,
TLSConfig: &tls.Config{GetCertificate: m.GetCertificate},
}
deps.Info("Listening for HTTP1.1 Connections", "addr", ":3001")
deps.Info("Listening for HTTP2 Connections", "addr", config.HTTPBindAddr)
go http.ListenAndServe(":3001", secureRedirect())
s.ListenAndServeTLS("", "")
} else {
deps.Info("Listening for HTTP1.1 Connections", "addr", config.HTTPBindAddr)
http.ListenAndServe(config.HTTPBindAddr, Routes(deps))
}
}
func NewCLI(c *cli.Context) *CLI {
log := logrus.New()
log.Level = logrus.InfoLevel
log.Formatter = &logrus.TextFormatter{DisableColors: true}
return &CLI{c: c, log: log}
}
func (h *JSON) WriteErrorCode(ctx context.Context, w http.ResponseWriter, r *http.Request, code int, err error) {
id, _ := ctx.Value(RequestIDKey).(string)
if id == "" {
id = uuid.New()
}
if h.Logger == nil {
h.Logger = logrus.New()
}
if e, ok := err.(*Error); ok {
h.Logger.WithError(e).WithField("request_id", id).WithField("status", code).WithField("stack", e.Err.ErrorStack()).Printf("Got error.")
} else if e, ok := err.(*errors.Error); ok {
h.Logger.WithError(e).WithField("request_id", id).WithField("status", code).WithField("stack", e.ErrorStack()).Printf("Got error.")
} else {
h.Logger.WithError(err).WithField("request_id", id).WithField("status", code).Printf("Got error.")
}
if code == 0 {
code = http.StatusInternalServerError
}
h.WriteCode(ctx, w, r, code, &jsonError{
RequestID: id,
Error: err.Error(),
Code: code,
})
}
func NewLogger(level logrus.Level, formatter logrus.Formatter) *Logger {
log := logrus.New()
log.Level = level
log.Formatter = formatter
return &Logger{Logger: log}
}
func init() {
// do something here to set environment depending on an environment variable
// or command-line flag
if Environment == "debug" {
//logger := log.New(os.Stdout, "logger: ", log.Lshortfile)
log.SetPrefix("logger: ")
log.SetFlags(log.Lshortfile)
} else {
logger = logrus.New()
if Environment == "production" {
// Log as JSON instead of the default ASCII formatter.
//log.SetFormatter(&log.JSONFormatter{})
logger.Formatter = &logrus.JSONFormatter{}
// Output to stderr instead of stdout, could also be a file.
//log.SetOutput(os.Stderr)
// Only log the warning severity or above.
//log.SetLevel(log.WarnLevel)
logger.Level = logrus.WarnLevel
} else {
// The TextFormatter is default, you don't actually have to do this.
//log.SetFormatter(&log.TextFormatter{})
//log.SetOutput(os.Stderr)
//log.SetLevel(log.InfoLevel)
//logger.Formatter = &logrus.TextFormatter{}
//logger.Level = log.InfoLevel
//logger.Out = os.Stderr
}
}
}
// NewCustomMiddleware builds a *Middleware with the given level and formatter
func NewCustomMiddleware(level logrus.Level, formatter logrus.Formatter) *Middleware {
log := logrus.New()
log.Level = level
log.Formatter = formatter
return &Middleware{Logger: log}
}
// NewCustomMiddleware builds a *Middleware with the given level and formatter
func NewCustomMiddleware(level logrus.Level, formatter logrus.Formatter, name string) *Middleware {
log := logrus.New()
log.Level = level
log.Formatter = formatter
return &Middleware{Logger: log, Name: name, logStarting: true, clock: &realClock{}}
}
// NewDatabase creates a (postgres) *Database from a database URL string
func NewDatabase(url string, log *logrus.Logger) (*Database, error) {
db := &Database{
url: url,
st: &dbStatements{},
log: log,
migrations: defaultMigrations,
unpreparedStatements: defaultStatements,
}
if db.log == nil {
db.log = logrus.New()
}
parsedURL, err := pq.ParseURL(url)
if err != nil {
return nil, err
}
conn, err := sql.Open("postgres", parsedURL)
if err != nil {
return nil, err
}
db.conn = conn
return db, nil
}
func init_logger() *log.Logger {
l := log.New()
f := &log.TextFormatter{}
f.DisableColors = true
l.Level = log.InfoLevel
l.Formatter = f
/*
//add syslog.LOG_DEBUG as the lowest level of logging to syslog ( so no filtering is applied.
//All the log filtering is taken care on the local logger level
hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DEBUG, "")
if err == nil {
l.Hooks.Add(hook)
} else {
l.WithFields(log.Fields{
"error": err.Error(),
}).Error("Could not add syslog logging hook")
}
*/
return l
}
// NewCustomMiddleware builds a *Middleware with the given level and formatter
func NewCustomMiddleware(level logrus.Level, formatter logrus.Formatter, name string) *Middleware {
log := logrus.New()
log.Level = level
log.Formatter = formatter
return &Middleware{Logger: log, Name: name}
}
func (f *filePublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
logger := log.New()
logger.Println("Publishing started")
var metrics []plugin.MetricType
switch contentType {
case plugin.SnapGOBContentType:
dec := gob.NewDecoder(bytes.NewBuffer(content))
if err := dec.Decode(&metrics); err != nil {
logger.Printf("Error decoding: error=%v content=%v", err, content)
return err
}
default:
logger.Printf("Error unknown content type '%v'", contentType)
return errors.New(fmt.Sprintf("Unknown content type '%s'", contentType))
}
logger.Printf("publishing %v metrics to %v", len(metrics), config)
file, err := os.OpenFile(config["file"].(ctypes.ConfigValueStr).Value, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
defer file.Close()
if err != nil {
logger.Printf("Error: %v", err)
return err
}
w := bufio.NewWriter(file)
for _, m := range metrics {
formattedTags := formatMetricTagsAsString(m.Tags())
w.WriteString(fmt.Sprintf("%v|%v|%v|%v\n", m.Timestamp(), m.Namespace(), m.Data(), formattedTags))
}
w.Flush()
return nil
}