func main() {
var config LogConfig
flag.StringVar(&config.Name, "name", "log_server", "Server name")
flag.StringVar(&config.ListenPort, "port", ":1734", "Listening port")
flag.StringVar(&config.logForwarderConfig.URL.Scheme, "scheme", "http", "Set url scheme to http")
flag.StringVar(&config.logForwarderConfig.URL.Host, "influxdb_url", "localhost:8086", "Influxdb host and port")
flag.StringVar(&config.logForwarderConfig.Username, "influxdb_username", "", "Influxdb basic auth username")
flag.StringVar(&config.logForwarderConfig.Password, "influxdb_password", "", "Influxdb basic auth password")
flag.StringVar(&config.logForwarderConfig.UserAgent, "influxdb_user_agent", "", "User agent")
flag.DurationVar(&config.logForwarderConfig.Timeout, "influxdb_timeout", time.Duration(10)*time.Second, "Influxdb request time out")
flag.StringVar(&config.logForwarderConfig.RetentionPolicy, "influxdb_retention_policy", "3d", "Time to keep old data before clean it")
flag.StringVar(&config.logForwarderConfig.Precision, "influxdb_timestamp_precision", "n", "Timestamp precision")
flag.StringVar(&config.logForwarderConfig.Consistency, "influxdb_node_consistency", "one", "Influxdb nodes write consistency")
flag.Parse()
glog.Infof("Config: %v", config)
app := ripple.NewApplication()
logcontroller, err := controller.NewLogController(config.Name, config.logForwarderConfig)
if err != nil {
glog.Errorf("Log controller init failed: %v", err)
}
app.RegisterController("log", logcontroller)
//app.AddRoute(ripple.Route{Pattern: "_controller/_action"})
app.AddRoute(ripple.Route{Pattern: "upload", Controller: "log", Action: "Upload"})
app.SetBaseUrl("/")
http.HandleFunc("/", app.ServeHTTP)
glog.Infof("Starting centralized log server at ", config.ListenPort)
http.ListenAndServe(config.ListenPort, nil)
defer glog.Flush()
}
func main() {
var config LogClientConfig
flag.StringVar(&config.Name, "name", "log_client", "Log cilent name")
flag.StringVar(&config.Host, "server_host", "http://localhost", "Log server host")
flag.StringVar(&config.Port, "server_port", ":1734", "Log server listen port")
flag.StringVar(&config.Method, "method", "/upload", "Log client method")
flag.StringVar(&config.Dir, "dir", "/tmp/", "Upload Directory")
flag.StringVar(&config.ListenPort, "client_listen_port", ":1735", "Log client server listening port")
flag.DurationVar(&config.UploadInterval, "upload_interval", time.Duration(15)*time.Minute, "Upload file created before upload interval (better be smaller than keep_interval, has to be bigger than log file create interval)")
flag.DurationVar(&config.KeepInterval, "keep_interval", time.Duration(30)*time.Minute, "Log file kept time (better be bigger than sleep_interval and upload_interval)")
flag.DurationVar(&config.SleepInterval, "sleep_interval", time.Duration(10)*time.Minute, "Sleep time interval between every upload action (better smaller than keep_interval)")
flag.Parse()
// if config.KeepInterval < config.UploadInterval || config.KeepInterval < config.SleepInterval {
// if config.UploadInterval >= config.SleepInterval {
// config.KeepInterval = config.UploadInterval * 2
// // fmt.Printf("%v\n", config.KeepInterval)
// } else {
// config.KeepInterval = config.SleepInterval * 2
// // fmt.Printf("%v\n", config.KeepInterval)
// }
// }
if config.Dir[len(config.Dir)-1] != '/' {
config.Dir += "/"
}
glog.Infof("Log file will be kept for %v, log file will be uploaded %v after created, log client will run every %v", config.KeepInterval, config.UploadInterval, config.SleepInterval)
fmt.Printf("Log file will be kept for %v, log file will be uploaded %v after created, log client will run every %v\n", config.KeepInterval, config.UploadInterval, config.SleepInterval)
go func() {
for {
client.SendLog(config.Host+config.Port+config.Method, config.Dir, config.UploadInterval, config.KeepInterval)
time.Sleep(config.SleepInterval)
}
}()
app := ripple.NewApplication()
logclientcontroller, err := controller.NewLogClientController("logclientserver", config.Host+config.Port+config.Method, config.Dir)
if err != nil {
glog.Errorf("Log client controller init failed: %v", err)
}
app.RegisterController("logclient", logclientcontroller)
app.AddRoute(ripple.Route{Pattern: "sync", Controller: "logclient", Action: "Sync"})
app.AddRoute(ripple.Route{Pattern: "favicon.ico", Controller: "logclient", Action: "Favicon"})
app.AddRoute(ripple.Route{Pattern: "locallog", Controller: "logclient", Action: "LocalLogFile"})
app.SetBaseUrl("/")
http.HandleFunc("/", app.ServeHTTP)
glog.Infof("Starting log client server at ", config.ListenPort)
http.ListenAndServe(config.ListenPort, nil)
}
func SendLog(url, dir string, upload, keep time.Duration) error {
err := filepath.Walk(dir, func(filename string, f os.FileInfo, err error) error {
if f == nil {
info_err := errors.New("Filepath.Walk() returned no fileinfo")
glog.Errorf("%v", info_err)
return nil
}
if f.IsDir() {
return nil
}
if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
fields := strings.Split(f.Name(), ".")
t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[2])
time_int := t.Unix()
// glog.Infoln("Log created at: ", t, "\tTime now: ", time.Now())
// fmt.Println("Log created at: ", t, "\tTime now: ", time.Now())
if time.Now().Unix()-int64(upload.Seconds()) > time_int {
post_err := PostFile(filename, url)
if post_err != nil {
// glog.Errorf("Upload file failed, %v", post_err)
} else {
os.Rename(filename, path.Dir(filename)+"/OLD."+f.Name())
glog.Infof("%s upload succeed \n", filename)
fmt.Printf("%s upload succeed \n", filename)
}
}
return nil
}
if b, _ := path.Match("OLD.LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
fields := strings.Split(f.Name(), ".")
t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[3])
time_int := t.Unix()
if time.Now().Unix()-int64(keep.Seconds()) > time_int {
err := os.Remove(filename)
if err == nil {
glog.Infof("%s removed \n", filename)
fmt.Printf("%s removed \n", filename)
}
}
return nil
}
return nil
})
if err != nil {
return err
}
return nil
}
func SendLogNow(url, dir string) error {
err := filepath.Walk(dir, func(filename string, f os.FileInfo, err error) error {
if f == nil {
info_err := errors.New("Filepath.Walk() returned no fileinfo")
glog.Errorf("%v", info_err)
return nil
}
if f.IsDir() {
return nil
}
if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
post_err := PostFile(filename, url)
if post_err != nil {
glog.Errorf("Upload file failed, %v", post_err)
} else {
glog.Infof("%s upload succeed \n", filename)
fmt.Printf("%s upload succeed \n", filename)
}
return nil
}
return nil
})
if err != nil {
return err
}
return nil
}
func (this *LogController) PostUpload(ctx *ripple.Context) {
ctx.Request.ParseMultipartForm(32 << 20)
file, handler, err := ctx.Request.FormFile("uploadlogfile")
if err != nil {
glog.Errorf("Log file upload failed: %v", err)
ctx.Response.Status = 400
ctx.Response.Body = err
return
}
defer file.Close()
logforwarder, err := models.NewLogForwarder(this.logForwarderConfig)
status_code, err := logforwarder.SaveFile(file, handler) // decode file and forward to database
if err != nil {
ctx.Response.Status = status_code
ctx.Response.Body = err
return
} else {
glog.Infof("Save log file to database succeed")
ctx.Response.Status = status_code
ctx.Response.Body = "Save log file to database succeed"
return
}
}
