func (this *LogClientController) uploadlocal(interval time.Duration) ([]byte, error) {
var logs []byte
err := filepath.Walk(this.dir, func(filename string, f os.FileInfo, err error) error {
if f == nil {
glog.Errorf("No log file found")
return nil
}
if f.IsDir() {
return nil
}
if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
fields := strings.Split(f.Name(), ".")
t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[2])
time_int := t.Unix()
if time.Now().Unix()-int64(interval.Seconds()) < time_int {
// fmt.Println(filename)
log, err := ioutil.ReadFile(filename)
if err != nil {
glog.Errorf("%v", err)
return nil
}
logs = append(logs, log...)
// fmt.Println(string(logs))
// logs = tmp
}
}
return nil
})
return logs, err
}
func SendLogNow(url, dir string) error {
err := filepath.Walk(dir, func(filename string, f os.FileInfo, err error) error {
if f == nil {
info_err := errors.New("Filepath.Walk() returned no fileinfo")
glog.Errorf("%v", info_err)
return nil
}
if f.IsDir() {
return nil
}
if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
post_err := PostFile(filename, url)
if post_err != nil {
glog.Errorf("Upload file failed, %v", post_err)
} else {
glog.Infof("%s upload succeed \n", filename)
fmt.Printf("%s upload succeed \n", filename)
}
return nil
}
return nil
})
if err != nil {
return err
}
return nil
}
func (this *LogForwarder) Ping() (time.Duration, string, error) {
now := time.Now()
url := this.url
url.Path = "ping"
// fmt.Println(url.String())
req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
glog.Errorf("%v", err)
return 0, "", err
}
req.Header.Set("User-Agent", this.userAgent)
if this.username != "" {
req.SetBasicAuth(this.username, this.password)
}
resp, err := this.httpClient.Do(req)
if err != nil {
glog.Errorf("%v", err)
return 0, "", err
}
defer resp.Body.Close()
//fmt.Println(resp.StatusCode)
version := resp.Header.Get("X-Influxdb-Version")
return time.Since(now), version, nil
}
func main() {
var config LogConfig
flag.StringVar(&config.Name, "name", "log_server", "Server name")
flag.StringVar(&config.ListenPort, "port", ":1734", "Listening port")
flag.StringVar(&config.logForwarderConfig.URL.Scheme, "scheme", "http", "Set url scheme to http")
flag.StringVar(&config.logForwarderConfig.URL.Host, "influxdb_url", "localhost:8086", "Influxdb host and port")
flag.StringVar(&config.logForwarderConfig.Username, "influxdb_username", "", "Influxdb basic auth username")
flag.StringVar(&config.logForwarderConfig.Password, "influxdb_password", "", "Influxdb basic auth password")
flag.StringVar(&config.logForwarderConfig.UserAgent, "influxdb_user_agent", "", "User agent")
flag.DurationVar(&config.logForwarderConfig.Timeout, "influxdb_timeout", time.Duration(10)*time.Second, "Influxdb request time out")
flag.StringVar(&config.logForwarderConfig.RetentionPolicy, "influxdb_retention_policy", "3d", "Time to keep old data before clean it")
flag.StringVar(&config.logForwarderConfig.Precision, "influxdb_timestamp_precision", "n", "Timestamp precision")
flag.StringVar(&config.logForwarderConfig.Consistency, "influxdb_node_consistency", "one", "Influxdb nodes write consistency")
flag.Parse()
glog.Infof("Config: %v", config)
app := ripple.NewApplication()
logcontroller, err := controller.NewLogController(config.Name, config.logForwarderConfig)
if err != nil {
glog.Errorf("Log controller init failed: %v", err)
}
app.RegisterController("log", logcontroller)
//app.AddRoute(ripple.Route{Pattern: "_controller/_action"})
app.AddRoute(ripple.Route{Pattern: "upload", Controller: "log", Action: "Upload"})
app.SetBaseUrl("/")
http.HandleFunc("/", app.ServeHTTP)
glog.Infof("Starting centralized log server at ", config.ListenPort)
http.ListenAndServe(config.ListenPort, nil)
defer glog.Flush()
}
func main() {
var config LogClientConfig
flag.StringVar(&config.Name, "name", "log_client", "Log cilent name")
flag.StringVar(&config.Host, "server_host", "http://localhost", "Log server host")
flag.StringVar(&config.Port, "server_port", ":1734", "Log server listen port")
flag.StringVar(&config.Method, "method", "/upload", "Log client method")
flag.StringVar(&config.Dir, "dir", "/tmp/", "Upload Directory")
flag.StringVar(&config.ListenPort, "client_listen_port", ":1735", "Log client server listening port")
flag.DurationVar(&config.UploadInterval, "upload_interval", time.Duration(15)*time.Minute, "Upload file created before upload interval (better be smaller than keep_interval, has to be bigger than log file create interval)")
flag.DurationVar(&config.KeepInterval, "keep_interval", time.Duration(30)*time.Minute, "Log file kept time (better be bigger than sleep_interval and upload_interval)")
flag.DurationVar(&config.SleepInterval, "sleep_interval", time.Duration(10)*time.Minute, "Sleep time interval between every upload action (better smaller than keep_interval)")
flag.Parse()
// if config.KeepInterval < config.UploadInterval || config.KeepInterval < config.SleepInterval {
// if config.UploadInterval >= config.SleepInterval {
// config.KeepInterval = config.UploadInterval * 2
// // fmt.Printf("%v\n", config.KeepInterval)
// } else {
// config.KeepInterval = config.SleepInterval * 2
// // fmt.Printf("%v\n", config.KeepInterval)
// }
// }
if config.Dir[len(config.Dir)-1] != '/' {
config.Dir += "/"
}
glog.Infof("Log file will be kept for %v, log file will be uploaded %v after created, log client will run every %v", config.KeepInterval, config.UploadInterval, config.SleepInterval)
fmt.Printf("Log file will be kept for %v, log file will be uploaded %v after created, log client will run every %v\n", config.KeepInterval, config.UploadInterval, config.SleepInterval)
go func() {
for {
client.SendLog(config.Host+config.Port+config.Method, config.Dir, config.UploadInterval, config.KeepInterval)
time.Sleep(config.SleepInterval)
}
}()
app := ripple.NewApplication()
logclientcontroller, err := controller.NewLogClientController("logclientserver", config.Host+config.Port+config.Method, config.Dir)
if err != nil {
glog.Errorf("Log client controller init failed: %v", err)
}
app.RegisterController("logclient", logclientcontroller)
app.AddRoute(ripple.Route{Pattern: "sync", Controller: "logclient", Action: "Sync"})
app.AddRoute(ripple.Route{Pattern: "favicon.ico", Controller: "logclient", Action: "Favicon"})
app.AddRoute(ripple.Route{Pattern: "locallog", Controller: "logclient", Action: "LocalLogFile"})
app.SetBaseUrl("/")
http.HandleFunc("/", app.ServeHTTP)
glog.Infof("Starting log client server at ", config.ListenPort)
http.ListenAndServe(config.ListenPort, nil)
}
func (this *LogForwarder) Query(q *Query) (*http.Response, error) {
//fmt.Println(q.Command)
//fmt.Println(q.Database)
url := this.url
url.Path = "query"
params := url.Query()
params.Set("q", q.Command)
if q.Database != "" {
params.Set("db", q.Database)
}
url.RawQuery = params.Encode()
// fmt.Println(url.String())
req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
glog.Errorf("%v", err)
return nil, err
}
req.Header.Set("User-Agent", this.userAgent)
if this.username != "" {
req.SetBasicAuth(this.username, this.password)
}
resp, err := this.httpClient.Do(req)
if err != nil {
glog.Errorf("%v", err)
return nil, err
}
defer resp.Body.Close()
//fmt.Println(resp.StatusCode)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
query_err := errors.New("Query database failed")
glog.Errorf("%v", query_err)
return nil, query_err
}
return resp, nil
}
func PostFile(filename, targetUrl string) error {
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
fileWriter, err := bodyWriter.CreateFormFile("uploadlogfile", filename)
if err != nil {
glog.Errorf("Error writing to buffer")
return err
}
//var mu sync.Mutex
//mu.Lock()
//glog.Flush()
fh, err := os.Open(filename)
if err != nil {
glog.Errorf("Error opening file")
return err
}
defer fh.Close()
_, err = io.Copy(fileWriter, fh)
if err != nil {
glog.Errorf("%v", err)
return err
}
//mu.Unlock()
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
resp, err := http.Post(targetUrl, contentType, bodyBuf)
if err != nil {
glog.Errorf("%v", err)
return err
}
defer resp.Body.Close()
if resp.StatusCode != 204 && resp.StatusCode != 200 {
if resp.StatusCode == 400 {
code_err := errors.New("Bad filename format")
glog.Errorf("%s: %v %d", filename, code_err, resp.StatusCode)
return code_err
} else if resp.StatusCode == 500 {
code_err := errors.New("Log server internal error")
glog.Errorf("%v %d", code_err, resp.StatusCode)
return code_err
} else {
code_err := errors.New("Http status code error")
glog.Errorf("%v %d", code_err, resp.StatusCode)
return code_err
}
}
return nil
}
func SendLog(url, dir string, upload, keep time.Duration) error {
err := filepath.Walk(dir, func(filename string, f os.FileInfo, err error) error {
if f == nil {
info_err := errors.New("Filepath.Walk() returned no fileinfo")
glog.Errorf("%v", info_err)
return nil
}
if f.IsDir() {
return nil
}
if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
fields := strings.Split(f.Name(), ".")
t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[2])
time_int := t.Unix()
// glog.Infoln("Log created at: ", t, "\tTime now: ", time.Now())
// fmt.Println("Log created at: ", t, "\tTime now: ", time.Now())
if time.Now().Unix()-int64(upload.Seconds()) > time_int {
post_err := PostFile(filename, url)
if post_err != nil {
// glog.Errorf("Upload file failed, %v", post_err)
} else {
os.Rename(filename, path.Dir(filename)+"/OLD."+f.Name())
glog.Infof("%s upload succeed \n", filename)
fmt.Printf("%s upload succeed \n", filename)
}
}
return nil
}
if b, _ := path.Match("OLD.LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
fields := strings.Split(f.Name(), ".")
t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[3])
time_int := t.Unix()
if time.Now().Unix()-int64(keep.Seconds()) > time_int {
err := os.Remove(filename)
if err == nil {
glog.Infof("%s removed \n", filename)
fmt.Printf("%s removed \n", filename)
}
}
return nil
}
return nil
})
if err != nil {
return err
}
return nil
}
func (this *LogController) PostUpload(ctx *ripple.Context) {
ctx.Request.ParseMultipartForm(32 << 20)
file, handler, err := ctx.Request.FormFile("uploadlogfile")
if err != nil {
glog.Errorf("Log file upload failed: %v", err)
ctx.Response.Status = 400
ctx.Response.Body = err
return
}
defer file.Close()
logforwarder, err := models.NewLogForwarder(this.logForwarderConfig)
status_code, err := logforwarder.SaveFile(file, handler) // decode file and forward to database
if err != nil {
ctx.Response.Status = status_code
ctx.Response.Body = err
return
} else {
glog.Infof("Save log file to database succeed")
ctx.Response.Status = status_code
ctx.Response.Body = "Save log file to database succeed"
return
}
}
func (this *LogForwarder) Write(file multipart.File, filename string) (*http.Response, error) {
url := this.url
url.Path = "write"
//fields := strings.Split(filename, ".")
params := url.Query()
params.Set("consistency", this.consistency)
params.Set("db", filename)
params.Set("precision", this.precision)
//params.Set("rp", "default")
url.RawQuery = params.Encode()
//Send the multipart.file to influxdb directly
req, err := http.NewRequest("POST", url.String(), file)
//io.copy multipart.file to fd (unnecessary)
/*
fd, err := os.OpenFile("/tmp/logserver/"+filename, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
glog.Errorf("Create tmp log file failed: %v", err)
return nil, err
}
defer fd.Close()
_, err = io.Copy(fd, file)
if err != nil {
glog.Errorf("Copy to tmp log file failed: %v", err)
return nil, err
}
fd.Seek(0, 0)
// req, err := http.NewRequest("POST", url.String(), fd)
//Resolve JSON to LINE protocol, and sent to influxdb
scanner := bufio.NewScanner(fd)
bodyBuf := &bytes.Buffer{}
for scanner.Scan() {
var log LogModel
//fmt.Println(scanner.Bytes())
str := scanner.Bytes()
err = json.Unmarshal(str[:], &log)
if err != nil {
continue
}
query := log.Process + ",process=" + log.Process + ",type=" + log.Type + " value=\"" + log.Pid + "\",message=\"" + log.Message + "\",debug=\"" + log.Debug + "\" " + log.Time
//fmt.Println(query)
bodyBuf.WriteString(query)
bodyBuf.WriteByte('\n')
}
req, err := http.NewRequest("POST", url.String(), bodyBuf)
*/
if err != nil {
glog.Errorf("%v", err)
return nil, err
}
req.Header.Set("Content-Type", "")
req.Header.Set("User-Agent", this.userAgent)
if this.username != "" {
req.SetBasicAuth(this.username, this.password)
}
resp, err := this.httpClient.Do(req)
if err != nil {
return nil, err
} else {
//os.Remove("/tmp/logserver/" + filename)
}
defer resp.Body.Close()
//fmt.Println(resp.StatusCode)
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
fmt.Println(resp.StatusCode, resp, resp.Body)
write_err := errors.New("Write to influxdb failed, bad request format")
glog.Errorf("%v", write_err)
return nil, write_err
}
return resp, nil
}