func (this *LogClientController) uploadlocal(interval time.Duration) ([]byte, error) {
	var logs []byte
	err := filepath.Walk(this.dir, func(filename string, f os.FileInfo, err error) error {
		if f == nil {
			glog.Errorf("No log file found")
			return nil
		}
		if f.IsDir() {
			return nil
		}
		if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
			fields := strings.Split(f.Name(), ".")
			t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[2])
			time_int := t.Unix()
			if time.Now().Unix()-int64(interval.Seconds()) < time_int {
				// fmt.Println(filename)
				log, err := ioutil.ReadFile(filename)
				if err != nil {
					glog.Errorf("%v", err)
					return nil
				}
				logs = append(logs, log...)
				// fmt.Println(string(logs))
				// logs = tmp
			}
		}
		return nil
	})
	return logs, err
}
Esempio n. 2
0
func SendLogNow(url, dir string) error {
	err := filepath.Walk(dir, func(filename string, f os.FileInfo, err error) error {
		if f == nil {
			info_err := errors.New("Filepath.Walk() returned no fileinfo")
			glog.Errorf("%v", info_err)
			return nil
		}
		if f.IsDir() {
			return nil
		}
		if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
			post_err := PostFile(filename, url)
			if post_err != nil {
				glog.Errorf("Upload file failed, %v", post_err)
			} else {
				glog.Infof("%s upload succeed \n", filename)
				fmt.Printf("%s upload succeed \n", filename)
			}
			return nil
		}
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}
Esempio n. 3
0
func (this *LogForwarder) Ping() (time.Duration, string, error) {
	now := time.Now()
	url := this.url
	url.Path = "ping"
	// fmt.Println(url.String())

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		glog.Errorf("%v", err)
		return 0, "", err
	}
	req.Header.Set("User-Agent", this.userAgent)
	if this.username != "" {
		req.SetBasicAuth(this.username, this.password)
	}

	resp, err := this.httpClient.Do(req)
	if err != nil {
		glog.Errorf("%v", err)
		return 0, "", err
	}
	defer resp.Body.Close()
	//fmt.Println(resp.StatusCode)
	version := resp.Header.Get("X-Influxdb-Version")
	return time.Since(now), version, nil
}
Esempio n. 4
0
func main() {
	var config LogConfig
	flag.StringVar(&config.Name, "name", "log_server", "Server name")
	flag.StringVar(&config.ListenPort, "port", ":1734", "Listening port")
	flag.StringVar(&config.logForwarderConfig.URL.Scheme, "scheme", "http", "Set url scheme to http")
	flag.StringVar(&config.logForwarderConfig.URL.Host, "influxdb_url", "localhost:8086", "Influxdb host and port")
	flag.StringVar(&config.logForwarderConfig.Username, "influxdb_username", "", "Influxdb basic auth username")
	flag.StringVar(&config.logForwarderConfig.Password, "influxdb_password", "", "Influxdb basic auth password")
	flag.StringVar(&config.logForwarderConfig.UserAgent, "influxdb_user_agent", "", "User agent")
	flag.DurationVar(&config.logForwarderConfig.Timeout, "influxdb_timeout", time.Duration(10)*time.Second, "Influxdb request time out")
	flag.StringVar(&config.logForwarderConfig.RetentionPolicy, "influxdb_retention_policy", "3d", "Time to keep old data before clean it")
	flag.StringVar(&config.logForwarderConfig.Precision, "influxdb_timestamp_precision", "n", "Timestamp precision")
	flag.StringVar(&config.logForwarderConfig.Consistency, "influxdb_node_consistency", "one", "Influxdb nodes write consistency")
	flag.Parse()
	glog.Infof("Config: %v", config)

	app := ripple.NewApplication()
	logcontroller, err := controller.NewLogController(config.Name, config.logForwarderConfig)
	if err != nil {
		glog.Errorf("Log controller init failed: %v", err)
	}
	app.RegisterController("log", logcontroller)
	//app.AddRoute(ripple.Route{Pattern: "_controller/_action"})
	app.AddRoute(ripple.Route{Pattern: "upload", Controller: "log", Action: "Upload"})
	app.SetBaseUrl("/")
	http.HandleFunc("/", app.ServeHTTP)

	glog.Infof("Starting centralized log server at ", config.ListenPort)
	http.ListenAndServe(config.ListenPort, nil)
	defer glog.Flush()
}
Esempio n. 5
0
func main() {
	var config LogClientConfig
	flag.StringVar(&config.Name, "name", "log_client", "Log cilent name")
	flag.StringVar(&config.Host, "server_host", "http://localhost", "Log server host")
	flag.StringVar(&config.Port, "server_port", ":1734", "Log server listen port")
	flag.StringVar(&config.Method, "method", "/upload", "Log client method")
	flag.StringVar(&config.Dir, "dir", "/tmp/", "Upload Directory")
	flag.StringVar(&config.ListenPort, "client_listen_port", ":1735", "Log client server listening port")
	flag.DurationVar(&config.UploadInterval, "upload_interval", time.Duration(15)*time.Minute, "Upload file created before upload interval (better be smaller than keep_interval, has to be bigger than log file create interval)")
	flag.DurationVar(&config.KeepInterval, "keep_interval", time.Duration(30)*time.Minute, "Log file kept time (better be bigger than sleep_interval and upload_interval)")
	flag.DurationVar(&config.SleepInterval, "sleep_interval", time.Duration(10)*time.Minute, "Sleep time interval between every upload action (better smaller than keep_interval)")
	flag.Parse()

	// if config.KeepInterval < config.UploadInterval || config.KeepInterval < config.SleepInterval {
	// 	if config.UploadInterval >= config.SleepInterval {
	// 		config.KeepInterval = config.UploadInterval * 2
	// 		// fmt.Printf("%v\n", config.KeepInterval)
	// 	} else {
	// 		config.KeepInterval = config.SleepInterval * 2
	// 		// fmt.Printf("%v\n", config.KeepInterval)
	// 	}
	// }
	if config.Dir[len(config.Dir)-1] != '/' {
		config.Dir += "/"
	}

	glog.Infof("Log file will be kept for %v, log file will be uploaded %v after created, log client will run every %v", config.KeepInterval, config.UploadInterval, config.SleepInterval)
	fmt.Printf("Log file will be kept for %v, log file will be uploaded %v after created, log client will run every %v\n", config.KeepInterval, config.UploadInterval, config.SleepInterval)

	go func() {
		for {
			client.SendLog(config.Host+config.Port+config.Method, config.Dir, config.UploadInterval, config.KeepInterval)
			time.Sleep(config.SleepInterval)
		}
	}()

	app := ripple.NewApplication()
	logclientcontroller, err := controller.NewLogClientController("logclientserver", config.Host+config.Port+config.Method, config.Dir)
	if err != nil {
		glog.Errorf("Log client controller init failed: %v", err)
	}
	app.RegisterController("logclient", logclientcontroller)
	app.AddRoute(ripple.Route{Pattern: "sync", Controller: "logclient", Action: "Sync"})
	app.AddRoute(ripple.Route{Pattern: "favicon.ico", Controller: "logclient", Action: "Favicon"})
	app.AddRoute(ripple.Route{Pattern: "locallog", Controller: "logclient", Action: "LocalLogFile"})
	app.SetBaseUrl("/")
	http.HandleFunc("/", app.ServeHTTP)
	glog.Infof("Starting log client server at ", config.ListenPort)
	http.ListenAndServe(config.ListenPort, nil)

}
Esempio n. 6
0
func (this *LogForwarder) Query(q *Query) (*http.Response, error) {
	//fmt.Println(q.Command)
	//fmt.Println(q.Database)
	url := this.url
	url.Path = "query"
	params := url.Query()
	params.Set("q", q.Command)
	if q.Database != "" {
		params.Set("db", q.Database)
	}
	url.RawQuery = params.Encode()
	// fmt.Println(url.String())

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		glog.Errorf("%v", err)
		return nil, err
	}
	req.Header.Set("User-Agent", this.userAgent)
	if this.username != "" {
		req.SetBasicAuth(this.username, this.password)
	}

	resp, err := this.httpClient.Do(req)
	if err != nil {
		glog.Errorf("%v", err)
		return nil, err
	}
	defer resp.Body.Close()
	//fmt.Println(resp.StatusCode)

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
		query_err := errors.New("Query database failed")
		glog.Errorf("%v", query_err)
		return nil, query_err
	}
	return resp, nil
}
Esempio n. 7
0
func PostFile(filename, targetUrl string) error {
	bodyBuf := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(bodyBuf)

	fileWriter, err := bodyWriter.CreateFormFile("uploadlogfile", filename)
	if err != nil {
		glog.Errorf("Error writing to buffer")
		return err
	}
	//var mu sync.Mutex
	//mu.Lock()
	//glog.Flush()
	fh, err := os.Open(filename)
	if err != nil {
		glog.Errorf("Error opening file")
		return err
	}
	defer fh.Close()

	_, err = io.Copy(fileWriter, fh)
	if err != nil {
		glog.Errorf("%v", err)
		return err
	}
	//mu.Unlock()
	contentType := bodyWriter.FormDataContentType()
	bodyWriter.Close()

	resp, err := http.Post(targetUrl, contentType, bodyBuf)
	if err != nil {
		glog.Errorf("%v", err)
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 204 && resp.StatusCode != 200 {
		if resp.StatusCode == 400 {
			code_err := errors.New("Bad filename format")
			glog.Errorf("%s: %v %d", filename, code_err, resp.StatusCode)
			return code_err
		} else if resp.StatusCode == 500 {
			code_err := errors.New("Log server internal error")
			glog.Errorf("%v %d", code_err, resp.StatusCode)
			return code_err
		} else {
			code_err := errors.New("Http status code error")
			glog.Errorf("%v %d", code_err, resp.StatusCode)
			return code_err
		}
	}
	return nil
}
Esempio n. 8
0
func SendLog(url, dir string, upload, keep time.Duration) error {
	err := filepath.Walk(dir, func(filename string, f os.FileInfo, err error) error {
		if f == nil {
			info_err := errors.New("Filepath.Walk() returned no fileinfo")
			glog.Errorf("%v", info_err)
			return nil
		}
		if f.IsDir() {
			return nil
		}
		if b, _ := path.Match("LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
			fields := strings.Split(f.Name(), ".")
			t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[2])
			time_int := t.Unix()
			// glog.Infoln("Log created at: ", t, "\tTime now: ", time.Now())
			// fmt.Println("Log created at: ", t, "\tTime now: ", time.Now())
			if time.Now().Unix()-int64(upload.Seconds()) > time_int {
				post_err := PostFile(filename, url)
				if post_err != nil {
					// glog.Errorf("Upload file failed, %v", post_err)
				} else {
					os.Rename(filename, path.Dir(filename)+"/OLD."+f.Name())
					glog.Infof("%s upload succeed \n", filename)
					fmt.Printf("%s upload succeed \n", filename)
				}
			}
			return nil
		}
		if b, _ := path.Match("OLD.LOG.*.*????-??-??T??:??:??Z", f.Name()); b {
			fields := strings.Split(f.Name(), ".")
			t, _ := time.Parse("MST2006-01-02T15:04:05Z", fields[3])
			time_int := t.Unix()
			if time.Now().Unix()-int64(keep.Seconds()) > time_int {
				err := os.Remove(filename)
				if err == nil {
					glog.Infof("%s removed \n", filename)
					fmt.Printf("%s removed \n", filename)
				}
			}
			return nil
		}
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}
Esempio n. 9
0
func (this *LogController) PostUpload(ctx *ripple.Context) {
	ctx.Request.ParseMultipartForm(32 << 20)
	file, handler, err := ctx.Request.FormFile("uploadlogfile")
	if err != nil {
		glog.Errorf("Log file upload failed: %v", err)
		ctx.Response.Status = 400
		ctx.Response.Body = err
		return
	}
	defer file.Close()

	logforwarder, err := models.NewLogForwarder(this.logForwarderConfig)
	status_code, err := logforwarder.SaveFile(file, handler) // decode file and forward to database
	if err != nil {
		ctx.Response.Status = status_code
		ctx.Response.Body = err
		return
	} else {
		glog.Infof("Save log file to database succeed")
		ctx.Response.Status = status_code
		ctx.Response.Body = "Save log file to database succeed"
		return
	}
}
Esempio n. 10
0
func (this *LogForwarder) Write(file multipart.File, filename string) (*http.Response, error) {
	url := this.url
	url.Path = "write"

	//fields := strings.Split(filename, ".")

	params := url.Query()
	params.Set("consistency", this.consistency)
	params.Set("db", filename)
	params.Set("precision", this.precision)
	//params.Set("rp", "default")
	url.RawQuery = params.Encode()

	//Send the multipart.file to influxdb directly
	req, err := http.NewRequest("POST", url.String(), file)

	//io.copy multipart.file to fd (unnecessary)
	/*
		fd, err := os.OpenFile("/tmp/logserver/"+filename, os.O_RDWR|os.O_CREATE, 0666)
		if err != nil {
			glog.Errorf("Create tmp log file failed: %v", err)
			return nil, err
		}
		defer fd.Close()
		_, err = io.Copy(fd, file)
		if err != nil {
			glog.Errorf("Copy to tmp log file failed: %v", err)
			return nil, err
		}
		fd.Seek(0, 0)
		//	req, err := http.NewRequest("POST", url.String(), fd)

		//Resolve JSON to LINE protocol, and sent to influxdb
		scanner := bufio.NewScanner(fd)
		bodyBuf := &bytes.Buffer{}
		for scanner.Scan() {
			var log LogModel
			//fmt.Println(scanner.Bytes())
			str := scanner.Bytes()
			err = json.Unmarshal(str[:], &log)
			if err != nil {
				continue
			}
			query := log.Process + ",process=" + log.Process + ",type=" + log.Type + " value=\"" + log.Pid + "\",message=\"" + log.Message + "\",debug=\"" + log.Debug + "\" " + log.Time
			//fmt.Println(query)
			bodyBuf.WriteString(query)
			bodyBuf.WriteByte('\n')
		}
		req, err := http.NewRequest("POST", url.String(), bodyBuf)
	*/
	if err != nil {
		glog.Errorf("%v", err)
		return nil, err
	}
	req.Header.Set("Content-Type", "")
	req.Header.Set("User-Agent", this.userAgent)
	if this.username != "" {
		req.SetBasicAuth(this.username, this.password)
	}

	resp, err := this.httpClient.Do(req)
	if err != nil {
		return nil, err
	} else {
		//os.Remove("/tmp/logserver/" + filename)
	}
	defer resp.Body.Close()
	//fmt.Println(resp.StatusCode)
	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
		fmt.Println(resp.StatusCode, resp, resp.Body)
		write_err := errors.New("Write to influxdb failed, bad request format")
		glog.Errorf("%v", write_err)
		return nil, write_err
	}
	return resp, nil
}