Example #1
0
func (g *Grabber) GetConfig() (toolkit.M, error) {
	retValue := toolkit.M{}
	parm, found := g.Config.FormValues["formvalues"].(toolkit.M)
	if found {
		for key, val := range parm {
			switch {
			case strings.Contains(val.(string), "time.Now()"):
				// time.Now(), Date2String(YYYYMMDD) = time.Now().Date2String(YYYYMMDD)
				format := ""
				if strings.Contains(val.(string), "Date2String") {
					format = strings.Replace(strings.Replace(val.(string), "Date2String(time.Now(),", "", -1), ")", "", -1)
				}

				parm[key] = cast.Date2String(time.Now(), format)
			}
		}

		retValue.Set("formvalues", parm)
	}

	switch {
	case ((g.AuthType == "session" || g.AuthType == "cookie") && g.LoginValues != nil):
		tConfig := toolkit.M{}
		tConfig.Set("loginvalues", g.LoginValues)
		jar, e := toolkit.HttpGetCookieJar(g.LoginUrl, g.CallType, tConfig)
		if e != nil {
			return nil, e
		} else {
			retValue.Set("cookie", jar)
		}
	}

	return retValue, nil
}
Example #2
0
func (g *GrabService) AddRecHistory(key string, docs []toolkit.M) string {
	var config = map[string]interface{}{"useheader": true, "delimiter": ",", "newfile": true}
	file := fmt.Sprintf("%s%s.%s-%s.csv", g.HistoryRecPath, g.Name, key, cast.Date2String(time.Now(), "YYYYMMddHHmmss"))
	ci := &dbox.ConnectionInfo{file, "", "", "", config}
	c, e := dbox.NewConnection("csv", ci)
	if e != nil {
		g.ErrorNotes = fmt.Sprintf("[%s] Setup connection to Record history failed [csv-%s]:%s", g.Name, file, e)
		g.Log.AddLog(g.ErrorNotes, "ERROR")
		return ""
	}

	e = c.Connect()
	if e != nil {
		g.ErrorNotes = fmt.Sprintf("[%s] Setup connection to history failed [csv-%s]:%s", g.Name, file, e)
		g.Log.AddLog(g.ErrorNotes, "ERROR")
		return ""
	}

	// q := c.NewQuery().SetConfig("multiexec", true).Save()

	for _, doc := range docs {
		e = c.NewQuery().Insert().Exec(toolkit.M{"data": doc})
		if e != nil {
			g.ErrorNotes = fmt.Sprintf("[%s] Insert to history failed [csv-%s]:%s", g.Name, file, e)
			g.Log.AddLog(g.ErrorNotes, "ERROR")
			return ""
		}
	}
	c.Close()

	return file
}
Example #3
0
func (l *LogEngine) AddLog(msg string, logtype string) error {
	var e error
	logtype = strings.ToUpper(logtype) + " "

	if l.LogToStdOut {
		if logtype == "ERROR " {
			l.logError.Println(msg)
		} else if logtype == "WARNING " {
			l.logWarn.Println(msg)
		} else {
			l.logInfo.Println(msg)
		}
		if e != nil {
			return errors.New("Log.AddLog Error: " + e.Error())
		}
	}

	if l.LogToFile {
		filename := l.FileNamePattern
		if l.UseDateFormat != "" && strings.Contains(l.FileNamePattern, "%s") {
			filename = fmt.Sprintf(l.FileNamePattern, cast.Date2String(time.Now(), l.UseDateFormat))
		}
		filename = filepath.Join(l.Path, filename)
		f, e := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
		if e != nil {
			return errors.New("Log.AddLog Error: " + e.Error())
		}
		defer f.Close()
		logFile := log.New(f, logtype, log.Ldate|log.Ltime)
		logFile.Println(msg)
	}

	return nil
}
Example #4
0
func NewHistory(nameid string) *WebGrabberController {
	w := new(WebGrabberController)

	dateNow := cast.Date2String(time.Now(), "YYYYMMdd") //time.Now()
	path := wgHistoryPath + nameid + "-" + dateNow + ".csv"
	w.filepathName = path
	w.nameid = nameid
	return w
}
Example #5
0
func (h *HistoryModule) OpenHistory() interface{} {
	var config = map[string]interface{}{"useheader": true, "delimiter": ",", "dateformat": "MM-dd-YYYY"}

	ci := &dbox.ConnectionInfo{h.filepathName, "", "", "", config}
	c, e := dbox.NewConnection("csv", ci)
	if e != nil {
		return e.Error()
	}

	e = c.Connect()
	if e != nil {
		return e.Error()
	}
	defer c.Close()

	csr, e := c.NewQuery().Select("*").Cursor(nil)
	if e != nil {
		return e.Error()
	}
	if csr == nil {
		return "Cursor not initialized"
	}
	defer csr.Close()
	ds := []toolkit.M{}
	e = csr.Fetch(&ds, 0, false)
	if e != nil {
		return e.Error()
	}

	var history = []interface{}{} //toolkit.M{}
	for i, v := range ds {
		// layout := "2006/01/02 15:04:05"
		castDate := time.Now()
		if v.Has("grabdata") {
			castDate, _ = time.Parse(time.RFC3339, v.Get("grabdate").(string))
		}

		h.humanDate = cast.Date2String(castDate, "YYYY/MM/dd HH:mm:ss")
		h.rowgrabbed, _ = strconv.ParseFloat(fmt.Sprintf("%v", v.Get("rowgrabbed")), 64)
		h.rowsaved, _ = strconv.ParseFloat(fmt.Sprintf("%v", v.Get("rowgrabbed")), 64)

		var addToMap = toolkit.M{}
		addToMap.Set("id", i+1)
		addToMap.Set("datasettingname", v.Get("datasettingname"))
		addToMap.Set("grabdate", h.humanDate)
		addToMap.Set("grabstatus", v.Get("grabstatus"))
		addToMap.Set("rowgrabbed", h.rowgrabbed)
		addToMap.Set("rowsaved", h.rowsaved)
		addToMap.Set("notehistory", v.Get("note"))
		addToMap.Set("recfile", v.Get("recfile"))
		addToMap.Set("nameid", h.nameid)

		history = append(history, addToMap)
	}

	return history
}
Example #6
0
func NewHistory(nameid string) *HistoryModule {
	h := new(HistoryModule)

	dateNow := cast.Date2String(time.Now(), "YYYYMM") //time.Now()
	path := HistoryPath + nameid + "-" + dateNow + ".csv"
	h.filepathName = path
	h.nameid = nameid
	return h
}
Example #7
0
func NewHistory(nameid string) *Grabber {
	w := new(Grabber)

	dateNow := cast.Date2String(time.Now(), "YYYYMMdd") //time.Now()
	path := tLocation + nameid + "-" + dateNow + ".csv"
	w.filepathName = path
	w.nameid = nameid
	return w
}
Example #8
0
func (g *GrabModule) CheckStat(datas []interface{}) interface{} {
	var (
		grabStatus         interface{}
		lastDate, nextDate string
		//toolkit.M
	)
	var summaryNotes = toolkit.M{} //map[string]interface{}{}
	for _, v := range datas {
		vToMap, e := toolkit.ToM(v)
		if e != nil {
			return e.Error()
		}

		if knot.SharedObject().Get(vToMap["nameid"].(string)) != nil {
			i := knot.SharedObject().Get(vToMap["nameid"].(string)).(*sdt.GrabService)
			tLast := cast.Date2String(i.LastGrabExe, "YYYY/MM/dd HH:mm:ss") //i.LastGrabExe.Format("2006/01/02 15:04:05")
			if tLast != "0001/01/01 00:00:00" {
				lastDate = tLast
			}
			tNext := cast.Date2String(i.NextGrabExe, "YYYY/MM/dd HH:mm:ss") //i.NextGrabExe.Format("2006/01/02 15:04:05")
			if tNext != "0001/01/01 00:00:00" {
				nextDate = tNext
			}

			startdate := cast.Date2String(i.StartDate, "YYYY/MM/dd HH:mm:ss") //i.StartDate.Format("2006/01/02 15:04:05")
			enddate := cast.Date2String(i.EndDate, "YYYY/MM/dd HH:mm:ss")     //i.EndDate.Format("2006/01/02 15:04:05")
			summaryNotes.Set("startDate", startdate)
			summaryNotes.Set("endDate", enddate)
			summaryNotes.Set("grabCount", i.GrabCount)
			summaryNotes.Set("rowGrabbed", i.RowGrabbed)
			summaryNotes.Set("errorFound", i.ErrorFound)

			grabStatus = ReadLog(vToMap["logconf"], i.ServiceRunningStat, i.Name, lastDate, nextDate, i.LastGrabStat, summaryNotes)
		} else {
			summaryNotes.Set("errorFound", 0)
			grabStatus = ReadLog(vToMap["logconf"], false, vToMap["nameid"].(string), "", "", false, summaryNotes)
		}
	}

	return grabStatus
}
Example #9
0
func (a *DataFlowController) Save(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := map[string]interface{}{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	dataShapes := payload["DataShapes"].(map[string]interface{})
	actions := payload["Actions"].([]interface{})

	currentDataFlow := new(colonycore.DataFlow)
	currentDataFlow.DataShapes = dataShapes
	currentDataFlow.Actions = constructActions(actions)
	currentDataFlow.Name = tk.ToString(payload["Name"])
	currentDataFlow.Description = tk.ToString(payload["Description"])
	currentDataFlow.ID = tk.ToString(payload["ID"])
	currentDataFlow.GlobalParam = tk.M{}

	for _, val := range payload["GlobalParam"].([]interface{}) {
		tmp := val.(map[string]interface{})
		currentDataFlow.GlobalParam.Set(tk.ToString(tmp["key"]), tk.ToString(tmp["value"]))
	}

	dataDs := []colonycore.DataFlow{}
	cursor, err := colonycore.Find(new(colonycore.DataFlow), dbox.Eq("_id", currentDataFlow.ID))
	if cursor != nil {
		cursor.Fetch(&dataDs, 0, false)
		defer cursor.Close()
	}
	if err != nil && cursor != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	if len(dataDs) == 0 {
		currentDataFlow.CreatedDate = time.Now()
		currentDataFlow.CreatedBy = "Test User"
		currentDataFlow.ID = strings.Replace(currentDataFlow.Name, " ", "", -1) + cast.Date2String(time.Now(), "YYYYMMddHHmm")
	} else {
		currentDataFlow.CreatedDate = dataDs[0].CreatedDate
		currentDataFlow.CreatedBy = dataDs[0].CreatedBy
	}

	currentDataFlow.LastModified = time.Now()

	err = colonycore.Save(currentDataFlow)
	fmt.Println("")
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	return helper.CreateResult(true, currentDataFlow, "success")
}
Example #10
0
func (w *Grabber) OpenHistory() ([]interface{}, error) {
	var history = []interface{}{} //toolkit.M{}
	var config = map[string]interface{}{"useheader": true, "delimiter": ",", "dateformat": "MM-dd-YYYY"}

	ci := &dbox.ConnectionInfo{tLocation, "", "", "", config}
	c, err := dbox.NewConnection("csv", ci)
	if err != nil {
		return history, err
	}

	err = c.Connect()
	if err != nil {
		return history, err
	}
	defer c.Close()

	csr, err := c.NewQuery().Select("*").Cursor(nil)
	if err != nil {
		return history, err
	}
	if csr == nil {
		return history, errors.New("Cursor not initialized")
	}
	defer csr.Close()
	ds := []toolkit.M{}
	err = csr.Fetch(&ds, 0, false)
	if err != nil {
		return history, err
	}

	for i, v := range ds {
		castDate, _ := time.Parse(time.RFC3339, v.Get("grabdate").(string))

		w.humanDate = cast.Date2String(castDate, "YYYY/MM/dd HH:mm:ss")
		w.rowgrabbed, _ = strconv.ParseFloat(fmt.Sprintf("%v", v.Get("rowgrabbed")), 64)
		w.rowsaved, _ = strconv.ParseFloat(fmt.Sprintf("%v", v.Get("rowgrabbed")), 64)

		var addToMap = toolkit.M{}
		addToMap.Set("id", i+1)
		addToMap.Set("datasettingname", v.Get("datasettingname"))
		addToMap.Set("grabdate", w.humanDate)
		addToMap.Set("grabstatus", v.Get("grabstatus"))
		addToMap.Set("rowgrabbed", w.rowgrabbed)
		addToMap.Set("rowsaved", w.rowsaved)
		addToMap.Set("notehistory", v.Get("note"))
		addToMap.Set("recfile", v.Get("recfile"))
		addToMap.Set("nameid", w.nameid)

		history = append(history, addToMap)
	}
	return history, nil
}
Example #11
0
File: hive.go Project: masmeka/hdc
func CheckDataType(inputModel reflect.StructField, inputVal interface{}, dateFormat string) (output string) {
	if dateFormat == "" {
		dateFormat = "dd/MM/yyyy"
	}

	output = ""

	switch inputModel.Type.Kind() {
	case reflect.Int:
		temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
		output = strconv.FormatInt(temp, 10)
	case reflect.Int16:
		temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
		output = strconv.FormatInt(temp, 10)
	case reflect.Int32:
		temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
		output = strconv.FormatInt(temp, 10)
	case reflect.Int64:
		temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
		output = strconv.FormatInt(temp, 10)
	case reflect.Float32:
		temp, _ := strconv.ParseFloat(strconv.FormatFloat(inputVal.(float64), 'f', 3, 32), 32)
		output = strconv.FormatFloat(temp, 'f', 3, 32)
	case reflect.Float64:
		temp, _ := strconv.ParseFloat(strconv.FormatFloat(inputVal.(float64), 'f', 3, 64), 64)
		output = strconv.FormatFloat(temp, 'f', 3, 64)
	case reflect.Bool:
		temp, _ := strconv.ParseBool(strconv.FormatBool(inputVal.(bool)))
		output = strconv.FormatBool(temp)
	case reflect.String:
		output += "\"" + inputVal.(string) + "\""
	default:
		dtype := DetectDataType(inputVal.(string), dateFormat)
		if dtype == "date" {
			output = "\"" + cast.Date2String(cast.String2Date(inputVal.(string), dateFormat), dateFormat) + "\""
		}
	}

	return output
}
Example #12
0
func (g *GrabService) AddHistory(history toolkit.M) {
	mapHeader := make([]toolkit.M, 7)
	mapHeader[0] = toolkit.M{}.Set("datasettingname", "string")
	mapHeader[1] = toolkit.M{}.Set("grabdate", "date")
	mapHeader[2] = toolkit.M{}.Set("grabstatus", "string")
	mapHeader[3] = toolkit.M{}.Set("rowgrabbed", "int")
	mapHeader[4] = toolkit.M{}.Set("rowsaved", "int")
	mapHeader[5] = toolkit.M{}.Set("note", "string")
	mapHeader[6] = toolkit.M{}.Set("recfile", "string")

	var config = map[string]interface{}{"mapheader": mapHeader, "useheader": true, "delimiter": ",", "newfile": true}
	file := fmt.Sprintf("%s%s-%s.csv", g.HistoryPath, g.Name, cast.Date2String(time.Now(), "YYYYMM"))
	ci := &dbox.ConnectionInfo{file, "", "", "", config}
	c, e := dbox.NewConnection("csv", ci)
	if e != nil {
		g.ErrorNotes = fmt.Sprintf("[%s] Setup connection to history failed [csv-%s]:%s", g.Name, file, e)
		g.Log.AddLog(g.ErrorNotes, "ERROR")
		return
	}

	e = c.Connect()
	if e != nil {
		g.ErrorNotes = fmt.Sprintf("[%s] Setup connection to history failed [csv-%s]:%s", g.Name, file, e)
		g.Log.AddLog(g.ErrorNotes, "ERROR")
		return
	}

	e = c.NewQuery().Insert().Exec(toolkit.M{"data": history})
	if e != nil {
		g.ErrorNotes = fmt.Sprintf("[%s] Insert to history failed [csv-%s]:%s", g.Name, file, e)
		g.Log.AddLog(g.ErrorNotes, "ERROR")
		c.Close()
		return
	}

	c.Close()
}
Example #13
0
func (w *WebGrabberController) GetLog(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	arrcmd := make([]string, 0, 0)
	result := toolkit.M{}
	payload := struct {
		ID   string `json:"_id"`
		Date string `json:"date"`
	}{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	wg := new(colonycore.WebGrabber)
	err = colonycore.Get(wg, payload.ID)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	o, err := toolkit.ToM(wg)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	// history := NewHistory(payload.ID)
	// logs := history.GetLogHistory([]interface{}{o}, payload.Date)
	// apppath := ""
	// if runtime.GOOS == "windows" {
	// 	arrcmd = append(arrcmd, "cmd")
	// 	arrcmd = append(arrcmd, "/C")
	// 	apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
	// } else {
	// 	apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
	// }

	// arrcmd = append(arrcmd, apppath)
	// arrcmd = append(arrcmd, `-readtype=logfile`)
	// arrcmd = append(arrcmd, `-datetime=`+payload.Date)
	// arrcmd = append(arrcmd, `-nameid=`+payload.ID)
	// arrcmd = append(arrcmd, `-datas=`+toolkit.JsonString([]interface{}{o}))

	// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
	// byteoutput, err := cmd.CombinedOutput()
	// if err != nil {
	// 	return helper.CreateResult(false, nil, err.Error())
	// }
	logPath := ""
	for _, v := range []interface{}{o} {
		vMap, _ := toolkit.ToM(v)

		logConf := vMap["logconf"].(map[string]interface{})
		dateNowFormat := logConf["filepattern"].(string)
		logpathconfig := logConf["logpath"].(string)
		theDate := cast.String2Date(payload.Date, "YYYY/MM/dd HH:mm:ss")
		theDateString := cast.Date2String(theDate, dateNowFormat)
		fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
		logPath = logpathconfig + fileName
	}

	client, server, err := w.ConnectToSedotanServer()
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	SshClient := *client

	apppath := ""
	if server.OS == "linux" {
		apppath = server.AppPath + `/cli/sedotanread`
		arrcmd = append(arrcmd, apppath)
		arrcmd = append(arrcmd, `-readtype="logfile"`)
		arrcmd = append(arrcmd, `-nameid="`+payload.ID+`"`)
		arrcmd = append(arrcmd, `-pathfile="`+logPath+`"`)
		arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`)
	} else {
		apppath = server.AppPath + `\bin\sedotanread.exe`
		arrcmd = append(arrcmd, apppath)
		arrcmd = append(arrcmd, `-readtype="logfile"`)
		arrcmd = append(arrcmd, `-nameid=`+payload.ID+`"`)
		arrcmd = append(arrcmd, `-pathfile=`+logPath+`"`)
		arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`)
	}

	cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
	fmt.Println("====>", cmds)
	output, err := SshClient.GetOutputCommandSsh(cmds)
	if err != nil {
		fmt.Println(err)
	}

	err = toolkit.UnjsonFromString(output, &result)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(true, result["DATA"], "")
}
Example #14
0
func (w *WebGrabberController) GetHistory(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	var result = toolkit.M{}
	arrcmd := make([]string, 0, 0)
	payload := new(colonycore.WebGrabber)
	dateNow := cast.Date2String(time.Now(), "YYYYMMdd") //time.Now()
	err := r.GetPayload(payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	err = colonycore.Get(payload, payload.ID)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	// module := NewHistory(payload.HistConf.FileName)
	// history, err := module.OpenHistory()

	client, server, err := w.ConnectToSedotanServer()
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	SshClient := *client

	apppath := ""
	if server.OS == "linux" {
		apppath = server.AppPath + `/cli/sedotanread`
		arrcmd = append(arrcmd, apppath)
		arrcmd = append(arrcmd, `-readtype="history"`)
		arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`/webgrabber/history/`+payload.HistConf.FileName+`-`+dateNow+`.csv"`)
	} else {
		apppath = server.AppPath + `\bin\sedotanread.exe`
		arrcmd = append(arrcmd, apppath)
		arrcmd = append(arrcmd, `-readtype="history"`)
		arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`\webgrabber\history\`+payload.HistConf.FileName+`-`+dateNow+`.csv"`)
	}

	// apppath := ""
	// if runtime.GOOS == "windows" {
	// 	arrcmd = append(arrcmd, "cmd")
	// 	arrcmd = append(arrcmd, "/C")
	// 	apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
	// } else {
	// 	apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
	// }

	// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
	// byteoutput, err := cmd.CombinedOutput()
	// if err != nil {
	// 	return helper.CreateResult(false, nil, err.Error())
	// }

	// err = toolkit.UnjsonFromString(string(byteoutput), &result)
	// if err != nil {
	// 	return helper.CreateResult(false, nil, err.Error())
	// }

	// fmt.Println(strings.Join(append(arrcmd[:1],arrcmd[1:]...)," "))

	cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
	fmt.Println("====>", cmds)
	output, err := SshClient.GetOutputCommandSsh(cmds)
	if err != nil {
		fmt.Println(err)
	}

	err = toolkit.UnjsonFromString(output, &result)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(true, result["DATA"], "")
}
Example #15
0
func (w *WebGrabberController) SaveScrapperData(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := new(colonycore.WebGrabber)
	if err := r.GetPayload(&payload); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	payload.LogConf.FileName = "LOG-" + payload.ID
	payload.LogConf.LogPath = wgLogPath
	payload.LogConf.FilePattern = DateToString(time.Now())

	payload.HistConf.FileName = "HIST-" + payload.ID
	payload.HistConf.Histpath = wgHistoryPath
	payload.HistConf.RecPath = wgHistoryRecPath

	for i, each := range payload.DataSettings {
		if each.DestType == "csv" {
			payload.DataSettings[i].ConnectionInfo.Host = f.Join(wgOutputPath, payload.ID)
		}
	}

	if err := colonycore.Delete(payload); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	if payload.GrabConf["username"] == "" {
		delete(payload.GrabConf, "username")
	}

	if payload.GrabConf["password"] == "" {
		delete(payload.GrabConf, "password")
	}

	if payload.GrabConf["authtype"] == "AuthType_Basic" {
		delete(payload.GrabConf, "loginurl")
		delete(payload.GrabConf, "logouturl")
	} else if payload.GrabConf["authtype"] == "" {
		delete(payload.GrabConf, "loginurl")
		delete(payload.GrabConf, "logouturl")
		delete(payload.GrabConf, "username")
		delete(payload.GrabConf, "password")
	}

	castStartTime, err := time.Parse(time.RFC3339, payload.IntervalConf.StartTime)
	if err == nil {
		payload.IntervalConf.StartTime = cast.Date2String(castStartTime, "YYYY-MM-dd HH:mm:ss")

	}

	castExpTime, err := time.Parse(time.RFC3339, payload.IntervalConf.ExpiredTime)
	if err == nil {
		payload.IntervalConf.ExpiredTime = cast.Date2String(castExpTime, "YYYY-MM-dd HH:mm:ss")
	}

	if err := colonycore.Save(payload); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	if err := w.SyncConfig(); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(true, payload, "")
}
Example #16
0
func (w *WebGrabberController) GetLogHistory(datas []interface{}, date string) interface{} {

	for _, v := range datas {
		vMap, _ := toolkit.ToM(v)

		logConf := vMap["logconf"].(map[string]interface{})
		dateNowFormat := logConf["filepattern"].(string)
		theDate := cast.String2Date(date, "YYYY/MM/dd HH:mm:ss")
		theDateString := cast.Date2String(theDate, dateNowFormat)
		fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
		w.logPath = f.Join(EC_DATA_PATH, "webgrabber", "log", fileName)
	}

	file, err := os.Open(w.logPath)

	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	defer file.Close()
	addMinute := toolkit.String2Date(date, "YYYY/MM/dd HH:mm:ss").Add(1 * time.Minute)
	dateAddMinute := toolkit.Date2String(addMinute, "YYYY/MM/dd HH:mm:ss")
	getHours := strings.Split(date, ":")
	getAddMinute := strings.Split(dateAddMinute, ":")
	containString := getHours[0] + ":" + getHours[1]
	containString2 := getAddMinute[0] + ":" + getAddMinute[1]
	scanner := bufio.NewScanner(file)
	lines := 0
	containLines := 0
	logsseparator := ""

	add7Hours := func(s string) string {
		t, _ := time.Parse("2006/01/02 15:04", s)
		t = t.Add(time.Hour * 7)
		return t.Format("2006/01/02 15:04")
	}

	containString = add7Hours(containString)
	containString2 = add7Hours(containString2)

	var logs []interface{}
	for scanner.Scan() {
		lines++
		contains := strings.Contains(scanner.Text(), containString)
		contains2 := strings.Contains(scanner.Text(), containString2)

		if contains {
			containLines = lines
			logsseparator = containString
		}

		if contains2 {
			containLines = lines
			logsseparator = containString2
		}
		result := toolkit.M{}
		if lines == containLines {
			arrlogs := strings.Split(scanner.Text(), logsseparator)
			result.Set("Type", arrlogs[0])
			result.Set("Date", logsseparator+":"+arrlogs[1][1:3])
			result.Set("Desc", arrlogs[1][4:len(arrlogs[1])])
			logs = append(logs, result)
		}
	}

	if err := scanner.Err(); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	var addSlice = toolkit.M{}
	addSlice.Set("logs", logs)
	return addSlice
}