예제 #1
0
파일: main.go 프로젝트: novalagung/sedotan
func saverechistory(key string, dts []toolkit.M) (fullfilename string, err error) {
	err = nil
	filename := fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
	fullfilename = filepath.Join(toolkit.ToString(histConf.Get("recpath", "")), filename)
	if EC_DATA_PATH != "" {
		fullfilename = filepath.Join(EC_DATA_PATH, "webgrabber", "historyrec", filename)
	}

	cconfig := toolkit.M{"newfile": true, "useheader": true, "delimiter": ","}
	conn, err := prepareconnection("csv", fullfilename, "", "", "", cconfig)
	if err != nil {
		return
	}

	q := conn.NewQuery().SetConfig("multiexec", true).Insert()
	for _, dt := range dts {
		err = q.Exec(toolkit.M{}.Set("data", dt))
	}

	conn.Close()

	return
}
예제 #2
0
// watch, watch the process and mantain the link between the action in the flow
func watch(process colonycore.DataFlow) (e error) {
	act_result_path = ACT_RESULT_PATH + toolkit.Date2String(time.Now(), "dd/MM/yyyy - hh:mm:ss") + process.ID
	globalParam = process.GlobalParam

	var ListCurrentTierAction, ListLastTierAction, ListNextTierAction []colonycore.FlowAction

	//get initial current tier action
	for _, action := range process.Actions {
		if action.FirstAction {
			ListCurrentTierAction = append(ListCurrentTierAction, action)
		}
	}

	for len(ListCurrentTierAction) > 0 {
		nextIdx := []string{}
		for _, tier := range ListCurrentTierAction {
			CurrentAction := tier
			isFound := false
			var files []byte
			var filenames []string
			var ActionBefore []colonycore.FlowAction

			//check previous action result status file to make sure that previous action was complete
			for isFound == false {
				ActionBefore = getActionBefore(ListLastTierAction, CurrentAction)

				if ActionBefore != nil {
					for _, actx := range ActionBefore {
						files, e = getActionResultStatus(process, actx)
						if e != nil {
							return
							break
						}
						if files != nil {
							isFound = true
							filenames = append(filenames, getActionResultStatusPath(process, actx))
						} else {
							isFound = false
							filenames = []string{}
							break
						}
					}
					time.Sleep(time.Second * 5)
				} else {
					isFound = true
				}
			}

			//read previous action result status to get next process ID || get last tier action for next result check || run current action
			if len(filenames) > 0 {
				for _, fname := range filenames {
					files, e := ioutil.ReadFile(fname)

					if e != nil {
						return e
					}

					if strings.Contains(string(files), "OK") {
						for _, ok := range CurrentAction.OK {
							ListLastTierAction = append(ListLastTierAction, CurrentAction)
							nextIdx = append(nextIdx, GetAction(process, ok).Id)
						}
					} else {
						for _, nok := range CurrentAction.KO {
							ListLastTierAction = append(ListLastTierAction, CurrentAction)
							nextIdx = append(nextIdx, GetAction(process, nok).Id)
						}
					}

					go runProcess(process, CurrentAction, ActionBefore)

					//update globalParam
					if CurrentAction.OutputParam != nil {
						for key, val := range CurrentAction.OutputParam {
							if strings.Contains(strings.ToLower(key), GLOBAL_PARAM_KEYWORD) {
								globalParam.Set(key, val)
							}
						}
					}
				}
			}

			//get next tier action
			if len(nextIdx) > 0 {
				for _, idx := range nextIdx {
					ListNextTierAction = append(ListNextTierAction, GetAction(process, idx))
				}
			}
		}

		ListLastTierAction = []colonycore.FlowAction{}
		for _, curr := range ListCurrentTierAction {
			ListLastTierAction = append(ListLastTierAction, curr)
		}

		ListCurrentTierAction = []colonycore.FlowAction{}
		for _, next := range ListNextTierAction {
			ListCurrentTierAction = append(ListCurrentTierAction, next)
		}

		ListNextTierAction = []colonycore.FlowAction{}
	}

	return
}
예제 #3
0
파일: main.go 프로젝트: novalagung/sedotan
func savehistory(dt toolkit.M) (err error) {
	err = nil
	filename := fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))
	fullfilename := filepath.Join(toolkit.ToString(histConf.Get("histpath", "")), filename)
	if EC_DATA_PATH != "" {
		fullfilename = filepath.Join(EC_DATA_PATH, "webgrabber", "history", filename)
	}

	cconfig := toolkit.M{"newfile": true, "useheader": true, "delimiter": ","}
	conn, err := prepareconnection("csv", fullfilename, "", "", "", cconfig)
	if err != nil {
		return
	}

	err = conn.NewQuery().SetConfig("multiexec", true).Insert().Exec(toolkit.M{}.Set("data", dt))

	conn.Close()

	return
}
예제 #4
0
func DateMinutePress(tm time.Time) time.Time {
	return toolkit.String2Date(toolkit.Date2String(tm, "YYYY-MM-dd HH:mm"), "YYYY-MM-dd HH:mm").UTC()
}
예제 #5
0
func DateSecondPress(tm time.Time) time.Time {
	return toolkit.String2Date(toolkit.Date2String(tm, dateformat), dateformat).UTC()
}
예제 #6
0
func DateToString(tm time.Time) string {
	if tm.IsZero() {
		tm = TimeNow()
	}
	return toolkit.Date2String(tm, dateformat)
}
예제 #7
0
func DateToString(tm time.Time) string {
	return toolkit.Date2String(tm, dateformat)
}
예제 #8
0
func (w *WebGrabberController) GetLogHistory(datas []interface{}, date string) interface{} {

	for _, v := range datas {
		vMap, _ := toolkit.ToM(v)

		logConf := vMap["logconf"].(map[string]interface{})
		dateNowFormat := logConf["filepattern"].(string)
		theDate := cast.String2Date(date, "YYYY/MM/dd HH:mm:ss")
		theDateString := cast.Date2String(theDate, dateNowFormat)
		fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
		w.logPath = f.Join(EC_DATA_PATH, "webgrabber", "log", fileName)
	}

	file, err := os.Open(w.logPath)

	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	defer file.Close()
	addMinute := toolkit.String2Date(date, "YYYY/MM/dd HH:mm:ss").Add(1 * time.Minute)
	dateAddMinute := toolkit.Date2String(addMinute, "YYYY/MM/dd HH:mm:ss")
	getHours := strings.Split(date, ":")
	getAddMinute := strings.Split(dateAddMinute, ":")
	containString := getHours[0] + ":" + getHours[1]
	containString2 := getAddMinute[0] + ":" + getAddMinute[1]
	scanner := bufio.NewScanner(file)
	lines := 0
	containLines := 0
	logsseparator := ""

	add7Hours := func(s string) string {
		t, _ := time.Parse("2006/01/02 15:04", s)
		t = t.Add(time.Hour * 7)
		return t.Format("2006/01/02 15:04")
	}

	containString = add7Hours(containString)
	containString2 = add7Hours(containString2)

	var logs []interface{}
	for scanner.Scan() {
		lines++
		contains := strings.Contains(scanner.Text(), containString)
		contains2 := strings.Contains(scanner.Text(), containString2)

		if contains {
			containLines = lines
			logsseparator = containString
		}

		if contains2 {
			containLines = lines
			logsseparator = containString2
		}
		result := toolkit.M{}
		if lines == containLines {
			arrlogs := strings.Split(scanner.Text(), logsseparator)
			result.Set("Type", arrlogs[0])
			result.Set("Date", logsseparator+":"+arrlogs[1][1:3])
			result.Set("Desc", arrlogs[1][4:len(arrlogs[1])])
			logs = append(logs, result)
		}
	}

	if err := scanner.Err(); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	var addSlice = toolkit.M{}
	addSlice.Set("logs", logs)
	return addSlice
}
예제 #9
0
// watch, watch the process and mantain the link between the action in the flow
func watch(process colonycore.DataFlow, user string) (e error) {
	flowprocess := colonycore.DataFlowProcess{
		Id:        generateProcessID(process.ID),
		Flow:      process,
		StartDate: time.Now(),
		StartedBy: user,
		Status:    PROCESS_STATUS_RUN,
	}

	flowprocess.GlobalParam = globalParam
	saveFlowProcess(flowprocess, e)

	ACT_RESULT_PATH, e = filepath.Abs(filepath.Dir(os.Args[0]))
	if e != nil {
		saveFlowProcess(flowprocess, e)
	}
	act_result_path = ACT_RESULT_PATH + "/eaciit/dataflow/result/" + toolkit.Date2String(time.Now(), "dd/MM/yyyy - hh:mm:ss") + process.ID
	var ListCurrentTierAction, ListLastTierAction, ListNextTierAction []colonycore.FlowAction

	//get initial current tier action
	for _, action := range process.Actions {
		if action.FirstAction {
			ListCurrentTierAction = append(ListCurrentTierAction, action)
		}
	}

	for len(ListCurrentTierAction) > 0 {
		nextIdx := []string{}
		for _, tier := range ListCurrentTierAction {
			CurrentAction := tier
			isFound := false
			var files []byte
			var filenames []string
			var ActionBefore []colonycore.FlowAction

			//check previous action result status file to make sure that previous action was complete
			for isFound == false {
				ActionBefore = getActionBefore(ListLastTierAction, CurrentAction)

				if ActionBefore != nil {
					for _, actx := range ActionBefore {
						files, e = getActionResultStatus(process, actx)
						if e != nil {
							saveFlowProcess(flowprocess, e)
							break
						}
						if files != nil {
							isFound = true
							filenames = append(filenames, getActionResultStatusPath(process, actx))
						} else {
							isFound = false
							filenames = []string{}
							break
						}
					}
					time.Sleep(time.Second * 5)
				} else {
					isFound = true
				}

				if isFound && ActionBefore != nil {
					for _, actx := range ActionBefore {
						flowprocess.Steps = append(flowprocess.Steps, actx)
						saveFlowProcess(flowprocess, e)
					}
				}
			}

			var v reflect.Type
			v = reflect.TypeOf(CurrentAction.Action).Elem()

			isFork := false
			isForkAction := false
			if v.Kind() == reflect.Struct {
				for i := 0; i < v.NumField(); i++ {
					if strings.ToLower(v.Field(i).Name) == "isfork" {
						isFork = reflect.ValueOf(CurrentAction.Action).Elem().Field(i).Bool()
						isForkAction = true
						break
					}
				}
			}

			//read previous action result status to get next process ID || get last tier action for next result check || run current action
			if len(filenames) > 0 {
				for _, fname := range filenames {
					files, e := ioutil.ReadFile(fname)

					if e != nil {
						saveFlowProcess(flowprocess, e)
						break
					}

					// if action
					if isForkAction {
						if isFork {
							if strings.Contains(string(files), "OK") {
								for _, ok := range CurrentAction.OK {
									ListLastTierAction = append(ListLastTierAction, CurrentAction)
									nextIdx = append(nextIdx, GetAction(process, ok).Id)
								}
							} else {
								for _, nok := range CurrentAction.KO {
									ListLastTierAction = append(ListLastTierAction, CurrentAction)
									nextIdx = append(nextIdx, GetAction(process, nok).Id)
								}
							}
						} else {
							//decision logic :
							ActionBefore = getActionBefore(ListLastTierAction, CurrentAction)
							ThisAction := CurrentAction.Action.(*colonycore.ActionDecision)
							var DecisionClause []string

							flowprocess.Steps = append(flowprocess.Steps, ThisAction)
							saveFlowProcess(flowprocess, e)

							for _, actionbefore := range ActionBefore {
								//get previous action's result
								outres, e := decodeOutputFile(actionbefore)
								_ = e
								//replace decision variable with result value
								for _, condition := range ThisAction.Conditions {
									for _, outdet := range outres {
										for key, val := range outdet {
											strings.Replace(condition.Stat, string(key), val.(string), -1)
										}
									}

									DecisionClause = append(DecisionClause, "if "+condition.Stat+" {return \"true\"} else {return \"false\"} ")
								}
							}

							//set nextID as per applied condition
							condIdx := 0
							for _, clause := range DecisionClause {
								goClause := `
									func main(){
										` + clause + `
									}
								`
								resultExec, e := golpal.New().Execute(goClause)

								if e != nil {
									saveFlowProcess(flowprocess, e)
								}

								if resultExec == "true" {
									destAction := strings.Split(ThisAction.Conditions[condIdx].FlowAction, ",")

									for _, dest := range destAction {
										nextIdx = append(nextIdx, dest)
									}
								}

								condIdx++
							}
						}
					} else {
						if strings.Contains(string(files), "OK") {
							for _, ok := range CurrentAction.OK {
								ListLastTierAction = append(ListLastTierAction, CurrentAction)
								nextIdx = append(nextIdx, GetAction(process, ok).Id)
							}
						} else {
							for _, nok := range CurrentAction.KO {
								ListLastTierAction = append(ListLastTierAction, CurrentAction)
								nextIdx = append(nextIdx, GetAction(process, nok).Id)
							}
						}
					}

					go runProcess(process, CurrentAction)

					//update globalParam
					if CurrentAction.OutputParam != nil {
						for key, val := range CurrentAction.OutputParam {
							if strings.Contains(strings.ToLower(key), GLOBAL_PARAM_KEYWORD) {
								globalParam.Set(key, val)
								flowprocess.GlobalParam = globalParam
								saveFlowProcess(flowprocess, e)
							}
						}
					}
				}
			}

			//get next tier action
			if len(nextIdx) > 0 {
				for _, idx := range nextIdx {
					ListNextTierAction = append(ListNextTierAction, GetAction(process, idx))
				}
			}
		}

		ListLastTierAction = []colonycore.FlowAction{}
		for _, curr := range ListCurrentTierAction {
			ListLastTierAction = append(ListLastTierAction, curr)
		}

		ListCurrentTierAction = []colonycore.FlowAction{}
		for _, next := range ListNextTierAction {
			ListCurrentTierAction = append(ListCurrentTierAction, next)
		}

		ListNextTierAction = []colonycore.FlowAction{}
	}

	return
}
예제 #10
0
파일: main.go 프로젝트: novalagung/sedotan
func savedatagrab() (err error) {
	var wg, wgstream sync.WaitGroup
	// if len(sGrabber.CollectionSettings) > 0 {
	// 	wg.Add(len(sGrabber.CollectionSettings))
	// }

	mapRecHistory = make(map[string]string, 0)
	historyFileName = fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))

	for key, _ := range sGrabber.CollectionSettings {
		//set history name
		mapRecHistory[key] = fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
		//================
		err = nil
		note := ""
		dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
			Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", mapRecHistory[key])

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data source", key), "INFO")
		iQ, err := sGrabber.GetQuery(key)
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get query data : %s", key, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		defer sGrabber.CloseConn()
		csr, err := iQ.Cursor(nil)
		if err != nil || csr == nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to create cursor or cursor nil to get data : %s", key, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data souce done", key), "INFO")
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save", key), "INFO")
		err = destDboxs[key].IConnection.Connect()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
		if destDboxs[key].collection != "" {
			q = q.From(destDboxs[key].collection)
		}
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save done", key), "INFO")

		//Update Total Process
		mutex.Lock()
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
		err = getsnapshot()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}

		if pid == snapshotdata.Pid {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, csr.Count()), "INFO")
			snapshotdata.Cgtotal += csr.Count()
			err = savesnapshot()
		}

		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}
		mutex.Unlock()

		outtm := make(chan toolkit.M)
		wgstream.Add(1)
		go func(intm chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
			streamsavedata(intm, sQ, key, dt)
			wgstream.Done()
		}(outtm, q, key, dt)

		wg.Add(1)
		go func(outtm chan toolkit.M, csr dbox.ICursor) {
			condition := true

			for condition {
				results := make([]toolkit.M, 0)
				err = csr.Fetch(&results, 1000, false)
				if err != nil {
					note = fmt.Sprintf("[savedatagrab.%s] Unable to fetch data :%s", key, err.Error())
					Log.AddLog(note, "ERROR")
					condition = false
					continue
				}

				for _, result := range results {
					xresult := toolkit.M{}
					for _, column := range sGrabber.CollectionSettings[key].MapsColumns {
						var val, tval interface{}
						strsplits := strings.Split(column.Source, "|")

						ttm := toolkit.M{}
						lskey := ""
						for i, strsplit := range strsplits {
							lskey = strsplit
							if i == 0 && result.Has(strsplit) {
								ttm.Set(strsplit, result[strsplit])
							} else if ttm.Has(strsplit) {
								ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
							}
						}

						strsplits = strings.Split(column.Destination, "|")
						if ttm.Has(lskey) {
							tval = ttm[lskey]
						} else {
							tval = sedotan.DefaultValue(column.DType)
						}

						// if len(strsplits) > 1 {
						// 	if xresult.Has(strsplits[0]) {
						// 		val = xre
						// 	} else {

						// 	}
						// }

						tm := toolkit.M{}
						if xresult.Has(strsplits[0]) {
							tm, _ = toolkit.ToM(xresult[strsplits[0]])
						}
						// xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
						switch column.DType {
						case "int":
							tval = toolkit.ToInt(tval, toolkit.RoundingAuto)
						case "float":
							tval = toolkit.ToFloat64(tval, 2, toolkit.RoundingAuto)
						case "string":
							tval = toolkit.ToString(tval)
						default:
							tval = tval
						}

						val = getresultobj(strsplits, tval, tm)

						// if len(strsplits) == 1 {
						// 	val = tval
						// } else {
						// 	val = getresultobj(strsplits[1:], tval)
						// }

						// if len(strsplits) > 1 && {

						// }
						xresult.Set(strsplits[0], val)

						// for i, strsplit := range strsplits {
						// 	if i == 0 && ttm.Has(lskey) {
						// 		xresult.Set(strsplit, ttm[lskey])
						// 	} else if (i + 1) == len(strsplits) {

						// 	}
						// 	// else if ttm.Has(strsplit) {
						// 	// 	ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
						// 	// }
						// }

					}
					outtm <- xresult
				}

				if len(results) < 1000 || err != nil {
					condition = false
				}
			}
			//
			//

			// ms := []toolkit.M{}
			// for _, val := range results {
			// 	m := toolkit.M{}
			// 	for _, column := range g.CollectionSettings[dataSettingId].MapsColumns {
			// 		m.Set(column.Source, "")
			// 		if val.Has(column.Destination) {
			// 			m.Set(column.Source, val[column.Destination])
			// 		}
			// 	}
			// 	ms = append(ms, m)
			// }

			close(outtm)
			wg.Done()
		}(outtm, csr)

	}

	wgstream.Wait()
	wg.Wait()

	return
}