func saverechistory(key string, dts []toolkit.M) (fullfilename string, err error) {
err = nil
filename := fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
fullfilename = filepath.Join(toolkit.ToString(histConf.Get("recpath", "")), filename)
if EC_DATA_PATH != "" {
fullfilename = filepath.Join(EC_DATA_PATH, "webgrabber", "historyrec", filename)
}
cconfig := toolkit.M{"newfile": true, "useheader": true, "delimiter": ","}
conn, err := prepareconnection("csv", fullfilename, "", "", "", cconfig)
if err != nil {
return
}
q := conn.NewQuery().SetConfig("multiexec", true).Insert()
for _, dt := range dts {
err = q.Exec(toolkit.M{}.Set("data", dt))
}
conn.Close()
return
}
// watch, watch the process and mantain the link between the action in the flow
func watch(process colonycore.DataFlow) (e error) {
act_result_path = ACT_RESULT_PATH + toolkit.Date2String(time.Now(), "dd/MM/yyyy - hh:mm:ss") + process.ID
globalParam = process.GlobalParam
var ListCurrentTierAction, ListLastTierAction, ListNextTierAction []colonycore.FlowAction
//get initial current tier action
for _, action := range process.Actions {
if action.FirstAction {
ListCurrentTierAction = append(ListCurrentTierAction, action)
}
}
for len(ListCurrentTierAction) > 0 {
nextIdx := []string{}
for _, tier := range ListCurrentTierAction {
CurrentAction := tier
isFound := false
var files []byte
var filenames []string
var ActionBefore []colonycore.FlowAction
//check previous action result status file to make sure that previous action was complete
for isFound == false {
ActionBefore = getActionBefore(ListLastTierAction, CurrentAction)
if ActionBefore != nil {
for _, actx := range ActionBefore {
files, e = getActionResultStatus(process, actx)
if e != nil {
return
break
}
if files != nil {
isFound = true
filenames = append(filenames, getActionResultStatusPath(process, actx))
} else {
isFound = false
filenames = []string{}
break
}
}
time.Sleep(time.Second * 5)
} else {
isFound = true
}
}
//read previous action result status to get next process ID || get last tier action for next result check || run current action
if len(filenames) > 0 {
for _, fname := range filenames {
files, e := ioutil.ReadFile(fname)
if e != nil {
return e
}
if strings.Contains(string(files), "OK") {
for _, ok := range CurrentAction.OK {
ListLastTierAction = append(ListLastTierAction, CurrentAction)
nextIdx = append(nextIdx, GetAction(process, ok).Id)
}
} else {
for _, nok := range CurrentAction.KO {
ListLastTierAction = append(ListLastTierAction, CurrentAction)
nextIdx = append(nextIdx, GetAction(process, nok).Id)
}
}
go runProcess(process, CurrentAction, ActionBefore)
//update globalParam
if CurrentAction.OutputParam != nil {
for key, val := range CurrentAction.OutputParam {
if strings.Contains(strings.ToLower(key), GLOBAL_PARAM_KEYWORD) {
globalParam.Set(key, val)
}
}
}
}
}
//get next tier action
if len(nextIdx) > 0 {
for _, idx := range nextIdx {
ListNextTierAction = append(ListNextTierAction, GetAction(process, idx))
}
}
}
ListLastTierAction = []colonycore.FlowAction{}
for _, curr := range ListCurrentTierAction {
ListLastTierAction = append(ListLastTierAction, curr)
}
ListCurrentTierAction = []colonycore.FlowAction{}
for _, next := range ListNextTierAction {
ListCurrentTierAction = append(ListCurrentTierAction, next)
}
ListNextTierAction = []colonycore.FlowAction{}
}
return
}
func savehistory(dt toolkit.M) (err error) {
err = nil
filename := fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))
fullfilename := filepath.Join(toolkit.ToString(histConf.Get("histpath", "")), filename)
if EC_DATA_PATH != "" {
fullfilename = filepath.Join(EC_DATA_PATH, "webgrabber", "history", filename)
}
cconfig := toolkit.M{"newfile": true, "useheader": true, "delimiter": ","}
conn, err := prepareconnection("csv", fullfilename, "", "", "", cconfig)
if err != nil {
return
}
err = conn.NewQuery().SetConfig("multiexec", true).Insert().Exec(toolkit.M{}.Set("data", dt))
conn.Close()
return
}
func DateMinutePress(tm time.Time) time.Time {
return toolkit.String2Date(toolkit.Date2String(tm, "YYYY-MM-dd HH:mm"), "YYYY-MM-dd HH:mm").UTC()
}
func DateSecondPress(tm time.Time) time.Time {
return toolkit.String2Date(toolkit.Date2String(tm, dateformat), dateformat).UTC()
}
func DateToString(tm time.Time) string {
if tm.IsZero() {
tm = TimeNow()
}
return toolkit.Date2String(tm, dateformat)
}
func DateToString(tm time.Time) string {
return toolkit.Date2String(tm, dateformat)
}
func (w *WebGrabberController) GetLogHistory(datas []interface{}, date string) interface{} {
for _, v := range datas {
vMap, _ := toolkit.ToM(v)
logConf := vMap["logconf"].(map[string]interface{})
dateNowFormat := logConf["filepattern"].(string)
theDate := cast.String2Date(date, "YYYY/MM/dd HH:mm:ss")
theDateString := cast.Date2String(theDate, dateNowFormat)
fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
w.logPath = f.Join(EC_DATA_PATH, "webgrabber", "log", fileName)
}
file, err := os.Open(w.logPath)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
defer file.Close()
addMinute := toolkit.String2Date(date, "YYYY/MM/dd HH:mm:ss").Add(1 * time.Minute)
dateAddMinute := toolkit.Date2String(addMinute, "YYYY/MM/dd HH:mm:ss")
getHours := strings.Split(date, ":")
getAddMinute := strings.Split(dateAddMinute, ":")
containString := getHours[0] + ":" + getHours[1]
containString2 := getAddMinute[0] + ":" + getAddMinute[1]
scanner := bufio.NewScanner(file)
lines := 0
containLines := 0
logsseparator := ""
add7Hours := func(s string) string {
t, _ := time.Parse("2006/01/02 15:04", s)
t = t.Add(time.Hour * 7)
return t.Format("2006/01/02 15:04")
}
containString = add7Hours(containString)
containString2 = add7Hours(containString2)
var logs []interface{}
for scanner.Scan() {
lines++
contains := strings.Contains(scanner.Text(), containString)
contains2 := strings.Contains(scanner.Text(), containString2)
if contains {
containLines = lines
logsseparator = containString
}
if contains2 {
containLines = lines
logsseparator = containString2
}
result := toolkit.M{}
if lines == containLines {
arrlogs := strings.Split(scanner.Text(), logsseparator)
result.Set("Type", arrlogs[0])
result.Set("Date", logsseparator+":"+arrlogs[1][1:3])
result.Set("Desc", arrlogs[1][4:len(arrlogs[1])])
logs = append(logs, result)
}
}
if err := scanner.Err(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
var addSlice = toolkit.M{}
addSlice.Set("logs", logs)
return addSlice
}
// watch, watch the process and mantain the link between the action in the flow
func watch(process colonycore.DataFlow, user string) (e error) {
flowprocess := colonycore.DataFlowProcess{
Id: generateProcessID(process.ID),
Flow: process,
StartDate: time.Now(),
StartedBy: user,
Status: PROCESS_STATUS_RUN,
}
flowprocess.GlobalParam = globalParam
saveFlowProcess(flowprocess, e)
ACT_RESULT_PATH, e = filepath.Abs(filepath.Dir(os.Args[0]))
if e != nil {
saveFlowProcess(flowprocess, e)
}
act_result_path = ACT_RESULT_PATH + "/eaciit/dataflow/result/" + toolkit.Date2String(time.Now(), "dd/MM/yyyy - hh:mm:ss") + process.ID
var ListCurrentTierAction, ListLastTierAction, ListNextTierAction []colonycore.FlowAction
//get initial current tier action
for _, action := range process.Actions {
if action.FirstAction {
ListCurrentTierAction = append(ListCurrentTierAction, action)
}
}
for len(ListCurrentTierAction) > 0 {
nextIdx := []string{}
for _, tier := range ListCurrentTierAction {
CurrentAction := tier
isFound := false
var files []byte
var filenames []string
var ActionBefore []colonycore.FlowAction
//check previous action result status file to make sure that previous action was complete
for isFound == false {
ActionBefore = getActionBefore(ListLastTierAction, CurrentAction)
if ActionBefore != nil {
for _, actx := range ActionBefore {
files, e = getActionResultStatus(process, actx)
if e != nil {
saveFlowProcess(flowprocess, e)
break
}
if files != nil {
isFound = true
filenames = append(filenames, getActionResultStatusPath(process, actx))
} else {
isFound = false
filenames = []string{}
break
}
}
time.Sleep(time.Second * 5)
} else {
isFound = true
}
if isFound && ActionBefore != nil {
for _, actx := range ActionBefore {
flowprocess.Steps = append(flowprocess.Steps, actx)
saveFlowProcess(flowprocess, e)
}
}
}
var v reflect.Type
v = reflect.TypeOf(CurrentAction.Action).Elem()
isFork := false
isForkAction := false
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
if strings.ToLower(v.Field(i).Name) == "isfork" {
isFork = reflect.ValueOf(CurrentAction.Action).Elem().Field(i).Bool()
isForkAction = true
break
}
}
}
//read previous action result status to get next process ID || get last tier action for next result check || run current action
if len(filenames) > 0 {
for _, fname := range filenames {
files, e := ioutil.ReadFile(fname)
if e != nil {
saveFlowProcess(flowprocess, e)
break
}
// if action
if isForkAction {
if isFork {
if strings.Contains(string(files), "OK") {
for _, ok := range CurrentAction.OK {
ListLastTierAction = append(ListLastTierAction, CurrentAction)
nextIdx = append(nextIdx, GetAction(process, ok).Id)
}
} else {
for _, nok := range CurrentAction.KO {
ListLastTierAction = append(ListLastTierAction, CurrentAction)
nextIdx = append(nextIdx, GetAction(process, nok).Id)
}
}
} else {
//decision logic :
ActionBefore = getActionBefore(ListLastTierAction, CurrentAction)
ThisAction := CurrentAction.Action.(*colonycore.ActionDecision)
var DecisionClause []string
flowprocess.Steps = append(flowprocess.Steps, ThisAction)
saveFlowProcess(flowprocess, e)
for _, actionbefore := range ActionBefore {
//get previous action's result
outres, e := decodeOutputFile(actionbefore)
_ = e
//replace decision variable with result value
for _, condition := range ThisAction.Conditions {
for _, outdet := range outres {
for key, val := range outdet {
strings.Replace(condition.Stat, string(key), val.(string), -1)
}
}
DecisionClause = append(DecisionClause, "if "+condition.Stat+" {return \"true\"} else {return \"false\"} ")
}
}
//set nextID as per applied condition
condIdx := 0
for _, clause := range DecisionClause {
goClause := `
func main(){
` + clause + `
}
`
resultExec, e := golpal.New().Execute(goClause)
if e != nil {
saveFlowProcess(flowprocess, e)
}
if resultExec == "true" {
destAction := strings.Split(ThisAction.Conditions[condIdx].FlowAction, ",")
for _, dest := range destAction {
nextIdx = append(nextIdx, dest)
}
}
condIdx++
}
}
} else {
if strings.Contains(string(files), "OK") {
for _, ok := range CurrentAction.OK {
ListLastTierAction = append(ListLastTierAction, CurrentAction)
nextIdx = append(nextIdx, GetAction(process, ok).Id)
}
} else {
for _, nok := range CurrentAction.KO {
ListLastTierAction = append(ListLastTierAction, CurrentAction)
nextIdx = append(nextIdx, GetAction(process, nok).Id)
}
}
}
go runProcess(process, CurrentAction)
//update globalParam
if CurrentAction.OutputParam != nil {
for key, val := range CurrentAction.OutputParam {
if strings.Contains(strings.ToLower(key), GLOBAL_PARAM_KEYWORD) {
globalParam.Set(key, val)
flowprocess.GlobalParam = globalParam
saveFlowProcess(flowprocess, e)
}
}
}
}
}
//get next tier action
if len(nextIdx) > 0 {
for _, idx := range nextIdx {
ListNextTierAction = append(ListNextTierAction, GetAction(process, idx))
}
}
}
ListLastTierAction = []colonycore.FlowAction{}
for _, curr := range ListCurrentTierAction {
ListLastTierAction = append(ListLastTierAction, curr)
}
ListCurrentTierAction = []colonycore.FlowAction{}
for _, next := range ListNextTierAction {
ListCurrentTierAction = append(ListCurrentTierAction, next)
}
ListNextTierAction = []colonycore.FlowAction{}
}
return
}
func savedatagrab() (err error) {
var wg, wgstream sync.WaitGroup
// if len(sGrabber.CollectionSettings) > 0 {
// wg.Add(len(sGrabber.CollectionSettings))
// }
mapRecHistory = make(map[string]string, 0)
historyFileName = fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))
for key, _ := range sGrabber.CollectionSettings {
//set history name
mapRecHistory[key] = fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
//================
err = nil
note := ""
dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", mapRecHistory[key])
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data source", key), "INFO")
iQ, err := sGrabber.GetQuery(key)
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get query data : %s", key, err.Error())
errlogsavehistory(note, dt)
continue
}
defer sGrabber.CloseConn()
csr, err := iQ.Cursor(nil)
if err != nil || csr == nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to create cursor or cursor nil to get data : %s", key, err.Error())
errlogsavehistory(note, dt)
continue
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data souce done", key), "INFO")
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save", key), "INFO")
err = destDboxs[key].IConnection.Connect()
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
errlogsavehistory(note, dt)
continue
}
q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
if destDboxs[key].collection != "" {
q = q.From(destDboxs[key].collection)
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save done", key), "INFO")
//Update Total Process
mutex.Lock()
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
err = getsnapshot()
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
Log.AddLog(note, "ERROR")
}
if pid == snapshotdata.Pid {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, csr.Count()), "INFO")
snapshotdata.Cgtotal += csr.Count()
err = savesnapshot()
}
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
Log.AddLog(note, "ERROR")
}
mutex.Unlock()
outtm := make(chan toolkit.M)
wgstream.Add(1)
go func(intm chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
streamsavedata(intm, sQ, key, dt)
wgstream.Done()
}(outtm, q, key, dt)
wg.Add(1)
go func(outtm chan toolkit.M, csr dbox.ICursor) {
condition := true
for condition {
results := make([]toolkit.M, 0)
err = csr.Fetch(&results, 1000, false)
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to fetch data :%s", key, err.Error())
Log.AddLog(note, "ERROR")
condition = false
continue
}
for _, result := range results {
xresult := toolkit.M{}
for _, column := range sGrabber.CollectionSettings[key].MapsColumns {
var val, tval interface{}
strsplits := strings.Split(column.Source, "|")
ttm := toolkit.M{}
lskey := ""
for i, strsplit := range strsplits {
lskey = strsplit
if i == 0 && result.Has(strsplit) {
ttm.Set(strsplit, result[strsplit])
} else if ttm.Has(strsplit) {
ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
}
}
strsplits = strings.Split(column.Destination, "|")
if ttm.Has(lskey) {
tval = ttm[lskey]
} else {
tval = sedotan.DefaultValue(column.DType)
}
// if len(strsplits) > 1 {
// if xresult.Has(strsplits[0]) {
// val = xre
// } else {
// }
// }
tm := toolkit.M{}
if xresult.Has(strsplits[0]) {
tm, _ = toolkit.ToM(xresult[strsplits[0]])
}
// xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
switch column.DType {
case "int":
tval = toolkit.ToInt(tval, toolkit.RoundingAuto)
case "float":
tval = toolkit.ToFloat64(tval, 2, toolkit.RoundingAuto)
case "string":
tval = toolkit.ToString(tval)
default:
tval = tval
}
val = getresultobj(strsplits, tval, tm)
// if len(strsplits) == 1 {
// val = tval
// } else {
// val = getresultobj(strsplits[1:], tval)
// }
// if len(strsplits) > 1 && {
// }
xresult.Set(strsplits[0], val)
// for i, strsplit := range strsplits {
// if i == 0 && ttm.Has(lskey) {
// xresult.Set(strsplit, ttm[lskey])
// } else if (i + 1) == len(strsplits) {
// }
// // else if ttm.Has(strsplit) {
// // ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
// // }
// }
}
outtm <- xresult
}
if len(results) < 1000 || err != nil {
condition = false
}
}
//
//
// ms := []toolkit.M{}
// for _, val := range results {
// m := toolkit.M{}
// for _, column := range g.CollectionSettings[dataSettingId].MapsColumns {
// m.Set(column.Source, "")
// if val.Has(column.Destination) {
// m.Set(column.Source, val[column.Destination])
// }
// }
// ms = append(ms, m)
// }
close(outtm)
wg.Done()
}(outtm, csr)
}
wgstream.Wait()
wg.Wait()
return
}