func genGo(fi os.FileInfo, source, out string) error { log.Info("Processing " + fi.Name()) fn := filepath.Join(source, fi.Name()) var ( bs []byte e error ) if bs, e = ioutil.ReadFile(fn); e != nil { return errors.New("Open error " + e.Error()) } pkg := new(PackageModel) e = toolkit.UnjsonFromString(string(bs), pkg) if e != nil { return errors.New("Unmarshal JSON: " + e.Error()) } for _, sm := range pkg.Structs { e = sm.Write(pkg, out) if e != nil { return errors.New(toolkit.Sprintf("Write model %s: %s", sm.Name, e.Error())) } log.Info(toolkit.Sprintf("Writing %s.%s", pkg.Name, sm.Name)) } log.Info("Processing " + fi.Name() + " done") return nil }
func TestRunSedotanReadRecHistory(t *testing.T) { t.Skip("Skip : Comment this line to do test") arrcmd := make([]string, 0, 0) result := toolkit.M{} arrcmd = append(arrcmd, EC_APP_PATH+`\bin\sedotanread.exe`) arrcmd = append(arrcmd, `-readtype=rechistory`) arrcmd = append(arrcmd, `-recfile=E:\EACIIT\src\github.com\eaciit\colony-app\data-root\webgrabber\historyrec\irondcecomcn.Iron01-20160316022830.csv`) if runtime.GOOS == "windows" { cmd = exec.Command(arrcmd[0], arrcmd[1:]...) rechistory, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:]...) err = toolkit.UnjsonFromString(rechistory, &result) if err != nil { t.Errorf("Error, %s \n", err) } byteoutput, err := cmd.CombinedOutput() if err != nil { // Log.AddLog(fmt.Sprintf("[%v] run at %v, found error : %v", eid, sedotan.DateToString(thistime), err.Error()), "ERROR") } err = toolkit.UnjsonFromString(string(byteoutput), &result) } else { // cmd = exec.Command("sudo", "../daemon/sedotandaemon", `-config="`+tbasepath+`\config-daemon.json"`, `-logpath="`+tbasepath+`\log"`) } fmt.Println(result) }
func (q *Query) openFile() error { if q.fileHasBeenOpened { return nil } _, e := os.Stat(q.jsonPath) if e != nil && (strings.Contains(e.Error(), "does not exist") || strings.Contains(e.Error(), "no such file or directory")) { q.data = []toolkit.M{} return nil } else if e != nil { return err.Error(packageName, modQuery, "openFile: Open file fail", e.Error()) } bs, e := ioutil.ReadFile(q.jsonPath) if e != nil { return err.Error(packageName, modQuery, "openFile: Read file data fail", e.Error()) } jsonText := string(bs) var tempData []toolkit.M e = toolkit.UnjsonFromString(jsonText, &tempData) if e != nil { return err.Error(packageName, modQuery, "openFile: Serializaion fail", e.Error()) } q.data = tempData q.fileHasBeenOpened = true return nil }
func (d *DataBrowserController) parseQuery(conn dbox.IConnection, dbrowser colonycore.DataBrowser, datacon *colonycore.Connection) (dbox.IQuery, error) { var dataQuery dbox.IQuery if dbrowser.QueryType == "nonQueryText" { dataQuery = conn.NewQuery().From(dbrowser.TableNames) } else if dbrowser.QueryType == "SQL" { if toolkit.HasMember(rdbms, datacon.Driver) { dataQuery = conn.NewQuery().Command("freequery", toolkit.M{}. Set("syntax", dbrowser.QueryText)) } else { return nil, errors.New("Free Text Query with SQL only for RDBMS, please use Dbox") } } else if dbrowser.QueryType == "Dbox" { queryInfo := toolkit.M{} toolkit.UnjsonFromString(dbrowser.QueryText, &queryInfo) toolkit.Println("queryinfo", queryInfo) if qFrom := queryInfo.Get("from", "").(string); qFrom != "" { dataQuery = conn.NewQuery() dataQuery = dataQuery.From(qFrom) } if qSelect := queryInfo.Get("select", "").(string); qSelect != "" { if qSelect != "*" { dataQuery = dataQuery.Select(strings.Split(qSelect, ",")...) } } } return dataQuery, nil }
func main() { jsonarg := toolkit.M{} err := toolkit.UnjsonFromString(os.Args[1], &jsonarg) if err != nil { return } jsonarg["Date"] = "000000" jsonarg[`MB 62% Fe`] = "POST " + toolkit.ToString(jsonarg[`MB 62% Fe`]) jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto) * 5) fmt.Println(toolkit.JsonString(jsonarg)) }
func main() { jsonarg := toolkit.M{} err := toolkit.UnjsonFromString(os.Args[1], &jsonarg) if err != nil { return } jsonarg["Date"] = time.Now().Format("020106") jsonarg[`MB 62% Fe`] = "FE " + toolkit.ToString(jsonarg[`MB 62% Fe`]) jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto)) fmt.Println(toolkit.JsonString(jsonarg)) }
func TestRunSedotanReadHistory(t *testing.T) { t.Skip("Skip : Comment this line to do test") var history = toolkit.M{} arrcmd := make([]string, 0, 0) arrcmd = append(arrcmd, EC_APP_PATH+`\bin\sedotanread.exe`) arrcmd = append(arrcmd, `-readtype=history`) arrcmd = append(arrcmd, `-pathfile=`+EC_DATA_PATH+`\webgrabber\history\HIST-GRABDCE-20160316.csv`) if runtime.GOOS == "windows" { historystring, _ := toolkit.RunCommand(arrcmd[0], arrcmd[1:]...) err := toolkit.UnjsonFromString(historystring, &history) if err != nil { t.Errorf("Error, %s \n", err) } } else { // cmd = exec.Command("sudo", "../daemon/sedotandaemon", `-config="`+tbasepath+`\config-daemon.json"`, `-logpath="`+tbasepath+`\log"`) } fmt.Println(history) }
func (q *Query) openFile(commandtype string) error { if q.fileHasBeenOpened { return nil } _, e := os.Stat(q.jsonPath) setting := q.Connection().Info().Settings if e != nil && toolkit.HasMember([]interface{}{dbox.QueryPartSave, dbox.QueryPartInsert}, commandtype) && strings.Contains(e.Error(), "cannot find the file specified") && setting != nil { newfile := setting.Get("newfile", false).(bool) if newfile { e = q.writeFile() if e != nil { return err.Error(packageName, modQuery, "openFile: "+commandtype+" Write fail", e.Error()) } } else { return err.Error(packageName, modQuery, "openFile: "+commandtype+" Create new file is false", e.Error()) } } else if e != nil && (strings.Contains(e.Error(), "does not exist") || strings.Contains(e.Error(), "no such file or directory")) { q.data = []toolkit.M{} return nil } else if e != nil { return err.Error(packageName, modQuery, "openFile: Open file fail", e.Error()) } bs, e := ioutil.ReadFile(q.jsonPath) if e != nil { return err.Error(packageName, modQuery, "openFile: Read file data fail", e.Error()) } jsonText := string(bs) var tempData []toolkit.M e = toolkit.UnjsonFromString(jsonText, &tempData) if e != nil { return err.Error(packageName, modQuery, "openFile: Serializaion fail", e.Error()) } q.data = tempData q.fileHasBeenOpened = true return nil }
func TestRunSedotanReadSnapshot(t *testing.T) { t.Skip("Skip : Comment this line to do test") arrcmd := make([]string, 0, 0) result := toolkit.M{} arrcmd = append(arrcmd, EC_APP_PATH+`\bin\sedotanread.exe`) arrcmd = append(arrcmd, `-readtype=snapshot`) arrcmd = append(arrcmd, `-pathfile=`+EC_DATA_PATH+`\daemon\daemonsnapshot.csv`) arrcmd = append(arrcmd, `-nameid=irondcecomcn`) if runtime.GOOS == "windows" { SnapShot, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:]...) err = toolkit.UnjsonFromString(SnapShot, &result) if err != nil { t.Errorf("Error, %s \n", err) } } else { // cmd = exec.Command("sudo", "../daemon/sedotandaemon", `-config="`+tbasepath+`\config-daemon.json"`, `-logpath="`+tbasepath+`\log"`) } fmt.Println(result) }
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) { var hasLookup bool toolkit.Println("payload : ", payload) if payload.Has("haslookup") { hasLookup = payload.Get("haslookup").(bool) } _id := toolkit.ToString(payload.Get("browserid", "")) sort := payload.Get("sort") search := payload.Get("search") _ = search take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto) skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto) TblName := toolkit.M{} payload.Unset("browserid") //sorter = "" if sort != nil { tmsort, _ := toolkit.ToM(sort.([]interface{})[0]) fmt.Printf("====== sort %#v\n", tmsort["dir"]) if tmsort["dir"] == "asc" { sorter = tmsort["field"].(string) } else if tmsort["dir"] == "desc" { sorter = "-" + tmsort["field"].(string) } else if tmsort["dir"] == nil { sorter = " " } } else { sorter = " " } dataDS := new(colonycore.DataBrowser) err := colonycore.Get(dataDS, _id) if err != nil { return 0, nil, nil, err } dataConn := new(colonycore.Connection) err = colonycore.Get(dataConn, dataDS.ConnectionID) if err != nil { return 0, nil, nil, err } if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil { return 0, nil, nil, err } connection, err := helper.ConnectUsingDataConn(dataConn).Connect() if err != nil { return 0, nil, nil, err } if dataDS.QueryType == "" { TblName.Set("from", dataDS.TableNames) payload.Set("from", dataDS.TableNames) } else if dataDS.QueryType == "Dbox" { getTableName := toolkit.M{} toolkit.UnjsonFromString(dataDS.QueryText, &getTableName) payload.Set("from", getTableName.Get("from").(string)) if qSelect := getTableName.Get("select", "").(string); qSelect != "" { payload.Set("select", getTableName.Get("select").(string)) } } else if dataDS.QueryType == "SQL" { var QueryString string if dataConn.Driver == "mysql" || dataConn.Driver == "hive" { QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip) } else if dataConn.Driver == "mssql" { QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " + toolkit.ToString(take) + " ROWS ONLY " } else if dataConn.Driver == "postgres" { QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip) } stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString) payload.Set("freetext", stringQuery) // toolkit.Println(stringQuery) } qcount, _ := d.parseQuery(connection.NewQuery(), TblName) query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload) var selectfield string for _, metadata := range dataDS.MetaData { tField := metadata.Field if payload.Has(tField) { selectfield = toolkit.ToString(tField) if toolkit.IsSlice(payload[tField]) { query = query.Where(dbox.In(tField, payload[tField].([]interface{})...)) qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...)) } else if !toolkit.IsNilOrEmpty(payload[tField]) { var hasPattern bool for _, val := range querypattern { if strings.Contains(toolkit.ToString(payload[tField]), val) { hasPattern = true } } if hasPattern { query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]), toolkit.ToString(metadata.DataType), "")) qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]), toolkit.ToString(metadata.DataType), "")) } else { switch toolkit.ToString(metadata.DataType) { case "int": query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto))) qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto))) case "float32": query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto))) qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto))) case "float64": query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto))) qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto))) default: query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField]))) qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField]))) } } } } } if hasLookup && selectfield != "" { if toolkit.HasMember(ds_flatfile, dataConn.Driver) { query = query.Select(selectfield) qcount = qcount.Select(selectfield) } else { query = query.Select(selectfield).Group(selectfield) qcount = qcount.Select(selectfield).Group(selectfield) } } ccount, err := qcount.Cursor(nil) if err != nil { return 0, nil, nil, err } defer ccount.Close() dcount := ccount.Count() cursor, err := query.Cursor(nil) if err != nil { return 0, nil, nil, err } defer cursor.Close() data := []toolkit.M{} cursor.Fetch(&data, 0, false) if err != nil { return 0, nil, nil, err } if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) && !toolkit.HasMember(ds_flatfile, dataConn.Driver) { dataMongo := []toolkit.M{} for _, val := range data { mVal, _ := toolkit.ToM(val.Get("_id")) dataMongo = append(dataMongo, mVal) } data = dataMongo } else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) { /*distinct value for flat file*/ dataFlat := []toolkit.M{} var existingVal = []string{""} for _, val := range data { valString := toolkit.ToString(val.Get(selectfield)) if !toolkit.HasMember(existingVal, valString) { dataFlat = append(dataFlat, val) existingVal = append(existingVal, valString) } } data = dataFlat } return dcount, data, dataDS, nil }
func (w *WebGrabberController) GetLog(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson arrcmd := make([]string, 0, 0) result := toolkit.M{} payload := struct { ID string `json:"_id"` Date string `json:"date"` }{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } wg := new(colonycore.WebGrabber) err = colonycore.Get(wg, payload.ID) if err != nil { return helper.CreateResult(false, nil, err.Error()) } o, err := toolkit.ToM(wg) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // history := NewHistory(payload.ID) // logs := history.GetLogHistory([]interface{}{o}, payload.Date) // apppath := "" // if runtime.GOOS == "windows" { // arrcmd = append(arrcmd, "cmd") // arrcmd = append(arrcmd, "/C") // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe") // } else { // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread") // } // arrcmd = append(arrcmd, apppath) // arrcmd = append(arrcmd, `-readtype=logfile`) // arrcmd = append(arrcmd, `-datetime=`+payload.Date) // arrcmd = append(arrcmd, `-nameid=`+payload.ID) // arrcmd = append(arrcmd, `-datas=`+toolkit.JsonString([]interface{}{o})) // cmd := exec.Command(arrcmd[0], arrcmd[1:]...) // byteoutput, err := cmd.CombinedOutput() // if err != nil { // return helper.CreateResult(false, nil, err.Error()) // } logPath := "" for _, v := range []interface{}{o} { vMap, _ := toolkit.ToM(v) logConf := vMap["logconf"].(map[string]interface{}) dateNowFormat := logConf["filepattern"].(string) logpathconfig := logConf["logpath"].(string) theDate := cast.String2Date(payload.Date, "YYYY/MM/dd HH:mm:ss") theDateString := cast.Date2String(theDate, dateNowFormat) fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString) logPath = logpathconfig + fileName } client, server, err := w.ConnectToSedotanServer() if err != nil { return helper.CreateResult(false, nil, err.Error()) } SshClient := *client apppath := "" if server.OS == "linux" { apppath = server.AppPath + `/cli/sedotanread` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="logfile"`) arrcmd = append(arrcmd, `-nameid="`+payload.ID+`"`) arrcmd = append(arrcmd, `-pathfile="`+logPath+`"`) arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`) } else { apppath = server.AppPath + `\bin\sedotanread.exe` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="logfile"`) arrcmd = append(arrcmd, `-nameid=`+payload.ID+`"`) arrcmd = append(arrcmd, `-pathfile=`+logPath+`"`) arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`) } cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ") fmt.Println("====>", cmds) output, err := SshClient.GetOutputCommandSsh(cmds) if err != nil { fmt.Println(err) } err = toolkit.UnjsonFromString(output, &result) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, result["DATA"], "") }
func (p *Page) Save(payload toolkit.M, isDesigner bool, widgetPath string) error { if isDesigner { p.ID = payload.Get("_id", "").(string) if err := p.GetById(); err != nil { return err } var datasources []string wpArray := p.Widget if payload.Get("dataSourceId") != nil { /*save configuration page*/ for _, ds := range payload.Get("dataSourceId", "").([]interface{}) { datasources = append(datasources, ds.(string)) } p.DataSources = datasources } var mode string if payload.Get("mode") != nil { mode = payload.Get("mode", "").(string) } if mode == "new widget" { /*save new widget*/ newWidget, err := p.SaveNewWidget(payload, widgetPath) wpArray = append(wpArray, newWidget) if err != nil { return err } } else if mode == "delete widget" { widget := p.Widget wpArray = nil for _, val := range widget { if payload.Get("widgetPageId", "") == val.ID { continue } wpArray = append(wpArray, val) } } else if mode == "save widget" { if payload.Get("widget") != nil { getwp := payload.Get("widget").(map[string]interface{}) wp := &WidgetPage{} toolkit.UnjsonFromString(toolkit.JsonString(getwp), wp) p.ID = payload.Get("pageId", "").(string) if err := p.GetById(); err != nil { return err } widget := p.Widget wpArray = nil for _, val := range widget { if wp.ID == val.ID { val = wp } wpArray = append(wpArray, val) } } } p.Widget = wpArray // page.ThemeColor = payload.Get("themeColor", "").(string) // page.SendFiles(EC_DATA_PATH, payload.Get("serverId", "").(string)) } if err := Save(p); err != nil { return err } return nil }
func (w *WebGrabberController) GetSnapshot(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson arrcmd := make([]string, 0, 0) result := toolkit.M{} payload := struct { Nameid string }{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // module := GetDirSnapshot("daemonsnapshot") // SnapShot, err := module.OpenSnapShot(payload.Nameid) // ===================LOCALHOST TEST================================= // apppath := "" // if runtime.GOOS == "windows" { // arrcmd = append(arrcmd, "cmd") // arrcmd = append(arrcmd, "/C") // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe") // } else { // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread") // } // arrcmd = append(arrcmd, apppath) // arrcmd = append(arrcmd, `-readtype=snapshot`) // arrcmd = append(arrcmd, `-pathfile=`+EC_DATA_PATH+`\daemon\daemonsnapshot.csv`) // arrcmd = append(arrcmd, `-nameid=`+payload.Nameid) // cmd := exec.Command(arrcmd[0], arrcmd[1:]...) // byteoutput, err := cmd.CombinedOutput() // if err != nil { // return helper.CreateResult(false, nil, err.Error()) // } // err = toolkit.UnjsonFromString(string(byteoutput), &result) // if err != nil { // return helper.CreateResult(false, nil, err.Error()) // } // ===================END LOCALHOST TEST=============================== client, server, err := w.ConnectToSedotanServer() if err != nil { return helper.CreateResult(false, nil, err.Error()) } SshClient := *client apppath := "" if server.OS == "linux" { apppath = server.AppPath + `/cli/sedotanread` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="snapshot"`) arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`/daemon/daemonsnapshot.csv"`) arrcmd = append(arrcmd, `-nameid="`+payload.Nameid+`"`) } else { apppath = server.AppPath + `\bin\sedotanread.exe` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="snapshot"`) arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`\daemon\daemonsnapshot.csv"`) arrcmd = append(arrcmd, `-nameid="`+payload.Nameid+`"`) } cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ") fmt.Println("====>", cmds) output, err := SshClient.GetOutputCommandSsh(cmds) if err != nil { fmt.Println(err) } err = toolkit.UnjsonFromString(output, &result) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, result["DATA"], "") }
func (w *WebGrabberController) GetHistory(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson var result = toolkit.M{} arrcmd := make([]string, 0, 0) payload := new(colonycore.WebGrabber) dateNow := cast.Date2String(time.Now(), "YYYYMMdd") //time.Now() err := r.GetPayload(payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } err = colonycore.Get(payload, payload.ID) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // module := NewHistory(payload.HistConf.FileName) // history, err := module.OpenHistory() client, server, err := w.ConnectToSedotanServer() if err != nil { return helper.CreateResult(false, nil, err.Error()) } SshClient := *client apppath := "" if server.OS == "linux" { apppath = server.AppPath + `/cli/sedotanread` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="history"`) arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`/webgrabber/history/`+payload.HistConf.FileName+`-`+dateNow+`.csv"`) } else { apppath = server.AppPath + `\bin\sedotanread.exe` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="history"`) arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`\webgrabber\history\`+payload.HistConf.FileName+`-`+dateNow+`.csv"`) } // apppath := "" // if runtime.GOOS == "windows" { // arrcmd = append(arrcmd, "cmd") // arrcmd = append(arrcmd, "/C") // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe") // } else { // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread") // } // cmd := exec.Command(arrcmd[0], arrcmd[1:]...) // byteoutput, err := cmd.CombinedOutput() // if err != nil { // return helper.CreateResult(false, nil, err.Error()) // } // err = toolkit.UnjsonFromString(string(byteoutput), &result) // if err != nil { // return helper.CreateResult(false, nil, err.Error()) // } // fmt.Println(strings.Join(append(arrcmd[:1],arrcmd[1:]...)," ")) cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ") fmt.Println("====>", cmds) output, err := SshClient.GetOutputCommandSsh(cmds) if err != nil { fmt.Println(err) } err = toolkit.UnjsonFromString(output, &result) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, result["DATA"], "") }
func (d *DataBrowserController) hasAggr(ctx dbox.IConnection, data *colonycore.DataBrowser, conn *colonycore.Connection) (*colonycore.DataBrowser, error) { var fieldArr, aggrArr []string var indexAggr []map[int]string var query dbox.IQuery fieldAggr := toolkit.M{} for i, v := range data.MetaData { if v.Aggregate != "" { result := toolkit.M{} toolkit.UnjsonFromString(v.Aggregate, &result) cursor := []toolkit.M{} if data.QueryType == "" { aggregate, e := d.dboxAggr(data.TableNames, v.Field, ctx, query, result, fieldAggr, cursor, conn) if e != nil { return nil, e } v.Aggregate = toolkit.JsonString(aggregate) } else if data.QueryType == "SQL" { names := map[int]string{} fieldArr = append(fieldArr, v.Field) if _, sumOK := result["SUM"]; sumOK { aggrArr = append(aggrArr, "SUM("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "sum"}) } else { names[i] = "sum" } } if _, avgOK := result["AVG"]; avgOK { aggrArr = append(aggrArr, "AVG("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "avg"}) } else { names[i] = "avg" } } if _, maxOK := result["MAX"]; maxOK { aggrArr = append(aggrArr, "MAX("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "max"}) } else { names[i] = "max" } } if _, minOK := result["MIN"]; minOK { aggrArr = append(aggrArr, "MIN("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "min"}) } else { names[i] = "min" } } if _, minOK := result["COUNT"]; minOK { aggrArr = append(aggrArr, "COUNT("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "count"}) } else { names[i] = "count" } } if len(result) > 1 { fieldAggr.Set(v.Field, indexAggr) } else { fieldAggr.Set(v.Field, names) } } else if data.QueryType == "Dbox" { getQuery := toolkit.M{} toolkit.UnjsonFromString(data.QueryText, &getQuery) aggregate, e := d.dboxAggr(getQuery.Get("from").(string), v.Field, ctx, query, result, fieldAggr, cursor, conn) if e != nil { return nil, e } v.Aggregate = toolkit.JsonString(aggregate) } } } if data.QueryType == "SQL" { // fieldString := strings.Join(fieldArr, ", ") aggrString := strings.Join(aggrArr, ", ") var queryText string r := regexp.MustCompile(`(([Ff][Rr][Oo][Mm])) (?P<from>([a-zA-Z][_a-zA-Z]+[_a-zA-Z0-1].*))`) temparray := r.FindStringSubmatch(data.QueryText) sqlpart := toolkit.M{} for i, val := range r.SubexpNames() { if val != "" { sqlpart.Set(val, temparray[i]) } } if fromOK := sqlpart.Get("from", "").(string); fromOK != "" { queryText = toolkit.Sprintf("select %s FROM %s", aggrString, sqlpart.Get("from", "").(string)) // toolkit.Printf("queryString:%v\n", queryString) } query = ctx.NewQuery().Command("freequery", toolkit.M{}. Set("syntax", queryText)) csr, e := query.Cursor(nil) if e != nil { return nil, e } defer csr.Close() cursor := []toolkit.M{} e = csr.Fetch(&cursor, 0, false) if e != nil { return nil, e } for f, m := range fieldAggr { aggrData := toolkit.M{} for _, aggs := range cursor { for k, agg := range aggs { if toolkit.SliceLen(m) > 0 { for _, vals := range m.([]map[int]string) { for key, val := range vals { if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) { aggrData.Set(val, agg) data.MetaData[key].Aggregate = toolkit.JsonString(aggrData) } } } } else { for key, val := range m.(map[int]string) { if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) { aggrData.Set(val, agg) data.MetaData[key].Aggregate = toolkit.JsonString(aggrData) // toolkit.Printf("k:%v f:%v key:%v val:%v agg:%v\n", k, f, key, val, data.MetaData[key].Aggregate) } } } } } } } return data, nil }
func streamsavedata(intms <-chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) { var err error iN, note := 0, "" for intm := range intms { if destDboxs[key].desttype == "mongo" { intm.Set("_id", toolkit.GenerateRandomString("", 32)) } if len(intm) == 0 { continue } //Pre Execute Program if extCommand.Has("pre") && toolkit.ToString(extCommand["pre"]) != "" { sintm := toolkit.JsonString(intm) arrcmd := make([]string, 0, 0) // if runtime.GOOS == "windows" { // arrcmd = append(arrcmd, "cmd") // arrcmd = append(arrcmd, "/C") // } arrcmd = append(arrcmd, toolkit.ToString(extCommand["pre"])) arrcmd = append(arrcmd, sintm) // output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:]) output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1]) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute pre external command :%s", key, err.Error()), "ERROR") note = "Error Found" continue } err = toolkit.UnjsonFromString(output, &intm) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get pre external command output :%s", key, err.Error()), "ERROR") note = "Error Found" continue } } err = sQ.Exec(toolkit.M{ "data": intm, }) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR") note = "Error Found" continue } err = saverechistory(key, intm) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert record data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR") note = "Error Found" } iN += 1 if math.Mod(float64(iN), 100) == 0 { _ = updatesnapshot(iN, key) dt = dt.Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN)) iN = 0 } //Post Execute Program if extCommand.Has("post") { sintm := toolkit.JsonString(intm) arrcmd := make([]string, 0, 0) // if runtime.GOOS == "windows" { // arrcmd = append(arrcmd, "cmd") // arrcmd = append(arrcmd, "/C") // } arrcmd = append(arrcmd, toolkit.ToString(extCommand["post"])) arrcmd = append(arrcmd, sintm) // output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:]) output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1]) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute post external command :%s", key, err.Error()), "ERROR") note = "Error Found" continue } err = toolkit.UnjsonFromString(output, &intm) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get post external command output :%s", key, err.Error()), "ERROR") note = "Error Found" continue } } } dt = dt.Set("note", note). Set("grabstatus", "done"). Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN)) _ = updatesnapshot(iN, key) err = savehistory(dt) if err != nil { Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save history : %s", key), "ERROR") } Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Finish save data", key), "INFO") destDboxs[key].IConnection.Close() }
func (w *WebGrabberController) GetFetchedData(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson arrcmd := make([]string, 0, 0) result := toolkit.M{} payload := struct { RecFile string `json:"recfile"` }{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // var data []toolkit.M // config := toolkit.M{"useheader": true, "delimiter": ","} // query := helper.Query("csv", payload.RecFile, "", "", "", config) // data, err = query.SelectAll("") // apppath := "" // if runtime.GOOS == "windows" { // arrcmd = append(arrcmd, "cmd") // arrcmd = append(arrcmd, "/C") // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe") // } else { // apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread") // } // arrcmd = append(arrcmd, apppath) // arrcmd = append(arrcmd, `-readtype=rechistory`) // arrcmd = append(arrcmd, `-pathfile=`+payload.RecFile) // cmd := exec.Command(arrcmd[0], arrcmd[1:]...) // byteoutput, err := cmd.CombinedOutput() // if err != nil { // return helper.CreateResult(false, nil, err.Error()) // } client, server, err := w.ConnectToSedotanServer() if err != nil { return helper.CreateResult(false, nil, err.Error()) } SshClient := *client payload.RecFile = strings.Replace(payload.RecFile, EC_DATA_PATH, server.DataPath, -1) apppath := "" if server.OS == "linux" { apppath = server.AppPath + `/cli/sedotanread` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="rechistory"`) arrcmd = append(arrcmd, `-pathfile="`+strings.Replace(payload.RecFile, `\`, `/`, -1)+`"`) } else { apppath = server.AppPath + `\bin\sedotanread.exe` arrcmd = append(arrcmd, apppath) arrcmd = append(arrcmd, `-readtype="rechistory"`) arrcmd = append(arrcmd, `-pathfile="`+payload.RecFile+`"`) } cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ") fmt.Println("====>", cmds) output, err := SshClient.GetOutputCommandSsh(cmds) if err != nil { fmt.Println(err) } err = toolkit.UnjsonFromString(output, &result) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, result["DATA"], "") }
func main() { var err error var datastring string var datatemp []interface{} var logs interface{} container := toolkit.M{} dataset := make([]toolkit.M, 0, 0) // fReadType := flag.String("readtype", "", "read type sedotan file") //snapshot,history,rechistory,logfile,[daemonlog] // fLocation := flag.String("pathfile", "", "Full path file location include filename and pattern") //support using environment variable EC_DATA_PATH // fDateTime := flag.String("datetime", "", "Date time for log file") // fTake := flag.Int("take", 0, "take for limit data") // fSkip := flag.Int("skip", 0, "skip for limit data") flag.Parse() tReadType := toolkit.ToString(*fReadType) tLocation = toolkit.ToString(*fLocation) tNameid = toolkit.ToString(*fNameid) tDateTime = toolkit.ToString(*fDateTime) tDatas = toolkit.ToString(*fDatas) //snapshot,history,rechistory,logfile,[daemonlog] //=========== Parse other flag =========== // HERE //======================================== switch tReadType { case "snapshot": module := GetDirSnapshot(tNameid) SnapShot, err := module.OpenSnapShot(tNameid) if err != nil { fmt.Sprintf("ERROR: %s", err) } datastring = toolkit.JsonString(SnapShot) err = toolkit.UnjsonFromString(datastring, &dataset) container.Set("DATA", dataset) case "history": module := NewHistory(tLocation) datatemp, err = module.OpenHistory() if err != nil { fmt.Sprintf("ERROR: %s", err) } datastring = toolkit.JsonString(datatemp) err = toolkit.UnjsonFromString(datastring, &dataset) container.Set("DATA", dataset) case "rechistory": var data []toolkit.M config := toolkit.M{"useheader": true, "delimiter": ","} query := helper.Query("csv", tLocation, "", "", "", config) data, err = query.SelectAll("") container.Set("DATA", data) case "logfile": history := NewHistory(tNameid) err = toolkit.UnjsonFromString(tDatas, &datatemp) logs = history.GetLogHistory(datatemp, tDateTime) container.Set("DATA", logs) case "daemonlog": // logs, err := GetLogs() // if err != nil { // fmt.Sprintf("ERROR: %s", err) // } // datatemp = append(datatemp,logs) // datastring = toolkit.JsonString(datatemp) // container.Set("DATA", datastring) container.Set("DATA", dataset) err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format")) default: container.Set("DATA", dataset) err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format")) } container.Set("ERROR", err) outputstring := toolkit.JsonString(container) fmt.Printf("%s", outputstring) }