func genGo(fi os.FileInfo, source, out string) error {
log.Info("Processing " + fi.Name())
fn := filepath.Join(source, fi.Name())
var (
bs []byte
e error
)
if bs, e = ioutil.ReadFile(fn); e != nil {
return errors.New("Open error " + e.Error())
}
pkg := new(PackageModel)
e = toolkit.UnjsonFromString(string(bs), pkg)
if e != nil {
return errors.New("Unmarshal JSON: " + e.Error())
}
for _, sm := range pkg.Structs {
e = sm.Write(pkg, out)
if e != nil {
return errors.New(toolkit.Sprintf("Write model %s: %s", sm.Name, e.Error()))
}
log.Info(toolkit.Sprintf("Writing %s.%s", pkg.Name, sm.Name))
}
log.Info("Processing " + fi.Name() + " done")
return nil
}
func TestRunSedotanReadRecHistory(t *testing.T) {
t.Skip("Skip : Comment this line to do test")
arrcmd := make([]string, 0, 0)
result := toolkit.M{}
arrcmd = append(arrcmd, EC_APP_PATH+`\bin\sedotanread.exe`)
arrcmd = append(arrcmd, `-readtype=rechistory`)
arrcmd = append(arrcmd, `-recfile=E:\EACIIT\src\github.com\eaciit\colony-app\data-root\webgrabber\historyrec\irondcecomcn.Iron01-20160316022830.csv`)
if runtime.GOOS == "windows" {
cmd = exec.Command(arrcmd[0], arrcmd[1:]...)
rechistory, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:]...)
err = toolkit.UnjsonFromString(rechistory, &result)
if err != nil {
t.Errorf("Error, %s \n", err)
}
byteoutput, err := cmd.CombinedOutput()
if err != nil {
// Log.AddLog(fmt.Sprintf("[%v] run at %v, found error : %v", eid, sedotan.DateToString(thistime), err.Error()), "ERROR")
}
err = toolkit.UnjsonFromString(string(byteoutput), &result)
} else {
// cmd = exec.Command("sudo", "../daemon/sedotandaemon", `-config="`+tbasepath+`\config-daemon.json"`, `-logpath="`+tbasepath+`\log"`)
}
fmt.Println(result)
}
func (q *Query) openFile() error {
if q.fileHasBeenOpened {
return nil
}
_, e := os.Stat(q.jsonPath)
if e != nil && (strings.Contains(e.Error(), "does not exist") || strings.Contains(e.Error(), "no such file or directory")) {
q.data = []toolkit.M{}
return nil
} else if e != nil {
return err.Error(packageName, modQuery, "openFile: Open file fail", e.Error())
}
bs, e := ioutil.ReadFile(q.jsonPath)
if e != nil {
return err.Error(packageName, modQuery, "openFile: Read file data fail", e.Error())
}
jsonText := string(bs)
var tempData []toolkit.M
e = toolkit.UnjsonFromString(jsonText, &tempData)
if e != nil {
return err.Error(packageName, modQuery, "openFile: Serializaion fail", e.Error())
}
q.data = tempData
q.fileHasBeenOpened = true
return nil
}
func (d *DataBrowserController) parseQuery(conn dbox.IConnection, dbrowser colonycore.DataBrowser, datacon *colonycore.Connection) (dbox.IQuery, error) {
var dataQuery dbox.IQuery
if dbrowser.QueryType == "nonQueryText" {
dataQuery = conn.NewQuery().From(dbrowser.TableNames)
} else if dbrowser.QueryType == "SQL" {
if toolkit.HasMember(rdbms, datacon.Driver) {
dataQuery = conn.NewQuery().Command("freequery", toolkit.M{}.
Set("syntax", dbrowser.QueryText))
} else {
return nil, errors.New("Free Text Query with SQL only for RDBMS, please use Dbox")
}
} else if dbrowser.QueryType == "Dbox" {
queryInfo := toolkit.M{}
toolkit.UnjsonFromString(dbrowser.QueryText, &queryInfo)
toolkit.Println("queryinfo", queryInfo)
if qFrom := queryInfo.Get("from", "").(string); qFrom != "" {
dataQuery = conn.NewQuery()
dataQuery = dataQuery.From(qFrom)
}
if qSelect := queryInfo.Get("select", "").(string); qSelect != "" {
if qSelect != "*" {
dataQuery = dataQuery.Select(strings.Split(qSelect, ",")...)
}
}
}
return dataQuery, nil
}
func main() {
jsonarg := toolkit.M{}
err := toolkit.UnjsonFromString(os.Args[1], &jsonarg)
if err != nil {
return
}
jsonarg["Date"] = "000000"
jsonarg[`MB 62% Fe`] = "POST " + toolkit.ToString(jsonarg[`MB 62% Fe`])
jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto) * 5)
fmt.Println(toolkit.JsonString(jsonarg))
}
func main() {
jsonarg := toolkit.M{}
err := toolkit.UnjsonFromString(os.Args[1], &jsonarg)
if err != nil {
return
}
jsonarg["Date"] = time.Now().Format("020106")
jsonarg[`MB 62% Fe`] = "FE " + toolkit.ToString(jsonarg[`MB 62% Fe`])
jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto))
fmt.Println(toolkit.JsonString(jsonarg))
}
func TestRunSedotanReadHistory(t *testing.T) {
t.Skip("Skip : Comment this line to do test")
var history = toolkit.M{}
arrcmd := make([]string, 0, 0)
arrcmd = append(arrcmd, EC_APP_PATH+`\bin\sedotanread.exe`)
arrcmd = append(arrcmd, `-readtype=history`)
arrcmd = append(arrcmd, `-pathfile=`+EC_DATA_PATH+`\webgrabber\history\HIST-GRABDCE-20160316.csv`)
if runtime.GOOS == "windows" {
historystring, _ := toolkit.RunCommand(arrcmd[0], arrcmd[1:]...)
err := toolkit.UnjsonFromString(historystring, &history)
if err != nil {
t.Errorf("Error, %s \n", err)
}
} else {
// cmd = exec.Command("sudo", "../daemon/sedotandaemon", `-config="`+tbasepath+`\config-daemon.json"`, `-logpath="`+tbasepath+`\log"`)
}
fmt.Println(history)
}
func (q *Query) openFile(commandtype string) error {
if q.fileHasBeenOpened {
return nil
}
_, e := os.Stat(q.jsonPath)
setting := q.Connection().Info().Settings
if e != nil && toolkit.HasMember([]interface{}{dbox.QueryPartSave, dbox.QueryPartInsert}, commandtype) &&
strings.Contains(e.Error(), "cannot find the file specified") && setting != nil {
newfile := setting.Get("newfile", false).(bool)
if newfile {
e = q.writeFile()
if e != nil {
return err.Error(packageName, modQuery, "openFile: "+commandtype+" Write fail", e.Error())
}
} else {
return err.Error(packageName, modQuery, "openFile: "+commandtype+" Create new file is false", e.Error())
}
} else if e != nil && (strings.Contains(e.Error(), "does not exist") || strings.Contains(e.Error(), "no such file or directory")) {
q.data = []toolkit.M{}
return nil
} else if e != nil {
return err.Error(packageName, modQuery, "openFile: Open file fail", e.Error())
}
bs, e := ioutil.ReadFile(q.jsonPath)
if e != nil {
return err.Error(packageName, modQuery, "openFile: Read file data fail", e.Error())
}
jsonText := string(bs)
var tempData []toolkit.M
e = toolkit.UnjsonFromString(jsonText, &tempData)
if e != nil {
return err.Error(packageName, modQuery, "openFile: Serializaion fail", e.Error())
}
q.data = tempData
q.fileHasBeenOpened = true
return nil
}
func TestRunSedotanReadSnapshot(t *testing.T) {
t.Skip("Skip : Comment this line to do test")
arrcmd := make([]string, 0, 0)
result := toolkit.M{}
arrcmd = append(arrcmd, EC_APP_PATH+`\bin\sedotanread.exe`)
arrcmd = append(arrcmd, `-readtype=snapshot`)
arrcmd = append(arrcmd, `-pathfile=`+EC_DATA_PATH+`\daemon\daemonsnapshot.csv`)
arrcmd = append(arrcmd, `-nameid=irondcecomcn`)
if runtime.GOOS == "windows" {
SnapShot, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:]...)
err = toolkit.UnjsonFromString(SnapShot, &result)
if err != nil {
t.Errorf("Error, %s \n", err)
}
} else {
// cmd = exec.Command("sudo", "../daemon/sedotandaemon", `-config="`+tbasepath+`\config-daemon.json"`, `-logpath="`+tbasepath+`\log"`)
}
fmt.Println(result)
}
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
var hasLookup bool
toolkit.Println("payload : ", payload)
if payload.Has("haslookup") {
hasLookup = payload.Get("haslookup").(bool)
}
_id := toolkit.ToString(payload.Get("browserid", ""))
sort := payload.Get("sort")
search := payload.Get("search")
_ = search
take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)
TblName := toolkit.M{}
payload.Unset("browserid")
//sorter = ""
if sort != nil {
tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
fmt.Printf("====== sort %#v\n", tmsort["dir"])
if tmsort["dir"] == "asc" {
sorter = tmsort["field"].(string)
} else if tmsort["dir"] == "desc" {
sorter = "-" + tmsort["field"].(string)
} else if tmsort["dir"] == nil {
sorter = " "
}
} else {
sorter = " "
}
dataDS := new(colonycore.DataBrowser)
err := colonycore.Get(dataDS, _id)
if err != nil {
return 0, nil, nil, err
}
dataConn := new(colonycore.Connection)
err = colonycore.Get(dataConn, dataDS.ConnectionID)
if err != nil {
return 0, nil, nil, err
}
if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
return 0, nil, nil, err
}
connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
if err != nil {
return 0, nil, nil, err
}
if dataDS.QueryType == "" {
TblName.Set("from", dataDS.TableNames)
payload.Set("from", dataDS.TableNames)
} else if dataDS.QueryType == "Dbox" {
getTableName := toolkit.M{}
toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
payload.Set("from", getTableName.Get("from").(string))
if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
payload.Set("select", getTableName.Get("select").(string))
}
} else if dataDS.QueryType == "SQL" {
var QueryString string
if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
} else if dataConn.Driver == "mssql" {
QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
toolkit.ToString(take) + " ROWS ONLY "
} else if dataConn.Driver == "postgres" {
QueryString = " LIMIT " + toolkit.ToString(take) +
" OFFSET " + toolkit.ToString(skip)
}
stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
payload.Set("freetext", stringQuery)
// toolkit.Println(stringQuery)
}
qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)
var selectfield string
for _, metadata := range dataDS.MetaData {
tField := metadata.Field
if payload.Has(tField) {
selectfield = toolkit.ToString(tField)
if toolkit.IsSlice(payload[tField]) {
query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
} else if !toolkit.IsNilOrEmpty(payload[tField]) {
var hasPattern bool
for _, val := range querypattern {
if strings.Contains(toolkit.ToString(payload[tField]), val) {
hasPattern = true
}
}
if hasPattern {
query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
} else {
switch toolkit.ToString(metadata.DataType) {
case "int":
query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
case "float32":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
case "float64":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
default:
query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
}
}
}
}
}
if hasLookup && selectfield != "" {
if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
query = query.Select(selectfield)
qcount = qcount.Select(selectfield)
} else {
query = query.Select(selectfield).Group(selectfield)
qcount = qcount.Select(selectfield).Group(selectfield)
}
}
ccount, err := qcount.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer ccount.Close()
dcount := ccount.Count()
cursor, err := query.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer cursor.Close()
data := []toolkit.M{}
cursor.Fetch(&data, 0, false)
if err != nil {
return 0, nil, nil, err
}
if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
dataMongo := []toolkit.M{}
for _, val := range data {
mVal, _ := toolkit.ToM(val.Get("_id"))
dataMongo = append(dataMongo, mVal)
}
data = dataMongo
} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
/*distinct value for flat file*/
dataFlat := []toolkit.M{}
var existingVal = []string{""}
for _, val := range data {
valString := toolkit.ToString(val.Get(selectfield))
if !toolkit.HasMember(existingVal, valString) {
dataFlat = append(dataFlat, val)
existingVal = append(existingVal, valString)
}
}
data = dataFlat
}
return dcount, data, dataDS, nil
}
func (w *WebGrabberController) GetLog(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
arrcmd := make([]string, 0, 0)
result := toolkit.M{}
payload := struct {
ID string `json:"_id"`
Date string `json:"date"`
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
wg := new(colonycore.WebGrabber)
err = colonycore.Get(wg, payload.ID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
o, err := toolkit.ToM(wg)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
// history := NewHistory(payload.ID)
// logs := history.GetLogHistory([]interface{}{o}, payload.Date)
// apppath := ""
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
// } else {
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
// }
// arrcmd = append(arrcmd, apppath)
// arrcmd = append(arrcmd, `-readtype=logfile`)
// arrcmd = append(arrcmd, `-datetime=`+payload.Date)
// arrcmd = append(arrcmd, `-nameid=`+payload.ID)
// arrcmd = append(arrcmd, `-datas=`+toolkit.JsonString([]interface{}{o}))
// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
// byteoutput, err := cmd.CombinedOutput()
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
logPath := ""
for _, v := range []interface{}{o} {
vMap, _ := toolkit.ToM(v)
logConf := vMap["logconf"].(map[string]interface{})
dateNowFormat := logConf["filepattern"].(string)
logpathconfig := logConf["logpath"].(string)
theDate := cast.String2Date(payload.Date, "YYYY/MM/dd HH:mm:ss")
theDateString := cast.Date2String(theDate, dateNowFormat)
fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
logPath = logpathconfig + fileName
}
client, server, err := w.ConnectToSedotanServer()
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
SshClient := *client
apppath := ""
if server.OS == "linux" {
apppath = server.AppPath + `/cli/sedotanread`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="logfile"`)
arrcmd = append(arrcmd, `-nameid="`+payload.ID+`"`)
arrcmd = append(arrcmd, `-pathfile="`+logPath+`"`)
arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`)
} else {
apppath = server.AppPath + `\bin\sedotanread.exe`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="logfile"`)
arrcmd = append(arrcmd, `-nameid=`+payload.ID+`"`)
arrcmd = append(arrcmd, `-pathfile=`+logPath+`"`)
arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`)
}
cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
fmt.Println("====>", cmds)
output, err := SshClient.GetOutputCommandSsh(cmds)
if err != nil {
fmt.Println(err)
}
err = toolkit.UnjsonFromString(output, &result)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, result["DATA"], "")
}
func (p *Page) Save(payload toolkit.M, isDesigner bool, widgetPath string) error {
if isDesigner {
p.ID = payload.Get("_id", "").(string)
if err := p.GetById(); err != nil {
return err
}
var datasources []string
wpArray := p.Widget
if payload.Get("dataSourceId") != nil { /*save configuration page*/
for _, ds := range payload.Get("dataSourceId", "").([]interface{}) {
datasources = append(datasources, ds.(string))
}
p.DataSources = datasources
}
var mode string
if payload.Get("mode") != nil {
mode = payload.Get("mode", "").(string)
}
if mode == "new widget" { /*save new widget*/
newWidget, err := p.SaveNewWidget(payload, widgetPath)
wpArray = append(wpArray, newWidget)
if err != nil {
return err
}
} else if mode == "delete widget" {
widget := p.Widget
wpArray = nil
for _, val := range widget {
if payload.Get("widgetPageId", "") == val.ID {
continue
}
wpArray = append(wpArray, val)
}
} else if mode == "save widget" {
if payload.Get("widget") != nil {
getwp := payload.Get("widget").(map[string]interface{})
wp := &WidgetPage{}
toolkit.UnjsonFromString(toolkit.JsonString(getwp), wp)
p.ID = payload.Get("pageId", "").(string)
if err := p.GetById(); err != nil {
return err
}
widget := p.Widget
wpArray = nil
for _, val := range widget {
if wp.ID == val.ID {
val = wp
}
wpArray = append(wpArray, val)
}
}
}
p.Widget = wpArray
// page.ThemeColor = payload.Get("themeColor", "").(string)
// page.SendFiles(EC_DATA_PATH, payload.Get("serverId", "").(string))
}
if err := Save(p); err != nil {
return err
}
return nil
}
func (w *WebGrabberController) GetSnapshot(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
arrcmd := make([]string, 0, 0)
result := toolkit.M{}
payload := struct {
Nameid string
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
// module := GetDirSnapshot("daemonsnapshot")
// SnapShot, err := module.OpenSnapShot(payload.Nameid)
// ===================LOCALHOST TEST=================================
// apppath := ""
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
// } else {
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
// }
// arrcmd = append(arrcmd, apppath)
// arrcmd = append(arrcmd, `-readtype=snapshot`)
// arrcmd = append(arrcmd, `-pathfile=`+EC_DATA_PATH+`\daemon\daemonsnapshot.csv`)
// arrcmd = append(arrcmd, `-nameid=`+payload.Nameid)
// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
// byteoutput, err := cmd.CombinedOutput()
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
// err = toolkit.UnjsonFromString(string(byteoutput), &result)
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
// ===================END LOCALHOST TEST===============================
client, server, err := w.ConnectToSedotanServer()
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
SshClient := *client
apppath := ""
if server.OS == "linux" {
apppath = server.AppPath + `/cli/sedotanread`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="snapshot"`)
arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`/daemon/daemonsnapshot.csv"`)
arrcmd = append(arrcmd, `-nameid="`+payload.Nameid+`"`)
} else {
apppath = server.AppPath + `\bin\sedotanread.exe`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="snapshot"`)
arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`\daemon\daemonsnapshot.csv"`)
arrcmd = append(arrcmd, `-nameid="`+payload.Nameid+`"`)
}
cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
fmt.Println("====>", cmds)
output, err := SshClient.GetOutputCommandSsh(cmds)
if err != nil {
fmt.Println(err)
}
err = toolkit.UnjsonFromString(output, &result)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, result["DATA"], "")
}
func (w *WebGrabberController) GetHistory(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
var result = toolkit.M{}
arrcmd := make([]string, 0, 0)
payload := new(colonycore.WebGrabber)
dateNow := cast.Date2String(time.Now(), "YYYYMMdd") //time.Now()
err := r.GetPayload(payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
err = colonycore.Get(payload, payload.ID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
// module := NewHistory(payload.HistConf.FileName)
// history, err := module.OpenHistory()
client, server, err := w.ConnectToSedotanServer()
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
SshClient := *client
apppath := ""
if server.OS == "linux" {
apppath = server.AppPath + `/cli/sedotanread`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="history"`)
arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`/webgrabber/history/`+payload.HistConf.FileName+`-`+dateNow+`.csv"`)
} else {
apppath = server.AppPath + `\bin\sedotanread.exe`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="history"`)
arrcmd = append(arrcmd, `-pathfile="`+server.DataPath+`\webgrabber\history\`+payload.HistConf.FileName+`-`+dateNow+`.csv"`)
}
// apppath := ""
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
// } else {
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
// }
// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
// byteoutput, err := cmd.CombinedOutput()
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
// err = toolkit.UnjsonFromString(string(byteoutput), &result)
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
// fmt.Println(strings.Join(append(arrcmd[:1],arrcmd[1:]...)," "))
cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
fmt.Println("====>", cmds)
output, err := SshClient.GetOutputCommandSsh(cmds)
if err != nil {
fmt.Println(err)
}
err = toolkit.UnjsonFromString(output, &result)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, result["DATA"], "")
}
func (d *DataBrowserController) hasAggr(ctx dbox.IConnection, data *colonycore.DataBrowser, conn *colonycore.Connection) (*colonycore.DataBrowser, error) {
var fieldArr, aggrArr []string
var indexAggr []map[int]string
var query dbox.IQuery
fieldAggr := toolkit.M{}
for i, v := range data.MetaData {
if v.Aggregate != "" {
result := toolkit.M{}
toolkit.UnjsonFromString(v.Aggregate, &result)
cursor := []toolkit.M{}
if data.QueryType == "" {
aggregate, e := d.dboxAggr(data.TableNames, v.Field, ctx, query, result, fieldAggr, cursor, conn)
if e != nil {
return nil, e
}
v.Aggregate = toolkit.JsonString(aggregate)
} else if data.QueryType == "SQL" {
names := map[int]string{}
fieldArr = append(fieldArr, v.Field)
if _, sumOK := result["SUM"]; sumOK {
aggrArr = append(aggrArr, "SUM("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "sum"})
} else {
names[i] = "sum"
}
}
if _, avgOK := result["AVG"]; avgOK {
aggrArr = append(aggrArr, "AVG("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "avg"})
} else {
names[i] = "avg"
}
}
if _, maxOK := result["MAX"]; maxOK {
aggrArr = append(aggrArr, "MAX("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "max"})
} else {
names[i] = "max"
}
}
if _, minOK := result["MIN"]; minOK {
aggrArr = append(aggrArr, "MIN("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "min"})
} else {
names[i] = "min"
}
}
if _, minOK := result["COUNT"]; minOK {
aggrArr = append(aggrArr, "COUNT("+v.Field+")")
if len(result) > 1 {
indexAggr = append(indexAggr, map[int]string{i: "count"})
} else {
names[i] = "count"
}
}
if len(result) > 1 {
fieldAggr.Set(v.Field, indexAggr)
} else {
fieldAggr.Set(v.Field, names)
}
} else if data.QueryType == "Dbox" {
getQuery := toolkit.M{}
toolkit.UnjsonFromString(data.QueryText, &getQuery)
aggregate, e := d.dboxAggr(getQuery.Get("from").(string), v.Field, ctx, query, result, fieldAggr, cursor, conn)
if e != nil {
return nil, e
}
v.Aggregate = toolkit.JsonString(aggregate)
}
}
}
if data.QueryType == "SQL" {
// fieldString := strings.Join(fieldArr, ", ")
aggrString := strings.Join(aggrArr, ", ")
var queryText string
r := regexp.MustCompile(`(([Ff][Rr][Oo][Mm])) (?P<from>([a-zA-Z][_a-zA-Z]+[_a-zA-Z0-1].*))`)
temparray := r.FindStringSubmatch(data.QueryText)
sqlpart := toolkit.M{}
for i, val := range r.SubexpNames() {
if val != "" {
sqlpart.Set(val, temparray[i])
}
}
if fromOK := sqlpart.Get("from", "").(string); fromOK != "" {
queryText = toolkit.Sprintf("select %s FROM %s", aggrString, sqlpart.Get("from", "").(string))
// toolkit.Printf("queryString:%v\n", queryString)
}
query = ctx.NewQuery().Command("freequery", toolkit.M{}.
Set("syntax", queryText))
csr, e := query.Cursor(nil)
if e != nil {
return nil, e
}
defer csr.Close()
cursor := []toolkit.M{}
e = csr.Fetch(&cursor, 0, false)
if e != nil {
return nil, e
}
for f, m := range fieldAggr {
aggrData := toolkit.M{}
for _, aggs := range cursor {
for k, agg := range aggs {
if toolkit.SliceLen(m) > 0 {
for _, vals := range m.([]map[int]string) {
for key, val := range vals {
if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) {
aggrData.Set(val, agg)
data.MetaData[key].Aggregate = toolkit.JsonString(aggrData)
}
}
}
} else {
for key, val := range m.(map[int]string) {
if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) {
aggrData.Set(val, agg)
data.MetaData[key].Aggregate = toolkit.JsonString(aggrData)
// toolkit.Printf("k:%v f:%v key:%v val:%v agg:%v\n", k, f, key, val, data.MetaData[key].Aggregate)
}
}
}
}
}
}
}
return data, nil
}
func streamsavedata(intms <-chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
var err error
iN, note := 0, ""
for intm := range intms {
if destDboxs[key].desttype == "mongo" {
intm.Set("_id", toolkit.GenerateRandomString("", 32))
}
if len(intm) == 0 {
continue
}
//Pre Execute Program
if extCommand.Has("pre") && toolkit.ToString(extCommand["pre"]) != "" {
sintm := toolkit.JsonString(intm)
arrcmd := make([]string, 0, 0)
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// }
arrcmd = append(arrcmd, toolkit.ToString(extCommand["pre"]))
arrcmd = append(arrcmd, sintm)
// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute pre external command :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
err = toolkit.UnjsonFromString(output, &intm)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get pre external command output :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
}
err = sQ.Exec(toolkit.M{
"data": intm,
})
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
note = "Error Found"
continue
}
err = saverechistory(key, intm)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert record data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
note = "Error Found"
}
iN += 1
if math.Mod(float64(iN), 100) == 0 {
_ = updatesnapshot(iN, key)
dt = dt.Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
iN = 0
}
//Post Execute Program
if extCommand.Has("post") {
sintm := toolkit.JsonString(intm)
arrcmd := make([]string, 0, 0)
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// }
arrcmd = append(arrcmd, toolkit.ToString(extCommand["post"]))
arrcmd = append(arrcmd, sintm)
// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute post external command :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
err = toolkit.UnjsonFromString(output, &intm)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get post external command output :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
}
}
dt = dt.Set("note", note).
Set("grabstatus", "done").
Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
_ = updatesnapshot(iN, key)
err = savehistory(dt)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save history : %s", key), "ERROR")
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Finish save data", key), "INFO")
destDboxs[key].IConnection.Close()
}
func (w *WebGrabberController) GetFetchedData(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
arrcmd := make([]string, 0, 0)
result := toolkit.M{}
payload := struct {
RecFile string `json:"recfile"`
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
// var data []toolkit.M
// config := toolkit.M{"useheader": true, "delimiter": ","}
// query := helper.Query("csv", payload.RecFile, "", "", "", config)
// data, err = query.SelectAll("")
// apppath := ""
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
// } else {
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
// }
// arrcmd = append(arrcmd, apppath)
// arrcmd = append(arrcmd, `-readtype=rechistory`)
// arrcmd = append(arrcmd, `-pathfile=`+payload.RecFile)
// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
// byteoutput, err := cmd.CombinedOutput()
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
client, server, err := w.ConnectToSedotanServer()
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
SshClient := *client
payload.RecFile = strings.Replace(payload.RecFile, EC_DATA_PATH, server.DataPath, -1)
apppath := ""
if server.OS == "linux" {
apppath = server.AppPath + `/cli/sedotanread`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="rechistory"`)
arrcmd = append(arrcmd, `-pathfile="`+strings.Replace(payload.RecFile, `\`, `/`, -1)+`"`)
} else {
apppath = server.AppPath + `\bin\sedotanread.exe`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="rechistory"`)
arrcmd = append(arrcmd, `-pathfile="`+payload.RecFile+`"`)
}
cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
fmt.Println("====>", cmds)
output, err := SshClient.GetOutputCommandSsh(cmds)
if err != nil {
fmt.Println(err)
}
err = toolkit.UnjsonFromString(output, &result)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, result["DATA"], "")
}
func main() {
var err error
var datastring string
var datatemp []interface{}
var logs interface{}
container := toolkit.M{}
dataset := make([]toolkit.M, 0, 0)
// fReadType := flag.String("readtype", "", "read type sedotan file") //snapshot,history,rechistory,logfile,[daemonlog]
// fLocation := flag.String("pathfile", "", "Full path file location include filename and pattern") //support using environment variable EC_DATA_PATH
// fDateTime := flag.String("datetime", "", "Date time for log file")
// fTake := flag.Int("take", 0, "take for limit data")
// fSkip := flag.Int("skip", 0, "skip for limit data")
flag.Parse()
tReadType := toolkit.ToString(*fReadType)
tLocation = toolkit.ToString(*fLocation)
tNameid = toolkit.ToString(*fNameid)
tDateTime = toolkit.ToString(*fDateTime)
tDatas = toolkit.ToString(*fDatas)
//snapshot,history,rechistory,logfile,[daemonlog]
//=========== Parse other flag ===========
// HERE
//========================================
switch tReadType {
case "snapshot":
module := GetDirSnapshot(tNameid)
SnapShot, err := module.OpenSnapShot(tNameid)
if err != nil {
fmt.Sprintf("ERROR: %s", err)
}
datastring = toolkit.JsonString(SnapShot)
err = toolkit.UnjsonFromString(datastring, &dataset)
container.Set("DATA", dataset)
case "history":
module := NewHistory(tLocation)
datatemp, err = module.OpenHistory()
if err != nil {
fmt.Sprintf("ERROR: %s", err)
}
datastring = toolkit.JsonString(datatemp)
err = toolkit.UnjsonFromString(datastring, &dataset)
container.Set("DATA", dataset)
case "rechistory":
var data []toolkit.M
config := toolkit.M{"useheader": true, "delimiter": ","}
query := helper.Query("csv", tLocation, "", "", "", config)
data, err = query.SelectAll("")
container.Set("DATA", data)
case "logfile":
history := NewHistory(tNameid)
err = toolkit.UnjsonFromString(tDatas, &datatemp)
logs = history.GetLogHistory(datatemp, tDateTime)
container.Set("DATA", logs)
case "daemonlog":
// logs, err := GetLogs()
// if err != nil {
// fmt.Sprintf("ERROR: %s", err)
// }
// datatemp = append(datatemp,logs)
// datastring = toolkit.JsonString(datatemp)
// container.Set("DATA", datastring)
container.Set("DATA", dataset)
err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format"))
default:
container.Set("DATA", dataset)
err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format"))
}
container.Set("ERROR", err)
outputstring := toolkit.JsonString(container)
fmt.Printf("%s", outputstring)
}