func (a *SessionController) GetSession(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
_ = a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
}
tSession := new(acl.Session)
if find := toolkit.ToString(payload["find"]); find != "" {
filter = new(dbox.Filter)
filter = dbox.Contains("loginid", find)
}
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
// c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
// c, err := acl.Find(tSession, nil, toolkit.M{}.Set("take", payload["take"].(int)).Set("skip", payload["skip"].(int)))
c, err := acl.Find(tSession, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
err = c.Fetch(&arrm, 0, false)
for i, val := range arrm {
arrm[i].Set("duration", time.Since(val["created"].(time.Time)).Hours())
arrm[i].Set("status", "ACTIVE")
if val["expired"].(time.Time).Before(time.Now().UTC()) {
arrm[i].Set("duration", val["expired"].(time.Time).Sub(val["created"].(time.Time)).Hours())
arrm[i].Set("status", "EXPIRED")
}
}
c.Close()
c, err = acl.Find(tSession, filter, nil)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
} else {
return helper.CreateResult(true, data, "")
}
}
func TestAggr(t *testing.T) {
skipIfNil(t)
c2 := c.Min(fn).Max(fn).Sum(fn).Avg(fn).Exec()
check(t, c2.Error, "Aggr")
if toolkit.ToInt(c2.Result.Min, toolkit.RoundingAuto) != min ||
toolkit.ToInt(c2.Result.Max, toolkit.RoundingAuto) != max ||
c2.Result.Sum != toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) ||
c2.Result.Avg != avg {
t.Fatalf("Error aggr. Got %v\n", toolkit.JsonString(c2.Result))
}
toolkit.Println("Value: ", toolkit.JsonString(c2.Result))
}
func (a *AdministrationController) GetAccess(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
}
tAccess := new(acl.Access)
if find := toolkit.ToString(payload["find"]); find != "" {
filter = new(dbox.Filter)
filter = dbox.Or(dbox.Contains("id", find),
dbox.Contains("title", find),
dbox.Contains("group1", find),
dbox.Contains("group2", find),
dbox.Contains("group3", find),
dbox.Contains("specialaccess1", find),
dbox.Contains("specialaccess2", find),
dbox.Contains("specialaccess3", find),
dbox.Contains("specialaccess4", find))
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err == nil {
err = c.Fetch(&arrm, 0, false)
}
c.Close()
c, err = acl.Find(tAccess, filter, nil)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
} else {
return helper.CreateResult(true, data, "")
}
}
func (a *UserController) GetUser(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
_ = a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
}
tUser := new(acl.User)
if find := toolkit.ToString(payload["find"]); find != "" {
filter = new(dbox.Filter)
filter = dbox.Or(dbox.Contains("id", find),
dbox.Contains("fullname", find),
dbox.Contains("email", find))
}
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
c, err := acl.Find(tUser, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
err = c.Fetch(&arrm, 0, false)
c.Close()
c, err = acl.Find(tUser, filter, nil)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
} else {
return helper.CreateResult(true, data, "")
}
}
func (a *Controller) KendoGridSettings(ins toolkit.M) toolkit.M {
if ins == nil {
ins = toolkit.M{}
}
s := toolkit.M{}
q_skip := a.Ctx.Input.Query("skip")
q_page := a.Ctx.Input.Query("page")
q_size := a.Ctx.Input.Query("pageSize")
if q_skip != "" {
s.Set("skip", toolkit.ToInt(q_skip))
}
if q_page != "" {
s.Set("page", toolkit.ToInt(q_page))
}
if q_size != "" {
s.Set("limit", toolkit.ToInt(q_size))
}
sortField := strings.ToLower(a.Ctx.Input.Query("sort[0][field]"))
sortDir := a.Ctx.Input.Query("sort[0][dir]")
if sortField != "" {
if sortField == "id" {
sortField = "_id"
}
if sortDir == "" || sortDir == "asc" {
s.Set("order", []string{sortField})
} else {
s.Set("order", []string{"-" + sortField})
}
}
if fqe := a.KendoGridFilter("filter"); fqe != nil {
if ins.Has("where") {
fqe = dbs.And(fqe, ins.Get("where").(*dbs.QE))
}
s.Set("where", fqe)
}
return s
}
func (a *GroupController) Search(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
_ = a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if strings.Contains(toolkit.TypeName(payload["search"]), "float") {
payload["search"] = toolkit.ToInt(payload["search"], toolkit.RoundingAuto)
}
tGroup := new(acl.Group)
if search := toolkit.ToString(payload["search"]); search != "" {
filter = new(dbox.Filter)
filter = dbox.Or(dbox.Contains("_id", search), dbox.Contains("title", search), dbox.Contains("owner", search))
}
fmt.Println(filter)
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
c, err := acl.Find(tGroup, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
err = c.Fetch(&arrm, 0, false)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
return helper.CreateResult(true, data, "")
}
func ToString(reflectIs reflect.Kind, i interface{}) string {
var s string
if reflectIs != reflect.String {
toI := toolkit.ToInt(i)
s = strconv.Itoa(toI)
} else {
s = i.(string)
}
return s
}
func toInterface(data string, dataType string, dateFormat string) interface{} {
if dataType == "" {
if strings.HasPrefix(data, "#") && strings.HasSuffix(data, "#") {
dataType = DataDate
} else {
vfloat := toolkit.ToFloat64(dataType, 2, toolkit.RoundingAuto)
vint := toolkit.ToInt(dataType, toolkit.RoundingAuto)
if int(vfloat) == vint && vint != 0 {
dataType = DataInt
} else if vfloat != 0 {
// dataType = DataFloat
b, i := toolkit.MemberIndex(DataFloats, dataType)
if b {
for idx, dataFloat := range DataFloats {
if idx == i {
dataType = dataFloat
}
}
}
} else {
dataType = DataString
}
}
}
if dataType == DataDate {
return toolkit.String2Date(data, dateFormat)
} else if dataType == DataInt {
return toolkit.ToInt(data, toolkit.RoundingAuto)
} else if toolkit.HasMember(DataFloats, dataType) {
return toolkit.ToFloat64(data, 2, toolkit.RoundingAuto)
} else {
return data
}
return nil
}
func checkisonprocess(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
cond = false
if _, f := mapsnapshot[id]; !f {
return
}
mtkdata := mapsnapshot[id]
mapcron, _ := toolkit.ToM(intervalconf["cronconf"])
if mtkdata.Grabstatus == "running" && len(mapcron) <= 0 {
cond = true
}
mtkstarttime := sedotan.StringToDate(mtkdata.Starttime)
timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
timeoutsec := time.Second * time.Duration(timeoutint)
if cond && (thistime.After(mtkstarttime.Add(timeoutsec)) && timeoutint > 0) {
cond = false
}
return
}
//Check Time run and record to snapshot
func checkistimerun(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
cond = false
var mtklaststarttime, mtklastupdate time.Time
tempss := Snapshot{Id: id,
Starttime: "",
Laststartgrab: "",
Lastupdate: "",
Grabcount: 1,
Rowgrabbed: 0,
Errorfound: 0,
Lastgrabstatus: "",
Grabstatus: "running",
Cgtotal: 0,
Cgprocess: 0,
Note: "",
Pid: 0}
// Lastgrabstatus string //[success|failed]
// Grabstatus string //[running|done]
strintervalconf := intervalconf.Get("starttime", "").(string)
intervalstart := sedotan.StringToDate(strintervalconf)
strintervaltype := intervalconf.Get("intervaltype", "").(string)
grabinterval := toolkit.ToInt(intervalconf.Get("grabinterval", 0), toolkit.RoundingAuto)
mtkdata := Snapshot{}
if _, f := mapsnapshot[id]; f {
mtkdata = mapsnapshot[id]
tempss.Starttime = mtkdata.Starttime
//for data timeout
mtklaststarttime = sedotan.StringToDate(mtkdata.Laststartgrab)
mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
timeoutsec := time.Second * time.Duration(timeoutint)
if mtkdata.Grabstatus == "running" && thistime.After(mtklaststarttime.Add(timeoutsec)) && timeoutint > 0 {
mtkdata.Lastupdate = sedotan.DateToString(mtklaststarttime.Add(timeoutsec))
mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
mtkdata.Lastgrabstatus = "failed"
mtkdata.Grabstatus = "done"
}
// set initial - found a long time ago snapshot
if intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)) {
tempss.Starttime = sedotan.DateToString(thistime)
mtkdata.Grabcount = 0
mtkdata.Rowgrabbed = 0
mtkdata.Errorfound = 0
mtkdata.Lastgrabstatus = ""
}
}
if mtkdata.Lastgrabstatus == "failed" {
grabinterval = toolkit.ToInt(intervalconf.Get("timeoutinterval", 0), toolkit.RoundingAuto)
}
if strintervalconf != "" && intervalstart.Before(thistime) {
_, fcond := mapsnapshot[id]
switch {
case !fcond || intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)):
cond = true
case intervalconf.Get("grabinterval", 0).(float64) > 0:
var durationgrab time.Duration
switch strintervaltype {
case "seconds":
durationgrab = time.Second * time.Duration(grabinterval)
case "minutes":
durationgrab = time.Minute * time.Duration(grabinterval)
case "hours":
durationgrab = time.Hour * time.Duration(grabinterval)
}
nextgrab := mtklastupdate.Add(durationgrab)
if nextgrab.Before(thistime) && !mtklastupdate.IsZero() {
cond = true
tempss.Grabcount = mtkdata.Grabcount + 1
tempss.Rowgrabbed = mtkdata.Rowgrabbed
tempss.Errorfound = mtkdata.Errorfound
tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
}
}
}
mapcronconf, _ := toolkit.ToM(intervalconf.Get("cronconf", nil))
minutetime := sedotan.DateMinutePress(thistime) //review this and the usage and parsing in cron
if len(mapcronconf) > 0 {
//min hour dayofmonth month dayofweek
cond = true
arrstr := [6]string{"min", "hour", "dayofmonth", "month", "dayofweek"}
// arrstr := [6]string{"month", "dayofmonth", "dayofweek", "hour", "min", "second"}
for _, str := range arrstr {
sval := toolkit.ToString(mapcronconf.Get(str, ""))
ival := toolkit.ToInt(sval, toolkit.RoundingAuto)
var valcom int
switch str {
// case "second":
// valcom = thistime.Second()
case "min":
valcom = thistime.Minute()
case "hour":
valcom = thistime.Hour()
case "dayofmonth":
valcom = thistime.Day()
case "month":
valcom = toolkit.ToInt(thistime.Month(), toolkit.RoundingAuto)
case "dayofweek":
valcom = toolkit.ToInt(thistime.Weekday(), toolkit.RoundingAuto)
}
if sval != "*" && valcom != ival {
cond = false
} else {
cond = cond && true
}
}
if mtkdata.Laststartgrab != "" {
cond = cond && minutetime.After(sedotan.StringToDate(mtkdata.Laststartgrab))
}
if cond && minutetime.Before(sedotan.StringToDate(mtkdata.Starttime)) {
tempss.Grabcount = mtkdata.Grabcount + 1
tempss.Rowgrabbed = mtkdata.Rowgrabbed
tempss.Errorfound = mtkdata.Errorfound
tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
}
}
if cond {
mapsnapshot[id] = tempss
}
return
}
func (d *DataBrowserController) TestQuery(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
data := colonycore.DataBrowser{} //map[string]interface{}{}
err := r.GetPayload(&data)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
conn, datacon, err := d.connToDatabase(data.ConnectionID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
query, err := d.parseQuery(conn, data, datacon)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
cursor, err := query.Cursor(nil)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
defer cursor.Close()
dataFetch := []toolkit.M{}
err = cursor.Fetch(&dataFetch, 1, false)
if err != nil {
cursor.ResetFetch()
err = cursor.Fetch(&dataFetch, 0, false)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
metadata := []*colonycore.StructInfo{}
// var dt string
for i, dataFields := range dataFetch {
if i > 0 {
break
}
j := 1
for keyField, dataField := range dataFields {
if strings.Contains(keyField, "id") && !strings.Contains(data.QueryText, "id") &&
!strings.Contains(data.QueryText, "*") && data.TableNames == "" {
continue
}
sInfo := &colonycore.StructInfo{}
sInfo.Field = keyField
sInfo.Label = keyField
rf := "string"
if dataField != nil {
rf = toolkit.TypeName(dataField)
if rf == "time.Time" {
rf = "date"
}
}
sInfo.DataType = rf
sInfo.Format = ""
sInfo.Align = "Left"
sInfo.ShowIndex = toolkit.ToInt(j, toolkit.RoundingAuto)
sInfo.Sortable = false
sInfo.SimpleFilter = false
sInfo.AdvanceFilter = false
sInfo.Aggregate = ""
metadata = append(metadata, sInfo)
j++
}
}
data.MetaData = metadata
return helper.CreateResult(true, data, "")
}
func GrabHtmlConfig(data toolkit.M) (*sdt.GrabService, error) {
var e error
var gi, ti time.Duration
xGrabService := sdt.NewGrabService()
xGrabService.Name = data["nameid"].(string) //"irondcecom"
xGrabService.Url = data["url"].(string) //"http://www.dce.com.cn/PublicWeb/MainServlet"
xGrabService.SourceType = sdt.SourceType_HttpHtml
grabintervalToInt := toolkit.ToInt(data["grabinterval"], toolkit.RoundingAuto)
timeintervalToInt := toolkit.ToInt(data["timeoutinterval"], toolkit.RoundingAuto)
if data["intervaltype"].(string) == "seconds" {
gi = time.Duration(grabintervalToInt) * time.Second
ti = time.Duration(timeintervalToInt) * time.Second
} else if data["intervaltype"].(string) == "minutes" {
gi = time.Duration(grabintervalToInt) * time.Minute
ti = time.Duration(timeintervalToInt) * time.Minute
} else if data["intervaltype"].(string) == "hours" {
gi = time.Duration(grabintervalToInt) * time.Hour
ti = time.Duration(timeintervalToInt) * time.Hour
}
xGrabService.GrabInterval = gi //* time.Minute
xGrabService.TimeOutInterval = ti //* time.Minute //time.Hour, time.Minute, time.Second
xGrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", timeintervalToInt, data["intervaltype"] /*"seconds"*/)
grabConfig := sdt.Config{}
if data["calltype"].(string) == "POST" {
dataurl := toolkit.M{}
for _, grabconf := range data["grabconf"].(map[string]interface{}) {
grabDataConf, e := toolkit.ToM(grabconf)
if e != nil {
return nil, e
}
for key, subGrabDataConf := range grabDataConf {
if reflect.ValueOf(subGrabDataConf).Kind() == reflect.Float64 {
i := toolkit.ToInt(subGrabDataConf, toolkit.RoundingAuto)
toString := strconv.Itoa(i)
dataurl[key] = toString
} else {
dataurl[key] = subGrabDataConf
}
}
}
grabConfig.SetFormValues(dataurl)
}
grabDataConf, e := toolkit.ToM(data["grabconf"])
if e != nil {
return nil, e
}
isAuthType := grabDataConf.Has("authtype")
if isAuthType {
grabConfig.AuthType = grabDataConf["authtype"].(string)
grabConfig.LoginUrl = grabDataConf["loginurl"].(string) //"http://localhost:8000/login"
grabConfig.LogoutUrl = grabDataConf["logouturl"].(string) //"http://localhost:8000/logout"
grabConfig.LoginValues = toolkit.M{}.
Set("name", grabDataConf["loginvalues"].(map[string]interface{})["name"].(string)).
Set("password", grabDataConf["loginvalues"].(map[string]interface{})["password"].(string))
}
xGrabService.ServGrabber = sdt.NewGrabber(xGrabService.Url, data["calltype"].(string), &grabConfig)
logconfToMap, e := toolkit.ToM(data["logconf"])
if e != nil {
return nil, e
}
logpath := logconfToMap["logpath"].(string) //"E:\\data\\vale\\log"
filename := logconfToMap["filename"].(string) + "-%s" //"LOG-GRABDCETEST"
filepattern := logconfToMap["filepattern"].(string) //"20060102"
logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
if e != nil {
return nil, e
}
xGrabService.Log = logconf
xGrabService.ServGrabber.DataSettings = make(map[string]*sdt.DataSetting)
xGrabService.DestDbox = make(map[string]*sdt.DestInfo)
tempDataSetting := sdt.DataSetting{}
tempDestInfo := sdt.DestInfo{}
// isCondition := []interface{}{}
tempFilterCond := toolkit.M{}
// var condition string
for _, dataSet := range data["datasettings"].([]interface{}) {
dataToMap, _ := toolkit.ToM(dataSet)
tempDataSetting.RowSelector = dataToMap["rowselector"].(string)
for _, columnSet := range dataToMap["columnsettings"].([]interface{}) {
columnToMap, e := toolkit.ToM(columnSet)
if e != nil {
return nil, e
}
i := toolkit.ToInt(columnToMap["index"], toolkit.RoundingAuto)
tempDataSetting.Column(i, &sdt.GrabColumn{Alias: columnToMap["alias"].(string), Selector: columnToMap["selector"].(string)})
}
/*if data["calltype"].(string) == "POST" {
// orCondition := []interface{}{}
isRowdeletecond := dataToMap.Has("rowdeletecond")
fmt.Println("isRowdeletecond>", isRowdeletecond)
if hasRowdeletecond := dataToMap.Has("rowdeletecond"); hasRowdeletecond {
for key, rowDeleteMap := range dataToMap["rowdeletecond"].(map[string]interface{}) {
if key == "$or" || key == "$and" {
for _, subDataRowDelete := range rowDeleteMap.([]interface{}) {
for subIndex, getValueRowDelete := range subDataRowDelete.(map[string]interface{}) {
orCondition = append(orCondition, map[string]interface{}{subIndex: getValueRowDelete})
}
}
condition = key
}
}
}
// tempDataSetting.RowDeleteCond = toolkit.M{}.Set(condition, orCondition)
tempFilterCond := toolkit.M{}.Set(condition, orCondition)
tempDataSetting.SetFilterCond(tempFilterCond)
}*/
if hasRowdeletecond := dataToMap.Has("rowdeletecond"); hasRowdeletecond {
rowToM, e := toolkit.ToM(dataToMap["rowdeletecond"])
if e != nil {
return nil, e
}
tempFilterCond, e = toolkit.ToM(rowToM.Get("filtercond", nil))
tempDataSetting.SetFilterCond(tempFilterCond)
}
if hasRowincludecond := dataToMap.Has("rowincludecond"); hasRowincludecond {
rowToM, e := toolkit.ToM(dataToMap["rowincludecond"])
if e != nil {
return nil, e
}
tempFilterCond, e = toolkit.ToM(rowToM.Get("filtercond", nil))
tempDataSetting.SetFilterCond(tempFilterCond)
}
xGrabService.ServGrabber.DataSettings[dataToMap["name"].(string)] = &tempDataSetting //DATA01 use name in datasettings
connToMap, e := toolkit.ToM(dataToMap["connectioninfo"])
if e != nil {
return nil, e
}
var db, usr, pwd string
if hasDb := connToMap.Has("database"); !hasDb {
db = ""
} else {
db = connToMap["database"].(string)
}
if hasUser := connToMap.Has("username"); !hasUser {
usr = ""
} else {
usr = connToMap["username"].(string)
}
if hasPwd := connToMap.Has("password"); !hasPwd {
pwd = ""
} else {
pwd = connToMap["username"].(string)
}
ci := dbox.ConnectionInfo{}
ci.Host = connToMap["host"].(string) //"E:\\data\\vale\\Data_GrabIronTest.csv"
ci.Database = db
ci.UserName = usr
ci.Password = pwd
if hasSettings := connToMap.Has("settings"); !hasSettings {
ci.Settings = nil
} else {
settingToMap, e := toolkit.ToM(connToMap["settings"])
if e != nil {
return nil, e
}
ci.Settings = settingToMap //toolkit.M{}.Set("useheader", settingToMap["useheader"].(bool)).Set("delimiter", settingToMap["delimiter"])
}
if hasCollection := connToMap.Has("collection"); !hasCollection {
tempDestInfo.Collection = ""
} else {
tempDestInfo.Collection = connToMap["collection"].(string)
}
tempDestInfo.Desttype = dataToMap["desttype"].(string)
tempDestInfo.IConnection, e = dbox.NewConnection(tempDestInfo.Desttype, &ci)
if e != nil {
return nil, e
}
xGrabService.DestDbox[dataToMap["name"].(string)] = &tempDestInfo
//=History===========================================================
xGrabService.HistoryPath = HistoryPath //"E:\\data\\vale\\history\\"
xGrabService.HistoryRecPath = HistoryRecPath //"E:\\data\\vale\\historyrec\\"
//===================================================================
}
return xGrabService, nil
}
func fetchConfig() (err error) {
switch toolkit.ToString(config.Get("sourcetype", "")) {
case "SourceType_HttpHtml":
SourceType = SourceType_HttpHtml
case "SourceType_HttpJson":
SourceType = SourceType_HttpJson
default:
err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
return
}
Log.AddLog("Start fetch grabconf", "INFO")
if !config.Has("grabconf") {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
return
}
tconfgrab := toolkit.M{}
tconfgrab, err = toolkit.ToM(config["grabconf"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
return
}
grabconfig := sedotan.Config{}
if tconfgrab.Has("formvalues") {
tfromvalues := toolkit.M{}
tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
return
}
grabconfig.SetFormValues(tfromvalues)
}
if tconfgrab.Has("loginvalues") {
grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
}
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
return
}
grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))
grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))
grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))
Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")
wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)
Log.AddLog("Start fetch datasettings", "INFO")
if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
destDboxs = make(map[string]*DestInfo)
for i, xVal := range config["datasettings"].([]interface{}) {
err = nil
tDataSetting := sedotan.DataSetting{}
tDestDbox := DestInfo{}
mVal := toolkit.M{}
mVal, err = toolkit.ToM(xVal)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
continue
}
tnameid := toolkit.ToString(mVal.Get("nameid", ""))
if tnameid == "" {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
continue
}
tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))
// Fetch columnsettings
if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
continue
}
tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
mValcs := toolkit.M{}
mValcs, err = toolkit.ToM(Valcs)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
continue
}
tgrabcolumn := sedotan.GrabColumn{}
tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))
tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
tDataSetting.Column(tindex, &tgrabcolumn)
}
//Fetch Filter Condition
if mVal.Has("filtercond") {
tfiltercond := toolkit.M{}
tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
} else {
tDataSetting.SetFilterCond(tfiltercond)
}
}
//Fetch Connection Info
tConnInfo := toolkit.M{}
tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))
tHost := toolkit.ToString(tConnInfo.Get("host", ""))
tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
tSettings := toolkit.M{}
tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection.Close()
destDboxs[tnameid] = &tDestDbox
wGrabber.DataSettings[tnameid] = &tDataSetting
}
err = nil
if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
if !config.Has("histconf") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
Log.AddLog("Start fetch histconf", "INFO")
histConf, err = toolkit.ToM(config.Get("histconf", nil))
if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
return
}
func constructActions(list []interface{}) (flows []colonycore.FlowAction) {
for _, act := range list {
flowAction := act.(map[string]interface{})
server := colonycore.Server{}
if flowAction["server"] != nil {
dsServer := []colonycore.Server{}
cursor, err := colonycore.Find(new(colonycore.Server), dbox.Eq("_id", tk.ToString(flowAction["server"])))
if err != nil {
//
} else {
if cursor != nil {
cursor.Fetch(&dsServer, 0, false)
defer cursor.Close()
}
if len(dsServer) > 0 {
server = dsServer[0]
}
}
}
inputParam := tk.M{}
outputParam := tk.M{}
outputType := ""
for _, in := range flowAction["inputparam"].([]interface{}) {
inputParam.Set("", in)
}
if flowAction["outputtype"] != nil {
outputType = tk.ToString(flowAction["outputtype"])
}
for _, out := range flowAction["outputparam"].([]interface{}) {
outputParam.Set("", out)
}
OK := []string{}
KO := []string{}
for _, str := range flowAction["OK"].([]interface{}) {
OK = append(OK, tk.ToString(str))
}
for _, str := range flowAction["KO"].([]interface{}) {
KO = append(KO, tk.ToString(str))
}
actionType := tk.ToString(flowAction["type"])
flow := colonycore.FlowAction{
Id: tk.ToString(flowAction["id"]),
Name: tk.ToString(flowAction["name"]) + "-" + tk.ToString(flowAction["id"]),
Description: tk.ToString(flowAction["description"]),
Type: actionType,
Server: server,
OK: OK,
KO: KO,
Retry: tk.ToInt(flowAction["Retry"], ""),
Interval: tk.ToInt(flowAction["Interval"], ""),
FirstAction: flowAction["firstaction"].(bool),
InputParam: inputParam,
OutputParam: outputParam,
OutputType: outputType,
OutputPath: "",
}
dataAction := flowAction["action"].(map[string]interface{})
switch actionType {
case dataflow.ACTION_TYPE_SPARK:
spark := colonycore.ActionSpark{
Master: tk.ToString(dataAction["master"]),
Mode: tk.ToString(dataAction["mode"]),
File: tk.ToString(dataAction["appfiles"]),
MainClass: tk.ToString(dataAction["mainclass"]),
Args: tk.ToString(dataAction["args"]),
}
flow.Action = spark
break
case dataflow.ACTION_TYPE_HIVE:
hive := colonycore.ActionHive{
ScriptPath: tk.ToString(dataAction),
// Params: ,
}
flow.Action = hive
break
case dataflow.ACTION_TYPE_SSH:
ssh := colonycore.ActionSSH{
Command: tk.ToString(dataAction["script"]),
}
flow.Action = ssh
break
case dataflow.ACTION_TYPE_HDFS:
hdfs := colonycore.ActionHDFS{
Command: tk.ToString(dataAction["script"]),
}
flow.Action = hdfs
break
case dataflow.ACTION_TYPE_JAVA:
java := colonycore.ActionJavaApp{
Jar: tk.ToString(dataAction["jar"]),
}
flow.Action = java
break
case dataflow.ACTION_TYPE_MAP_REDUCE:
mr := colonycore.ActionHadoopStreaming{
// Jar: ,
Mapper: tk.ToString(dataAction["mapper"]),
Reducer: tk.ToString(dataAction["reducer"]),
Input: tk.ToString(dataAction["input"]),
Output: tk.ToString(dataAction["output"]),
Params: tk.ToString(dataAction["params"]),
}
flow.Action = mr
break
case dataflow.ACTION_TYPE_KAFKA:
kafka := colonycore.ActionKafka{}
flow.Action = kafka
break
case dataflow.ACTION_TYPE_EMAIL:
email := colonycore.ActionEmail{
// To:,
// CC:,
// Subject:,
// Body:,
}
flow.Action = email
break
// case "Fork":
// fork := colonycore.ActionFork{}
// action.Action = fork
// action.Type = dataflow.ACTION_TYPE_FORK
// break
case dataflow.ACTION_TYPE_DECISION:
conditions := []colonycore.Condition{}
/*for _, val := range dataAction["conditions"].([]interface{}) {
condition := val.(colonycore.Condition)
conditions = append(conditions, condition)
}*/
decision := colonycore.ActionDecision{
Conditions: conditions,
// IsFork: dataAction["isfork"].(bool),
}
flow.Action = decision
break
case dataflow.ACTION_TYPE_STOP:
stop := colonycore.ActionStop{
Message: tk.ToString(dataAction["message"]),
}
flow.Action = stop
break
}
flows = append(flows, flow)
}
return
}
func GrabDocConfig(data toolkit.M) (*sdt.GrabService, error) {
var e error
var gi, ti time.Duration
GrabService := sdt.NewGrabService()
GrabService.Name = data["nameid"].(string) //"iopriceindices"
GrabService.SourceType = sdt.SourceType_DocExcel
grabintervalToInt := toolkit.ToInt(data["grabinterval"], toolkit.RoundingAuto)
timeintervalToInt := toolkit.ToInt(data["timeoutinterval"], toolkit.RoundingAuto)
if data["intervaltype"].(string) == "seconds" {
gi = time.Duration(grabintervalToInt) * time.Second
ti = time.Duration(timeintervalToInt) * time.Second
} else if data["intervaltype"].(string) == "minutes" {
gi = time.Duration(grabintervalToInt) * time.Minute
ti = time.Duration(timeintervalToInt) * time.Minute
} else if data["intervaltype"].(string) == "hours" {
gi = time.Duration(grabintervalToInt) * time.Hour
ti = time.Duration(timeintervalToInt) * time.Hour
}
GrabService.GrabInterval = gi
GrabService.TimeOutInterval = ti //time.Hour, time.Minute, time.Second
GrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", timeintervalToInt, data["intervaltype"])
ci := dbox.ConnectionInfo{}
grabDataConf, e := toolkit.ToM(data["grabconf"])
if e != nil {
return nil, e
}
isDoctype := grabDataConf.Has("doctype")
if isDoctype {
connToMap, e := toolkit.ToM(grabDataConf["connectioninfo"])
if e != nil {
return nil, e
}
ci.Host = connToMap["host"].(string) //"E:\\data\\sample\\IO Price Indices.xlsm"
if hasSettings := connToMap.Has("settings"); !hasSettings {
ci.Settings = nil
} else {
settingToMap, e := toolkit.ToM(connToMap["settings"])
if e != nil {
return nil, e
}
ci.Settings = settingToMap //toolkit.M{}.Set("useheader", settingToMap["useheader"].(bool)).Set("delimiter", settingToMap["delimiter"])
}
GrabService.ServGetData, e = sdt.NewGetDatabase(ci.Host, grabDataConf["doctype"].(string), &ci)
}
logconfToMap, e := toolkit.ToM(data["logconf"])
if e != nil {
return nil, e
}
logpath := logconfToMap["logpath"].(string) //"E:\\data\\vale\\log"
filename := logconfToMap["filename"].(string) + "-%s" //"LOG-LOCALXLSX-%s"
filepattern := logconfToMap["filepattern"].(string) //"20060102"
logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
if e != nil {
return nil, e
}
GrabService.Log = logconf
GrabService.ServGetData.CollectionSettings = make(map[string]*sdt.CollectionSetting)
GrabService.DestDbox = make(map[string]*sdt.DestInfo)
tempDataSetting := sdt.CollectionSetting{}
tempDestInfo := sdt.DestInfo{}
for _, dataSet := range data["datasettings"].([]interface{}) {
dataToMap, e := toolkit.ToM(dataSet)
if e != nil {
return nil, e
}
tempDataSetting.Collection = dataToMap["rowselector"].(string) //"HIST"
for _, columnSet := range dataToMap["columnsettings"].([]interface{}) {
columnToMap, e := toolkit.ToM(columnSet)
if e != nil {
return nil, e
}
tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &sdt.GrabColumn{Alias: columnToMap["alias"].(string), Selector: columnToMap["selector"].(string)})
}
GrabService.ServGetData.CollectionSettings[dataToMap["name"].(string)] = &tempDataSetting //DATA01 use name in datasettings
// fmt.Println("doctype>", grabDataConf["doctype"])
connToMap, e := toolkit.ToM(dataToMap["connectioninfo"])
if e != nil {
return nil, e
}
var db, usr, pwd string
if hasDb := connToMap.Has("database"); !hasDb {
db = ""
} else {
db = connToMap["database"].(string)
}
if hasUser := connToMap.Has("username"); !hasUser {
usr = ""
} else {
usr = connToMap["username"].(string)
}
if hasPwd := connToMap.Has("password"); !hasPwd {
pwd = ""
} else {
pwd = connToMap["username"].(string)
}
ci = dbox.ConnectionInfo{}
ci.Host = connToMap["host"].(string) //"localhost:27017"
ci.Database = db //"valegrab"
ci.UserName = usr //""
ci.Password = pwd //""
//tempDestInfo.Collection = "iopriceindices"
if hasCollection := connToMap.Has("collection"); !hasCollection {
tempDestInfo.Collection = ""
} else {
tempDestInfo.Collection = connToMap["collection"].(string)
}
tempDestInfo.Desttype = dataToMap["desttype"].(string) //"mongo"
tempDestInfo.IConnection, e = dbox.NewConnection(tempDestInfo.Desttype, &ci)
if e != nil {
return nil, e
}
GrabService.DestDbox[dataToMap["name"].(string)] = &tempDestInfo
//=History===========================================================
GrabService.HistoryPath = HistoryPath //"E:\\data\\vale\\history\\"
GrabService.HistoryRecPath = HistoryRecPath //"E:\\data\\vale\\historyrec\\"
//===================================================================
}
return GrabService, e
}
func fetchConfig() (err error) {
switch toolkit.ToString(config.Get("sourcetype", "")) {
case "SourceType_DocExcel":
SourceType = SourceType_DocExcel
case "SourceType_DocMongo":
SourceType = SourceType_DocMongo
default:
err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
return
}
Log.AddLog("Start fetch grabconf", "INFO")
if !config.Has("grabconf") {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
return
}
tconfgrab := toolkit.M{}
tconfgrab, err = toolkit.ToM(config["grabconf"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
return
}
if !tconfgrab.Has("doctype") {
err = errors.New("Fetch Config, doctype not found")
return
}
ci := dbox.ConnectionInfo{}
mapconninfo := toolkit.M{}
mapconninfo, err = toolkit.ToM(tconfgrab.Get("connectioninfo", nil))
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo found error : %v", err.Error()))
return
}
ci.Host = toolkit.ToString(mapconninfo.Get("host", ""))
ci.Database = toolkit.ToString(mapconninfo.Get("database", ""))
ci.UserName = toolkit.ToString(mapconninfo.Get("userName", ""))
ci.Password = toolkit.ToString(mapconninfo.Get("password", ""))
ci.Settings, err = toolkit.ToM(mapconninfo.Get("settings", nil))
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo.settings found error : %v", err.Error()))
return
}
sGrabber, err = sedotan.NewGetDatabase(ci.Host, toolkit.ToString(tconfgrab.Get("doctype", "")), &ci)
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, create new get database found error : %v", err.Error()))
}
Log.AddLog("Start fetch datasettings", "INFO")
if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
sGrabber.CollectionSettings = make(map[string]*sedotan.CollectionSetting)
destDboxs = make(map[string]*DestInfo)
for i, xVal := range config["datasettings"].([]interface{}) {
err = nil
tCollectionSetting := sedotan.CollectionSetting{}
tDestDbox := DestInfo{}
mVal := toolkit.M{}
mVal, err = toolkit.ToM(xVal)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
continue
}
tnameid := toolkit.ToString(mVal.Get("nameid", ""))
if tnameid == "" {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
continue
}
tCollectionSetting.Collection = toolkit.ToString(mVal.Get("collection", ""))
// Fetch mapssettings
if !mVal.Has("mapssettings") || !(toolkit.TypeName(mVal["mapssettings"]) == "[]interface {}") {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : mapssettings is not found or incorrect", i, tnameid), "ERROR")
continue
}
tCollectionSetting.MapsColumns = make([]*sedotan.MapColumn, 0, 0)
for xi, Valcs := range mVal["mapssettings"].([]interface{}) {
mValcs := toolkit.M{}
mValcs, err = toolkit.ToM(Valcs)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : mapssettings is not found or incorrect", i, tnameid, xi), "ERROR")
continue
}
tgrabcolumn := sedotan.MapColumn{}
tgrabcolumn.Source = toolkit.ToString(mValcs.Get("source", ""))
tgrabcolumn.SType = toolkit.ToString(mValcs.Get("sourcetype", ""))
tgrabcolumn.Destination = toolkit.ToString(mValcs.Get("destination", ""))
tgrabcolumn.DType = toolkit.ToString(mValcs.Get("destinationtype", ""))
// tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
tCollectionSetting.MapsColumns = append(tCollectionSetting.MapsColumns, &tgrabcolumn)
}
//Fetch Filter Condition
if mVal.Has("filtercond") {
tfiltercond := toolkit.M{}
tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
} else {
tCollectionSetting.SetFilterCond(tfiltercond)
}
}
//Fetch limit data
if mVal.Has("limitrow") {
tLimitrow, err := toolkit.ToM(mVal["limitrow"])
if err == nil {
tCollectionSetting.Take = toolkit.ToInt(tLimitrow.Get("take", 0), toolkit.RoundingAuto)
tCollectionSetting.Skip = toolkit.ToInt(tLimitrow.Get("skip", 0), toolkit.RoundingAuto)
}
}
//Fetch Connection Info
tConnInfo := toolkit.M{}
tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))
tHost := toolkit.ToString(tConnInfo.Get("host", ""))
tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
tSettings := toolkit.M{}
tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection.Close()
destDboxs[tnameid] = &tDestDbox
sGrabber.CollectionSettings[tnameid] = &tCollectionSetting
}
err = nil
if len(destDboxs) == 0 || len(sGrabber.CollectionSettings) == 0 {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
if !config.Has("histconf") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
Log.AddLog("Start fetch histconf", "INFO")
histConf, err = toolkit.ToM(config.Get("histconf", nil))
if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
if !config.Has("extcommand") {
return
}
Log.AddLog("Start fetch extcommand", "INFO")
extCommand, _ = toolkit.ToM(config.Get("extcommand", nil))
return
}
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
var hasLookup bool
toolkit.Println("payload : ", payload)
if payload.Has("haslookup") {
hasLookup = payload.Get("haslookup").(bool)
}
_id := toolkit.ToString(payload.Get("browserid", ""))
sort := payload.Get("sort")
search := payload.Get("search")
_ = search
take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)
TblName := toolkit.M{}
payload.Unset("browserid")
//sorter = ""
if sort != nil {
tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
fmt.Printf("====== sort %#v\n", tmsort["dir"])
if tmsort["dir"] == "asc" {
sorter = tmsort["field"].(string)
} else if tmsort["dir"] == "desc" {
sorter = "-" + tmsort["field"].(string)
} else if tmsort["dir"] == nil {
sorter = " "
}
} else {
sorter = " "
}
dataDS := new(colonycore.DataBrowser)
err := colonycore.Get(dataDS, _id)
if err != nil {
return 0, nil, nil, err
}
dataConn := new(colonycore.Connection)
err = colonycore.Get(dataConn, dataDS.ConnectionID)
if err != nil {
return 0, nil, nil, err
}
if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
return 0, nil, nil, err
}
connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
if err != nil {
return 0, nil, nil, err
}
if dataDS.QueryType == "" {
TblName.Set("from", dataDS.TableNames)
payload.Set("from", dataDS.TableNames)
} else if dataDS.QueryType == "Dbox" {
getTableName := toolkit.M{}
toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
payload.Set("from", getTableName.Get("from").(string))
if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
payload.Set("select", getTableName.Get("select").(string))
}
} else if dataDS.QueryType == "SQL" {
var QueryString string
if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
} else if dataConn.Driver == "mssql" {
QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
toolkit.ToString(take) + " ROWS ONLY "
} else if dataConn.Driver == "postgres" {
QueryString = " LIMIT " + toolkit.ToString(take) +
" OFFSET " + toolkit.ToString(skip)
}
stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
payload.Set("freetext", stringQuery)
// toolkit.Println(stringQuery)
}
qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)
var selectfield string
for _, metadata := range dataDS.MetaData {
tField := metadata.Field
if payload.Has(tField) {
selectfield = toolkit.ToString(tField)
if toolkit.IsSlice(payload[tField]) {
query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
} else if !toolkit.IsNilOrEmpty(payload[tField]) {
var hasPattern bool
for _, val := range querypattern {
if strings.Contains(toolkit.ToString(payload[tField]), val) {
hasPattern = true
}
}
if hasPattern {
query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
} else {
switch toolkit.ToString(metadata.DataType) {
case "int":
query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
case "float32":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
case "float64":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
default:
query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
}
}
}
}
}
if hasLookup && selectfield != "" {
if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
query = query.Select(selectfield)
qcount = qcount.Select(selectfield)
} else {
query = query.Select(selectfield).Group(selectfield)
qcount = qcount.Select(selectfield).Group(selectfield)
}
}
ccount, err := qcount.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer ccount.Close()
dcount := ccount.Count()
cursor, err := query.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer cursor.Close()
data := []toolkit.M{}
cursor.Fetch(&data, 0, false)
if err != nil {
return 0, nil, nil, err
}
if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
dataMongo := []toolkit.M{}
for _, val := range data {
mVal, _ := toolkit.ToM(val.Get("_id"))
dataMongo = append(dataMongo, mVal)
}
data = dataMongo
} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
/*distinct value for flat file*/
dataFlat := []toolkit.M{}
var existingVal = []string{""}
for _, val := range data {
valString := toolkit.ToString(val.Get(selectfield))
if !toolkit.HasMember(existingVal, valString) {
dataFlat = append(dataFlat, val)
existingVal = append(existingVal, valString)
}
}
data = dataFlat
}
return dcount, data, dataDS, nil
}
func (b *BaseController) ConvertMGOToSQLServer(m orm.IModel) error {
tStart := time.Now()
tk.Printf("\nConvertMGOToSQLServer: Converting %v \n", m.TableName())
tk.Println("ConvertMGOToSQLServer: Starting to convert...\n")
c, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Cursor(nil)
if e != nil {
return e
}
defer c.Close()
totalData := c.Count()
processIter := tk.ToInt(tk.ToFloat64(totalData/maxDataEachProcess, 0, tk.RoundingUp), tk.RoundingUp)
if maxDataEachProcess == 0 {
processIter = 0
}
for iter := 0; iter < processIter+1; iter++ {
skip := iter * maxDataEachProcess
take := maxDataEachProcess
if maxDataEachProcess == 0 {
take = totalData
} else if iter == processIter {
take = totalData - skip
}
csr, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Skip(skip).Take(take).Cursor(nil)
if e != nil {
return e
}
result := []tk.M{}
e = csr.Fetch(&result, 0, false)
csr.Close()
if e != nil {
return e
}
dtLen := len(result)
resPart := make([][]tk.M, worker)
if dtLen < worker {
resPart = make([][]tk.M, 1)
resPart[0] = result
} else {
workerTaskCount := tk.ToInt(tk.ToFloat64(dtLen/worker, 0, tk.RoundingAuto), tk.RoundingAuto)
count := 0
for i := 0; i < worker; i++ {
if i == worker-1 {
resPart[i] = result[count:]
} else {
resPart[i] = result[count : count+workerTaskCount]
}
count += workerTaskCount
}
}
wg := &sync.WaitGroup{}
wg.Add(len(resPart))
for _, val := range resPart {
// go b.Insert(val, m, wg)
go b.InsertBulk(val, m, wg, tStart)
}
wg.Wait()
}
tk.Println("\nConvertMGOToSQLServer: Finish.")
tk.Printf("Completed Success in %v \n", time.Since(tStart))
return nil
}
func savedatagrab() (err error) {
var wg, wgstream sync.WaitGroup
// if len(sGrabber.CollectionSettings) > 0 {
// wg.Add(len(sGrabber.CollectionSettings))
// }
mapRecHistory = make(map[string]string, 0)
historyFileName = fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))
for key, _ := range sGrabber.CollectionSettings {
//set history name
mapRecHistory[key] = fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
//================
err = nil
note := ""
dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", mapRecHistory[key])
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data source", key), "INFO")
iQ, err := sGrabber.GetQuery(key)
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get query data : %s", key, err.Error())
errlogsavehistory(note, dt)
continue
}
defer sGrabber.CloseConn()
csr, err := iQ.Cursor(nil)
if err != nil || csr == nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to create cursor or cursor nil to get data : %s", key, err.Error())
errlogsavehistory(note, dt)
continue
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data souce done", key), "INFO")
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save", key), "INFO")
err = destDboxs[key].IConnection.Connect()
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
errlogsavehistory(note, dt)
continue
}
q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
if destDboxs[key].collection != "" {
q = q.From(destDboxs[key].collection)
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save done", key), "INFO")
//Update Total Process
mutex.Lock()
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
err = getsnapshot()
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
Log.AddLog(note, "ERROR")
}
if pid == snapshotdata.Pid {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, csr.Count()), "INFO")
snapshotdata.Cgtotal += csr.Count()
err = savesnapshot()
}
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
Log.AddLog(note, "ERROR")
}
mutex.Unlock()
outtm := make(chan toolkit.M)
wgstream.Add(1)
go func(intm chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
streamsavedata(intm, sQ, key, dt)
wgstream.Done()
}(outtm, q, key, dt)
wg.Add(1)
go func(outtm chan toolkit.M, csr dbox.ICursor) {
condition := true
for condition {
results := make([]toolkit.M, 0)
err = csr.Fetch(&results, 1000, false)
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to fetch data :%s", key, err.Error())
Log.AddLog(note, "ERROR")
condition = false
continue
}
for _, result := range results {
xresult := toolkit.M{}
for _, column := range sGrabber.CollectionSettings[key].MapsColumns {
var val, tval interface{}
strsplits := strings.Split(column.Source, "|")
ttm := toolkit.M{}
lskey := ""
for i, strsplit := range strsplits {
lskey = strsplit
if i == 0 && result.Has(strsplit) {
ttm.Set(strsplit, result[strsplit])
} else if ttm.Has(strsplit) {
ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
}
}
strsplits = strings.Split(column.Destination, "|")
if ttm.Has(lskey) {
tval = ttm[lskey]
} else {
tval = sedotan.DefaultValue(column.DType)
}
// if len(strsplits) > 1 {
// if xresult.Has(strsplits[0]) {
// val = xre
// } else {
// }
// }
tm := toolkit.M{}
if xresult.Has(strsplits[0]) {
tm, _ = toolkit.ToM(xresult[strsplits[0]])
}
// xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
switch column.DType {
case "int":
tval = toolkit.ToInt(tval, toolkit.RoundingAuto)
case "float":
tval = toolkit.ToFloat64(tval, 2, toolkit.RoundingAuto)
case "string":
tval = toolkit.ToString(tval)
default:
tval = tval
}
val = getresultobj(strsplits, tval, tm)
// if len(strsplits) == 1 {
// val = tval
// } else {
// val = getresultobj(strsplits[1:], tval)
// }
// if len(strsplits) > 1 && {
// }
xresult.Set(strsplits[0], val)
// for i, strsplit := range strsplits {
// if i == 0 && ttm.Has(lskey) {
// xresult.Set(strsplit, ttm[lskey])
// } else if (i + 1) == len(strsplits) {
// }
// // else if ttm.Has(strsplit) {
// // ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
// // }
// }
}
outtm <- xresult
}
if len(results) < 1000 || err != nil {
condition = false
}
}
//
//
// ms := []toolkit.M{}
// for _, val := range results {
// m := toolkit.M{}
// for _, column := range g.CollectionSettings[dataSettingId].MapsColumns {
// m.Set(column.Source, "")
// if val.Has(column.Destination) {
// m.Set(column.Source, val[column.Destination])
// }
// }
// ms = append(ms, m)
// }
close(outtm)
wg.Done()
}(outtm, csr)
}
wgstream.Wait()
wg.Wait()
return
}
func streamsavedata(intms <-chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
var err error
iN, note := 0, ""
for intm := range intms {
if destDboxs[key].desttype == "mongo" {
intm.Set("_id", toolkit.GenerateRandomString("", 32))
}
if len(intm) == 0 {
continue
}
//Pre Execute Program
if extCommand.Has("pre") && toolkit.ToString(extCommand["pre"]) != "" {
sintm := toolkit.JsonString(intm)
arrcmd := make([]string, 0, 0)
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// }
arrcmd = append(arrcmd, toolkit.ToString(extCommand["pre"]))
arrcmd = append(arrcmd, sintm)
// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute pre external command :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
err = toolkit.UnjsonFromString(output, &intm)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get pre external command output :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
}
err = sQ.Exec(toolkit.M{
"data": intm,
})
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
note = "Error Found"
continue
}
err = saverechistory(key, intm)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert record data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
note = "Error Found"
}
iN += 1
if math.Mod(float64(iN), 100) == 0 {
_ = updatesnapshot(iN, key)
dt = dt.Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
iN = 0
}
//Post Execute Program
if extCommand.Has("post") {
sintm := toolkit.JsonString(intm)
arrcmd := make([]string, 0, 0)
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// }
arrcmd = append(arrcmd, toolkit.ToString(extCommand["post"]))
arrcmd = append(arrcmd, sintm)
// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute post external command :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
err = toolkit.UnjsonFromString(output, &intm)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get post external command output :%s", key, err.Error()), "ERROR")
note = "Error Found"
continue
}
}
}
dt = dt.Set("note", note).
Set("grabstatus", "done").
Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
_ = updatesnapshot(iN, key)
err = savehistory(dt)
if err != nil {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save history : %s", key), "ERROR")
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Finish save data", key), "INFO")
destDboxs[key].IConnection.Close()
}
func doGenerateMaintenanceDataBrowser(wg *sync.WaitGroup, d *GenDataBrowser, plant PowerPlantCoordinates) {
var e error
ctx := d.BaseController.Ctx
c := ctx.Connection
saveDatas := []orm.IModel{}
plantCodeStr := plant.PlantCode
dataCount := 0
tmpPlantCondition := conditions.Get(plantCodeStr)
if tmpPlantCondition != nil {
plantCondition := tmpPlantCondition.(tk.M)
length := plantCondition.GetInt("length")
dets := plantCondition.Get("det").([]tk.M)
dets = append(dets, tk.M{})
turbinesCodes := []string{}
genIDTempTable := tk.GenerateRandomString("", 20)
detsLen := len(dets) - 1
tk.Printf("detsLen: %v \n", detsLen)
for i, det := range dets {
freeQuery := false
tk.Println(i)
tk.Printf("dets: %#v \n", det)
assets := []FunctionalLocation{}
systemAssets := []FunctionalLocation{}
desc := det.GetString("desc")
query := []*dbox.Filter{}
query = append(query, dbox.Contains("FunctionalLocationCode", plantCodeStr))
query = append(query, dbox.Eq("PIPI", plantCodeStr))
queryStr := "select * from FunctionalLocation where FunctionalLocationCode like('%" + plantCodeStr + "%') and PIPI = '" + plantCodeStr + "' "
if i == 0 {
query = append(query, dbox.Contains("Description", desc))
if plantCodeStr == "2220" {
query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
} else {
query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
}
} else if i == 1 && detsLen == 2 {
query = append(query, dbox.Contains("Description", desc))
if plantCodeStr == "2220" {
query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
} else {
query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
}
} else {
tk.Printf("turbinesCodes: %v \n", len(turbinesCodes))
if len(turbinesCodes) > 1 {
maxLength := 1000
maxLoop := tk.ToInt(tk.ToFloat64(len(turbinesCodes)/maxLength, 0, tk.RoundingUp), tk.RoundingUp)
for i := 0; i <= maxLoop; i++ {
datas := []string{}
if i != maxLoop {
datas = turbinesCodes[i*maxLength : (i*maxLength)+maxLength]
} else {
datas = turbinesCodes[i*maxLength:]
}
tmpNotIn := []orm.IModel{}
for _, val := range datas {
tmpData := new(GenDataBrowserNotInTmp)
tmpData.ID = genIDTempTable
tmpData.FLCode = val
tmpNotIn = append(tmpNotIn, tmpData)
}
// tk.Printf("tmpNotIn: %v \n", len(tmpNotIn))
e = ctx.InsertBulk(tmpNotIn)
ErrorHandler(e, "generateMaintenanceDataBrowser")
}
queryStr = queryStr + " and FunctionalLocationCode not in(select flcode from GenDataBrowserNotInTmp where ID = '" + genIDTempTable + "')"
freeQuery = true
}
}
if freeQuery {
csrDet, e := c.NewQuery().Command("freequery", tk.M{}.Set("syntax", queryStr)).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&assets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
} else {
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(dbox.And(query...)).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&assets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
}
tk.Printf("-- assets: %v \n", len(assets))
if len(assets) > 0 {
relatedAssets := []FunctionalLocation{}
for _, asset := range assets {
if plantCodeStr == "2110" {
if len(asset.FunctionalLocationCode) <= 13 {
query = []*dbox.Filter{}
query = append(query,
dbox.And(
dbox.Eq("PG", "MP1"),
dbox.Eq("PIPI", plantCodeStr),
dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode),
dbox.And(
dbox.Lte("LEN(FunctionalLocationCode)", 13),
dbox.Gte("LEN(FunctionalLocationCode)", 12),
),
),
)
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&systemAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
}
if systemAssets == nil || len(systemAssets) == 0 {
query = []*dbox.Filter{}
if i != 2 {
query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
} else {
query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
}
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&relatedAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
for _, relasset := range relatedAssets {
isTurbineSystem := false
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
isTurbineSystem = true
}
newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)
if newEquipment != "" {
if i != 2 {
turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
}
for _, year := range years {
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = relasset.FunctionalLocationCode
data.FLDescription = relasset.Description
data.IsTurbine = false
// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
data.IsTurbine = true
// tk.Println(" isTurbine: TRUE")
} else {
// tk.Println(" isTurbine: FALSE")
data.TurbineParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 {
data.AssetType = "Gas"
}
data.EquipmentType = newEquipment
data.EquipmentTypeDescription = newEquipment
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
substrValInt = 1
substr = "ST"
} else if data.AssetType == "Gas" {
substrValInt = 2
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 1")*/
}
}
}
} else {
// with system assets
if i != 2 {
turbinesCodes = append(turbinesCodes, asset.FunctionalLocationCode)
}
for _, year := range years {
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = asset.FunctionalLocationCode
data.FLDescription = asset.Description
data.IsTurbine = false
data.IsSystem = false
// tk.Printf("%v | %v | ", asset.FunctionalLocationCode, i)
if asset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
data.IsTurbine = true
data.IsSystem = true
// tk.Println(" isTurbine: TRUE")
} else {
// tk.Println(" isTurbine: FALSE")
data.TurbineParent = asset.FunctionalLocationCode
data.SystemParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 {
data.AssetType = "Gas"
}
data.EquipmentType = "System"
data.EquipmentTypeDescription = "System"
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
substrValInt = 1
substr = "ST"
} else if data.AssetType == "Gas" {
substrValInt = 2
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
// dataCount++
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 2")*/
}
for _, sysAsset := range systemAssets {
query = []*dbox.Filter{}
if i != 2 {
query = append(query, dbox.Contains("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
} else {
query = append(query, dbox.Eq("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
}
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&relatedAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
for _, relasset := range relatedAssets {
isTurbineSystem := false
if (relasset.FunctionalLocationCode == asset.FunctionalLocationCode || relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode) && i != 2 {
isTurbineSystem = true
}
newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)
if newEquipment != "" {
if i != 2 {
turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
}
for _, year := range years {
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = relasset.FunctionalLocationCode
data.FLDescription = relasset.Description
data.IsTurbine = false
data.IsSystem = false
// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)
if relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode && i != 2 {
data.IsTurbine = true
data.IsSystem = true
// tk.Println(" isTurbine: TRUE")
} else {
// tk.Println(" isTurbine: FALSE")
data.TurbineParent = asset.FunctionalLocationCode
data.SystemParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 {
data.AssetType = "Gas"
}
data.EquipmentType = newEquipment
data.EquipmentTypeDescription = newEquipment
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
substrValInt = 1
substr = "ST"
} else if data.AssetType == "Gas" {
substrValInt = 2
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 3")*/
}
}
}
}
}
} else {
// another plant(s)
query = []*dbox.Filter{}
if i != detsLen {
query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
} else {
query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
}
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&relatedAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
tk.Printf("-- related assets: %v \n", len(relatedAssets))
for _, relasset := range relatedAssets {
isTurbineSystem := false
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
isTurbineSystem = true
}
newEquipment := ""
newEquipment = d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)
if newEquipment != "" {
if i != detsLen {
turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
}
for _, year := range years {
_ = year
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = relasset.FunctionalLocationCode
data.FLDescription = relasset.Description
data.IsTurbine = false
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
data.IsTurbine = true
} else {
data.TurbineParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 && detsLen > 1 {
data.AssetType = "Gas"
}
data.EquipmentType = newEquipment
data.EquipmentTypeDescription = newEquipment
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
// substrValInt = 1
substrValInt = 2
substr = "ST" + substr
} else if data.AssetType == "Gas" {
substrValInt = 2
substr = "GT" + substr
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 4")*/
}
}
}
}
}
}
}
e = c.NewQuery().Delete().From(new(GenDataBrowserNotInTmp).TableName()).SetConfig("multiexec", true).Where(dbox.Eq("ID", genIDTempTable)).Exec(nil)
if len(saveDatas) > 0 {
e = ctx.InsertBulk(saveDatas)
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
}
}
mugen.Lock()
dataCounts.Set(plantCodeStr, dataCount)
mugen.Unlock()
wg.Done()
}
func (a *DataFlowController) GetDataMonitoring(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
e := r.GetPayload(&payload)
if e != nil {
return helper.CreateResult(false, nil, e.Error())
}
status := tk.ToString(payload["status"])
filters := []*dbox.Filter{}
var filter *dbox.Filter
filter = new(dbox.Filter)
if strings.Contains(strings.ToLower(status), "run") {
filters = append(filters, dbox.Eq("status", "RUN"))
} else {
filters = append(filters, dbox.Ne("status", "RUN"))
}
take := tk.ToInt(payload["take"], tk.RoundingAuto)
skip := tk.ToInt(payload["skip"], tk.RoundingAuto)
start := tk.ToString(payload["startdate"])
end := tk.ToString(payload["enddate"])
search := tk.ToString(payload["search"])
startdate := time.Now()
enddate := time.Now()
if start != "" {
startdate, _ = time.Parse(time.RFC3339, start)
filters = append(filters, dbox.Gte("startdate", startdate))
startdate = startdate.AddDate(0, 0, 1)
startdate = startdate.Add(time.Duration(-1) * time.Second)
filters = append(filters, dbox.Lte("startdate", startdate))
}
if end != "" && !strings.Contains(strings.ToLower(status), "run") {
enddate, _ = time.Parse(time.RFC3339, end)
filters = append(filters, dbox.Gte("enddate", enddate))
enddate = enddate.AddDate(0, 0, 1)
enddate = enddate.Add(time.Duration(-1) * time.Second)
filters = append(filters, dbox.Lte("enddate", enddate))
}
if search != "" {
filters = append(filters, dbox.Or(dbox.Contains("flow.name", search), dbox.Contains("flow.description", search)))
}
filter = dbox.And(filters...)
dataDs := []colonycore.DataFlowProcess{}
cursor, err := colonycore.Finds(new(colonycore.DataFlowProcess), tk.M{}.Set("where", filter).Set("take", take).Set("skip", skip).Set("order", []string{"-startdate"}))
cursorCount, _ := colonycore.Finds(new(colonycore.DataFlowProcess), tk.M{}.Set("where", filter))
res := tk.M{}
if cursor != nil {
cursor.Fetch(&dataDs, 0, false)
defer cursor.Close()
defer cursorCount.Close()
res.Set("total", cursorCount.Count())
} else {
res.Set("total", 0)
}
if err != nil && cursor != nil {
return helper.CreateResult(false, nil, err.Error())
}
res.Set("data", dataDs)
return helper.CreateResult(true, res, "success")
}