Пример #1
0
func (a *SessionController) GetSession(r *knot.WebContext) interface{} {
	var filter *dbox.Filter
	r.Config.OutputType = knot.OutputJson
	_ = a.InitialSetDatabase()

	payload := map[string]interface{}{}
	err := r.GetForms(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
		payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
	}

	tSession := new(acl.Session)
	if find := toolkit.ToString(payload["find"]); find != "" {
		filter = new(dbox.Filter)
		filter = dbox.Contains("loginid", find)
	}
	take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)

	// c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
	// c, err := acl.Find(tSession, nil, toolkit.M{}.Set("take", payload["take"].(int)).Set("skip", payload["skip"].(int)))
	c, err := acl.Find(tSession, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}

	data := toolkit.M{}
	arrm := make([]toolkit.M, 0, 0)
	err = c.Fetch(&arrm, 0, false)
	for i, val := range arrm {
		arrm[i].Set("duration", time.Since(val["created"].(time.Time)).Hours())
		arrm[i].Set("status", "ACTIVE")
		if val["expired"].(time.Time).Before(time.Now().UTC()) {
			arrm[i].Set("duration", val["expired"].(time.Time).Sub(val["created"].(time.Time)).Hours())
			arrm[i].Set("status", "EXPIRED")
		}
	}
	c.Close()

	c, err = acl.Find(tSession, filter, nil)

	data.Set("Datas", arrm)
	data.Set("total", c.Count())

	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	} else {
		return helper.CreateResult(true, data, "")
	}

}
Пример #2
0
func TestAggr(t *testing.T) {
	skipIfNil(t)
	c2 := c.Min(fn).Max(fn).Sum(fn).Avg(fn).Exec()
	check(t, c2.Error, "Aggr")
	if toolkit.ToInt(c2.Result.Min, toolkit.RoundingAuto) != min ||
		toolkit.ToInt(c2.Result.Max, toolkit.RoundingAuto) != max ||
		c2.Result.Sum != toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) ||
		c2.Result.Avg != avg {
		t.Fatalf("Error aggr. Got %v\n", toolkit.JsonString(c2.Result))
	}
	toolkit.Println("Value: ", toolkit.JsonString(c2.Result))
}
Пример #3
0
func (a *AdministrationController) GetAccess(r *knot.WebContext) interface{} {
	var filter *dbox.Filter
	r.Config.OutputType = knot.OutputJson
	a.InitialSetDatabase()

	payload := map[string]interface{}{}
	err := r.GetForms(&payload)

	if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
		payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
	}

	tAccess := new(acl.Access)
	if find := toolkit.ToString(payload["find"]); find != "" {
		filter = new(dbox.Filter)
		filter = dbox.Or(dbox.Contains("id", find),
			dbox.Contains("title", find),
			dbox.Contains("group1", find),
			dbox.Contains("group2", find),
			dbox.Contains("group3", find),
			dbox.Contains("specialaccess1", find),
			dbox.Contains("specialaccess2", find),
			dbox.Contains("specialaccess3", find),
			dbox.Contains("specialaccess4", find))
	}

	data := toolkit.M{}
	arrm := make([]toolkit.M, 0, 0)

	take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)

	c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
	if err == nil {
		err = c.Fetch(&arrm, 0, false)
	}
	c.Close()

	c, err = acl.Find(tAccess, filter, nil)
	data.Set("Datas", arrm)
	data.Set("total", c.Count())

	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	} else {
		return helper.CreateResult(true, data, "")
	}

}
Пример #4
0
func (a *UserController) GetUser(r *knot.WebContext) interface{} {
	var filter *dbox.Filter
	r.Config.OutputType = knot.OutputJson
	_ = a.InitialSetDatabase()

	payload := map[string]interface{}{}
	err := r.GetForms(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
		payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
	}

	tUser := new(acl.User)
	if find := toolkit.ToString(payload["find"]); find != "" {
		filter = new(dbox.Filter)
		filter = dbox.Or(dbox.Contains("id", find),
			dbox.Contains("fullname", find),
			dbox.Contains("email", find))
	}
	take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)

	c, err := acl.Find(tUser, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}

	data := toolkit.M{}
	arrm := make([]toolkit.M, 0, 0)
	err = c.Fetch(&arrm, 0, false)
	c.Close()

	c, err = acl.Find(tUser, filter, nil)

	data.Set("Datas", arrm)
	data.Set("total", c.Count())

	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	} else {
		return helper.CreateResult(true, data, "")
	}
}
Пример #5
0
func (a *Controller) KendoGridSettings(ins toolkit.M) toolkit.M {
	if ins == nil {
		ins = toolkit.M{}
	}
	s := toolkit.M{}
	q_skip := a.Ctx.Input.Query("skip")
	q_page := a.Ctx.Input.Query("page")
	q_size := a.Ctx.Input.Query("pageSize")

	if q_skip != "" {
		s.Set("skip", toolkit.ToInt(q_skip))
	}

	if q_page != "" {
		s.Set("page", toolkit.ToInt(q_page))
	}

	if q_size != "" {
		s.Set("limit", toolkit.ToInt(q_size))
	}

	sortField := strings.ToLower(a.Ctx.Input.Query("sort[0][field]"))
	sortDir := a.Ctx.Input.Query("sort[0][dir]")

	if sortField != "" {
		if sortField == "id" {
			sortField = "_id"
		}
		if sortDir == "" || sortDir == "asc" {
			s.Set("order", []string{sortField})
		} else {
			s.Set("order", []string{"-" + sortField})
		}
	}

	if fqe := a.KendoGridFilter("filter"); fqe != nil {
		if ins.Has("where") {
			fqe = dbs.And(fqe, ins.Get("where").(*dbs.QE))
		}
		s.Set("where", fqe)
	}

	return s
}
Пример #6
0
func (a *GroupController) Search(r *knot.WebContext) interface{} {
	var filter *dbox.Filter
	r.Config.OutputType = knot.OutputJson
	_ = a.InitialSetDatabase()

	payload := map[string]interface{}{}

	err := r.GetForms(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	if strings.Contains(toolkit.TypeName(payload["search"]), "float") {
		payload["search"] = toolkit.ToInt(payload["search"], toolkit.RoundingAuto)
	}

	tGroup := new(acl.Group)
	if search := toolkit.ToString(payload["search"]); search != "" {
		filter = new(dbox.Filter)
		filter = dbox.Or(dbox.Contains("_id", search), dbox.Contains("title", search), dbox.Contains("owner", search))

	}
	fmt.Println(filter)
	take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)

	c, err := acl.Find(tGroup, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}

	data := toolkit.M{}
	arrm := make([]toolkit.M, 0, 0)
	err = c.Fetch(&arrm, 0, false)

	data.Set("Datas", arrm)
	data.Set("total", c.Count())

	if err != nil {
		return helper.CreateResult(true, nil, err.Error())
	}

	return helper.CreateResult(true, data, "")

}
Пример #7
0
func ToString(reflectIs reflect.Kind, i interface{}) string {
	var s string
	if reflectIs != reflect.String {
		toI := toolkit.ToInt(i)
		s = strconv.Itoa(toI)
	} else {
		s = i.(string)
	}
	return s
}
Пример #8
0
func toInterface(data string, dataType string, dateFormat string) interface{} {
	if dataType == "" {
		if strings.HasPrefix(data, "#") && strings.HasSuffix(data, "#") {
			dataType = DataDate
		} else {
			vfloat := toolkit.ToFloat64(dataType, 2, toolkit.RoundingAuto)
			vint := toolkit.ToInt(dataType, toolkit.RoundingAuto)
			if int(vfloat) == vint && vint != 0 {
				dataType = DataInt
			} else if vfloat != 0 {
				// dataType = DataFloat
				b, i := toolkit.MemberIndex(DataFloats, dataType)
				if b {
					for idx, dataFloat := range DataFloats {
						if idx == i {
							dataType = dataFloat
						}
					}
				}
			} else {
				dataType = DataString
			}
		}
	}

	if dataType == DataDate {
		return toolkit.String2Date(data, dateFormat)
	} else if dataType == DataInt {
		return toolkit.ToInt(data, toolkit.RoundingAuto)
	} else if toolkit.HasMember(DataFloats, dataType) {
		return toolkit.ToFloat64(data, 2, toolkit.RoundingAuto)
	} else {
		return data
	}

	return nil
}
Пример #9
0
func checkisonprocess(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
	cond = false
	if _, f := mapsnapshot[id]; !f {
		return
	}

	mtkdata := mapsnapshot[id]
	mapcron, _ := toolkit.ToM(intervalconf["cronconf"])
	if mtkdata.Grabstatus == "running" && len(mapcron) <= 0 {
		cond = true
	}

	mtkstarttime := sedotan.StringToDate(mtkdata.Starttime)

	timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
	timeoutsec := time.Second * time.Duration(timeoutint)

	if cond && (thistime.After(mtkstarttime.Add(timeoutsec)) && timeoutint > 0) {
		cond = false
	}

	return
}
Пример #10
0
//Check Time run and record to snapshot
func checkistimerun(id string, intervalconf toolkit.M, grabconf toolkit.M) (cond bool) {
	cond = false
	var mtklaststarttime, mtklastupdate time.Time
	tempss := Snapshot{Id: id,
		Starttime:      "",
		Laststartgrab:  "",
		Lastupdate:     "",
		Grabcount:      1,
		Rowgrabbed:     0,
		Errorfound:     0,
		Lastgrabstatus: "",
		Grabstatus:     "running",
		Cgtotal:        0,
		Cgprocess:      0,
		Note:           "",
		Pid:            0}
	// 	Lastgrabstatus string //[success|failed]
	// 	Grabstatus     string //[running|done]

	strintervalconf := intervalconf.Get("starttime", "").(string)
	intervalstart := sedotan.StringToDate(strintervalconf)
	strintervaltype := intervalconf.Get("intervaltype", "").(string)
	grabinterval := toolkit.ToInt(intervalconf.Get("grabinterval", 0), toolkit.RoundingAuto)

	mtkdata := Snapshot{}

	if _, f := mapsnapshot[id]; f {
		mtkdata = mapsnapshot[id]
		tempss.Starttime = mtkdata.Starttime
		//for data timeout
		mtklaststarttime = sedotan.StringToDate(mtkdata.Laststartgrab)
		mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)

		timeoutint := toolkit.ToInt(grabconf.Get("timeout", 0), toolkit.RoundingAuto)
		timeoutsec := time.Second * time.Duration(timeoutint)
		if mtkdata.Grabstatus == "running" && thistime.After(mtklaststarttime.Add(timeoutsec)) && timeoutint > 0 {
			mtkdata.Lastupdate = sedotan.DateToString(mtklaststarttime.Add(timeoutsec))
			mtklastupdate = sedotan.StringToDate(mtkdata.Lastupdate)
			mtkdata.Lastgrabstatus = "failed"
			mtkdata.Grabstatus = "done"
		}

		// set initial - found a long time ago snapshot
		if intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)) {
			tempss.Starttime = sedotan.DateToString(thistime)

			mtkdata.Grabcount = 0
			mtkdata.Rowgrabbed = 0
			mtkdata.Errorfound = 0
			mtkdata.Lastgrabstatus = ""
		}
	}

	if mtkdata.Lastgrabstatus == "failed" {
		grabinterval = toolkit.ToInt(intervalconf.Get("timeoutinterval", 0), toolkit.RoundingAuto)
	}

	if strintervalconf != "" && intervalstart.Before(thistime) {
		_, fcond := mapsnapshot[id]

		switch {
		case !fcond || intervalstart.After(sedotan.StringToDate(mtkdata.Starttime)):
			cond = true
		case intervalconf.Get("grabinterval", 0).(float64) > 0:
			var durationgrab time.Duration

			switch strintervaltype {
			case "seconds":
				durationgrab = time.Second * time.Duration(grabinterval)
			case "minutes":
				durationgrab = time.Minute * time.Duration(grabinterval)
			case "hours":
				durationgrab = time.Hour * time.Duration(grabinterval)
			}
			nextgrab := mtklastupdate.Add(durationgrab)

			if nextgrab.Before(thistime) && !mtklastupdate.IsZero() {
				cond = true
				tempss.Grabcount = mtkdata.Grabcount + 1
				tempss.Rowgrabbed = mtkdata.Rowgrabbed
				tempss.Errorfound = mtkdata.Errorfound
				tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
			}
		}
	}

	mapcronconf, _ := toolkit.ToM(intervalconf.Get("cronconf", nil))
	minutetime := sedotan.DateMinutePress(thistime) //review this and the usage and parsing in cron
	if len(mapcronconf) > 0 {
		//min hour dayofmonth month dayofweek
		cond = true
		arrstr := [6]string{"min", "hour", "dayofmonth", "month", "dayofweek"}
		// arrstr := [6]string{"month", "dayofmonth", "dayofweek", "hour", "min", "second"}
		for _, str := range arrstr {
			sval := toolkit.ToString(mapcronconf.Get(str, ""))
			ival := toolkit.ToInt(sval, toolkit.RoundingAuto)

			var valcom int
			switch str {
			// case "second":
			// 	valcom = thistime.Second()
			case "min":
				valcom = thistime.Minute()
			case "hour":
				valcom = thistime.Hour()
			case "dayofmonth":
				valcom = thistime.Day()
			case "month":
				valcom = toolkit.ToInt(thistime.Month(), toolkit.RoundingAuto)
			case "dayofweek":
				valcom = toolkit.ToInt(thistime.Weekday(), toolkit.RoundingAuto)
			}

			if sval != "*" && valcom != ival {
				cond = false
			} else {
				cond = cond && true
			}
		}

		if mtkdata.Laststartgrab != "" {
			cond = cond && minutetime.After(sedotan.StringToDate(mtkdata.Laststartgrab))
		}

		if cond && minutetime.Before(sedotan.StringToDate(mtkdata.Starttime)) {
			tempss.Grabcount = mtkdata.Grabcount + 1
			tempss.Rowgrabbed = mtkdata.Rowgrabbed
			tempss.Errorfound = mtkdata.Errorfound
			tempss.Lastgrabstatus = mtkdata.Lastgrabstatus
		}
	}

	if cond {
		mapsnapshot[id] = tempss
	}

	return
}
Пример #11
0
func (d *DataBrowserController) TestQuery(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	data := colonycore.DataBrowser{} //map[string]interface{}{}
	err := r.GetPayload(&data)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	conn, datacon, err := d.connToDatabase(data.ConnectionID)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	query, err := d.parseQuery(conn, data, datacon)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	cursor, err := query.Cursor(nil)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}
	defer cursor.Close()

	dataFetch := []toolkit.M{}
	err = cursor.Fetch(&dataFetch, 1, false)
	if err != nil {
		cursor.ResetFetch()
		err = cursor.Fetch(&dataFetch, 0, false)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}
	}

	metadata := []*colonycore.StructInfo{}
	// var dt string
	for i, dataFields := range dataFetch {
		if i > 0 {
			break
		}

		j := 1
		for keyField, dataField := range dataFields {
			if strings.Contains(keyField, "id") && !strings.Contains(data.QueryText, "id") &&
				!strings.Contains(data.QueryText, "*") && data.TableNames == "" {
				continue
			}
			sInfo := &colonycore.StructInfo{}
			sInfo.Field = keyField
			sInfo.Label = keyField

			rf := "string"
			if dataField != nil {
				rf = toolkit.TypeName(dataField)

				if rf == "time.Time" {
					rf = "date"
				}
			}
			sInfo.DataType = rf
			sInfo.Format = ""
			sInfo.Align = "Left"
			sInfo.ShowIndex = toolkit.ToInt(j, toolkit.RoundingAuto)
			sInfo.Sortable = false
			sInfo.SimpleFilter = false
			sInfo.AdvanceFilter = false
			sInfo.Aggregate = ""
			metadata = append(metadata, sInfo)
			j++
		}
	}

	data.MetaData = metadata

	return helper.CreateResult(true, data, "")
}
Пример #12
0
func GrabHtmlConfig(data toolkit.M) (*sdt.GrabService, error) {
	var e error
	var gi, ti time.Duration
	xGrabService := sdt.NewGrabService()
	xGrabService.Name = data["nameid"].(string) //"irondcecom"
	xGrabService.Url = data["url"].(string)     //"http://www.dce.com.cn/PublicWeb/MainServlet"

	xGrabService.SourceType = sdt.SourceType_HttpHtml

	grabintervalToInt := toolkit.ToInt(data["grabinterval"], toolkit.RoundingAuto)
	timeintervalToInt := toolkit.ToInt(data["timeoutinterval"], toolkit.RoundingAuto)
	if data["intervaltype"].(string) == "seconds" {
		gi = time.Duration(grabintervalToInt) * time.Second
		ti = time.Duration(timeintervalToInt) * time.Second
	} else if data["intervaltype"].(string) == "minutes" {
		gi = time.Duration(grabintervalToInt) * time.Minute
		ti = time.Duration(timeintervalToInt) * time.Minute
	} else if data["intervaltype"].(string) == "hours" {
		gi = time.Duration(grabintervalToInt) * time.Hour
		ti = time.Duration(timeintervalToInt) * time.Hour
	}

	xGrabService.GrabInterval = gi    //* time.Minute
	xGrabService.TimeOutInterval = ti //* time.Minute //time.Hour, time.Minute, time.Second

	xGrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", timeintervalToInt, data["intervaltype"] /*"seconds"*/)

	grabConfig := sdt.Config{}

	if data["calltype"].(string) == "POST" {
		dataurl := toolkit.M{}
		for _, grabconf := range data["grabconf"].(map[string]interface{}) {
			grabDataConf, e := toolkit.ToM(grabconf)
			if e != nil {
				return nil, e
			}
			for key, subGrabDataConf := range grabDataConf {
				if reflect.ValueOf(subGrabDataConf).Kind() == reflect.Float64 {
					i := toolkit.ToInt(subGrabDataConf, toolkit.RoundingAuto)
					toString := strconv.Itoa(i)
					dataurl[key] = toString
				} else {
					dataurl[key] = subGrabDataConf
				}
			}
		}

		grabConfig.SetFormValues(dataurl)
	}

	grabDataConf, e := toolkit.ToM(data["grabconf"])
	if e != nil {
		return nil, e
	}

	isAuthType := grabDataConf.Has("authtype")
	if isAuthType {
		grabConfig.AuthType = grabDataConf["authtype"].(string)
		grabConfig.LoginUrl = grabDataConf["loginurl"].(string)   //"http://localhost:8000/login"
		grabConfig.LogoutUrl = grabDataConf["logouturl"].(string) //"http://localhost:8000/logout"

		grabConfig.LoginValues = toolkit.M{}.
			Set("name", grabDataConf["loginvalues"].(map[string]interface{})["name"].(string)).
			Set("password", grabDataConf["loginvalues"].(map[string]interface{})["password"].(string))

	}

	xGrabService.ServGrabber = sdt.NewGrabber(xGrabService.Url, data["calltype"].(string), &grabConfig)

	logconfToMap, e := toolkit.ToM(data["logconf"])
	if e != nil {
		return nil, e
	}
	logpath := logconfToMap["logpath"].(string)           //"E:\\data\\vale\\log"
	filename := logconfToMap["filename"].(string) + "-%s" //"LOG-GRABDCETEST"
	filepattern := logconfToMap["filepattern"].(string)   //"20060102"

	logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
	if e != nil {
		return nil, e
	}

	xGrabService.Log = logconf

	xGrabService.ServGrabber.DataSettings = make(map[string]*sdt.DataSetting)
	xGrabService.DestDbox = make(map[string]*sdt.DestInfo)

	tempDataSetting := sdt.DataSetting{}
	tempDestInfo := sdt.DestInfo{}
	// isCondition := []interface{}{}
	tempFilterCond := toolkit.M{}
	// var condition string

	for _, dataSet := range data["datasettings"].([]interface{}) {
		dataToMap, _ := toolkit.ToM(dataSet)
		tempDataSetting.RowSelector = dataToMap["rowselector"].(string)

		for _, columnSet := range dataToMap["columnsettings"].([]interface{}) {
			columnToMap, e := toolkit.ToM(columnSet)
			if e != nil {
				return nil, e
			}
			i := toolkit.ToInt(columnToMap["index"], toolkit.RoundingAuto)
			tempDataSetting.Column(i, &sdt.GrabColumn{Alias: columnToMap["alias"].(string), Selector: columnToMap["selector"].(string)})
		}

		/*if data["calltype"].(string) == "POST" {
			// orCondition := []interface{}{}
			isRowdeletecond := dataToMap.Has("rowdeletecond")
			fmt.Println("isRowdeletecond>", isRowdeletecond)
			if hasRowdeletecond := dataToMap.Has("rowdeletecond"); hasRowdeletecond {
				for key, rowDeleteMap := range dataToMap["rowdeletecond"].(map[string]interface{}) {
					if key == "$or" || key == "$and" {
						for _, subDataRowDelete := range rowDeleteMap.([]interface{}) {
							for subIndex, getValueRowDelete := range subDataRowDelete.(map[string]interface{}) {
								orCondition = append(orCondition, map[string]interface{}{subIndex: getValueRowDelete})
							}
						}
						condition = key
					}
				}
			}
			// tempDataSetting.RowDeleteCond = toolkit.M{}.Set(condition, orCondition)
			tempFilterCond := toolkit.M{}.Set(condition, orCondition)
			tempDataSetting.SetFilterCond(tempFilterCond)
		}*/

		if hasRowdeletecond := dataToMap.Has("rowdeletecond"); hasRowdeletecond {
			rowToM, e := toolkit.ToM(dataToMap["rowdeletecond"])
			if e != nil {
				return nil, e
			}
			tempFilterCond, e = toolkit.ToM(rowToM.Get("filtercond", nil))
			tempDataSetting.SetFilterCond(tempFilterCond)
		}

		if hasRowincludecond := dataToMap.Has("rowincludecond"); hasRowincludecond {
			rowToM, e := toolkit.ToM(dataToMap["rowincludecond"])
			if e != nil {
				return nil, e
			}

			tempFilterCond, e = toolkit.ToM(rowToM.Get("filtercond", nil))
			tempDataSetting.SetFilterCond(tempFilterCond)
		}

		xGrabService.ServGrabber.DataSettings[dataToMap["name"].(string)] = &tempDataSetting //DATA01 use name in datasettings

		connToMap, e := toolkit.ToM(dataToMap["connectioninfo"])
		if e != nil {
			return nil, e
		}
		var db, usr, pwd string

		if hasDb := connToMap.Has("database"); !hasDb {
			db = ""
		} else {
			db = connToMap["database"].(string)
		}

		if hasUser := connToMap.Has("username"); !hasUser {
			usr = ""
		} else {
			usr = connToMap["username"].(string)
		}

		if hasPwd := connToMap.Has("password"); !hasPwd {
			pwd = ""
		} else {
			pwd = connToMap["username"].(string)
		}
		ci := dbox.ConnectionInfo{}
		ci.Host = connToMap["host"].(string) //"E:\\data\\vale\\Data_GrabIronTest.csv"
		ci.Database = db
		ci.UserName = usr
		ci.Password = pwd

		if hasSettings := connToMap.Has("settings"); !hasSettings {
			ci.Settings = nil
		} else {
			settingToMap, e := toolkit.ToM(connToMap["settings"])
			if e != nil {
				return nil, e
			}
			ci.Settings = settingToMap //toolkit.M{}.Set("useheader", settingToMap["useheader"].(bool)).Set("delimiter", settingToMap["delimiter"])
		}

		if hasCollection := connToMap.Has("collection"); !hasCollection {
			tempDestInfo.Collection = ""
		} else {
			tempDestInfo.Collection = connToMap["collection"].(string)
		}

		tempDestInfo.Desttype = dataToMap["desttype"].(string)

		tempDestInfo.IConnection, e = dbox.NewConnection(tempDestInfo.Desttype, &ci)
		if e != nil {
			return nil, e
		}

		xGrabService.DestDbox[dataToMap["name"].(string)] = &tempDestInfo

		//=History===========================================================
		xGrabService.HistoryPath = HistoryPath       //"E:\\data\\vale\\history\\"
		xGrabService.HistoryRecPath = HistoryRecPath //"E:\\data\\vale\\historyrec\\"
		//===================================================================
	}

	return xGrabService, nil
}
Пример #13
0
func fetchConfig() (err error) {

	switch toolkit.ToString(config.Get("sourcetype", "")) {
	case "SourceType_HttpHtml":
		SourceType = SourceType_HttpHtml
	case "SourceType_HttpJson":
		SourceType = SourceType_HttpJson
	default:
		err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
		return
	}

	Log.AddLog("Start fetch grabconf", "INFO")
	if !config.Has("grabconf") {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
		return
	}

	tconfgrab := toolkit.M{}
	tconfgrab, err = toolkit.ToM(config["grabconf"])
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
		return
	}

	grabconfig := sedotan.Config{}
	if tconfgrab.Has("formvalues") {
		tfromvalues := toolkit.M{}
		tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
		if err != nil {
			err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
			return
		}
		grabconfig.SetFormValues(tfromvalues)
	}

	if tconfgrab.Has("loginvalues") {
		grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
	}

	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
		return
	}

	grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
	grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))

	grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
	grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
	grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))

	grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
	grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))

	Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")

	wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)

	Log.AddLog("Start fetch datasettings", "INFO")
	if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
	destDboxs = make(map[string]*DestInfo)

	for i, xVal := range config["datasettings"].([]interface{}) {
		err = nil
		tDataSetting := sedotan.DataSetting{}
		tDestDbox := DestInfo{}

		mVal := toolkit.M{}
		mVal, err = toolkit.ToM(xVal)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
			continue
		}

		tnameid := toolkit.ToString(mVal.Get("nameid", ""))
		if tnameid == "" {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
			continue
		}
		tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))

		// Fetch columnsettings
		if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
			continue
		}

		tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
		for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
			mValcs := toolkit.M{}
			mValcs, err = toolkit.ToM(Valcs)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
				continue
			}

			tgrabcolumn := sedotan.GrabColumn{}
			tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
			tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
			tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
			tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))

			tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
			tDataSetting.Column(tindex, &tgrabcolumn)
		}

		//Fetch Filter Condition
		if mVal.Has("filtercond") {
			tfiltercond := toolkit.M{}
			tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
			} else {
				tDataSetting.SetFilterCond(tfiltercond)
			}
		}

		//Fetch Connection Info
		tConnInfo := toolkit.M{}
		tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
		tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))

		tHost := toolkit.ToString(tConnInfo.Get("host", ""))
		tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
		tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
		tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
		tSettings := toolkit.M{}
		tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}

		tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.IConnection.Close()

		destDboxs[tnameid] = &tDestDbox
		wGrabber.DataSettings[tnameid] = &tDataSetting

	}
	err = nil

	if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	if !config.Has("histconf") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	Log.AddLog("Start fetch histconf", "INFO")
	histConf, err = toolkit.ToM(config.Get("histconf", nil))
	if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	return
}
Пример #14
0
func constructActions(list []interface{}) (flows []colonycore.FlowAction) {
	for _, act := range list {
		flowAction := act.(map[string]interface{})
		server := colonycore.Server{}
		if flowAction["server"] != nil {
			dsServer := []colonycore.Server{}
			cursor, err := colonycore.Find(new(colonycore.Server), dbox.Eq("_id", tk.ToString(flowAction["server"])))

			if err != nil {
				//
			} else {
				if cursor != nil {
					cursor.Fetch(&dsServer, 0, false)
					defer cursor.Close()
				}

				if len(dsServer) > 0 {
					server = dsServer[0]
				}
			}
		}

		inputParam := tk.M{}
		outputParam := tk.M{}
		outputType := ""

		for _, in := range flowAction["inputparam"].([]interface{}) {
			inputParam.Set("", in)
		}

		if flowAction["outputtype"] != nil {
			outputType = tk.ToString(flowAction["outputtype"])
		}

		for _, out := range flowAction["outputparam"].([]interface{}) {
			outputParam.Set("", out)
		}

		OK := []string{}
		KO := []string{}

		for _, str := range flowAction["OK"].([]interface{}) {
			OK = append(OK, tk.ToString(str))
		}

		for _, str := range flowAction["KO"].([]interface{}) {
			KO = append(KO, tk.ToString(str))
		}

		actionType := tk.ToString(flowAction["type"])

		flow := colonycore.FlowAction{
			Id:          tk.ToString(flowAction["id"]),
			Name:        tk.ToString(flowAction["name"]) + "-" + tk.ToString(flowAction["id"]),
			Description: tk.ToString(flowAction["description"]),
			Type:        actionType,
			Server:      server,
			OK:          OK,
			KO:          KO,
			Retry:       tk.ToInt(flowAction["Retry"], ""),
			Interval:    tk.ToInt(flowAction["Interval"], ""),
			FirstAction: flowAction["firstaction"].(bool),
			InputParam:  inputParam,
			OutputParam: outputParam,
			OutputType:  outputType,
			OutputPath:  "",
		}

		dataAction := flowAction["action"].(map[string]interface{})

		switch actionType {
		case dataflow.ACTION_TYPE_SPARK:
			spark := colonycore.ActionSpark{
				Master:    tk.ToString(dataAction["master"]),
				Mode:      tk.ToString(dataAction["mode"]),
				File:      tk.ToString(dataAction["appfiles"]),
				MainClass: tk.ToString(dataAction["mainclass"]),
				Args:      tk.ToString(dataAction["args"]),
			}
			flow.Action = spark
			break
		case dataflow.ACTION_TYPE_HIVE:
			hive := colonycore.ActionHive{
				ScriptPath: tk.ToString(dataAction),
				// Params:     ,
			}
			flow.Action = hive
			break
		case dataflow.ACTION_TYPE_SSH:
			ssh := colonycore.ActionSSH{
				Command: tk.ToString(dataAction["script"]),
			}
			flow.Action = ssh
			break
		case dataflow.ACTION_TYPE_HDFS:
			hdfs := colonycore.ActionHDFS{
				Command: tk.ToString(dataAction["script"]),
			}
			flow.Action = hdfs
			break
		case dataflow.ACTION_TYPE_JAVA:
			java := colonycore.ActionJavaApp{
				Jar: tk.ToString(dataAction["jar"]),
			}
			flow.Action = java
			break
		case dataflow.ACTION_TYPE_MAP_REDUCE:
			mr := colonycore.ActionHadoopStreaming{
				// Jar: ,
				Mapper:  tk.ToString(dataAction["mapper"]),
				Reducer: tk.ToString(dataAction["reducer"]),
				Input:   tk.ToString(dataAction["input"]),
				Output:  tk.ToString(dataAction["output"]),
				Params:  tk.ToString(dataAction["params"]),
			}
			flow.Action = mr
			break
		case dataflow.ACTION_TYPE_KAFKA:
			kafka := colonycore.ActionKafka{}
			flow.Action = kafka
			break
		case dataflow.ACTION_TYPE_EMAIL:
			email := colonycore.ActionEmail{
			// To:,
			//   CC:,
			//   Subject:,
			//   Body:,
			}
			flow.Action = email
			break
		// case "Fork":
		// 	fork := colonycore.ActionFork{}
		// 	action.Action = fork
		// 	action.Type = dataflow.ACTION_TYPE_FORK
		// 	break
		case dataflow.ACTION_TYPE_DECISION:
			conditions := []colonycore.Condition{}

			/*for _, val := range dataAction["conditions"].([]interface{}) {
				condition := val.(colonycore.Condition)
				conditions = append(conditions, condition)
			}*/

			decision := colonycore.ActionDecision{
				Conditions: conditions,
				// IsFork:     dataAction["isfork"].(bool),
			}
			flow.Action = decision
			break
		case dataflow.ACTION_TYPE_STOP:
			stop := colonycore.ActionStop{
				Message: tk.ToString(dataAction["message"]),
			}
			flow.Action = stop
			break
		}

		flows = append(flows, flow)
	}

	return
}
Пример #15
0
func GrabDocConfig(data toolkit.M) (*sdt.GrabService, error) {
	var e error
	var gi, ti time.Duration

	GrabService := sdt.NewGrabService()
	GrabService.Name = data["nameid"].(string) //"iopriceindices"
	GrabService.SourceType = sdt.SourceType_DocExcel

	grabintervalToInt := toolkit.ToInt(data["grabinterval"], toolkit.RoundingAuto)
	timeintervalToInt := toolkit.ToInt(data["timeoutinterval"], toolkit.RoundingAuto)
	if data["intervaltype"].(string) == "seconds" {
		gi = time.Duration(grabintervalToInt) * time.Second
		ti = time.Duration(timeintervalToInt) * time.Second
	} else if data["intervaltype"].(string) == "minutes" {
		gi = time.Duration(grabintervalToInt) * time.Minute
		ti = time.Duration(timeintervalToInt) * time.Minute
	} else if data["intervaltype"].(string) == "hours" {
		gi = time.Duration(grabintervalToInt) * time.Hour
		ti = time.Duration(timeintervalToInt) * time.Hour
	}
	GrabService.GrabInterval = gi
	GrabService.TimeOutInterval = ti //time.Hour, time.Minute, time.Second
	GrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", timeintervalToInt, data["intervaltype"])

	ci := dbox.ConnectionInfo{}

	grabDataConf, e := toolkit.ToM(data["grabconf"])
	if e != nil {
		return nil, e
	}
	isDoctype := grabDataConf.Has("doctype")
	if isDoctype {
		connToMap, e := toolkit.ToM(grabDataConf["connectioninfo"])
		if e != nil {
			return nil, e
		}
		ci.Host = connToMap["host"].(string) //"E:\\data\\sample\\IO Price Indices.xlsm"
		if hasSettings := connToMap.Has("settings"); !hasSettings {
			ci.Settings = nil
		} else {
			settingToMap, e := toolkit.ToM(connToMap["settings"])
			if e != nil {
				return nil, e
			}
			ci.Settings = settingToMap //toolkit.M{}.Set("useheader", settingToMap["useheader"].(bool)).Set("delimiter", settingToMap["delimiter"])
		}
		GrabService.ServGetData, e = sdt.NewGetDatabase(ci.Host, grabDataConf["doctype"].(string), &ci)
	}

	logconfToMap, e := toolkit.ToM(data["logconf"])
	if e != nil {
		return nil, e
	}
	logpath := logconfToMap["logpath"].(string)           //"E:\\data\\vale\\log"
	filename := logconfToMap["filename"].(string) + "-%s" //"LOG-LOCALXLSX-%s"
	filepattern := logconfToMap["filepattern"].(string)   //"20060102"

	logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
	if e != nil {
		return nil, e
	}

	GrabService.Log = logconf

	GrabService.ServGetData.CollectionSettings = make(map[string]*sdt.CollectionSetting)
	GrabService.DestDbox = make(map[string]*sdt.DestInfo)

	tempDataSetting := sdt.CollectionSetting{}
	tempDestInfo := sdt.DestInfo{}

	for _, dataSet := range data["datasettings"].([]interface{}) {
		dataToMap, e := toolkit.ToM(dataSet)
		if e != nil {
			return nil, e
		}
		tempDataSetting.Collection = dataToMap["rowselector"].(string) //"HIST"
		for _, columnSet := range dataToMap["columnsettings"].([]interface{}) {
			columnToMap, e := toolkit.ToM(columnSet)
			if e != nil {
				return nil, e
			}
			tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &sdt.GrabColumn{Alias: columnToMap["alias"].(string), Selector: columnToMap["selector"].(string)})
		}
		GrabService.ServGetData.CollectionSettings[dataToMap["name"].(string)] = &tempDataSetting //DATA01 use name in datasettings

		// fmt.Println("doctype>", grabDataConf["doctype"])
		connToMap, e := toolkit.ToM(dataToMap["connectioninfo"])
		if e != nil {
			return nil, e
		}
		var db, usr, pwd string

		if hasDb := connToMap.Has("database"); !hasDb {
			db = ""
		} else {
			db = connToMap["database"].(string)
		}
		if hasUser := connToMap.Has("username"); !hasUser {
			usr = ""
		} else {
			usr = connToMap["username"].(string)
		}

		if hasPwd := connToMap.Has("password"); !hasPwd {
			pwd = ""
		} else {
			pwd = connToMap["username"].(string)
		}
		ci = dbox.ConnectionInfo{}
		ci.Host = connToMap["host"].(string) //"localhost:27017"
		ci.Database = db                     //"valegrab"
		ci.UserName = usr                    //""
		ci.Password = pwd                    //""

		//tempDestInfo.Collection = "iopriceindices"
		if hasCollection := connToMap.Has("collection"); !hasCollection {
			tempDestInfo.Collection = ""
		} else {
			tempDestInfo.Collection = connToMap["collection"].(string)
		}
		tempDestInfo.Desttype = dataToMap["desttype"].(string) //"mongo"

		tempDestInfo.IConnection, e = dbox.NewConnection(tempDestInfo.Desttype, &ci)
		if e != nil {
			return nil, e
		}

		GrabService.DestDbox[dataToMap["name"].(string)] = &tempDestInfo
		//=History===========================================================
		GrabService.HistoryPath = HistoryPath       //"E:\\data\\vale\\history\\"
		GrabService.HistoryRecPath = HistoryRecPath //"E:\\data\\vale\\historyrec\\"
		//===================================================================
	}

	return GrabService, e
}
Пример #16
0
func fetchConfig() (err error) {

	switch toolkit.ToString(config.Get("sourcetype", "")) {
	case "SourceType_DocExcel":
		SourceType = SourceType_DocExcel
	case "SourceType_DocMongo":
		SourceType = SourceType_DocMongo
	default:
		err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
		return
	}

	Log.AddLog("Start fetch grabconf", "INFO")
	if !config.Has("grabconf") {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
		return
	}

	tconfgrab := toolkit.M{}
	tconfgrab, err = toolkit.ToM(config["grabconf"])
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
		return
	}

	if !tconfgrab.Has("doctype") {
		err = errors.New("Fetch Config, doctype not found")
		return
	}

	ci := dbox.ConnectionInfo{}
	mapconninfo := toolkit.M{}

	mapconninfo, err = toolkit.ToM(tconfgrab.Get("connectioninfo", nil))
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo found error : %v", err.Error()))
		return
	}

	ci.Host = toolkit.ToString(mapconninfo.Get("host", ""))
	ci.Database = toolkit.ToString(mapconninfo.Get("database", ""))
	ci.UserName = toolkit.ToString(mapconninfo.Get("userName", ""))
	ci.Password = toolkit.ToString(mapconninfo.Get("password", ""))
	ci.Settings, err = toolkit.ToM(mapconninfo.Get("settings", nil))
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo.settings found error : %v", err.Error()))
		return
	}

	sGrabber, err = sedotan.NewGetDatabase(ci.Host, toolkit.ToString(tconfgrab.Get("doctype", "")), &ci)
	if err != nil {
		err = errors.New(fmt.Sprintf("Fetch Config, create new get database found error : %v", err.Error()))
	}

	Log.AddLog("Start fetch datasettings", "INFO")
	if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	sGrabber.CollectionSettings = make(map[string]*sedotan.CollectionSetting)
	destDboxs = make(map[string]*DestInfo)

	for i, xVal := range config["datasettings"].([]interface{}) {
		err = nil
		tCollectionSetting := sedotan.CollectionSetting{}
		tDestDbox := DestInfo{}

		mVal := toolkit.M{}
		mVal, err = toolkit.ToM(xVal)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
			continue
		}

		tnameid := toolkit.ToString(mVal.Get("nameid", ""))
		if tnameid == "" {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
			continue
		}
		tCollectionSetting.Collection = toolkit.ToString(mVal.Get("collection", ""))

		// Fetch mapssettings
		if !mVal.Has("mapssettings") || !(toolkit.TypeName(mVal["mapssettings"]) == "[]interface {}") {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : mapssettings is not found or incorrect", i, tnameid), "ERROR")
			continue
		}

		tCollectionSetting.MapsColumns = make([]*sedotan.MapColumn, 0, 0)
		for xi, Valcs := range mVal["mapssettings"].([]interface{}) {
			mValcs := toolkit.M{}
			mValcs, err = toolkit.ToM(Valcs)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : mapssettings is not found or incorrect", i, tnameid, xi), "ERROR")
				continue
			}

			tgrabcolumn := sedotan.MapColumn{}

			tgrabcolumn.Source = toolkit.ToString(mValcs.Get("source", ""))
			tgrabcolumn.SType = toolkit.ToString(mValcs.Get("sourcetype", ""))
			tgrabcolumn.Destination = toolkit.ToString(mValcs.Get("destination", ""))
			tgrabcolumn.DType = toolkit.ToString(mValcs.Get("destinationtype", ""))

			// tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
			tCollectionSetting.MapsColumns = append(tCollectionSetting.MapsColumns, &tgrabcolumn)
		}

		//Fetch Filter Condition
		if mVal.Has("filtercond") {
			tfiltercond := toolkit.M{}
			tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
			if err != nil {
				Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
			} else {
				tCollectionSetting.SetFilterCond(tfiltercond)
			}
		}

		//Fetch limit data
		if mVal.Has("limitrow") {
			tLimitrow, err := toolkit.ToM(mVal["limitrow"])
			if err == nil {
				tCollectionSetting.Take = toolkit.ToInt(tLimitrow.Get("take", 0), toolkit.RoundingAuto)
				tCollectionSetting.Skip = toolkit.ToInt(tLimitrow.Get("skip", 0), toolkit.RoundingAuto)
			}
		}

		//Fetch Connection Info
		tConnInfo := toolkit.M{}
		tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
		tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))

		tHost := toolkit.ToString(tConnInfo.Get("host", ""))
		tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
		tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
		tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
		tSettings := toolkit.M{}
		tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}

		tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
			continue
		}
		tDestDbox.IConnection.Close()

		destDboxs[tnameid] = &tDestDbox
		sGrabber.CollectionSettings[tnameid] = &tCollectionSetting

	}
	err = nil

	if len(destDboxs) == 0 || len(sGrabber.CollectionSettings) == 0 {
		err = errors.New("Fetch Config, datasettings is not found or have wrong format")
		return
	}

	if !config.Has("histconf") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	Log.AddLog("Start fetch histconf", "INFO")
	histConf, err = toolkit.ToM(config.Get("histconf", nil))
	if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
		err = errors.New("Fetch Config, history configuration is not found or have wrong format")
		return
	}

	if !config.Has("extcommand") {
		return
	}

	Log.AddLog("Start fetch extcommand", "INFO")
	extCommand, _ = toolkit.ToM(config.Get("extcommand", nil))

	return
}
Пример #17
0
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
	var hasLookup bool
	toolkit.Println("payload : ", payload)
	if payload.Has("haslookup") {
		hasLookup = payload.Get("haslookup").(bool)
	}
	_id := toolkit.ToString(payload.Get("browserid", ""))
	sort := payload.Get("sort")
	search := payload.Get("search")
	_ = search
	take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)

	TblName := toolkit.M{}
	payload.Unset("browserid")
	//sorter = ""
	if sort != nil {
		tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
		fmt.Printf("====== sort %#v\n", tmsort["dir"])
		if tmsort["dir"] == "asc" {
			sorter = tmsort["field"].(string)
		} else if tmsort["dir"] == "desc" {
			sorter = "-" + tmsort["field"].(string)
		} else if tmsort["dir"] == nil {
			sorter = " "
		}
	} else {
		sorter = " "
	}

	dataDS := new(colonycore.DataBrowser)
	err := colonycore.Get(dataDS, _id)
	if err != nil {
		return 0, nil, nil, err
	}

	dataConn := new(colonycore.Connection)
	err = colonycore.Get(dataConn, dataDS.ConnectionID)
	if err != nil {
		return 0, nil, nil, err
	}

	if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
		return 0, nil, nil, err
	}

	connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
	if err != nil {
		return 0, nil, nil, err
	}

	if dataDS.QueryType == "" {
		TblName.Set("from", dataDS.TableNames)
		payload.Set("from", dataDS.TableNames)
	} else if dataDS.QueryType == "Dbox" {
		getTableName := toolkit.M{}
		toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
		payload.Set("from", getTableName.Get("from").(string))

		if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
			payload.Set("select", getTableName.Get("select").(string))
		}
	} else if dataDS.QueryType == "SQL" {
		var QueryString string
		if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
			QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
		} else if dataConn.Driver == "mssql" {
			QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
				toolkit.ToString(take) + " ROWS ONLY "

		} else if dataConn.Driver == "postgres" {
			QueryString = " LIMIT " + toolkit.ToString(take) +
				" OFFSET " + toolkit.ToString(skip)
		}
		stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
		payload.Set("freetext", stringQuery)
		// toolkit.Println(stringQuery)
	}

	qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
	query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)

	var selectfield string
	for _, metadata := range dataDS.MetaData {
		tField := metadata.Field
		if payload.Has(tField) {
			selectfield = toolkit.ToString(tField)
			if toolkit.IsSlice(payload[tField]) {
				query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
				qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
			} else if !toolkit.IsNilOrEmpty(payload[tField]) {
				var hasPattern bool
				for _, val := range querypattern {
					if strings.Contains(toolkit.ToString(payload[tField]), val) {
						hasPattern = true
					}
				}
				if hasPattern {
					query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
						toolkit.ToString(metadata.DataType), ""))
					qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
						toolkit.ToString(metadata.DataType), ""))
				} else {
					switch toolkit.ToString(metadata.DataType) {
					case "int":
						query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
					case "float32":
						query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
					case "float64":
						query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
					default:
						query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
						qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
					}
				}
			}
		}
	}

	if hasLookup && selectfield != "" {
		if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
			query = query.Select(selectfield)
			qcount = qcount.Select(selectfield)
		} else {
			query = query.Select(selectfield).Group(selectfield)
			qcount = qcount.Select(selectfield).Group(selectfield)
		}

	}

	ccount, err := qcount.Cursor(nil)
	if err != nil {
		return 0, nil, nil, err
	}
	defer ccount.Close()

	dcount := ccount.Count()

	cursor, err := query.Cursor(nil)
	if err != nil {
		return 0, nil, nil, err
	}
	defer cursor.Close()

	data := []toolkit.M{}
	cursor.Fetch(&data, 0, false)

	if err != nil {
		return 0, nil, nil, err
	}

	if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
		!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
		dataMongo := []toolkit.M{}
		for _, val := range data {
			mVal, _ := toolkit.ToM(val.Get("_id"))
			dataMongo = append(dataMongo, mVal)
		}
		data = dataMongo
	} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
		/*distinct value for flat file*/
		dataFlat := []toolkit.M{}
		var existingVal = []string{""}
		for _, val := range data {
			valString := toolkit.ToString(val.Get(selectfield))
			if !toolkit.HasMember(existingVal, valString) {
				dataFlat = append(dataFlat, val)
				existingVal = append(existingVal, valString)
			}
		}
		data = dataFlat
	}

	return dcount, data, dataDS, nil
}
Пример #18
0
func (b *BaseController) ConvertMGOToSQLServer(m orm.IModel) error {
	tStart := time.Now()

	tk.Printf("\nConvertMGOToSQLServer: Converting %v \n", m.TableName())
	tk.Println("ConvertMGOToSQLServer: Starting to convert...\n")

	c, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Cursor(nil)

	if e != nil {
		return e
	}

	defer c.Close()

	totalData := c.Count()
	processIter := tk.ToInt(tk.ToFloat64(totalData/maxDataEachProcess, 0, tk.RoundingUp), tk.RoundingUp)

	if maxDataEachProcess == 0 {
		processIter = 0
	}

	for iter := 0; iter < processIter+1; iter++ {

		skip := iter * maxDataEachProcess
		take := maxDataEachProcess

		if maxDataEachProcess == 0 {
			take = totalData
		} else if iter == processIter {
			take = totalData - skip
		}

		csr, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Skip(skip).Take(take).Cursor(nil)

		if e != nil {
			return e
		}

		result := []tk.M{}
		e = csr.Fetch(&result, 0, false)
		csr.Close()

		if e != nil {
			return e
		}

		dtLen := len(result)

		resPart := make([][]tk.M, worker)

		if dtLen < worker {
			resPart = make([][]tk.M, 1)
			resPart[0] = result
		} else {
			workerTaskCount := tk.ToInt(tk.ToFloat64(dtLen/worker, 0, tk.RoundingAuto), tk.RoundingAuto)
			count := 0

			for i := 0; i < worker; i++ {
				if i == worker-1 {
					resPart[i] = result[count:]
				} else {
					resPart[i] = result[count : count+workerTaskCount]
				}
				count += workerTaskCount
			}
		}

		wg := &sync.WaitGroup{}
		wg.Add(len(resPart))

		for _, val := range resPart {
			// go b.Insert(val, m, wg)
			go b.InsertBulk(val, m, wg, tStart)
		}

		wg.Wait()
	}

	tk.Println("\nConvertMGOToSQLServer: Finish.")
	tk.Printf("Completed Success in %v \n", time.Since(tStart))
	return nil
}
Пример #19
0
func savedatagrab() (err error) {
	var wg, wgstream sync.WaitGroup
	// if len(sGrabber.CollectionSettings) > 0 {
	// 	wg.Add(len(sGrabber.CollectionSettings))
	// }

	mapRecHistory = make(map[string]string, 0)
	historyFileName = fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))

	for key, _ := range sGrabber.CollectionSettings {
		//set history name
		mapRecHistory[key] = fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
		//================
		err = nil
		note := ""
		dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
			Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", mapRecHistory[key])

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data source", key), "INFO")
		iQ, err := sGrabber.GetQuery(key)
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get query data : %s", key, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		defer sGrabber.CloseConn()
		csr, err := iQ.Cursor(nil)
		if err != nil || csr == nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to create cursor or cursor nil to get data : %s", key, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data souce done", key), "INFO")
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save", key), "INFO")
		err = destDboxs[key].IConnection.Connect()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
		if destDboxs[key].collection != "" {
			q = q.From(destDboxs[key].collection)
		}
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save done", key), "INFO")

		//Update Total Process
		mutex.Lock()
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
		err = getsnapshot()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}

		if pid == snapshotdata.Pid {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, csr.Count()), "INFO")
			snapshotdata.Cgtotal += csr.Count()
			err = savesnapshot()
		}

		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}
		mutex.Unlock()

		outtm := make(chan toolkit.M)
		wgstream.Add(1)
		go func(intm chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
			streamsavedata(intm, sQ, key, dt)
			wgstream.Done()
		}(outtm, q, key, dt)

		wg.Add(1)
		go func(outtm chan toolkit.M, csr dbox.ICursor) {
			condition := true

			for condition {
				results := make([]toolkit.M, 0)
				err = csr.Fetch(&results, 1000, false)
				if err != nil {
					note = fmt.Sprintf("[savedatagrab.%s] Unable to fetch data :%s", key, err.Error())
					Log.AddLog(note, "ERROR")
					condition = false
					continue
				}

				for _, result := range results {
					xresult := toolkit.M{}
					for _, column := range sGrabber.CollectionSettings[key].MapsColumns {
						var val, tval interface{}
						strsplits := strings.Split(column.Source, "|")

						ttm := toolkit.M{}
						lskey := ""
						for i, strsplit := range strsplits {
							lskey = strsplit
							if i == 0 && result.Has(strsplit) {
								ttm.Set(strsplit, result[strsplit])
							} else if ttm.Has(strsplit) {
								ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
							}
						}

						strsplits = strings.Split(column.Destination, "|")
						if ttm.Has(lskey) {
							tval = ttm[lskey]
						} else {
							tval = sedotan.DefaultValue(column.DType)
						}

						// if len(strsplits) > 1 {
						// 	if xresult.Has(strsplits[0]) {
						// 		val = xre
						// 	} else {

						// 	}
						// }

						tm := toolkit.M{}
						if xresult.Has(strsplits[0]) {
							tm, _ = toolkit.ToM(xresult[strsplits[0]])
						}
						// xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
						switch column.DType {
						case "int":
							tval = toolkit.ToInt(tval, toolkit.RoundingAuto)
						case "float":
							tval = toolkit.ToFloat64(tval, 2, toolkit.RoundingAuto)
						case "string":
							tval = toolkit.ToString(tval)
						default:
							tval = tval
						}

						val = getresultobj(strsplits, tval, tm)

						// if len(strsplits) == 1 {
						// 	val = tval
						// } else {
						// 	val = getresultobj(strsplits[1:], tval)
						// }

						// if len(strsplits) > 1 && {

						// }
						xresult.Set(strsplits[0], val)

						// for i, strsplit := range strsplits {
						// 	if i == 0 && ttm.Has(lskey) {
						// 		xresult.Set(strsplit, ttm[lskey])
						// 	} else if (i + 1) == len(strsplits) {

						// 	}
						// 	// else if ttm.Has(strsplit) {
						// 	// 	ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
						// 	// }
						// }

					}
					outtm <- xresult
				}

				if len(results) < 1000 || err != nil {
					condition = false
				}
			}
			//
			//

			// ms := []toolkit.M{}
			// for _, val := range results {
			// 	m := toolkit.M{}
			// 	for _, column := range g.CollectionSettings[dataSettingId].MapsColumns {
			// 		m.Set(column.Source, "")
			// 		if val.Has(column.Destination) {
			// 			m.Set(column.Source, val[column.Destination])
			// 		}
			// 	}
			// 	ms = append(ms, m)
			// }

			close(outtm)
			wg.Done()
		}(outtm, csr)

	}

	wgstream.Wait()
	wg.Wait()

	return
}
Пример #20
0
func streamsavedata(intms <-chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
	var err error
	iN, note := 0, ""

	for intm := range intms {
		if destDboxs[key].desttype == "mongo" {
			intm.Set("_id", toolkit.GenerateRandomString("", 32))
		}

		if len(intm) == 0 {
			continue
		}
		//Pre Execute Program
		if extCommand.Has("pre") && toolkit.ToString(extCommand["pre"]) != "" {
			sintm := toolkit.JsonString(intm)
			arrcmd := make([]string, 0, 0)

			// if runtime.GOOS == "windows" {
			// 	arrcmd = append(arrcmd, "cmd")
			// 	arrcmd = append(arrcmd, "/C")
			// }

			arrcmd = append(arrcmd, toolkit.ToString(extCommand["pre"]))
			arrcmd = append(arrcmd, sintm)

			// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
			output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute pre external command :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}

			err = toolkit.UnjsonFromString(output, &intm)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get pre external command output :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}
		}

		err = sQ.Exec(toolkit.M{
			"data": intm,
		})

		if err != nil {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
			note = "Error Found"
			continue
		}

		err = saverechistory(key, intm)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert record data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
			note = "Error Found"
		}

		iN += 1
		if math.Mod(float64(iN), 100) == 0 {
			_ = updatesnapshot(iN, key)
			dt = dt.Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
			iN = 0
		}

		//Post Execute Program
		if extCommand.Has("post") {
			sintm := toolkit.JsonString(intm)
			arrcmd := make([]string, 0, 0)

			// if runtime.GOOS == "windows" {
			// 	arrcmd = append(arrcmd, "cmd")
			// 	arrcmd = append(arrcmd, "/C")
			// }

			arrcmd = append(arrcmd, toolkit.ToString(extCommand["post"]))
			arrcmd = append(arrcmd, sintm)

			// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
			output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute post external command :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}

			err = toolkit.UnjsonFromString(output, &intm)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get post external command output :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}
		}
	}
	dt = dt.Set("note", note).
		Set("grabstatus", "done").
		Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
	_ = updatesnapshot(iN, key)
	err = savehistory(dt)
	if err != nil {
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save history : %s", key), "ERROR")
	}
	Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Finish save data", key), "INFO")
	destDboxs[key].IConnection.Close()
}
Пример #21
0
func doGenerateMaintenanceDataBrowser(wg *sync.WaitGroup, d *GenDataBrowser, plant PowerPlantCoordinates) {
	var e error
	ctx := d.BaseController.Ctx
	c := ctx.Connection

	saveDatas := []orm.IModel{}
	plantCodeStr := plant.PlantCode
	dataCount := 0

	tmpPlantCondition := conditions.Get(plantCodeStr)
	if tmpPlantCondition != nil {
		plantCondition := tmpPlantCondition.(tk.M)

		length := plantCondition.GetInt("length")
		dets := plantCondition.Get("det").([]tk.M)
		dets = append(dets, tk.M{})

		turbinesCodes := []string{}
		genIDTempTable := tk.GenerateRandomString("", 20)

		detsLen := len(dets) - 1

		tk.Printf("detsLen: %v \n", detsLen)

		for i, det := range dets {
			freeQuery := false
			tk.Println(i)
			tk.Printf("dets: %#v \n", det)
			assets := []FunctionalLocation{}
			systemAssets := []FunctionalLocation{}
			desc := det.GetString("desc")

			query := []*dbox.Filter{}
			query = append(query, dbox.Contains("FunctionalLocationCode", plantCodeStr))
			query = append(query, dbox.Eq("PIPI", plantCodeStr))

			queryStr := "select * from FunctionalLocation where FunctionalLocationCode like('%" + plantCodeStr + "%') and PIPI = '" + plantCodeStr + "' "

			if i == 0 {
				query = append(query, dbox.Contains("Description", desc))
				if plantCodeStr == "2220" {
					query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
				} else {
					query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
				}
			} else if i == 1 && detsLen == 2 {
				query = append(query, dbox.Contains("Description", desc))
				if plantCodeStr == "2220" {
					query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
				} else {
					query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
				}
			} else {
				tk.Printf("turbinesCodes: %v \n", len(turbinesCodes))
				if len(turbinesCodes) > 1 {
					maxLength := 1000
					maxLoop := tk.ToInt(tk.ToFloat64(len(turbinesCodes)/maxLength, 0, tk.RoundingUp), tk.RoundingUp)

					for i := 0; i <= maxLoop; i++ {
						datas := []string{}

						if i != maxLoop {
							datas = turbinesCodes[i*maxLength : (i*maxLength)+maxLength]

						} else {
							datas = turbinesCodes[i*maxLength:]
						}

						tmpNotIn := []orm.IModel{}
						for _, val := range datas {
							tmpData := new(GenDataBrowserNotInTmp)
							tmpData.ID = genIDTempTable
							tmpData.FLCode = val
							tmpNotIn = append(tmpNotIn, tmpData)
						}
						// tk.Printf("tmpNotIn: %v \n", len(tmpNotIn))
						e = ctx.InsertBulk(tmpNotIn)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
					}

					queryStr = queryStr + " and FunctionalLocationCode not in(select flcode from GenDataBrowserNotInTmp where ID = '" + genIDTempTable + "')"
					freeQuery = true
				}
			}

			if freeQuery {
				csrDet, e := c.NewQuery().Command("freequery", tk.M{}.Set("syntax", queryStr)).Cursor(nil)
				ErrorHandler(e, "generateMaintenanceDataBrowser")

				e = csrDet.Fetch(&assets, 0, false)
				ErrorHandler(e, "generateMaintenanceDataBrowser")
				csrDet.Close()
			} else {
				csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(dbox.And(query...)).Cursor(nil)
				ErrorHandler(e, "generateMaintenanceDataBrowser")

				e = csrDet.Fetch(&assets, 0, false)
				ErrorHandler(e, "generateMaintenanceDataBrowser")
				csrDet.Close()
			}

			tk.Printf("-- assets: %v \n", len(assets))

			if len(assets) > 0 {
				relatedAssets := []FunctionalLocation{}

				for _, asset := range assets {

					if plantCodeStr == "2110" {
						if len(asset.FunctionalLocationCode) <= 13 {
							query = []*dbox.Filter{}
							query = append(query,
								dbox.And(
									dbox.Eq("PG", "MP1"),
									dbox.Eq("PIPI", plantCodeStr),
									dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode),
									dbox.And(
										dbox.Lte("LEN(FunctionalLocationCode)", 13),
										dbox.Gte("LEN(FunctionalLocationCode)", 12),
									),
								),
							)

							csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							e = csrDet.Fetch(&systemAssets, 0, false)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							csrDet.Close()
						}

						if systemAssets == nil || len(systemAssets) == 0 {
							query = []*dbox.Filter{}

							if i != 2 {
								query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
							} else {
								query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
							}

							csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							e = csrDet.Fetch(&relatedAssets, 0, false)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							csrDet.Close()

							for _, relasset := range relatedAssets {
								isTurbineSystem := false
								if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
									isTurbineSystem = true
								}

								newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

								if newEquipment != "" {

									if i != 2 {
										turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
									}

									for _, year := range years {
										data := DataBrowser{}
										data.PeriodYear = year
										data.FunctionalLocation = relasset.FunctionalLocationCode
										data.FLDescription = relasset.Description
										data.IsTurbine = false

										// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)

										if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
											data.IsTurbine = true
											// tk.Println(" isTurbine: TRUE")
										} else {
											// tk.Println(" isTurbine: FALSE")
											data.TurbineParent = asset.FunctionalLocationCode
										}

										data.AssetType = "Other"

										if i == 0 {
											data.AssetType = "Steam"
										} else if i == 1 {
											data.AssetType = "Gas"
										}

										data.EquipmentType = newEquipment
										data.EquipmentTypeDescription = newEquipment
										data.Plant = plant
										data.PlantCode = plant.PlantCode

										if data.IsTurbine {
											info := GeneralInfo{}
											substr := ""
											substrValInt := 0
											if data.AssetType == "Steam" {
												substrValInt = 1
												substr = "ST"
											} else if data.AssetType == "Gas" {
												substrValInt = 2
											}

											if substrValInt != 0 {
												tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfo)
													substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
													return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
												}).Exec().Result.Data().([]GeneralInfo)
												if len(tmpInfo) > 0 {
													info = tmpInfo[0]
												}

												if info.Id != "" {
													data.TInfShortName = info.Unit
													data.TInfManufacturer = info.Manufacturer
													data.TInfModel = info.Model
													data.TInfUnitType = info.UnitType
													data.TInfInstalledCapacity = info.InstalledCapacity
													data.TInfOperationalCapacity = info.OperationalCapacity
													data.TInfPrimaryFuel = info.PrimaryFuel1
													data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
													data.TInfBackupFuel = info.BackupFuel
													data.TInfHeatRate = info.HeatRate
													data.TInfEfficiency = info.Efficiency

													commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
													ErrorHandler(e, "generateMaintenanceDataBrowser")
													data.TInfCommisioningDate = commDate

													if info.RetirementPlan != "" {
														retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
														retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
														ErrorHandler(e, "generateMaintenanceDataBrowser")
														data.TInfRetirementPlan = retirementPlan
													}

													installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfInstalledMWH = installedMWH.Value

													actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

													capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfCapacityFactor = capacityFactor.Value

													actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoActualFuelConsumption)
														return y.GenID == info.Id && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

													data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
													data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
													data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
													data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

													fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
														y := x.(FuelCost)
														unitID := strings.Replace(
															strings.Replace(
																strings.Replace(
																	strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

														return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
													})

													data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
														y := x.(FuelCost)
														return y.EnergyNetProduction
													}).Exec().Result.Data().(float64)

													data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
														y := x.(FuelCost)
														return y.PrimaryFuelConsumed
													}).Exec().Result.Data().(float64)
												}
											}
											// Vibrations handled by sql query
										}

										// Maintenance handled by sql query
										// FailureNotifications handled by sql query
										// MROElements handled by sql query
										// Operationals handled by sql query

										// e = ctx.Insert(&data)
										saveDatas = append(saveDatas, &data)
										dataCount++
										if len(saveDatas) == 1000 {
											mugen.Lock()
											e = ctx.InsertBulk(saveDatas)
											mugen.Unlock()
											ErrorHandler(e, "generateMaintenanceDataBrowser")
											tk.Printf("%v : %v \n", plantCodeStr, dataCount)

											saveDatas = []orm.IModel{}
										}

										/*ErrorHandler(e, "generateMaintenanceDataBrowser")
										tk.Println("save 1")*/
									}
								}
							}

						} else {
							// with system assets
							if i != 2 {
								turbinesCodes = append(turbinesCodes, asset.FunctionalLocationCode)
							}

							for _, year := range years {
								data := DataBrowser{}
								data.PeriodYear = year
								data.FunctionalLocation = asset.FunctionalLocationCode
								data.FLDescription = asset.Description
								data.IsTurbine = false
								data.IsSystem = false

								// tk.Printf("%v | %v | ", asset.FunctionalLocationCode, i)

								if asset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
									data.IsTurbine = true
									data.IsSystem = true
									// tk.Println(" isTurbine: TRUE")
								} else {
									// tk.Println(" isTurbine: FALSE")
									data.TurbineParent = asset.FunctionalLocationCode
									data.SystemParent = asset.FunctionalLocationCode
								}

								data.AssetType = "Other"

								if i == 0 {
									data.AssetType = "Steam"
								} else if i == 1 {
									data.AssetType = "Gas"
								}

								data.EquipmentType = "System"
								data.EquipmentTypeDescription = "System"
								data.Plant = plant
								data.PlantCode = plant.PlantCode

								if data.IsTurbine {
									info := GeneralInfo{}
									substr := ""
									substrValInt := 0
									if data.AssetType == "Steam" {
										substrValInt = 1
										substr = "ST"
									} else if data.AssetType == "Gas" {
										substrValInt = 2
									}

									if substrValInt != 0 {
										tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
											y := x.(GeneralInfo)
											substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
											return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
										}).Exec().Result.Data().([]GeneralInfo)
										if len(tmpInfo) > 0 {
											info = tmpInfo[0]
										}

										if info.Id != "" {
											data.TInfShortName = info.Unit
											data.TInfManufacturer = info.Manufacturer
											data.TInfModel = info.Model
											data.TInfUnitType = info.UnitType
											data.TInfInstalledCapacity = info.InstalledCapacity
											data.TInfOperationalCapacity = info.OperationalCapacity
											data.TInfPrimaryFuel = info.PrimaryFuel1
											data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
											data.TInfBackupFuel = info.BackupFuel
											data.TInfHeatRate = info.HeatRate
											data.TInfEfficiency = info.Efficiency

											commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
											ErrorHandler(e, "generateMaintenanceDataBrowser")
											data.TInfCommisioningDate = commDate

											if info.RetirementPlan != "" {
												retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
												retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
												ErrorHandler(e, "generateMaintenanceDataBrowser")
												data.TInfRetirementPlan = retirementPlan
											}

											installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfInstalledMWH = installedMWH.Value

											actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

											capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfCapacityFactor = capacityFactor.Value

											actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoActualFuelConsumption)
												return y.GenID == info.Id && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

											data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
											data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
											data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
											data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

											fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
												y := x.(FuelCost)
												unitID := strings.Replace(
													strings.Replace(
														strings.Replace(
															strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

												return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
											})

											data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
												y := x.(FuelCost)
												return y.EnergyNetProduction
											}).Exec().Result.Data().(float64)

											data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
												y := x.(FuelCost)
												return y.PrimaryFuelConsumed
											}).Exec().Result.Data().(float64)
										}
									}
									// Vibrations handled by sql query
								}

								// Maintenance handled by sql query
								// FailureNotifications handled by sql query
								// MROElements handled by sql query
								// Operationals handled by sql query

								// e = ctx.Insert(&data)
								// dataCount++
								saveDatas = append(saveDatas, &data)
								dataCount++
								if len(saveDatas) == 1000 {
									mugen.Lock()
									e = ctx.InsertBulk(saveDatas)
									mugen.Unlock()
									ErrorHandler(e, "generateMaintenanceDataBrowser")
									tk.Printf("%v : %v \n", plantCodeStr, dataCount)

									saveDatas = []orm.IModel{}
								}

								/*ErrorHandler(e, "generateMaintenanceDataBrowser")
								tk.Println("save 2")*/

							}

							for _, sysAsset := range systemAssets {
								query = []*dbox.Filter{}

								if i != 2 {
									query = append(query, dbox.Contains("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
								} else {
									query = append(query, dbox.Eq("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
								}

								csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
								ErrorHandler(e, "generateMaintenanceDataBrowser")
								e = csrDet.Fetch(&relatedAssets, 0, false)
								ErrorHandler(e, "generateMaintenanceDataBrowser")
								csrDet.Close()

								for _, relasset := range relatedAssets {
									isTurbineSystem := false
									if (relasset.FunctionalLocationCode == asset.FunctionalLocationCode || relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode) && i != 2 {
										isTurbineSystem = true
									}

									newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

									if newEquipment != "" {

										if i != 2 {
											turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
										}

										for _, year := range years {
											data := DataBrowser{}
											data.PeriodYear = year
											data.FunctionalLocation = relasset.FunctionalLocationCode
											data.FLDescription = relasset.Description
											data.IsTurbine = false
											data.IsSystem = false

											// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)

											if relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode && i != 2 {
												data.IsTurbine = true
												data.IsSystem = true
												// tk.Println(" isTurbine: TRUE")
											} else {
												// tk.Println(" isTurbine: FALSE")
												data.TurbineParent = asset.FunctionalLocationCode
												data.SystemParent = asset.FunctionalLocationCode
											}

											data.AssetType = "Other"

											if i == 0 {
												data.AssetType = "Steam"
											} else if i == 1 {
												data.AssetType = "Gas"
											}

											data.EquipmentType = newEquipment
											data.EquipmentTypeDescription = newEquipment
											data.Plant = plant
											data.PlantCode = plant.PlantCode

											if data.IsTurbine {
												info := GeneralInfo{}
												substr := ""
												substrValInt := 0
												if data.AssetType == "Steam" {
													substrValInt = 1
													substr = "ST"
												} else if data.AssetType == "Gas" {
													substrValInt = 2
												}

												if substrValInt != 0 {
													tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfo)
														substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
														return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
													}).Exec().Result.Data().([]GeneralInfo)
													if len(tmpInfo) > 0 {
														info = tmpInfo[0]
													}

													if info.Id != "" {
														data.TInfShortName = info.Unit
														data.TInfManufacturer = info.Manufacturer
														data.TInfModel = info.Model
														data.TInfUnitType = info.UnitType
														data.TInfInstalledCapacity = info.InstalledCapacity
														data.TInfOperationalCapacity = info.OperationalCapacity
														data.TInfPrimaryFuel = info.PrimaryFuel1
														data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
														data.TInfBackupFuel = info.BackupFuel
														data.TInfHeatRate = info.HeatRate
														data.TInfEfficiency = info.Efficiency

														commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
														ErrorHandler(e, "generateMaintenanceDataBrowser")
														data.TInfCommisioningDate = commDate

														if info.RetirementPlan != "" {
															retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
															retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
															ErrorHandler(e, "generateMaintenanceDataBrowser")
															data.TInfRetirementPlan = retirementPlan
														}

														installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfInstalledMWH = installedMWH.Value

														actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

														capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfCapacityFactor = capacityFactor.Value

														actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoActualFuelConsumption)
															return y.GenID == info.Id && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

														data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
														data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
														data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
														data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

														fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
															y := x.(FuelCost)
															unitID := strings.Replace(
																strings.Replace(
																	strings.Replace(
																		strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

															return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
														})

														data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
															y := x.(FuelCost)
															return y.EnergyNetProduction
														}).Exec().Result.Data().(float64)

														data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
															y := x.(FuelCost)
															return y.PrimaryFuelConsumed
														}).Exec().Result.Data().(float64)
													}
												}
												// Vibrations handled by sql query
											}

											// Maintenance handled by sql query
											// FailureNotifications handled by sql query
											// MROElements handled by sql query
											// Operationals handled by sql query

											// e = ctx.Insert(&data)
											saveDatas = append(saveDatas, &data)
											dataCount++
											if len(saveDatas) == 1000 {
												mugen.Lock()
												e = ctx.InsertBulk(saveDatas)
												mugen.Unlock()
												ErrorHandler(e, "generateMaintenanceDataBrowser")
												tk.Printf("%v : %v \n", plantCodeStr, dataCount)

												saveDatas = []orm.IModel{}
											}

											/*ErrorHandler(e, "generateMaintenanceDataBrowser")
											tk.Println("save 3")*/
										}
									}
								}
							}
						}
					} else {
						// another plant(s)

						query = []*dbox.Filter{}

						if i != detsLen {
							query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
						} else {
							query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
						}

						csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
						e = csrDet.Fetch(&relatedAssets, 0, false)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
						csrDet.Close()
						tk.Printf("-- related assets: %v \n", len(relatedAssets))

						for _, relasset := range relatedAssets {
							isTurbineSystem := false
							if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
								isTurbineSystem = true
							}

							newEquipment := ""
							newEquipment = d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

							if newEquipment != "" {

								if i != detsLen {
									turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
								}

								for _, year := range years {
									_ = year
									data := DataBrowser{}
									data.PeriodYear = year
									data.FunctionalLocation = relasset.FunctionalLocationCode
									data.FLDescription = relasset.Description
									data.IsTurbine = false

									if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
										data.IsTurbine = true
									} else {
										data.TurbineParent = asset.FunctionalLocationCode
									}

									data.AssetType = "Other"

									if i == 0 {
										data.AssetType = "Steam"
									} else if i == 1 && detsLen > 1 {
										data.AssetType = "Gas"
									}

									data.EquipmentType = newEquipment
									data.EquipmentTypeDescription = newEquipment
									data.Plant = plant
									data.PlantCode = plant.PlantCode

									if data.IsTurbine {
										info := GeneralInfo{}
										substr := ""
										substrValInt := 0
										if data.AssetType == "Steam" {
											// substrValInt = 1
											substrValInt = 2
											substr = "ST" + substr
										} else if data.AssetType == "Gas" {
											substrValInt = 2
											substr = "GT" + substr
										}

										if substrValInt != 0 {
											tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfo)
												substr = data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
												return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
											}).Exec().Result.Data().([]GeneralInfo)
											if len(tmpInfo) > 0 {
												info = tmpInfo[0]
											}

											if info.Id != "" {
												data.TInfShortName = info.Unit
												data.TInfManufacturer = info.Manufacturer
												data.TInfModel = info.Model
												data.TInfUnitType = info.UnitType
												data.TInfInstalledCapacity = info.InstalledCapacity
												data.TInfOperationalCapacity = info.OperationalCapacity
												data.TInfPrimaryFuel = info.PrimaryFuel1
												data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
												data.TInfBackupFuel = info.BackupFuel
												data.TInfHeatRate = info.HeatRate
												data.TInfEfficiency = info.Efficiency

												commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
												ErrorHandler(e, "generateMaintenanceDataBrowser")

												data.TInfCommisioningDate = commDate

												if info.RetirementPlan != "" {
													retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
													retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
													ErrorHandler(e, "generateMaintenanceDataBrowser")
													data.TInfRetirementPlan = retirementPlan
												}

												installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfInstalledMWH = installedMWH.Value

												actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

												capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfCapacityFactor = capacityFactor.Value

												actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoActualFuelConsumption)
													return y.GenID == info.Id && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

												data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
												data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
												data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
												data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

												fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
													y := x.(FuelCost)
													unitID := strings.Replace(
														strings.Replace(
															strings.Replace(
																strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

													return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
												})

												data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
													y := x.(FuelCost)
													return y.EnergyNetProduction
												}).Exec().Result.Data().(float64)

												data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
													y := x.(FuelCost)
													return y.PrimaryFuelConsumed
												}).Exec().Result.Data().(float64)
											}
										}
										// Vibrations handled by sql query
									}

									// Maintenance handled by sql query
									// FailureNotifications handled by sql query
									// MROElements handled by sql query
									// Operationals handled by sql query

									// e = ctx.Insert(&data)
									saveDatas = append(saveDatas, &data)
									dataCount++
									if len(saveDatas) == 1000 {
										mugen.Lock()
										e = ctx.InsertBulk(saveDatas)
										mugen.Unlock()
										ErrorHandler(e, "generateMaintenanceDataBrowser")
										tk.Printf("%v : %v \n", plantCodeStr, dataCount)

										saveDatas = []orm.IModel{}
									}

									/*ErrorHandler(e, "generateMaintenanceDataBrowser")
									tk.Println("save 4")*/
								}
							}
						}
					}

				}
			}
		}

		e = c.NewQuery().Delete().From(new(GenDataBrowserNotInTmp).TableName()).SetConfig("multiexec", true).Where(dbox.Eq("ID", genIDTempTable)).Exec(nil)

		if len(saveDatas) > 0 {
			e = ctx.InsertBulk(saveDatas)
			ErrorHandler(e, "generateMaintenanceDataBrowser")
			tk.Printf("%v : %v \n", plantCodeStr, dataCount)
		}

	}

	mugen.Lock()
	dataCounts.Set(plantCodeStr, dataCount)
	mugen.Unlock()

	wg.Done()
}
Пример #22
0
func (a *DataFlowController) GetDataMonitoring(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := map[string]interface{}{}
	e := r.GetPayload(&payload)
	if e != nil {
		return helper.CreateResult(false, nil, e.Error())
	}

	status := tk.ToString(payload["status"])

	filters := []*dbox.Filter{}
	var filter *dbox.Filter
	filter = new(dbox.Filter)

	if strings.Contains(strings.ToLower(status), "run") {
		filters = append(filters, dbox.Eq("status", "RUN"))
	} else {
		filters = append(filters, dbox.Ne("status", "RUN"))
	}

	take := tk.ToInt(payload["take"], tk.RoundingAuto)
	skip := tk.ToInt(payload["skip"], tk.RoundingAuto)

	start := tk.ToString(payload["startdate"])
	end := tk.ToString(payload["enddate"])

	search := tk.ToString(payload["search"])

	startdate := time.Now()
	enddate := time.Now()

	if start != "" {
		startdate, _ = time.Parse(time.RFC3339, start)
		filters = append(filters, dbox.Gte("startdate", startdate))
		startdate = startdate.AddDate(0, 0, 1)
		startdate = startdate.Add(time.Duration(-1) * time.Second)
		filters = append(filters, dbox.Lte("startdate", startdate))
	}

	if end != "" && !strings.Contains(strings.ToLower(status), "run") {
		enddate, _ = time.Parse(time.RFC3339, end)
		filters = append(filters, dbox.Gte("enddate", enddate))
		enddate = enddate.AddDate(0, 0, 1)
		enddate = enddate.Add(time.Duration(-1) * time.Second)
		filters = append(filters, dbox.Lte("enddate", enddate))
	}

	if search != "" {
		filters = append(filters, dbox.Or(dbox.Contains("flow.name", search), dbox.Contains("flow.description", search)))
	}

	filter = dbox.And(filters...)
	dataDs := []colonycore.DataFlowProcess{}
	cursor, err := colonycore.Finds(new(colonycore.DataFlowProcess), tk.M{}.Set("where", filter).Set("take", take).Set("skip", skip).Set("order", []string{"-startdate"}))
	cursorCount, _ := colonycore.Finds(new(colonycore.DataFlowProcess), tk.M{}.Set("where", filter))
	res := tk.M{}
	if cursor != nil {
		cursor.Fetch(&dataDs, 0, false)
		defer cursor.Close()
		defer cursorCount.Close()
		res.Set("total", cursorCount.Count())
	} else {
		res.Set("total", 0)
	}

	if err != nil && cursor != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	res.Set("data", dataDs)

	return helper.CreateResult(true, res, "success")
}