func (d *DataGrabberController) FindDataGrabber(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson //~ payload := map[string]string{"inputText":"test"} payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } text := payload["inputText"].(string) textLow := strings.ToLower(text) var query *dbox.Filter valueInt, errv := strconv.Atoi(text) fmt.Printf("", valueInt) fmt.Printf("", errv) if errv == nil { // == try useing Eq for support integer query = dbox.Or(dbox.Eq("GrabInterval", valueInt), dbox.Eq("TimeoutInterval", valueInt)) } else { // == try useing Contains for support autocomplite query = dbox.Or(dbox.Contains("_id", text), dbox.Contains("_id", textLow), dbox.Contains("DataSourceOrigin", text), dbox.Contains("DataSourceOrigin", textLow), dbox.Contains("DataSourceDestination", text), dbox.Contains("DataSourceDestination", textLow), dbox.Contains("IntervalType", text), dbox.Contains("IntervalType", textLow)) } data := []colonycore.DataGrabber{} cursor, err := colonycore.Find(new(colonycore.DataGrabber), query) cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, data, "") }
func (a *DataFlowController) GetListData(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } search := tk.ToString(payload["search"]) var query *dbox.Filter if search != "" { query = dbox.Or(dbox.Contains("name", search), dbox.Contains("description", search), dbox.Contains("createdby", search)) } cursor, err := colonycore.Find(new(colonycore.DataFlow), query) dataDs := []colonycore.DataFlow{} if cursor != nil { cursor.Fetch(&dataDs, 0, false) defer cursor.Close() } if err != nil && cursor != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, dataDs, "success") }
func (d *DataSourceController) FindConnection(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson //~ payload := map[string]string{"inputText":"tes","inputDrop":""} payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } text := payload["inputText"].(string) pilih := payload["inputDrop"].(string) textLow := strings.ToLower(text) // == try useing Contains for support autocomplite var query *dbox.Filter if text != "" { query = dbox.Or(dbox.Contains("_id", text), dbox.Contains("_id", textLow), dbox.Contains("Database", text), dbox.Contains("Database", textLow), dbox.Contains("Driver", text), dbox.Contains("Driver", textLow), dbox.Contains("Host", text), dbox.Contains("Host", textLow), dbox.Contains("UserName", text), dbox.Contains("UserName", textLow), dbox.Contains("Password", text), dbox.Contains("Password", textLow)) } if pilih != "" { query = dbox.And(query, dbox.Eq("Driver", pilih)) } data := []colonycore.Connection{} cursor, err := colonycore.Find(new(colonycore.Connection), query) cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, data, "") }
func (d *DataSourceController) RemoveDataSource(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } id := payload["_id"].(string) dg := new(colonycore.DataGrabber) filter := dbox.Or(dbox.Eq("DataSourceOrigin", id), dbox.Eq("DataSourceDestination", id)) cursor, err := colonycore.Find(dg, filter) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() if cursor.Count() > 0 { return helper.CreateResult(false, nil, "Cannot delete data source because used on data grabber") } o := new(colonycore.DataSource) o.ID = id err = colonycore.Delete(o) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, nil, "") }
func (d *DataSourceController) GetDataSources(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } search := payload["search"].(string) var query *dbox.Filter query = dbox.Or(dbox.Contains("_id", search), dbox.Contains("ConnectionID", search)) cursor, err := colonycore.Find(new(colonycore.DataSource), query) if err != nil { return helper.CreateResult(false, nil, err.Error()) } data := []colonycore.DataSource{} err = cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, data, "") }
func TestSelectFilter(t *testing.T) { t.Skip() c, e := prepareConnection() if e != nil { t.Errorf("Unable to connect %s \n", e.Error()) return } defer c.Close() csr, e := c.NewQuery(). Select("empno", "ename", "mgr", "hiredate"). Where(dbox.Or(dbox.Eq("empno", 7521), dbox.Eq("ename", "ADAMS"))). From(tableName).Cursor(nil) if e != nil { t.Errorf("Cursor pre error: %s \n", e.Error()) return } if csr == nil { t.Errorf("Cursor not initialized") return } defer csr.Close() rets := /*[]customers{}*/ []toolkit.M{} e = csr.Fetch(&rets, 0, false) if e != nil { t.Errorf("Unable to fetch: %s \n", e.Error()) } else { toolkit.Printf("Filter OK. Result: %v \n", toolkit.JsonString(rets)) } }
func (l *LangenvironmentController) UninstallLang(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := new(colonycore.LanguageEnvironmentPayload) err := r.GetPayload(payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } dataServers := new(colonycore.Server) err = colonycore.Get(dataServers, payload.ServerId) if err != nil { return helper.CreateResult(false, nil, err.Error()) } serverPathSeparator := CreateApplicationController(l.Server).GetServerPathSeparator(dataServers) sshSetting, sshClient, err := dataServers.Connect() if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer sshClient.Close() var query *dbox.Filter if payload.Lang == LANG_SCALA { var IsInstalled bool for _, eachLang := range dataServers.InstalledLang { if eachLang.Lang == LANG_JAVA { IsInstalled = eachLang.IsInstalled break } } if !IsInstalled { query = dbox.Or(dbox.Eq("language", LANG_JAVA), dbox.Eq("language", LANG_SCALA)) } else { query = dbox.Eq("language", payload.Lang) } } else { query = dbox.Eq("language", payload.Lang) } result, err := l.ProcessSetup(dataServers, query, serverPathSeparator, sshSetting) if err != nil { helper.CreateResult(false, nil, err.Error()) } fmt.Println("result :: ", result) for _, eachLang := range dataServers.InstalledLang { if eachLang.Lang == payload.Lang { eachLang.IsInstalled = false break } } err = colonycore.Save(dataServers) if err != nil { helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, payload, "") }
func (d *WebGrabberController) FindWebGrabber(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson //~ payload := map[string]string{"inputText": "GRAB_TEST", "inputRequest": "", "inputType": ""} payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } text := payload["inputText"].(string) req := payload["inputRequest"].(string) tipe := payload["inputType"].(string) textLow := strings.ToLower(text) // == bug, cant find if autocomplite, just full text can be get result var query *dbox.Filter if text != "" { valueInt, errv := strconv.Atoi(text) if errv == nil { // == try useing Eq for support integer query = dbox.Or(dbox.Eq("GrabInterval", valueInt), dbox.Eq("TimeoutInterval", valueInt)) } else { // == try useing Contains for support autocomplite query = dbox.Or(dbox.Contains("_id", text), dbox.Contains("_id", textLow), dbox.Contains("Calltype", text), dbox.Contains("Calltype", textLow), dbox.Contains("SourceType", text), dbox.Contains("SourceType", textLow), dbox.Contains("IntervalType", text), dbox.Contains("IntervalType", textLow)) } } if req != "" { query = dbox.And(query, dbox.Eq("Calltype", req)) } if tipe != "" { query = dbox.And(query, dbox.Eq("SourceType", tipe)) } data := []colonycore.WebGrabber{} cursor, err := colonycore.Find(new(colonycore.WebGrabber), query) cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, payload, "") }
func TestDelete(t *testing.T) { t.Skip("Skip : Comment this line to do test") skipIfConnectionIsNil(t) e := ctx.NewQuery().Delete().From("Data_CUD").SetConfig("multiexec", true).Where(dbox.Or(dbox.Eq("Id", "ID-11"), dbox.Eq("Id", "ID-13"))).Exec(nil) if e != nil { t.Fatalf("Delete fail: %s", e.Error()) } }
func TestFilter(t *testing.T) { fb := dbox.NewFilterBuilder(new(FilterBuilder)) fb.AddFilter(dbox.Or( dbox.Eq("_id", 1), dbox.Eq("group", "administrators"))) b, e := fb.Build() if e != nil { t.Errorf("Error %s", e.Error()) } else { fmt.Printf("Result:\n%v\n", toolkit.JsonString(b)) } }
func (s *ServerController) GetServers(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := struct { Search string `json:"search"` ServerOS string `json:"serverOS"` ServerType string `json:"serverType"` SSHType string `json:"sshType"` }{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } filters := []*dbox.Filter{} if payload.Search != "" { filters = append(filters, dbox.Or( dbox.Contains("_id", payload.Search), dbox.Contains("os", payload.Search), dbox.Contains("host", payload.Search), dbox.Contains("serverType", payload.Search), dbox.Contains("sshtype", payload.Search), )) } if payload.ServerOS != "" { filters = append(filters, dbox.Eq("os", payload.ServerOS)) } if payload.ServerType != "" { filters = append(filters, dbox.Eq("serverType", payload.ServerType)) } if payload.SSHType != "" { filters = append(filters, dbox.Eq("sshtype", payload.SSHType)) } var query *dbox.Filter if len(filters) > 0 { query = dbox.And(filters...) } cursor, err := colonycore.Find(new(colonycore.Server), query) if err != nil { return helper.CreateResult(false, nil, err.Error()) } data := []colonycore.Server{} err = cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, data, "") }
func TestFilter(t *testing.T) { fb := dbox.NewFilterBuilder(new(FilterBuilder)) fb.AddFilter(dbox.Or( dbox.Contains("regfield", "1"), dbox.Ne("nefield", 1), dbox.Eq("group", "administrators"))) b, e := fb.Build() if e != nil { t.Errorf("Error %s", e.Error()) } else { fmt.Printf("Result:\n%v\n", toolkit.JsonString(b)) } fb = dbox.NewFilterBuilder(new(FilterBuilder)) fb.AddFilter(dbox.And(dbox.Or(dbox.Eq("EmployeeId", "101-102-10"), dbox.Eq("EmployeeId", "101-102-3"), dbox.Eq("EmployeeId", "101-102-4")), dbox.Eq("Age", "30"))) c, e := fb.Build() if e != nil { t.Errorf("Error %s", e.Error()) } else { fmt.Printf("Result:\n%v\n", toolkit.JsonString(c)) } }
func filterCondition(cond toolkit.M) *dbox.Filter { fb := new(dbox.Filter) for key, val := range cond { if key == "$and" || key == "$or" { afb := []*dbox.Filter{} for _, sVal := range val.([]interface{}) { rVal := sVal.(map[string]interface{}) mVal := toolkit.M{} for rKey, mapVal := range rVal { mVal.Set(rKey, mapVal) } afb = append(afb, filterCondition(mVal)) } if key == "$and" { fb = dbox.And(afb...) } else { fb = dbox.Or(afb...) } } else { if reflect.ValueOf(val).Kind() == reflect.Map { mVal := val.(map[string]interface{}) tomVal, _ := toolkit.ToM(mVal) switch { case tomVal.Has("$eq"): fb = dbox.Eq(key, tomVal["$eq"].(string)) case tomVal.Has("$ne"): fb = dbox.Ne(key, tomVal["$ne"].(string)) case tomVal.Has("$regex"): fb = dbox.Contains(key, tomVal["$regex"].(string)) case tomVal.Has("$gt"): fb = dbox.Gt(key, tomVal["$gt"].(string)) case tomVal.Has("$gte"): fb = dbox.Gte(key, tomVal["$gte"].(string)) case tomVal.Has("$lt"): fb = dbox.Lt(key, tomVal["$lt"].(string)) case tomVal.Has("$lte"): fb = dbox.Lte(key, tomVal["$lte"].(string)) } } else { fb = dbox.Eq(key, val) } } } return fb }
func (a *AdministrationController) GetAccess(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if strings.Contains(toolkit.TypeName(payload["find"]), "float") { payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto) } tAccess := new(acl.Access) if find := toolkit.ToString(payload["find"]); find != "" { filter = new(dbox.Filter) filter = dbox.Or(dbox.Contains("id", find), dbox.Contains("title", find), dbox.Contains("group1", find), dbox.Contains("group2", find), dbox.Contains("group3", find), dbox.Contains("specialaccess1", find), dbox.Contains("specialaccess2", find), dbox.Contains("specialaccess3", find), dbox.Contains("specialaccess4", find)) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err == nil { err = c.Fetch(&arrm, 0, false) } c.Close() c, err = acl.Find(tAccess, filter, nil) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, data, "") } }
func (a *UserController) GetUser(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson _ = a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if strings.Contains(toolkit.TypeName(payload["find"]), "float") { payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto) } tUser := new(acl.User) if find := toolkit.ToString(payload["find"]); find != "" { filter = new(dbox.Filter) filter = dbox.Or(dbox.Contains("id", find), dbox.Contains("fullname", find), dbox.Contains("email", find)) } take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) c, err := acl.Find(tUser, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err != nil { return helper.CreateResult(true, nil, err.Error()) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) err = c.Fetch(&arrm, 0, false) c.Close() c, err = acl.Find(tUser, filter, nil) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, data, "") } }
func (a *GroupController) Search(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson _ = a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if strings.Contains(toolkit.TypeName(payload["search"]), "float") { payload["search"] = toolkit.ToInt(payload["search"], toolkit.RoundingAuto) } tGroup := new(acl.Group) if search := toolkit.ToString(payload["search"]); search != "" { filter = new(dbox.Filter) filter = dbox.Or(dbox.Contains("_id", search), dbox.Contains("title", search), dbox.Contains("owner", search)) } fmt.Println(filter) take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) c, err := acl.Find(tGroup, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err != nil { return helper.CreateResult(true, nil, err.Error()) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) err = c.Fetch(&arrm, 0, false) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } return helper.CreateResult(true, data, "") }
func (d *DataSourceController) GetConnections(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } search := "" if payload["search"] != nil { search = payload["search"].(string) } driver := "" if payload["driver"] != nil { search = payload["driver"].(string) } // search := payload["search"] // driver := payload["driver"] var query *dbox.Filter query = dbox.Or(dbox.Contains("_id", search), dbox.Contains("Driver", search), dbox.Contains("Host", search), dbox.Contains("Database", search), dbox.Contains("UserName", search)) if driver != "" { query = dbox.And(query, dbox.Eq("Driver", driver)) } data := []colonycore.Connection{} cursor, err := colonycore.Find(new(colonycore.Connection), query) if err != nil { return helper.CreateResult(false, nil, err.Error()) } err = cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, data, "") }
func (w *WebGrabberController) GetScrapperData(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } search := payload["search"].(string) requesttype := payload["requesttype"].(string) sourcetype := payload["sourcetype"].(string) var query *dbox.Filter query = dbox.Or(dbox.Contains("_id", search)) if sourcetype == "" { //default sourcetype == "SourceType_HttpHtml" query = dbox.And(query, dbox.Eq("sourcetype", "SourceType_HttpHtml")) } else { query = dbox.And(query, dbox.Eq("sourcetype", sourcetype)) } if requesttype != "" { query = dbox.And(query, dbox.Eq("grabconf.calltype", requesttype)) } cursor, err := colonycore.Find(new(colonycore.WebGrabber), query) if err != nil { return helper.CreateResult(false, nil, err.Error()) } data := []colonycore.WebGrabber{} err = cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() return helper.CreateResult(true, data, "") }
func (a *UserController) Search(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetPayload(&payload) find := payload["search"].(string) bfind, err := strconv.ParseBool(find) tUser := new(acl.User) arrm := make([]toolkit.M, 0, 0) filter := dbox.Or(dbox.Contains("_id", find), dbox.Contains("id", find), dbox.Contains("loginid", find), dbox.Contains("fullname", find), dbox.Contains("email", find), dbox.Eq("enable", bfind)) c, e := acl.Find(tUser, filter, toolkit.M{}.Set("take", 0)) if e == nil { e = c.Fetch(&arrm, 0, false) } if e != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, arrm, "") } }
func (d *DataSourceController) FindDataSource(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } text := payload["inputText"].(string) textLow := strings.ToLower(text) // == try useing Contains for support autocomplite var query *dbox.Filter query = dbox.Or(dbox.Contains("_id", text), dbox.Contains("_id", textLow), dbox.Contains("ConnectionID", text), dbox.Contains("ConnectionID", textLow)) data := []colonycore.DataSource{} cursor, err := colonycore.Find(new(colonycore.DataSource), query) cursor.Fetch(&data, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // == bug, i dont know what i can to do if find by database name.== //~ if data == nil { //~ query = dbox.Eq("Database",text) //~ data := []colonycore.Connection{} //~ cursor, err := colonycore.Find(new(colonycore.Connection), query) //~ cursor.Fetch(&data, 0, false) //~ if err != nil { //~ return helper.CreateResult(false, nil, err.Error()) //~ } //~ fmt.Printf("========asdasd=======%#v",data) //~ } defer cursor.Close() return helper.CreateResult(true, data, "") }
func (a *AdministrationController) Search(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetPayload(&payload) find := payload["search"].(string) bfind, err := strconv.ParseBool(find) tAccess := new(acl.Access) arrm := make([]toolkit.M, 0, 0) filter := dbox.Or(dbox.Contains("_id", find), dbox.Contains("id", find), dbox.Contains("title", find), dbox.Contains("group1", find), dbox.Contains("group2", find), dbox.Contains("group3", find), dbox.Contains("specialaccess1", find), dbox.Contains("specialaccess2", find), dbox.Contains("specialaccess3", find), dbox.Contains("specialaccess4", find), dbox.Eq("enable", bfind)) c, e := acl.Find(tAccess, filter, toolkit.M{}.Set("take", 0)) if e == nil { e = c.Fetch(&arrm, 0, false) } if e != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, arrm, "") } }
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) { logConf, err := d.getLogger(dataGrabber) if err != nil { logConf.AddLog(err.Error(), "ERROR") defer logConf.Close() } message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType) logConf.AddLog(message, "SUCCESS") fmt.Println(message) dsOrigin := new(colonycore.DataSource) err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } dsDestination := new(colonycore.DataSource) err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } dataDS, _, conn, query, metaSave, err := new(DataSourceController). ConnectToDataSource(dataGrabber.DataSourceOrigin) if len(dataDS.QueryInfo) == 0 { message := "Data source origin has invalid query" logConf.AddLog(message, "ERROR") return false, nil, message } if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } defer conn.Close() if metaSave.keyword != "" { message := `Data source origin query is not "Select"` logConf.AddLog(message, "ERROR") return false, nil, message } cursor, err := query.Cursor(nil) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } defer cursor.Close() data := []toolkit.M{} err = cursor.Fetch(&data, 0, false) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } connDesc := new(colonycore.Connection) err = colonycore.Get(connDesc, dsDestination.ConnectionID) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } const FLAG_ARG_DATA string = `%1` transformedData := []toolkit.M{} for _, each := range data { eachTransformedData := toolkit.M{} for _, eachMap := range dataGrabber.Maps { var valueEachSourceField interface{} // ============================================ SOURCE if !strings.Contains(eachMap.Source, "|") { // source could be: field, object, array valueEachSourceField = each.Get(eachMap.Source) } else { // source could be: field of object, field of array-objects prev := strings.Split(eachMap.Source, "|")[0] next := strings.Split(eachMap.Source, "|")[1] var fieldInfoDes *colonycore.FieldInfo = nil for _, eds := range dsOrigin.MetaData { if eds.ID == prev { fieldInfoDes = eds break } } if fieldInfoDes != nil { // source is field of array-objects if fieldInfoDes.Type == "array-objects" { valueObjects := []interface{}{} if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil { valueObjects = make([]interface{}, len(temp)) for i, each := range temp { if tempSub, _ := toolkit.ToM(each); tempSub != nil { valueObjects[i] = tempSub.Get(next) } } } valueEachSourceField = valueObjects } else { // source is field of object valueObject := toolkit.M{} if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil { valueEachSourceField = valueObject.Get(next) } } } } // ============================================ DESTINATION if !strings.Contains(eachMap.Destination, "|") { if eachMap.SourceType == "object" { sourceObject, _ := toolkit.ToM(valueEachSourceField) if sourceObject == nil { sourceObject = toolkit.M{} } valueObject := toolkit.M{} for _, desMeta := range dsDestination.MetaData { if desMeta.ID == eachMap.Destination { for _, eachMetaSub := range desMeta.Sub { // valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID)) valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType)) } break } } eachTransformedData.Set(eachMap.Destination, valueObject) } else if eachMap.SourceType == "array-objects" { sourceObjects, _ := valueEachSourceField.([]interface{}) if sourceObjects == nil { sourceObjects = []interface{}{} } valueObjects := []interface{}{} for _, sourceObjectRaw := range sourceObjects { sourceObject, _ := toolkit.ToM(sourceObjectRaw) if sourceObject == nil { sourceObject = toolkit.M{} } valueObject := toolkit.M{} for _, desMeta := range dsDestination.MetaData { if desMeta.ID == eachMap.Destination { for _, eachMetaSub := range desMeta.Sub { // valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID)) valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType)) } break } } valueObjects = append(valueObjects, valueObject) } eachTransformedData.Set(eachMap.Destination, valueObjects) } else { if strings.Contains(eachMap.DestinationType, "array") { valueObjects := each.Get(eachMap.Source) eachTransformedData.Set(eachMap.Destination, valueObjects) } else { // eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each)) eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType)) } } } else { prev := strings.Split(eachMap.Destination, "|")[0] next := strings.Split(eachMap.Destination, "|")[1] var fieldInfoDes *colonycore.FieldInfo = nil for _, eds := range dsDestination.MetaData { if eds.ID == prev { fieldInfoDes = eds break } } if fieldInfoDes != nil { if fieldInfoDes.Type == "array-objects" { valueObjects := []interface{}{} if temp := eachTransformedData.Get(prev, nil); temp == nil { valueObjects = []interface{}{} } else { valueObjects, _ = temp.([]interface{}) if valueObjects == nil { valueObjects = []interface{}{} } } if temp, _ := valueEachSourceField.([]interface{}); temp != nil { for i, eachVal := range temp { valueObject := toolkit.M{} if len(valueObjects) > i { if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil { valueObject = temp2 // valueObject.Set(next, eachVal) valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType)) } valueObjects[i] = valueObject } else { if fieldInfoDes.Sub != nil { for _, subMeta := range fieldInfoDes.Sub { valueObject.Set(subMeta.ID, nil) } } // valueObject.Set(next, eachVal) valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType)) valueObjects = append(valueObjects, valueObject) } } } eachTransformedData.Set(prev, valueObjects) } else { valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev)) if valueObject == nil { valueObject = toolkit.M{} } //tambahan prevSource := strings.Split(eachMap.Source, "|")[0] nextSource := strings.Split(eachMap.Source, "|")[1] mval, _ := toolkit.ToM(each.Get(prevSource, nil)) //========= valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType)) // valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval)) eachTransformedData.Set(prev, valueObject) } } } } transformedData = append(transformedData, eachTransformedData) dataToSave := eachTransformedData // ================ pre transfer command if dataGrabber.PreTransferCommand != "" { // jsonTranformedDataBytes, err := json.Marshal(each) jsonTranformedDataBytes, err := json.Marshal(eachTransformedData) if err != nil { return false, nil, err.Error() } jsonTranformedData := string(jsonTranformedDataBytes) var preCommand = dataGrabber.PreTransferCommand if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) { preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1)) } dataToSave = toolkit.M{} output, err := toolkit.RunCommand(preCommand, jsonTranformedData) fmt.Printf("===> Pre Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", preCommand, jsonTranformedData, output) if err == nil { postData := toolkit.M{} if err := json.Unmarshal([]byte(output), &postData); err == nil { dataToSave = postData } } } // ================ if len(dataToSave) == 0 { continue } nilFieldDest := eachTransformedData for _, metadataDest := range dsDestination.MetaData { if temp := eachTransformedData.Get(metadataDest.ID); temp == nil { if metadataDest.ID != "_id" { if metadataDest.Type == "object" { valueObject := toolkit.M{} for _, eachMetaSub := range metadataDest.Sub { valueObject.Set(eachMetaSub.ID, nil) } nilFieldDest.Set(metadataDest.ID, valueObject) } else if metadataDest.Type == "array-objects" { valueEachSourceField := each.Get(metadataDest.ID) sourceObjects, _ := valueEachSourceField.([]interface{}) if sourceObjects == nil { sourceObjects = []interface{}{} } valueObjects := []interface{}{} for _, sourceObjectRaw := range sourceObjects { sourceObject, _ := toolkit.ToM(sourceObjectRaw) if sourceObject == nil { sourceObject = toolkit.M{} } valueObject := toolkit.M{} for keyss, _ := range sourceObject { valueObject.Set(keyss, nil) } valueObjects = append(valueObjects, valueObject) } nilFieldDest.Set(metadataDest.ID, valueObjects) } else { if strings.Contains(metadataDest.Type, "array") { valueObjects := []interface{}{} nilFieldDest.Set(metadataDest.ID, valueObjects) } else { nilFieldDest.Set(metadataDest.ID, nil) } } } } } tableName := dsDestination.QueryInfo.GetString("from") queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings) if dataGrabber.InsertMode == "fresh" { queryWrapper.Delete(tableName, dbox.Or()) } if eachTransformedData.Has("_id") { err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id"))) } if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") { queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings) } else { queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings) } if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" { nilFieldDest.Set("_id", helper.RandomIDWithPrefix("")) } err = queryWrapper.Save(tableName, nilFieldDest) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } // ================ post transfer command if dataGrabber.PostTransferCommand != "" { eachTransformedData = dataToSave jsonTranformedDataBytes, err := json.Marshal(eachTransformedData) if err != nil { return false, nil, err.Error() } jsonTranformedData := string(jsonTranformedDataBytes) var postCommand = dataGrabber.PostTransferCommand if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) { postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1)) } output, err := toolkit.RunCommand(postCommand, jsonTranformedData) fmt.Printf("===> Post Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", postCommand, jsonTranformedData, output) } } message = fmt.Sprintf("===> Success transforming %v data", len(transformedData)) logConf.AddLog(message, "SUCCESS") fmt.Println(message) return true, transformedData, "" }
func (d *DataSourceController) filterParse(where toolkit.M) *dbox.Filter { field := where.Get("field", "").(string) value := fmt.Sprintf("%v", where["value"]) if key := where.Get("key", "").(string); key == "Eq" { valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"])) if errv == nil { return dbox.Eq(field, valueInt) } else { return dbox.Eq(field, value) } } else if key == "Ne" { valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"])) if errv == nil { return dbox.Ne(field, valueInt) } else { return dbox.Ne(field, value) } } else if key == "Lt" { valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"])) if errv == nil { return dbox.Lt(field, valueInt) } else { return dbox.Lt(field, value) } } else if key == "Lte" { valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"])) if errv == nil { return dbox.Lte(field, valueInt) } else { return dbox.Lte(field, value) } } else if key == "Gt" { valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"])) if errv == nil { return dbox.Gt(field, valueInt) } else { return dbox.Gt(field, value) } } else if key == "Gte" { valueInt, errv := strconv.Atoi(fmt.Sprintf("%v", where["value"])) if errv == nil { return dbox.Gte(field, valueInt) } else { return dbox.Gte(field, value) } } else if key == "In" { valueArray := []interface{}{} for _, e := range strings.Split(value, ",") { valueArray = append(valueArray, strings.Trim(e, "")) } return dbox.In(field, valueArray...) } else if key == "Nin" { valueArray := []interface{}{} for _, e := range strings.Split(value, ",") { valueArray = append(valueArray, strings.Trim(e, "")) } return dbox.Nin(field, valueArray...) } else if key == "Contains" { return dbox.Contains(field, value) } else if key == "Or" { subs := where.Get("value", []interface{}{}).([]interface{}) filtersToMerge := []*dbox.Filter{} for _, eachSub := range subs { eachWhere, _ := toolkit.ToM(eachSub) filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere)) } return dbox.Or(filtersToMerge...) } else if key == "And" { subs := where.Get("value", []interface{}{}).([]interface{}) filtersToMerge := []*dbox.Filter{} for _, eachSub := range subs { eachWhere, _ := toolkit.ToM(eachSub) filtersToMerge = append(filtersToMerge, d.filterParse(eachWhere)) } return dbox.And(filtersToMerge...) } return nil }
func (l *LangenvironmentController) SetupFromSH(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := new(colonycore.LanguageEnvironmentPayload) err := r.GetPayload(payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } dataServers := new(colonycore.Server) err = colonycore.Get(dataServers, payload.ServerId) if err != nil { return helper.CreateResult(false, nil, err.Error()) } serverPathSeparator := CreateApplicationController(l.Server).GetServerPathSeparator(dataServers) sshSetting, sshClient, err := dataServers.Connect() if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer sshClient.Close() var query *dbox.Filter if payload.Lang == LANG_SCALA { var IsInstalled bool for _, eachLang := range dataServers.InstalledLang { if eachLang.Lang == LANG_JAVA { IsInstalled = eachLang.IsInstalled break } } if !IsInstalled { query = dbox.Or(dbox.Eq("language", LANG_JAVA), dbox.Eq("language", LANG_SCALA)) } else { query = dbox.Eq("language", payload.Lang) } } else { query = dbox.Eq("language", payload.Lang) } dataLanguage := []colonycore.LanguageEnviroment{} cursor, err := colonycore.Find(new(colonycore.LanguageEnviroment), query) cursor.Fetch(&dataLanguage, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() if cursor.Count() > 0 { for _, eachLang := range dataLanguage { for _, dataInstaller := range eachLang.Installer { var sourcePath string var destinationPath string pathstring := []string{dataServers.DataPath, "langenvironment", "installer"} var installShPath, uninstallShPath string if strings.ToLower(dataServers.OS) == strings.ToLower(dataInstaller.OS) { // fmt.Println(ii, " : -- > ", eachLang.Language) targetLang := new(colonycore.InstalledLang) targetLang.Lang = eachLang.Language for _, eachLang := range dataServers.InstalledLang { if eachLang.Lang == targetLang.Lang { targetLang = eachLang break } } targetLang.IsInstalled = true if eachLang.Language == LANG_GO { pathstring = append(pathstring, LANG_GO) pathstring = append(pathstring, dataServers.OS) sourcePath = filepath.Join(leSourcePath, LANG_GO, dataServers.OS, dataInstaller.InstallerSource) destinationPath = strings.Join(pathstring, serverPathSeparator) installShPath = filepath.Join(leSourcePath, LANG_GO, dataServers.OS, "install.sh") uninstallShPath = filepath.Join(leSourcePath, LANG_GO, dataServers.OS, "uninstall.sh") } else if eachLang.Language == LANG_JAVA { pathstring = append(pathstring, LANG_JAVA) pathstring = append(pathstring, dataServers.OS) sourcePath = filepath.Join(leSourcePath, LANG_JAVA, dataServers.OS, dataInstaller.InstallerSource) destinationPath = strings.Join(pathstring, serverPathSeparator) installShPath = filepath.Join(leSourcePath, LANG_JAVA, dataServers.OS, "install.sh") uninstallShPath = filepath.Join(leSourcePath, LANG_JAVA, dataServers.OS, "uninstall.sh") } else if eachLang.Language == LANG_SCALA { pathstring = append(pathstring, LANG_SCALA) pathstring = append(pathstring, dataServers.OS) sourcePath = filepath.Join(leSourcePath, LANG_SCALA, dataServers.OS, dataInstaller.InstallerSource) destinationPath = strings.Join(pathstring, serverPathSeparator) installShPath = filepath.Join(leSourcePath, LANG_SCALA, dataServers.OS, "install.sh") uninstallShPath = filepath.Join(leSourcePath, LANG_SCALA, dataServers.OS, "uninstall.sh") } installShdestPath := strings.Join(append(pathstring, "install.sh"), serverPathSeparator) uninstallShdestPath := strings.Join(append(pathstring, "uninstall.sh"), serverPathSeparator) installFilePath := strings.Join(append(pathstring, dataInstaller.InstallerSource), serverPathSeparator) err = sshSetting.SshCopyByPath(sourcePath, destinationPath) if err != nil { return helper.CreateResult(false, nil, err.Error()) } err = sshSetting.SshCopyByPath(installShPath, destinationPath) if err != nil { return helper.CreateResult(false, nil, err.Error()) } err = sshSetting.SshCopyByPath(uninstallShPath, destinationPath) if err != nil { return helper.CreateResult(false, nil, err.Error()) } //sed -i 's/^M//' install.sh cmdSedInstall := fmt.Sprintf("sed -i 's/\r//g' %s", installShdestPath) _, err = sshSetting.GetOutputCommandSsh(cmdSedInstall) if err != nil { return helper.CreateResult(false, nil, err.Error()) } //sed -i 's/^M//' uninstall.sh cmdSedUninstall := fmt.Sprintf("sed -i 's/\r//g' %s", uninstallShdestPath) _, err = sshSetting.GetOutputCommandSsh(cmdSedUninstall) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // // chmod +x install.sh cmdChmodCliInstall := fmt.Sprintf("chmod +x %s", installShdestPath) _, err = sshSetting.GetOutputCommandSsh(cmdChmodCliInstall) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // // chmod +x uninstall.sh cmdChmodCliUninstall := fmt.Sprintf("chmod +x %s", uninstallShdestPath) _, err = sshSetting.GetOutputCommandSsh(cmdChmodCliUninstall) if err != nil { return helper.CreateResult(false, nil, err.Error()) } // // sh install.sh installFilePath DESTINSTALL_PATH projectpath cmdShCli := fmt.Sprintf("bash %s %s %s", installShdestPath, installFilePath, DESTINSTALL_PATH) outputCmd, err := sshSetting.RunCommandSshAsMap(cmdShCli) if err != nil { return helper.CreateResult(false, nil, err.Error()) } fmt.Println(" :: : ", outputCmd[0].Output) newInstalledLangs := []*colonycore.InstalledLang{} for _, each := range dataServers.InstalledLang { if each.Lang == targetLang.Lang { continue } newInstalledLangs = append(newInstalledLangs, each) } newInstalledLangs = append(newInstalledLangs, targetLang) dataServers.InstalledLang = newInstalledLangs colonycore.Save(dataServers) } } } } return helper.CreateResult(true, payload, "") }
func filterCondition(cond toolkit.M) *dbox.Filter { fb := new(dbox.Filter) for key, val := range cond { if key == "$and" || key == "$or" { afb := []*dbox.Filter{} for _, sVal := range val.([]interface{}) { rVal := sVal.(map[string]interface{}) mVal := toolkit.M{} for rKey, mapVal := range rVal { mVal.Set(rKey, mapVal) } afb = append(afb, filterCondition(mVal)) } if key == "$and" { fb = dbox.And(afb...) } else { fb = dbox.Or(afb...) } } else { if toolkit.TypeName(val) == "map[string]interface {}" { mVal := val.(map[string]interface{}) tomVal, _ := toolkit.ToM(mVal) switch { case tomVal.Has("$eq"): fb = dbox.Eq(key, tomVal["$eq"]) case tomVal.Has("$ne"): fb = dbox.Ne(key, tomVal["$ne"]) case tomVal.Has("$regex"): fb = dbox.Contains(key, toolkit.ToString(tomVal["$regex"])) case tomVal.Has("$gt"): fb = dbox.Gt(key, tomVal["$gt"]) case tomVal.Has("$gte"): fb = dbox.Gte(key, tomVal["$gte"]) case tomVal.Has("$lt"): fb = dbox.Lt(key, tomVal["$lt"]) case tomVal.Has("$lte"): fb = dbox.Lte(key, tomVal["$lte"]) case tomVal.Has("$in"): tval := make([]interface{}, 0, 0) if toolkit.TypeName(tomVal["$in"]) == "[]interface {}" { for _, tin := range tomVal["$in"].([]interface{}) { tval = append(tval, tin) } } else { tval = append(tval, tomVal["$in"]) } fb = dbox.In(key, tval...) case tomVal.Has("$nin"): tval := make([]interface{}, 0, 0) if toolkit.TypeName(tomVal["$nin"]) == "[]interface {}" { for _, tin := range tomVal["$nin"].([]interface{}) { tval = append(tval, tin) } } else { tval = append(tval, tomVal["$nin"]) } fb = dbox.Nin(key, tval...) } } else { fb = dbox.Eq(key, val) } } } return fb }
func (a *DataFlowController) GetDataMonitoring(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} e := r.GetPayload(&payload) if e != nil { return helper.CreateResult(false, nil, e.Error()) } status := tk.ToString(payload["status"]) filters := []*dbox.Filter{} var filter *dbox.Filter filter = new(dbox.Filter) if strings.Contains(strings.ToLower(status), "run") { filters = append(filters, dbox.Eq("status", "RUN")) } else { filters = append(filters, dbox.Ne("status", "RUN")) } take := tk.ToInt(payload["take"], tk.RoundingAuto) skip := tk.ToInt(payload["skip"], tk.RoundingAuto) start := tk.ToString(payload["startdate"]) end := tk.ToString(payload["enddate"]) search := tk.ToString(payload["search"]) startdate := time.Now() enddate := time.Now() if start != "" { startdate, _ = time.Parse(time.RFC3339, start) filters = append(filters, dbox.Gte("startdate", startdate)) startdate = startdate.AddDate(0, 0, 1) startdate = startdate.Add(time.Duration(-1) * time.Second) filters = append(filters, dbox.Lte("startdate", startdate)) } if end != "" && !strings.Contains(strings.ToLower(status), "run") { enddate, _ = time.Parse(time.RFC3339, end) filters = append(filters, dbox.Gte("enddate", enddate)) enddate = enddate.AddDate(0, 0, 1) enddate = enddate.Add(time.Duration(-1) * time.Second) filters = append(filters, dbox.Lte("enddate", enddate)) } if search != "" { filters = append(filters, dbox.Or(dbox.Contains("flow.name", search), dbox.Contains("flow.description", search))) } filter = dbox.And(filters...) dataDs := []colonycore.DataFlowProcess{} cursor, err := colonycore.Finds(new(colonycore.DataFlowProcess), tk.M{}.Set("where", filter).Set("take", take).Set("skip", skip).Set("order", []string{"-startdate"})) cursorCount, _ := colonycore.Finds(new(colonycore.DataFlowProcess), tk.M{}.Set("where", filter)) res := tk.M{} if cursor != nil { cursor.Fetch(&dataDs, 0, false) defer cursor.Close() defer cursorCount.Close() res.Set("total", cursorCount.Count()) } else { res.Set("total", 0) } if err != nil && cursor != nil { return helper.CreateResult(false, nil, err.Error()) } res.Set("data", dataDs) return helper.CreateResult(true, res, "success") }