func saverechistory(key string, dt toolkit.M) (err error) { err = nil fullfilename := filepath.Join(toolkit.ToString(histConf.Get("recpath", "")), mapRecHistory[key]) if EC_DATA_PATH != "" { fullfilename = filepath.Join(EC_DATA_PATH, "datagrabber", "historyrec", mapRecHistory[key]) } // fmt.Println(fullfilename, " - Key - ", key, " - filename - ", mapRecHistory) cconfig := toolkit.M{"newfile": true, "useheader": true, "delimiter": ","} conn, err := prepareconnection("csv", fullfilename, "", "", "", cconfig) if err != nil { return } q := conn.NewQuery().SetConfig("multiexec", true).Insert() for k, v := range dt { if toolkit.TypeName(v) == "toolkit.M" { dt.Set(k, fmt.Sprintf("%v", v)) } } err = q.Exec(toolkit.M{}.Set("data", dt)) conn.Close() return }
func GetFieldsFromDS(_id string, opt ...string) ([]string, error) { withSubFields := false if len(opt) > 0 { if opt[0] == "with sub fields" { withSubFields = true } } dataFetch, err := FetchDataFromDS(_id, 1) if err != nil { return nil, err } var fields []string for i, val := range dataFetch { if i > 0 { break } for field, fieldValue := range val { if withSubFields && (toolkit.TypeName(fieldValue) == "map[string]interface {}") { for subField := range fieldValue.(map[string]interface{}) { fields = append(fields, fmt.Sprintf("%s.%s", field, subField)) } continue } fields = append(fields, field) } } return fields, nil }
func (a *SessionController) GetSession(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson _ = a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if strings.Contains(toolkit.TypeName(payload["find"]), "float") { payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto) } tSession := new(acl.Session) if find := toolkit.ToString(payload["find"]); find != "" { filter = new(dbox.Filter) filter = dbox.Contains("loginid", find) } take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) // c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) // c, err := acl.Find(tSession, nil, toolkit.M{}.Set("take", payload["take"].(int)).Set("skip", payload["skip"].(int))) c, err := acl.Find(tSession, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err != nil { return helper.CreateResult(true, nil, err.Error()) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) err = c.Fetch(&arrm, 0, false) for i, val := range arrm { arrm[i].Set("duration", time.Since(val["created"].(time.Time)).Hours()) arrm[i].Set("status", "ACTIVE") if val["expired"].(time.Time).Before(time.Now().UTC()) { arrm[i].Set("duration", val["expired"].(time.Time).Sub(val["created"].(time.Time)).Hours()) arrm[i].Set("status", "EXPIRED") } } c.Close() c, err = acl.Find(tSession, filter, nil) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, data, "") } }
func (a *AdministrationController) GetAccess(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if strings.Contains(toolkit.TypeName(payload["find"]), "float") { payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto) } tAccess := new(acl.Access) if find := toolkit.ToString(payload["find"]); find != "" { filter = new(dbox.Filter) filter = dbox.Or(dbox.Contains("id", find), dbox.Contains("title", find), dbox.Contains("group1", find), dbox.Contains("group2", find), dbox.Contains("group3", find), dbox.Contains("specialaccess1", find), dbox.Contains("specialaccess2", find), dbox.Contains("specialaccess3", find), dbox.Contains("specialaccess4", find)) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err == nil { err = c.Fetch(&arrm, 0, false) } c.Close() c, err = acl.Find(tAccess, filter, nil) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, data, "") } }
func getConfig() (err error) { var result interface{} if strings.Contains(configpath, "http") { res, err := http.Get(configpath) checkfatalerror(err) defer res.Body.Close() decoder := json.NewDecoder(res.Body) err = decoder.Decode(&result) checkfatalerror(err) } else { bytes, err := ioutil.ReadFile(configpath) checkfatalerror(err) err = json.Unmarshal(bytes, &result) checkfatalerror(err) } switch toolkit.TypeName(result) { case "[]interface {}": isFound := false for _, eachRaw := range result.([]interface{}) { each := eachRaw.(map[string]interface{}) if each["_id"].(string) == _id { m := toolkit.M{} m, err = toolkit.ToM(each) checkfatalerror(err) config = m isFound = true } } if !isFound { checkfatalerror(errors.New(fmt.Sprintf("config with _id %s is not found\n%#v", _id, result))) } case "map[string]interface {}": m := toolkit.M{} m, err = toolkit.ToM(result) checkfatalerror(err) config = m default: checkfatalerror(errors.New(fmt.Sprintf("invalid config file\n%#v", result))) } return }
func (q *Query) DataType(data interface{}) interface{} { if data != nil { rf := toolkit.TypeName(data) // toolkit.Println("data>", rf) if rf == "[]uint8" { uintToString := string(data.([]uint8)) spChar := strings.Contains(uintToString, "\x00") if spChar { uintToString = strings.Replace(uintToString, "\x00", "", 1) } floatVal, e := strconv.ParseFloat(uintToString, 64) if e != nil { } else { data = floatVal } } else { intVal, e := strconv.Atoi(toolkit.ToString(data)) if e != nil { e = nil floatVal, e := strconv.ParseFloat(toolkit.ToString(data), 64) if e != nil { e = nil boolVal, e := strconv.ParseBool(toolkit.ToString(data)) if e != nil { e = nil dateVal, e := time.Parse(q.DateFormat, toolkit.ToString(data)) if e != nil { data = data } else { /*if string is date*/ data = dateVal } } else { /*if string is bool*/ data = boolVal } } else { /*if string is float*/ data = floatVal } } else { /*if string is int*/ data = intVal } } } return data }
func (a *UserController) GetUser(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson _ = a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if strings.Contains(toolkit.TypeName(payload["find"]), "float") { payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto) } tUser := new(acl.User) if find := toolkit.ToString(payload["find"]); find != "" { filter = new(dbox.Filter) filter = dbox.Or(dbox.Contains("id", find), dbox.Contains("fullname", find), dbox.Contains("email", find)) } take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) c, err := acl.Find(tUser, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err != nil { return helper.CreateResult(true, nil, err.Error()) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) err = c.Fetch(&arrm, 0, false) c.Close() c, err = acl.Find(tUser, filter, nil) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } else { return helper.CreateResult(true, data, "") } }
func (a *GroupController) Search(r *knot.WebContext) interface{} { var filter *dbox.Filter r.Config.OutputType = knot.OutputJson _ = a.InitialSetDatabase() payload := map[string]interface{}{} err := r.GetForms(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if strings.Contains(toolkit.TypeName(payload["search"]), "float") { payload["search"] = toolkit.ToInt(payload["search"], toolkit.RoundingAuto) } tGroup := new(acl.Group) if search := toolkit.ToString(payload["search"]); search != "" { filter = new(dbox.Filter) filter = dbox.Or(dbox.Contains("_id", search), dbox.Contains("title", search), dbox.Contains("owner", search)) } fmt.Println(filter) take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto) skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto) c, err := acl.Find(tGroup, filter, toolkit.M{}.Set("take", take).Set("skip", skip)) if err != nil { return helper.CreateResult(true, nil, err.Error()) } data := toolkit.M{} arrm := make([]toolkit.M, 0, 0) err = c.Fetch(&arrm, 0, false) data.Set("Datas", arrm) data.Set("total", c.Count()) if err != nil { return helper.CreateResult(true, nil, err.Error()) } return helper.CreateResult(true, data, "") }
/* ========================================== var payload = { sessionid:"t7AuS0YIE9w8gOWY22HPJaj1pSxEjBNU", accesscheck:[""], accessid:"" }; app.ajaxPost("/acl/authenticate", payload) ============================================= */ func (a *AclController) Authenticate(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson var iaccenum acl.AccessTypeEnum payload := toolkit.M{} result := toolkit.M{} result.Set("hasaccess", false) err := r.GetPayload(&payload) switch { case err != nil: return helper.CreateResult(false, nil, err.Error()) } switch toolkit.TypeName(payload["accesscheck"]) { case "[]interface {}": for _, val := range payload["accesscheck"].([]interface{}) { tacc := acl.GetAccessEnum(toolkit.ToString(val)) if !acl.Matchaccess(int(tacc), int(iaccenum)) { iaccenum += tacc } } default: iaccenum = acl.GetAccessEnum(toolkit.ToString(payload["accesscheck"])) } // toolkit.Println("Type name : ", toolkit.TypeName(payload["accesscheck"])) found := acl.HasAccess(toolkit.ToString(payload["sessionid"]), acl.IDTypeSession, toolkit.ToString(payload["accessid"]), iaccenum) if found { result.Set("hasaccess", found) } return helper.CreateResult(true, result, "") }
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) { // log.Printf("start parse:\n") if !toolkit.IsPointer(m) { // log.Printf("not pointer\n") return errorlib.Error("", "", "Fetch", "Model object should be pointer") } // log.Printf("pointer\n") slice := false var ins []string if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" { slice = true ins = in.([]string) } else { ins = append(ins, in.(string)) } // log.Printf("outputType: %v\n", outputType) if outputType == CSV { var v reflect.Type if slice { v = reflect.TypeOf(m).Elem().Elem() } else { v = reflect.TypeOf(m).Elem() } ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0) for _, data := range ins { appendData := toolkit.M{} iv := reflect.New(v).Interface() reader := csv.NewReader(strings.NewReader("")) if strings.Contains(data, "','") { reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\"")) } else { reader = csv.NewReader(strings.NewReader(data)) } record, e := reader.Read() if e != nil { return e } if v.Kind() == reflect.Struct { for i := 0; i < v.NumField(); i++ { appendData[v.Field(i).Name] = strings.TrimSpace(record[i]) } for i := 0; i < v.NumField(); i++ { tag := v.Field(i).Tag if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) { valthis := appendData[v.Field(i).Name] if valthis == nil { valthis = appendData[tag.Get("tag_name")] } switch v.Field(i).Type.Kind() { case reflect.Int: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int16: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int32: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int64: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Float32: valf, _ := strconv.ParseFloat(valthis.(string), 32) appendData.Set(v.Field(i).Name, valf) case reflect.Float64: valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(v.Field(i).Name, valf) } dtype := DetectFormat(valthis.(string), dateFormat) if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(v.Field(i).Name, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(v.Field(i).Name, valf) } } } } else { for i, val := range header { appendData[val] = strings.TrimSpace(record[i]) } if len(header) == 0 { e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct") return e } for _, val := range header { valthis := appendData[val] dtype := DetectFormat(valthis.(string), dateFormat) if dtype == "int" { appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto)) } else if dtype == "float" { valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(val, valf) } else if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(val, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(val, valf) } } } toolkit.Serde(appendData, iv, JSON) ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem()) } if slice { reflect.ValueOf(m).Elem().Set(ivs) } else { reflect.ValueOf(m).Elem().Set(ivs.Index(0)) } } else if outputType == JSON { var temp interface{} ins = InspectJson(ins) //for catch multi json in one line if JsonPart != "" && slice { for { tempjsonpart := JsonPart JsonPart = "" tempIn := InspectJson([]string{tempjsonpart}) if len(tempIn) == 0 { break } else { for _, tin := range tempIn { ins = append(ins, tin) } } } } forSerde := strings.Join(ins, ",") if slice { forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ",")) } if len(ins) > 0 { e := json.Unmarshal([]byte(forSerde), &temp) if e != nil { return e } e = toolkit.Serde(temp, m, "json") if e != nil { return e } } } else { var v reflect.Type if slice { v = reflect.TypeOf(m).Elem().Elem() } else { v = reflect.TypeOf(m).Elem() } // log.Printf("v: %v\n", v) ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0) // log.Printf("ivs: %v\n", ivs) for _, data := range ins { appendData := toolkit.M{} iv := reflect.New(v).Interface() /*log.Printf("data: %v\n", data) log.Printf("iv: %v\n", iv)*/ splitted := strings.Split(data, "\t") /*log.Printf("appendData: %v\n", appendData) log.Printf("kind: %v\n", v.Kind()) log.Printf("test: %v", fmt.Sprintf("%v", v)) //log.Printf("v.Name: %T\n", v) if fmt.Sprintf("%v", v) == "reflect.Value" { log.Printf("else: %v\n", "reflect.Value") for _, val := range header { log.Printf("val: %v\n", val) valthis := appendData[val] dtype := DetectFormat(valthis.(string), dateFormat) if dtype == "int" { appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto)) } else if dtype == "float" { valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(val, valf) } else if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(val, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(val, valf) } } log.Printf("appendData: %v\n", appendData) } else */ if v.Kind() == reflect.Struct { for i := 0; i < v.NumField(); i++ { appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '")) } // log.Printf("struct: %v\n", v.Kind()) for i := 0; i < v.NumField(); i++ { tag := v.Field(i).Tag // log.Printf("i: %v\n", i) // log.Printf("name: (%v) tag: (%v)\n", appendData.Has(v.Field(i).Name), appendData.Has(tag.Get("tag_name"))) if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) { valthis := appendData[v.Field(i).Name] if valthis == nil { valthis = appendData[tag.Get("tag_name")] } // log.Printf("valthis: %v\n", valthis) switch v.Field(i).Type.Kind() { case reflect.Int: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int16: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int32: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int64: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Float32: valf, _ := strconv.ParseFloat(valthis.(string), 32) appendData.Set(v.Field(i).Name, valf) case reflect.Float64: valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(v.Field(i).Name, valf) } dtype := DetectFormat(valthis.(string), dateFormat) if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(v.Field(i).Name, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(v.Field(i).Name, valf) } } } } else { for i, val := range header { appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '")) } if len(header) == 0 { e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct") return e } // log.Printf("else: %v\n", v.Kind()) for _, val := range header { // log.Printf("val: %v\n", val) valthis := appendData[val] dtype := DetectFormat(valthis.(string), dateFormat) if dtype == "int" { appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto)) } else if dtype == "float" { valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(val, valf) } else if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(val, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(val, valf) } } } toolkit.Serde(appendData, iv, JSON) // log.Printf("iv result: %v\n", iv) ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem()) // log.Printf("ivs result: %v\n", ivs) } if slice { reflect.ValueOf(m).Elem().Set(ivs) } else { reflect.ValueOf(m).Elem().Set(ivs.Index(0)) } // log.Printf("result: %v\n", m) } return nil }
func fetchConfig() (err error) { switch toolkit.ToString(config.Get("sourcetype", "")) { case "SourceType_HttpHtml": SourceType = SourceType_HttpHtml case "SourceType_HttpJson": SourceType = SourceType_HttpJson default: err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", ""))) return } Log.AddLog("Start fetch grabconf", "INFO") if !config.Has("grabconf") { err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error")) return } tconfgrab := toolkit.M{} tconfgrab, err = toolkit.ToM(config["grabconf"]) if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error())) return } grabconfig := sedotan.Config{} if tconfgrab.Has("formvalues") { tfromvalues := toolkit.M{} tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"]) if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error())) return } grabconfig.SetFormValues(tfromvalues) } if tconfgrab.Has("loginvalues") { grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"]) } if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error())) return } grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", "")) grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", "")) grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", "")) grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", "")) grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", "")) grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", "")) grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", "")) Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO") wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig) Log.AddLog("Start fetch datasettings", "INFO") if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") { err = errors.New("Fetch Config, datasettings is not found or have wrong format") return } wGrabber.DataSettings = make(map[string]*sedotan.DataSetting) destDboxs = make(map[string]*DestInfo) for i, xVal := range config["datasettings"].([]interface{}) { err = nil tDataSetting := sedotan.DataSetting{} tDestDbox := DestInfo{} mVal := toolkit.M{} mVal, err = toolkit.ToM(xVal) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR") continue } tnameid := toolkit.ToString(mVal.Get("nameid", "")) if tnameid == "" { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR") continue } tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", "")) // Fetch columnsettings if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR") continue } tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0) for xi, Valcs := range mVal["columnsettings"].([]interface{}) { mValcs := toolkit.M{} mValcs, err = toolkit.ToM(Valcs) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR") continue } tgrabcolumn := sedotan.GrabColumn{} tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", "")) tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", "")) tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", "")) tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", "")) tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto) tDataSetting.Column(tindex, &tgrabcolumn) } //Fetch Filter Condition if mVal.Has("filtercond") { tfiltercond := toolkit.M{} tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{})) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR") } else { tDataSetting.SetFilterCond(tfiltercond) } } //Fetch Connection Info tConnInfo := toolkit.M{} tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{})) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR") continue } tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", "")) tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", "")) tHost := toolkit.ToString(tConnInfo.Get("host", "")) tDatabase := toolkit.ToString(tConnInfo.Get("database", "")) tUserName := toolkit.ToString(tConnInfo.Get("username", "")) tPassword := toolkit.ToString(tConnInfo.Get("password", "")) tSettings := toolkit.M{} tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil)) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR") continue } tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR") continue } tDestDbox.IConnection.Close() destDboxs[tnameid] = &tDestDbox wGrabber.DataSettings[tnameid] = &tDataSetting } err = nil if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 { err = errors.New("Fetch Config, datasettings is not found or have wrong format") return } if !config.Has("histconf") { err = errors.New("Fetch Config, history configuration is not found or have wrong format") return } Log.AddLog("Start fetch histconf", "INFO") histConf, err = toolkit.ToM(config.Get("histconf", nil)) if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") { err = errors.New("Fetch Config, history configuration is not found or have wrong format") return } return }
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) { if !toolkit.IsPointer(m) { return errorlib.Error("", "", "Fetch", "Model object should be pointer") } slice := false var ins []string if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" { slice = true ins = in.([]string) } else { ins = append(ins, in.(string)) } if outputType == CSV { var v reflect.Type if slice && toolkit.TypeName(m) != "*interface {}" { v = reflect.TypeOf(m).Elem().Elem() } else { v = reflect.TypeOf(m).Elem() } ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0) for _, data := range ins { appendData := toolkit.M{} iv := reflect.New(v).Interface() reader := csv.NewReader(strings.NewReader("")) if strings.Contains(data, "','") { reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\"")) } else { reader = csv.NewReader(strings.NewReader(data)) } record, e := reader.Read() if e != nil { return e } if v.Kind() == reflect.Struct { for i := 0; i < v.NumField(); i++ { appendData[v.Field(i).Name] = strings.TrimSpace(record[i]) valthis := appendData[v.Field(i).Name] switch v.Field(i).Type.Kind() { case reflect.Int: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int16: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int32: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int64: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Float32: valf, _ := strconv.ParseFloat(valthis.(string), 32) appendData.Set(v.Field(i).Name, valf) case reflect.Float64: valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(v.Field(i).Name, valf) case reflect.Bool: valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(v.Field(i).Name, valf) default: dtype := DetectDataType(valthis.(string), dateFormat) if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(v.Field(i).Name, valf) } } } } else { if len(header) == 0 { e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct") return e } for i, val := range header { appendData[val] = strings.TrimSpace(record[i]) valthis := appendData[val] dtype := DetectDataType(valthis.(string), dateFormat) if dtype == "int" { appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto)) } else if dtype == "float" { valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(val, valf) } else if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(val, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(val, valf) } } } toolkit.Serde(appendData, iv, JSON) ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem()) } if slice { reflect.ValueOf(m).Elem().Set(ivs) } else { reflect.ValueOf(m).Elem().Set(ivs.Index(0)) } } else if outputType == JSON { var temp interface{} ins = InspectJson(ins) //for catch multi json in one line if JsonPart != "" && slice { for { tempjsonpart := JsonPart JsonPart = "" tempIn := InspectJson([]string{tempjsonpart}) if len(tempIn) == 0 { break } else { for _, tin := range tempIn { ins = append(ins, tin) } } } } forSerde := strings.Join(ins, ",") if slice { forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ",")) } if len(ins) > 0 { e := json.Unmarshal([]byte(forSerde), &temp) if e != nil { return e } e = toolkit.Serde(temp, m, "json") if e != nil { return e } } } else { var v reflect.Type if slice && toolkit.TypeName(m) != "*interface {}" { v = reflect.TypeOf(m).Elem().Elem() } else { v = reflect.TypeOf(m).Elem() } ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0) for _, data := range ins { appendData := toolkit.M{} iv := reflect.New(v).Interface() splitted := strings.Split(data, "\t") if v.Kind() == reflect.Struct { for i := 0; i < v.NumField(); i++ { appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '")) valthis := appendData[v.Field(i).Name] switch v.Field(i).Type.Kind() { case reflect.Int: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int16: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int32: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Int64: appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto)) case reflect.Float32: valf, _ := strconv.ParseFloat(valthis.(string), 32) appendData.Set(v.Field(i).Name, valf) case reflect.Float64: valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(v.Field(i).Name, valf) case reflect.Bool: valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(v.Field(i).Name, valf) default: dtype := DetectDataType(valthis.(string), dateFormat) if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(v.Field(i).Name, valf) } } } } else { if len(header) == 0 { e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct") return e } for i, val := range header { appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '")) valthis := appendData[val] dtype := DetectDataType(valthis.(string), dateFormat) if dtype == "int" { appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto)) } else if dtype == "float" { valf, _ := strconv.ParseFloat(valthis.(string), 64) appendData.Set(val, valf) } else if dtype == "date" { valf := cast.String2Date(valthis.(string), dateFormat) appendData.Set(val, valf) } else if dtype == "bool" { valf, _ := strconv.ParseBool(valthis.(string)) appendData.Set(val, valf) } } } toolkit.Serde(appendData, iv, JSON) ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem()) } if slice { reflect.ValueOf(m).Elem().Set(ivs) } else { reflect.ValueOf(m).Elem().Set(ivs.Index(0)) } } return nil }
func filterCondition(cond toolkit.M) *dbox.Filter { fb := new(dbox.Filter) for key, val := range cond { if key == "$and" || key == "$or" { afb := []*dbox.Filter{} for _, sVal := range val.([]interface{}) { rVal := sVal.(map[string]interface{}) mVal := toolkit.M{} for rKey, mapVal := range rVal { mVal.Set(rKey, mapVal) } afb = append(afb, filterCondition(mVal)) } if key == "$and" { fb = dbox.And(afb...) } else { fb = dbox.Or(afb...) } } else { if toolkit.TypeName(val) == "map[string]interface {}" { mVal := val.(map[string]interface{}) tomVal, _ := toolkit.ToM(mVal) switch { case tomVal.Has("$eq"): fb = dbox.Eq(key, tomVal["$eq"]) case tomVal.Has("$ne"): fb = dbox.Ne(key, tomVal["$ne"]) case tomVal.Has("$regex"): fb = dbox.Contains(key, toolkit.ToString(tomVal["$regex"])) case tomVal.Has("$gt"): fb = dbox.Gt(key, tomVal["$gt"]) case tomVal.Has("$gte"): fb = dbox.Gte(key, tomVal["$gte"]) case tomVal.Has("$lt"): fb = dbox.Lt(key, tomVal["$lt"]) case tomVal.Has("$lte"): fb = dbox.Lte(key, tomVal["$lte"]) case tomVal.Has("$in"): tval := make([]interface{}, 0, 0) if toolkit.TypeName(tomVal["$in"]) == "[]interface {}" { for _, tin := range tomVal["$in"].([]interface{}) { tval = append(tval, tin) } } else { tval = append(tval, tomVal["$in"]) } fb = dbox.In(key, tval...) case tomVal.Has("$nin"): tval := make([]interface{}, 0, 0) if toolkit.TypeName(tomVal["$nin"]) == "[]interface {}" { for _, tin := range tomVal["$nin"].([]interface{}) { tval = append(tval, tin) } } else { tval = append(tval, tomVal["$nin"]) } fb = dbox.Nin(key, tval...) } } else { fb = dbox.Eq(key, val) } } } return fb }
func (d *DataBrowserController) TestQuery(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson data := colonycore.DataBrowser{} //map[string]interface{}{} err := r.GetPayload(&data) if err != nil { return helper.CreateResult(false, nil, err.Error()) } conn, datacon, err := d.connToDatabase(data.ConnectionID) if err != nil { return helper.CreateResult(false, nil, err.Error()) } query, err := d.parseQuery(conn, data, datacon) if err != nil { return helper.CreateResult(false, nil, err.Error()) } cursor, err := query.Cursor(nil) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer cursor.Close() dataFetch := []toolkit.M{} err = cursor.Fetch(&dataFetch, 1, false) if err != nil { cursor.ResetFetch() err = cursor.Fetch(&dataFetch, 0, false) if err != nil { return helper.CreateResult(false, nil, err.Error()) } } metadata := []*colonycore.StructInfo{} // var dt string for i, dataFields := range dataFetch { if i > 0 { break } j := 1 for keyField, dataField := range dataFields { if strings.Contains(keyField, "id") && !strings.Contains(data.QueryText, "id") && !strings.Contains(data.QueryText, "*") && data.TableNames == "" { continue } sInfo := &colonycore.StructInfo{} sInfo.Field = keyField sInfo.Label = keyField rf := "string" if dataField != nil { rf = toolkit.TypeName(dataField) if rf == "time.Time" { rf = "date" } } sInfo.DataType = rf sInfo.Format = "" sInfo.Align = "Left" sInfo.ShowIndex = toolkit.ToInt(j, toolkit.RoundingAuto) sInfo.Sortable = false sInfo.SimpleFilter = false sInfo.AdvanceFilter = false sInfo.Aggregate = "" metadata = append(metadata, sInfo) j++ } } data.MetaData = metadata return helper.CreateResult(true, data, "") }
func fetchConfig() (err error) { switch toolkit.ToString(config.Get("sourcetype", "")) { case "SourceType_DocExcel": SourceType = SourceType_DocExcel case "SourceType_DocMongo": SourceType = SourceType_DocMongo default: err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", ""))) return } Log.AddLog("Start fetch grabconf", "INFO") if !config.Has("grabconf") { err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error")) return } tconfgrab := toolkit.M{} tconfgrab, err = toolkit.ToM(config["grabconf"]) if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error())) return } if !tconfgrab.Has("doctype") { err = errors.New("Fetch Config, doctype not found") return } ci := dbox.ConnectionInfo{} mapconninfo := toolkit.M{} mapconninfo, err = toolkit.ToM(tconfgrab.Get("connectioninfo", nil)) if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo found error : %v", err.Error())) return } ci.Host = toolkit.ToString(mapconninfo.Get("host", "")) ci.Database = toolkit.ToString(mapconninfo.Get("database", "")) ci.UserName = toolkit.ToString(mapconninfo.Get("userName", "")) ci.Password = toolkit.ToString(mapconninfo.Get("password", "")) ci.Settings, err = toolkit.ToM(mapconninfo.Get("settings", nil)) if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo.settings found error : %v", err.Error())) return } sGrabber, err = sedotan.NewGetDatabase(ci.Host, toolkit.ToString(tconfgrab.Get("doctype", "")), &ci) if err != nil { err = errors.New(fmt.Sprintf("Fetch Config, create new get database found error : %v", err.Error())) } Log.AddLog("Start fetch datasettings", "INFO") if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") { err = errors.New("Fetch Config, datasettings is not found or have wrong format") return } sGrabber.CollectionSettings = make(map[string]*sedotan.CollectionSetting) destDboxs = make(map[string]*DestInfo) for i, xVal := range config["datasettings"].([]interface{}) { err = nil tCollectionSetting := sedotan.CollectionSetting{} tDestDbox := DestInfo{} mVal := toolkit.M{} mVal, err = toolkit.ToM(xVal) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR") continue } tnameid := toolkit.ToString(mVal.Get("nameid", "")) if tnameid == "" { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR") continue } tCollectionSetting.Collection = toolkit.ToString(mVal.Get("collection", "")) // Fetch mapssettings if !mVal.Has("mapssettings") || !(toolkit.TypeName(mVal["mapssettings"]) == "[]interface {}") { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : mapssettings is not found or incorrect", i, tnameid), "ERROR") continue } tCollectionSetting.MapsColumns = make([]*sedotan.MapColumn, 0, 0) for xi, Valcs := range mVal["mapssettings"].([]interface{}) { mValcs := toolkit.M{} mValcs, err = toolkit.ToM(Valcs) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : mapssettings is not found or incorrect", i, tnameid, xi), "ERROR") continue } tgrabcolumn := sedotan.MapColumn{} tgrabcolumn.Source = toolkit.ToString(mValcs.Get("source", "")) tgrabcolumn.SType = toolkit.ToString(mValcs.Get("sourcetype", "")) tgrabcolumn.Destination = toolkit.ToString(mValcs.Get("destination", "")) tgrabcolumn.DType = toolkit.ToString(mValcs.Get("destinationtype", "")) // tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto) tCollectionSetting.MapsColumns = append(tCollectionSetting.MapsColumns, &tgrabcolumn) } //Fetch Filter Condition if mVal.Has("filtercond") { tfiltercond := toolkit.M{} tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{})) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR") } else { tCollectionSetting.SetFilterCond(tfiltercond) } } //Fetch limit data if mVal.Has("limitrow") { tLimitrow, err := toolkit.ToM(mVal["limitrow"]) if err == nil { tCollectionSetting.Take = toolkit.ToInt(tLimitrow.Get("take", 0), toolkit.RoundingAuto) tCollectionSetting.Skip = toolkit.ToInt(tLimitrow.Get("skip", 0), toolkit.RoundingAuto) } } //Fetch Connection Info tConnInfo := toolkit.M{} tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{})) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR") continue } tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", "")) tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", "")) tHost := toolkit.ToString(tConnInfo.Get("host", "")) tDatabase := toolkit.ToString(tConnInfo.Get("database", "")) tUserName := toolkit.ToString(tConnInfo.Get("username", "")) tPassword := toolkit.ToString(tConnInfo.Get("password", "")) tSettings := toolkit.M{} tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil)) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR") continue } tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings) if err != nil { Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR") continue } tDestDbox.IConnection.Close() destDboxs[tnameid] = &tDestDbox sGrabber.CollectionSettings[tnameid] = &tCollectionSetting } err = nil if len(destDboxs) == 0 || len(sGrabber.CollectionSettings) == 0 { err = errors.New("Fetch Config, datasettings is not found or have wrong format") return } if !config.Has("histconf") { err = errors.New("Fetch Config, history configuration is not found or have wrong format") return } Log.AddLog("Start fetch histconf", "INFO") histConf, err = toolkit.ToM(config.Get("histconf", nil)) if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") { err = errors.New("Fetch Config, history configuration is not found or have wrong format") return } if !config.Has("extcommand") { return } Log.AddLog("Start fetch extcommand", "INFO") extCommand, _ = toolkit.ToM(config.Get("extcommand", nil)) return }
func checkconfig(cm toolkit.M) (err error) { err = nil if !cm.Has("_id") { err = errors.New(fmt.Sprintf("_id is not found")) return } if !cm.Has("sourcetype") { err = errors.New(fmt.Sprintf("sourcetype is not found")) return } if cm.Has("grabconf") { _, err = toolkit.ToM(cm["grabconf"]) } else { err = errors.New(fmt.Sprintf("grab config is not found")) } if err != nil { return } if cm.Has("intervalconf") { _, err = toolkit.ToM(cm["intervalconf"]) } else { err = errors.New(fmt.Sprintf("interval configuration is not found")) } if err != nil { return } if cm.Has("logconf") { _, err = toolkit.ToM(cm["logconf"]) } else { err = errors.New(fmt.Sprintf("log configuration is not found")) } if err != nil { return } if cm.Has("datasettings") { if toolkit.TypeName(cm["datasettings"]) != "[]interface {}" { err = errors.New(fmt.Sprintf("data settings must be []interface {}")) } } else { err = errors.New(fmt.Sprintf("data settings is not found")) } if err != nil { return } if cm.Has("histconf") { _, err = toolkit.ToM(cm["histconf"]) } else { err = errors.New(fmt.Sprintf("history configuration is not found")) } if err != nil { return } return }
func fetchconfig() { var result interface{} if strings.Contains(configPath, "http") { res, configerr := http.Get(configPath) if configerr != nil { return } defer res.Body.Close() decoder := json.NewDecoder(res.Body) configerr = decoder.Decode(&result) } else { bytes, configerr := ioutil.ReadFile(configPath) if configerr != nil { return } configerr = json.Unmarshal(bytes, &result) } if configerr != nil { return } switch toolkit.TypeName(result) { case "[]interface {}": for i, each := range result.([]interface{}) { m, err := toolkit.ToM(each) if err == nil { err = checkconfig(m) } if err == nil { validconfig += 1 if m.Get("running", false).(bool) { config = append(config, m) } } else { // tstring := fmt.Sprintf("%v;(Config index %d. %v)", configerr, i, err) // configerr = errors.New(fmt.Sprintf("%v", tstring)) Log.AddLog(fmt.Sprintf("Config index %d. %v", i, err), "ERROR") } } case "map[string]interface {}": m, err := toolkit.ToM(result) if err == nil { err = checkconfig(m) } if err == nil { validconfig += 1 if m.Get("running", false).(bool) { config = append(config, m) } } else { Log.AddLog(fmt.Sprintf("Fetch Config Error Found : %v", err), "ERROR") } default: Log.AddLog(fmt.Sprintf("invalid config file\n%#v", result), "ERROR") } }
func getJsonFilePath(o interface{}) string { tns := strings.Split(strings.ToLower(toolkit.TypeName(o)), ".") fn := tns[len(tns)-1] + ".json" return filepath.Join(ConfigPath, fn) }
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error { tableData := []toolkit.M{} var e error h := c.sessionHive if h != nil { e = h.Exec(c.QueryString, func(x hive.HiveResult) error { tableData = append(tableData, x.ResultObj.(map[string]interface{})) return nil }) } else { rows, e := c.session.Query(c.QueryString) var valueType reflect.Type if n == 1 { valueType = reflect.TypeOf(m).Elem() } else { valueType = reflect.TypeOf(m).Elem().Elem() } dataTypeList := toolkit.M{} var isStruct bool if valueType.Kind() == reflect.Struct { for i := 0; i < valueType.NumField(); i++ { namaField := strings.ToLower(valueType.Field(i).Name) dataType := valueType.Field(i).Type.String() dataTypeList.Set(namaField, dataType) } isStruct = true } if e != nil { return e } defer rows.Close() columns, e := rows.Columns() if e != nil { return e } count := len(columns) values := make([]interface{}, count) valuePtrs := make([]interface{}, count) for rows.Next() { for i := 0; i < count; i++ { valuePtrs[i] = &values[i] } rows.Scan(valuePtrs...) entry := toolkit.M{} for i, col := range columns { var v interface{} val := values[i] var ok bool var b []byte if val == nil { v = nil } else { b, ok = val.([]byte) if ok { v = string(b) } else { v = val } } /*mysql always byte, postgres only string that byte, oracle always string, mssql agree with field datatype*/ if (ok && (c.driver == "mysql" || c.driver == "postgres")) || c.driver == "oci8" { if isStruct { v = c.structValue(dataTypeList, col, v) } else { intVal, e := strconv.Atoi(toolkit.ToString(v)) if e != nil { e = nil floatVal, e := strconv.ParseFloat(toolkit.ToString(v), 64) if e != nil { e = nil boolVal, e := strconv.ParseBool(toolkit.ToString(v)) if e != nil { e = nil dateVal, e := time.Parse(c.DateFormat, toolkit.ToString(v)) if e != nil { v = v } else { /*if string is date*/ v = dateVal } } else { /*if string is bool*/ v = boolVal } } else { /*if string is float*/ v = floatVal } } else { /*if string is int*/ v = intVal } } } toolkit.Println(col, toolkit.TypeName(v), v) entry.Set(strings.ToLower(col), v) } tableData = append(tableData, entry) } } maxIndex := toolkit.SliceLen(tableData) if e != nil { return e } end := c.start + n if end > maxIndex || n == 0 { end = maxIndex } if c.start >= maxIndex { e = errors.New("No more data to fetched!") } else { e = toolkit.Serde(tableData[c.start:end], m, "json") } c.start = end return e }
func (d *DataSourceController) parseMetadata(data toolkit.M) []*colonycore.FieldInfo { metadata := []*colonycore.FieldInfo{} for key, val := range data { label := key lookup := new(colonycore.Lookup) lookup.LookupFields = []string{} meta := new(colonycore.FieldInfo) meta.ID = key meta.Label = label meta.Type, _ = helper.GetBetterType(val) meta.Format = "" meta.Lookup = lookup meta.Type = "string" if val != nil { if toolkit.TypeName(val) == "map[string]interface {}" { val, _ = toolkit.ToM(val) } switch toolkit.TypeName(val) { case "toolkit.M": meta.Type = "object" meta.Sub = d.parseMetadata(val.(toolkit.M)) case "[]interface {}": meta.Type = "array" valArray := val.([]interface{}) if len(valArray) > 0 { if toolkit.TypeName(valArray[0]) == "map[string]interface {}" { valArray[0], _ = toolkit.ToM(valArray[0]) } if toolkit.TypeName(valArray[0]) == "toolkit.M" { meta.Type = "array-objects" meta.Sub = d.parseMetadata(valArray[0].(toolkit.M)) } else { switch target := toolkit.TypeName(valArray[0]); { case strings.Contains(target, "interface"): case strings.Contains(target, "string"): meta.Type = "array-string" case strings.Contains(target, "int"): meta.Type = "array-int" case strings.Contains(target, "float"): case strings.Contains(target, "double"): meta.Type = "array-double" default: meta.Type = "array-string" } } } } } metadata = append(metadata, meta) } return metadata }