func (q *Query) HasPartExec() error { var e error var lastJson []toolkit.M q.ReadFile(&lastJson, q.Connection().(*Connection).filePath) if toolkit.SliceLen(lastJson) > 0 { getWhere := []*dbox.Filter{} for _, v := range q.whereData { getWhere = []*dbox.Filter{v} i := dbox.Find(q.sliceData, getWhere) for idSlice, _ := range q.sliceData { if toolkit.HasMember(i, idSlice) { idata := dbox.Find(lastJson, getWhere) for idx, _ := range lastJson { if toolkit.HasMember(idata, idx) { lastJson[idx] = q.sliceData[idSlice] } } if toolkit.SliceLen(idata) == 0 { lastJson = append(lastJson, q.sliceData[idSlice]) // toolkit.Printf("idata>%v\n", q.sliceData[idSlice]) } } } } q.sliceData = lastJson } e = q.WriteFile(q.sliceData) if e != nil { return errorlib.Error(packageName, modQuery+".Exec", "HasPartExec", e.Error()) } return nil }
func (q *Query) Cursor(in toolkit.M) (dbox.ICursor, error) { var cursor *Cursor setting, e := q.prepare(in) if e != nil { return nil, err.Error(packageName, modQuery, "Cursor", e.Error()) } commandtype := setting.GetString("commandtype") if commandtype != dbox.QueryPartSelect { return nil, err.Error(packageName, modQuery, "Cursor", "Cursor is only working with select command, for "+commandtype+" please use .Exec instead") } e = q.openFile(commandtype) if e != nil { return nil, err.Error(packageName, modQuery, "Cursor", e.Error()) } cursor = newCursor(q) skip := 0 if skip = setting.Get("skip").(int); skip > 0 { cursor.skip = skip } take := 0 if take = setting.Get("take").(int); take > 0 { cursor.take = take } if sort := setting.Get("sort").([]string); toolkit.SliceLen(sort) > 0 { fb := new(json.FilterBuilder) sorter := fb.SortFetch(sort, q.data) q.data = sorter } cursor.jsonSelect = setting.Get("fields").([]string) var count int count = toolkit.SliceLen(q.data) where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter) if len(where) > 0 { cursor.where = where cursor.indexes = dbox.Find(q.data, where) count = toolkit.SliceLen(cursor.indexes) } if count <= skip { count = 0 } else { count -= skip } if count >= take && take > 0 { count = take } cursor.count = count return cursor, nil }
func (q *Query) HasPartExec() error { var e error var lastJson []toolkit.M q.ReadFile(&lastJson, q.Connection().(*Connection).filePath) if toolkit.SliceLen(lastJson) > 0 { getWhere := []*dbox.Filter{} for _, v := range q.whereData { getWhere = []*dbox.Filter{v} i := dbox.Find(q.sliceData, getWhere) for idSlice := range q.sliceData { if toolkit.HasMember(i, idSlice) { idata := dbox.Find(lastJson, getWhere) for idx := range lastJson { if toolkit.HasMember(idata, idx) { lastJson[idx] = q.sliceData[idSlice] } } if toolkit.SliceLen(idata) == 0 { lastJson = append(lastJson, q.sliceData[idSlice]) } } } } q.sliceData = lastJson } else { idx := []int{} for _, v := range q.whereData { getWhere := []*dbox.Filter{v} idx = dbox.Find(q.sliceData, getWhere) } // toolkit.Printf("newdata>%v\n", idx) if toolkit.SliceLen(idx) > 1 { newdata := toolkit.M{} for idslice, dataslice := range q.sliceData { if toolkit.HasMember(idx, idslice) { idf, _ := toolkit.IdInfo(dataslice) newdata = q.sliceData[idslice] toolkit.CopyM(&dataslice, &newdata, false, []string{idf}) } } q.sliceData = []toolkit.M{} q.sliceData = append(q.sliceData, newdata) } } e = q.WriteFile(q.sliceData) if e != nil { return errorlib.Error(packageName, modQuery+".Exec", "HasPartExec", e.Error()) } return nil }
func TestSelect(t *testing.T) { t.Skip() c, e := prepareConnection() if e != nil { t.Errorf("Unable to connect %s \n", e.Error()) } defer c.Close() // csr, e := c.NewQuery().Select().From("tes").Where(dbox.Eq("id", "3")).Cursor(nil) csr, e := c.NewQuery(). // Select("empno", "ename", "hiredate"). From(tableCustomers).Cursor(nil) if e != nil { t.Errorf("Cursor pre error: %s \n", e.Error()) return } if csr == nil { t.Errorf("Cursor not initialized") return } defer csr.Close() rets := []toolkit.M{} e = csr.Fetch(&rets, 0, false) if e != nil { t.Errorf("Unable to fetch N: %s \n", e.Error()) } else { toolkit.Printf("Fetch N OK. Result: %v \n", toolkit.JsonString(rets)) toolkit.Printf("Total Fetch OK : %v \n", toolkit.SliceLen(rets)) } }
func TestTakeSkip(t *testing.T) { t.Skip() c, e := prepareConnection() if e != nil { t.Errorf("Unable to connect %s \n", e.Error()) } defer c.Close() csr, e := c.NewQuery(). Select("id", "productname"). From(tableProducts). Take(5). Skip(10). Cursor(nil) if e != nil { t.Errorf("Cursor pre error: %s \n", e.Error()) return } if csr == nil { t.Errorf("Cursor not initialized") return } defer csr.Close() rets := []toolkit.M{} e = csr.Fetch(&rets, 0, false) if e != nil { t.Errorf("Unable to fetch: %s \n", e.Error()) } else { toolkit.Printf("Fetch OK. Result: %v \n", toolkit.JsonString(rets)) toolkit.Printf("Total Record OK. Result: %v \n", toolkit.SliceLen(rets)) } }
func checkDir(basepath string, scanDir string, dirName string) error { dirList, err := ioutil.ReadDir(scanDir) if err != nil { return err } if toolkit.SliceLen(dirList) == 1 { for _, f := range dirList { if f.IsDir() { oldpath := filepath.Join(scanDir, f.Name()) temp_oldpath := filepath.Join(scanDir, dirName) if err := os.Rename(oldpath, temp_oldpath); err != nil { return err } if err := toolkit.ZipCompress(temp_oldpath, scanDir+".zip"); err != nil { return err } if err := os.RemoveAll(scanDir); err != nil { return err } if err := toolkit.ZipExtract(scanDir+".zip", basepath); err != nil { return err } if err := os.Remove(scanDir + ".zip"); err != nil { return err } } } } return nil }
//func Find(ms []toolkit.M, filters []*Filter) (output []int) { func Find(ms interface{}, filters []*Filter) (output []int) { //-- is not a slice if !toolkit.IsSlice(ms) { toolkit.Println("Data is not slice") return []int{} } //toolkit.Printf("Find:%s Filter:%s\n", toolkit.JsonString(ms), toolkit.JsonString(filters)) sliceLen := toolkit.SliceLen(ms) for i := 0; i < sliceLen; i++ { var v toolkit.M item := toolkit.SliceItem(ms, i) e := toolkit.Serde(item, &v, "json") if e == nil { match := MatchM(v, filters) if match { output = append(output, i) } } else { //toolkit.Println("Serde Fail: ", e.Error(), " Data: ", item) } } return }
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error { var source []toolkit.M var lower, upper, lenData, lenIndex int if c.where == nil { lenData = len(c.q.data) if c.currentIndex == 0 { c.maxIndex = lenData } } else { lenIndex = len(c.indexes) if c.currentIndex == 0 { c.maxIndex = lenIndex } } if c.currentIndex == 0 && (c.skip > 0 || c.take > 0) { /*determine max data allowed to be fetched*/ c.maxIndex = c.skip + c.take } lower = c.currentIndex upper = lower + n if c.skip > 0 && c.currentIndex < 1 { lower += c.skip } if n == 0 { if c.where == nil { upper = lenData } else { upper = lenIndex } if c.take > 0 { upper = lower + c.take } } else if n == 1 { upper = lower + 1 } else { upper = lower + n if c.take > 0 && n > c.take { upper = lower + c.take } } if c.where == nil { if toolkit.SliceLen(c.q.data) > 0 { if lower >= lenData { return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!") } if upper >= lenData { upper = lenData } } } else { if toolkit.SliceLen(c.indexes) > 0 { if lower >= lenIndex { return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!") } if upper >= lenIndex { upper = lenIndex } } } if upper >= c.maxIndex { upper = c.maxIndex } if c.where == nil { source = c.q.data[lower:upper] } else if len(c.indexes) > 0 { for _, v := range c.indexes[lower:upper] { /* toolkit.Printf("Add index: %d. Source info now: %s \n", v, func() string { var ret []string for _, id := range source { ret = append(ret, id.Get("_id").(string)) } return strings.Join(ret, ",") }()) */ if v < len(c.q.data) { source = append(source, c.q.data[v]) } } } if toolkit.SliceLen(c.jsonSelect) > 0 { source = getSelected(source, c.jsonSelect) } var e error if n == 1 && !toolkit.IsSlice(m) { if len(source) > 0 { e = toolkit.Serde(&source[0], m, "json") } } else { e = toolkit.Serde(&source, m, "json") } c.currentIndex = upper if e != nil { return errorlib.Error(packageName, modCursor, "Fetch", e.Error()) } //toolkit.Printf("Data: %s\nLower, Upper = %d, %d\nSource: %s\nResult:%s\n\n", toolkit.JsonString(c.q.data), lower, upper, toolkit.JsonString(source), toolkit.JsonString(m)) return nil }
func (q *Query) Exec(in toolkit.M) error { setting, e := q.prepare(in) commandType := setting["commandtype"].(string) //toolkit.Printf("Command type: %s\n", commandType) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error()) } if setting.GetString("commandtype") == dbox.QueryPartSelect { return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead") } q.Lock() defer q.Unlock() var dataM toolkit.M var dataMs []toolkit.M hasData := in.Has("data") dataIsSlice := false data := in.Get("data") if toolkit.IsSlice(data) { dataIsSlice = true e = toolkit.Unjson(toolkit.Jsonify(data), dataMs) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error()) } } else { dataM, e = toolkit.ToM(data) dataMs = append(dataMs, dataM) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error()) } } hasWhere := setting.Has("where") where := setting.Get("where", []*dbox.Filter{}).([]*dbox.Filter) if hasWhere && len(where) == 0 { inWhere := in.Get("where") if inWhere == nil { hasWhere = false where = nil } else { if !toolkit.IsSlice(inWhere) { where = append(where, inWhere.(*dbox.Filter)) } else { where = inWhere.([]*dbox.Filter) } } } if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartDelete, dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) { hasWhere = true //toolkit.Println("check where") if toolkit.IsSlice(data) { ids := []interface{}{} idField := "" if idField == "" { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data send is a slice, but its element has no ID") } dataCount := toolkit.SliceLen(data) for i := 0; i < dataCount; i++ { dataI := toolkit.SliceItem(data, i) if i == 0 { idField = toolkit.IdField(dataI) } ids = append(ids, toolkit.Id(dataI)) } where = []*dbox.Filter{dbox.In(idField, ids)} } else { idfield := "_id" id := toolkit.Id(data) if !toolkit.IsNilOrEmpty(id) { where = []*dbox.Filter{dbox.Eq(idfield, id)} } else { where = nil hasWhere = false } } } /* toolkit.Printf("CommandType: %s HasData: %v HasWhere: %v Where: %s\n", commandType, hasData, hasWhere, toolkit.JsonString(where)) */ e = q.openFile(commandType) //toolkit.Printf(commandType+" Open File, found record: %d\nData:%s\n", len(q.data), toolkit.JsonString(q.data)) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error()) } var indexes []interface{} if hasWhere && commandType != dbox.QueryPartInsert { whereIndex := dbox.Find(q.data, where) indexes = toolkit.ToInterfaceArray(&whereIndex) //toolkit.Printf("Where Index: %s Index:%s\n", toolkit.JsonString(whereIndex), toolkit.JsonString(indexes)) } if commandType == dbox.QueryPartInsert { if !hasData { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty") } if !dataIsSlice { dataMs = []toolkit.M{dataM} } //-- validate for _, datam := range dataMs { idField, idValue := toolkit.IdInfo(datam) toolkit.Serde(dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)}), &indexes, "") if len(indexes) > 0 { return err.Error(packageName, modQuery, "Exec: "+commandType, toolkit.Sprintf("Data %v already exist", idValue)) } } //-- insert the data q.data = append(q.data, dataMs...) } else if commandType == dbox.QueryPartUpdate { //-- valida if !hasData { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty") } var dataUpdate toolkit.M var updateDataIndex int // if it is a slice then we need to update each data passed on its slice isDataSlice := toolkit.IsSlice(data) if isDataSlice == false { isDataSlice = false e = toolkit.Serde(data, &dataUpdate, "") if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail"+e.Error()) } } var idField string //toolkit.Printf("Indexes: %s\n", toolkit.JsonString(indexes)) for i, v := range q.data { // update only data that match given inde if toolkit.HasMember(indexes, i) || !hasWhere { if idField == "" { idField = toolkit.IdField(v) if idField == "" { return err.Error(packageName, modQuery, "Exec: "+commandType, "No ID") } } // If dataslice is sent, iterate f if isDataSlice { e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "") if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, "Serde data fail "+e.Error()) } updateDataIndex++ } dataOrigin := q.data[i] toolkit.CopyM(&dataUpdate, &dataOrigin, false, []string{"_id"}) toolkit.Serde(dataOrigin, &v, "") q.data[i] = v } } } else if commandType == dbox.QueryPartDelete { if hasWhere && len(where) > 0 { indexes := dbox.Find(q.data, where) if len(indexes) > 0 { newdata := []toolkit.M{} for index, v := range q.data { partOfIndex := toolkit.HasMember(indexes, index) if partOfIndex == false { newdata = append(newdata, v) } //toolkit.Println("i:", indexes, ", index:", index, ", p.ofIndex: ", partOfIndex, ", data: ", toolkit.JsonString(newdata)) } q.data = newdata } } else { q.data = []toolkit.M{} } //toolkit.Printf("Data now: %s\n", toolkit.JsonString(q.data)) } else if commandType == dbox.QueryPartSave { if !hasData { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data is empty") } var dataMs []toolkit.M var dataM toolkit.M if !toolkit.IsSlice(data) { e = toolkit.Serde(&data, &dataM, "json") if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error()) } dataMs = append(dataMs, dataM) } else { e = toolkit.Serde(&data, &dataMs, "json") if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType+" Serde data fail", e.Error()) } } //toolkit.Printf("Saving: %s\n", toolkit.JsonString(dataMs)) for _, v := range dataMs { idField, idValue := toolkit.IdInfo(v) indexes := dbox.Find(q.data, []*dbox.Filter{dbox.Eq(idField, idValue)}) if len(indexes) == 0 { q.data = append(q.data, v) } else { dataOrigin := q.data[indexes[0]] //toolkit.Printf("Copy data %s to %s\n", toolkit.JsonString(v), toolkit.JsonString(dataOrigin)) toolkit.CopyM(&v, &dataOrigin, false, []string{idField}) q.data[indexes[0]] = dataOrigin } } } e = q.writeFile() if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType+" Write fail", e.Error()) } return nil }
func (s *SliceBase) Len() int { return toolkit.SliceLen(s.data) }
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error { if closeWhenDone { c.Close() } var source []toolkit.M var lower, upper, lenData, lenIndex int if !c.isWhere { lenData = len(c.datas) if c.lastFetched == 0 { c.maxIndex = lenData } } else { lenIndex = len(c.indexes) if c.lastFetched == 0 { c.maxIndex = lenIndex } } if c.lastFetched == 0 && (c.skip > 0 || c.take > 0) { /*determine max data allowed to be fetched*/ c.maxIndex = c.skip + c.take } lower = c.lastFetched upper = lower + n if c.skip > 0 && c.lastFetched < 1 { lower += c.skip } if n == 0 { if !c.isWhere { upper = lenData } else { upper = lenIndex } if c.take > 0 { upper = lower + c.take } } else if n == 1 { upper = lower + 1 } else { upper = lower + n if c.take > 0 && n > c.take { upper = lower + c.take } } if !c.isWhere { if toolkit.SliceLen(c.datas) > 0 { if lower >= lenData { return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!") } if upper >= lenData { upper = lenData } } } else { if toolkit.SliceLen(c.indexes) > 0 { if lower >= lenIndex { return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!") } if upper >= lenIndex { upper = lenIndex } } } if upper >= c.maxIndex { upper = c.maxIndex } if !c.isWhere { source = c.datas[lower:upper] } else { for _, v := range c.indexes[lower:upper] { if v < len(c.datas) { source = append(source, c.datas[v]) } } } if toolkit.SliceLen(c.jsonSelect) > 0 { source = c.GetSelected(source, c.jsonSelect) } var e error e = toolkit.Serde(&source, m, "json") c.lastFetched = upper if e != nil { return errorlib.Error(packageName, modCursor, "Fetch", e.Error()) } // var first, last int // dataJson := []toolkit.M{} // c.count = len(c.datas) // c.lastFetched = c.count // if c.skip > 0 { // if c.take > 0 { // c.count = c.skip + c.take // } else { // first = c.skip // } // } else { // c.count = c.take // } // if c.take > 0 { // if c.take == c.skip { // first = c.skip // } else { // first = c.count - c.take // } // } // // toolkit.Printf("first = skip>%v last = take>%v lastfetched>%v count>%v\n", first, last, c.lastFetched, c.count) // if n == 0 { // last = c.count // if c.lastFetched <= c.count || c.count == 0 { // last = c.lastFetched // } // } else if n > 0 { // switch { // case c.lastFetched == 0: // last = n // c.lastFetched = n // case n > c.lastFetched || n < c.lastFetched || n == c.lastFetched: // first = c.lastFetched // last = c.lastFetched + n // c.lastFetched = last // if c.lastFetched > c.count { // if first > c.count { // return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!") // } // last = c.count // c.lastFetched = last // } // } // if first > last { // return errorlib.Error(packageName, modCursor, "Fetch", "Wrong fetched data!") // } // } // if c.isWhere { // i := dbox.Find(c.datas, c.whereFields) // c.lastFetched = len(i) // if c.lastFetched < c.count || c.count == 0 { // last = c.lastFetched // } // for _, index := range i[first:last] { // dataJson = append(dataJson, c.datas[index]) // } // } else { // dataJson = c.datas[first:last] // } // if toolkit.SliceLen(c.jsonSelect) > 0 { // dataJson = c.GetSelected(dataJson, c.jsonSelect) // } // e := toolkit.Serde(dataJson, m, "json") // if e != nil { // return errorlib.Error(packageName, modCursor, "Fetch", e.Error()) // } return nil }
func (q *Query) Exec(in toolkit.M) error { setting, e := q.prepare(in) commandType := setting["commandtype"].(string) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, e.Error()) } if setting.GetString("commandtype") == dbox.QueryPartSelect { return err.Error(packageName, modQuery, "Exec: "+commandType, "Exec is not working with select command, please use .Cursor instead") } q.Lock() defer q.Unlock() var dataM toolkit.M var dataMs []toolkit.M hasData := in.Has("data") dataIsSlice := false data := in.Get("data") if toolkit.IsSlice(data) { dataIsSlice = true e = toolkit.Unjson(toolkit.Jsonify(data), dataMs) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error()) } } else { dataM, e = toolkit.ToM(data) dataMs = append(dataMs, dataM) if e != nil { return err.Error(packageName, modQuery, "Exec: "+commandType, "Data encoding error: "+e.Error()) } } hasWhere := in.Has("where") where := in.Get("where", []*dbox.Filter{}).([]*dbox.Filter) if hasData && hasWhere == false && toolkit.HasMember([]interface{}{dbox.QueryPartInsert, dbox.QueryPartUpdate, dbox.QueryPartSave}, commandType) { hasWhere = true if toolkit.IsSlice(data) { ids := []interface{}{} idField := "" if idField == "" { return err.Error(packageName, modQuery, "Exec:"+commandType, "Data send is a slice, but its element has no ID") } dataCount := toolkit.SliceLen(data) for i := 0; i < dataCount; i++ { dataI := toolkit.SliceItem(data, i) if i == 0 { idField = toolkit.IdField(dataI) } ids = append(ids, toolkit.Id(dataI)) } where = []*dbox.Filter{dbox.In(idField, ids)} } else { id := toolkit.Id(data) if toolkit.IsNilOrEmpty(id) { where = []*dbox.Filter{dbox.Eq(toolkit.IdField(id), id)} } else { where = nil hasWhere = false } } } q.openFile() if commandType == dbox.QueryPartInsert { if !hasData { return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty") } if dataIsSlice { q.data = append(q.data, dataMs...) } else { q.data = append(q.data, dataM) } } else if commandType == dbox.QueryPartUpdate { if !hasData { return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty") } var indexes []interface{} if hasWhere { toolkit.Serde(dbox.Find(q.data, where), &indexes, "") } var dataUpdate toolkit.M var updateDataIndex int isDataSlice := toolkit.IsSlice(data) if isDataSlice == false { isDataSlice = false e = toolkit.Serde(data, &dataUpdate, "") if e != nil { return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data. "+e.Error()) } } var idField string for i, v := range q.data { if toolkit.HasMember(indexes, i) || len(indexes) == 0 { if idField == "" { idField = toolkit.IdField(v) if idField == "" { return err.Error(packageName, modQuery, "Exec:"+commandType, "No ID") } } var dataOrigin toolkit.M e = toolkit.Serde(v, &dataOrigin, "") if e != nil { return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data origin. "+e.Error()) } if isDataSlice { e = toolkit.Serde(toolkit.SliceItem(data, updateDataIndex), &dataUpdate, "") if e != nil { return err.Error(packageName, modQuery, "Exec:"+commandType, "Unable to serialize data. "+e.Error()) } updateDataIndex++ } for fieldName, fieldValue := range dataUpdate { if fieldName != idField { if dataOrigin.Has(fieldName) { dataOrigin.Set(fieldName, fieldValue) } } } toolkit.Serde(dataOrigin, &v, "") q.data[i] = v } } } else if commandType == dbox.QueryPartDelete { if hasWhere { var indexes []interface{} toolkit.Serde(dbox.Find(q.data, where), &indexes, "") if len(indexes) > 0 { newdata := []toolkit.M{} for index, v := range q.data { if toolkit.HasMember(indexes, index) == false { newdata = append(newdata, v) } } q.data = newdata } } else { q.data = []toolkit.M{} } } else if commandType == dbox.QueryPartSave { if !hasData { return err.Error(packageName, modQuery, "Exec:"+commandType, "Data is empty") } } q.writeFile() return nil }
func (c *Crowd) Len() int { if c.data == nil { return 0 } return toolkit.SliceLen(c.data) }
func (q *Query) Cursor(in toolkit.M) (dbox.ICursor, error) { var ( e error dataMaps []toolkit.M ) q.ReadFile(&dataMaps, q.Connection().(*Connection).filePath) cursor := dbox.NewCursor(new(Cursor)) cursor = cursor.SetConnection(q.Connection()) filters, e := q.Filters(in) if e != nil { return nil, errorlib.Error(packageName, modQuery, "Cursor", e.Error()) } commandType := filters.GetString("cmdType") if commandType != dbox.QueryPartSelect { return nil, errorlib.Error(packageName, modQuery, "Cursor", "Cursor is only working with select command, for "+commandType+" please use .Exec instead") } aggregate := false hasWhere := filters.Has("where") hasAggregate := filters.Get("aggregate").(bool) if hasAggregate { aggregate = true } if !aggregate { if hasWhere { // toolkit.Println("where:", toolkit.JsonString(filters.Get("where"))) cursor.(*Cursor).whereFields = filters.Get("where").([]*dbox.Filter) cursor.(*Cursor).isWhere = true cursor.(*Cursor).indexes = dbox.Find(dataMaps, filters.Get("where", []*dbox.Filter{}).([]*dbox.Filter)) } // toolkit.Println("skip:", toolkit.JsonString(filters.Get("skip"))) skip := 0 if skip = filters.Get("skip").(int); skip > 0 { cursor.(*Cursor).skip = skip } // toolkit.Println("take:", toolkit.JsonString(filters.Get("take"))) take := 0 if take = filters.Get("take").(int); take > 0 { cursor.(*Cursor).take = take } if sort := filters.Get("sort").([]string); toolkit.SliceLen(sort) > 0 { fb := new(FilterBuilder) // toolkit.Printf("sorter:%v\n", sort) sorter := fb.SortFetch(sort, dataMaps) cursor.(*Cursor).datas = sorter } else { cursor.(*Cursor).datas = dataMaps } var count int if hasWhere { count = toolkit.SliceLen(cursor.(*Cursor).indexes) } else { count = toolkit.SliceLen(cursor.(*Cursor).datas) } if count <= skip { count = 0 } else { count -= skip } if count >= take && take > 0 { count = take } cursor.(*Cursor).count = count cursor.(*Cursor).jsonSelect = filters.Get("select").([]string) } else { return nil, errorlib.Error(packageName, modQuery, "Cursor", "No Aggregate function") } return cursor, nil }
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error { if closeWhenDone { c.Close() } var first, last int dataJson := []toolkit.M{} var datas []toolkit.M toolkit.Unjson(c.readFile, &datas) c.count = len(datas) if n == 0 { last = c.count } else if n > 0 { switch { case c.lastFeteched == 0: last = n c.lastFeteched = n case n > c.lastFeteched || n < c.lastFeteched || n == c.lastFeteched: first = c.lastFeteched last = c.lastFeteched + n c.lastFeteched = last if c.lastFeteched > c.count { if first > c.count { return errorlib.Error(packageName, modCursor, "Fetch", "No more data to fetched!") } last = c.count } // toolkit.Printf("first>%v last>%v lastfetched>%v count>%v\n", first, last, c.lastFeteched, c.count) } } if c.isWhere { i := dbox.Find(datas, c.whereFields) last = len(i) for _, index := range i[first:last] { dataJson = append(dataJson, datas[index]) } } else { dataJson = datas[first:last] } if toolkit.SliceLen(c.jsonSelect) > 0 { var getRemField = toolkit.M{} for _, v := range dataJson { for i, _ := range v { getRemField.Set(i, i) } if c.jsonSelect[0] != "*" { fields := c.removeDuplicatesUnordered(getRemField, c.jsonSelect) for _, field := range fields { v.Unset(field) } } } } e := toolkit.Serde(dataJson, m, "json") if e != nil { return errorlib.Error(packageName, modCursor, "Fetch", e.Error()) } return nil }
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error { tableData := []toolkit.M{} // var e error rows, e := c.session.Query(c.QueryString) var valueType reflect.Type if n == 1 { valueType = reflect.TypeOf(m).Elem() } else { valueType = reflect.TypeOf(m).Elem().Elem() } if e != nil { return e } defer rows.Close() columns, e := rows.Columns() if e != nil { return e } count := len(columns) values := make([]interface{}, count) valuePtrs := make([]interface{}, count) for rows.Next() { for i := 0; i < count; i++ { valuePtrs[i] = &values[i] } rows.Scan(valuePtrs...) entry := toolkit.M{} for i, col := range columns { var v interface{} val := values[i] // b, ok := val.([]byte) // // toolkit.Println("i : ", i, " :col: ", col, " :val: ", val, " :b : ", b, " :type data : ", toolkit.Value(val)) // var out interface{} // e = toolkit.Unjson(b, &out) // // toolkit.Println("i : ", i, "b : ", b, " :out: ", v, " :: error : ", e) // if e != nil { // ok = false // } // if ok { // v = out // toolkit.Println("error OK :: ", ok, " :v :", v) // } else { // toolkit.Println("error OK :: ", ok, " :b :", b) // v = string(b) // } v = val entry.Set(strings.ToLower(col), v) } if valueType.Kind() == reflect.Struct { for i := 0; i < valueType.NumField(); i++ { namaField := strings.ToLower(valueType.Field(i).Name) dataType := strings.ToLower(valueType.Field(i).Type.String()) if entry.Has(namaField) { if strings.Contains(dataType, "int") { entry.Set(namaField, cast.ToInt(entry[namaField], cast.RoundingAuto)) } else if strings.Contains(dataType, "time.time") { entry.Set(namaField, cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05")) } } } } tableData = append(tableData, entry) } // toolkit.Println("... ::: ", tableData) maxIndex := toolkit.SliceLen(tableData) var e2 error if e2 != nil { return e2 } end := c.start + n if end > maxIndex || n == 0 { end = maxIndex } if c.start >= maxIndex { e2 = errors.New("No more data to fetched!") } else { e2 = toolkit.Serde(tableData[c.start:end], m, "json") } c.start = end return e2 }
func (d *DataBrowserController) hasAggr(ctx dbox.IConnection, data *colonycore.DataBrowser, conn *colonycore.Connection) (*colonycore.DataBrowser, error) { var fieldArr, aggrArr []string var indexAggr []map[int]string var query dbox.IQuery fieldAggr := toolkit.M{} for i, v := range data.MetaData { if v.Aggregate != "" { result := toolkit.M{} toolkit.UnjsonFromString(v.Aggregate, &result) cursor := []toolkit.M{} if data.QueryType == "" { aggregate, e := d.dboxAggr(data.TableNames, v.Field, ctx, query, result, fieldAggr, cursor, conn) if e != nil { return nil, e } v.Aggregate = toolkit.JsonString(aggregate) } else if data.QueryType == "SQL" { names := map[int]string{} fieldArr = append(fieldArr, v.Field) if _, sumOK := result["SUM"]; sumOK { aggrArr = append(aggrArr, "SUM("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "sum"}) } else { names[i] = "sum" } } if _, avgOK := result["AVG"]; avgOK { aggrArr = append(aggrArr, "AVG("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "avg"}) } else { names[i] = "avg" } } if _, maxOK := result["MAX"]; maxOK { aggrArr = append(aggrArr, "MAX("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "max"}) } else { names[i] = "max" } } if _, minOK := result["MIN"]; minOK { aggrArr = append(aggrArr, "MIN("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "min"}) } else { names[i] = "min" } } if _, minOK := result["COUNT"]; minOK { aggrArr = append(aggrArr, "COUNT("+v.Field+")") if len(result) > 1 { indexAggr = append(indexAggr, map[int]string{i: "count"}) } else { names[i] = "count" } } if len(result) > 1 { fieldAggr.Set(v.Field, indexAggr) } else { fieldAggr.Set(v.Field, names) } } else if data.QueryType == "Dbox" { getQuery := toolkit.M{} toolkit.UnjsonFromString(data.QueryText, &getQuery) aggregate, e := d.dboxAggr(getQuery.Get("from").(string), v.Field, ctx, query, result, fieldAggr, cursor, conn) if e != nil { return nil, e } v.Aggregate = toolkit.JsonString(aggregate) } } } if data.QueryType == "SQL" { // fieldString := strings.Join(fieldArr, ", ") aggrString := strings.Join(aggrArr, ", ") var queryText string r := regexp.MustCompile(`(([Ff][Rr][Oo][Mm])) (?P<from>([a-zA-Z][_a-zA-Z]+[_a-zA-Z0-1].*))`) temparray := r.FindStringSubmatch(data.QueryText) sqlpart := toolkit.M{} for i, val := range r.SubexpNames() { if val != "" { sqlpart.Set(val, temparray[i]) } } if fromOK := sqlpart.Get("from", "").(string); fromOK != "" { queryText = toolkit.Sprintf("select %s FROM %s", aggrString, sqlpart.Get("from", "").(string)) // toolkit.Printf("queryString:%v\n", queryString) } query = ctx.NewQuery().Command("freequery", toolkit.M{}. Set("syntax", queryText)) csr, e := query.Cursor(nil) if e != nil { return nil, e } defer csr.Close() cursor := []toolkit.M{} e = csr.Fetch(&cursor, 0, false) if e != nil { return nil, e } for f, m := range fieldAggr { aggrData := toolkit.M{} for _, aggs := range cursor { for k, agg := range aggs { if toolkit.SliceLen(m) > 0 { for _, vals := range m.([]map[int]string) { for key, val := range vals { if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) { aggrData.Set(val, agg) data.MetaData[key].Aggregate = toolkit.JsonString(aggrData) } } } } else { for key, val := range m.(map[int]string) { if strings.Contains(k, f) && strings.Contains(k, data.MetaData[key].Field) && strings.Contains(k, val) { aggrData.Set(val, agg) data.MetaData[key].Aggregate = toolkit.JsonString(aggrData) // toolkit.Printf("k:%v f:%v key:%v val:%v agg:%v\n", k, f, key, val, data.MetaData[key].Aggregate) } } } } } } } return data, nil }
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error { tableData := []toolkit.M{} var e error h := c.sessionHive if h != nil { e = h.Exec(c.QueryString, func(x hive.HiveResult) error { tableData = append(tableData, x.ResultObj.(map[string]interface{})) return nil }) } else { rows, e := c.session.Query(c.QueryString) var valueType reflect.Type if n == 1 { valueType = reflect.TypeOf(m).Elem() } else { valueType = reflect.TypeOf(m).Elem().Elem() } dataTypeList := toolkit.M{} var isStruct bool if valueType.Kind() == reflect.Struct { for i := 0; i < valueType.NumField(); i++ { namaField := strings.ToLower(valueType.Field(i).Name) dataType := valueType.Field(i).Type.String() dataTypeList.Set(namaField, dataType) } isStruct = true } if e != nil { return e } defer rows.Close() columns, e := rows.Columns() if e != nil { return e } count := len(columns) values := make([]interface{}, count) valuePtrs := make([]interface{}, count) for rows.Next() { for i := 0; i < count; i++ { valuePtrs[i] = &values[i] } rows.Scan(valuePtrs...) entry := toolkit.M{} for i, col := range columns { var v interface{} val := values[i] var ok bool var b []byte if val == nil { v = nil } else { b, ok = val.([]byte) if ok { v = string(b) } else { v = val } } /*mysql always byte, postgres only string that byte, oracle always string, mssql agree with field datatype*/ if (ok && (c.driver == "mysql" || c.driver == "postgres")) || c.driver == "oci8" { if isStruct { v = c.structValue(dataTypeList, col, v) } else { intVal, e := strconv.Atoi(toolkit.ToString(v)) if e != nil { e = nil floatVal, e := strconv.ParseFloat(toolkit.ToString(v), 64) if e != nil { e = nil boolVal, e := strconv.ParseBool(toolkit.ToString(v)) if e != nil { e = nil dateVal, e := time.Parse(c.DateFormat, toolkit.ToString(v)) if e != nil { v = v } else { /*if string is date*/ v = dateVal } } else { /*if string is bool*/ v = boolVal } } else { /*if string is float*/ v = floatVal } } else { /*if string is int*/ v = intVal } } } toolkit.Println(col, toolkit.TypeName(v), v) entry.Set(strings.ToLower(col), v) } tableData = append(tableData, entry) } } maxIndex := toolkit.SliceLen(tableData) if e != nil { return e } end := c.start + n if end > maxIndex || n == 0 { end = maxIndex } if c.start >= maxIndex { e = errors.New("No more data to fetched!") } else { e = toolkit.Serde(tableData[c.start:end], m, "json") } c.start = end return e }
func (cmd *Command) Exec(c *Crowd) error { if c.data == nil { return errors.New("Exec: Data is empty") } l := c.Len() if cmd.CommandType == CommandSum { fn := cmd.Fns[0] sum := float64(0) for i := 0; i < l; i++ { el := fn(c.Item(i)) if !toolkit.IsNumber(el) { c.Result.Sum = 0 return nil } item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto) sum += item } c.Result.Sum = sum } else if cmd.CommandType == CommandMin { fn := cmd.Fns[0] var ret interface{} for i := 0; i < l; i++ { item := fn(c.Item(i)) if i == 0 { ret = item } else if toolkit.Compare(ret, item, "$gt") { ret = item } } c.Result.Min = ret } else if cmd.CommandType == CommandMax { fn := cmd.Fns[0] var ret interface{} for i := 0; i < l; i++ { item := fn(c.Item(i)) if i == 0 { ret = item } else if toolkit.Compare(ret, item, "$lt") { ret = item } } c.Result.Max = ret } else if cmd.CommandType == CommandAvg { fn := cmd.Fns[0] ret := float64(0) for i := 0; i < l; i++ { el := fn(c.Item(i)) if !toolkit.IsNumber(el) { c.Result.Sum = 0 return nil } item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto) ret += item } c.Result.Avg = toolkit.Div(ret, toolkit.ToFloat64(l, 0, toolkit.RoundingAuto)) } else if cmd.CommandType == CommandWhere { fn := cmd.Fns[0] el, _ := toolkit.GetEmptySliceElement(c.data) tel := reflect.TypeOf(el) array := reflect.MakeSlice(reflect.SliceOf(tel), 0, 0) for i := 0; i < l; i++ { item := c.Item(i) if fn(item).(bool) { //toolkit.Printfn("Where data %d: %v", i, item) array = reflect.Append(array, reflect.ValueOf(item)) } } c.Result.data = array.Interface() } else if cmd.CommandType == CommandApply { fn := cmd.Fns[0] var array reflect.Value for i := 0; i < l; i++ { item := fn(c.Item(i)) //toolkit.Printfn("Applying data %d of %d: %v", i, l, item) if i == 0 { //toolkit.Println(reflect.ValueOf(item).Type().String()) array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(item)), 0, 0) } array = reflect.Append(array, reflect.ValueOf(item)) } c.Result.data = array.Interface() c.data = c.Result.data } else if cmd.CommandType == CommandGroup { fng := cmd.Fns[0] fnc := cmd.Fns[1] mvs := map[interface{}]reflect.Value{} //mvo := map[interface{}]interface{}{} var mvo []KV for i := 0; i < l; i++ { item := c.Item(i) //toolkit.Printfn("Processing data %d of %d: %v", i, l, item) g := fng(item) gi := fnc(item) array, exist := mvs[g] if !exist { //array = []interface{}{} array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(gi)), 0, 0) } array = reflect.Append(array, reflect.ValueOf(gi)) //toolkit.Println("Data:",g,array) mvs[g] = array } for k, v := range mvs { mvo = append(mvo, KV{k, v.Interface()}) } c.Result.data = mvo c.data = mvo } else if cmd.CommandType == CommandSort { sorter, e := NewSorter(c.data, cmd.Fns[0]) if e != nil { return e } var direction SortDirection direction = cmd.Parms.Get("direction", SortAscending).(SortDirection) c.Result.data = sorter.Sort(direction) c.data = c.Result.data } else if cmd.CommandType == CommandJoin { joinData := cmd.Parms.Get("joindata") if joinData == nil { return errors.New("crowd.join: Right side join data is nil") } if cmd.FnJoinKey == nil { cmd.FnJoinKey = func(x, y interface{}) bool { return x == y } } if cmd.FnJoinSelect == nil { cmd.FnJoinSelect = func(x, y interface{}) interface{} { return toolkit.M{}.Set("data1", x).Set("data2", y) } } l1 := toolkit.SliceLen(joinData) var array reflect.Value arrayBuilt := false for i := 0; i < l; i++ { item1 := c.Item(i) for i1 := 0; i1 < l1; i1++ { item2 := toolkit.SliceItem(joinData, i1) joinOK := cmd.FnJoinKey(item1, item2) if joinOK { outObj := cmd.FnJoinSelect(item1, item2) if !arrayBuilt { arrayBuilt = true array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(outObj)), 0, 0) } array = reflect.Append(array, reflect.ValueOf(outObj)) } } } if !arrayBuilt { return errors.New("crowd.join: No match") } c.Result.data = array.Interface() c.data = array.Interface() } else { return errors.New(string(cmd.CommandType) + ": not yet applicable") } return nil }