func (w *WebGrabberController) GetFetchedData(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson w.PrepareHistoryPath() payload := struct { Driver string Host string Database string Collection string Username string Password string }{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } var data []toolkit.M if payload.Driver == "csv" { config := toolkit.M{"useheader": true, "delimiter": ","} query := helper.Query("csv", payload.Host, "", "", "", config) data, err = query.SelectAll("") } else { query := helper.Query("mongo", payload.Host, payload.Database, payload.Username, payload.Password) data, err = query.SelectAll(payload.Collection) } if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, data, "") }
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) { logConf, err := d.getLogger(dataGrabber) if err != nil { logConf.AddLog(err.Error(), "ERROR") defer logConf.Close() } message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType) logConf.AddLog(message, "SUCCESS") fmt.Println(message) dsOrigin := new(colonycore.DataSource) err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } dsDestination := new(colonycore.DataSource) err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } dataDS, _, conn, query, metaSave, err := new(DataSourceController). ConnectToDataSource(dataGrabber.DataSourceOrigin) if len(dataDS.QueryInfo) == 0 { message := "Data source origin has invalid query" logConf.AddLog(message, "ERROR") return false, nil, message } if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } defer conn.Close() if metaSave.keyword != "" { message := `Data source origin query is not "Select"` logConf.AddLog(message, "ERROR") return false, nil, message } cursor, err := query.Cursor(nil) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } defer cursor.Close() data := []toolkit.M{} err = cursor.Fetch(&data, 0, false) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } arrayContains := func(slice []string, key string) bool { for _, each := range slice { if each == key { return true } } return false } connDesc := new(colonycore.Connection) err = colonycore.Get(connDesc, dsDestination.ConnectionID) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } transformedData := []toolkit.M{} for _, each := range data { eachTransformedData := toolkit.M{} for _, eachMeta := range dsDestination.MetaData { if arrayContains(dataGrabber.IgnoreFieldsDestination, eachMeta.ID) { continue } fieldFrom := eachMeta.ID checkMap: for _, eachMap := range dataGrabber.Map { if eachMap.FieldDestination == eachMeta.ID { fieldFrom = eachMap.FieldOrigin break checkMap } } eachTransformedData.Set(eachMeta.ID, each.Get(fieldFrom)) } transformedData = append(transformedData, eachTransformedData) tableName := dsDestination.QueryInfo.GetString("from") queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings) err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.GetString("_id"))) queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings) err = queryWrapper.Save(tableName, eachTransformedData) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } } message = fmt.Sprintf("===> Success transforming %v data", len(transformedData)) logConf.AddLog(message, "SUCCESS") fmt.Println(message) return true, transformedData, "" }
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) { logConf, err := d.getLogger(dataGrabber) if err != nil { logConf.AddLog(err.Error(), "ERROR") defer logConf.Close() } message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType) logConf.AddLog(message, "SUCCESS") fmt.Println(message) dsOrigin := new(colonycore.DataSource) err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } dsDestination := new(colonycore.DataSource) err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } dataDS, _, conn, query, metaSave, err := new(DataSourceController). ConnectToDataSource(dataGrabber.DataSourceOrigin) if len(dataDS.QueryInfo) == 0 { message := "Data source origin has invalid query" logConf.AddLog(message, "ERROR") return false, nil, message } if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } defer conn.Close() if metaSave.keyword != "" { message := `Data source origin query is not "Select"` logConf.AddLog(message, "ERROR") return false, nil, message } cursor, err := query.Cursor(nil) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } defer cursor.Close() data := []toolkit.M{} err = cursor.Fetch(&data, 0, false) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } connDesc := new(colonycore.Connection) err = colonycore.Get(connDesc, dsDestination.ConnectionID) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } const FLAG_ARG_DATA string = `%1` transformedData := []toolkit.M{} for _, each := range data { eachTransformedData := toolkit.M{} for _, eachMap := range dataGrabber.Maps { var valueEachSourceField interface{} // ============================================ SOURCE if !strings.Contains(eachMap.Source, "|") { // source could be: field, object, array valueEachSourceField = each.Get(eachMap.Source) } else { // source could be: field of object, field of array-objects prev := strings.Split(eachMap.Source, "|")[0] next := strings.Split(eachMap.Source, "|")[1] var fieldInfoDes *colonycore.FieldInfo = nil for _, eds := range dsOrigin.MetaData { if eds.ID == prev { fieldInfoDes = eds break } } if fieldInfoDes != nil { // source is field of array-objects if fieldInfoDes.Type == "array-objects" { valueObjects := []interface{}{} if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil { valueObjects = make([]interface{}, len(temp)) for i, each := range temp { if tempSub, _ := toolkit.ToM(each); tempSub != nil { valueObjects[i] = tempSub.Get(next) } } } valueEachSourceField = valueObjects } else { // source is field of object valueObject := toolkit.M{} if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil { valueEachSourceField = valueObject.Get(next) } } } } // ============================================ DESTINATION if !strings.Contains(eachMap.Destination, "|") { if eachMap.SourceType == "object" { sourceObject, _ := toolkit.ToM(valueEachSourceField) if sourceObject == nil { sourceObject = toolkit.M{} } valueObject := toolkit.M{} for _, desMeta := range dsDestination.MetaData { if desMeta.ID == eachMap.Destination { for _, eachMetaSub := range desMeta.Sub { // valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID)) valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType)) } break } } eachTransformedData.Set(eachMap.Destination, valueObject) } else if eachMap.SourceType == "array-objects" { sourceObjects, _ := valueEachSourceField.([]interface{}) if sourceObjects == nil { sourceObjects = []interface{}{} } valueObjects := []interface{}{} for _, sourceObjectRaw := range sourceObjects { sourceObject, _ := toolkit.ToM(sourceObjectRaw) if sourceObject == nil { sourceObject = toolkit.M{} } valueObject := toolkit.M{} for _, desMeta := range dsDestination.MetaData { if desMeta.ID == eachMap.Destination { for _, eachMetaSub := range desMeta.Sub { // valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID)) valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType)) } break } } valueObjects = append(valueObjects, valueObject) } eachTransformedData.Set(eachMap.Destination, valueObjects) } else { if strings.Contains(eachMap.DestinationType, "array") { valueObjects := each.Get(eachMap.Source) eachTransformedData.Set(eachMap.Destination, valueObjects) } else { // eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each)) eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType)) } } } else { prev := strings.Split(eachMap.Destination, "|")[0] next := strings.Split(eachMap.Destination, "|")[1] var fieldInfoDes *colonycore.FieldInfo = nil for _, eds := range dsDestination.MetaData { if eds.ID == prev { fieldInfoDes = eds break } } if fieldInfoDes != nil { if fieldInfoDes.Type == "array-objects" { valueObjects := []interface{}{} if temp := eachTransformedData.Get(prev, nil); temp == nil { valueObjects = []interface{}{} } else { valueObjects, _ = temp.([]interface{}) if valueObjects == nil { valueObjects = []interface{}{} } } if temp, _ := valueEachSourceField.([]interface{}); temp != nil { for i, eachVal := range temp { valueObject := toolkit.M{} if len(valueObjects) > i { if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil { valueObject = temp2 // valueObject.Set(next, eachVal) valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType)) } valueObjects[i] = valueObject } else { if fieldInfoDes.Sub != nil { for _, subMeta := range fieldInfoDes.Sub { valueObject.Set(subMeta.ID, nil) } } // valueObject.Set(next, eachVal) valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType)) valueObjects = append(valueObjects, valueObject) } } } eachTransformedData.Set(prev, valueObjects) } else { valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev)) if valueObject == nil { valueObject = toolkit.M{} } //tambahan prevSource := strings.Split(eachMap.Source, "|")[0] nextSource := strings.Split(eachMap.Source, "|")[1] mval, _ := toolkit.ToM(each.Get(prevSource, nil)) //========= valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType)) // valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval)) eachTransformedData.Set(prev, valueObject) } } } } transformedData = append(transformedData, eachTransformedData) dataToSave := eachTransformedData // ================ pre transfer command if dataGrabber.PreTransferCommand != "" { // jsonTranformedDataBytes, err := json.Marshal(each) jsonTranformedDataBytes, err := json.Marshal(eachTransformedData) if err != nil { return false, nil, err.Error() } jsonTranformedData := string(jsonTranformedDataBytes) var preCommand = dataGrabber.PreTransferCommand if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) { preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1)) } dataToSave = toolkit.M{} output, err := toolkit.RunCommand(preCommand, jsonTranformedData) fmt.Printf("===> Pre Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", preCommand, jsonTranformedData, output) if err == nil { postData := toolkit.M{} if err := json.Unmarshal([]byte(output), &postData); err == nil { dataToSave = postData } } } // ================ if len(dataToSave) == 0 { continue } nilFieldDest := eachTransformedData for _, metadataDest := range dsDestination.MetaData { if temp := eachTransformedData.Get(metadataDest.ID); temp == nil { if metadataDest.ID != "_id" { if metadataDest.Type == "object" { valueObject := toolkit.M{} for _, eachMetaSub := range metadataDest.Sub { valueObject.Set(eachMetaSub.ID, nil) } nilFieldDest.Set(metadataDest.ID, valueObject) } else if metadataDest.Type == "array-objects" { valueEachSourceField := each.Get(metadataDest.ID) sourceObjects, _ := valueEachSourceField.([]interface{}) if sourceObjects == nil { sourceObjects = []interface{}{} } valueObjects := []interface{}{} for _, sourceObjectRaw := range sourceObjects { sourceObject, _ := toolkit.ToM(sourceObjectRaw) if sourceObject == nil { sourceObject = toolkit.M{} } valueObject := toolkit.M{} for keyss, _ := range sourceObject { valueObject.Set(keyss, nil) } valueObjects = append(valueObjects, valueObject) } nilFieldDest.Set(metadataDest.ID, valueObjects) } else { if strings.Contains(metadataDest.Type, "array") { valueObjects := []interface{}{} nilFieldDest.Set(metadataDest.ID, valueObjects) } else { nilFieldDest.Set(metadataDest.ID, nil) } } } } } tableName := dsDestination.QueryInfo.GetString("from") queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings) if dataGrabber.InsertMode == "fresh" { queryWrapper.Delete(tableName, dbox.Or()) } if eachTransformedData.Has("_id") { err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id"))) } if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") { queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings) } else { queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings) } if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" { nilFieldDest.Set("_id", helper.RandomIDWithPrefix("")) } err = queryWrapper.Save(tableName, nilFieldDest) if err != nil { logConf.AddLog(err.Error(), "ERROR") return false, nil, err.Error() } // ================ post transfer command if dataGrabber.PostTransferCommand != "" { eachTransformedData = dataToSave jsonTranformedDataBytes, err := json.Marshal(eachTransformedData) if err != nil { return false, nil, err.Error() } jsonTranformedData := string(jsonTranformedDataBytes) var postCommand = dataGrabber.PostTransferCommand if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) { postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1)) } output, err := toolkit.RunCommand(postCommand, jsonTranformedData) fmt.Printf("===> Post Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", postCommand, jsonTranformedData, output) } } message = fmt.Sprintf("===> Success transforming %v data", len(transformedData)) logConf.AddLog(message, "SUCCESS") fmt.Println(message) return true, transformedData, "" }
func main() { var err error var datastring string var datatemp []interface{} var logs interface{} container := toolkit.M{} dataset := make([]toolkit.M, 0, 0) // fReadType := flag.String("readtype", "", "read type sedotan file") //snapshot,history,rechistory,logfile,[daemonlog] // fLocation := flag.String("pathfile", "", "Full path file location include filename and pattern") //support using environment variable EC_DATA_PATH // fDateTime := flag.String("datetime", "", "Date time for log file") // fTake := flag.Int("take", 0, "take for limit data") // fSkip := flag.Int("skip", 0, "skip for limit data") flag.Parse() tReadType := toolkit.ToString(*fReadType) tLocation = toolkit.ToString(*fLocation) tNameid = toolkit.ToString(*fNameid) tDateTime = toolkit.ToString(*fDateTime) tDatas = toolkit.ToString(*fDatas) //snapshot,history,rechistory,logfile,[daemonlog] //=========== Parse other flag =========== // HERE //======================================== switch tReadType { case "snapshot": module := GetDirSnapshot(tNameid) SnapShot, err := module.OpenSnapShot(tNameid) if err != nil { fmt.Sprintf("ERROR: %s", err) } datastring = toolkit.JsonString(SnapShot) err = toolkit.UnjsonFromString(datastring, &dataset) container.Set("DATA", dataset) case "history": module := NewHistory(tLocation) datatemp, err = module.OpenHistory() if err != nil { fmt.Sprintf("ERROR: %s", err) } datastring = toolkit.JsonString(datatemp) err = toolkit.UnjsonFromString(datastring, &dataset) container.Set("DATA", dataset) case "rechistory": var data []toolkit.M config := toolkit.M{"useheader": true, "delimiter": ","} query := helper.Query("csv", tLocation, "", "", "", config) data, err = query.SelectAll("") container.Set("DATA", data) case "logfile": history := NewHistory(tNameid) err = toolkit.UnjsonFromString(tDatas, &datatemp) logs = history.GetLogHistory(datatemp, tDateTime) container.Set("DATA", logs) case "daemonlog": // logs, err := GetLogs() // if err != nil { // fmt.Sprintf("ERROR: %s", err) // } // datatemp = append(datatemp,logs) // datastring = toolkit.JsonString(datatemp) // container.Set("DATA", datastring) container.Set("DATA", dataset) err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format")) default: container.Set("DATA", dataset) err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format")) } container.Set("ERROR", err) outputstring := toolkit.JsonString(container) fmt.Printf("%s", outputstring) }