func (d *DataSourceController) SaveConnection(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } o := new(colonycore.Connection) o.ID = payload["_id"].(string) o.Driver = payload["Driver"].(string) o.Host = payload["Host"].(string) o.Database = payload["Database"].(string) o.UserName = payload["UserName"].(string) o.Password = payload["Password"].(string) o.Settings = d.parseSettings(payload["Settings"], map[string]interface{}{}).(map[string]interface{}) err = colonycore.Delete(o) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if toolkit.HasMember([]string{"csv", "json"}, o.Driver) { if strings.HasPrefix(o.Host, "http") { fileType := helper.GetFileExtension(o.Host) o.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(EC_DATA_PATH, "datasource", "upload", o.ID), fileType) file, err := os.Create(o.FileLocation) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer file.Close() resp, err := http.Get(o.Host) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer resp.Body.Close() _, err = io.Copy(file, resp.Body) if err != nil { return helper.CreateResult(false, nil, err.Error()) } } else { o.FileLocation = o.Host } } err = colonycore.Save(o) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, nil, "") }
func (d *DataSourceController) SaveConnection(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } o := new(colonycore.Connection) o.ID = payload["_id"].(string) o.Driver = payload["Driver"].(string) o.Host = payload["Host"].(string) o.Database = payload["Database"].(string) o.UserName = payload["UserName"].(string) o.Password = payload["Password"].(string) o.Settings = d.parseSettings(payload["Settings"], map[string]interface{}{}).(map[string]interface{}) err = colonycore.Delete(o) if err != nil { return helper.CreateResult(false, nil, err.Error()) } if o.Driver == "weblink" { fileType := helper.GetFileExtension(o.Host) fileLocation := fmt.Sprintf("%s.%s", filepath.Join(AppBasePath, "config", "etc", o.ID), fileType) file, err := os.Create(fileLocation) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer file.Close() resp, err := http.Get(o.Host) if err != nil { return helper.CreateResult(false, nil, err.Error()) } defer resp.Body.Close() _, err = io.Copy(file, resp.Body) if err != nil { return helper.CreateResult(false, nil, err.Error()) } } err = colonycore.Save(o) if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, nil, "") }
func (d *DataSourceController) TestConnection(r *knot.WebContext) interface{} { r.Config.OutputType = knot.OutputJson payload := map[string]interface{}{} err := r.GetPayload(&payload) if err != nil { return helper.CreateResult(false, nil, err.Error()) } driver := payload["Driver"].(string) host := payload["Host"].(string) database := payload["Database"].(string) username := payload["UserName"].(string) password := payload["Password"].(string) var settings toolkit.M = nil if settingsRaw := d.parseSettings(payload["Settings"], nil); settingsRaw != nil { settings, err = toolkit.ToM(settingsRaw) if err != nil { return helper.CreateResult(false, nil, err.Error()) } } if err := d.checkIfDriverIsSupported(driver); err != nil { return helper.CreateResult(false, nil, err.Error()) } fakeDataConn := &colonycore.Connection{ Database: database, Driver: driver, Host: host, UserName: username, Password: password, Settings: settings, } if driver == "json" || driver == "csv" { if strings.HasPrefix(host, "http") { fileTempID := helper.RandomIDWithPrefix("f") fileType := helper.GetFileExtension(host) fakeDataConn.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(EC_DATA_PATH, "datasource", "upload", fileTempID), fileType) file, err := os.Create(fakeDataConn.FileLocation) if err != nil { os.Remove(fakeDataConn.FileLocation) return helper.CreateResult(false, nil, err.Error()) } defer file.Close() resp, err := http.Get(host) if err != nil { os.Remove(fakeDataConn.FileLocation) return helper.CreateResult(false, nil, err.Error()) } defer resp.Body.Close() _, err = io.Copy(file, resp.Body) if err != nil { os.Remove(fakeDataConn.FileLocation) return helper.CreateResult(false, nil, err.Error()) } } else { fakeDataConn.FileLocation = host } } err = helper.ConnectUsingDataConn(fakeDataConn).CheckIfConnected() if fakeDataConn.FileLocation != "" && strings.HasPrefix(host, "http") { os.Remove(fakeDataConn.FileLocation) } if err != nil { return helper.CreateResult(false, nil, err.Error()) } return helper.CreateResult(true, nil, "") }
func (d *DataGrabberController) AutoGenerateDataSources(payload *colonycore.DataGrabberWizardPayload, formatTime string) ([]*colonycore.DataGrabber, error) { var connSource dbox.IConnection var connDest dbox.IConnection var collObjectNames []string var dirpath string isNosql := false result := []*colonycore.DataGrabber{} dataConnSource := new(colonycore.Connection) //pengambilan data untuk mengecek driver destination (mongo, json, csv, sql dan lain-lain) dataConnDest := new(colonycore.Connection) err := colonycore.Get(dataConnDest, payload.ConnectionDestination) if err != nil { return result, err } if !toolkit.HasMember([]string{"mysql", "hive"}, dataConnDest.Driver) { //mengambil nilai object atau tabel yang ada didestination connDest, err = helper.ConnectUsingDataConn(dataConnDest).Connect() if err != nil { return result, err } defer connDest.Close() if toolkit.HasMember([]string{"json", "csv"}, dataConnDest.Driver) { var filedata string dirpath, filedata = filepath.Split(dataConnDest.FileLocation) collObjectNames = []string{strings.Split(filedata, ".")[0]} } else { collObjectNames = connDest.ObjectNames(dbox.ObjTypeAll) } //connection source/from err := colonycore.Get(dataConnSource, payload.ConnectionSource) if err != nil { return result, err } connSource, err = helper.ConnectUsingDataConn(dataConnSource).Connect() if err != nil { return result, err } defer connSource.Close() isNosql = true } for key, each := range payload.Transformations { if tDest := strings.TrimSpace(each.TableDestination); tDest != "" { var connectionIDDest string if isNosql { //pengecekan tidak adanya tabel di connection destination if each.TableDestination != "" && !toolkit.HasMember(collObjectNames, each.TableDestination) { //pengambilan ds metadata sesuai dengan table source var queryS = connSource.NewQuery().Take(1) if !toolkit.HasMember([]string{"csv", "json"}, dataConnSource.Driver) { queryS = queryS.From(each.TableSource) } csr, err := queryS.Cursor(nil) if err != nil { return result, err } defer csr.Close() data := toolkit.M{} if !toolkit.HasMember([]string{"json", "mysql"}, dataConnSource.Driver) { err = csr.Fetch(&data, 1, false) } else { dataAll := []toolkit.M{} err = csr.Fetch(&dataAll, 1, false) if err != nil { return result, err } if len(dataAll) > 0 { data = dataAll[0] } } if toolkit.HasMember([]string{"csv", "json"}, dataConnDest.Driver) { // filepath.WalkFunc o := new(colonycore.Connection) exts := filepath.Ext(dataConnDest.FileLocation) extstrim := strings.TrimPrefix(exts, ".") newpath := filepath.Join(dirpath, each.TableDestination+exts) connectionIDDest = fmt.Sprintf("conn_%s_%s", extstrim, formatTime) o.ID = connectionIDDest o.Driver = extstrim o.Host = newpath if dataConnDest.Driver == "csv" { o.Settings = toolkit.M{"newfile": true, "useheader": true, "delimiter": ","} } else { o.Settings = toolkit.M{"newfile": true} } if strings.HasPrefix(o.Host, "http") { fileType := helper.GetFileExtension(o.Host) o.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(EC_DATA_PATH, "datasource", "upload", o.ID), fileType) file, err := os.Create(o.FileLocation) if err != nil { return nil, err } defer file.Close() resp, err := http.Get(o.Host) if err != nil { return nil, err } defer resp.Body.Close() _, err = io.Copy(file, resp.Body) if err != nil { return nil, err } } else { o.FileLocation = o.Host } err := colonycore.Save(o) if err != nil { return result, err } newconnDest, err := helper.ConnectUsingDataConn(o).Connect() if err != nil { return result, err } defer newconnDest.Close() err = newconnDest.NewQuery().Save().Exec(toolkit.M{"data": data}) if err != nil { return result, err } } else { err = connDest.NewQuery().Save().From(each.TableDestination).Exec(toolkit.M{"data": data}) if err != nil { return result, err } } } } var prevDS string var nextDS string mapGrabber := []*colonycore.Map{} for i := 0; i < 2; i++ { var valueFrom string var connectionID string prefix := "" if t := strings.TrimSpace(payload.Prefix); t != "" { prefix = fmt.Sprintf("%s_", t) } cdsID := fmt.Sprintf("%sDS_%d_%d_%s", prefix, i, key, formatTime) if i == 0 { //table source valueFrom = each.TableSource connectionID = payload.ConnectionSource } else { //table destination valueFrom = each.TableDestination if !toolkit.HasMember([]string{"csv", "json"}, dataConnDest.Driver) { connectionID = payload.ConnectionDestination } else { connectionID = connectionIDDest } } squery := fmt.Sprintf(`{"from":"%s", "select":"*"}`, valueFrom) queryinf := toolkit.M{} json.Unmarshal([]byte(squery), &queryinf) dataDs := []colonycore.DataSource{} cursor, err := colonycore.Find(new(colonycore.DataSource), dbox.Eq("ConnectionID", connectionID)) cursor.Fetch(&dataDs, 0, false) if err != nil { return result, err } defer cursor.Close() dataConn := new(colonycore.Connection) err = colonycore.Get(dataConn, connectionID) if err != nil { return result, err } resultDataRaw := []toolkit.M{} if cursor.Count() > 0 { for _, eachData := range dataDs { resultEachData := toolkit.M{} qFrom := eachData.QueryInfo.Get("from").(string) isSelectExists := eachData.QueryInfo.Has("select") if qFrom == valueFrom && isSelectExists { resultEachData.Set(valueFrom, eachData.ID) resultDataRaw = append(resultDataRaw, resultEachData) if i == 1 { for _, eachMetadata := range eachData.MetaData { mapGrabberField := colonycore.Map{Source: eachMetadata.ID, SourceType: eachMetadata.Type, Destination: eachMetadata.ID, DestinationType: eachMetadata.Type} mapGrabber = append(mapGrabber, &mapGrabberField) } } } } if len(resultDataRaw) == 0 { resultEachData := toolkit.M{} resultEachData.Set(valueFrom, cdsID) resultDataRaw = append(resultDataRaw, resultEachData) cds := new(colonycore.DataSource) cds.ID = cdsID cds.ConnectionID = connectionID cds.MetaData = []*colonycore.FieldInfo{} cds.QueryInfo = queryinf _, metadata, err := CreateDataSourceController(d.Server).DoFetchDataSourceMetaData(dataConn, valueFrom) if err != nil { return result, err } cds.MetaData = metadata err = colonycore.Save(cds) if err != nil { return result, err } if i == 1 { for _, eachMetadata := range metadata { mapGrabberField := colonycore.Map{Source: eachMetadata.ID, SourceType: eachMetadata.Type, Destination: eachMetadata.ID, DestinationType: eachMetadata.Type} mapGrabber = append(mapGrabber, &mapGrabberField) } } } } else { resultEachData := toolkit.M{} resultEachData.Set(valueFrom, cdsID) resultDataRaw = append(resultDataRaw, resultEachData) cds := new(colonycore.DataSource) cds.ID = cdsID cds.ConnectionID = connectionID cds.MetaData = []*colonycore.FieldInfo{} cds.QueryInfo = queryinf _, metadata, err := CreateDataSourceController(d.Server).DoFetchDataSourceMetaData(dataConn, valueFrom) if err != nil { return result, err } cds.MetaData = metadata err = colonycore.Save(cds) if err != nil { return result, err } if i == 1 { for _, eachMetadata := range metadata { mapGrabberField := colonycore.Map{Source: eachMetadata.ID, SourceType: eachMetadata.Type, Destination: eachMetadata.ID, DestinationType: eachMetadata.Type} mapGrabber = append(mapGrabber, &mapGrabberField) } } } for _, resd := range resultDataRaw { if i == 0 { //table source prevDS = resd.Get(valueFrom).(string) break } else { //table destination nextDS = resd.Get(valueFrom).(string) break } } } prefix := "" if t := strings.TrimSpace(payload.Prefix); t != "" { prefix = fmt.Sprintf("%s_", t) } owiz := new(colonycore.DataGrabber) owiz.ID = fmt.Sprintf("%sDG_%s_%s", prefix, strconv.Itoa(key), formatTime) owiz.DataSourceOrigin = prevDS owiz.DataSourceDestination = nextDS owiz.IsFromWizard = true owiz.GrabInterval = 20 owiz.IntervalType = "seconds" owiz.InsertMode = "append" // owiz.Maps = []*colonycore.Map{} owiz.Maps = mapGrabber owiz.PostTransferCommand = "" owiz.PreTransferCommand = "" owiz.TimeoutInterval = 20 owiz.UseInterval = false owiz.RunAt = []string{} err := colonycore.Save(owiz) if err != nil { return result, err } result = append(result, owiz) } } return result, nil }