Пример #1
0
func ConnectUsingDataConn(dataConn *colonycore.Connection) *queryWrapper {
	if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, dataConn.Driver) && strings.HasPrefix(dataConn.Host, "http") {

		if toolkit.IsFileExist(dataConn.FileLocation) || dataConn.FileLocation == "" {
			fileTempID := RandomIDWithPrefix("f")
			fileType := GetFileExtension(dataConn.Host)
			dataConn.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(os.Getenv("EC_DATA_PATH"), "datasource", "upload", fileTempID), fileType)

			file, err := os.Create(dataConn.FileLocation)
			if err != nil {
				os.Remove(dataConn.FileLocation)
			} else {
				defer file.Close()
			}

			resp, err := http.Get(dataConn.Host)
			if err != nil {
				os.Remove(dataConn.FileLocation)
			} else {
				defer resp.Body.Close()
			}

			_, err = io.Copy(file, resp.Body)
			if err != nil {
				os.Remove(dataConn.FileLocation)
			}

			colonycore.Save(dataConn)
		}

		return Query(dataConn.Driver, dataConn.FileLocation, "", "", "", dataConn.Settings)
	}

	return Query(dataConn.Driver, dataConn.Host, dataConn.Database, dataConn.UserName, dataConn.Password, dataConn.Settings)
}
Пример #2
0
func (d *DataGrabberController) StartTransformation(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	mutex.Lock()

	dataGrabber := new(colonycore.DataGrabber)
	err := r.GetPayload(dataGrabber)
	if err != nil {
		mutex.Unlock()
		return helper.CreateResult(false, nil, err.Error())
	}

	err = colonycore.Get(dataGrabber, dataGrabber.ID)
	if err != nil {
		mutex.Unlock()
		return helper.CreateResult(false, nil, err.Error())
	}

	if _, ok := serviceHolder[dataGrabber.ID]; ok {
		serviceHolder[dataGrabber.ID] = false
		delete(serviceHolder, dataGrabber.ID)

		message := fmt.Sprintf("===> Transformation stopped! %s -> %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination)
		fmt.Println(message)

		logConf, err := d.getLogger(dataGrabber)
		if err != nil {
			logConf.AddLog(err.Error(), "ERROR")
			defer logConf.Close()
		}

		logConf.AddLog(message, "SUCCESS")
	} else {
		message := fmt.Sprintf("===> Starting transform! %s -> %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination)
		fmt.Println(message)

		logConf, err := d.getLogger(dataGrabber)
		if err != nil {
			logConf.AddLog(err.Error(), "ERROR")
			defer logConf.Close()
		}

		logConf.AddLog(message, "SUCCESS")
	}

	yo := func() {
		logAt := time.Now().Format("20060102-150405")
		logConfTransformation, err := d.getLogger(dataGrabber)
		if err != nil {
			logConfTransformation.AddLog(err.Error(), "ERROR")
			defer logConfTransformation.Close()
		}

		success, data, message := d.Transform(dataGrabber)
		_, _, _ = success, data, message
		// timeout not yet implemented

		dataPath := filepath.Join(dgOutputPath, fmt.Sprintf("%s.json", logAt))
		if toolkit.IsFileExist(dataPath) {
			if err = os.Remove(dataPath); err != nil {
				logConfTransformation.AddLog(err.Error(), "ERROR")
			}
		}

		if _, err = os.Create(dataPath); err != nil {
			logConfTransformation.AddLog(err.Error(), "ERROR")
		} else {
			if bytes, err := json.Marshal(data); err != nil {
				logConfTransformation.AddLog(err.Error(), "ERROR")
			} else {
				if err = ioutil.WriteFile(dataPath, bytes, 0666); err != nil {
					logConfTransformation.AddLog(err.Error(), "ERROR")
				}
			}
		}
	}
	yo()

	if !dataGrabber.UseInterval {
		mutex.Unlock()
		return helper.CreateResult(true, nil, "")
	}

	serviceHolder[dataGrabber.ID] = true

	go func(dg *colonycore.DataGrabber) {
		for true {
			if flag, _ := serviceHolder[dg.ID]; !flag {
				break
			}

			var interval time.Duration
			switch dataGrabber.IntervalType {
			case "seconds":
				interval = time.Duration(dataGrabber.GrabInterval) * time.Second
			case "minutes":
				interval = time.Duration(dataGrabber.GrabInterval) * time.Minute
			case "hours":
				interval = time.Duration(dataGrabber.GrabInterval) * time.Hour
			}

			time.Sleep(interval)
			yo()
		}
	}(dataGrabber)

	mutex.Unlock()
	return helper.CreateResult(true, nil, "")
}
Пример #3
0
func (g *GrabService) execService() {
	g.LastGrabStat = false
	go func(g *GrabService) {
		for g.ServiceRunningStat {

			if g.LastGrabStat {
				<-time.After(g.GrabInterval)
			} else {
				<-time.After(g.TimeOutInterval)
			}

			if !g.ServiceRunningStat {
				continue
			}

			g.ErrorNotes = ""
			g.LastGrabExe = time.Now()
			g.NextGrabExe = time.Now().Add(g.GrabInterval)
			g.LastGrabStat = true
			g.Log.AddLog(fmt.Sprintf("[%s] Grab Started %s", g.Name, g.Url), "INFO")
			g.GrabCount += 1

			keySetting := []string{}
			switch g.SourceType {
			case SourceType_HttpHtml, SourceType_HttpJson:
				if e := g.ServGrabber.Grab(nil); e != nil {
					g.ErrorNotes = fmt.Sprintf("[%s] Grab Failed %s, repeat after %s :%s", g.Name, g.Url, g.TimeOutIntervalInfo, e)
					g.Log.AddLog(g.ErrorNotes, "ERROR")
					g.NextGrabExe = time.Now().Add(g.TimeOutInterval)
					g.LastGrabStat = false
					g.ErrorFound += 1
				} else {
					g.Log.AddLog(fmt.Sprintf("[%s] Grab Success %s", g.Name, g.Url), "INFO")
				}
				for key, _ := range g.ServGrabber.Config.DataSettings {
					keySetting = append(keySetting, key)
				}
				// keySetting = g.sGrabber.Config.DataSettings
			case SourceType_DocExcel:
				// e = g.sGetData.ResultFromDatabase(key, &docs)
				// if e != nil {
				// 	g.LastGrabStat = false
				// }
				for key, _ := range g.ServGetData.CollectionSettings {
					keySetting = append(keySetting, key)
				}
			}

			// if e := g.ServGrabber.Grab(nil); e != nil {
			// 	g.ErrorNotes = fmt.Sprintf("[%s] Grab Failed %s, repeat after %s :%s", g.Name, g.Url, g.TimeOutIntervalInfo, e)
			// 	g.Log.AddLog(g.ErrorNotes, "ERROR")
			// 	g.NextGrabExe = time.Now().Add(g.TimeOutInterval)
			// 	g.LastGrabStat = false
			// 	g.ErrorFound += 1
			// 	continue
			// } else {
			// 	g.Log.AddLog(fmt.Sprintf("[%s] Grab Success %s", g.Name, g.Url), "INFO")
			// }

			if g.LastGrabStat {
				for _, key := range keySetting {
					var e error
					g.Log.AddLog(fmt.Sprintf("[%s-%s] Fetch Data to destination started", g.Name, key), "INFO")

					docs := []toolkit.M{}
					switch g.SourceType {
					case SourceType_HttpHtml, SourceType_HttpJson:
						e = g.ServGrabber.ResultFromHtml(key, &docs)
					case SourceType_DocExcel:
						e = g.ServGetData.ResultFromDatabase(key, &docs)
						if e != nil {
							g.LastGrabStat = false
						}
					}
					if e != nil || !(g.LastGrabStat) {
						g.ErrorNotes = fmt.Sprintf("[%s-%s] Fetch Result Failed : ", g.Name, key, e)
						g.Log.AddLog(g.ErrorNotes, "ERROR")
					}

					// cannot insert data into output file, not sure why
					// solution: if the file is empty, then it's need to be deleted first
					if g.DestDbox[key].Desttype == "csv" {
						outputCSVpath := g.DestDbox[key].IConnection.Info().Host
						if toolkit.IsFileExist(outputCSVpath) {
							outputCSVcontentBytes, e := ioutil.ReadFile(outputCSVpath)
							if e != nil {
								g.ErrorNotes = fmt.Sprintf("[%s-%s] Connect to destination failed [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
								g.Log.AddLog(g.ErrorNotes, "ERROR")
							}

							if strings.TrimSpace(string(outputCSVcontentBytes)) == "" {
								os.Remove(outputCSVpath)
							}
						}
					}

					e = g.DestDbox[key].IConnection.Connect()
					if e != nil {
						g.ErrorNotes = fmt.Sprintf("[%s-%s] Connect to destination failed [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
						g.Log.AddLog(g.ErrorNotes, "ERROR")
					}

					var q dbox.IQuery
					if g.DestDbox[key].Collection == "" {
						q = g.DestDbox[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
					} else {
						q = g.DestDbox[key].IConnection.NewQuery().SetConfig("multiexec", true).From(g.DestDbox[key].Collection).Save()
					}

					if g.DestDbox[key].Desttype == "csv" {
						q = q.Insert()
					}

					xN := 0
					iN := 0
					for _, doc := range docs {
						for key, val := range doc {
							doc[key] = strings.TrimSpace(fmt.Sprintf("%s", val))
						}

						if g.DestDbox[key].Desttype == "mongo" {
							doc["_id"] = toolkit.GenerateRandomString("1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnpqrstuvwxyz", 32)
						}

						e = q.Exec(toolkit.M{
							"data": doc,
						})

						if g.DestDbox[key].Desttype == "mongo" {
							delete(doc, "_id")
						}

						if e != nil {
							g.ErrorNotes = fmt.Sprintf("[%s-%s] Unable to insert [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
							g.Log.AddLog(g.ErrorNotes, "ERROR")
							g.ErrorFound += 1
						} else {
							iN += 1
						}
						xN++
					}
					g.RowGrabbed += xN
					q.Close()
					g.DestDbox[key].IConnection.Close()

					g.Log.AddLog(fmt.Sprintf("[%s-%s] Fetch Data to destination finished with %d record fetch", g.Name, key, xN), "INFO")

					if g.HistoryPath != "" && g.HistoryRecPath != "" {
						recfile := g.AddRecHistory(key, docs)
						historyservice := toolkit.M{}.Set("datasettingname", key).Set("grabdate", g.LastGrabExe).Set("rowgrabbed", g.RowGrabbed).
							Set("rowsaved", iN).Set("note", g.ErrorNotes).Set("grabstatus", "SUCCESS").Set("recfile", recfile)
						if !(g.LastGrabStat) {
							historyservice.Set("grabstatus", "FAILED")
						}
						g.AddHistory(historyservice)
					}
				}
			} else {
				if g.HistoryPath != "" {
					historyservice := toolkit.M{}.Set("datasettingname", "-").Set("grabdate", g.LastGrabExe).Set("rowgrabbed", g.RowGrabbed).
						Set("rowsaved", 0).Set("note", g.ErrorNotes).Set("grabstatus", "FAILED").Set("recfile", "")
					g.AddHistory(historyservice)
				}
			}
		}
	}(g)
}