func ConnectUsingDataConn(dataConn *colonycore.Connection) *queryWrapper {
if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, dataConn.Driver) && strings.HasPrefix(dataConn.Host, "http") {
if toolkit.IsFileExist(dataConn.FileLocation) || dataConn.FileLocation == "" {
fileTempID := RandomIDWithPrefix("f")
fileType := GetFileExtension(dataConn.Host)
dataConn.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(os.Getenv("EC_DATA_PATH"), "datasource", "upload", fileTempID), fileType)
file, err := os.Create(dataConn.FileLocation)
if err != nil {
os.Remove(dataConn.FileLocation)
} else {
defer file.Close()
}
resp, err := http.Get(dataConn.Host)
if err != nil {
os.Remove(dataConn.FileLocation)
} else {
defer resp.Body.Close()
}
_, err = io.Copy(file, resp.Body)
if err != nil {
os.Remove(dataConn.FileLocation)
}
colonycore.Save(dataConn)
}
return Query(dataConn.Driver, dataConn.FileLocation, "", "", "", dataConn.Settings)
}
return Query(dataConn.Driver, dataConn.Host, dataConn.Database, dataConn.UserName, dataConn.Password, dataConn.Settings)
}
func (d *DataGrabberController) StartTransformation(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
mutex.Lock()
dataGrabber := new(colonycore.DataGrabber)
err := r.GetPayload(dataGrabber)
if err != nil {
mutex.Unlock()
return helper.CreateResult(false, nil, err.Error())
}
err = colonycore.Get(dataGrabber, dataGrabber.ID)
if err != nil {
mutex.Unlock()
return helper.CreateResult(false, nil, err.Error())
}
if _, ok := serviceHolder[dataGrabber.ID]; ok {
serviceHolder[dataGrabber.ID] = false
delete(serviceHolder, dataGrabber.ID)
message := fmt.Sprintf("===> Transformation stopped! %s -> %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination)
fmt.Println(message)
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
logConf.AddLog(message, "SUCCESS")
} else {
message := fmt.Sprintf("===> Starting transform! %s -> %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination)
fmt.Println(message)
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
logConf.AddLog(message, "SUCCESS")
}
yo := func() {
logAt := time.Now().Format("20060102-150405")
logConfTransformation, err := d.getLogger(dataGrabber)
if err != nil {
logConfTransformation.AddLog(err.Error(), "ERROR")
defer logConfTransformation.Close()
}
success, data, message := d.Transform(dataGrabber)
_, _, _ = success, data, message
// timeout not yet implemented
dataPath := filepath.Join(dgOutputPath, fmt.Sprintf("%s.json", logAt))
if toolkit.IsFileExist(dataPath) {
if err = os.Remove(dataPath); err != nil {
logConfTransformation.AddLog(err.Error(), "ERROR")
}
}
if _, err = os.Create(dataPath); err != nil {
logConfTransformation.AddLog(err.Error(), "ERROR")
} else {
if bytes, err := json.Marshal(data); err != nil {
logConfTransformation.AddLog(err.Error(), "ERROR")
} else {
if err = ioutil.WriteFile(dataPath, bytes, 0666); err != nil {
logConfTransformation.AddLog(err.Error(), "ERROR")
}
}
}
}
yo()
if !dataGrabber.UseInterval {
mutex.Unlock()
return helper.CreateResult(true, nil, "")
}
serviceHolder[dataGrabber.ID] = true
go func(dg *colonycore.DataGrabber) {
for true {
if flag, _ := serviceHolder[dg.ID]; !flag {
break
}
var interval time.Duration
switch dataGrabber.IntervalType {
case "seconds":
interval = time.Duration(dataGrabber.GrabInterval) * time.Second
case "minutes":
interval = time.Duration(dataGrabber.GrabInterval) * time.Minute
case "hours":
interval = time.Duration(dataGrabber.GrabInterval) * time.Hour
}
time.Sleep(interval)
yo()
}
}(dataGrabber)
mutex.Unlock()
return helper.CreateResult(true, nil, "")
}
func (g *GrabService) execService() {
g.LastGrabStat = false
go func(g *GrabService) {
for g.ServiceRunningStat {
if g.LastGrabStat {
<-time.After(g.GrabInterval)
} else {
<-time.After(g.TimeOutInterval)
}
if !g.ServiceRunningStat {
continue
}
g.ErrorNotes = ""
g.LastGrabExe = time.Now()
g.NextGrabExe = time.Now().Add(g.GrabInterval)
g.LastGrabStat = true
g.Log.AddLog(fmt.Sprintf("[%s] Grab Started %s", g.Name, g.Url), "INFO")
g.GrabCount += 1
keySetting := []string{}
switch g.SourceType {
case SourceType_HttpHtml, SourceType_HttpJson:
if e := g.ServGrabber.Grab(nil); e != nil {
g.ErrorNotes = fmt.Sprintf("[%s] Grab Failed %s, repeat after %s :%s", g.Name, g.Url, g.TimeOutIntervalInfo, e)
g.Log.AddLog(g.ErrorNotes, "ERROR")
g.NextGrabExe = time.Now().Add(g.TimeOutInterval)
g.LastGrabStat = false
g.ErrorFound += 1
} else {
g.Log.AddLog(fmt.Sprintf("[%s] Grab Success %s", g.Name, g.Url), "INFO")
}
for key, _ := range g.ServGrabber.Config.DataSettings {
keySetting = append(keySetting, key)
}
// keySetting = g.sGrabber.Config.DataSettings
case SourceType_DocExcel:
// e = g.sGetData.ResultFromDatabase(key, &docs)
// if e != nil {
// g.LastGrabStat = false
// }
for key, _ := range g.ServGetData.CollectionSettings {
keySetting = append(keySetting, key)
}
}
// if e := g.ServGrabber.Grab(nil); e != nil {
// g.ErrorNotes = fmt.Sprintf("[%s] Grab Failed %s, repeat after %s :%s", g.Name, g.Url, g.TimeOutIntervalInfo, e)
// g.Log.AddLog(g.ErrorNotes, "ERROR")
// g.NextGrabExe = time.Now().Add(g.TimeOutInterval)
// g.LastGrabStat = false
// g.ErrorFound += 1
// continue
// } else {
// g.Log.AddLog(fmt.Sprintf("[%s] Grab Success %s", g.Name, g.Url), "INFO")
// }
if g.LastGrabStat {
for _, key := range keySetting {
var e error
g.Log.AddLog(fmt.Sprintf("[%s-%s] Fetch Data to destination started", g.Name, key), "INFO")
docs := []toolkit.M{}
switch g.SourceType {
case SourceType_HttpHtml, SourceType_HttpJson:
e = g.ServGrabber.ResultFromHtml(key, &docs)
case SourceType_DocExcel:
e = g.ServGetData.ResultFromDatabase(key, &docs)
if e != nil {
g.LastGrabStat = false
}
}
if e != nil || !(g.LastGrabStat) {
g.ErrorNotes = fmt.Sprintf("[%s-%s] Fetch Result Failed : ", g.Name, key, e)
g.Log.AddLog(g.ErrorNotes, "ERROR")
}
// cannot insert data into output file, not sure why
// solution: if the file is empty, then it's need to be deleted first
if g.DestDbox[key].Desttype == "csv" {
outputCSVpath := g.DestDbox[key].IConnection.Info().Host
if toolkit.IsFileExist(outputCSVpath) {
outputCSVcontentBytes, e := ioutil.ReadFile(outputCSVpath)
if e != nil {
g.ErrorNotes = fmt.Sprintf("[%s-%s] Connect to destination failed [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
g.Log.AddLog(g.ErrorNotes, "ERROR")
}
if strings.TrimSpace(string(outputCSVcontentBytes)) == "" {
os.Remove(outputCSVpath)
}
}
}
e = g.DestDbox[key].IConnection.Connect()
if e != nil {
g.ErrorNotes = fmt.Sprintf("[%s-%s] Connect to destination failed [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
g.Log.AddLog(g.ErrorNotes, "ERROR")
}
var q dbox.IQuery
if g.DestDbox[key].Collection == "" {
q = g.DestDbox[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
} else {
q = g.DestDbox[key].IConnection.NewQuery().SetConfig("multiexec", true).From(g.DestDbox[key].Collection).Save()
}
if g.DestDbox[key].Desttype == "csv" {
q = q.Insert()
}
xN := 0
iN := 0
for _, doc := range docs {
for key, val := range doc {
doc[key] = strings.TrimSpace(fmt.Sprintf("%s", val))
}
if g.DestDbox[key].Desttype == "mongo" {
doc["_id"] = toolkit.GenerateRandomString("1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnpqrstuvwxyz", 32)
}
e = q.Exec(toolkit.M{
"data": doc,
})
if g.DestDbox[key].Desttype == "mongo" {
delete(doc, "_id")
}
if e != nil {
g.ErrorNotes = fmt.Sprintf("[%s-%s] Unable to insert [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
g.Log.AddLog(g.ErrorNotes, "ERROR")
g.ErrorFound += 1
} else {
iN += 1
}
xN++
}
g.RowGrabbed += xN
q.Close()
g.DestDbox[key].IConnection.Close()
g.Log.AddLog(fmt.Sprintf("[%s-%s] Fetch Data to destination finished with %d record fetch", g.Name, key, xN), "INFO")
if g.HistoryPath != "" && g.HistoryRecPath != "" {
recfile := g.AddRecHistory(key, docs)
historyservice := toolkit.M{}.Set("datasettingname", key).Set("grabdate", g.LastGrabExe).Set("rowgrabbed", g.RowGrabbed).
Set("rowsaved", iN).Set("note", g.ErrorNotes).Set("grabstatus", "SUCCESS").Set("recfile", recfile)
if !(g.LastGrabStat) {
historyservice.Set("grabstatus", "FAILED")
}
g.AddHistory(historyservice)
}
}
} else {
if g.HistoryPath != "" {
historyservice := toolkit.M{}.Set("datasettingname", "-").Set("grabdate", g.LastGrabExe).Set("rowgrabbed", g.RowGrabbed).
Set("rowsaved", 0).Set("note", g.ErrorNotes).Set("grabstatus", "FAILED").Set("recfile", "")
g.AddHistory(historyservice)
}
}
}
}(g)
}
