// test worker
func TestWorker(t *testing.T) {
file, _ := os.Open("worker_test.txt")
defer file.Close()
scanner := bufio.NewScanner(file)
// initialize manager and workers
totalworker := 100
manager := w.NewManager(totalworker)
for i := 0; i < totalworker; i++ {
manager.FreeWorkers <- &w.Worker{i, manager.TimeProcess, manager.FreeWorkers}
}
// monitoring worker thats free
go manager.DoMonitor()
// reading file
for scanner.Scan() {
// getting data per line
data := scanner.Text()
// send task to free worker
manager.Tasks <- func() {
// do something here
fmt.Println(data)
}
}
// waiting for tasks has been done
go manager.Timeout(0)
<-manager.Done
}
// loading file with worker
func (h *Hive) LoadFileWithWorker(FilePath, TableName, fileType string, TableModel interface{}, TotalWorker int) (retVal string, err error) {
retVal = "process failed"
isMatch := false
hr, err := h.fetch("select '1' from " + TableName + " limit 1;")
if err != nil {
return retVal, err
}
if hr.Result == nil {
tempQuery := ""
var v reflect.Type
v = reflect.TypeOf(TableModel).Elem()
if v.Kind() == reflect.Struct {
tempQuery = "create table " + TableName + " ("
for i := 0; i < v.NumField(); i++ {
if i == (v.NumField() - 1) {
tempQuery += v.Field(i).Name + " " + v.Field(i).Type.String() + ");"
} else {
tempQuery += v.Field(i).Name + " " + v.Field(i).Type.String() + ", "
}
}
_, err = h.fetch(tempQuery)
}
}
isMatch, err = h.CheckDataStructure(TableName, TableModel)
if isMatch == false {
return retVal, err
}
if err == nil {
file, err := os.Open(FilePath)
if err != nil {
fmt.Println(err)
}
defer file.Close()
if err != nil {
log.Println(err)
}
scanner := bufio.NewScanner(file)
// initiate dispatcher
manager := wk.NewManager(TotalWorker)
// initiate workers
for x := 0; x < TotalWorker; x++ {
manager.FreeWorkers <- &wk.Worker{x, manager.TimeProcess, manager.FreeWorkers}
}
// monitoring worker whos free
go manager.DoMonitor()
for scanner.Scan() {
err = Parse([]string{}, scanner.Text(), TableModel, "csv", "")
if err != nil {
log.Println(err)
}
insertValues := ""
var v reflect.Type
v = reflect.TypeOf(TableModel).Elem()
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
if v.Field(i).Type.String() == "string" {
insertValues += "\"" + reflect.ValueOf(TableModel).Elem().Field(i).String() + "\""
} else if v.Field(i).Type.String() == "int" {
temp, _ := strconv.ParseInt(reflect.ValueOf(TableModel).Elem().Field(i).String(), 32, 32)
insertValues += strconv.FormatInt(temp, 10)
} else if v.Field(i).Type.String() == "float" {
insertValues += strconv.FormatFloat(reflect.ValueOf(TableModel).Elem().Field(i).Float(), 'f', 6, 64)
} else {
insertValues += "\"" + reflect.ValueOf(TableModel).Elem().Field(i).Interface().(string) + "\""
}
if i < v.NumField()-1 {
insertValues += ", "
}
}
}
manager.Tasks <- func() {
if insertValues != "" {
retQuery := QueryBuilder("insert", TableName, insertValues, TableModel)
_, err = h.fetch(retQuery)
}
}
}
// waiting for tasks has been done
go manager.Timeout(3)
<-manager.Done
if err == nil {
retVal = "success"
}
}
return retVal, err
}