Golang NewManager Examples

Example #1

File: worker_test.go Project: yanda15/hdc

// test worker
func TestWorker(t *testing.T) {
	file, _ := os.Open("worker_test.txt")
	defer file.Close()
	scanner := bufio.NewScanner(file)

	// initialize manager and workers
	totalworker := 100
	manager := w.NewManager(totalworker)
	for i := 0; i < totalworker; i++ {
		manager.FreeWorkers <- &w.Worker{i, manager.TimeProcess, manager.FreeWorkers}
	}

	// monitoring worker thats free
	go manager.DoMonitor()

	// reading file
	for scanner.Scan() {
		// getting data per line
		data := scanner.Text()

		// send task to free worker
		manager.Tasks <- func() {
			// do something here
			fmt.Println(data)
		}
	}

	// waiting for tasks has been done
	go manager.Timeout(0)
	<-manager.Done
}

Example #2

File: hive.go Project: yanda15/hdc

// loading file with worker
func (h *Hive) LoadFileWithWorker(FilePath, TableName, fileType string, TableModel interface{}, TotalWorker int) (retVal string, err error) {
	retVal = "process failed"
	isMatch := false
	hr, err := h.fetch("select '1' from " + TableName + " limit 1;")

	if err != nil {
		return retVal, err
	}

	if hr.Result == nil {
		tempQuery := ""

		var v reflect.Type
		v = reflect.TypeOf(TableModel).Elem()

		if v.Kind() == reflect.Struct {
			tempQuery = "create table " + TableName + " ("
			for i := 0; i < v.NumField(); i++ {
				if i == (v.NumField() - 1) {
					tempQuery += v.Field(i).Name + " " + v.Field(i).Type.String() + ");"
				} else {
					tempQuery += v.Field(i).Name + " " + v.Field(i).Type.String() + ", "
				}
			}
			_, err = h.fetch(tempQuery)
		}
	}

	isMatch, err = h.CheckDataStructure(TableName, TableModel)

	if isMatch == false {
		return retVal, err
	}

	if err == nil {
		file, err := os.Open(FilePath)
		if err != nil {
			fmt.Println(err)
		}
		defer file.Close()

		if err != nil {
			log.Println(err)
		}

		scanner := bufio.NewScanner(file)

		// initiate dispatcher
		manager := wk.NewManager(TotalWorker)

		// initiate workers
		for x := 0; x < TotalWorker; x++ {
			manager.FreeWorkers <- &wk.Worker{x, manager.TimeProcess, manager.FreeWorkers}
		}

		// monitoring worker whos free
		go manager.DoMonitor()

		for scanner.Scan() {
			err = Parse([]string{}, scanner.Text(), TableModel, "csv", "")

			if err != nil {
				log.Println(err)
			}
			insertValues := ""

			var v reflect.Type
			v = reflect.TypeOf(TableModel).Elem()

			if v.Kind() == reflect.Struct {
				for i := 0; i < v.NumField(); i++ {
					if v.Field(i).Type.String() == "string" {
						insertValues += "\"" + reflect.ValueOf(TableModel).Elem().Field(i).String() + "\""
					} else if v.Field(i).Type.String() == "int" {
						temp, _ := strconv.ParseInt(reflect.ValueOf(TableModel).Elem().Field(i).String(), 32, 32)
						insertValues += strconv.FormatInt(temp, 10)
					} else if v.Field(i).Type.String() == "float" {
						insertValues += strconv.FormatFloat(reflect.ValueOf(TableModel).Elem().Field(i).Float(), 'f', 6, 64)
					} else {
						insertValues += "\"" + reflect.ValueOf(TableModel).Elem().Field(i).Interface().(string) + "\""
					}

					if i < v.NumField()-1 {
						insertValues += ", "
					}
				}
			}

			manager.Tasks <- func() {
				if insertValues != "" {
					retQuery := QueryBuilder("insert", TableName, insertValues, TableModel)
					_, err = h.fetch(retQuery)
				}
			}
		}

		// waiting for tasks has been done
		go manager.Timeout(3)
		<-manager.Done

		if err == nil {
			retVal = "success"
		}
	}

	return retVal, err
}