Пример #1
0
func createRecordFn(
	serviceClient *services.Client,
	createFunction func(*services.Client, Record) (migration.Id, error),
	successChan chan LoadSuccess,
	errorChan chan error,
) func(Record) {
	return func(record Record) {
		id, err := createFunction(serviceClient, record)
		switch {
		case err != nil:
			errorChan <- err
		case err == nil:
			successChan <- LoadSuccess{createdId: migration.Id(id)}
		}
	}
}
Пример #2
0
func main() {
	c := streamFile("/Users/dan/Code/ETL/diablo-migration/input-files/members.csv")
	transformed := pipeline(c, []func(Record) Record{correctTimestamps})
	toImport, toCreate := segmentByLoadMethod(transformed)
	toImport = pipeline(toImport, []func(Record) Record{removeFields([]string{"first_name", "last_name", "email", "phone"})})

	success := make(chan LoadSuccess)
	errors := make(chan error)
	importFn := importRecordFn(
		db,
		"customer_accounts",
		[]string{"member_id", "braintree_token", "created_at", "updated_at"},
		success,
		errors,
	)
	apiFn := func(
		client *services.Client,
		record Record,
	) (migration.Id, error) {
		memberId, err := strconv.ParseFloat(record["member_id"].(string), 64)
		if err != nil {
			return 0, err
		}
		account, err := client.Finance.CreateCustomerAccount(
			memberId,
			record["first_name"].(string),
			record["last_name"].(string),
			record["email"].(string),
			record["phone"].(string),
		)
		if err != nil {
			return 0, err
		}
		return migration.Id(account.Id), nil
	}
	migration.CheckError(err)
	createFn := createRecordFn(
		client,
		apiFn,
		success,
		errors,
	)

	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		for record := range toImport {
			importFn(record)
		}
		wg.Done()
	}()

	go func() {
		for record := range toCreate {
			createFn(record)
		}
		wg.Done()
	}()

	go func() {
		for success := range success {
			log.Println(success.createdId)
		}
	}()

	go func() {
		for error := range errors {
			log.Println(error)
		}
	}()

	wg.Wait()
	close(success)
	close(errors)
}