func init() {
var err error
db, err = sql.Open(DatabaseDriver, DatabaseUrl)
migration.CheckError(err)
client, err = services.NewServiceClient(app)
migration.CheckError(err)
}
func main() {
c := streamFile("/Users/dan/Code/ETL/diablo-migration/input-files/members.csv")
transformed := pipeline(c, []func(Record) Record{correctTimestamps})
toImport, toCreate := segmentByLoadMethod(transformed)
toImport = pipeline(toImport, []func(Record) Record{removeFields([]string{"first_name", "last_name", "email", "phone"})})
success := make(chan LoadSuccess)
errors := make(chan error)
importFn := importRecordFn(
db,
"customer_accounts",
[]string{"member_id", "braintree_token", "created_at", "updated_at"},
success,
errors,
)
apiFn := func(
client *services.Client,
record Record,
) (migration.Id, error) {
memberId, err := strconv.ParseFloat(record["member_id"].(string), 64)
if err != nil {
return 0, err
}
account, err := client.Finance.CreateCustomerAccount(
memberId,
record["first_name"].(string),
record["last_name"].(string),
record["email"].(string),
record["phone"].(string),
)
if err != nil {
return 0, err
}
return migration.Id(account.Id), nil
}
migration.CheckError(err)
createFn := createRecordFn(
client,
apiFn,
success,
errors,
)
var wg sync.WaitGroup
wg.Add(2)
go func() {
for record := range toImport {
importFn(record)
}
wg.Done()
}()
go func() {
for record := range toCreate {
createFn(record)
}
wg.Done()
}()
go func() {
for success := range success {
log.Println(success.createdId)
}
}()
go func() {
for error := range errors {
log.Println(error)
}
}()
wg.Wait()
close(success)
close(errors)
}