// RefreshTables refreshes multiple tables in parallel and returns an error if any of the copies
// fail.
func (r *Redshift) RefreshTables(
tables map[string]postgres.TableSchema, schema, tmpschema, s3prefix, awsRegion string, delim rune) error {
if _, err := r.logAndExec(fmt.Sprintf(`CREATE SCHEMA "%s"`, tmpschema), false); err != nil {
return err
}
group := new(errgroup.Group)
for name, ts := range tables {
group.Add(1)
go func(name string, ts postgres.TableSchema) {
if err := r.refreshTable(schema, name, tmpschema, postgres.S3Filename(s3prefix, name), awsRegion, ts, delim); err != nil {
group.Error(err)
}
group.Done()
}(name, ts)
}
errs := new(errgroup.Group)
if err := group.Wait(); err != nil {
errs.Error(err)
}
if _, err := r.logAndExec(fmt.Sprintf(`DROP SCHEMA "%s" CASCADE`, tmpschema), false); err != nil {
errs.Error(err)
}
// Use errs.Wait() to group the two errors into a single error object.
return errs.Wait()
}
func TestAddDone(t *testing.T) {
t.Parallel()
var g errgroup.Group
l := 10
g.Add(l)
for i := 0; i < l; i++ {
go g.Done()
}
ensure.Nil(t, g.Wait())
}
// DumpTablesToS3 dumps multiple tables to s3 in parallel.
func (db *DB) DumpTablesToS3(tables []string, s3prefix string) error {
group := new(errgroup.Group)
for _, table := range tables {
group.Add(1)
go func(table string) {
if err := db.DumpTableToS3(table, S3Filename(s3prefix, table)); err != nil {
group.Error(err)
}
group.Done()
}(table)
}
return group.Wait()
}
func (d *deployCmd) computeChecksums(files []string,
normalizeName func(string) string) (map[string]string, error) {
var wg errgroup.Group
maxParallel := make(chan struct{}, maxOpenFD)
wg.Add(len(files))
var mutex sync.Mutex
checksums := make(map[string]string)
computeChecksum := func(name string) {
defer func() {
wg.Done()
<-maxParallel
}()
file, err := os.Open(name)
defer file.Close()
if err != nil {
wg.Error(stackerr.Wrap(err))
return
}
h := md5.New()
if _, err := io.Copy(h, file); err != nil {
wg.Error(stackerr.Wrap(err))
return
}
if err := file.Close(); err != nil {
wg.Error(stackerr.Wrap(err))
return
}
mutex.Lock()
checksums[normalizeName(name)] = fmt.Sprintf("%x", h.Sum(nil))
defer mutex.Unlock()
}
for _, file := range files {
maxParallel <- struct{}{}
go computeChecksum(file)
}
err := wg.Wait()
if err != nil {
return checksums, err
}
return checksums, nil
}
// GetTableSchemas returns a map from a tablename to its schema. Gets schemas for different tables
// in parallel.
func (db *DB) GetTableSchemas(tables []string, namespace string) (map[string]TableSchema, error) {
group := new(errgroup.Group)
tsmap := map[string]TableSchema{}
for _, table := range tables {
group.Add(1)
go func(table string) {
ts, err := db.GetTableSchema(table, namespace)
if err != nil {
group.Error(err)
}
tsmap[table] = ts
group.Done()
}(table)
}
err := group.Wait()
return tsmap, err
}
func (h *herokuLink) herokuAppNames(ids []string, e *parsecli.Env) (nameIDs, []string, error) {
var wg errgroup.Group
wg.Add(len(ids))
maxParallel := make(chan struct{}, maxRequests)
var (
ret nameIDs
deletedLinks []string
retMutex sync.Mutex
deletedLinksMutex sync.Mutex
)
getAppName := func(id string) {
defer func() {
wg.Done()
<-maxParallel
}()
appName, err := parsecli.FetchHerokuAppName(id, e)
if err != nil {
if stackerr.HasUnderlying(err, stackerr.MatcherFunc(parsecli.HerokuAppNotFound)) {
deletedLinksMutex.Lock()
defer deletedLinksMutex.Unlock()
deletedLinks = append(deletedLinks, id)
return
}
wg.Error(err) // ignore error if corresponding heroku app was deleted
return
}
retMutex.Lock()
defer retMutex.Unlock()
ret = append(ret, nameID{id: id, name: appName})
}
for _, id := range ids {
go getAppName(id)
}
err := wg.Wait()
sort.Sort(ret)
return ret, deletedLinks, stackerr.Wrap(err)
}
func uploadSymbolFiles(files []string, commonHeaders map[string]string, removeFiles bool, e *env) error {
var wg errgroup.Group
uploadFile := func(filename string, e *env) {
defer wg.Done()
name := filepath.Base(filepath.Clean(filename))
file, err := os.Open(filename)
if err != nil {
wg.Error(stackerr.Wrap(err))
return
}
defer file.Close()
req, err := http.NewRequest("POST", path.Join("symbolFiles", name), bufio.NewReader(file))
if err != nil {
wg.Error(stackerr.Wrap(err))
return
}
if req.Header == nil {
req.Header = make(http.Header)
}
for key, val := range commonHeaders {
req.Header.Add(key, val)
}
hash, err := base64MD5OfFile(filename)
if err != nil {
wg.Error(err)
return
}
req.Header.Add("Content-MD5", hash)
mimeType := mime.TypeByExtension(filepath.Ext(name))
if mimeType == "" {
mimeType = "application/octet-stream"
}
req.Header.Add("Content-Type", mimeType)
res := make(map[string]interface{})
if _, err := e.ParseAPIClient.Do(req, nil, &res); err != nil {
wg.Error(err)
return
}
if removeFiles {
if err := file.Close(); err != nil {
wg.Error(err)
return
}
if err := os.Remove(filename); err != nil {
wg.Error(err)
return
}
}
}
for _, file := range files {
wg.Add(1)
go uploadFile(file, e)
}
err := wg.Wait()
if err != nil {
return err
}
fmt.Fprintln(e.Out, "Uploaded symbol files.")
return nil
}
func (d *downloadCmd) moveFiles(
e *env,
destination string,
release *deployInfo) error {
var wg errgroup.Group
maxParallel := make(chan struct{}, maxOpenFD)
numFiles := len(release.Versions.Cloud) + len(release.Versions.Public)
wg.Add(numFiles)
var numErrors int32
moveFile := func(destination, kind, file, checksum string) {
defer func() {
wg.Done()
<-maxParallel
}()
err := os.MkdirAll(
filepath.Join(e.Root, kind, filepath.Dir(file)),
0755,
)
if err != nil {
atomic.AddInt32(&numErrors, 1)
wg.Error(stackerr.Wrap(err))
return
}
err = os.Rename(
filepath.Join(destination, kind, file),
filepath.Join(e.Root, kind, file),
)
if err != nil {
atomic.AddInt32(&numErrors, 1)
wg.Error(stackerr.Wrap(err))
return
}
err = d.verifyChecksum(
filepath.Join(e.Root, kind, file),
checksum,
)
if err != nil {
atomic.AddInt32(&numErrors, 1)
wg.Error(err)
return
}
}
for file, checksum := range release.Checksums.Cloud {
maxParallel <- struct{}{}
go moveFile(
destination,
cloudDir,
file,
checksum,
)
}
for file, checksum := range release.Checksums.Public {
maxParallel <- struct{}{}
go moveFile(
destination,
hostingDir,
file,
checksum,
)
}
if err := wg.Wait(); err != nil {
// could not move a single file so no corruption:w
if int(numErrors) == numFiles {
fmt.Fprintf(
e.Out,
`Failed to download Cloud Code to
%q
Try "parse download" and manually move contents from
the temporary download location.
`,
e.Root,
)
return nil
}
fmt.Fprintf(
e.Out,
`Failed to download Cloud Code to
%q
It might have corrupted contents, due to partially moved files.
Try "parse download" and manually move contents from
the temporary download location.
`,
e.Root,
)
return err
}
return nil
}
func (d *downloadCmd) download(e *env, destination string, release *deployInfo) error {
var wg errgroup.Group
maxParallel := make(chan struct{}, maxOpenFD)
wg.Add(len(release.Versions.Cloud) + len(release.Versions.Public))
downloadHosted := func(file, version, checksum string) {
defer func() {
wg.Done()
<-maxParallel
}()
v := make(url.Values)
v.Set("version", version)
v.Set("checksum", checksum)
u := &url.URL{
Path: path.Join("hosted_files", file),
RawQuery: v.Encode(),
}
var content []byte
_, err := e.ParseAPIClient.Get(u, &content)
if err != nil {
wg.Error(err)
return
}
path := path.Join(destination, hostingDir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
wg.Error(stackerr.Wrap(err))
return
}
if err := ioutil.WriteFile(path, content, 0644); err != nil {
wg.Error(stackerr.Wrap(err))
return
}
if err := d.verifyChecksum(path, checksum); err != nil {
wg.Error(err)
return
}
}
downloadScript := func(file, version, checksum string) {
defer func() {
wg.Done()
<-maxParallel
}()
v := make(url.Values)
v.Set("version", version)
v.Set("checksum", checksum)
u := &url.URL{
Path: path.Join("scripts", file),
RawQuery: v.Encode(),
}
var content string
_, err := e.ParseAPIClient.Get(u, &content)
if err != nil {
wg.Error(err)
return
}
path := path.Join(destination, cloudDir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
wg.Error(stackerr.Wrap(err))
return
}
if err := ioutil.WriteFile(path, []byte(content), 0644); err != nil {
wg.Error(stackerr.Wrap(err))
return
}
}
for file, version := range release.Versions.Public {
checksum, ok := release.Checksums.Public[file]
if !ok {
continue
}
maxParallel <- struct{}{}
go downloadHosted(file, version, checksum)
}
for file, version := range release.Versions.Cloud {
checksum, ok := release.Checksums.Cloud[file]
if !ok {
continue
}
maxParallel <- struct{}{}
go downloadScript(file, version, checksum)
}
return wg.Wait()
}
// Pair returns a TransformFunc that pairs all the elements in the table with another table, based
// on the given identifier functions and join type.
func Pair(rightTable optimus.Table, leftID, rightID RowIdentifier, filterFn func(optimus.Row) (bool, error)) optimus.TransformFunc {
// Map of everything in the right table
right := make(map[interface{}][]optimus.Row)
// Track whether or not rows in the right table were joined against
joined := make(map[interface{}]bool)
// Start building the map right away, because it could be slow.
mapResult := make(chan error)
go func() {
defer close(mapResult)
for row := range rightTable.Rows() {
id, err := rightID(row)
if err != nil {
mapResult <- err
return
}
if val := right[id]; val == nil {
right[id] = []optimus.Row{}
joined[id] = false
}
right[id] = append(right[id], row)
}
mapResult <- rightTable.Err()
}()
return func(in <-chan optimus.Row, out chan<- optimus.Row) error {
if err := <-mapResult; err != nil {
return err
}
// The channel of paired rows from the left and right tables
pairedRows := make(chan optimus.Row)
wg := errgroup.Group{}
// Pair the left table with the right table based on the ids
wg.Add(1)
go func() {
defer close(pairedRows)
defer wg.Done()
for leftRow := range in {
id, err := leftID(leftRow)
if err != nil {
wg.Error(err)
return
}
if rightRows := right[id]; rightRows != nil && id != nil {
joined[id] = true
for _, rightRow := range rightRows {
pairedRows <- optimus.Row{"left": leftRow, "right": rightRow}
}
} else {
pairedRows <- optimus.Row{"left": leftRow}
}
}
for id, joined := range joined {
if joined {
continue
}
for _, rightRow := range right[id] {
pairedRows <- optimus.Row{"right": rightRow}
}
}
return
}()
// Filter the paired rows based on our join type
wg.Add(1)
go func() {
defer wg.Done()
if err := Select(filterFn)(pairedRows, out); err != nil {
wg.Error(err)
}
}()
return wg.Wait()
}
}
func (d *deployCmd) uploadSourceFiles(u *uploader) (map[string]string,
map[string]string, error) {
sourceFiles, ignoredFiles, err := d.getSourceFiles(filepath.Join(u.Env.Root, u.DirName), u.Suffixes, u.Env)
if err != nil {
return nil, nil, err
}
namePrefixLen := len(filepath.Join(u.Env.Root, u.DirName, "1")) - 1
normalizeName := func(name string) string {
name = filepath.ToSlash(filepath.Clean(name))
return name[namePrefixLen:]
}
currentChecksums, err := d.computeChecksums(sourceFiles, normalizeName)
if err != nil {
return nil, nil, err
}
var mutex sync.Mutex
maxParallel := make(chan struct{}, maxOpenFD)
var wg errgroup.Group
currentVersions := make(map[string]string)
uploadFile := func(sourceFile string) {
defer func() {
wg.Done()
<-maxParallel
}()
version, err := d.uploadFile(sourceFile, u.EndPoint, u.Env, normalizeName)
if err != nil {
wg.Error(err)
return
}
mutex.Lock()
currentVersions[normalizeName(sourceFile)] = version
defer mutex.Unlock()
}
changed := false
var changedFiles []string
for _, sourceFile := range sourceFiles {
if !d.Force { // if not forced, verify changed content using checksums
name := normalizeName(sourceFile)
var noUpload bool
if prevChecksum, ok := u.PrevChecksums[name]; ok {
noUpload = prevChecksum == currentChecksums[name]
}
if prevVersion, ok := u.PrevVersions[name]; ok && noUpload {
currentVersions[name] = prevVersion
continue
}
}
changed = true
wg.Add(1)
changedFiles = append(changedFiles, sourceFile)
maxParallel <- struct{}{}
go uploadFile(sourceFile)
}
if changed && d.Verbose {
var message string
switch u.DirName {
case "cloud":
message = "scripts"
case "public":
message = "hosting"
}
fmt.Fprintf(u.Env.Out,
`Uploading recent changes to %s...
The following files will be uploaded:
%s
`,
message,
strings.Join(changedFiles, "\n"),
)
if len(ignoredFiles) != 0 {
fmt.Fprintln(u.Env.Out, "The following files will be ignored:")
for _, file := range ignoredFiles {
fmt.Fprintln(u.Env.Out, file)
}
}
}
if err := wg.Wait(); err != nil {
return nil, nil, err
}
return currentChecksums, currentVersions, nil
}