Beispiel #1
0
func diagnoseNotReady(parent *Package, p *Package) {
	switch p.State {
	case PackageBuilding, PackageBuildingButDirty, PackageUpdating, PackageUpdatingButDirty, PackageUpdateQueued, PackageBuildQueued:
		// Just wait. Though it may be helpful to inform the user of this?
	case PackageDirtyIdle:
		alog.Printf("@(dim:Can't build) %s @(dim:because) %s @(dim:isn't ready.)\n", parent.Name, p.Name)
		p.LastBuildInputsModTime = time.Time{} // Force a build even if a previous one failed, so that we can get the output again
		queueUpdate(p)
	default:
		alog.Panicf("diagnoseNotReady encountered unexpected state %s", p.State)
	}
}
Beispiel #2
0
func diagnoseCircularDependency(p *Package) {
	importNameQuoted := strconv.Quote(p.ImportName)
	absPkgPath := filepath.Join(srcRoot, p.Name)
	files, _ := ioutil.ReadDir(absPkgPath)
	for _, filename := range files {
		if filepath.Ext(filename.Name()) != ".go" {
			continue
		}
		path := filepath.Join(absPkgPath, filename.Name())
		fileSet := token.NewFileSet()
		ast, err := parser.ParseFile(fileSet, path, nil, parser.ImportsOnly)
		if err != nil {
			alog.Printf("Error parsing %s: %v\n", path, err)
		}
		for _, imp := range ast.Imports {
			if imp.Path.Value == importNameQuoted {
				position := fileSet.Position(imp.Pos())
				alog.Printf("@(error:Circular import found on line %d of %s)\n", position.Line, path)
			}
		}
	}
}
Beispiel #3
0
func processPathTriggers(notifyChan chan watcher.PathEvent) {
	for pathEvent := range notifyChan {
		path := pathEvent.Path
		moduleName, err := filepath.Rel(srcRoot, filepath.Dir(path))
		if err != nil {
			alog.Bail(err)
		}
		if buildExtensions.Has(filepath.Ext(path)) {
			if Opts.Verbose {
				alog.Printf("@(dim:Triggering module) @(cyan:%s) @(dim:due to update of) @(cyan:%s)\n", moduleName, path)
			}
			moduleUpdateChan <- moduleName
		}
	}
}
Beispiel #4
0
func getMostRecentDep(p *Package) (*Package, error) {
	var recentDep *Package
	for importName, _ := range p.Imports.Raw() {
		depPackage := resolveImport(p, importName)
		if depPackage == nil || depPackage.State != PackageReady {
			if depPackage == p {
				diagnoseCircularDependency(p)
			} else if beVerbose() {
				if depPackage == nil {
					alog.Printf("%s @(dim:requires) %s@(dim:, which could not be found.)\n", p.Name, importName)
				} else {
					diagnoseNotReady(p, depPackage)
				}
			}
			return nil, dependenciesNotReadyError
		}
		if recentDep == nil || depPackage.BuiltModTime.After(recentDep.BuiltModTime) {
			recentDep = depPackage
		}
	}
	return recentDep, nil
}
Beispiel #5
0
func (p *Package) build() {
	ctx := bismuth2.New()
	ctx.Verbose = Opts.Verbose

	timer := alog.NewTimer()
	// Just in case it gets deleted for some reason:
	absPath := filepath.Join(srcRoot, p.Name)

	if beVerbose() {
		alog.Printf("@(dim:Building) %s@(dim:...)\n", p.Name)
	}
	tmpTargetPath := filepath.Join(tmpdir, RandStr(20))
	fail := func() {
		os.Remove(tmpTargetPath)
		buildFailure <- p
	}
	args := []string{"go", "build", "-o", tmpTargetPath}
	if Opts.Tags != "" {
		args = append(args, "-tags")
		args = append(args, Opts.Tags)
	}
	var err error
	if beVerbose() {
		err = ctx.QuoteCwd("go-build:"+p.Name, absPath, args...)
	} else {
		_, _, err = ctx.RunCwd(absPath, args...)
	}
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			if waitStatus, ok := exitErr.Sys().(syscall.WaitStatus); ok {
				if beVerbose() {
					alog.Printf("@(error:Failed to build) %s @(dim)(status=%d)@(r)\n", p.Name, waitStatus.ExitStatus())
				}
				fail()
				return
			}
		}
		alog.Printf("@(error:Failed to install) %s@(error:: %s)\n", p.Name, err)
		fail()
		return
	}
	err = os.Chtimes(tmpTargetPath, time.Now(), p.UpdateStartTime)
	if err != nil {
		alog.Printf("@(error:Error setting atime/mtime of) %s@(error::) %v\n", tmpTargetPath, err)
		fail()
		return
	}
	targetPath := p.getAbsTargetPath()
	targetDir := filepath.Dir(targetPath)
	err = os.MkdirAll(targetDir, 0750)
	if err != nil {
		alog.Printf("@(error:Error creating directory %s for build target: %v)\n", targetDir, err)
		fail()
		return
	}
	err = os.Rename(tmpTargetPath, targetPath)
	if err != nil {
		alog.Printf("@(error:Error renaming %q to %q: %v)\n", tmpTargetPath, targetPath, err)
		fail()
		return
	}
	durationStr := timer.FormatElapsedColor(2*time.Second, 10*time.Second)
	if Opts.Verbose {
		alog.Printf("@(dim:[)%s@(dim:]) @(green:Successfully built) %s @(dim:->) @(time:%s)\n", durationStr, p.Name, p.UpdateStartTime.Format(DATE_FORMAT))
	} else {
		alog.Printf("@(dim:[)%s@(dim:]) @(green:Successfully built) %s\n", durationStr, p.Name)
	}
	if p.HasTests && shouldRunTests() {
		args := []string{"go", "test"}
		if Opts.TestArgShort {
			args = append(args, "-short")
		}
		if Opts.TestArgRun != "" {
			args = append(args, "-run")
			args = append(args, Opts.TestArgRun)
		}
		go ctx.QuoteCwd("go-test:"+p.Name, absPath, args...)
	}

	buildSuccess <- p
}
Beispiel #6
0
func update(pkgName string) {
	pUpdate := NewPackage(pkgName)
	defer func() {
		updateFinished <- pUpdate
	}()
	pUpdate.UpdateStartTime = time.Now()

	absPkgPath := filepath.Join(srcRoot, pkgName)
	pkg, err := build.ImportDir(absPkgPath, build.ImportComment)
	if err != nil {
		pUpdate.UpdateError = err
		// If the directory no longer exists, then tell the dispatcher to remove this package from the index
		_, statErr := os.Stat(absPkgPath)
		if statErr != nil && os.IsNotExist(statErr) {
			pUpdate.RemovePackage = true
		} else if beVerbose() {
			alog.Printf("@(warn:Error parsing import of module %s: %s)\n", pkgName, err)
		}
		return
	}
	pUpdate.Imports = stringset.New()
	for _, importName := range pkg.Imports {
		if !goStdLibPackages.Has(importName) {
			pUpdate.Imports.Add(importName)
		}
	}
	statFiles := func(files []string) {
		for _, filename := range files {
			path := filepath.Join(absPkgPath, filename)
			fileinfo, err := os.Stat(path)
			if err != nil {
				alog.Printf("@(error:Error stat-ing %s: %v)\n", path, err)
				pUpdate.UpdateError = err
				return
			}
			modTime := fileinfo.ModTime()
			if modTime.After(pUpdate.UpdateStartTime) {
				if modTime.After(time.Now()) {
					alog.Printf("@(warn:File has future modification time: %q mod %s)\n", path, modTime.String())
					alog.Printf("@(warn:Correct triggering of builds depends on correctly-set system clocks.)\n")
					// Assume that it was not actually modified in the future, but that the system clock is just wrong
					// This will allow us to build the package, but we'll keep re-building every time autoinstall
					// restarts until the system clock gets past the file's time.
					modTime = pUpdate.UpdateStartTime.Add(-1 * time.Microsecond)
				}
			}
			if modTime.After(pUpdate.SourceModTime) {
				pUpdate.SourceModTime = modTime
				pUpdate.RecentSrcName = fileinfo.Name()
			}
		}
	}
	statFiles(pkg.GoFiles)
	statFiles(pkg.CgoFiles)
	statFiles(pkg.CFiles)
	statFiles(pkg.CXXFiles)
	statFiles(pkg.MFiles)
	statFiles(pkg.HFiles)
	statFiles(pkg.SFiles)
	statFiles(pkg.SwigFiles)
	statFiles(pkg.SwigCXXFiles)
	statFiles(pkg.SysoFiles)
	if shouldRunTests() {
		statFiles(pkg.TestGoFiles)
	}

	if len(pkg.TestGoFiles) > 0 {
		pUpdate.HasTests = true
	}
	pUpdate.IsProgram = pkg.Name == "main"

	targetPath := pUpdate.getAbsTargetPath()
	fileinfo, err := os.Stat(targetPath)
	if err != nil && !os.IsNotExist(err) {
		alog.Printf("@(error:Error stat-ing %s: %v)\n", targetPath, err)
		pUpdate.UpdateError = err
		return
	} else if err == nil {
		pUpdate.BuiltModTime = fileinfo.ModTime()
	}
}
Beispiel #7
0
func main() {
	sighup := autorestart.NotifyOnSighup()
	_, err := flags.ParseArgs(&Opts, os.Args)
	if err != nil {
		err2, ok := err.(*flags.Error)
		if ok && err2.Type == flags.ErrHelp {
			return
		}
		alog.Printf("Error parsing command-line options: %s\n", err)
		return
	}
	if goPath == "" {
		alog.Printf("GOPATH is not set in the environment. Please set GOPATH first, then retry.\n")
		alog.Printf("For help setting GOPATH, see https://golang.org/doc/code.html\n")
		return
	}
	if Opts.NoColor {
		alog.DisableColor()
	} else {
		alog.AddAnsiColorCode("time", alog.ColorBlue)
	}
	alog.Printf("@(dim:autoinstall started.)\n")
	if Opts.MaxWorkers == 0 {
		Opts.MaxWorkers = runtime.GOMAXPROCS(0)
	}
	pluralProcess := ""
	if Opts.MaxWorkers != 1 {
		pluralProcess = "es"
	}
	alog.Printf("@(dim:Building all packages in) @(dim,cyan:%s)@(dim: using up to )@(dim,cyan:%d)@(dim: process%s.)\n", goPath, Opts.MaxWorkers, pluralProcess)
	if !Opts.Verbose {
		alog.Printf("@(dim:Use) --verbose @(dim:to show all messages during startup.)\n")
	}

	listener := watcher.NewListener()
	listener.Path = srcRoot
	// "_workspace" is a kludge to avoid recursing into Godeps workspaces
	// "node_modules" is a kludge to avoid walking into typically-huge node_modules trees
	listener.IgnorePart = stringset.New(".git", ".hg", "node_modules", "_workspace", "etld")
	listener.NotifyOnStartup = true
	listener.DebounceDuration = 200 * time.Millisecond
	listener.Start()

	// Delete any straggler tmp files, carefully
	files, err := ioutil.ReadDir(tmpdir)
	if err == nil {
		for _, file := range files {
			if filepath.Ext(file.Name()) == ".tmp" {
				os.Remove(filepath.Join(tmpdir, file.Name()))
			}
		}
	} else if os.IsNotExist(err) {
		err = os.MkdirAll(tmpdir, 0700)
		if err != nil {
			alog.Printf("@(error:Error creating temp directory at %s: %v)\n", tmpdir, err)
			return
		}
	} else {
		alog.Printf("@(error:Error checking contents of temp directory at %s: %v)\n", tmpdir, err)
		return
	}

	go processPathTriggers(listener.NotifyChan)
	go dispatcher()

	<-sighup
	startupLogger.Close()
}
Beispiel #8
0
func printStartupSummary() {
	alog.Printf("@(dim:Finished initial pass of all packages.)\n")
	updateStartupText(true)
}
Beispiel #9
0
func pushWork() {
	for numWorkersActive() < Opts.MaxWorkers && len(updateQueue) > 0 {
		var pkg *Package
		pkg, updateQueue = updateQueue[0], updateQueue[1:]
		if pkg.State != PackageUpdateQueued {
			alog.Panicf("Package %s was in updateQueue but had state %s", pkg.Name, pkg.State)
		}
		chState(pkg, PackageUpdating)
		numUpdatesActive++
		go update(pkg.Name)
	}
	for numWorkersActive() < Opts.MaxWorkers && len(buildQueue) > 0 {
		var pkg *Package
		pkg, buildQueue = buildQueue[0], buildQueue[1:]
		if pkg.State != PackageBuildQueued {
			alog.Panicf("Package %s was in buildQueue but had state %s", pkg.Name, pkg.State)
		}
		if !pkg.shouldBuild() {
			chState(pkg, PackageDirtyIdle)
		} else {
			recentDep, err := getMostRecentDep(pkg)
			if err == dependenciesNotReadyError {
				// At least one dependency is not ready
				chState(pkg, PackageDirtyIdle)
			} else if err != nil {
				alog.Panicf("@(error:Encountered unexpected error received from calcDepsModTime: %v)", err)
			} else {
				var inputsModTime time.Time
				if recentDep != nil {
					inputsModTime = recentDep.BuiltModTime
				}
				if pkg.SourceModTime.After(inputsModTime) {
					inputsModTime = pkg.SourceModTime
				}
				printTimes := func() {
					alog.Printf("    @(dim:Target ModTime) @(time:%s)\n", pkg.BuiltModTime.Format(DATE_FORMAT))
					if recentDep != nil {
						alog.Printf("    @(dim:  deps ModTime) @(time:%s) %s\n", recentDep.BuiltModTime.Format(DATE_FORMAT), recentDep.Name)
					} else {
						alog.Printf("    @(dim:  deps ModTime) n/a @(dim:no dependencies)\n")
					}
					alog.Printf("    @(dim:   src ModTime) @(time:%s) %s\n", pkg.SourceModTime.Format(DATE_FORMAT), pkg.RecentSrcName)
				}
				if !pkg.UpdateStartTime.After(inputsModTime) {
					// This package last updated after some of its inputs. Send it back to update again.
					queueUpdate(pkg)
				} else if !pkg.BuiltModTime.IsZero() && !inputsModTime.After(pkg.BuiltModTime) {
					// No need to build, as this package is already up to date.
					if Opts.Verbose {
						alog.Printf("@(dim:No need to build) %s\n", pkg.Name)
						printTimes()
					}
					chState(pkg, PackageReady)
					triggerDependentPackages(pkg.ImportName)
				} else if !inputsModTime.After(pkg.LastBuildInputsModTime) {
					if Opts.Verbose {
						// Sometimes, package updates/builds can be unnecessary triggered repeatedly. For example, sometimes builds themselves
						// can cause files to be touched in depended-upon packages, resulting in a cycle of endless failed builds (successful
						// builds would not be retried already because we would compare the timestamps and determine that the target was up to
						// date).
						alog.Printf("@(dim:Not building) %s@(dim:, as the package and its dependencies have not changed since its last build, which failed.)\n", pkg.Name)
					}
					chState(pkg, PackageDirtyIdle)
				} else {
					if Opts.Verbose && !pkg.BuiltModTime.IsZero() {
						alog.Printf("@(dim:Building) %s@(dim::)\n", pkg.Name)
						printTimes()
					}
					chState(pkg, PackageBuilding)
					numBuildsActive++
					pkg.LastBuildInputsModTime = inputsModTime
					go pkg.build()
				}
			}
		}
	}
}
Beispiel #10
0
func dispatcher() {
	startupLogger = alog.New(os.Stderr, "@(dim:{isodate}) ", 0)

	for {
		if !finishedInitialPass {
			updateStartupText(false)
		}
		dispatchState := getDispatchState()
		timeout := getTimeout(dispatchState)

		select {
		case p := <-buildSuccess:
			numBuildSucesses++
			numBuildsActive--
			switch p.State {
			case PackageBuilding:
				chState(p, PackageReady)
				p.BuiltModTime = p.UpdateStartTime
				p.LastBuildInputsModTime = time.Time{}
				triggerDependentPackages(p.ImportName)
			case PackageBuildingButDirty:
				queueUpdate(p)
			default:
				alog.Panicf("buildSuccess with state %s", p.State)
			}
		case p := <-buildFailure:
			numBuildFailures++
			numBuildsActive--
			switch p.State {
			case PackageBuilding:
				chState(p, PackageDirtyIdle)
			case PackageBuildingButDirty:
				queueUpdate(p)
			default:
				alog.Panicf("buildFailure with state %s", p.State)
			}
		case pUpdate := <-updateFinished:
			numUpdatesActive--
			p := packages[pUpdate.Name]
			if p == nil {
				alog.Panicf("Couldn't find package %s, yet dispatcher received an update for it.", pUpdate.Name)
			}
			switch p.State {
			case PackageUpdating:
				if pUpdate.UpdateError != nil {
					if pUpdate.RemovePackage {
						alog.Printf("@(dim:Removing package %s from index, as it has been removed from the filesystem.)\n", pUpdate.Name)
						delete(packages, p.Name)
						// Trigger updates of any packages that depend on this import name
						// XXX this should be modified if triggerDependentPackages is made more specific in the future
						triggerDependentPackages(p.ImportName)
					} else {
						chState(p, PackageDirtyIdle)
					}
				} else {
					p.mergeUpdate(pUpdate)
					queueBuild(p)
				}
			case PackageUpdatingButDirty:
				queueUpdate(p)
			default:
				alog.Panicf("updateFinished with state %s", p.State)
			}
		case pName := <-moduleUpdateChan:
			p := packages[pName]
			if p == nil {
				p = NewPackage(pName)
				p.init()
				packages[pName] = p
				numUnready++
			}
			switch p.State {
			case PackageReady, PackageDirtyIdle:
				queueUpdate(p)
			case PackageUpdating:
				chState(p, PackageUpdatingButDirty)
			case PackageBuilding:
				chState(p, PackageBuildingButDirty)
			case PackageUpdateQueued, PackageUpdatingButDirty, PackageBuildingButDirty:
				// Already have an update queued (or will), no need to change
			case PackageBuildQueued:
				// Has a build queued, but we need to do update first. Splice it out of the buildQueue, then queue the update.
				unqueueBuild(p)
				queueUpdate(p)
			default:
				alog.Panicf("moduleUpdateChan encountered unexpected state %s", p.State)
			}
		case <-timeout:
			switch dispatchState {
			case DispatchMaybeFinishedInitialPass:
				// We've reached the conclusion of the initial pass
				finishedInitialPass = true
				printStartupSummary()
			case DispatchCanPushWork:
				pushWork()
			case DispatchWaitingForWork:
				for _, p := range packages {
					switch p.State {
					case PackageBuilding, PackageBuildingButDirty:
						alog.Printf("@(dim:Still building %s...)\n", p.Name)
					case PackageUpdating, PackageUpdatingButDirty:
						alog.Printf("@(dim:Still checking %s...)\n", p.Name)
					}
				}
			default:
				alog.Panicf("dispatch hit timeout with unexpected dispatchState %s", dispatchState)
			}
		}
	}
}