func diagnoseNotReady(parent *Package, p *Package) {
switch p.State {
case PackageBuilding, PackageBuildingButDirty, PackageUpdating, PackageUpdatingButDirty, PackageUpdateQueued, PackageBuildQueued:
// Just wait. Though it may be helpful to inform the user of this?
case PackageDirtyIdle:
alog.Printf("@(dim:Can't build) %s @(dim:because) %s @(dim:isn't ready.)\n", parent.Name, p.Name)
p.LastBuildInputsModTime = time.Time{} // Force a build even if a previous one failed, so that we can get the output again
queueUpdate(p)
default:
alog.Panicf("diagnoseNotReady encountered unexpected state %s", p.State)
}
}
func diagnoseCircularDependency(p *Package) {
importNameQuoted := strconv.Quote(p.ImportName)
absPkgPath := filepath.Join(srcRoot, p.Name)
files, _ := ioutil.ReadDir(absPkgPath)
for _, filename := range files {
if filepath.Ext(filename.Name()) != ".go" {
continue
}
path := filepath.Join(absPkgPath, filename.Name())
fileSet := token.NewFileSet()
ast, err := parser.ParseFile(fileSet, path, nil, parser.ImportsOnly)
if err != nil {
alog.Printf("Error parsing %s: %v\n", path, err)
}
for _, imp := range ast.Imports {
if imp.Path.Value == importNameQuoted {
position := fileSet.Position(imp.Pos())
alog.Printf("@(error:Circular import found on line %d of %s)\n", position.Line, path)
}
}
}
}
func processPathTriggers(notifyChan chan watcher.PathEvent) {
for pathEvent := range notifyChan {
path := pathEvent.Path
moduleName, err := filepath.Rel(srcRoot, filepath.Dir(path))
if err != nil {
alog.Bail(err)
}
if buildExtensions.Has(filepath.Ext(path)) {
if Opts.Verbose {
alog.Printf("@(dim:Triggering module) @(cyan:%s) @(dim:due to update of) @(cyan:%s)\n", moduleName, path)
}
moduleUpdateChan <- moduleName
}
}
}
func getMostRecentDep(p *Package) (*Package, error) {
var recentDep *Package
for importName, _ := range p.Imports.Raw() {
depPackage := resolveImport(p, importName)
if depPackage == nil || depPackage.State != PackageReady {
if depPackage == p {
diagnoseCircularDependency(p)
} else if beVerbose() {
if depPackage == nil {
alog.Printf("%s @(dim:requires) %s@(dim:, which could not be found.)\n", p.Name, importName)
} else {
diagnoseNotReady(p, depPackage)
}
}
return nil, dependenciesNotReadyError
}
if recentDep == nil || depPackage.BuiltModTime.After(recentDep.BuiltModTime) {
recentDep = depPackage
}
}
return recentDep, nil
}
func (p *Package) build() {
ctx := bismuth2.New()
ctx.Verbose = Opts.Verbose
timer := alog.NewTimer()
// Just in case it gets deleted for some reason:
absPath := filepath.Join(srcRoot, p.Name)
if beVerbose() {
alog.Printf("@(dim:Building) %s@(dim:...)\n", p.Name)
}
tmpTargetPath := filepath.Join(tmpdir, RandStr(20))
fail := func() {
os.Remove(tmpTargetPath)
buildFailure <- p
}
args := []string{"go", "build", "-o", tmpTargetPath}
if Opts.Tags != "" {
args = append(args, "-tags")
args = append(args, Opts.Tags)
}
var err error
if beVerbose() {
err = ctx.QuoteCwd("go-build:"+p.Name, absPath, args...)
} else {
_, _, err = ctx.RunCwd(absPath, args...)
}
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
if waitStatus, ok := exitErr.Sys().(syscall.WaitStatus); ok {
if beVerbose() {
alog.Printf("@(error:Failed to build) %s @(dim)(status=%d)@(r)\n", p.Name, waitStatus.ExitStatus())
}
fail()
return
}
}
alog.Printf("@(error:Failed to install) %s@(error:: %s)\n", p.Name, err)
fail()
return
}
err = os.Chtimes(tmpTargetPath, time.Now(), p.UpdateStartTime)
if err != nil {
alog.Printf("@(error:Error setting atime/mtime of) %s@(error::) %v\n", tmpTargetPath, err)
fail()
return
}
targetPath := p.getAbsTargetPath()
targetDir := filepath.Dir(targetPath)
err = os.MkdirAll(targetDir, 0750)
if err != nil {
alog.Printf("@(error:Error creating directory %s for build target: %v)\n", targetDir, err)
fail()
return
}
err = os.Rename(tmpTargetPath, targetPath)
if err != nil {
alog.Printf("@(error:Error renaming %q to %q: %v)\n", tmpTargetPath, targetPath, err)
fail()
return
}
durationStr := timer.FormatElapsedColor(2*time.Second, 10*time.Second)
if Opts.Verbose {
alog.Printf("@(dim:[)%s@(dim:]) @(green:Successfully built) %s @(dim:->) @(time:%s)\n", durationStr, p.Name, p.UpdateStartTime.Format(DATE_FORMAT))
} else {
alog.Printf("@(dim:[)%s@(dim:]) @(green:Successfully built) %s\n", durationStr, p.Name)
}
if p.HasTests && shouldRunTests() {
args := []string{"go", "test"}
if Opts.TestArgShort {
args = append(args, "-short")
}
if Opts.TestArgRun != "" {
args = append(args, "-run")
args = append(args, Opts.TestArgRun)
}
go ctx.QuoteCwd("go-test:"+p.Name, absPath, args...)
}
buildSuccess <- p
}
func update(pkgName string) {
pUpdate := NewPackage(pkgName)
defer func() {
updateFinished <- pUpdate
}()
pUpdate.UpdateStartTime = time.Now()
absPkgPath := filepath.Join(srcRoot, pkgName)
pkg, err := build.ImportDir(absPkgPath, build.ImportComment)
if err != nil {
pUpdate.UpdateError = err
// If the directory no longer exists, then tell the dispatcher to remove this package from the index
_, statErr := os.Stat(absPkgPath)
if statErr != nil && os.IsNotExist(statErr) {
pUpdate.RemovePackage = true
} else if beVerbose() {
alog.Printf("@(warn:Error parsing import of module %s: %s)\n", pkgName, err)
}
return
}
pUpdate.Imports = stringset.New()
for _, importName := range pkg.Imports {
if !goStdLibPackages.Has(importName) {
pUpdate.Imports.Add(importName)
}
}
statFiles := func(files []string) {
for _, filename := range files {
path := filepath.Join(absPkgPath, filename)
fileinfo, err := os.Stat(path)
if err != nil {
alog.Printf("@(error:Error stat-ing %s: %v)\n", path, err)
pUpdate.UpdateError = err
return
}
modTime := fileinfo.ModTime()
if modTime.After(pUpdate.UpdateStartTime) {
if modTime.After(time.Now()) {
alog.Printf("@(warn:File has future modification time: %q mod %s)\n", path, modTime.String())
alog.Printf("@(warn:Correct triggering of builds depends on correctly-set system clocks.)\n")
// Assume that it was not actually modified in the future, but that the system clock is just wrong
// This will allow us to build the package, but we'll keep re-building every time autoinstall
// restarts until the system clock gets past the file's time.
modTime = pUpdate.UpdateStartTime.Add(-1 * time.Microsecond)
}
}
if modTime.After(pUpdate.SourceModTime) {
pUpdate.SourceModTime = modTime
pUpdate.RecentSrcName = fileinfo.Name()
}
}
}
statFiles(pkg.GoFiles)
statFiles(pkg.CgoFiles)
statFiles(pkg.CFiles)
statFiles(pkg.CXXFiles)
statFiles(pkg.MFiles)
statFiles(pkg.HFiles)
statFiles(pkg.SFiles)
statFiles(pkg.SwigFiles)
statFiles(pkg.SwigCXXFiles)
statFiles(pkg.SysoFiles)
if shouldRunTests() {
statFiles(pkg.TestGoFiles)
}
if len(pkg.TestGoFiles) > 0 {
pUpdate.HasTests = true
}
pUpdate.IsProgram = pkg.Name == "main"
targetPath := pUpdate.getAbsTargetPath()
fileinfo, err := os.Stat(targetPath)
if err != nil && !os.IsNotExist(err) {
alog.Printf("@(error:Error stat-ing %s: %v)\n", targetPath, err)
pUpdate.UpdateError = err
return
} else if err == nil {
pUpdate.BuiltModTime = fileinfo.ModTime()
}
}
func main() {
sighup := autorestart.NotifyOnSighup()
_, err := flags.ParseArgs(&Opts, os.Args)
if err != nil {
err2, ok := err.(*flags.Error)
if ok && err2.Type == flags.ErrHelp {
return
}
alog.Printf("Error parsing command-line options: %s\n", err)
return
}
if goPath == "" {
alog.Printf("GOPATH is not set in the environment. Please set GOPATH first, then retry.\n")
alog.Printf("For help setting GOPATH, see https://golang.org/doc/code.html\n")
return
}
if Opts.NoColor {
alog.DisableColor()
} else {
alog.AddAnsiColorCode("time", alog.ColorBlue)
}
alog.Printf("@(dim:autoinstall started.)\n")
if Opts.MaxWorkers == 0 {
Opts.MaxWorkers = runtime.GOMAXPROCS(0)
}
pluralProcess := ""
if Opts.MaxWorkers != 1 {
pluralProcess = "es"
}
alog.Printf("@(dim:Building all packages in) @(dim,cyan:%s)@(dim: using up to )@(dim,cyan:%d)@(dim: process%s.)\n", goPath, Opts.MaxWorkers, pluralProcess)
if !Opts.Verbose {
alog.Printf("@(dim:Use) --verbose @(dim:to show all messages during startup.)\n")
}
listener := watcher.NewListener()
listener.Path = srcRoot
// "_workspace" is a kludge to avoid recursing into Godeps workspaces
// "node_modules" is a kludge to avoid walking into typically-huge node_modules trees
listener.IgnorePart = stringset.New(".git", ".hg", "node_modules", "_workspace", "etld")
listener.NotifyOnStartup = true
listener.DebounceDuration = 200 * time.Millisecond
listener.Start()
// Delete any straggler tmp files, carefully
files, err := ioutil.ReadDir(tmpdir)
if err == nil {
for _, file := range files {
if filepath.Ext(file.Name()) == ".tmp" {
os.Remove(filepath.Join(tmpdir, file.Name()))
}
}
} else if os.IsNotExist(err) {
err = os.MkdirAll(tmpdir, 0700)
if err != nil {
alog.Printf("@(error:Error creating temp directory at %s: %v)\n", tmpdir, err)
return
}
} else {
alog.Printf("@(error:Error checking contents of temp directory at %s: %v)\n", tmpdir, err)
return
}
go processPathTriggers(listener.NotifyChan)
go dispatcher()
<-sighup
startupLogger.Close()
}
func printStartupSummary() {
alog.Printf("@(dim:Finished initial pass of all packages.)\n")
updateStartupText(true)
}
func pushWork() {
for numWorkersActive() < Opts.MaxWorkers && len(updateQueue) > 0 {
var pkg *Package
pkg, updateQueue = updateQueue[0], updateQueue[1:]
if pkg.State != PackageUpdateQueued {
alog.Panicf("Package %s was in updateQueue but had state %s", pkg.Name, pkg.State)
}
chState(pkg, PackageUpdating)
numUpdatesActive++
go update(pkg.Name)
}
for numWorkersActive() < Opts.MaxWorkers && len(buildQueue) > 0 {
var pkg *Package
pkg, buildQueue = buildQueue[0], buildQueue[1:]
if pkg.State != PackageBuildQueued {
alog.Panicf("Package %s was in buildQueue but had state %s", pkg.Name, pkg.State)
}
if !pkg.shouldBuild() {
chState(pkg, PackageDirtyIdle)
} else {
recentDep, err := getMostRecentDep(pkg)
if err == dependenciesNotReadyError {
// At least one dependency is not ready
chState(pkg, PackageDirtyIdle)
} else if err != nil {
alog.Panicf("@(error:Encountered unexpected error received from calcDepsModTime: %v)", err)
} else {
var inputsModTime time.Time
if recentDep != nil {
inputsModTime = recentDep.BuiltModTime
}
if pkg.SourceModTime.After(inputsModTime) {
inputsModTime = pkg.SourceModTime
}
printTimes := func() {
alog.Printf(" @(dim:Target ModTime) @(time:%s)\n", pkg.BuiltModTime.Format(DATE_FORMAT))
if recentDep != nil {
alog.Printf(" @(dim: deps ModTime) @(time:%s) %s\n", recentDep.BuiltModTime.Format(DATE_FORMAT), recentDep.Name)
} else {
alog.Printf(" @(dim: deps ModTime) n/a @(dim:no dependencies)\n")
}
alog.Printf(" @(dim: src ModTime) @(time:%s) %s\n", pkg.SourceModTime.Format(DATE_FORMAT), pkg.RecentSrcName)
}
if !pkg.UpdateStartTime.After(inputsModTime) {
// This package last updated after some of its inputs. Send it back to update again.
queueUpdate(pkg)
} else if !pkg.BuiltModTime.IsZero() && !inputsModTime.After(pkg.BuiltModTime) {
// No need to build, as this package is already up to date.
if Opts.Verbose {
alog.Printf("@(dim:No need to build) %s\n", pkg.Name)
printTimes()
}
chState(pkg, PackageReady)
triggerDependentPackages(pkg.ImportName)
} else if !inputsModTime.After(pkg.LastBuildInputsModTime) {
if Opts.Verbose {
// Sometimes, package updates/builds can be unnecessary triggered repeatedly. For example, sometimes builds themselves
// can cause files to be touched in depended-upon packages, resulting in a cycle of endless failed builds (successful
// builds would not be retried already because we would compare the timestamps and determine that the target was up to
// date).
alog.Printf("@(dim:Not building) %s@(dim:, as the package and its dependencies have not changed since its last build, which failed.)\n", pkg.Name)
}
chState(pkg, PackageDirtyIdle)
} else {
if Opts.Verbose && !pkg.BuiltModTime.IsZero() {
alog.Printf("@(dim:Building) %s@(dim::)\n", pkg.Name)
printTimes()
}
chState(pkg, PackageBuilding)
numBuildsActive++
pkg.LastBuildInputsModTime = inputsModTime
go pkg.build()
}
}
}
}
}
func dispatcher() {
startupLogger = alog.New(os.Stderr, "@(dim:{isodate}) ", 0)
for {
if !finishedInitialPass {
updateStartupText(false)
}
dispatchState := getDispatchState()
timeout := getTimeout(dispatchState)
select {
case p := <-buildSuccess:
numBuildSucesses++
numBuildsActive--
switch p.State {
case PackageBuilding:
chState(p, PackageReady)
p.BuiltModTime = p.UpdateStartTime
p.LastBuildInputsModTime = time.Time{}
triggerDependentPackages(p.ImportName)
case PackageBuildingButDirty:
queueUpdate(p)
default:
alog.Panicf("buildSuccess with state %s", p.State)
}
case p := <-buildFailure:
numBuildFailures++
numBuildsActive--
switch p.State {
case PackageBuilding:
chState(p, PackageDirtyIdle)
case PackageBuildingButDirty:
queueUpdate(p)
default:
alog.Panicf("buildFailure with state %s", p.State)
}
case pUpdate := <-updateFinished:
numUpdatesActive--
p := packages[pUpdate.Name]
if p == nil {
alog.Panicf("Couldn't find package %s, yet dispatcher received an update for it.", pUpdate.Name)
}
switch p.State {
case PackageUpdating:
if pUpdate.UpdateError != nil {
if pUpdate.RemovePackage {
alog.Printf("@(dim:Removing package %s from index, as it has been removed from the filesystem.)\n", pUpdate.Name)
delete(packages, p.Name)
// Trigger updates of any packages that depend on this import name
// XXX this should be modified if triggerDependentPackages is made more specific in the future
triggerDependentPackages(p.ImportName)
} else {
chState(p, PackageDirtyIdle)
}
} else {
p.mergeUpdate(pUpdate)
queueBuild(p)
}
case PackageUpdatingButDirty:
queueUpdate(p)
default:
alog.Panicf("updateFinished with state %s", p.State)
}
case pName := <-moduleUpdateChan:
p := packages[pName]
if p == nil {
p = NewPackage(pName)
p.init()
packages[pName] = p
numUnready++
}
switch p.State {
case PackageReady, PackageDirtyIdle:
queueUpdate(p)
case PackageUpdating:
chState(p, PackageUpdatingButDirty)
case PackageBuilding:
chState(p, PackageBuildingButDirty)
case PackageUpdateQueued, PackageUpdatingButDirty, PackageBuildingButDirty:
// Already have an update queued (or will), no need to change
case PackageBuildQueued:
// Has a build queued, but we need to do update first. Splice it out of the buildQueue, then queue the update.
unqueueBuild(p)
queueUpdate(p)
default:
alog.Panicf("moduleUpdateChan encountered unexpected state %s", p.State)
}
case <-timeout:
switch dispatchState {
case DispatchMaybeFinishedInitialPass:
// We've reached the conclusion of the initial pass
finishedInitialPass = true
printStartupSummary()
case DispatchCanPushWork:
pushWork()
case DispatchWaitingForWork:
for _, p := range packages {
switch p.State {
case PackageBuilding, PackageBuildingButDirty:
alog.Printf("@(dim:Still building %s...)\n", p.Name)
case PackageUpdating, PackageUpdatingButDirty:
alog.Printf("@(dim:Still checking %s...)\n", p.Name)
}
}
default:
alog.Panicf("dispatch hit timeout with unexpected dispatchState %s", dispatchState)
}
}
}
}