// CleanCache inspects the compilation cache and removes all packages
// which are not referenced (anymore).
func (f *Fissile) CleanCache(targetPath string) error {
// 1. Collect list of packages referenced by the releases. A
// variant of the code in ListPackages, we keep only the
// hashes.
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
referenced := make(map[string]int)
for _, release := range f.releases {
for _, pkg := range release.Packages {
referenced[pkg.Version] = 1
}
}
/// 2. Scan local compilation cache, compare to referenced,
/// remove anything not found.
f.UI.Printf("Cleaning up %s\n", color.MagentaString(targetPath))
cached, err := filepath.Glob(targetPath + "/*")
if err != nil {
return err
}
removed := 0
for _, cache := range cached {
key := filepath.Base(cache)
if _, ok := referenced[key]; ok {
continue
}
f.UI.Printf("- Removing %s\n", color.YellowString(key))
if err := os.RemoveAll(cache); err != nil {
return err
}
removed++
}
if removed == 0 {
f.UI.Println("Nothing found to remove")
return nil
}
plural := ""
if removed > 1 {
plural = "s"
}
f.UI.Printf("Removed %s package%s\n",
color.MagentaString(fmt.Sprintf("%d", removed)),
plural)
return nil
}
func pollEvents() {
errorCount := 0
for {
// Use this block for polling of uchiwa
events, err := fetchEvents(cfg.Alerts)
if err != nil {
log.Println(color.YellowString("Could not pollEvents from Uchiwa"))
errorCount++
} else {
errorCount = 0
}
eventCount := len(events)
if eventCount > 0 {
log.Println(color.MagentaString(fmt.Sprintf("Found %d Uchiwa events for processing...", eventCount)))
} else {
log.Println(color.MagentaString("Found 0 Uchiwa events, nothing to process currently."))
}
for i, ev := range events {
//TODO: send to stathat
//TODO: may have to aggregate stats manually to see per datacenter (not sure stathat can do that for us)
key := ev.Check.Name + "-" + ev.Client.Name
// Only sending 5 keys for now since we're on a demo plan
if i < 5 {
stathat.PostEZCount(key, cfg.StatsAccount, 1)
fmt.Println("stathat -> " + key)
}
// results := outputRegexp.FindStringSubmatch(ev.Check.Output)
// if len(results) == 2 {
// if staleCount, err := strconv.Atoi(strings.TrimSpace(results[1])); err == nil {
// log.Println(color.MagentaString(fmt.Sprintf("Host %s has %d stale files > 1 hour old.", ev.Client.Name, staleCount)))
// }
// }
}
if errorCount > cfg.Uchiwa.MaxErrors {
log.Fatalln(color.MagentaString("Fatal: Reached max consecutive allowable errors with Uchiwa."))
}
time.Sleep(time.Second * time.Duration(cfg.Uchiwa.Interval))
}
}
func main() {
// Kick off uchiwa poller
go pollEvents()
// Poor-man wait: Just block until user presses enter
log.Println(color.MagentaString(fmt.Sprintf("Started: Waiting for Uchiwa events to process with %d second interval.", cfg.Uchiwa.Interval)))
blockUntilSignal()
}
func makeStack() (ret string) {
st := make([]uintptr, 50)
cnt := runtime.Callers(4, st)
st = st[0:cnt]
for _, r := range st {
fnc := runtime.FuncForPC(r)
file, line := fnc.FileLine(r - 1)
stackLine := fmt.Sprintf("\t%s:%d\n", file, line)
stackFunc := fmt.Sprintf("\t\t%s\n", fnc.Name())
if colored {
stackLine = color.MagentaString(stackLine)
stackFunc = color.MagentaString(stackFunc)
}
ret += stackLine + stackFunc
}
return
}
// Logo with color
func Logo() string {
var logo string
logo += "\n\n"
logo += color.GreenString(" ██╗ ██╗ ██████╗ ███╗ ███╗ █████╗ ███╗ ██╗██████╗ █████╗\n")
logo += color.MagentaString(" ██║ ██╔╝██╔═══██╗████╗ ████║██╔══██╗████╗ ██║██╔══██╗██╔══██╗\n")
logo += color.YellowString(" █████╔╝ ██║ ██║██╔████╔██║███████║██╔██╗ ██║██║ ██║███████║\n")
logo += color.CyanString(" ██╔═██╗ ██║ ██║██║╚██╔╝██║██╔══██║██║╚██╗██║██║ ██║██╔══██║\n")
logo += color.BlueString(" ██║ ██╗╚██████╔╝██║ ╚═╝ ██║██║ ██║██║ ╚████║██████╔╝██║ ██║\n")
logo += color.RedString(" ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝╚═════╝ ╚═╝ ╚═╝")
return logo
}
func loadConfig() {
b, err := ioutil.ReadFile("config.json")
if err != nil {
log.Fatal(color.RedString("Couldn't load config.json data."))
}
c := &config{}
err = json.Unmarshal(b, c)
if err != nil {
log.Fatal(color.RedString("Couldn't unmarshal config.json data."))
}
cfg = c
log.Println(color.MagentaString("Loaded config.json file from disk."))
}
func (f *Fissile) listPropertiesForHuman() {
// Human readable output.
for _, release := range f.releases {
f.UI.Println(color.GreenString("Dev release %s (%s)",
color.YellowString(release.Name), color.MagentaString(release.Version)))
for _, job := range release.Jobs {
f.UI.Printf("%s (%s): %s\n", color.YellowString(job.Name),
color.WhiteString(job.Version), job.Description)
for _, property := range job.Properties {
f.UI.Printf("\t%s: %v\n", color.YellowString(property.Name),
property.Default)
}
}
}
}
// Compile will compile a list of dev BOSH releases
func (f *Fissile) Compile(repository, targetPath, roleManifestPath, metricsPath string, workerCount int) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
if metricsPath != "" {
stampy.Stamp(metricsPath, "fissile", "compile-packages", "start")
defer stampy.Stamp(metricsPath, "fissile", "compile-packages", "done")
}
dockerManager, err := docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
roleManifest, err := model.LoadRoleManifest(roleManifestPath, f.releases)
if err != nil {
return fmt.Errorf("Error loading roles manifest: %s", err.Error())
}
f.UI.Println(color.GreenString("Compiling packages for dev releases:"))
for _, release := range f.releases {
f.UI.Printf(" %s (%s)\n", color.YellowString(release.Name), color.MagentaString(release.Version))
}
comp, err := compilator.NewCompilator(dockerManager, targetPath, metricsPath, repository, compilation.UbuntuBase, f.Version, false, f.UI)
if err != nil {
return fmt.Errorf("Error creating a new compilator: %s", err.Error())
}
if err := comp.Compile(workerCount, f.releases, roleManifest); err != nil {
return fmt.Errorf("Error compiling packages: %s", err.Error())
}
return nil
}
//
// print a set of articles
//
func printArticles(page int, articles []Article) {
fmt.Fprintf(color.Output, "\n-----------------------------------------------------------(%s)--\n", color.MagentaString("page:%d", page))
for id, entry := range articles {
source := entry.Information.SourceName
if len(source) == 0 {
source = "N/A"
}
fmt.Fprintf(color.Output, "%s: %s (%s)\n",
color.CyanString("%02d", id+1),
color.WhiteString(entry.Information.Title),
color.YellowString("%s", source))
}
fmt.Fprintf(color.Output, "-----------------------------------------------------------(%s)--\n\n", color.MagentaString("page%d", page))
}
// ColoredBuildStringFunc returns a formatting function for colorizing strings.
func ColoredBuildStringFunc(buildName string) StringFormatter {
return func(s string) string {
return color.GreenString("build-%s > %s", color.MagentaString(buildName), color.WhiteString("%s", s))
}
}
func (o *Decorator) DecorateDetails(entity log.Log) {
fmt.Println()
c := color.New(color.FgGreen, color.Bold)
c.Println("Response")
fmt.Printf(" • Id: %s", color.GreenString(entity.Id))
fmt.Println()
level := o.colorizeLevel(entity.Level)
fmt.Printf(" • Level: %s", level)
fmt.Println()
date := entity.Time.Format(time.Stamp)
fmt.Printf(" • Time: %s", color.WhiteString(date))
fmt.Println()
fmt.Printf(" • Host: %s", color.WhiteString(entity.Host))
fmt.Println()
if len(entity.SessionId) > 0 {
fmt.Printf(" • Session id: %s", color.GreenString(entity.SessionId))
fmt.Println()
}
message := o.replacePlaceholders(entity.Message, entity.Source)
fmt.Printf(" • Message: %s", color.CyanString(message))
fmt.Println()
fmt.Printf(" • Script id: %s", color.YellowString(entity.ScriptId))
fmt.Println()
fmt.Printf(" • Object: %s", (entity.Object))
fmt.Println()
fmt.Println(" • Source:")
for key, value := range entity.Source {
if value == nil || key == "log-level" || key == "message" ||
key == "script-id" || key == "@version" || key == "@timestamp" ||
key == "object" || key == "type" || key == "host" {
continue
}
switch reflect.TypeOf(value).String() {
case "string":
value = color.CyanString(fmt.Sprint(value))
break
case "int64":
value = color.BlueString(fmt.Sprintf("%d", value))
case "float64":
if regexp.MustCompile("^[0-9]+(.[0]+)?").MatchString(fmt.Sprintf("%f", value)) {
value = fmt.Sprintf("%d", int64(value.(float64)))
} else {
value = fmt.Sprintf("%f", value)
}
value = color.BlueString(value.(string))
break
default:
value = fmt.Sprint(value)
}
fmt.Printf(" • %s: %s", color.MagentaString(key), value)
fmt.Println()
}
if entity.Exception != nil && entity.Exception.Code != 0 {
fmt.Println(" • Exception:")
style1 := color.New(color.FgWhite, color.BgBlack)
style1.Printf(" • Message: %s", entity.Exception.Message)
style1.Println()
style1.Printf(" • Code: %d", entity.Exception.Code)
style1.Println()
for _, trace := range entity.Exception.Trace {
style1.Printf(" • File: %s:%d", trace.File, trace.Line)
style1.Println()
}
}
fmt.Println()
}
// ListJobs will list all jobs within a list of dev releases
func (f *Fissile) ListJobs() error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
for _, release := range f.releases {
f.UI.Println(color.GreenString("Dev release %s (%s)", color.YellowString(release.Name), color.MagentaString(release.Version)))
for _, job := range release.Jobs {
f.UI.Printf("%s (%s): %s\n", color.YellowString(job.Name), color.WhiteString(job.Version), job.Description)
}
f.UI.Printf(
"There are %s jobs present.\n\n",
color.GreenString("%d", len(release.Jobs)),
)
}
return nil
}
// ListPackages will list all BOSH packages within a list of dev releases
func (f *Fissile) ListPackages() error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
for _, release := range f.releases {
f.UI.Println(color.GreenString("Dev release %s (%s)", color.YellowString(release.Name), color.MagentaString(release.Version)))
for _, pkg := range release.Packages {
f.UI.Printf("%s (%s)\n", color.YellowString(pkg.Name), color.WhiteString(pkg.Version))
}
f.UI.Printf(
"There are %s packages present.\n\n",
color.GreenString("%d", len(release.Packages)),
)
}
return nil
}
func (c *Compilator) compilePackage(pkg *model.Package) (err error) {
// Prepare input dir (package plus deps)
if err := c.createCompilationDirStructure(pkg); err != nil {
return err
}
if err := c.copyDependencies(pkg); err != nil {
return err
}
// Generate a compilation script
targetScriptName := "compile.sh"
hostScriptPath := filepath.Join(pkg.GetTargetPackageSourcesDir(c.hostWorkDir), targetScriptName)
containerScriptPath := filepath.Join(docker.ContainerInPath, targetScriptName)
if err := compilation.SaveScript(c.baseType, compilation.CompilationScript, hostScriptPath); err != nil {
return err
}
// Extract package
extractDir := c.getSourcePackageDir(pkg)
if _, err := pkg.Extract(extractDir); err != nil {
return err
}
// Run compilation in container
containerName := c.getPackageContainerName(pkg)
// in-memory buffer of the log
log := new(bytes.Buffer)
stdoutWriter := docker.NewFormattingWriter(
log,
func(line string) string {
return color.GreenString("compilation-%s > %s", color.MagentaString("%s", pkg.Name), color.WhiteString("%s", line))
},
)
stderrWriter := docker.NewFormattingWriter(
log,
func(line string) string {
return color.GreenString("compilation-%s > %s", color.MagentaString("%s", pkg.Name), color.RedString("%s", line))
},
)
sourceMountName := fmt.Sprintf("source_mount-%s", uuid.New())
mounts := map[string]string{
pkg.GetTargetPackageSourcesDir(c.hostWorkDir): docker.ContainerInPath,
pkg.GetPackageCompiledTempDir(c.hostWorkDir): docker.ContainerOutPath,
// Add the volume mount to work around AUFS issues. We will clean
// the volume up (as long as we're not trying to keep the container
// around for debugging). We don't give it an actual directory to mount
// from, so it will be in some docker-maintained storage.
sourceMountName: ContainerSourceDir,
}
exitCode, container, err := c.dockerManager.RunInContainer(docker.RunInContainerOpts{
ContainerName: containerName,
ImageName: c.BaseImageName(),
Cmd: []string{"bash", containerScriptPath, pkg.Name, pkg.Version},
Mounts: mounts,
Volumes: map[string]map[string]string{sourceMountName: nil},
KeepContainer: c.keepContainer,
StdoutWriter: stdoutWriter,
StderrWriter: stderrWriter,
})
if container != nil && (!c.keepContainer || err == nil || exitCode == 0) {
// Attention. While the assignments to 'err' in the
// deferal below take effect after the 'return'
// statements coming later they are visible to the
// caller, i.e. override the 'return'ed value,
// because 'err' is a __named__ return parameter.
defer func() {
// Remove container - dockerManager.RemoveContainer does a force-rm
if removeErr := c.dockerManager.RemoveContainer(container.ID); removeErr != nil {
if err == nil {
err = removeErr
} else {
err = fmt.Errorf("Error compiling package: %s. Error removing package: %s", err.Error(), removeErr.Error())
}
}
if removeErr := c.dockerManager.RemoveVolumes(container); removeErr != nil {
if err == nil {
err = removeErr
} else {
err = fmt.Errorf("%s: Error removing volumes for package %s: %s", err, pkg.Name, removeErr)
}
}
}()
}
if err != nil {
log.WriteTo(c.ui)
return fmt.Errorf("Error compiling package %s: %s", pkg.Name, err.Error())
}
if exitCode != 0 {
log.WriteTo(c.ui)
return fmt.Errorf("Error - compilation for package %s exited with code %d", pkg.Name, exitCode)
}
return os.Rename(
pkg.GetPackageCompiledTempDir(c.hostWorkDir),
pkg.GetPackageCompiledDir(c.hostWorkDir))
}
func (j compileJob) Run() {
c := j.compilator
// Metrics: Overall time for the specific job
var waitSeriesName string
var runSeriesName string
if c.metricsPath != "" {
seriesName := fmt.Sprintf("compile-packages::%s/%s", j.pkg.Release.Name, j.pkg.Name)
waitSeriesName = fmt.Sprintf("compile-packages::wait::%s/%s", j.pkg.Release.Name, j.pkg.Name)
runSeriesName = fmt.Sprintf("compile-packages::run::%s/%s", j.pkg.Release.Name, j.pkg.Name)
stampy.Stamp(c.metricsPath, "fissile", seriesName, "start")
defer stampy.Stamp(c.metricsPath, "fissile", seriesName, "done")
stampy.Stamp(c.metricsPath, "fissile", waitSeriesName, "start")
}
// (xx) Wait for our deps. Note how without deps the killCh is
// not checked and ignored. It is also in a race with (**)
// draining doneCh and actually signaling the kill.
// Time spent waiting
for _, dep := range j.pkg.Dependencies {
done := false
for !done {
select {
case <-j.killCh:
c.ui.Printf("killed: %s/%s\n",
color.MagentaString(j.pkg.Release.Name),
color.MagentaString(j.pkg.Name))
j.doneCh <- compileResult{pkg: j.pkg, err: errWorkerAbort}
if c.metricsPath != "" {
stampy.Stamp(c.metricsPath, "fissile", waitSeriesName, "done")
}
return
case <-time.After(5 * time.Second):
c.ui.Printf("waiting: %s/%s - %s\n",
color.MagentaString(j.pkg.Release.Name),
color.MagentaString(j.pkg.Name),
color.MagentaString(dep.Name))
case <-c.signalDependencies[dep.Fingerprint]:
c.ui.Printf("depdone: %s/%s - %s\n",
color.MagentaString(j.pkg.Release.Name),
color.MagentaString(j.pkg.Name),
color.MagentaString(dep.Name))
done = true
}
}
}
if c.metricsPath != "" {
stampy.Stamp(c.metricsPath, "fissile", waitSeriesName, "done")
}
c.ui.Printf("compile: %s/%s\n",
color.MagentaString(j.pkg.Release.Name),
color.MagentaString(j.pkg.Name))
// Time spent in actual compilation
if c.metricsPath != "" {
stampy.Stamp(c.metricsPath, "fissile", runSeriesName, "start")
}
workerErr := compilePackageHarness(c, j.pkg)
if c.metricsPath != "" {
stampy.Stamp(c.metricsPath, "fissile", runSeriesName, "done")
}
c.ui.Printf("done: %s/%s\n",
color.MagentaString(j.pkg.Release.Name),
color.MagentaString(j.pkg.Name))
j.doneCh <- compileResult{pkg: j.pkg, err: workerErr}
}
func LogDebug(format string, a ...interface{}) {
log.Println("[DEBUG]", color.MagentaString(format, a...))
}
bracketSP = termcolor.New(termcolor.FgBlue).SprintFunc()
quoteSP = termcolor.New(termcolor.FgBlue, termcolor.Bold, termcolor.Italic).SprintFunc()
kvNumSP = termcolor.New(termcolor.FgYellow, termcolor.Bold).SprintFunc()
kvSepSP = termcolor.New(termcolor.FgWhite, termcolor.Bold, termcolor.Italic).SprintFunc()
strSP = termcolor.New(termcolor.FgWhite, termcolor.Bold).SprintFunc()
dateSP = termcolor.New(termcolor.FgRed, termcolor.Italic).SprintFunc()
packSP = termcolor.New(termcolor.FgWhite, termcolor.Italic, termcolor.Bold).SprintFunc()
escapeSP = termcolor.New(termcolor.FgCyan, termcolor.Bold).SprintFunc()
escapedSP = termcolor.New(termcolor.FgMagenta, termcolor.Bold).SprintFunc()
logHeading = fmt.Sprintf("%s%s%s", logSqBracketSP("["), logSP("LOG"), logSqBracketSP("]"))
debugL = termcolor.CyanString("DEBUG")
errorL = termcolor.RedString("ERROR")
infoL = termcolor.GreenString("INFO")
warnL = termcolor.MagentaString("WARN")
)
type nologArgs struct {
outToFile bool
outFileName string
color bool
verbose bool
gocheck string
args []string
}
var nlArgs nologArgs
func init() {
flag.BoolVar(&nlArgs.outToFile, "f", false, "setting this flag will output the logs to a file.")