func (r CIRepository) Pull() (ImportRec, bool, error) {
var relSourceRecs []relSourceRec
err := r.index.ListKeys(&relSourceRecs)
if err != nil {
return ImportRec{}, false, bosherr.WrapError(err, "Finding release sources")
}
if len(relSourceRecs) == 0 {
return ImportRec{}, false, nil
}
rec := relSourceRecs[0]
r.logger.Debug(ciRepositoryLogTag, "Pulling import '%v'", rec)
importRec := ImportRec{
RelSource: rec.RelSource,
Version: rec.Version,
}
err = r.index.Remove(relSourceRec{rec.RelSource, rec.Version})
if err != nil {
return ImportRec{}, false, bosherr.WrapError(err, "Removing import")
}
return importRec, true, nil
}
func (p sfdiskPartitioner) diskMatchesPartitions(devicePath string, partitionsToMatch []Partition) (result bool) {
existingPartitions, err := p.getPartitions(devicePath)
if err != nil {
err = bosherr.WrapError(err, "Getting partitions for %s", devicePath)
return
}
if len(existingPartitions) < len(partitionsToMatch) {
return
}
remainingDiskSpace, err := p.GetDeviceSizeInMb(devicePath)
if err != nil {
err = bosherr.WrapError(err, "Getting device size for %s", devicePath)
return
}
for index, partitionToMatch := range partitionsToMatch {
if index == len(partitionsToMatch)-1 {
partitionToMatch.SizeInMb = remainingDiskSpace
}
existingPartition := existingPartitions[index]
switch {
case existingPartition.Type != partitionToMatch.Type:
return
case notWithinDelta(existingPartition.SizeInMb, partitionToMatch.SizeInMb, 20):
return
}
remainingDiskSpace = remainingDiskSpace - partitionToMatch.SizeInMb
}
return true
}
func (c Config) validate() error {
if c.AssetsDir == "" {
return bosherr.New("Must provide non-empty assets_dir")
}
if c.ReposDir == "" {
return bosherr.New("Must provide non-empty repos_dir")
}
err := c.EventLog.Validate()
if err != nil {
return bosherr.WrapError(err, "Validating event_log configuration")
}
if c.Blobstore.Type != bpprov.BlobstoreConfigTypeLocal {
return bosherr.New("Blobstore type must be local")
}
err = c.Blobstore.Validate()
if err != nil {
return bosherr.WrapError(err, "Validating blobstore configuration")
}
return nil
}
func (p ReleaseCompiler) compileRelease(pkgsCompiler bppkgscomp.PackagesCompiler, depRelease bpdep.Release) error {
relReader := p.releaseReaderFactory.NewReader(
depRelease.Name,
depRelease.Version,
depRelease.URL,
)
relRelease, err := relReader.Read()
if err != nil {
return bosherr.WrapError(err, "Reading release")
}
defer relReader.Close()
err = pkgsCompiler.Compile(relRelease)
if err != nil {
return bosherr.WrapError(err, "Compiling release packages")
}
err = p.templatesCompiler.Precompile(relRelease)
if err != nil {
return bosherr.WrapError(err, "Precompiling release job templates")
}
return nil
}
func (i PeriodicGithubNoteImporter) importNotes() error {
// todo getting all releases; could be big
sources, err := i.releasesRepo.ListAll()
if err != nil {
return bosherr.WrapError(err, "Listing releases")
}
for _, source := range sources {
ghSource, valid := newGithubSource(source)
if !valid {
continue
}
// todo convert Source to string; argh
relVerRecs, err := i.releasesRepo.FindAll(source.Full)
if err != nil {
return bosherr.WrapError(err, "Listing all versions for release source '%s'", source)
}
err = i.importNotesForRelease(ghSource, relVerRecs)
if err != nil {
return bosherr.WrapError(err, "Importing notes for release source '%s'", source)
}
}
return nil
}
func (p BlobstoreProvider) Get(provider string, options map[string]interface{}) (boshblob.Blobstore, error) {
configDir, err := p.fs.TempDir("blobstore-s3-config")
if err != nil {
return nil, bosherr.WrapError(err, "Cerating tmp dir for blobstore config")
}
configPath := filepath.Join(configDir, "config.json")
blobstore := boshblob.NewExternalBlobstore(
provider,
options,
p.fs,
p.runner,
p.uuidGen,
configPath,
)
blobstore = boshblob.NewSHA1VerifiableBlobstore(blobstore)
blobstore = boshblob.NewRetryableBlobstore(blobstore, 3, p.logger)
err = blobstore.Validate()
if err != nil {
return nil, bosherr.WrapError(err, "Validating blobstore")
}
return blobstore, nil
}
func (p *PlainBucketPage) fetchOnce() (plainBucketPage_ListBucketResult, error) {
// Return memoized results; not thread safe
if p.fetchedResult != nil || p.fetchedErr != nil {
return *p.fetchedResult, p.fetchedErr
}
fullURL := p.fullURL()
p.logger.Debug(p.logTag, "Fetching bucket page from url '%s'", fullURL)
var result plainBucketPage_ListBucketResult
resp, err := http.Get(fullURL)
if err != nil {
p.fetchedErr = err
return result, bosherr.WrapError(err, "Requesting bucket page")
}
defer resp.Body.Close()
decoder := xml.NewDecoder(resp.Body)
decoder.CharsetReader = charset.NewReader
err = decoder.Decode(&result)
if err != nil {
p.fetchedErr = err
return result, bosherr.WrapError(err, "Parsing bucket page")
}
p.logger.Debug(p.logTag, "Saving bucket page result '%v'", result)
p.fetchedResult = &result
return result, nil
}
func (r S3StemcellsRepository) SaveAll(s3Files []bhs3.File) error {
var s3StemcellRecs []s3StemcellRec
for _, s3File := range s3Files {
url, err := s3File.URL()
if err != nil {
return bosherr.WrapError(err, "Generating S3 stemcell URL")
}
rec := s3StemcellRec{
Key: s3File.Key(),
ETag: s3File.ETag(),
Size: s3File.Size(),
LastModified: s3File.LastModified(),
URL: url,
}
s3StemcellRecs = append(s3StemcellRecs, rec)
}
err := r.index.Save(s3StemcellsKeyLatest, s3StemcellRecs)
if err != nil {
return bosherr.WrapError(err, "Saving all S3 stemcell recs")
}
return nil
}
func (a MountDiskAction) Run(diskCid string) (interface{}, error) {
err := a.settingsService.LoadSettings()
if err != nil {
return nil, bosherr.WrapError(err, "Refreshing the settings")
}
settings := a.settingsService.GetSettings()
devicePath, found := settings.Disks.Persistent[diskCid]
if !found {
return nil, bosherr.New("Persistent disk with volume id '%s' could not be found", diskCid)
}
mountPoint := a.dirProvider.StoreDir()
isMountPoint, err := a.mountPoints.IsMountPoint(mountPoint)
if err != nil {
return nil, bosherr.WrapError(err, "Checking mount point")
}
if isMountPoint {
mountPoint = a.dirProvider.StoreMigrationDir()
}
err = a.diskMounter.MountPersistentDisk(devicePath, mountPoint)
if err != nil {
return nil, bosherr.WrapError(err, "Mounting persistent disk")
}
return map[string]string{}, nil
}
func (e Extractor) extractReleaseAndJobs(tgzPath string) (bprel.Release, []bpreljob.Job, error) {
var rel bprel.Release
relReader := e.releaseReaderFactory.NewTarReader("file://" + tgzPath)
rel, err := relReader.Read()
if err != nil {
return rel, nil, bosherr.WrapError(err, "Reading release")
}
defer relReader.Close()
var relJobs []bpreljob.Job
for _, j := range rel.Jobs {
relJobReader := e.jobReaderFactory.NewTarReader("file://" + j.TarPath)
relJob, err := relJobReader.Read()
if err != nil {
return rel, nil, bosherr.WrapError(err, "Reading release job '%s'", j.Name)
}
defer relJobReader.Close()
relJobs = append(relJobs, relJob)
}
return rel, relJobs, nil
}
func (s JobState) buildPackageSpecs() (map[string]boshas.PackageSpec, error) {
specs := map[string]boshas.PackageSpec{}
for _, template := range s.depJob.Templates {
pkgs, err := s.templatesCompiler.FindPackages(template)
if err != nil {
return specs, bosherr.WrapError(err, "Finding packages for template %s", template.Name)
}
for _, pkg := range pkgs {
rec, err := s.packagesCompiler.FindCompiledPackage(pkg)
if err != nil {
return specs, bosherr.WrapError(err, "Finding compiled package %s", pkg.Name)
}
specs[pkg.Name] = boshas.PackageSpec{
Name: pkg.Name,
Version: pkg.Version,
Sha1: rec.SHA1,
BlobstoreID: rec.BlobID,
}
}
}
return specs, nil
}
func (e Extractor) Extract(url, tgzPath string) (bhrelsrepo.ReleaseVersionRec, error) {
e.logger.Debug(e.logTag, "Extracting from '%s' (url '%s')", tgzPath, url)
var relVerRec bhrelsrepo.ReleaseVersionRec
rel, relJobs, err := e.extractReleaseAndJobs(tgzPath)
if err != nil {
return relVerRec, bosherr.WrapError(err, "Extracting release and jobs")
}
relVerRec, err = e.releasesRepo.Find(url, rel.Version)
if err != nil {
return relVerRec, bosherr.WrapError(err, "Finding release")
}
err = e.jobsRepo.SaveAll(relVerRec, relJobs)
if err != nil {
return relVerRec, bosherr.WrapError(err, "Saving release jobs into jobs repository")
}
err = e.releaseVersionsRepo.Save(relVerRec, rel)
if err != nil {
return relVerRec, bosherr.WrapError(err, "Saving release into releases repository")
}
return relVerRec, nil
}
func (p RunitProvisioner) Provision(name string, stopTimeout time.Duration) error {
p.logger.Info(runitProvisionerLogTag, "Provisioning %s service", name)
err := p.depsProvisioner.InstallRunit()
if err != nil {
return bosherr.WrapError(err, "Installing runit")
}
servicePath, enableServicePath := p.buildServicePaths(name)
err = p.stopRunAndLog(servicePath, enableServicePath, name, stopTimeout)
if err != nil {
return bosherr.WrapError(err, "Stopping run and log")
}
err = p.setUpRun(servicePath, name)
if err != nil {
return bosherr.WrapError(err, "Setting up run")
}
err = p.setUpLog(servicePath, name)
if err != nil {
return bosherr.WrapError(err, "Setting up log")
}
err = p.startRunAndLog(servicePath, enableServicePath, name)
if err != nil {
return bosherr.WrapError(err, "Starting run and log")
}
return nil
}
func (tc ConcreteTemplatesCompiler) buildJobReaders(job bpdep.Job) ([]jobReader, error) {
var readers []jobReader
for _, template := range job.Templates {
rec, found, err := tc.tplToJobRepo.FindByTemplate(template)
if err != nil {
return readers, bosherr.WrapError(err, "Finding dep-template -> release-job record %s", template.Name)
} else if !found {
return readers, bosherr.New("Expected to find dep-template -> release-job record %s", template.Name)
}
jobRec, found, err := tc.jobsRepo.FindByReleaseJob(rec)
if err != nil {
return readers, bosherr.WrapError(err, "Finding job source blob %s", template.Name)
} else if !found {
return readers, bosherr.New("Expected to find job source blob %s -- %s", template.Name, rec)
}
jobURL := fmt.Sprintf("blobstore:///%s?fingerprint=%s", jobRec.BlobID, jobRec.SHA1)
reader := jobReader{
rec: rec,
tarReader: tc.jobReaderFactory.NewReader(jobURL),
}
readers = append(readers, reader)
}
return readers, nil
}
func (tr TarballRelease) Import(url string) error {
tr.logger.Debug(tr.logTag, "Importing tarball release from '%s'", url)
tgzPath, err := tr.builder.Build(tr.manifestPath)
if err != nil {
return bosherr.WrapError(err, "Building release tarball from manifest '%s'", tr.manifestPath)
}
defer tr.builder.CleanUp(tgzPath)
relVerRec, err := tr.extractor.Extract(url, tgzPath)
if err != nil {
return bosherr.WrapError(err, "Importing release version from tgz '%s'", tgzPath)
}
err = tr.uploader.Upload(relVerRec, tgzPath)
if err != nil {
return bosherr.WrapError(err, "Uploading release verison '%v'", relVerRec)
}
// Save release version only after everything else was successfully imported
err = tr.releasesRepo.Add(relVerRec)
if err != nil {
return bosherr.WrapError(err, "Saving release version into release versions repository")
}
return nil
}
func (u Updater) SetUp() error {
stage := u.eventLog.BeginStage(fmt.Sprintf("Setting up instance %s", u.instanceDesc), 3)
task := stage.BeginTask("Applying")
err := task.End(u.applier.Apply())
if err != nil {
return bosherr.WrapError(err, "Applying")
}
task = stage.BeginTask("Starting")
err = task.End(u.starter.Start())
if err != nil {
return bosherr.WrapError(err, "Starting")
}
task = stage.BeginTask("Waiting")
err = task.End(u.waiter.Wait())
if err != nil {
return bosherr.WrapError(err, "Waiting")
}
return nil
}
func (c httpClient) status() (status, error) {
url := c.monitURL("/_status2")
url.RawQuery = "format=xml"
response, err := c.makeRequest(url, "GET", "")
if err != nil {
return status{}, bosherr.WrapError(err, "Sending status request to monit")
}
defer response.Body.Close()
err = c.validateResponse(response)
if err != nil {
return status{}, bosherr.WrapError(err, "Getting monit status")
}
decoder := xml.NewDecoder(response.Body)
decoder.CharsetReader = charset.NewReader
var st status
err = decoder.Decode(&st)
if err != nil {
return status{}, bosherr.WrapError(err, "Unmarshalling Monit status")
}
return st, nil
}
func (v SemanticValidator) Validate() error {
for _, net := range v.deployment.Networks {
err := v.validateNetwork(net)
if err != nil {
return bosherr.WrapError(err, "Network %s", net.Name)
}
}
for _, release := range v.deployment.Releases {
err := v.validateRelease(release)
if err != nil {
return bosherr.WrapError(err, "Release %s", release.Name)
}
}
err := v.validateInstance(v.deployment.CompilationInstance)
if err != nil {
return bosherr.WrapError(err, "Compilation instance")
}
for _, job := range v.deployment.Jobs {
err := v.validateJob(job)
if err != nil {
return bosherr.WrapError(err, "Job %s", job.Name)
}
}
return nil
}
func (p MonitProvisioner) Provision() error {
p.logger.Info(monitProvisionerLogTag, "Provisioning monit")
path := "/var/vcap/monit"
err := p.cmds.MkdirP(path)
if err != nil {
return err
}
err = p.configureMonitrc()
if err != nil {
return bosherr.WrapError(err, "Configuring monitrc")
}
err = p.runitProvisioner.Provision(
monitProvisionerRunitName,
monitProvisionerRunitStopTime,
)
if err != nil {
return bosherr.WrapError(err, "Provisioning monit with runit")
}
return nil
}
func (r ERBRenderer) Render(srcPath, dstPath string) error {
r.logger.Debug(erbRendererLogTag, "Rendering template %s", dstPath)
dirPath := filepath.Dir(dstPath)
err := r.fs.MkdirAll(dirPath, os.FileMode(0755))
if err != nil {
return bosherr.WrapError(err, "Creating directory %s", dirPath)
}
rendererScriptPath, err := r.writeRendererScript()
if err != nil {
return err
}
contextPath, err := r.writeContext()
if err != nil {
return err
}
// Use ruby to compile job templates
command := boshsys.Command{
Name: r.determineRubyExePath(),
Args: []string{rendererScriptPath, contextPath, srcPath, dstPath},
}
_, _, _, err = r.runner.RunComplexCommand(command)
if err != nil {
return bosherr.WrapError(err, "Running ruby")
}
return nil
}
func (f ConcreteFetcher) Fetch(relSource string) (ReleaseDir, error) {
var releaseDir ReleaseDir
f.logger.Debug(concreteFetcherLogTag, "Starting fetching release '%s'", relSource)
// todo take identifier
downloadPath, err := f.downloader.Download("git://" + relSource)
if err != nil {
f.logger.Error(concreteFetcherLogTag,
"Failed to download release '%s': %s", relSource, err)
return releaseDir, bosherr.WrapError(err, "Downloading release")
}
cleanUp := func() error {
err := f.downloader.CleanUp(downloadPath)
if err != nil {
f.logger.Error(concreteFetcherLogTag,
"Failed to clean up downloaded release '%s': %s", relSource, err)
return bosherr.WrapError(err, "Cleaning up downloaded release")
}
return nil
}
releaseDir = NewReleaseDir(downloadPath, cleanUp, f.fs, f.logger)
return releaseDir, nil
}
func (a ApplyAction) Run(desiredSpec boshas.V1ApplySpec) (string, error) {
settings := a.settingsService.GetSettings()
resolvedDesiredSpec, err := a.specService.PopulateDynamicNetworks(desiredSpec, settings)
if err != nil {
return "", bosherr.WrapError(err, "Resolving dynamic networks")
}
if desiredSpec.ConfigurationHash != "" {
currentSpec, err := a.specService.Get()
if err != nil {
return "", bosherr.WrapError(err, "Getting current spec")
}
err = a.applier.Apply(currentSpec, resolvedDesiredSpec)
if err != nil {
return "", bosherr.WrapError(err, "Applying")
}
}
err = a.specService.Set(resolvedDesiredSpec)
if err != nil {
return "", bosherr.WrapError(err, "Persisting apply spec")
}
return "applied", nil
}
func (r defaultNetworkResolver) GetDefaultNetwork() (boshsettings.Network, error) {
network := boshsettings.Network{}
routes, err := r.routesSearcher.SearchRoutes()
if err != nil {
return network, bosherr.WrapError(err, "Searching routes")
}
if len(routes) == 0 {
return network, bosherr.New("No routes found")
}
for _, route := range routes {
if !route.IsDefault() {
continue
}
ip, err := r.ipResolver.GetPrimaryIPv4(route.InterfaceName)
if err != nil {
return network, bosherr.WrapError(
err, "Getting primary IPv4 for interface '%s'", route.InterfaceName)
}
return boshsettings.Network{
IP: ip.IP.String(),
Netmask: gonet.IP(ip.Mask).String(),
Gateway: route.Gateway,
}, nil
}
return network, bosherr.New("Failed to find default route")
}
func (p AptDepsProvisioner) installPkg(name string) error {
p.logger.Debug(aptDepsProvisionerLogTag, "Installing package %s", name)
_, _, _, err := p.runner.RunCommand("apt-get", "-y", "install", name)
if err == nil {
return nil
}
unableToFetch := strings.Contains(err.Error(), aptDepsProvisionerUnableToFetchMsg)
unableToLocate := strings.Contains(err.Error(), aptDepsProvisionerUnableToLocateMsg)
// Avoid running 'apt-get update' since it usually takes 30sec
if unableToFetch || unableToLocate {
_, _, _, err := p.runner.RunCommand("apt-get", "-y", "update")
if err != nil {
return bosherr.WrapError(err, "Updating sources")
}
var lastInstallErr error
// For some reason libssl-dev was really hard to install on the first try
for i := 0; i < 3; i++ {
_, _, _, lastInstallErr = p.runner.RunCommand("apt-get", "-y", "install", name)
if lastInstallErr == nil {
return nil
}
time.Sleep(1 * time.Second)
}
return bosherr.WrapError(lastInstallErr, "Installing %s after updating", name)
}
return err
}
func (ac HTTPClient) makeQuickRequest(method reqMethod, args reqArgs) (responseEnvelope, error) {
var responseBody responseEnvelope
requestBody := requestEnvelope{
Method: method,
Arguments: args,
ReplyTo: "n-a",
}
requestBytes, err := json.Marshal(requestBody)
if err != nil {
return responseBody, bosherr.WrapError(err, "Marshalling request body")
}
responseBytes, err := ac.makePlainRequest(string(requestBytes), "application/json")
if err != nil {
return responseBody, bosherr.WrapError(err, "Making plain request")
}
err = json.Unmarshal(responseBytes, &responseBody)
if err != nil {
return responseBody, bosherr.WrapError(err, "Unmarshalling response body")
}
if responseBody.HasException() {
return responseBody, bosherr.New("Ended with exception %#v", responseBody.Exception)
}
return responseBody, nil
}
func (p AptDepsProvisioner) Provision() error {
pkgNames := aptDepsProvisionerPkgsForMinimumStemcellCompatibility
if p.fullStemcellCompatibility {
pkgNames = append(pkgNames, aptDepsProvisionerPkgsForFullStemcellCompatibility...)
}
stage := p.eventLog.BeginStage("Installing dependencies", len(pkgNames))
installedPkgNames, err := p.listInstalledPkgNames()
if err != nil {
return bosherr.WrapError(err, "Listing installed packages")
}
for _, pkgName := range pkgNames {
task := stage.BeginTask(fmt.Sprintf("Package %s", pkgName))
if p.isPkgInstalled(pkgName, installedPkgNames) {
p.logger.Debug(aptDepsProvisionerLogTag, "Package %s is already installed", pkgName)
task.End(nil)
continue
}
err := task.End(p.installPkg(pkgName))
if err != nil {
return bosherr.WrapError(err, "Installing %s", pkgName)
}
}
return nil
}
func NewConfigFromPath(path string, fs boshsys.FileSystem) (Config, error) {
var config Config
bytes, err := fs.ReadFile(path)
if err != nil {
return config, bosherr.WrapError(err, "Reading config %s", path)
}
config = DefaultConfig
err = json.Unmarshal(bytes, &config)
if err != nil {
return config, bosherr.WrapError(err, "Unmarshalling config")
}
if config.VMProvisioner.AgentProvisioner.Configuration == nil {
config.VMProvisioner.AgentProvisioner.Configuration = DefaultAgentConfiguration
}
err = config.validate()
if err != nil {
return config, bosherr.WrapError(err, "Validating config")
}
return config, nil
}
func NewWatchTimeFromString(str string) (WatchTime, error) {
var watchTime WatchTime
parts := strings.Split(str, "-")
if len(parts) != 2 {
return watchTime, bosherr.New("Invalid watch time range %s", str)
}
min, err := strconv.Atoi(strings.Trim(parts[0], " "))
if err != nil {
return watchTime, bosherr.WrapError(
err, "Non-positive number as watch time minimum %s", parts[0])
}
max, err := strconv.Atoi(strings.Trim(parts[1], " "))
if err != nil {
return watchTime, bosherr.WrapError(
err, "Non-positive number as watch time maximum %s", parts[1])
}
if max < min {
return watchTime, bosherr.New(
"Watch time must have maximum greater than or equal minimum %s", str)
}
watchTime[0], watchTime[1] = min, max
return watchTime, nil
}
func (v SyntaxValidator) validateJob(job *Job) error {
if job.Name == "" {
return bosherr.New("Missing job name")
}
if job.Template != nil {
return bosherr.New("'template' is deprecated in favor of 'templates'")
}
err := v.validateUpdate(&job.Update)
if err != nil {
return bosherr.WrapError(err, "Update")
}
props, err := bputil.NewStringKeyed().ConvertMap(job.PropertiesRaw)
if err != nil {
return bosherr.WrapError(err, "Properties")
}
job.Properties = props
for i, na := range job.NetworkAssociations {
err := v.validateNetworkAssociation(&job.NetworkAssociations[i])
if err != nil {
return bosherr.WrapError(err, "Network association %s (%d)", na.NetworkName, i)
}
}
return nil
}
func (net centosNetManager) SetupManualNetworking(networks boshsettings.Networks, errCh chan error) error {
net.logger.Debug(centosNetManagerLogTag, "Configuring manual networking")
modifiedNetworks, err := net.writeIfcfgs(networks)
if err != nil {
return bosherr.WrapError(err, "Writing network interfaces")
}
net.restartNetwork()
err = net.writeResolvConf(networks)
if err != nil {
return bosherr.WrapError(err, "Writing resolv.conf")
}
addresses := toInterfaceAddresses(modifiedNetworks)
go func() {
net.addressBroadcaster.BroadcastMACAddresses(addresses)
if errCh != nil {
errCh <- nil
}
}()
return nil
}