// New returns a new instance of OnBuild builder
func New(config *api.Config, overrides build.Overrides) (*OnBuild, error) {
dockerHandler, err := docker.New(config.DockerConfig, config.PullAuthentication)
if err != nil {
return nil, err
}
b := &OnBuild{
docker: dockerHandler,
git: git.New(),
fs: util.NewFileSystem(),
tar: tar.New(),
}
// Use STI Prepare() and download the 'run' script optionally.
s, err := sti.New(config, overrides)
s.SetScripts([]string{}, []string{api.Assemble, api.Run})
downloader := overrides.Downloader
if downloader == nil {
d, sourceURL, err := scm.DownloaderForSource(config.Source)
if err != nil {
return nil, err
}
downloader = d
config.Source = sourceURL
}
b.source = onBuildSourceHandler{
Downloader: downloader,
Preparer: s,
Ignorer: &ignore.DockerIgnorer{},
}
b.garbage = &build.DefaultCleaner{b.fs, b.docker}
return b, nil
}
// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string) (build.Downloader, string, error) {
glog.V(4).Infof("DownloadForSource %s", s)
details, mods := git.ParseFile(s)
glog.V(4).Infof("return from ParseFile file exists %v proto specified %v use copy %v", details.FileExists, details.ProtoSpecified, details.UseCopy)
if details.FileExists && details.BadRef {
return nil, s, fmt.Errorf("local location referenced by %s exists but the input after the # is malformed", s)
}
if details.FileExists && mods != nil {
glog.V(4).Infof("new path from parse file %s", mods.Path)
s = mods.Path
}
if details.FileExists && details.UseCopy {
return &file.File{util.NewFileSystem()}, s, nil
}
// If the source is valid GIT protocol (file://, ssh://, git://, git@, etc..) use GIT
// binary to download the sources
g := git.New()
if g.ValidCloneSpec(s) {
return &git.Clone{g, util.NewFileSystem()}, s, nil
}
return nil, s, fmt.Errorf("no downloader defined for location: %q", s)
}
// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string, forceCopy bool) (build.Downloader, string, error) {
glog.V(4).Infof("DownloadForSource %s", s)
details, mods := git.ParseFile(s)
glog.V(4).Infof("return from ParseFile file exists %v proto specified %v use copy %v", details.FileExists, details.ProtoSpecified, details.UseCopy)
if details.FileExists && details.BadRef {
return nil, s, fmt.Errorf("local location referenced by %s exists but the input after the # is malformed", s)
}
if details.FileExists && mods != nil {
glog.V(4).Infof("new source from parse file %s", mods.Path)
if details.ProtoSpecified {
s = mods.Path
} else {
// prepending with file:// is a precautionary step which previous incarnations of this code did; we
// preserve that behavior (it is more explicit, if not absolutely necessary; but we do it here as was done before
// vs. down in our generic git layer (which is leveraged separately in origin)
s = "file://" + mods.Path
}
}
if details.FileExists && (details.UseCopy || forceCopy) {
return &file.File{util.NewFileSystem()}, s, nil
}
// If the source is valid GIT protocol (file://, ssh://, git://, git@, etc..) use GIT
// binary to download the sources
g := git.New()
if g.ValidCloneSpec(s) {
return &git.Clone{g, util.NewFileSystem()}, s, nil
}
return nil, s, fmt.Errorf("no downloader defined for location: %q", s)
}
// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string) (build.Downloader, string, error) {
details, _ := git.ParseFile(s)
if details.FileExists && details.UseCopy {
if !details.ProtoSpecified {
// since not using git, any resulting URLs need to be explicit with file:// protocol specified
s = "file://" + s
}
return &file.File{util.NewFileSystem()}, s, nil
}
if details.ProtoSpecified && !details.FileExists {
return nil, s, fmt.Errorf("local location: %s does not exist", s)
}
if !details.ProtoSpecified && details.FileExists {
// if local file system, without file://, when using git, should not need file://, but we'll be safe;
// satisfies previous constructed test case in scm_test.go as well
s = "file://" + s
}
// If the source is valid GIT remote protocol (ssh://, git://, git@, etc..) use GIT
// binary to download the sources
g := git.New()
if g.ValidCloneSpec(s) {
return &git.Clone{g, util.NewFileSystem()}, s, nil
}
return nil, s, fmt.Errorf("no downloader defined for location: %q", s)
}
// New returns the instance of STI builder strategy for the given config.
// If the layeredBuilder parameter is specified, then the builder provided will
// be used for the case that the base Docker image does not have 'tar' or 'bash'
// installed.
func New(req *api.Config, overrides build.Overrides) (*STI, error) {
docker, err := dockerpkg.New(req.DockerConfig, req.PullAuthentication)
if err != nil {
return nil, err
}
var incrementalDocker dockerpkg.Docker
if req.Incremental {
incrementalDocker, err = dockerpkg.New(req.DockerConfig, req.IncrementalAuthentication)
if err != nil {
return nil, err
}
}
inst := scripts.NewInstaller(req.BuilderImage, req.ScriptsURL, docker, req.PullAuthentication)
b := &STI{
installer: inst,
config: req,
docker: docker,
incrementalDocker: incrementalDocker,
git: git.New(),
fs: util.NewFileSystem(),
tar: tar.New(),
callbackInvoker: util.NewCallbackInvoker(),
requiredScripts: []string{api.Assemble, api.Run},
optionalScripts: []string{api.SaveArtifacts},
externalScripts: map[string]bool{},
installedScripts: map[string]bool{},
scriptsURL: map[string]string{},
}
// The sources are downloaded using the GIT downloader.
// TODO: Add more SCM in future.
// TODO: explicit decision made to customize processing for usage specifically vs.
// leveraging overrides; also, we ultimately want to simplify s2i usage a good bit,
// which would lead to replacing this quick short circuit (so this change is tactical)
b.source = overrides.Downloader
if b.source == nil && !req.Usage {
downloader, sourceURL, err := scm.DownloaderForSource(req.Source, req.ForceCopy)
if err != nil {
return nil, err
}
b.source = downloader
req.Source = sourceURL
}
b.garbage = &build.DefaultCleaner{b.fs, b.docker}
b.layered, err = layered.New(req, b, overrides)
// Set interfaces
b.preparer = b
// later on, if we support say .gitignore func in addition to .dockerignore func, setting
// ignorer will be based on config setting
b.ignorer = &ignore.DockerIgnorer{}
b.artifacts = b
b.scripts = b
b.postExecutor = b
return b, err
}
// NewDockerBuilder creates a new instance of DockerBuilder
func NewDockerBuilder(dockerClient DockerClient, build *api.Build) *DockerBuilder {
return &DockerBuilder{
dockerClient: dockerClient,
build: build,
git: git.New(),
tar: tar.New(),
urlTimeout: urlCheckTimeout,
}
}
// New returns the instance of STI builder strategy for the given config.
// If the layeredBuilder parameter is specified, then the builder provided will
// be used for the case that the base Docker image does not have 'tar' or 'bash'
// installed.
func New(req *api.Config, overrides build.Overrides) (*STI, error) {
docker, err := dockerpkg.New(req.DockerConfig, req.PullAuthentication)
if err != nil {
return nil, err
}
var incrementalDocker dockerpkg.Docker
if req.Incremental {
incrementalDocker, err = dockerpkg.New(req.DockerConfig, req.IncrementalAuthentication)
if err != nil {
return nil, err
}
}
inst := scripts.NewInstaller(req.BuilderImage, req.ScriptsURL, docker, req.PullAuthentication)
b := &STI{
installer: inst,
config: req,
docker: docker,
incrementalDocker: incrementalDocker,
git: git.New(),
fs: util.NewFileSystem(),
tar: tar.New(),
callbackInvoker: util.NewCallbackInvoker(),
requiredScripts: []string{api.Assemble, api.Run},
optionalScripts: []string{api.SaveArtifacts},
externalScripts: map[string]bool{},
installedScripts: map[string]bool{},
scriptsURL: map[string]string{},
}
// The sources are downloaded using the GIT downloader.
// TODO: Add more SCM in future.
b.source = overrides.Downloader
if b.source == nil {
downloader, sourceURL, err := scm.DownloaderForSource(req.Source)
if err != nil {
return nil, err
}
b.source = downloader
req.Source = sourceURL
}
b.garbage = &build.DefaultCleaner{b.fs, b.docker}
b.layered, err = layered.New(req, b, overrides)
// Set interfaces
b.preparer = b
// later on, if we support say .gitignore func in addition to .dockerignore func, setting
// ignorer will be based on config setting
b.ignorer = &ignore.DockerIgnorer{}
b.artifacts = b
b.scripts = b
b.postExecutor = b
return b, err
}
// IsRemoteRepository checks whether the provided string is a remote repository or not
func IsRemoteRepository(s string) bool {
if !s2igit.New().ValidCloneSpecRemoteOnly(s) {
return false
}
gitRepo := git.NewRepository()
if err := gitRepo.ListRemote(s); err != nil {
return false
}
return true
}
// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
ok, err := s2igit.New().ValidCloneSpec(rawurl)
if err != nil {
return fmt.Errorf("Invalid git source url %q: %v", rawurl, err)
}
if !ok {
return fmt.Errorf("Invalid git source url: %s", rawurl)
}
return checkRemoteGit(gitClient, rawurl, timeout)
}
// newSTIBuilder is the internal factory function to create STIBuilder based on parameters. Used for testing.
func newSTIBuilder(client DockerClient, dockerSocket string, build *api.Build,
builder builderFactory, validator validator) *STIBuilder {
// just create instance
return &STIBuilder{
builder: builder,
validator: validator,
git: git.New(),
dockerClient: client,
dockerSocket: dockerSocket,
build: build,
}
}
// newS2IBuilder is the internal factory function to create STIBuilder based on parameters. Used for testing.
func newS2IBuilder(dockerClient DockerClient, dockerSocket string, buildsClient client.BuildInterface, build *api.Build,
builder builderFactory, validator validator) *S2IBuilder {
// just create instance
return &S2IBuilder{
builder: builder,
validator: validator,
git: git.New(),
dockerClient: dockerClient,
dockerSocket: dockerSocket,
build: build,
client: buildsClient,
}
}
// IsRemoteRepository checks whether the provided string is a remote repository or not
func IsRemoteRepository(s string) bool {
if !s2igit.New(s2iutil.NewFileSystem()).ValidCloneSpecRemoteOnly(s) {
return false
}
url, err := url.Parse(s)
if err != nil {
return false
}
url.Fragment = ""
gitRepo := git.NewRepository()
if _, _, err := gitRepo.ListRemote(url.String()); err != nil {
return false
}
return true
}
// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string) build.Downloader {
// If the source starts with file:// and there is no GIT binary, use 'file'
// SCM plugin
if (strings.HasPrefix(s, "file://") || strings.HasPrefix(s, "/")) && !hasGitBinary() {
return &file.File{util.NewFileSystem()}
}
g := git.New()
if g.ValidCloneSpec(s) {
return &git.Clone{g, util.NewFileSystem()}
}
glog.Errorf("No downloader defined for %q source URL", s)
return nil
}
// ParseRepository parses a string that may be in the Git format (git@) or URL format
// and extracts the appropriate value. Any fragment on the URL is preserved.
//
// Protocols returned:
// - http, https
// - file
// - git
// - ssh
func ParseRepository(s string) (*url.URL, error) {
uri, err := url.Parse(s)
if err != nil {
return nil, err
}
// There are some shortcomings with url.Parse when it comes to GIT, namely wrt
// the GIT local/file and ssh protocols - it does not handle implied schema (i.e. no <proto>:// prefix)well;
// We handle those caveats here
err = s2igit.New().MungeNoProtocolURL(s, uri)
if err != nil {
return nil, err
}
return uri, nil
}
// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(fs util.FileSystem, s string, forceCopy bool) (build.Downloader, string, error) {
glog.V(4).Infof("DownloadForSource %s", s)
if len(s) == 0 {
return &empty.Noop{}, s, nil
}
details, mods, err := git.ParseFile(fs, s)
glog.V(4).Infof("return from ParseFile file exists %v proto specified %v use copy %v", details.FileExists, details.ProtoSpecified, details.UseCopy)
if err != nil {
if e, ok := err.(errors.Error); !forceCopy || !(ok && (e.ErrorCode == errors.EmptyGitRepositoryError)) {
return nil, s, err
}
}
if details.FileExists && details.BadRef {
return nil, s, fmt.Errorf("local location referenced by %s exists but the input after the # is malformed", s)
}
if details.FileExists && mods != nil {
glog.V(4).Infof("new source from parse file %s", mods.Path)
s = "file://" + mods.Path
}
if details.FileExists && (details.UseCopy || forceCopy) {
return &file.File{FileSystem: fs}, s, nil
}
// If the source is valid Git protocol (file://, ssh://, git://, git@, etc..) use Git
// binary to download the sources
g := git.New(fs)
ok, err := g.ValidCloneSpec(s)
if err != nil {
return nil, s, err
}
if ok {
return &git.Clone{Git: g, FileSystem: fs}, s, nil
}
return nil, s, fmt.Errorf("no downloader defined for location: %q", s)
}
// ParseRepository parses a string that may be in the Git format (git@) or URL format
// and extracts the appropriate value. Any fragment on the URL is preserved.
//
// Protocols returned:
// - http, https
// - file
// - git
// - ssh
func ParseRepository(s string) (*url.URL, error) {
uri, err := url.Parse(s)
if err != nil {
return nil, err
}
// There are some shortcomings with url.Parse when it comes to GIT, namely wrt
// the GIT local/file and ssh protocols - it does not handle implied schema (i.e. no <proto>:// prefix)well;
// We handle those caveats here
err = s2igit.New().MungeNoProtocolURL(s, uri)
if err != nil {
return nil, err
}
//TODO temporary work around for s2i's MungeNoProtocolURL (which is not used in s2i at the moment)
// there is some overloaded usage between the uri Path and the Path needed for DownloaderForSource
// this temporary work around can sit here even after the s2i change (including duplicating tests in this dir's git_test.go to source-to-image/pkg/scm/git/git_test.go)
// gets Godeps into origin, and then we can remove this
uri.Path = strings.TrimPrefix(uri.Path, "file://")
return uri, nil
}
// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string) (build.Downloader, string, error) {
// If the source is using file:// protocol but it is not a GIT repository,
// trim the prefix and treat it as a file copy.
if strings.HasPrefix(s, "file://") && !isLocalGitRepository(s) {
s = strings.TrimPrefix(s, "file://")
}
// If the source is file:// protocol and it is GIT repository, but we don't
// have GIT binary to fetch it, treat it as file copy.
if strings.HasPrefix(s, "file://") && !hasGitBinary() {
s = strings.TrimPrefix(s, "file://")
}
// If the source is valid GIT protocol (file://, git://, git@, etc..) use GIT
// binary to download the sources
if g := git.New(); g.ValidCloneSpec(s) {
return &git.Clone{g, util.NewFileSystem()}, s, nil
}
// Convert relative path to absolute path.
if !strings.HasPrefix(s, "/") {
if absolutePath, err := filepath.Abs(s); err == nil {
s = absolutePath
}
}
if isLocalGitRepository(s) {
return DownloaderForSource("file://" + s)
}
// If we have local directory and that directory exists, use file copy
if _, err := os.Stat(s); err == nil {
return &file.File{util.NewFileSystem()}, s, nil
}
return nil, s, fmt.Errorf("No downloader defined for location: %q", s)
}
// New returns the instance of STI builder strategy for the given config.
// If the layeredBuilder parameter is specified, then the builder provided will
// be used for the case that the base Docker image does not have 'tar' or 'bash'
// installed.
func New(config *api.Config, overrides build.Overrides) (*STI, error) {
docker, err := dockerpkg.New(config.DockerConfig, config.PullAuthentication)
if err != nil {
return nil, err
}
var incrementalDocker dockerpkg.Docker
if config.Incremental {
incrementalDocker, err = dockerpkg.New(config.DockerConfig, config.IncrementalAuthentication)
if err != nil {
return nil, err
}
}
inst := scripts.NewInstaller(config.BuilderImage, config.ScriptsURL, config.ScriptDownloadProxyConfig, docker, config.PullAuthentication)
tarHandler := tar.New()
tarHandler.SetExclusionPattern(regexp.MustCompile(config.ExcludeRegExp))
builder := &STI{
installer: inst,
config: config,
docker: docker,
incrementalDocker: incrementalDocker,
git: git.New(),
fs: util.NewFileSystem(),
tar: tarHandler,
callbackInvoker: util.NewCallbackInvoker(),
requiredScripts: []string{api.Assemble, api.Run},
optionalScripts: []string{api.SaveArtifacts},
optionalRuntimeScripts: []string{api.AssembleRuntime},
externalScripts: map[string]bool{},
installedScripts: map[string]bool{},
scriptsURL: map[string]string{},
}
if len(config.RuntimeImage) > 0 {
builder.runtimeInstaller = scripts.NewInstaller(config.RuntimeImage, config.ScriptsURL, config.ScriptDownloadProxyConfig, docker, config.PullAuthentication)
builder.runtimeDocker, err = dockerpkg.New(config.DockerConfig, config.RuntimeAuthentication)
if err != nil {
return builder, err
}
}
// The sources are downloaded using the Git downloader.
// TODO: Add more SCM in future.
// TODO: explicit decision made to customize processing for usage specifically vs.
// leveraging overrides; also, we ultimately want to simplify s2i usage a good bit,
// which would lead to replacing this quick short circuit (so this change is tactical)
builder.source = overrides.Downloader
if builder.source == nil && !config.Usage {
downloader, sourceURL, err := scm.DownloaderForSource(config.Source, config.ForceCopy)
if err != nil {
return nil, err
}
builder.source = downloader
config.Source = sourceURL
}
builder.garbage = build.NewDefaultCleaner(builder.fs, builder.docker)
builder.layered, err = layered.New(config, builder, overrides)
// Set interfaces
builder.preparer = builder
// later on, if we support say .gitignore func in addition to .dockerignore func, setting
// ignorer will be based on config setting
builder.ignorer = &ignore.DockerIgnorer{}
builder.artifacts = builder
builder.scripts = builder
builder.postExecutor = builder
builder.initPostExecutorSteps()
return builder, err
}
// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
if !s2igit.New().ValidCloneSpec(rawurl) {
return fmt.Errorf("Invalid git source url: %s", rawurl)
}
return checkRemoteGit(gitClient, rawurl, timeout)
}