// DownloadStep ...
func DownloadStep(collectionURI string, collection models.StepCollectionModel, id, version, commithash string) error {
downloadLocations, err := collection.GetDownloadLocations(id, version)
if err != nil {
return err
}
route, found := ReadRoute(collectionURI)
if !found {
return fmt.Errorf("No routing found for lib: %s", collectionURI)
}
stepPth := GetStepCacheDirPath(route, id, version)
if exist, err := pathutil.IsPathExists(stepPth); err != nil {
return err
} else if exist {
return nil
}
success := false
for _, downloadLocation := range downloadLocations {
switch downloadLocation.Type {
case "zip":
err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
return cmdex.DownloadAndUnZIP(downloadLocation.Src, stepPth)
})
if err != nil {
log.Warn("Failed to download step.zip: ", err)
} else {
success = true
return nil
}
case "git":
err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
return cmdex.GitCloneTagOrBranchAndValidateCommitHash(downloadLocation.Src, stepPth, version, commithash)
})
if err != nil {
log.Warnf("Failed to clone step (%s): %v", downloadLocation.Src, err)
} else {
success = true
return nil
}
default:
return fmt.Errorf("Failed to download: Invalid download location (%#v) for step %#v (%#v)", downloadLocation, id, version)
}
}
if !success {
return errors.New("Failed to download step")
}
return nil
}
// DownloadStep ...
func DownloadStep(collectionURI string, collection models.StepCollectionModel, id, version, commithash string) error {
log.Debugf("Download Step: %#v (%#v)\n", id, version)
downloadLocations, err := collection.GetDownloadLocations(id, version)
if err != nil {
return err
}
route, found := ReadRoute(collectionURI)
if !found {
return fmt.Errorf("No routing found for lib: %s", collectionURI)
}
stepPth := GetStepCacheDirPath(route, id, version)
if exist, err := pathutil.IsPathExists(stepPth); err != nil {
return err
} else if exist {
log.Debug("[STEPMAN] - Step already downloaded")
return nil
}
success := false
for _, downloadLocation := range downloadLocations {
switch downloadLocation.Type {
case "zip":
log.Debug("[STEPMAN] - Downloading step from: ", downloadLocation.Src)
if err := cmdex.DownloadAndUnZIP(downloadLocation.Src, stepPth); err != nil {
log.Warn("[STEPMAN] - Failed to download step.zip: ", err)
} else {
success = true
return nil
}
case "git":
log.Debug("[STEPMAN] - Git clone step from: ", downloadLocation.Src)
if err := cmdex.GitCloneTagOrBranchAndValidateCommitHash(downloadLocation.Src, stepPth, version, commithash); err != nil {
log.Warnf("[STEPMAN] - Failed to clone step (%s): %v", downloadLocation.Src, err)
} else {
success = true
return nil
}
default:
return fmt.Errorf("[STEPMAN] - Failed to download: Invalid download location (%#v) for step %#v (%#v)", downloadLocation, id, version)
}
}
if !success {
return errors.New("Failed to download step")
}
return nil
}
func convertToMinimalSpec(stepLib models.StepCollectionModel) models.StepCollectionModel {
steps := stepLib.Steps
minimalSteps := models.StepHash{}
for stepID := range steps {
minimalSteps[stepID] = models.StepGroupModel{}
}
stepLib.Steps = minimalSteps
return stepLib
}
func convertToLatestSpec(stepLib models.StepCollectionModel) models.StepCollectionModel {
steps := stepLib.Steps
latestSteps := models.StepHash{}
for stepID, stepGroup := range steps {
groupInfo := stepGroup.Info
versions := stepGroup.Versions
latestVersionStr := stepGroup.LatestVersionNumber
latestStep := versions[latestVersionStr]
latestSteps[stepID] = models.StepGroupModel{
Versions: map[string]models.StepModel{
latestVersionStr: latestStep,
},
Info: groupInfo,
}
}
stepLib.Steps = latestSteps
return stepLib
}
func generateStepLib(route SteplibRoute, templateCollection models.StepCollectionModel) (models.StepCollectionModel, error) {
collection := models.StepCollectionModel{
FormatVersion: templateCollection.FormatVersion,
GeneratedAtTimeStamp: time.Now().Unix(),
SteplibSource: templateCollection.SteplibSource,
DownloadLocations: templateCollection.DownloadLocations,
AssetsDownloadBaseURI: templateCollection.AssetsDownloadBaseURI,
}
stepHash := models.StepHash{}
stepsSpecDirPth := GetCollectionBaseDirPath(route)
err := filepath.Walk(stepsSpecDirPth, func(pth string, f os.FileInfo, err error) error {
truncatedPath := strings.Replace(pth, stepsSpecDirPth+"/", "", -1)
match, matchErr := regexp.MatchString("([a-z]+).yml", truncatedPath)
if matchErr != nil {
return matchErr
}
if match {
components := strings.Split(truncatedPath, "/")
if len(components) == 4 {
stepsDirName := components[0]
stepID := components[1]
stepVersion := components[2]
step, parseErr := ParseStepYml(pth, true)
if parseErr != nil {
return parseErr
}
stepGroupInfo := models.StepGroupInfoModel{}
// Check for step-info.yml - STEP_SPEC_DIR/steps/step-id/step-info.yml
stepGroupInfoPth := filepath.Join(stepsSpecDirPth, stepsDirName, stepID, "step-info.yml")
if exist, err := pathutil.IsPathExists(stepGroupInfoPth); err != nil {
return err
} else if exist {
deprecationInfo, err := ParseStepGroupInfo(stepGroupInfoPth)
if err != nil {
return err
}
stepGroupInfo.RemovalDate = deprecationInfo.RemovalDate
stepGroupInfo.DeprecateNotes = deprecationInfo.DeprecateNotes
}
// Check for assets - STEP_SPEC_DIR/steps/step-id/assets
if collection.AssetsDownloadBaseURI != "" {
assetsFolderPth := path.Join(stepsSpecDirPth, stepsDirName, stepID, "assets")
exist, err := pathutil.IsPathExists(assetsFolderPth)
if err != nil {
return err
}
if exist {
assetsMap := map[string]string{}
err := filepath.Walk(assetsFolderPth, func(pth string, f os.FileInfo, err error) error {
_, file := filepath.Split(pth)
if pth != assetsFolderPth && file != "" {
assetURI, err := urlutil.Join(collection.AssetsDownloadBaseURI, stepID, "assets", file)
if err != nil {
return err
}
assetsMap[file] = assetURI
}
return nil
})
if err != nil {
return err
}
step.AssetURLs = assetsMap
stepGroupInfo.AssetURLs = assetsMap
}
}
// Add to stepgroup
stepGroup, found := stepHash[stepID]
if !found {
stepGroup = models.StepGroupModel{
Versions: map[string]models.StepModel{},
}
}
stepGroup, err = addStepVersionToStepGroup(step, stepVersion, stepGroup)
if err != nil {
return err
}
stepGroup.Info = stepGroupInfo
stepHash[stepID] = stepGroup
} else {
}
}
return err
})
if err != nil {
log.Error("Failed to walk through path:", err)
return models.StepCollectionModel{}, err
}
collection.Steps = stepHash
return collection, nil
}
func generateStepLib(route SteplibRoute, templateCollection models.StepCollectionModel) (models.StepCollectionModel, error) {
collection := models.StepCollectionModel{
FormatVersion: templateCollection.FormatVersion,
GeneratedAtTimeStamp: time.Now().Unix(),
SteplibSource: templateCollection.SteplibSource,
DownloadLocations: templateCollection.DownloadLocations,
AssetsDownloadBaseURI: templateCollection.AssetsDownloadBaseURI,
}
stepHash := models.StepHash{}
stepsSpecDir := GetCollectionBaseDirPath(route)
log.Debugln(" stepsSpecDir: ", stepsSpecDir)
err := filepath.Walk(stepsSpecDir, func(pth string, f os.FileInfo, err error) error {
truncatedPath := strings.Replace(pth, stepsSpecDir+"/", "", -1)
match, matchErr := regexp.MatchString("([a-z]+).yml", truncatedPath)
if matchErr != nil {
return matchErr
}
if match {
components := strings.Split(truncatedPath, "/")
if len(components) == 4 {
id := components[1]
version := components[2]
log.Debugf("Start parsing (StepId:%s) (Version:%s)", id, version)
step, parseErr := ParseStepYml(pth, true)
if parseErr != nil {
log.Debugf(" Failed to parse StepId: %v Version: %v", id, version)
return parseErr
}
// Check for assets
if collection.AssetsDownloadBaseURI != "" {
assetsFolderPth := path.Join(stepsSpecDir, components[0], components[1], "assets")
exist, err := pathutil.IsPathExists(assetsFolderPth)
if err != nil {
return err
}
if exist {
assetsMap := map[string]string{}
err := filepath.Walk(assetsFolderPth, func(pth string, f os.FileInfo, err error) error {
_, file := filepath.Split(pth)
if pth != assetsFolderPth && file != "" {
assetURI, err := urlutil.Join(collection.AssetsDownloadBaseURI, id, "assets", file)
if err != nil {
return err
}
assetsMap[file] = assetURI
}
return nil
})
if err != nil {
log.Debugf(" Failed to add assets, at (%s) | Error: %v", assetsFolderPth, err)
return err
}
step.AssetURLs = assetsMap
}
}
// Add to stepgroup
stepGroup, found := stepHash[id]
if !found {
stepGroup = models.StepGroupModel{
Versions: map[string]models.StepModel{},
}
}
stepGroup, err = addStepVersionToStepGroup(step, version, stepGroup)
if err != nil {
log.Debugf(" Failed to add step to step-group. (StepId:%v) (Version: %v) | Error: %v", id, version, err)
return err
}
stepHash[id] = stepGroup
} else {
log.Debug(" * Path:", truncatedPath)
log.Debug(" * Legth:", len(components))
}
}
return err
})
if err != nil {
log.Error("[STEPMAN] - Failed to walk through path:", err)
return models.StepCollectionModel{}, err
}
collection.Steps = stepHash
return collection, nil
}