func getScripts(dirPath string) (scripts []Script, err error) {
files, err := ioutil.ReadDir(dirPath)
if err != nil {
except.Warn(err, ": Error in reading contents of ", dirPath)
return
}
for _, file := range files {
file.Name()
//figure out type of script
var script Script
switch {
case strings.HasSuffix(file.Name(), ".sh"):
blog.Debug("dirpath: " + dirPath + " after removing prefix: " + config.BANYANDIR() + " looks like: " + strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"))
script = newBashScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
case strings.HasSuffix(file.Name(), ".py"):
script = newPythonScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
default:
except.Warn("Unknown script file type for: " + file.Name())
//Ignore this file...
continue
}
scripts = append(scripts, script)
}
return
}
// GetRegistryURL determines the full URL, with or without HTTP Basic Auth, needed to
// access the registry or Docker Hub.
func GetRegistryURL() (URL string, hubAPI bool, BasicAuth string, XRegistryAuth string) {
basicAuth, fullRegistry, XRegistryAuth := RegAuth(RegistrySpec)
if *AuthRegistry == true {
if basicAuth == "" {
except.Fail("Registry auth could not be determined from docker config.")
}
BasicAuth = basicAuth
}
if *HTTPSRegistry == false {
URL = "http://" + RegistrySpec
} else {
// HTTPS is required
if strings.HasPrefix(fullRegistry, "https://") {
URL = fullRegistry
} else {
URL = "https://" + RegistrySpec
}
if *RegistryTokenAuth == true {
hubAPI = true
}
if strings.Contains(URL, "docker.io") || strings.Contains(URL, "gcr.io") {
hubAPI = true
if *RegistryTokenAuth == false {
except.Warn("Forcing --registrytokenauth=true, as required for Docker Hub and Google Container Registry")
*RegistryTokenAuth = true
}
}
}
return
}
// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
blog.Info("Writing image (pkg and other) data into file...")
for imageID, scriptMap := range outMapMap {
for scriptName, out := range scriptMap {
scriptDir := f.dir + "/" + trimExtension(scriptName)
err := fsutil.CreateDirIfNotExist(scriptDir)
if err != nil {
except.Error(err, ": Error creating script dir: ", scriptDir)
continue
}
image := string(imageID)
if len(image) < 12 {
except.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
} else {
image = string(imageID)[0:12]
}
filenamePath := scriptDir + "/" + image
if _, ok := out.([]byte); ok {
f.format = "txt"
filenamePath += "-miscdata"
} else {
// by default it is json. But f.format could get overwritten at any point
// in the for loop if the output type is []byte, hence the (re)assignment
f.format = "json"
// NOTE: If we start using json for output other than imageData, change this
filenamePath += "-pkgdata"
}
f.writeFileInFormat(filenamePath, &out)
}
}
return
}
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) {
switch f.format {
case "json":
err := jsonifyAndAppendToFile(filenamePath+".json", data)
if err != nil {
except.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
return
}
default:
except.Warn("Currently only supporting json output to write to files")
}
}
func (f *FileWriter) handleImageMetadata(imageMetadata []ImageMetadataInfo, action string) {
if len(imageMetadata) == 0 {
except.Warn("No image metadata to append to file...")
return
}
// If output directory does not exist, first create it
fsutil.CreateDirIfNotExist(f.dir)
filenamePath := f.dir + "/" + "metadata"
data := ImageMetadataAndAction{action, imageMetadata}
f.appendFileInFormat(filenamePath, data)
}
// RemoveObsoleteMetadata removes obsolete metadata from the Banyan service.
func RemoveObsoleteMetadata(obsolete []ImageMetadataInfo) {
if len(obsolete) == 0 {
except.Warn("No image metadata to save!")
return
}
config.BanyanUpdate("Remove Metadata", statusMessageMD(obsolete))
for _, writer := range WriterList {
writer.RemoveImageMetadata(obsolete)
}
return
}
// SaveImageMetadata saves image metadata to selected storage location
// (standard output, Banyan service, etc.).
func SaveImageMetadata(metadataSlice []ImageMetadataInfo) {
if len(metadataSlice) == 0 {
except.Warn("No image metadata to save!")
return
}
config.BanyanUpdate("Save Image Metadata", statusMessageMD(metadataSlice))
for _, writer := range WriterList {
writer.AppendImageMetadata(metadataSlice)
}
return
}
// getDistroID takes a distribution "pretty name" as input and returns the corresponding
// distribution ID, or "Unknown" if no match can be found.
func getDistroID(distroName string) string {
if id, ok := DistroMap[distroName]; ok {
return id
}
//Exceptions to the rule: There are many such cases, so bucketing them together
if strings.HasPrefix(distroName, `Ubuntu 14.04`) {
return "UBUNTU-trusty"
}
if strings.HasPrefix(distroName, `Ubuntu precise`) {
return "UBUNTU-precise"
}
if strings.HasPrefix(distroName, `Ubuntu 12.04`) {
return "UBUNTU-precise"
}
if strings.HasPrefix(distroName, `Ubuntu 10.04`) {
return "UBUNTU-lucid"
}
if strings.HasPrefix(distroName, `CentOS release 5`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 5`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 5`) {
m := distroRegexp[rel5z].FindStringSubmatch(distroName)
if len(m) > 1 {
return "REDHAT-5Server-5." + m[1]
}
return "REDHAT-5Server"
}
if strings.HasPrefix(distroName, `CentOS release 6`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 6`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 6`) {
m := distroRegexp[rel6z].FindStringSubmatch(distroName)
if len(m) > 1 {
return "REDHAT-6Server-6." + m[1]
}
return "REDHAT-6Server"
}
if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 7`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 7`) {
return "REDHAT-7Server"
}
if strings.HasPrefix(distroName, `Ubuntu Vivid`) {
return "UBUNTU-vivid"
}
if strings.HasPrefix(distroName, `Ubuntu Wily`) {
return "UBUNTU-wily"
}
except.Warn("DISTRO %s UNKNOWN", distroName)
return "Unknown"
}
// GetLocalImageMetadata returns image metadata queried from a local Docker host.
// Query the local docker daemon to detect new image builds on the host and new images pulled from registry by users.
func GetLocalImageMetadata(oldMetadataSet MetadataSet) (metadataSlice []ImageMetadataInfo) {
for {
blog.Info("Get a list of images from local Docker daemon")
imageMap, e := GetLocalImages(true, true)
if e != nil {
except.Warn(e, " GetLocalImages")
except.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
blog.Info("Get Image Metadata from local Docker daemon")
// Get image metadata
metadataSlice, e = GetImageMetadataSpecified(imageMap, oldMetadataSet)
if e != nil {
except.Warn(e, " GetImageMetadata")
except.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
break
}
return
}
func getScriptsToRun() (scripts []Script) {
// get default scripts
defaultScripts, err := getScripts(DefaultScriptsDir)
if err != nil {
except.Fail(err, ": Error in getting default scripts")
}
// get user-specified scripts
userScripts, err := getScripts(UserScriptsDir)
if err != nil {
except.Warn(err, ": Error in getting user-specified scripts")
}
scripts = append(defaultScripts, userScripts...)
return
}
// getImageList reads the list of previously processed images from the imageList file.
func getImageList(processedImages collector.ImageSet) (e error) {
f, e := os.Open(*imageList)
if e != nil {
except.Warn(e, ": Error in opening", *imageList, ": perhaps a fresh start?")
return
}
defer f.Close()
r := bufio.NewReader(f)
data, e := ioutil.ReadAll(r)
if e != nil {
except.Error(e, ": Error in reading file ", *imageList)
return
}
for _, str := range strings.Split(string(data), "\n") {
if len(str) != 0 {
blog.Debug("Previous image: %s", str)
processedImages[collector.ImageIDType(str)] = true
}
}
return
}
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) {
blog.Info("Writing " + filenamePath + "...")
switch f.format {
case "json":
err := jsonifyAndWriteToFile(filenamePath+".json", data)
if err != nil {
except.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
return
}
case "txt":
// what's passed in is ptr to interface{}. First get interface{} out of it and then
// typecast that to []byte
err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644)
if err != nil {
except.Error(err, ": Error in writing to file: ", filenamePath)
return
}
default:
except.Warn("Currently only supporting json output to write to files")
}
}
// getImageManifestHashList reads the list of previously processed images (manifest hash) from the imageList_ManifestHash file.
func getImageManifestHashList(processedImagesManifestHash collector.ImageSet) (e error) {
filename := *imageList + "_ManifestHash"
f, e := os.Open(filename)
if e != nil {
except.Warn(e, ": Error in opening", filename, ": perhaps a fresh start?")
return
}
defer f.Close()
r := bufio.NewReader(f)
data, e := ioutil.ReadAll(r)
if e != nil {
except.Error(e, ": Error in reading file ", filename)
return
}
for _, str := range strings.Split(string(data), "\n") {
if len(str) != 0 {
blog.Debug("Previous image: %s", str)
processedImagesManifestHash.Insert(collector.ImageIDType(str))
}
}
return
}
// SaveImageMetadata saves image metadata to selected storage location
// (standard output, Banyan service, etc.).
func SaveImageMetadata(metadataSlice []ImageMetadataInfo) {
if len(metadataSlice) == 0 {
except.Warn("No image metadata to save!")
return
}
config.BanyanUpdate("Save Image Metadata", statusMessageMD(metadataSlice))
slice := []ImageMetadataInfo{}
for _, metadata := range metadataSlice {
if len(metadata.Image) > 0 {
slice = append(slice, metadata)
}
}
if len(slice) == 0 {
return
}
for _, writer := range WriterList {
writer.AppendImageMetadata(slice)
}
return
}
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
if len(ReposToProcess) == 0 {
return
}
client := &http.Client{}
metadataMap := NewImageToMetadataMap(oldMetadataSet)
allRepos := []RepoType{}
// Check if we need to use the search API, i.e. only one repo given, and ends in wildcard "*".
if searchTerm := NeedRegistrySearch(); searchTerm != "" {
blog.Info("Using search API")
var e error
allRepos, e = registrySearchV1(client, searchTerm)
if e != nil {
except.Error(e, ":registry search")
return
}
}
// If search wasn't needed, the repos were individually specified.
if len(allRepos) == 0 {
for repo := range ReposToProcess {
allRepos = append(allRepos, repo)
}
}
for _, repo := range allRepos {
blog.Info("Get index and tag info for %s", string(repo))
config.BanyanUpdate("Get index and tag info for", string(repo))
var (
indexInfo IndexInfo
e error
repoTagSlice []TagInfo
repoMetadataSlice []ImageMetadataInfo
)
// loop until success
for {
indexInfo, e = getReposTokenAuthV1(repo, client)
if e != nil {
except.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
if e != nil {
except.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
if len(repoTagSlice) != 1 {
except.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
time.Sleep(config.RETRYDURATION)
continue
}
repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
if e != nil {
except.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
//success!
break
}
tagSlice = append(tagSlice, repoTagSlice...)
metadataSlice = append(metadataSlice, repoMetadataSlice...)
}
return
}
// GetImageMetadata determines which image metadata is of interest and then calls
// GetImageMetadataSpecified to obtain and return the appropriate metadata,
// queried from a Docker registry or Docker daemon.
// If the user has specified the repositories to examine, then no other repositories are examined.
// If the user has not specified repositories, then the registry search API is used to
// get the list of all repositories in the registry.
func GetImageMetadata(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
for {
blog.Info("Get Repos")
repoSlice, e := getRepos()
if e != nil {
except.Warn(e, " getRepos")
except.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
if len(repoSlice) == 0 {
// For some reason (like, registry search doesn't work), we are not
// seeing any repos in the registry.
// So, just reconstruct the list of repos that we saw earlier.
except.Warn("Empty repoSlice, reusing previous metadata")
repomap := make(map[string]bool)
for metadata := range oldMetadataSet {
if repomap[metadata.Repo] == false {
repoSlice = append(repoSlice, RepoType(metadata.Repo))
repomap[metadata.Repo] = true
}
}
}
// Now get a list of all the tags, and the image metadata/manifest
if *RegistryProto == "v1" {
blog.Info("Get Tags")
tagSlice, e = getTags(repoSlice)
if e != nil {
except.Warn(e, " getTags")
except.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
// get map from each imageID to all of its aliases (repo+tag)
imageMap := make(ImageToRepoTagMap)
for _, ti := range tagSlice {
for tag, imageID := range ti.TagMap {
repotag := RepoTagType{Repo: ti.Repo, Tag: tag}
imageMap.Insert(imageID, repotag)
}
}
blog.Info("Get Image Metadata")
// Get image metadata
metadataSlice, e = GetImageMetadataSpecified(imageMap, oldMetadataSet)
if e != nil {
except.Warn(e, " GetImageMetadataSpecified")
except.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
break
}
if *RegistryProto == "v2" {
blog.Info("Get Tags and Metadata")
tagSlice, metadataSlice, e = v2GetTagsMetadata(repoSlice)
if e != nil {
except.Warn(e)
except.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
break
}
}
return
}