func getScripts(dirPath string) (scripts []Script, err error) {
files, err := ioutil.ReadDir(dirPath)
if err != nil {
blog.Warn(err, ": Error in reading contents of ", dirPath)
return
}
for _, file := range files {
file.Name()
//figure out type of script
var script Script
switch {
case strings.HasSuffix(file.Name(), ".sh"):
blog.Debug("dirpath: " + dirPath + " after removing prefix: " + config.BANYANDIR() + " looks like: " + strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"))
script = newBashScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
case strings.HasSuffix(file.Name(), ".py"):
script = newPythonScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
default:
blog.Warn("Unknown script file type for: " + file.Name())
//Ignore this file...
continue
}
scripts = append(scripts, script)
}
return
}
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
if len(ReposToProcess) == 0 {
return
}
client := &http.Client{}
metadataMap := NewImageToMetadataMap(oldMetadataSet)
for repo := range ReposToProcess {
blog.Info("Get index and tag info for %s", string(repo))
config.BanyanUpdate("Get index and tag info for", string(repo))
var (
indexInfo IndexInfo
e error
repoTagSlice []TagInfo
repoMetadataSlice []ImageMetadataInfo
)
// loop until success
for {
indexInfo, e = getReposTokenAuthV1(repo, client)
if e != nil {
blog.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
if e != nil {
blog.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
if len(repoTagSlice) != 1 {
blog.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
time.Sleep(config.RETRYDURATION)
continue
}
repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
if e != nil {
blog.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
//success!
break
}
tagSlice = append(tagSlice, repoTagSlice...)
metadataSlice = append(metadataSlice, repoMetadataSlice...)
}
return
}
func v2GetMetadata(client *http.Client, repo, tag string) (metadata ImageMetadataInfo, e error) {
response, err := RegistryQueryV2(client, RegistryAPIURL+"/v2/"+repo+"/manifests/"+tag)
if err != nil {
except.Error(err)
if s, ok := err.(*HTTPStatusCodeError); ok {
except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode)
e = err
}
return
}
metadata.Repo = repo
metadata.Tag = tag
metadata.Image = ""
// Recent versions of Docker daemon (1.8.3+?) have complex code that calculates
// the image ID from the V2 manifest.
// This seems to be in flux as Docker moves toward content-addressable images in 1.10+,
// and as the registry image manifest schema itself is still evolving.
// As a temporary measure until Docker converges to a more stable state, collector
// will calculate its own hash over the V2 manifest and use the calculated
// value to try to filter out images that have previously been processed.
// The Docker-calculated image ID will get added to the metadata struct
// after the image is pulled.
sum := sha256.Sum256(response)
metadata.ManifestHash = hex.EncodeToString(sum[:])
var m ManifestV2Schema1
b := bytes.NewBuffer(response)
if e = json.NewDecoder(b).Decode(&m); e != nil {
blog.Warn("Failed to parse manifest")
return
}
if m.SchemaVersion != 1 {
blog.Warn("Manifest schema version %d is not yet supported\n", m.SchemaVersion)
return
}
if len(m.History) == 0 {
e = errors.New("repo " + repo + ":" + tag + " no images found in history")
return
}
var image ImageStruct
if e = json.Unmarshal([]byte(m.History[0].V1Compatibility), &image); e != nil {
blog.Warn("Failed to parse ImageStruct")
return
}
var creationTime time.Time
if creationTime, e = time.Parse(time.RFC3339Nano, image.Created); e != nil {
blog.Warn("Failed to parse creation time")
return
}
metadata.Datetime = creationTime
metadata.Size = image.Size
metadata.Author = image.Author
metadata.Checksum = image.Checksum
metadata.Comment = image.Comment
metadata.Parent = image.Parent
return
}
// getDistroID takes a distribution "pretty name" as input and returns the corresponding
// distribution ID, or "Unknown" if no match can be found.
func getDistroID(distroName string) string {
if id, ok := DistroMap[distroName]; ok {
return id
}
//Exceptions to the rule: There are many such cases, so bucketing them together
if strings.HasPrefix(distroName, `CentOS release 5`) {
return "REDHAT-5Server"
}
if strings.HasPrefix(distroName, `CentOS release 6`) {
return "REDHAT-6Server"
}
if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 5`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 5`) {
return "REDHAT-5Server"
}
if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 6`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 6`) {
return "REDHAT-6Server"
}
if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 7`) ||
strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 7`) {
return "REDHAT-7Server"
}
if strings.HasPrefix(distroName, `Ubuntu Vivid`) {
return "UBUNTU-vivid"
}
if strings.HasPrefix(distroName, `Ubuntu Wily`) {
return "UBUNTU-wily"
}
blog.Warn("DISTRO %s UNKNOWN", distroName)
return "Unknown"
}
// GetRegistryURL determines the full URL, with or without HTTP Basic Auth, needed to
// access the registry or Docker Hub.
func GetRegistryURL() (URL string, hubAPI bool, BasicAuth string, XRegistryAuth string) {
basicAuth, fullRegistry, XRegistryAuth := RegAuth(RegistrySpec)
if *AuthRegistry == true {
if basicAuth == "" {
blog.Exit("Registry auth could not be determined from docker config.")
}
BasicAuth = basicAuth
}
if *HTTPSRegistry == false {
URL = "http://" + RegistrySpec
} else {
// HTTPS is required
if strings.HasPrefix(fullRegistry, "https://") {
URL = fullRegistry
} else {
URL = "https://" + RegistrySpec
}
if *RegistryTokenAuth == true {
hubAPI = true
}
if strings.Contains(URL, "docker.io") || strings.Contains(URL, "gcr.io") {
hubAPI = true
if *RegistryTokenAuth == false {
blog.Warn("Forcing --registrytokenauth=true, as required for Docker Hub and Google Container Registry")
*RegistryTokenAuth = true
}
}
}
return
}
// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
blog.Info("Writing image (pkg and other) data into file...")
for imageID, scriptMap := range outMapMap {
for scriptName, out := range scriptMap {
scriptDir := f.dir + "/" + trimExtension(scriptName)
err := fsutil.CreateDirIfNotExist(scriptDir)
if err != nil {
blog.Error(err, ": Error creating script dir: ", scriptDir)
continue
}
image := string(imageID)
if len(image) < 12 {
blog.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
} else {
image = string(imageID)[0:12]
}
filenamePath := scriptDir + "/" + image
if _, ok := out.([]byte); ok {
f.format = "txt"
filenamePath += "-miscdata"
} else {
// by default it is json. But f.format could get overwritten at any point
// in the for loop if the output type is []byte, hence the (re)assignment
f.format = "json"
// NOTE: If we start using json for output other than imageData, change this
filenamePath += "-pkgdata"
}
f.writeFileInFormat(filenamePath, &out)
}
}
return
}
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) {
switch f.format {
case "json":
err := jsonifyAndAppendToFile(filenamePath+".json", data)
if err != nil {
blog.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
return
}
default:
blog.Warn("Currently only supporting json output to write to files")
}
}
func (f *FileWriter) handleImageMetadata(imageMetadata []ImageMetadataInfo, action string) {
if len(imageMetadata) == 0 {
blog.Warn("No image metadata to append to file...")
return
}
// If output directory does not exist, first create it
fsutil.CreateDirIfNotExist(f.dir)
filenamePath := f.dir + "/" + "metadata"
data := ImageMetadataAndAction{action, imageMetadata}
f.appendFileInFormat(filenamePath, data)
}
// Warn logs a warning message and generates a config.BanyanUpdate.
func Warn(arg0 interface{}, args ...interface{}) {
if len(args) == 0 {
blog.Warn(arg0)
s := fmt.Sprintf("WARN %v", arg0)
config.BanyanUpdate(s)
} else {
var s string
switch arg0.(type) {
case string:
blog.Warn(arg0.(string), args...)
s = fmt.Sprintf("WARN %s", arg0.(string))
s = fmt.Sprintf(s, args...)
default:
blog.Warn(arg0, args...)
s = fmt.Sprintf("WARN %v", arg0)
arr := []interface{}{s}
arr = append(arr, args...)
s = fmt.Sprintln(arr...)
}
s = strings.TrimRight(s, "\n")
config.BanyanUpdate(s)
}
}
// RemoveObsoleteMetadata removes obsolete metadata from the Banyan service.
func RemoveObsoleteMetadata(obsolete []ImageMetadataInfo) {
if len(obsolete) == 0 {
blog.Warn("No image metadata to save!")
return
}
config.BanyanUpdate("Remove Metadata", statusMessageMD(obsolete))
for _, writer := range WriterList {
writer.RemoveImageMetadata(obsolete)
}
return
}
// SaveImageMetadata saves image metadata to selected storage location
// (standard output, Banyan service, etc.).
func SaveImageMetadata(metadataSlice []ImageMetadataInfo) {
if len(metadataSlice) == 0 {
blog.Warn("No image metadata to save!")
return
}
config.BanyanUpdate("Save Image Metadata", statusMessageMD(metadataSlice))
for _, writer := range WriterList {
writer.AppendImageMetadata(metadataSlice)
}
return
}
// GetLocalImageMetadata returns image metadata queried from a local Docker host.
// Query the local docker daemon to detect new image builds on the host and new images pulled from registry by users.
func GetLocalImageMetadata(oldMetadataSet MetadataSet) (metadataSlice []ImageMetadataInfo) {
for {
blog.Info("Get a list of images from local Docker daemon")
imageMap, e := GetLocalImages()
if e != nil {
blog.Warn(e, " GetLocalImages")
blog.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
blog.Info("Get Image Metadata from local Docker daemon")
// Get image metadata
metadataSlice, e = getImageMetadata(imageMap, oldMetadataSet)
if e != nil {
blog.Warn(e, " GetImageMetadata")
blog.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
break
}
return
}
func getScriptsToRun() (scripts []Script) {
// get default scripts
defaultScripts, err := getScripts(DefaultScriptsDir)
if err != nil {
blog.Exit(err, ": Error in getting default scripts")
}
// get user-specified scripts
userScripts, err := getScripts(UserScriptsDir)
if err != nil {
blog.Warn(err, ": Error in getting user-specified scripts")
}
scripts = append(defaultScripts, userScripts...)
return
}
// getImageList reads the list of previously processed images from the imageList file.
func getImageList(processedImages collector.ImageSet) (e error) {
f, e := os.Open(*imageList)
if e != nil {
blog.Warn(e, ": Error in opening", *imageList, ": perhaps a fresh start?")
return
}
defer f.Close()
r := bufio.NewReader(f)
data, e := ioutil.ReadAll(r)
if e != nil {
blog.Error(e, ": Error in reading file ", *imageList)
return
}
for _, str := range strings.Split(string(data), "\n") {
if len(str) != 0 {
blog.Debug("Previous image: %s", str)
processedImages[collector.ImageIDType(str)] = true
}
}
return
}
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) {
blog.Info("Writing " + filenamePath + "...")
switch f.format {
case "json":
err := jsonifyAndWriteToFile(filenamePath+".json", data)
if err != nil {
blog.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
return
}
case "txt":
// what's passed in is ptr to interface{}. First get interface{} out of it and then
// typecast that to []byte
err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644)
if err != nil {
blog.Error(err, ": Error in writing to file: ", filenamePath)
return
}
default:
blog.Warn("Currently only supporting json output to write to files")
}
}
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
if len(ReposToProcess) == 0 {
return
}
client := &http.Client{}
metadataMap := NewImageToMetadataMap(oldMetadataSet)
allRepos := []RepoType{}
// Check if we need to use the search API, i.e. only one repo given, and ends in wildcard "*".
if searchTerm := NeedRegistrySearch(); searchTerm != "" {
blog.Info("Using search API")
var e error
allRepos, e = registrySearchV1(client, searchTerm)
if e != nil {
blog.Error(e, ":registry search")
return
}
}
// If search wasn't needed, the repos were individually specified.
if len(allRepos) == 0 {
for repo := range ReposToProcess {
allRepos = append(allRepos, repo)
}
}
for _, repo := range allRepos {
blog.Info("Get index and tag info for %s", string(repo))
config.BanyanUpdate("Get index and tag info for", string(repo))
var (
indexInfo IndexInfo
e error
repoTagSlice []TagInfo
repoMetadataSlice []ImageMetadataInfo
)
// loop until success
for {
indexInfo, e = getReposTokenAuthV1(repo, client)
if e != nil {
blog.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
if e != nil {
blog.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
if len(repoTagSlice) != 1 {
blog.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
time.Sleep(config.RETRYDURATION)
continue
}
repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
if e != nil {
blog.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
time.Sleep(config.RETRYDURATION)
continue
}
//success!
break
}
tagSlice = append(tagSlice, repoTagSlice...)
metadataSlice = append(metadataSlice, repoMetadataSlice...)
}
return
}
// GetImageMetadata returns repository/tag/image metadata queried from a Docker registry.
// If the user has specified the repositories to examine, then no other repositories are examined.
// If the user has not specified repositories, then the registry search API is used to
// get the list of all repositories in the registry.
func GetImageMetadata(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
for {
blog.Info("Get Repos")
repoSlice, e := getRepos()
if e != nil {
blog.Warn(e, " getRepos")
blog.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
if len(repoSlice) == 0 {
// For some reason (like, registry search doesn't work), we are not
// seeing any repos in the registry.
// So, just reconstruct the list of repos that we saw earlier.
blog.Warn("Empty repoSlice, reusing previous metadata")
repomap := make(map[string]bool)
for metadata := range oldMetadataSet {
if repomap[metadata.Repo] == false {
repoSlice = append(repoSlice, RepoType(metadata.Repo))
repomap[metadata.Repo] = true
}
}
}
// Now get a list of all the tags, and the image metadata/manifest
if *RegistryProto == "v1" {
blog.Info("Get Tags")
tagSlice, e = getTags(repoSlice)
if e != nil {
blog.Warn(e, " getTags")
blog.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
// get map from each imageID to all of its aliases (repo+tag)
imageMap := make(ImageToRepoTagMap)
for _, ti := range tagSlice {
for tag, imageID := range ti.TagMap {
repotag := RepoTagType{Repo: ti.Repo, Tag: tag}
imageMap.Insert(imageID, repotag)
}
}
blog.Info("Get Image Metadata")
// Get image metadata
metadataSlice, e = getImageMetadata(imageMap, oldMetadataSet)
if e != nil {
blog.Warn(e, " getImageMetadata")
blog.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
break
}
if *RegistryProto == "v2" {
blog.Info("Get Tags and Metadata")
tagSlice, metadataSlice, e = v2GetTagsMetadata(repoSlice)
if e != nil {
blog.Warn(e)
blog.Warn("Retrying")
time.Sleep(config.RETRYDURATION)
continue
}
break
}
}
return
}