Esempio n. 1
0
func getScripts(dirPath string) (scripts []Script, err error) {
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		except.Warn(err, ": Error in reading contents of ", dirPath)
		return
	}

	for _, file := range files {
		file.Name()
		//figure out type of script
		var script Script
		switch {
		case strings.HasSuffix(file.Name(), ".sh"):
			blog.Debug("dirpath: " + dirPath + " after removing prefix: " + config.BANYANDIR() + " looks like: " + strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"))
			script = newBashScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
		case strings.HasSuffix(file.Name(), ".py"):
			script = newPythonScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
		default:
			except.Warn("Unknown script file type for: " + file.Name())
			//Ignore this file...
			continue
		}
		scripts = append(scripts, script)
	}

	return
}
Esempio n. 2
0
// GetRegistryURL determines the full URL, with or without HTTP Basic Auth, needed to
// access the registry or Docker Hub.
func GetRegistryURL() (URL string, hubAPI bool, BasicAuth string, XRegistryAuth string) {
	basicAuth, fullRegistry, XRegistryAuth := RegAuth(RegistrySpec)
	if *AuthRegistry == true {
		if basicAuth == "" {
			except.Fail("Registry auth could not be determined from docker config.")
		}
		BasicAuth = basicAuth
	}
	if *HTTPSRegistry == false {
		URL = "http://" + RegistrySpec
	} else {
		// HTTPS is required
		if strings.HasPrefix(fullRegistry, "https://") {
			URL = fullRegistry
		} else {
			URL = "https://" + RegistrySpec
		}
		if *RegistryTokenAuth == true {
			hubAPI = true
		}
		if strings.Contains(URL, "docker.io") || strings.Contains(URL, "gcr.io") {
			hubAPI = true
			if *RegistryTokenAuth == false {
				except.Warn("Forcing --registrytokenauth=true, as required for Docker Hub and Google Container Registry")
				*RegistryTokenAuth = true
			}
		}
	}
	return
}
Esempio n. 3
0
// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
	blog.Info("Writing image (pkg and other) data into file...")

	for imageID, scriptMap := range outMapMap {
		for scriptName, out := range scriptMap {
			scriptDir := f.dir + "/" + trimExtension(scriptName)
			err := fsutil.CreateDirIfNotExist(scriptDir)
			if err != nil {
				except.Error(err, ": Error creating script dir: ", scriptDir)
				continue
			}
			image := string(imageID)
			if len(image) < 12 {
				except.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
			} else {
				image = string(imageID)[0:12]
			}
			filenamePath := scriptDir + "/" + image
			if _, ok := out.([]byte); ok {
				f.format = "txt"
				filenamePath += "-miscdata"
			} else {
				// by default it is json. But f.format could get overwritten at any point
				// in the for loop if the output type is []byte, hence the (re)assignment
				f.format = "json"
				// NOTE: If we start using json for output other than imageData, change this
				filenamePath += "-pkgdata"
			}
			f.writeFileInFormat(filenamePath, &out)
		}
	}
	return
}
Esempio n. 4
0
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) {
	switch f.format {
	case "json":
		err := jsonifyAndAppendToFile(filenamePath+".json", data)
		if err != nil {
			except.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
			return
		}
	default:
		except.Warn("Currently only supporting json output to write to files")
	}
}
Esempio n. 5
0
func (f *FileWriter) handleImageMetadata(imageMetadata []ImageMetadataInfo, action string) {
	if len(imageMetadata) == 0 {
		except.Warn("No image metadata to append to file...")
		return
	}

	// If output directory does not exist, first create it
	fsutil.CreateDirIfNotExist(f.dir)
	filenamePath := f.dir + "/" + "metadata"

	data := ImageMetadataAndAction{action, imageMetadata}
	f.appendFileInFormat(filenamePath, data)
}
Esempio n. 6
0
// RemoveObsoleteMetadata removes obsolete metadata from the Banyan service.
func RemoveObsoleteMetadata(obsolete []ImageMetadataInfo) {
	if len(obsolete) == 0 {
		except.Warn("No image metadata to save!")
		return
	}

	config.BanyanUpdate("Remove Metadata", statusMessageMD(obsolete))

	for _, writer := range WriterList {
		writer.RemoveImageMetadata(obsolete)
	}

	return
}
Esempio n. 7
0
// SaveImageMetadata saves image metadata to selected storage location
// (standard output, Banyan service, etc.).
func SaveImageMetadata(metadataSlice []ImageMetadataInfo) {
	if len(metadataSlice) == 0 {
		except.Warn("No image metadata to save!")
		return
	}

	config.BanyanUpdate("Save Image Metadata", statusMessageMD(metadataSlice))

	for _, writer := range WriterList {
		writer.AppendImageMetadata(metadataSlice)
	}

	return
}
Esempio n. 8
0
// getDistroID takes a distribution "pretty name" as input and returns the corresponding
// distribution ID, or "Unknown" if no match can be found.
func getDistroID(distroName string) string {
	if id, ok := DistroMap[distroName]; ok {
		return id
	}

	//Exceptions to the rule: There are many such cases, so bucketing them together
	if strings.HasPrefix(distroName, `Ubuntu 14.04`) {
		return "UBUNTU-trusty"
	}
	if strings.HasPrefix(distroName, `Ubuntu precise`) {
		return "UBUNTU-precise"
	}
	if strings.HasPrefix(distroName, `Ubuntu 12.04`) {
		return "UBUNTU-precise"
	}
	if strings.HasPrefix(distroName, `Ubuntu 10.04`) {
		return "UBUNTU-lucid"
	}
	if strings.HasPrefix(distroName, `CentOS release 5`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 5`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 5`) {
		m := distroRegexp[rel5z].FindStringSubmatch(distroName)
		if len(m) > 1 {
			return "REDHAT-5Server-5." + m[1]
		}
		return "REDHAT-5Server"
	}
	if strings.HasPrefix(distroName, `CentOS release 6`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 6`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 6`) {
		m := distroRegexp[rel6z].FindStringSubmatch(distroName)
		if len(m) > 1 {
			return "REDHAT-6Server-6." + m[1]
		}
		return "REDHAT-6Server"
	}
	if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 7`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 7`) {
		return "REDHAT-7Server"
	}
	if strings.HasPrefix(distroName, `Ubuntu Vivid`) {
		return "UBUNTU-vivid"
	}
	if strings.HasPrefix(distroName, `Ubuntu Wily`) {
		return "UBUNTU-wily"
	}

	except.Warn("DISTRO %s UNKNOWN", distroName)
	return "Unknown"
}
Esempio n. 9
0
// GetLocalImageMetadata returns image metadata queried from a local Docker host.
// Query the local docker daemon to detect new image builds on the host and new images pulled from registry by users.
func GetLocalImageMetadata(oldMetadataSet MetadataSet) (metadataSlice []ImageMetadataInfo) {
	for {
		blog.Info("Get a list of images from local Docker daemon")
		imageMap, e := GetLocalImages(true, true)
		if e != nil {
			except.Warn(e, " GetLocalImages")
			except.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}

		blog.Info("Get Image Metadata from local Docker daemon")
		// Get image metadata
		metadataSlice, e = GetImageMetadataSpecified(imageMap, oldMetadataSet)
		if e != nil {
			except.Warn(e, " GetImageMetadata")
			except.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}
		break
	}
	return
}
Esempio n. 10
0
func getScriptsToRun() (scripts []Script) {
	// get default scripts
	defaultScripts, err := getScripts(DefaultScriptsDir)
	if err != nil {
		except.Fail(err, ": Error in getting default scripts")
	}

	// get user-specified scripts
	userScripts, err := getScripts(UserScriptsDir)
	if err != nil {
		except.Warn(err, ": Error in getting user-specified scripts")
	}

	scripts = append(defaultScripts, userScripts...)
	return
}
Esempio n. 11
0
// getImageList reads the list of previously processed images from the imageList file.
func getImageList(processedImages collector.ImageSet) (e error) {
	f, e := os.Open(*imageList)
	if e != nil {
		except.Warn(e, ": Error in opening", *imageList, ": perhaps a fresh start?")
		return
	}
	defer f.Close()
	r := bufio.NewReader(f)
	data, e := ioutil.ReadAll(r)
	if e != nil {
		except.Error(e, ": Error in reading file ", *imageList)
		return
	}
	for _, str := range strings.Split(string(data), "\n") {
		if len(str) != 0 {
			blog.Debug("Previous image: %s", str)
			processedImages[collector.ImageIDType(str)] = true
		}
	}
	return
}
Esempio n. 12
0
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) {
	blog.Info("Writing " + filenamePath + "...")
	switch f.format {
	case "json":
		err := jsonifyAndWriteToFile(filenamePath+".json", data)
		if err != nil {
			except.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
			return
		}
	case "txt":
		// what's passed in is ptr to interface{}. First get interface{} out of it and then
		// typecast that to []byte
		err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644)
		if err != nil {
			except.Error(err, ": Error in writing to file: ", filenamePath)
			return
		}
	default:
		except.Warn("Currently only supporting json output to write to files")
	}
}
Esempio n. 13
0
// getImageManifestHashList reads the list of previously processed images (manifest hash) from the imageList_ManifestHash file.
func getImageManifestHashList(processedImagesManifestHash collector.ImageSet) (e error) {
	filename := *imageList + "_ManifestHash"
	f, e := os.Open(filename)
	if e != nil {
		except.Warn(e, ": Error in opening", filename, ": perhaps a fresh start?")
		return
	}
	defer f.Close()
	r := bufio.NewReader(f)
	data, e := ioutil.ReadAll(r)
	if e != nil {
		except.Error(e, ": Error in reading file ", filename)
		return
	}
	for _, str := range strings.Split(string(data), "\n") {
		if len(str) != 0 {
			blog.Debug("Previous image: %s", str)
			processedImagesManifestHash.Insert(collector.ImageIDType(str))
		}
	}
	return
}
Esempio n. 14
0
// SaveImageMetadata saves image metadata to selected storage location
// (standard output, Banyan service, etc.).
func SaveImageMetadata(metadataSlice []ImageMetadataInfo) {
	if len(metadataSlice) == 0 {
		except.Warn("No image metadata to save!")
		return
	}

	config.BanyanUpdate("Save Image Metadata", statusMessageMD(metadataSlice))

	slice := []ImageMetadataInfo{}
	for _, metadata := range metadataSlice {
		if len(metadata.Image) > 0 {
			slice = append(slice, metadata)
		}
	}
	if len(slice) == 0 {
		return
	}

	for _, writer := range WriterList {
		writer.AppendImageMetadata(slice)
	}

	return
}
Esempio n. 15
0
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
	if len(ReposToProcess) == 0 {
		return
	}
	client := &http.Client{}

	metadataMap := NewImageToMetadataMap(oldMetadataSet)

	allRepos := []RepoType{}
	// Check if we need to use the search API, i.e. only one repo given, and ends in wildcard "*".
	if searchTerm := NeedRegistrySearch(); searchTerm != "" {
		blog.Info("Using search API")
		var e error
		allRepos, e = registrySearchV1(client, searchTerm)
		if e != nil {
			except.Error(e, ":registry search")
			return
		}
	}
	// If search wasn't needed, the repos were individually specified.
	if len(allRepos) == 0 {
		for repo := range ReposToProcess {
			allRepos = append(allRepos, repo)
		}
	}

	for _, repo := range allRepos {
		blog.Info("Get index and tag info for %s", string(repo))
		config.BanyanUpdate("Get index and tag info for", string(repo))

		var (
			indexInfo         IndexInfo
			e                 error
			repoTagSlice      []TagInfo
			repoMetadataSlice []ImageMetadataInfo
		)

		// loop until success
		for {
			indexInfo, e = getReposTokenAuthV1(repo, client)
			if e != nil {
				except.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
			if e != nil {
				except.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			if len(repoTagSlice) != 1 {
				except.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
				config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
			if e != nil {
				except.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			//success!
			break
		}
		tagSlice = append(tagSlice, repoTagSlice...)
		metadataSlice = append(metadataSlice, repoMetadataSlice...)
	}
	return
}
Esempio n. 16
0
// GetImageMetadata determines which image metadata is of interest and then calls
// GetImageMetadataSpecified to obtain and return the appropriate metadata,
// queried from a Docker registry or Docker daemon.
// If the user has specified the repositories to examine, then no other repositories are examined.
// If the user has not specified repositories, then the registry search API is used to
// get the list of all repositories in the registry.
func GetImageMetadata(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
	for {
		blog.Info("Get Repos")
		repoSlice, e := getRepos()
		if e != nil {
			except.Warn(e, " getRepos")
			except.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}
		if len(repoSlice) == 0 {
			// For some reason (like, registry search doesn't work), we are not
			// seeing any repos in the registry.
			// So, just reconstruct the list of repos that we saw earlier.
			except.Warn("Empty repoSlice, reusing previous metadata")
			repomap := make(map[string]bool)
			for metadata := range oldMetadataSet {
				if repomap[metadata.Repo] == false {
					repoSlice = append(repoSlice, RepoType(metadata.Repo))
					repomap[metadata.Repo] = true
				}
			}
		}

		// Now get a list of all the tags, and the image metadata/manifest

		if *RegistryProto == "v1" {
			blog.Info("Get Tags")
			tagSlice, e = getTags(repoSlice)
			if e != nil {
				except.Warn(e, " getTags")
				except.Warn("Retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}

			// get map from each imageID to all of its aliases (repo+tag)
			imageMap := make(ImageToRepoTagMap)
			for _, ti := range tagSlice {
				for tag, imageID := range ti.TagMap {
					repotag := RepoTagType{Repo: ti.Repo, Tag: tag}

					imageMap.Insert(imageID, repotag)
				}
			}

			blog.Info("Get Image Metadata")
			// Get image metadata
			metadataSlice, e = GetImageMetadataSpecified(imageMap, oldMetadataSet)
			if e != nil {
				except.Warn(e, " GetImageMetadataSpecified")
				except.Warn("Retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			break
		}
		if *RegistryProto == "v2" {
			blog.Info("Get Tags and Metadata")
			tagSlice, metadataSlice, e = v2GetTagsMetadata(repoSlice)
			if e != nil {
				except.Warn(e)
				except.Warn("Retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			break
		}
	}

	return
}