Exemplo n.º 1
0
func getScripts(dirPath string) (scripts []Script, err error) {
	files, err := ioutil.ReadDir(dirPath)
	if err != nil {
		blog.Warn(err, ": Error in reading contents of ", dirPath)
		return
	}

	for _, file := range files {
		file.Name()
		//figure out type of script
		var script Script
		switch {
		case strings.HasSuffix(file.Name(), ".sh"):
			blog.Debug("dirpath: " + dirPath + " after removing prefix: " + config.BANYANDIR() + " looks like: " + strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"))
			script = newBashScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
		case strings.HasSuffix(file.Name(), ".py"):
			script = newPythonScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
		default:
			blog.Warn("Unknown script file type for: " + file.Name())
			//Ignore this file...
			continue
		}
		scripts = append(scripts, script)
	}

	return
}
Exemplo n.º 2
0
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
	if len(ReposToProcess) == 0 {
		return
	}
	client := &http.Client{}

	metadataMap := NewImageToMetadataMap(oldMetadataSet)

	for repo := range ReposToProcess {
		blog.Info("Get index and tag info for %s", string(repo))
		config.BanyanUpdate("Get index and tag info for", string(repo))

		var (
			indexInfo         IndexInfo
			e                 error
			repoTagSlice      []TagInfo
			repoMetadataSlice []ImageMetadataInfo
		)

		// loop until success
		for {
			indexInfo, e = getReposTokenAuthV1(repo, client)
			if e != nil {
				blog.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
			if e != nil {
				blog.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			if len(repoTagSlice) != 1 {
				blog.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
				config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
			if e != nil {
				blog.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			//success!
			break
		}
		tagSlice = append(tagSlice, repoTagSlice...)
		metadataSlice = append(metadataSlice, repoMetadataSlice...)
	}
	return
}
Exemplo n.º 3
0
func v2GetMetadata(client *http.Client, repo, tag string) (metadata ImageMetadataInfo, e error) {
	response, err := RegistryQueryV2(client, RegistryAPIURL+"/v2/"+repo+"/manifests/"+tag)
	if err != nil {
		except.Error(err)
		if s, ok := err.(*HTTPStatusCodeError); ok {
			except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode)
			e = err
		}
		return
	}

	metadata.Repo = repo
	metadata.Tag = tag
	metadata.Image = ""
	// Recent versions of Docker daemon (1.8.3+?) have complex code that calculates
	// the image ID from the V2 manifest.
	// This seems to be in flux as Docker moves toward content-addressable images in 1.10+,
	// and as the registry image manifest schema itself is still evolving.
	// As a temporary measure until Docker converges to a more stable state, collector
	// will calculate its own hash over the V2 manifest and use the calculated
	// value to try to filter out images that have previously been processed.
	// The Docker-calculated image ID will get added to the metadata struct
	// after the image is pulled.
	sum := sha256.Sum256(response)
	metadata.ManifestHash = hex.EncodeToString(sum[:])
	var m ManifestV2Schema1
	b := bytes.NewBuffer(response)
	if e = json.NewDecoder(b).Decode(&m); e != nil {
		blog.Warn("Failed to parse manifest")
		return
	}
	if m.SchemaVersion != 1 {
		blog.Warn("Manifest schema version %d is not yet supported\n", m.SchemaVersion)
		return
	}
	if len(m.History) == 0 {
		e = errors.New("repo " + repo + ":" + tag + " no images found in history")
		return
	}
	var image ImageStruct
	if e = json.Unmarshal([]byte(m.History[0].V1Compatibility), &image); e != nil {
		blog.Warn("Failed to parse ImageStruct")
		return
	}
	var creationTime time.Time
	if creationTime, e = time.Parse(time.RFC3339Nano, image.Created); e != nil {
		blog.Warn("Failed to parse creation time")
		return
	}
	metadata.Datetime = creationTime
	metadata.Size = image.Size
	metadata.Author = image.Author
	metadata.Checksum = image.Checksum
	metadata.Comment = image.Comment
	metadata.Parent = image.Parent
	return
}
Exemplo n.º 4
0
// getDistroID takes a distribution "pretty name" as input and returns the corresponding
// distribution ID, or "Unknown" if no match can be found.
func getDistroID(distroName string) string {
	if id, ok := DistroMap[distroName]; ok {
		return id
	}

	//Exceptions to the rule: There are many such cases, so bucketing them together
	if strings.HasPrefix(distroName, `CentOS release 5`) {
		return "REDHAT-5Server"
	}
	if strings.HasPrefix(distroName, `CentOS release 6`) {
		return "REDHAT-6Server"
	}
	if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 5`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 5`) {
		return "REDHAT-5Server"
	}
	if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 6`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 6`) {
		return "REDHAT-6Server"
	}
	if strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server release 7`) ||
		strings.HasPrefix(distroName, `Red Hat Enterprise Linux Server 7`) {
		return "REDHAT-7Server"
	}
	if strings.HasPrefix(distroName, `Ubuntu Vivid`) {
		return "UBUNTU-vivid"
	}
	if strings.HasPrefix(distroName, `Ubuntu Wily`) {
		return "UBUNTU-wily"
	}

	blog.Warn("DISTRO %s UNKNOWN", distroName)
	return "Unknown"
}
Exemplo n.º 5
0
// GetRegistryURL determines the full URL, with or without HTTP Basic Auth, needed to
// access the registry or Docker Hub.
func GetRegistryURL() (URL string, hubAPI bool, BasicAuth string, XRegistryAuth string) {
	basicAuth, fullRegistry, XRegistryAuth := RegAuth(RegistrySpec)
	if *AuthRegistry == true {
		if basicAuth == "" {
			blog.Exit("Registry auth could not be determined from docker config.")
		}
		BasicAuth = basicAuth
	}
	if *HTTPSRegistry == false {
		URL = "http://" + RegistrySpec
	} else {
		// HTTPS is required
		if strings.HasPrefix(fullRegistry, "https://") {
			URL = fullRegistry
		} else {
			URL = "https://" + RegistrySpec
		}
		if *RegistryTokenAuth == true {
			hubAPI = true
		}
		if strings.Contains(URL, "docker.io") || strings.Contains(URL, "gcr.io") {
			hubAPI = true
			if *RegistryTokenAuth == false {
				blog.Warn("Forcing --registrytokenauth=true, as required for Docker Hub and Google Container Registry")
				*RegistryTokenAuth = true
			}
		}
	}
	return
}
Exemplo n.º 6
0
// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
	blog.Info("Writing image (pkg and other) data into file...")

	for imageID, scriptMap := range outMapMap {
		for scriptName, out := range scriptMap {
			scriptDir := f.dir + "/" + trimExtension(scriptName)
			err := fsutil.CreateDirIfNotExist(scriptDir)
			if err != nil {
				blog.Error(err, ": Error creating script dir: ", scriptDir)
				continue
			}
			image := string(imageID)
			if len(image) < 12 {
				blog.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
			} else {
				image = string(imageID)[0:12]
			}
			filenamePath := scriptDir + "/" + image
			if _, ok := out.([]byte); ok {
				f.format = "txt"
				filenamePath += "-miscdata"
			} else {
				// by default it is json. But f.format could get overwritten at any point
				// in the for loop if the output type is []byte, hence the (re)assignment
				f.format = "json"
				// NOTE: If we start using json for output other than imageData, change this
				filenamePath += "-pkgdata"
			}
			f.writeFileInFormat(filenamePath, &out)
		}
	}
	return
}
Exemplo n.º 7
0
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) {
	switch f.format {
	case "json":
		err := jsonifyAndAppendToFile(filenamePath+".json", data)
		if err != nil {
			blog.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
			return
		}
	default:
		blog.Warn("Currently only supporting json output to write to files")
	}
}
Exemplo n.º 8
0
func (f *FileWriter) handleImageMetadata(imageMetadata []ImageMetadataInfo, action string) {
	if len(imageMetadata) == 0 {
		blog.Warn("No image metadata to append to file...")
		return
	}

	// If output directory does not exist, first create it
	fsutil.CreateDirIfNotExist(f.dir)
	filenamePath := f.dir + "/" + "metadata"

	data := ImageMetadataAndAction{action, imageMetadata}
	f.appendFileInFormat(filenamePath, data)
}
Exemplo n.º 9
0
// Warn logs a warning message and generates a config.BanyanUpdate.
func Warn(arg0 interface{}, args ...interface{}) {
	if len(args) == 0 {
		blog.Warn(arg0)
		s := fmt.Sprintf("WARN %v", arg0)
		config.BanyanUpdate(s)
	} else {
		var s string
		switch arg0.(type) {
		case string:
			blog.Warn(arg0.(string), args...)
			s = fmt.Sprintf("WARN %s", arg0.(string))
			s = fmt.Sprintf(s, args...)
		default:
			blog.Warn(arg0, args...)
			s = fmt.Sprintf("WARN %v", arg0)
			arr := []interface{}{s}
			arr = append(arr, args...)
			s = fmt.Sprintln(arr...)
		}
		s = strings.TrimRight(s, "\n")
		config.BanyanUpdate(s)
	}
}
Exemplo n.º 10
0
// RemoveObsoleteMetadata removes obsolete metadata from the Banyan service.
func RemoveObsoleteMetadata(obsolete []ImageMetadataInfo) {
	if len(obsolete) == 0 {
		blog.Warn("No image metadata to save!")
		return
	}

	config.BanyanUpdate("Remove Metadata", statusMessageMD(obsolete))

	for _, writer := range WriterList {
		writer.RemoveImageMetadata(obsolete)
	}

	return
}
Exemplo n.º 11
0
// SaveImageMetadata saves image metadata to selected storage location
// (standard output, Banyan service, etc.).
func SaveImageMetadata(metadataSlice []ImageMetadataInfo) {
	if len(metadataSlice) == 0 {
		blog.Warn("No image metadata to save!")
		return
	}

	config.BanyanUpdate("Save Image Metadata", statusMessageMD(metadataSlice))

	for _, writer := range WriterList {
		writer.AppendImageMetadata(metadataSlice)
	}

	return
}
Exemplo n.º 12
0
// GetLocalImageMetadata returns image metadata queried from a local Docker host.
// Query the local docker daemon to detect new image builds on the host and new images pulled from registry by users.
func GetLocalImageMetadata(oldMetadataSet MetadataSet) (metadataSlice []ImageMetadataInfo) {
	for {
		blog.Info("Get a list of images from local Docker daemon")
		imageMap, e := GetLocalImages()
		if e != nil {
			blog.Warn(e, " GetLocalImages")
			blog.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}

		blog.Info("Get Image Metadata from local Docker daemon")
		// Get image metadata
		metadataSlice, e = getImageMetadata(imageMap, oldMetadataSet)
		if e != nil {
			blog.Warn(e, " GetImageMetadata")
			blog.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}
		break
	}
	return
}
Exemplo n.º 13
0
func getScriptsToRun() (scripts []Script) {
	// get default scripts
	defaultScripts, err := getScripts(DefaultScriptsDir)
	if err != nil {
		blog.Exit(err, ": Error in getting default scripts")
	}

	// get user-specified scripts
	userScripts, err := getScripts(UserScriptsDir)
	if err != nil {
		blog.Warn(err, ": Error in getting user-specified scripts")
	}

	scripts = append(defaultScripts, userScripts...)
	return
}
Exemplo n.º 14
0
// getImageList reads the list of previously processed images from the imageList file.
func getImageList(processedImages collector.ImageSet) (e error) {
	f, e := os.Open(*imageList)
	if e != nil {
		blog.Warn(e, ": Error in opening", *imageList, ": perhaps a fresh start?")
		return
	}
	defer f.Close()
	r := bufio.NewReader(f)
	data, e := ioutil.ReadAll(r)
	if e != nil {
		blog.Error(e, ": Error in reading file ", *imageList)
		return
	}
	for _, str := range strings.Split(string(data), "\n") {
		if len(str) != 0 {
			blog.Debug("Previous image: %s", str)
			processedImages[collector.ImageIDType(str)] = true
		}
	}
	return
}
Exemplo n.º 15
0
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) {
	blog.Info("Writing " + filenamePath + "...")
	switch f.format {
	case "json":
		err := jsonifyAndWriteToFile(filenamePath+".json", data)
		if err != nil {
			blog.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
			return
		}
	case "txt":
		// what's passed in is ptr to interface{}. First get interface{} out of it and then
		// typecast that to []byte
		err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644)
		if err != nil {
			blog.Error(err, ": Error in writing to file: ", filenamePath)
			return
		}
	default:
		blog.Warn("Currently only supporting json output to write to files")
	}
}
Exemplo n.º 16
0
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub
// or other registry using v1 token authorization.
// The user must have specified a set of repositories of interest.
// The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses
// the token to query the registry.
func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
	if len(ReposToProcess) == 0 {
		return
	}
	client := &http.Client{}

	metadataMap := NewImageToMetadataMap(oldMetadataSet)

	allRepos := []RepoType{}
	// Check if we need to use the search API, i.e. only one repo given, and ends in wildcard "*".
	if searchTerm := NeedRegistrySearch(); searchTerm != "" {
		blog.Info("Using search API")
		var e error
		allRepos, e = registrySearchV1(client, searchTerm)
		if e != nil {
			blog.Error(e, ":registry search")
			return
		}
	}
	// If search wasn't needed, the repos were individually specified.
	if len(allRepos) == 0 {
		for repo := range ReposToProcess {
			allRepos = append(allRepos, repo)
		}
	}

	for _, repo := range allRepos {
		blog.Info("Get index and tag info for %s", string(repo))
		config.BanyanUpdate("Get index and tag info for", string(repo))

		var (
			indexInfo         IndexInfo
			e                 error
			repoTagSlice      []TagInfo
			repoMetadataSlice []ImageMetadataInfo
		)

		// loop until success
		for {
			indexInfo, e = getReposTokenAuthV1(repo, client)
			if e != nil {
				blog.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo)
			if e != nil {
				blog.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			if len(repoTagSlice) != 1 {
				blog.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice))
				config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo))
				time.Sleep(config.RETRYDURATION)
				continue
			}

			repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo)
			if e != nil {
				blog.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.")
				config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			//success!
			break
		}
		tagSlice = append(tagSlice, repoTagSlice...)
		metadataSlice = append(metadataSlice, repoMetadataSlice...)
	}
	return
}
Exemplo n.º 17
0
// GetImageMetadata returns repository/tag/image metadata queried from a Docker registry.
// If the user has specified the repositories to examine, then no other repositories are examined.
// If the user has not specified repositories, then the registry search API is used to
// get the list of all repositories in the registry.
func GetImageMetadata(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) {
	for {
		blog.Info("Get Repos")
		repoSlice, e := getRepos()
		if e != nil {
			blog.Warn(e, " getRepos")
			blog.Warn("Retrying")
			time.Sleep(config.RETRYDURATION)
			continue
		}
		if len(repoSlice) == 0 {
			// For some reason (like, registry search doesn't work), we are not
			// seeing any repos in the registry.
			// So, just reconstruct the list of repos that we saw earlier.
			blog.Warn("Empty repoSlice, reusing previous metadata")
			repomap := make(map[string]bool)
			for metadata := range oldMetadataSet {
				if repomap[metadata.Repo] == false {
					repoSlice = append(repoSlice, RepoType(metadata.Repo))
					repomap[metadata.Repo] = true
				}
			}
		}

		// Now get a list of all the tags, and the image metadata/manifest

		if *RegistryProto == "v1" {
			blog.Info("Get Tags")
			tagSlice, e = getTags(repoSlice)
			if e != nil {
				blog.Warn(e, " getTags")
				blog.Warn("Retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}

			// get map from each imageID to all of its aliases (repo+tag)
			imageMap := make(ImageToRepoTagMap)
			for _, ti := range tagSlice {
				for tag, imageID := range ti.TagMap {
					repotag := RepoTagType{Repo: ti.Repo, Tag: tag}

					imageMap.Insert(imageID, repotag)
				}
			}

			blog.Info("Get Image Metadata")
			// Get image metadata
			metadataSlice, e = getImageMetadata(imageMap, oldMetadataSet)
			if e != nil {
				blog.Warn(e, " getImageMetadata")
				blog.Warn("Retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			break
		}
		if *RegistryProto == "v2" {
			blog.Info("Get Tags and Metadata")
			tagSlice, metadataSlice, e = v2GetTagsMetadata(repoSlice)
			if e != nil {
				blog.Warn(e)
				blog.Warn("Retrying")
				time.Sleep(config.RETRYDURATION)
				continue
			}
			break
		}
	}

	return
}