/* queryAuthServerV2 retrieves an authorization token from a V2 auth server */
func queryAuthServerV2(client *http.Client, fieldMap map[string]string, BasicAuth string) (token string, e error) {
authServer := fieldMap["realm"]
if authServer == "" {
e = errors.New("No registry token auth server specified")
return
}
blog.Debug("authServer=%s\n", authServer)
URL := authServer
first := true
for key, value := range fieldMap {
if key != "realm" {
if first {
URL = URL + "?"
first = false
} else {
URL = URL + "&"
}
URL = URL + key + "=" + value
}
}
blog.Debug("Auth server URL is %s", URL)
req, e := http.NewRequest("GET", URL, nil)
if e != nil {
return
}
req.Header.Set("Authorization", "Basic "+BasicAuth)
r, e := client.Do(req)
if e != nil {
return
}
defer r.Body.Close()
if r.StatusCode < 200 || r.StatusCode > 299 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
response, e := ioutil.ReadAll(r.Body)
if e != nil {
return
}
var parsedReply authServerResult
e = json.Unmarshal(response, &parsedReply)
if e != nil {
return
}
token = parsedReply.Token
return token, e
}
func testWriteToFile(t *testing.T, outMapMap map[string]map[string]interface{}, script, image, destDir, format string, suffix string) (b []byte) {
fw := NewFileWriter(format, destDir)
fw.WriteImageAllData(outMapMap)
// Test if correct output file exists
finalDir := destDir + "/" + trimExtension(script) + "/"
blog.Debug("final dir: " + finalDir)
var filenamePath string
if ok, e := fsutil.DirExists(finalDir); ok {
if len(image) > 12 {
image = image[0:12]
}
file := image + suffix + "." + format
filenamePath = finalDir + "/" + file
_, err := os.Stat(filenamePath)
if err != nil {
if os.IsNotExist(err) {
t.Fatal(err, ": File ", filenamePath, " doesn't exist")
}
t.Fatal(err, ": Unknown error while locating file: ", filenamePath)
}
} else {
t.Fatal(e, ": Directory: ", finalDir, " doesn't exist")
}
b, err := ioutil.ReadFile(filenamePath)
if err != nil {
t.Fatal(err, ": Error in reading file: ", filenamePath)
}
return b
}
// RemoveImages removes least recently pulled docker images from the local docker host.
func RemoveImages(PulledImages []ImageMetadataInfo, imageToMDMap map[string][]ImageMetadataInfo) {
numRemoved := 0
for _, imageMD := range PulledImages {
// Get all metadata (repo/tags) associated with that image
for _, metadata := range imageToMDMap[imageMD.Image] {
// basespec := RegistrySpec + "/" + string(t.Repo) + ":"
if ExcludeRepo[RepoType(metadata.Repo)] {
continue
}
blog.Debug("Removing the following registry/repo:tag: " + RegistrySpec + "/" +
metadata.Repo + ":" + metadata.Tag)
apipath := "/images/" + RegistrySpec + "/" + metadata.Repo + ":" + metadata.Tag
blog.Info("RemoveImages %s", apipath)
config.BanyanUpdate("Remove", apipath)
_, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
if err != nil {
blog.Error(err, "RemoveImages Repo:Tag", metadata.Repo, metadata.Tag,
"image", metadata.Image)
}
numRemoved++
}
}
blog.Info("Number of repo/tags removed this time around: %d", numRemoved)
return
}
func getScripts(dirPath string) (scripts []Script, err error) {
files, err := ioutil.ReadDir(dirPath)
if err != nil {
blog.Warn(err, ": Error in reading contents of ", dirPath)
return
}
for _, file := range files {
file.Name()
//figure out type of script
var script Script
switch {
case strings.HasSuffix(file.Name(), ".sh"):
blog.Debug("dirpath: " + dirPath + " after removing prefix: " + config.BANYANDIR() + " looks like: " + strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"))
script = newBashScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
case strings.HasSuffix(file.Name(), ".py"):
script = newPythonScript(file.Name(), TARGETCONTAINERDIR+strings.TrimPrefix(dirPath, config.BANYANDIR()+"/hosttarget"), []string{""})
default:
blog.Warn("Unknown script file type for: " + file.Name())
//Ignore this file...
continue
}
scripts = append(scripts, script)
}
return
}
// LogsContainer makes a docker remote API call to get logs from a container.
func LogsContainer(containerID string) (output []byte, err error) {
apipath := "/containers/" + containerID + "/logs?stdout=1"
resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath)
return
}
blog.Debug("Response from docker remote API call for logs: " + string(resp))
for {
if len(resp) < 8 {
break
}
header := resp[0:8]
var size int32
buf := bytes.NewBuffer(header[4:8])
binary.Read(buf, binary.BigEndian, &size)
payload := resp[8:(8 + size)]
// blog.Info(string(frame))
resp = resp[(8 + size):]
if header[0] == uint8(1) {
// 1=stdout: return only the stdout log
output = append(output, payload...)
}
}
return
}
// TestWriteImageAllData tests writing different types of image data to files
func TestWriteImageAllData(t *testing.T) {
cases := []struct {
script, image, destDir, format string
}{
{"myscript", "image", "/tmp", "json"},
{"myscript.sh", "image1234", "/tmp", "json"},
{"myscript.abc.sh", "aaaabbbb", "/tmp", "json"},
}
outMap := make(map[string]interface{})
outMapMap := make(map[string]map[string]interface{})
// Testing imagedata...
var idata = []ImageDataInfo{{"111", "a", "b", "c", "dn1", "did1"}, {"111", "d", "e", "f", "dn2", "did2"}, {"121", "g", "h", "i", "dn3", "did3"}}
for _, c := range cases {
outMap[c.script] = idata
outMapMap[c.image] = outMap
b1 := testWriteToFile(t, outMapMap, c.script, c.image, c.destDir, c.format, "-pkgdata")
b2, err := json.MarshalIndent(idata, "", "\t")
if err != nil {
t.Fatal(err, ": Error in marshaling json for imagedata")
}
if !bytes.Equal(b1, b2) {
blog.Debug(b1)
blog.Debug(b2)
t.Fatal("Input/Output image data don't match: ", len(b1), len(b2))
}
}
blog.Info("Reaching here => writing imagedata to file works fine")
// Testing random output ([]byte)...
randOut := []byte("Testing random output from scripts")
for _, c := range cases {
script := "X" + c.script
outMap[script] = randOut
outMapMap[c.image] = outMap
b := testWriteToFile(t, outMapMap, script, c.image, c.destDir, "txt", "-miscdata")
if !bytes.Equal(b, randOut) {
blog.Debug(b)
blog.Debug(randOut)
t.Fatal("Input/Output image rand txt don't match", len(b), len(randOut))
}
}
//Pass...
return
}
// Run handles running of a script inside an image
func (sh ScriptInfo) Run(imageID ImageIDType) (b []byte, err error) {
jsonString, err := createCmd(imageID, sh.name, sh.staticBinary, sh.dirPath)
if err != nil {
except.Error(err, ": Error in creating command")
return
}
blog.Debug("Container spec: %s", string(jsonString))
containerID, err := CreateContainer(jsonString)
if err != nil {
except.Error(err, ": Error in creating container")
return
}
blog.Debug("New container ID: %s", containerID)
defer RemoveContainer(containerID)
jsonString, err = StartContainer(containerID)
if err != nil {
except.Error(err, ": Error in starting container")
return
}
blog.Debug("Response from StartContainer: %s", string(jsonString))
statusCode, err := WaitContainer(containerID)
if err != nil {
except.Error(err, ": Error in waiting for container to stop")
return
}
if statusCode != 0 {
err = errors.New("Bash script exit status: " + strconv.Itoa(statusCode))
return
}
b, err = LogsContainer(containerID)
if err != nil {
except.Error(err, ":Error in extracting output from container")
return
}
/*
_, err = removeContainer(containerID)
if err != nil {
except.Error(err, ":Error in removing container for image", containerID)
return
}
*/
return
}
// RemoveContainer makes a docker remote API call to remove a container.
func RemoveContainer(containerID string) (resp []byte, err error) {
apipath := "/containers/" + containerID
resp, err = DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
blog.Debug("Response from docker remote API call for remove: " + string(resp))
return
}
// listImages makes a docker remote API call to get a list of images
func listImages() (resp []byte, err error) {
apipath := "/images/json"
resp, err = DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
blog.Debug("Response from docker remote API call for list images: " + string(resp))
return
}
func InspectImage(imageID string) (resp []byte, err error) {
apipath := "/images/" + imageID + "/json"
resp, err = DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
blog.Debug("Response from docker remote API call for inspect image " + imageID + " : \n" + string(resp))
return
}
// StartContainer makes a docker remote API call to start a container.
func StartContainer(containerID string) (jsonOut []byte, err error) {
apipath := "/containers/" + containerID + "/start"
resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath)
return
}
blog.Debug("Response from docker remote API call for start: " + string(resp))
return
}
// DockerAPI performs an HTTP GET,POST,DELETE operation to the Docker daemon.
func DockerAPI(tr *http.Transport, operation, apipath string, jsonString []byte,
XRegistryAuth string) (resp []byte, e error) {
switch operation {
case "GET", "POST", "DELETE":
break
default:
e = errors.New("Operation " + operation + " not supported")
return
}
// for unix socket, URL (host.domain) is needed but can be anything
var host string
HTTP := "http://"
if DockerProto == "unix" {
host = dummydomain
} else {
host = DockerAddr
if DockerTLSVerify {
HTTP = "https://"
}
}
URL := HTTP + host + apipath
blog.Debug("DockerAPI %s", URL)
req, e := http.NewRequest(operation, URL, bytes.NewBuffer(jsonString))
if e != nil {
except.Error(e, ":DockerAPI failed to create http request")
return
}
req.Header.Add("Content-Type", "application/json")
if XRegistryAuth != "" {
req.Header.Add("X-Registry-Auth", XRegistryAuth)
}
//req.Header.Set("Authorization", "Bearer "+authToken)
client := &http.Client{Transport: tr, Timeout: DockerTimeout}
r, e := client.Do(req)
if e != nil {
except.Error(e, ":DockerAPI URL", URL, "client request failed")
return
}
defer r.Body.Close()
resp, e = ioutil.ReadAll(r.Body)
if e != nil {
except.Error(e, ":DockerAPI URL", URL, "invalid response body")
return
}
if r.StatusCode < 200 || r.StatusCode > 299 {
e = errors.New("DockerAPI URL: " + URL + " status code: " + strconv.Itoa(r.StatusCode) +
"error: " + string(resp))
return
}
return
}
func main() {
doFlags()
setupLogging()
//verifyVolumes()
copyBanyanData()
// setup connection to docker daemon's unix/tcp socket
var e error
collector.DockerTransport, e = collector.NewDockerTransport(*dockerProto, *dockerAddr)
if e != nil {
except.Fail(e, ": Error in connecting to docker remote API socket")
}
authToken := RegisterCollector()
// Set output writers
SetOutputWriters(authToken)
SetupBanyanStatus(authToken)
checkConfigUpdate(true)
if collector.LocalHost == false && collector.RegistryAPIURL == "" {
collector.RegistryAPIURL, collector.HubAPI, collector.BasicAuth, collector.XRegistryAuth = collector.GetRegistryURL()
blog.Info("registry API URL: %s", collector.RegistryAPIURL)
}
// Log the docker version
major, minor, revision, e := collector.DockerVersion()
if e != nil {
except.Error(e, ": Could not identify Docker version")
} else {
blog.Info("Docker version %d.%d.%d", major, minor, revision)
config.BanyanUpdate("Docker version", strconv.Itoa(major)+"."+strconv.Itoa(minor)+"."+strconv.Itoa(revision))
}
// Images we have processed already
processedImages := collector.NewImageSet()
e = getImageList(processedImages)
if e != nil {
blog.Info("Fresh start: No previously collected images were found in %s", *imageList)
}
_ = getImageManifestHashList(processedImages)
blog.Debug(processedImages)
// Main infinite loop.
InfLoop(authToken, processedImages)
}
// doFlags defines the cmdline Usage string and parses flag options.
func doFlags() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, " Usage: %s [OPTIONS] REGISTRY REPO [REPO...]\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\n REGISTRY:\n")
fmt.Fprintf(os.Stderr, "\tURL of your Docker registry; use index.docker.io for Docker Hub, use local.host to collect images from local Docker host\n")
fmt.Fprintf(os.Stderr, "\n REPO:\n")
fmt.Fprintf(os.Stderr, "\tOne or more repos to gather info about; if no repo is specified Collector will gather info on *all* repos in the Registry\n")
fmt.Fprintf(os.Stderr, "\n Environment variables:\n")
fmt.Fprintf(os.Stderr, "\tCOLLECTOR_DIR: (Required) Directory that contains the \"data\" folder with Collector default scripts, e.g., $GOPATH/src/github.com/banyanops/collector\n")
fmt.Fprintf(os.Stderr, "\tCOLLECTOR_ID: ID provided by Banyan web interface to register Collector with the Banyan service\n")
fmt.Fprintf(os.Stderr, "\tBANYAN_HOST_DIR: Host directory mounted into Collector/Target containers where results are stored (default: $HOME/.banyan)\n")
fmt.Fprintf(os.Stderr, "\tBANYAN_DIR: (Specify only in Dockerfile) Directory in the Collector container where host directory BANYAN_HOST_DIR is mounted\n")
fmt.Fprintf(os.Stderr, "\tDOCKER_{HOST,CERT_PATH,TLS_VERIFY}: If set, e.g., by docker-machine, then they take precedence over --dockerProto and --dockerAddr\n")
printExampleUsage()
fmt.Fprintf(os.Stderr, " Options:\n")
flag.PrintDefaults()
}
flag.Parse()
if config.COLLECTORDIR() == "" {
flag.Usage()
os.Exit(except.ErrorExitStatus)
}
if len(flag.Args()) < 1 {
flag.Usage()
os.Exit(except.ErrorExitStatus)
}
if *dockerProto != "unix" && *dockerProto != "tcp" {
flag.Usage()
os.Exit(except.ErrorExitStatus)
}
requiredDirs := []string{config.BANYANDIR(), filepath.Dir(*imageList), filepath.Dir(*repoList), *config.BanyanOutDir, collector.DefaultScriptsDir, collector.UserScriptsDir, collector.BinDir}
for _, dir := range requiredDirs {
blog.Debug("Creating directory: " + dir)
err := fsutil.CreateDirIfNotExist(dir)
if err != nil {
except.Fail(err, ": Error in creating a required directory: ", dir)
}
}
collector.RegistrySpec = flag.Arg(0)
// EqualFold: case insensitive comparison
if strings.EqualFold(flag.Arg(0), "local.host") {
collector.LocalHost = true
}
//nextMaxImages = *maxImages
}
func main() {
doFlags()
setupLogging()
//verifyVolumes()
copyBanyanData()
// setup connection to docker daemon's unix/tcp socket
var e error
collector.DockerTransport, e = collector.NewDockerTransport(*dockerProto, *dockerAddr)
if e != nil {
blog.Exit(e, ": Error in connecting to docker remote API socket")
}
authToken := RegisterCollector()
// Set output writers
SetOutputWriters(authToken)
SetupBanyanStatus(authToken)
checkConfigUpdate(true)
if collector.LocalHost == false && collector.RegistryAPIURL == "" {
collector.RegistryAPIURL, collector.HubAPI, collector.BasicAuth, collector.XRegistryAuth = collector.GetRegistryURL()
blog.Info("registry API URL: %s", collector.RegistryAPIURL)
}
// Images we have processed already
processedImages := collector.NewImageSet()
e = getImageList(processedImages)
if e != nil {
blog.Info("Fresh start: No previously collected images were found in %s", *imageList)
}
blog.Debug(processedImages)
// Image Metadata we have already seen
MetadataSet := collector.NewMetadataSet()
PulledList := []collector.ImageMetadataInfo{}
// Main infinite loop.
InfLoop(authToken, processedImages, MetadataSet, PulledList)
}
// WaitContainer makes a docker remote API call to wait for a container to finish running.
func WaitContainer(containerID string) (statusCode int, err error) {
apipath := "/containers/" + containerID + "/wait"
resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath)
return
}
blog.Debug("Response from docker remote API call for wait: " + string(resp))
var msg struct {
StatusCode int
}
err = json.Unmarshal(resp, &msg)
if err != nil {
except.Error(err, "waitContainer resp", string(resp))
return
}
blog.Info("Got StatusCode %d\n", msg.StatusCode)
statusCode = msg.StatusCode
return
}
// getImageList reads the list of previously processed images from the imageList file.
func getImageList(processedImages collector.ImageSet) (e error) {
f, e := os.Open(*imageList)
if e != nil {
except.Warn(e, ": Error in opening", *imageList, ": perhaps a fresh start?")
return
}
defer f.Close()
r := bufio.NewReader(f)
data, e := ioutil.ReadAll(r)
if e != nil {
except.Error(e, ": Error in reading file ", *imageList)
return
}
for _, str := range strings.Split(string(data), "\n") {
if len(str) != 0 {
blog.Debug("Previous image: %s", str)
processedImages[collector.ImageIDType(str)] = true
}
}
return
}
// CreateContainer makes a docker remote API call to create a container.
func CreateContainer(containerSpec []byte) (containerID string, err error) {
apipath := "/containers/create"
resp, err := DockerAPI(DockerTransport, "POST", apipath, containerSpec, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath, string(containerSpec))
return
}
blog.Debug("Response from docker remote API call for create: " + string(resp))
var msg struct {
Id string
Warnings string
}
err = json.Unmarshal(resp, &msg)
if err != nil {
except.Error(err, "createContainer resp", string(resp))
return
}
blog.Info("Got ID %s Warnings %s\n", msg.Id, msg.Warnings)
containerID = msg.Id
return
}
// GetLocalImages queries the local Docker daemon for list of images.
// The registry name gets stripped from the repo nam if stripRegistry is set to true.
// The repo has to appear in the list of repos to check if checkRepo is set to true.
func GetLocalImages(stripRegistry bool, checkRepo bool) (imageMap ImageToRepoTagMap, e error) {
// query a list of images from Docker daemon
response, e := listImages()
if e != nil {
return nil, e
}
// parse JSON
var localImageList []LocalImageStruct
if e = json.Unmarshal(response, &localImageList); e != nil {
return nil, e
}
// make map from each imageID to all of its aliases (repo+tag)
imageMap = make(ImageToRepoTagMap)
for _, localImage := range localImageList {
imageID := ImageIDType(localImage.ID)
for _, regRepoTag := range localImage.RepoTags {
// skip images with no repo:tag
if regRepoTag == "" || regRepoTag == "\u003cnone\u003e:\u003cnone\u003e" || regRepoTag == "<none>:<none>" {
blog.Debug("Image %s has a <none>:<none> repository:tag.", string(imageID))
continue
}
repoTag, e := ExtractRepoTag(regRepoTag, stripRegistry)
if e != nil {
return nil, e
}
if checkRepo {
if CheckRepoToProcess(repoTag.Repo) {
imageMap.Insert(imageID, repoTag)
}
} else {
imageMap.Insert(imageID, repoTag)
}
}
}
return
}
// getImageManifestHashList reads the list of previously processed images (manifest hash) from the imageList_ManifestHash file.
func getImageManifestHashList(processedImagesManifestHash collector.ImageSet) (e error) {
filename := *imageList + "_ManifestHash"
f, e := os.Open(filename)
if e != nil {
except.Warn(e, ": Error in opening", filename, ": perhaps a fresh start?")
return
}
defer f.Close()
r := bufio.NewReader(f)
data, e := ioutil.ReadAll(r)
if e != nil {
except.Error(e, ": Error in reading file ", filename)
return
}
for _, str := range strings.Split(string(data), "\n") {
if len(str) != 0 {
blog.Debug("Previous image: %s", str)
processedImagesManifestHash.Insert(collector.ImageIDType(str))
}
}
return
}
// RegistryQueryV2 performs an HTTP GET operation from the registry and returns the response.
// If the initial response code is 401 Unauthorized, then this function issues a call
// if indicated by an WWW-Authenticate header in the response to get a token, and
// then re-issues the initial call to get the final response.
func RegistryQueryV2(client *http.Client, URL string) (response []byte, e error) {
_, _, BasicAuth, XRegistryAuth = GetRegistryURL()
req, e := http.NewRequest("GET", URL, nil)
if e != nil {
return nil, e
}
req.Header.Set("Authorization", "Basic "+BasicAuth)
r, e := client.Do(req)
if e != nil {
return nil, e
}
if r.StatusCode == 401 {
blog.Debug("Registry Query %s got 401", URL)
// get the WWW-Authenticate header
WWWAuth := r.Header.Get("WWW-Authenticate")
if WWWAuth == "" {
except.Error("Empty WWW-Authenticate", URL)
return
}
arr := strings.Fields(WWWAuth)
if len(arr) != 2 {
e = errors.New("Invalid WWW-Authenticate format for " + WWWAuth)
except.Error(e)
return
}
authType := arr[0]
blog.Debug("Authorization type: %s", authType)
fieldMap := make(map[string]string)
e = parseAuthenticateFields(arr[1], fieldMap)
if e != nil {
except.Error(e)
return
}
r.Body.Close()
// access the authentication server to get a token
token, err := queryAuthServerV2(client, fieldMap, BasicAuth)
if err != nil {
except.Error(err)
return nil, err
}
// re-issue the original request, this time using the token
req, e = http.NewRequest("GET", URL, nil)
if e != nil {
return nil, e
}
req.Header.Set("Authorization", authType+" "+token)
r, e = client.Do(req)
if e != nil {
return nil, e
}
}
defer r.Body.Close()
if r.StatusCode < 200 || r.StatusCode > 299 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
response, e = ioutil.ReadAll(r.Body)
if e != nil {
return
}
return
}
// DoIteration runs one iteration of the main loop to get new images, extract data from them,
// and saves results.
func DoIteration(ReposToLimit RepoSet, authToken string,
processedImages collector.ImageSet, oldMetadataSet collector.MetadataSet,
PulledList []collector.ImageMetadataInfo) (currentMetadataSet collector.MetadataSet,
PulledNew []collector.ImageMetadataInfo) {
blog.Debug("DoIteration: processedImages is %v", processedImages)
PulledNew = PulledList
_ /*tagSlice*/, metadataSlice, currentMetadataSet := collector.GetNewImageMetadata(oldMetadataSet)
if len(metadataSlice) == 0 {
blog.Info("No new metadata in this iteration")
return
}
blog.Info("Obtained %d new metadata items in this iteration", len(metadataSlice))
collector.SaveImageMetadata(metadataSlice)
// number of images processed for each repository in this iteration
imageCount := make(map[collector.RepoType]int)
// Set of repos to stop limiting according to maxImages after this iteration completes.
StopLimiting := NewRepoSet()
// processed metadata
processedMetadata := collector.NewMetadataSet()
for {
pulledImages := collector.NewImageSet()
pullErrorMetadata := collector.NewMetadataSet()
for _, metadata := range metadataSlice {
processedMetadata.Insert(metadata)
if config.FilterRepos && !collector.ReposToProcess[collector.RepoType(metadata.Repo)] {
continue
}
// TODO: need to filter out images from ExcludedRepo also when collecting from local Docker host?
if collector.ExcludeRepo[collector.RepoType(metadata.Repo)] {
continue
}
if pulledImages[collector.ImageIDType(metadata.Image)] {
continue
}
// TODO: need to consider maxImages limit also when collecting from local Docker host?
repo := collector.RepoType(metadata.Repo)
if _, ok := ReposToLimit[repo]; !ok {
// new repo we haven't seen before; apply maxImages limit to repo
blog.Info("Starting to apply maxImages limit to repo %s", string(repo))
ReposToLimit[repo] = true
}
if ReposToLimit[repo] && *maxImages > 0 && imageCount[repo] >= *maxImages {
blog.Info("Max image count %d reached for %s, skipping :%s",
*maxImages, metadata.Repo, metadata.Tag)
// stop applying the maxImages limit to repo
StopLimiting[repo] = true
continue
}
if processedImages[collector.ImageIDType(metadata.Image)] {
continue
}
imageCount[collector.RepoType(metadata.Repo)]++
// docker pull image
if !collector.LocalHost {
err := collector.PullImage(metadata)
if err != nil {
// docker pull failed for some reason, possibly a transient failure.
// So we remove this metadata element from the current and processed sets,
// and move on to process any remaining metadata elements.
// In the next iteration, metadata
// lookup may rediscover this deleted metadata element
// and treat it as new, thus ensuring that the image pull will be retried.
// TODO: If the registry is corrupted, this can lead to an infinite
// loop in which the same image pull keeps getting tried and consistently fails.
currentMetadataSet.Delete(metadata)
processedMetadata.Delete(metadata)
// remember this pull error in order to demote this metadata to the end of the slice.
pullErrorMetadata.Insert(metadata)
err = collector.RemoveDanglingImages()
if err != nil {
except.Error(err, ": RemoveDanglingImages")
}
continue
}
}
PulledNew = append(PulledNew, metadata)
excess := len(PulledNew) - *removeThresh
if !collector.LocalHost && *removeThresh > 0 && excess > 0 {
config.BanyanUpdate("Removing " + strconv.Itoa(excess) + " pulled images")
collector.RemoveImages(PulledNew[0:excess])
PulledNew = PulledNew[excess:]
}
pulledImages[collector.ImageIDType(metadata.Image)] = true
if len(pulledImages) == IMAGEBATCH {
break
}
}
if len(pulledImages) == 0 {
blog.Info("No pulled images left to process in this iteration")
config.BanyanUpdate("No pulled images left to process in this iteration")
break
}
// reorder metadataSlice by moving images that couldn't be pulled to the end of the list
newMDSlice := []collector.ImageMetadataInfo{}
for _, metadata := range metadataSlice {
if !pullErrorMetadata.Exists(metadata) {
newMDSlice = append(newMDSlice, metadata)
}
}
for metadata := range pullErrorMetadata {
newMDSlice = append(newMDSlice, metadata)
}
metadataSlice = newMDSlice
// get and save image data for all the images in pulledimages
outMapMap := collector.GetImageAllData(pulledImages)
collector.SaveImageAllData(outMapMap)
for imageID := range pulledImages {
processedImages[imageID] = true
}
if e := persistImageList(pulledImages); e != nil {
except.Error(e, "Failed to persist list of collected images")
}
if checkConfigUpdate(false) == true {
// Config changed, and possibly did so before all current metadata was processed.
// Thus, remember only the metadata that has already been processed, and forget
// metadata that has not been processed yet.
// That way, the next time DoIteration() is entered, the metadata lookup
// will correctly schedule the forgotten metadata for processing, along with
// any new metadata.
currentMetadataSet = processedMetadata
break
}
}
for repo := range StopLimiting {
blog.Info("No longer enforcing maxImages limit on repo %s", repo)
ReposToLimit[repo] = false
}
return
}
// DoIteration runs one iteration of the main loop to get new images, extract data from them,
// and saves results.
func DoIteration(ReposToLimit RepoSet, authToken string,
processedImages collector.ImageSet, oldMetadataSet collector.MetadataSet,
PulledList []collector.ImageMetadataInfo) (currentMetadataSet collector.MetadataSet,
PulledNew []collector.ImageMetadataInfo) {
blog.Debug("DoIteration: processedImages is %v", processedImages)
PulledNew = PulledList
_ /*tagSlice*/, metadataSlice, currentMetadataSet := collector.GetNewImageMetadata(oldMetadataSet)
if len(metadataSlice) == 0 {
blog.Info("No new metadata in this iteration")
return
}
blog.Info("Obtained %d new metadata items in this iteration", len(metadataSlice))
collector.SaveImageMetadata(metadataSlice)
// number of images processed for each repository in this iteration
imageCount := make(map[collector.RepoType]int)
imageToMDMap := collector.GetImageToMDMap(metadataSlice)
// Set of repos to stop limiting according to maxImages after this iteration completes.
StopLimiting := NewRepoSet()
for {
pulledImages := collector.NewImageSet()
for _, metadata := range metadataSlice {
if config.FilterRepos && !collector.ReposToProcess[collector.RepoType(metadata.Repo)] {
continue
}
// TODO: need to filter out images from ExcludedRepo also when collecting from local Docker host?
if collector.ExcludeRepo[collector.RepoType(metadata.Repo)] {
continue
}
if pulledImages[collector.ImageIDType(metadata.Image)] {
continue
}
// TODO: need to consider maxImages limit also when collecting from local Docker host?
repo := collector.RepoType(metadata.Repo)
if _, ok := ReposToLimit[repo]; !ok {
// new repo we haven't seen before; apply maxImages limit to repo
blog.Info("Starting to apply maxImages limit to repo %s", string(repo))
ReposToLimit[repo] = true
}
if ReposToLimit[repo] && *maxImages > 0 && imageCount[repo] >= *maxImages {
blog.Info("Max image count %d reached for %s, skipping :%s",
*maxImages, metadata.Repo, metadata.Tag)
// stop applying the maxImages limit to repo
StopLimiting[repo] = true
continue
}
if processedImages[collector.ImageIDType(metadata.Image)] {
continue
}
imageCount[collector.RepoType(metadata.Repo)]++
// docker pull image
if !collector.LocalHost {
collector.PullImage(metadata)
}
PulledNew = append(PulledNew, metadata)
if !collector.LocalHost && *removeThresh > 0 && len(PulledNew) > *removeThresh {
collector.RemoveImages(PulledNew[0:*removeThresh], imageToMDMap)
PulledNew = PulledNew[*removeThresh:]
}
pulledImages[collector.ImageIDType(metadata.Image)] = true
if len(pulledImages) == IMAGEBATCH {
break
}
}
if len(pulledImages) == 0 {
break
}
// get and save image data for all the images in pulledimages
outMapMap := collector.GetImageAllData(pulledImages)
collector.SaveImageAllData(outMapMap)
for imageID := range pulledImages {
processedImages[imageID] = true
}
if e := persistImageList(pulledImages); e != nil {
blog.Error(e, "Failed to persist list of collected images")
}
if checkConfigUpdate(false) == true {
break
}
}
for repo := range StopLimiting {
blog.Info("No longer enforcing maxImages limit on repo %s", repo)
ReposToLimit[repo] = false
}
return
}