func v1GetTags(repoSlice []RepoType) (tagSlice []TagInfo, e error) { var client *http.Client if *RegistryTLSNoVerify { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client = &http.Client{Transport: tr} } else { client = &http.Client{} } for _, repo := range repoSlice { // get tags for one repo response, err := RegistryQuery(client, RegistryAPIURL+"/v1/repositories/"+string(repo)+"/tags") if err != nil { blog.Error(err) if s, ok := err.(*HTTPStatusCodeError); ok { blog.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode) continue } return } //parse JSON output var m map[TagType]ImageIDType if e = json.Unmarshal(response, &m); e != nil { return nil, e } var t TagInfo t.Repo = repo t.TagMap = m tagSlice = append(tagSlice, t) } return }
// RegistryRequestWithToken queries a Docker Registry that uses v1 Token Auth, e.g., Docker Hub. func RegistryRequestWithToken(client *http.Client, URL string, dockerToken string) (response []byte, e error) { var req *http.Request req, e = http.NewRequest("GET", URL, nil) if e != nil { blog.Error(e) return } req.Header.Set("Authorization", "Token "+dockerToken) var r *http.Response r, e = client.Do(req) if e != nil { blog.Error(e) return } defer r.Body.Close() if r.StatusCode != 200 { e = &HTTPStatusCodeError{StatusCode: r.StatusCode} return } response, e = ioutil.ReadAll(r.Body) if e != nil { blog.Error(e) return } return }
func runAllScripts(imageID ImageIDType) (outMap map[string]interface{}, err error) { //script name -> either byte array, or known types (e.g., ImageDataInfo) outMap = make(map[string]interface{}) scripts := getScriptsToRun() for _, script := range scripts { //run script output, err := script.Run(imageID) if err != nil { blog.Error(err, ": Error in running script: ", script.Name()) continue //continue trying to run other scripts } //analyze script output switch script.Name() { case PKGEXTRACTSCRIPT: imageDataInfo, err := parsePkgExtractOutput(output, imageID) if err != nil { blog.Error(err, ": Error in parsing PkgExtractOuput") return nil, err } outMap[script.Name()] = imageDataInfo default: //script name -> byte array outMap[script.Name()] = output } } return }
// ValidRepoName verifies that the name of a repo is in a legal format. func ValidRepoName(name string) bool { if len(name) == 0 { return false } if len(name) > 256 { blog.Error("Invalid repo name, too long: %s", name) return false } for _, c := range name { switch { case c >= 'a' && c <= 'z': continue case c >= 'A' && c <= 'Z': continue case c >= '0' && c <= '9': continue case c == '/' || c == '_' || c == '-' || c == '.': continue default: blog.Error("Invalid repo name %s", name) return false } } return true }
func copy(src, dest string) { // Read all content of src to data data, err := ioutil.ReadFile(src) if err != nil { blog.Error(err, ": Error in reading from file: ", src) return } // Write data to dest err = ioutil.WriteFile(dest, data, 0755) if err != nil { blog.Error(err, ": Error in writing to file: ", dest) return } }
// Fail prints an error message at blog.ERROR level and then quits with exit status ErrorExitStatus. func Fail(arg0 interface{}, args ...interface{}) { if len(args) == 0 { blog.Error(arg0) } else { switch arg0.(type) { case string: blog.Error(arg0.(string), args...) default: blog.Error(arg0, args...) } } blog.Close() os.Exit(ErrorExitStatus) }
// PullImage performs a docker pull on an image specified by repo/tag. // TODO: Detect if the pulled image has a different imageID than the value retrieved from // metadata, and if so correct the metadata, or at least skip processing the image. func PullImage(metadata ImageMetadataInfo) { tagspec := RegistrySpec + "/" + metadata.Repo + ":" + metadata.Tag apipath := "/images/create?fromImage=" + tagspec blog.Info("PullImage downloading %s, Image ID: %s", apipath, metadata.Image) config.BanyanUpdate("Pull", apipath, metadata.Image) resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, XRegistryAuth) if err != nil { blog.Error(err, "PullImage failed for", RegistrySpec, metadata.Repo, metadata.Tag, metadata.Image) } if strings.Contains(string(resp), `"error":`) { blog.Error("PullImage error for %s/%s/%s", RegistrySpec, metadata.Repo, metadata.Tag) } blog.Trace(string(resp)) return }
func jsonifyAndWriteToFile(filenamePath string, data interface{}) (err error) { b, err := json.MarshalIndent(data, "", "\t") if err != nil { blog.Error(err, ": Error in marshaling json") return err } err = ioutil.WriteFile(filenamePath, b, 0644) if err != nil { blog.Error(err, ": Error in writing to file: ", filenamePath) return err } return nil }
func CreateDirIfNotExist(dir string) (err error) { exists, err := DirExists(dir) if err != nil { blog.Error(err, ": Error while querying dir: ", dir) return err } if !exists { err = os.MkdirAll(dir, 0755) if err != nil { blog.Error(err, ": Error in creating dir: ", dir) return err } } return nil }
// WriteImageAllData writes image (pkg and other) data into file func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) { blog.Info("Writing image (pkg and other) data into file...") for imageID, scriptMap := range outMapMap { for scriptName, out := range scriptMap { scriptDir := f.dir + "/" + trimExtension(scriptName) err := fsutil.CreateDirIfNotExist(scriptDir) if err != nil { blog.Error(err, ": Error creating script dir: ", scriptDir) continue } image := string(imageID) if len(image) < 12 { blog.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?") } else { image = string(imageID)[0:12] } filenamePath := scriptDir + "/" + image if _, ok := out.([]byte); ok { f.format = "txt" filenamePath += "-miscdata" } else { // by default it is json. But f.format could get overwritten at any point // in the for loop if the output type is []byte, hence the (re)assignment f.format = "json" // NOTE: If we start using json for output other than imageData, change this filenamePath += "-pkgdata" } f.writeFileInFormat(filenamePath, &out) } } return }
func getTagsTokenAuthV1(repo RepoType, client *http.Client, indexInfo IndexInfo) (tagSlice []TagInfo, e error) { tagSlice, e = lookupTagsTokenAuthV1(client, indexInfo) if e != nil { blog.Error(e, ": Error in looking up tags in dockerhub") } return }
// getRepos queries the Docker registry for the list of the repositories it is currently hosting. // However, if the user specified a list of repositories, then getRepos() just returns that list // of specified repositories and does not query the Docker registry. func getRepos() (repoSlice []RepoType, err error) { if len(ReposToProcess) > 0 { for repo := range ReposToProcess { repoSlice = append(repoSlice, repo) } return } if *RegistryProto == "v2" { blog.Error("v2 registry search/catalog interface not yet supported in collector") return } // a query with an empty query string returns all the repos var client *http.Client if *RegistryTLSNoVerify { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client = &http.Client{Transport: tr} } else { client = &http.Client{} } return registrySearchV1(client, "") }
// lookupMetadataTokenAuthV1 takes as input the imageID, and Docker Hub auth/index info, // and it returns ImageMetadataInfo for that image by querying the indexed registry. func lookupMetadataTokenAuthV1(imageID ImageIDType, client *http.Client, indexInfo IndexInfo) ( metadata ImageMetadataInfo, e error) { blog.Info("Get Metadata for Image: %s", string(imageID)) URL := "https://" + indexInfo.RegistryURL + "/v1/images/" + string(imageID) + "/json" response, e := RegistryRequestWithToken(client, URL, indexInfo.DockerToken) if e != nil { blog.Error(e, "Unable to query metadata for image: "+string(imageID)) return } // log.Print("metadata query response: " + string(response)) var m ImageStruct if e = json.Unmarshal(response, &m); e != nil { return } var creationTime time.Time metadata.Image = string(imageID) if creationTime, e = time.Parse(time.RFC3339Nano, m.Created); e != nil { return } metadata.Datetime = creationTime metadata.Size = m.Size metadata.Author = m.Author metadata.Checksum = m.Checksum metadata.Comment = m.Comment metadata.Parent = m.Parent return }
// getReposTokenAuthV1 validates the user-specified list of repositories against an index server, e.g., Docker Hub. // It returns a list of IndexInfo structs with index info for each validated repository. func getReposTokenAuthV1(repo RepoType, client *http.Client) (indexInfo IndexInfo, e error) { _, _, BasicAuth, XRegistryAuth = GetRegistryURL() URL := RegistryAPIURL + "/v1/repositories/" + string(repo) + "/images" req, e := http.NewRequest("GET", URL, nil) req.Header.Set("X-Docker-Token", "true") if BasicAuth != "" { req.Header.Set("Authorization", "Basic "+BasicAuth) } r, e := client.Do(req) if e != nil { blog.Error(e, ":getReposTokenAuthV1 HTTP request failed") return } defer r.Body.Close() if r.StatusCode != 200 { e = &HTTPStatusCodeError{StatusCode: r.StatusCode} return } dockerToken := r.Header.Get("X-Docker-Token") if dockerToken == "" { e = errors.New("lookup error for repo " + string(repo)) return } registryURL := r.Header.Get("X-Docker-Endpoints") arr := strings.Split(registryURL, ",") if len(arr) == 0 { registryURL = "" e = errors.New("lookup error for repo " + string(repo)) return } registryURL = strings.TrimSpace(arr[0]) indexInfo = IndexInfo{Repo: repo, DockerToken: dockerToken, RegistryURL: registryURL} return }
// RemoveImages removes least recently pulled docker images from the local docker host. func RemoveImages(PulledImages []ImageMetadataInfo, imageToMDMap map[string][]ImageMetadataInfo) { numRemoved := 0 for _, imageMD := range PulledImages { // Get all metadata (repo/tags) associated with that image for _, metadata := range imageToMDMap[imageMD.Image] { // basespec := RegistrySpec + "/" + string(t.Repo) + ":" if ExcludeRepo[RepoType(metadata.Repo)] { continue } blog.Debug("Removing the following registry/repo:tag: " + RegistrySpec + "/" + metadata.Repo + ":" + metadata.Tag) apipath := "/images/" + RegistrySpec + "/" + metadata.Repo + ":" + metadata.Tag blog.Info("RemoveImages %s", apipath) config.BanyanUpdate("Remove", apipath) _, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "") if err != nil { blog.Error(err, "RemoveImages Repo:Tag", metadata.Repo, metadata.Tag, "image", metadata.Image) } numRemoved++ } } blog.Info("Number of repo/tags removed this time around: %d", numRemoved) return }
// getRepos queries the Docker registry for the list of the repositories it is currently hosting. // However, if the user specified a list of repositories, then getRepos() just returns that list // of specified repositories and does not query the Docker registry. func getRepos() (repoSlice []RepoType, err error) { if len(ReposToProcess) > 0 { for repo := range ReposToProcess { repoSlice = append(repoSlice, repo) } return } if *RegistryProto == "v2" { blog.Error("v2 registry search/catalog interface not yet supported in collector") return } // a query with an empty query string returns all the repos var client *http.Client if *RegistryTLSNoVerify { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client = &http.Client{Transport: tr} } else { client = &http.Client{} } response, err := RegistryQuery(client, RegistryAPIURL+"/v1/search?q=") if err != nil { blog.Error(err) if s, ok := err.(*HTTPStatusCodeError); ok { blog.Error("HTTP bad status code %d from registry %s using --registryhttps=%v --registryauth=%v --registryproto=%s", s.StatusCode, RegistryAPIURL, *HTTPSRegistry, *AuthRegistry, *RegistryProto) } return } // parse the JSON response body and populate repo slice var result registrySearchResult if err = json.Unmarshal(response, &result); err != nil { blog.Error(err, "unmarshal", string(response)) return } for _, elem := range result.Results { if ExcludeRepo[RepoType(elem.Name)] { continue } repoSlice = append(repoSlice, RepoType(elem.Name)) } return }
// GetImageMetadataTokenAuthV1 returns repositories/tags/image metadata from the Docker Hub // or other registry using v1 token authorization. // The user must have specified a set of repositories of interest. // The function queries the index server, e.g., Docker Hub, to get the token and registry, and then uses // the token to query the registry. func GetImageMetadataTokenAuthV1(oldMetadataSet MetadataSet) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo) { if len(ReposToProcess) == 0 { return } client := &http.Client{} metadataMap := NewImageToMetadataMap(oldMetadataSet) for repo := range ReposToProcess { blog.Info("Get index and tag info for %s", string(repo)) config.BanyanUpdate("Get index and tag info for", string(repo)) var ( indexInfo IndexInfo e error repoTagSlice []TagInfo repoMetadataSlice []ImageMetadataInfo ) // loop until success for { indexInfo, e = getReposTokenAuthV1(repo, client) if e != nil { blog.Warn(e, ":index lookup failed for repo", string(repo), "- retrying.") config.BanyanUpdate(e.Error(), ":index lookup failed, repo", string(repo), "- retrying") time.Sleep(config.RETRYDURATION) continue } repoTagSlice, e = getTagsTokenAuthV1(repo, client, indexInfo) if e != nil { blog.Warn(e, ":tag lookup failed for repo", string(repo), "- retrying.") config.BanyanUpdate(e.Error(), ":tag lookup failed for repo", string(repo), "- retrying") time.Sleep(config.RETRYDURATION) continue } if len(repoTagSlice) != 1 { blog.Error("Incorrect length of repoTagSlice: expected length=1, got length=%d", len(repoTagSlice)) config.BanyanUpdate("Incorrect length of repoTagSlice:", strconv.Itoa(len(repoTagSlice)), string(repo)) time.Sleep(config.RETRYDURATION) continue } repoMetadataSlice, e = getMetadataTokenAuthV1(repoTagSlice[0], metadataMap, client, indexInfo) if e != nil { blog.Warn(e, ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying.") config.BanyanUpdate(e.Error(), ":metadata lookup failed for", string(repoTagSlice[0].Repo), "- retrying") time.Sleep(config.RETRYDURATION) continue } //success! break } tagSlice = append(tagSlice, repoTagSlice...) metadataSlice = append(metadataSlice, repoMetadataSlice...) } return }
// Run handles running of a script inside an image func (sh ScriptInfo) Run(imageID ImageIDType) (b []byte, err error) { jsonString, err := createCmd(imageID, sh.name, sh.staticBinary, sh.dirPath) if err != nil { blog.Error(err, ": Error in creating command") return } blog.Debug("Container spec: %s", string(jsonString)) containerID, err := createContainer(jsonString) if err != nil { blog.Error(err, ": Error in creating container") return } blog.Debug("New container ID: %s", containerID) defer removeContainer(containerID) jsonString, err = startContainer(containerID) if err != nil { blog.Error(err, ": Error in starting container") return } blog.Debug("Response from startContainer: %s", string(jsonString)) statusCode, err := waitContainer(containerID) if err != nil { blog.Error(err, ": Error in waiting for container to stop") return } if statusCode != 0 { err = errors.New("Bash script exit status: " + strconv.Itoa(statusCode)) return } b, err = logsContainer(containerID) if err != nil { blog.Error(err, ":Error in extracting output from container") return } /* _, err = removeContainer(containerID) if err != nil { blog.Error(err, ":Error in removing container for image", containerID) return } */ return }
func getMetadataTokenAuthV1(repotag TagInfo, metadataMap ImageToMetadataMap, client *http.Client, indexInfo IndexInfo) (metadataSlice []ImageMetadataInfo, e error) { // for each tag, generate the current Image Metadata Info repo := repotag.Repo tagmap := repotag.TagMap for tag, imageID := range tagmap { if metadataMap.Exists(imageID) { continue } var metadata ImageMetadataInfo metadata, e = lookupMetadataTokenAuthV1(imageID, client, indexInfo) if e != nil { if s, ok := e.(*HTTPStatusCodeError); ok { blog.Error("Registry returned HTTP status code %d, skipping %s:%s image %s", s.StatusCode, string(repo), string(tag), string(imageID)) continue } // some other error (network broken?), so give up blog.Error(e, "Unable to lookup metadata for", repo, ":", tag, string(imageID)) return } metadata.Repo = string(repo) metadata.Tag = string(tag) metadataMap.Insert(ImageIDType(metadata.Image), metadata) } for tag, imageID := range tagmap { var curr ImageMetadataInfo if metadataMap.Exists(imageID) { // copy previous entry and fill in this repo/tag curr, _ = metadataMap.Metadata(imageID) curr.Repo = string(repo) curr.Tag = string(tag) metadataSlice = append(metadataSlice, curr) } else { e = errors.New("Missing metadata for image ID " + string(imageID)) return } } return }
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) { blog.Info("Writing " + filenamePath + "...") switch f.format { case "json": err := jsonifyAndWriteToFile(filenamePath+".json", data) if err != nil { blog.Error(err, ": Error in writing json output into file: ", filenamePath+".json") return } case "txt": // what's passed in is ptr to interface{}. First get interface{} out of it and then // typecast that to []byte err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644) if err != nil { blog.Error(err, ": Error in writing to file: ", filenamePath) return } default: blog.Warn("Currently only supporting json output to write to files") } }
// getTags queries the Docker registry for the list of the tags for each repository. func getTags(repoSlice []RepoType) (tagSlice []TagInfo, e error) { switch *RegistryProto { case "v1", "quay": return v1GetTags(repoSlice) case "v2": panic("Unreachable") default: blog.Error("Unknown registry protocol %s", *RegistryProto) return } panic("Unreachable") }
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) { switch f.format { case "json": err := jsonifyAndAppendToFile(filenamePath+".json", data) if err != nil { blog.Error(err, ": Error in writing json output into file: ", filenamePath+".json") return } default: blog.Warn("Currently only supporting json output to write to files") } }
func v2GetTagsMetadata(repoSlice []RepoType) (tagSlice []TagInfo, metadataSlice []ImageMetadataInfo, e error) { var client *http.Client if *RegistryTLSNoVerify { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client = &http.Client{Transport: tr} } else { client = &http.Client{} } for _, repo := range repoSlice { // get tags for one repo response, err := RegistryQuery(client, RegistryAPIURL+"/v2/"+string(repo)+"/tags/list") if err != nil { blog.Error(err) if s, ok := err.(*HTTPStatusCodeError); ok { blog.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode) continue } return } //parse JSON output var m V2Tag if e = json.Unmarshal(response, &m); e != nil { return } t := TagInfo{Repo: repo, TagMap: make(map[TagType]ImageIDType)} for _, tag := range m.Tags { metadata, e := v2GetMetadata(client, string(repo), tag) if e != nil { blog.Error(e, ":Unable to get metadata for repo", string(repo), "tag", tag) continue } t.TagMap[TagType(tag)] = ImageIDType(metadata.Image) metadataSlice = append(metadataSlice, metadata) } tagSlice = append(tagSlice, t) } return }
func jsonifyAndAppendToFile(filenamePath string, data ImageMetadataAndAction) (err error) { b, err := json.MarshalIndent(data, "", "\t") if err != nil { blog.Error(err, ": Error in marshaling json") return err } fd, err := os.OpenFile(filenamePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) if err != nil { blog.Error(err, ": Error in opening file: ", filenamePath) return err } defer fd.Close() _, err = fd.Write(b) if err != nil { blog.Error(err, ": Error in writing to file: ", filenamePath) return err } return nil }
// copyDir copies all files from srcDir to destDir func CopyDir(srcDir, destDir string) { existsSrc, err1 := DirExists(srcDir) existsDest, err2 := DirExists(destDir) if err1 != nil || err2 != nil { //detailed error handling inside DirExists return } if !existsSrc || !existsDest { blog.Error("Src/Dest directories don't exist: srcdir: " + srcDir + " destdir: " + destDir) return } files, err := ioutil.ReadDir(srcDir) if err != nil { blog.Error(err, ": Error in reading contents of ", srcDir) return } for _, file := range files { copy(srcDir+"/"+file.Name(), destDir+"/"+file.Name()) } }
func v2GetMetadata(client *http.Client, repo, tag string) (metadata ImageMetadataInfo, e error) { response, err := RegistryQuery(client, RegistryAPIURL+"/v2/"+repo+"/manifests/"+tag) if err != nil { blog.Error(err) if s, ok := err.(*HTTPStatusCodeError); ok { blog.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode) e = err } return } //parse JSON output var m V2Manifest b := bytes.NewBuffer(response) if e = json.NewDecoder(b).Decode(&m); e != nil { return } if len(m.History) == 0 { e = errors.New("repo " + repo + ":" + tag + " no images found in history") return } var image ImageStruct if e = json.Unmarshal([]byte(m.History[0].V1Compatibility), &image); e != nil { return } var creationTime time.Time metadata.Image = image.ID if creationTime, e = time.Parse(time.RFC3339Nano, image.Created); e != nil { return } metadata.Datetime = creationTime metadata.Repo = repo metadata.Tag = tag metadata.Size = image.Size metadata.Author = image.Author metadata.Checksum = image.Checksum metadata.Comment = image.Comment metadata.Parent = image.Parent return }
// Error logs an error message and generates a config.BanyanUpdate. func Error(arg0 interface{}, args ...interface{}) { if len(args) == 0 { blog.Error(arg0) s := fmt.Sprintf("ERROR %v", arg0) config.BanyanUpdate(s) } else { var s string switch arg0.(type) { case string: blog.Error(arg0.(string), args...) s = fmt.Sprintf("ERROR %s", arg0.(string)) s = fmt.Sprintf(s, args...) default: blog.Error(arg0, args...) s = fmt.Sprintf("ERROR %v", arg0) arr := []interface{}{s} arr = append(arr, args...) s = fmt.Sprintln(arr...) } s = strings.TrimRight(s, "\n") config.BanyanUpdate(s) } }
// registrySearchV1 queries the Docker registry, returning a slice of repos. func registrySearchV1(client *http.Client, searchTerm string) (repoSlice []RepoType, err error) { response, err := RegistryQuery(client, RegistryAPIURL+"/v1/search?q="+searchTerm) if err != nil { blog.Error(err) if s, ok := err.(*HTTPStatusCodeError); ok { blog.Error("HTTP bad status code %d from registry %s using --registryhttps=%v --registryauth=%v --registryproto=%s", s.StatusCode, RegistryAPIURL, *HTTPSRegistry, *AuthRegistry, *RegistryProto) } return } // parse the JSON response body and populate repo slice var result registrySearchResult if err = json.Unmarshal(response, &result); err != nil { blog.Error(err, "unmarshal", string(response)) return } for _, elem := range result.Results { if ExcludeRepo[RepoType(elem.Name)] { continue } repoSlice = append(repoSlice, RepoType(elem.Name)) } return }
func SetOutputWriters(authToken string) { dests := strings.Split(*config.Dests, ",") for _, dest := range dests { var writer collector.Writer switch dest { case "file": writer = collector.NewFileWriter("json", *config.BanyanOutDir) default: blog.Error("No such output writer!") //ignore the rest and keep going continue } collector.WriterList = append(collector.WriterList, writer) } }
// GetImageAllData extracts content info from each pulled image. Currently it gets system package info. func GetImageAllData(pulledImages ImageSet) (outMapMap map[string]map[string]interface{}) { //Map ImageID -> Script Map; Script Map: Script name -> output outMapMap = make(map[string]map[string]interface{}) for imageID := range pulledImages { config.BanyanUpdate("Scripts", string(imageID)) outMap, err := runAllScripts(imageID) if err != nil { blog.Error(err, ": Error processing image", string(imageID)) continue } outMapMap[string(imageID)] = outMap } return }