func v1GetTags(repoSlice []RepoType) (tagSlice []TagInfo, e error) { var client *http.Client if *RegistryTLSNoVerify { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client = &http.Client{Transport: tr} } else { client = &http.Client{} } for _, repo := range repoSlice { // get tags for one repo var response []byte response, e = RegistryQuery(client, RegistryAPIURL+"/v1/repositories/"+string(repo)+"/tags") if e != nil { except.Error(e) if s, ok := e.(*HTTPStatusCodeError); ok { except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode) continue } return } //parse JSON output var m map[TagType]ImageIDType if e = json.Unmarshal(response, &m); e != nil { return nil, e } var t TagInfo t.Repo = repo t.TagMap = m tagSlice = append(tagSlice, t) } return }
// ValidRepoName verifies that the name of a repo is in a legal format. // A valid name can optionally include a wildcard "*" but only as the last character. func ValidRepoName(name string) bool { if len(name) == 0 { return false } if len(name) > 256 { except.Error("Invalid repo name, too long: %s", name) return false } for i, c := range name { switch { case c >= 'a' && c <= 'z': continue case c >= 'A' && c <= 'Z': continue case c >= '0' && c <= '9': continue case c == '/' || c == '_' || c == '-' || c == '.': continue case c == '*' && i == len(name)-1: continue default: except.Error("Invalid repo name %s", name) return false } } return true }
func runAllScripts(imageID ImageIDType) (outMap map[string]interface{}, err error) { //script name -> either byte array, or known types (e.g., ImageDataInfo) outMap = make(map[string]interface{}) scripts := getScriptsToRun() for _, script := range scripts { //run script output, err := script.Run(imageID) if err != nil { except.Error(err, ": Error in running script: ", script.Name()) continue //continue trying to run other scripts } //analyze script output switch script.Name() { case PKGEXTRACTSCRIPT: imageDataInfo, err := parsePkgExtractOutput(output, imageID) if err != nil { except.Error(err, ": Error in parsing PkgExtractOuput") return nil, err } outMap[script.Name()] = imageDataInfo default: //script name -> byte array outMap[script.Name()] = output } } return }
func dockerImageID(regspec string, metadata *ImageMetadataInfo) (ID string, err error) { matchRepo := string(metadata.Repo) if regspec != config.DockerHub { matchRepo = regspec + "/" + matchRepo } matchTag := string(metadata.Tag) if strings.HasPrefix(matchRepo, "library/") { matchRepo = strings.Replace(matchRepo, "library/", "", 1) } // verify the image ID of the pulled image matches the expected metadata. imageMap, err := GetLocalImages(false, false) if err != nil { except.Error(err, ":unable to list local images") return } for imageID, repotagSlice := range imageMap { for _, repotag := range repotagSlice { if string(repotag.Repo) == matchRepo && string(repotag.Tag) == matchTag { ID = string(imageID) return } } } err = errors.New("Failed to find local image ID for " + metadata.Repo + ":" + metadata.Tag) except.Error(err) return }
// RegistryRequestWithToken queries a Docker Registry that uses v1 Token Auth, e.g., Docker Hub. func RegistryRequestWithToken(client *http.Client, URL string, dockerToken string) (response []byte, e error) { var req *http.Request req, e = http.NewRequest("GET", URL, nil) if e != nil { except.Error(e) return } req.Header.Set("Authorization", "Token "+dockerToken) var r *http.Response r, e = client.Do(req) if e != nil { except.Error(e) return } defer r.Body.Close() if r.StatusCode != 200 { e = &HTTPStatusCodeError{StatusCode: r.StatusCode} return } response, e = ioutil.ReadAll(r.Body) if e != nil { except.Error(e) return } return }
func v2GetMetadata(client *http.Client, repo, tag string) (metadata ImageMetadataInfo, e error) { response, err := RegistryQueryV2(client, RegistryAPIURL+"/v2/"+repo+"/manifests/"+tag) if err != nil { except.Error(err) if s, ok := err.(*HTTPStatusCodeError); ok { except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode) e = err } return } metadata.Repo = repo metadata.Tag = tag metadata.Image = "" // Recent versions of Docker daemon (1.8.3+?) have complex code that calculates // the image ID from the V2 manifest. // This seems to be in flux as Docker moves toward content-addressable images in 1.10+, // and as the registry image manifest schema itself is still evolving. // As a temporary measure until Docker converges to a more stable state, collector // will calculate its own hash over the V2 manifest and use the calculated // value to try to filter out images that have previously been processed. // The Docker-calculated image ID will get added to the metadata struct // after the image is pulled. sum := sha256.Sum256(response) metadata.ManifestHash = hex.EncodeToString(sum[:]) var m ManifestV2Schema1 b := bytes.NewBuffer(response) if e = json.NewDecoder(b).Decode(&m); e != nil { blog.Warn("Failed to parse manifest") return } if m.SchemaVersion != 1 { blog.Warn("Manifest schema version %d is not yet supported\n", m.SchemaVersion) return } if len(m.History) == 0 { e = errors.New("repo " + repo + ":" + tag + " no images found in history") return } var image ImageStruct if e = json.Unmarshal([]byte(m.History[0].V1Compatibility), &image); e != nil { blog.Warn("Failed to parse ImageStruct") return } var creationTime time.Time if creationTime, e = time.Parse(time.RFC3339Nano, image.Created); e != nil { blog.Warn("Failed to parse creation time") return } metadata.Datetime = creationTime metadata.Size = image.Size metadata.Author = image.Author metadata.Checksum = image.Checksum metadata.Comment = image.Comment metadata.Parent = image.Parent return }
// DockerAPI performs an HTTP GET,POST,DELETE operation to the Docker daemon. func DockerAPI(tr *http.Transport, operation, apipath string, jsonString []byte, XRegistryAuth string) (resp []byte, e error) { switch operation { case "GET", "POST", "DELETE": break default: e = errors.New("Operation " + operation + " not supported") return } // for unix socket, URL (host.domain) is needed but can be anything var host string HTTP := "http://" if DockerProto == "unix" { host = dummydomain } else { host = DockerAddr if DockerTLSVerify { HTTP = "https://" } } URL := HTTP + host + apipath blog.Info("DockerAPI %s", URL) req, e := http.NewRequest(operation, URL, bytes.NewBuffer(jsonString)) if e != nil { except.Error(e, ":DockerAPI failed to create http request") return } req.Header.Add("Content-Type", "application/json") if XRegistryAuth != "" { req.Header.Add("X-Registry-Auth", XRegistryAuth) } //req.Header.Set("Authorization", "Bearer "+authToken) client := &http.Client{Transport: tr, Timeout: DockerTimeout} r, e := client.Do(req) if e != nil { except.Error(e, ":DockerAPI URL", URL, "client request failed") return } defer r.Body.Close() resp, e = ioutil.ReadAll(r.Body) if e != nil { except.Error(e, ":DockerAPI URL", URL, "invalid response body") return } if r.StatusCode < 200 || r.StatusCode > 299 { e = errors.New("DockerAPI URL: " + URL + " status code: " + strconv.Itoa(r.StatusCode) + "error: " + string(resp)) return } return }
// RemoveImages removes least recently pulled docker images from the local docker host. func RemoveImages(PulledImages []ImageMetadataInfo) { numRemoved := 0 imageMap, err := GetLocalImages(false, false) if err != nil { except.Error(err, ": RemoveImages unable to list local images") } for _, metadata := range PulledImages { if strings.HasPrefix(metadata.Repo, "library/") { metadata.Repo = strings.Replace(metadata.Repo, "library/", "", 1) } imageID := ImageIDType(metadata.Image) if metadata.Image == "" { // unknown image ID. Search the repotags for a match var err error imageID, err = imageMap.Image(RepoType(metadata.Repo), TagType(metadata.Tag)) if err != nil { except.Error(err, ": RemoveImages unable to find image ID") break } } // Get all repo:tags associated with the image repoTagSlice := imageMap.RepoTags(imageID) if len(repoTagSlice) == 0 { except.Error("RemoveImages unable to find expected repo:tag " + metadata.Repo + ":" + metadata.Tag + " for image ID=" + string(imageID)) except.Error("imageMap is %v", imageMap) continue } for _, repotag := range repoTagSlice { // basespec := RegistrySpec + "/" + string(t.Repo) + ":" if ExcludeRepo[RepoType(repotag.Repo)] { continue } apipath := "/images/" + string(repotag.Repo) + ":" + string(repotag.Tag) blog.Info("RemoveImages %s", apipath) config.BanyanUpdate("Remove", apipath) _, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "") if err != nil { except.Error(err, "RemoveImages Repo:Tag", repotag.Repo, repotag.Tag, "image", metadata.Image) } numRemoved++ } } blog.Info("Number of repo/tags removed this time around: %d", numRemoved) RemoveDanglingImages() return }
func copy(src, dest string) { // Read all content of src to data data, err := ioutil.ReadFile(src) if err != nil { except.Error(err, ": Error in reading from file: ", src) return } // Write data to dest err = ioutil.WriteFile(dest, data, 0755) if err != nil { except.Error(err, ": Error in writing to file: ", dest) return } }
func jsonifyAndWriteToFile(filenamePath string, data interface{}) (err error) { b, err := json.MarshalIndent(data, "", "\t") if err != nil { except.Error(err, ": Error in marshaling json") return err } err = ioutil.WriteFile(filenamePath, b, 0644) if err != nil { except.Error(err, ": Error in writing to file: ", filenamePath) return err } return nil }
// LogsContainer makes a docker remote API call to get logs from a container. func LogsContainer(containerID string) (output []byte, err error) { apipath := "/containers/" + containerID + "/logs?stdout=1" resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "") if err != nil { except.Error(err, ": Error in Remote Docker API call: ", apipath) return } blog.Debug("Response from docker remote API call for logs: " + string(resp)) for { if len(resp) < 8 { break } header := resp[0:8] var size int32 buf := bytes.NewBuffer(header[4:8]) binary.Read(buf, binary.BigEndian, &size) payload := resp[8:(8 + size)] // blog.Info(string(frame)) resp = resp[(8 + size):] if header[0] == uint8(1) { // 1=stdout: return only the stdout log output = append(output, payload...) } } return }
// getRepos queries the Docker registry for the list of the repositories it is currently hosting. // However, if the user specified a list of repositories, then getRepos() just returns that list // of specified repositories and does not query the Docker registry. func getRepos() (repoSlice []RepoType, err error) { if len(ReposToProcess) > 0 { for repo := range ReposToProcess { repoSlice = append(repoSlice, repo) } return } if *RegistryProto == "v2" { except.Error("v2 registry search/catalog interface not yet supported in collector") return } // a query with an empty query string returns all the repos var client *http.Client if *RegistryTLSNoVerify { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client = &http.Client{Transport: tr} } else { client = &http.Client{} } return registrySearchV1(client, "") }
// getReposTokenAuthV1 validates the user-specified list of repositories against an index server, e.g., Docker Hub. // It returns a list of IndexInfo structs with index info for each validated repository. func getReposTokenAuthV1(repo RepoType, client *http.Client) (indexInfo IndexInfo, e error) { _, _, BasicAuth, XRegistryAuth = GetRegistryURL() URL := RegistryAPIURL + "/v1/repositories/" + string(repo) + "/images" req, e := http.NewRequest("GET", URL, nil) req.Header.Set("X-Docker-Token", "true") if BasicAuth != "" { req.Header.Set("Authorization", "Basic "+BasicAuth) } r, e := client.Do(req) if e != nil { except.Error(e, ":getReposTokenAuthV1 HTTP request failed") return } defer r.Body.Close() if r.StatusCode != 200 { e = &HTTPStatusCodeError{StatusCode: r.StatusCode} return } dockerToken := r.Header.Get("X-Docker-Token") if dockerToken == "" { e = errors.New("lookup error for repo " + string(repo)) return } registryURL := r.Header.Get("X-Docker-Endpoints") arr := strings.Split(registryURL, ",") if len(arr) == 0 { registryURL = "" e = errors.New("lookup error for repo " + string(repo)) return } registryURL = strings.TrimSpace(arr[0]) indexInfo = IndexInfo{Repo: repo, DockerToken: dockerToken, RegistryURL: registryURL} return }
// WriteImageAllData writes image (pkg and other) data into file func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) { blog.Info("Writing image (pkg and other) data into file...") for imageID, scriptMap := range outMapMap { for scriptName, out := range scriptMap { scriptDir := f.dir + "/" + trimExtension(scriptName) err := fsutil.CreateDirIfNotExist(scriptDir) if err != nil { except.Error(err, ": Error creating script dir: ", scriptDir) continue } image := string(imageID) if len(image) < 12 { except.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?") } else { image = string(imageID)[0:12] } filenamePath := scriptDir + "/" + image if _, ok := out.([]byte); ok { f.format = "txt" filenamePath += "-miscdata" } else { // by default it is json. But f.format could get overwritten at any point // in the for loop if the output type is []byte, hence the (re)assignment f.format = "json" // NOTE: If we start using json for output other than imageData, change this filenamePath += "-pkgdata" } f.writeFileInFormat(filenamePath, &out) } } return }
func CreateDirIfNotExist(dir string) (err error) { exists, err := DirExists(dir) if err != nil { except.Error(err, ": Error while querying dir: ", dir) return err } if !exists { err = os.MkdirAll(dir, 0755) if err != nil { except.Error(err, ": Error in creating dir: ", dir) return err } } return nil }
// lookupMetadataTokenAuthV1 takes as input the imageID, and Docker Hub auth/index info, // and it returns ImageMetadataInfo for that image by querying the indexed registry. func lookupMetadataTokenAuthV1(imageID ImageIDType, client *http.Client, indexInfo IndexInfo) ( metadata ImageMetadataInfo, e error) { blog.Info("Get Metadata for Image: %s", string(imageID)) URL := "https://" + indexInfo.RegistryURL + "/v1/images/" + string(imageID) + "/json" response, e := RegistryRequestWithToken(client, URL, indexInfo.DockerToken) if e != nil { except.Error(e, "Unable to query metadata for image: "+string(imageID)) return } // log.Print("metadata query response: " + string(response)) var m ImageStruct if e = json.Unmarshal(response, &m); e != nil { return } var creationTime time.Time metadata.Image = string(imageID) if creationTime, e = time.Parse(time.RFC3339Nano, m.Created); e != nil { return } metadata.Datetime = creationTime metadata.Size = m.Size metadata.Author = m.Author metadata.Checksum = m.Checksum metadata.Comment = m.Comment metadata.Parent = m.Parent return }
func getTagsTokenAuthV1(repo RepoType, client *http.Client, indexInfo IndexInfo) (tagSlice []TagInfo, e error) { tagSlice, e = lookupTagsTokenAuthV1(client, indexInfo) if e != nil { except.Error(e, ": Error in looking up tags in dockerhub") } return }
// Run handles running of a script inside an image func (sh ScriptInfo) Run(imageID ImageIDType) (b []byte, err error) { jsonString, err := createCmd(imageID, sh.name, sh.staticBinary, sh.dirPath) if err != nil { except.Error(err, ": Error in creating command") return } blog.Debug("Container spec: %s", string(jsonString)) containerID, err := CreateContainer(jsonString) if err != nil { except.Error(err, ": Error in creating container") return } blog.Debug("New container ID: %s", containerID) defer RemoveContainer(containerID) jsonString, err = StartContainer(containerID) if err != nil { except.Error(err, ": Error in starting container") return } blog.Debug("Response from StartContainer: %s", string(jsonString)) statusCode, err := WaitContainer(containerID) if err != nil { except.Error(err, ": Error in waiting for container to stop") return } if statusCode != 0 { err = errors.New("Bash script exit status: " + strconv.Itoa(statusCode)) return } b, err = LogsContainer(containerID) if err != nil { except.Error(err, ":Error in extracting output from container") return } /* _, err = removeContainer(containerID) if err != nil { except.Error(err, ":Error in removing container for image", containerID) return } */ return }
// RemoveImageByID calls Docker to remove an image specified by ID. func RemoveImageByID(image ImageIDType) (resp []byte, err error) { apipath := "/images/" + string(image) resp, err = DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "") if err != nil { except.Error(err, "RemoveImageByID") return } return }
func getMetadataTokenAuthV1(repotag TagInfo, metadataMap ImageToMetadataMap, client *http.Client, indexInfo IndexInfo) (metadataSlice []ImageMetadataInfo, e error) { // for each tag, generate the current Image Metadata Info repo := repotag.Repo tagmap := repotag.TagMap for tag, imageID := range tagmap { if metadataMap.Exists(imageID) { continue } var metadata ImageMetadataInfo metadata, e = lookupMetadataTokenAuthV1(imageID, client, indexInfo) if e != nil { if s, ok := e.(*HTTPStatusCodeError); ok { except.Error("Registry returned HTTP status code %d, skipping %s:%s image %s", s.StatusCode, string(repo), string(tag), string(imageID)) continue } // some other error (network broken?), so give up except.Error(e, "Unable to lookup metadata for", repo, ":", tag, string(imageID)) return } metadata.Repo = string(repo) metadata.Tag = string(tag) metadataMap.Insert(ImageIDType(metadata.Image), metadata) } for tag, imageID := range tagmap { var curr ImageMetadataInfo if metadataMap.Exists(imageID) { // copy previous entry and fill in this repo/tag curr, _ = metadataMap.Metadata(imageID) curr.Repo = string(repo) curr.Tag = string(tag) metadataSlice = append(metadataSlice, curr) } else { e = errors.New("Missing metadata for image ID " + string(imageID)) return } } return }
func InspectImage(imageID string) (resp []byte, err error) { apipath := "/images/" + imageID + "/json" resp, err = DockerAPI(DockerTransport, "GET", apipath, []byte{}, "") if err != nil { except.Error(err) return } blog.Debug("Response from docker remote API call for inspect image " + imageID + " : \n" + string(resp)) return }
func InspectContainer(containerID string) (containerSpec ContainerInspection, err error) { apipath := "/containers/" + containerID + "/json" resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "") if err != nil { except.Error(err) return } err = json.Unmarshal(resp, &containerSpec) return }
// listImages makes a docker remote API call to get a list of images func listImages() (resp []byte, err error) { apipath := "/images/json" resp, err = DockerAPI(DockerTransport, "GET", apipath, []byte{}, "") if err != nil { except.Error(err) return } blog.Debug("Response from docker remote API call for list images: " + string(resp)) return }
// RemoveContainer makes a docker remote API call to remove a container. func RemoveContainer(containerID string) (resp []byte, err error) { apipath := "/containers/" + containerID resp, err = DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "") if err != nil { except.Error(err) return } blog.Debug("Response from docker remote API call for remove: " + string(resp)) return }
// StartContainer makes a docker remote API call to start a container. func StartContainer(containerID string) (jsonOut []byte, err error) { apipath := "/containers/" + containerID + "/start" resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, "") if err != nil { except.Error(err, ": Error in Remote Docker API call: ", apipath) return } blog.Debug("Response from docker remote API call for start: " + string(resp)) return }
// WaitContainer makes a docker remote API call to wait for a container to finish running. func WaitContainer(containerID string) (statusCode int, err error) { apipath := "/containers/" + containerID + "/wait" resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, "") if err != nil { except.Error(err, ": Error in Remote Docker API call: ", apipath) return } blog.Debug("Response from docker remote API call for wait: " + string(resp)) var msg struct { StatusCode int } err = json.Unmarshal(resp, &msg) if err != nil { except.Error(err, "waitContainer resp", string(resp)) return } blog.Info("Got StatusCode %d\n", msg.StatusCode) statusCode = msg.StatusCode return }
// ListDanglingImages calls Docker to get the list of dangling images, and // returns a list of their image IDs. func ListDanglingImages() (imageList []ImageIDType, err error) { apipath := `/images/json?filters={"dangling":["true"]}` // apipath = "/images/json" resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "") if err != nil { except.Error(err, "ListDanglingImages") return } var localImageList []LocalImageStruct if err = json.Unmarshal(resp, &localImageList); err != nil { except.Error(err, "ListDanglingImages JSON unmarshal") return } for _, imInfo := range localImageList { imageList = append(imageList, ImageIDType(imInfo.ID)) } return }
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) { blog.Info("Writing " + filenamePath + "...") switch f.format { case "json": err := jsonifyAndWriteToFile(filenamePath+".json", data) if err != nil { except.Error(err, ": Error in writing json output into file: ", filenamePath+".json") return } case "txt": // what's passed in is ptr to interface{}. First get interface{} out of it and then // typecast that to []byte err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644) if err != nil { except.Error(err, ": Error in writing to file: ", filenamePath) return } default: except.Warn("Currently only supporting json output to write to files") } }
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) { switch f.format { case "json": err := jsonifyAndAppendToFile(filenamePath+".json", data) if err != nil { except.Error(err, ": Error in writing json output into file: ", filenamePath+".json") return } default: except.Warn("Currently only supporting json output to write to files") } }
// CreateContainer makes a docker remote API call to create a container. func CreateContainer(containerSpec []byte) (containerID string, err error) { apipath := "/containers/create" resp, err := DockerAPI(DockerTransport, "POST", apipath, containerSpec, "") if err != nil { except.Error(err, ": Error in Remote Docker API call: ", apipath, string(containerSpec)) return } blog.Debug("Response from docker remote API call for create: " + string(resp)) var msg struct { Id string Warnings string } err = json.Unmarshal(resp, &msg) if err != nil { except.Error(err, "createContainer resp", string(resp)) return } blog.Info("Got ID %s Warnings %s\n", msg.Id, msg.Warnings) containerID = msg.Id return }