func v1GetTags(repoSlice []RepoType) (tagSlice []TagInfo, e error) {
var client *http.Client
if *RegistryTLSNoVerify {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client = &http.Client{Transport: tr}
} else {
client = &http.Client{}
}
for _, repo := range repoSlice {
// get tags for one repo
var response []byte
response, e = RegistryQuery(client, RegistryAPIURL+"/v1/repositories/"+string(repo)+"/tags")
if e != nil {
except.Error(e)
if s, ok := e.(*HTTPStatusCodeError); ok {
except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode)
continue
}
return
}
//parse JSON output
var m map[TagType]ImageIDType
if e = json.Unmarshal(response, &m); e != nil {
return nil, e
}
var t TagInfo
t.Repo = repo
t.TagMap = m
tagSlice = append(tagSlice, t)
}
return
}
// ValidRepoName verifies that the name of a repo is in a legal format.
// A valid name can optionally include a wildcard "*" but only as the last character.
func ValidRepoName(name string) bool {
if len(name) == 0 {
return false
}
if len(name) > 256 {
except.Error("Invalid repo name, too long: %s", name)
return false
}
for i, c := range name {
switch {
case c >= 'a' && c <= 'z':
continue
case c >= 'A' && c <= 'Z':
continue
case c >= '0' && c <= '9':
continue
case c == '/' || c == '_' || c == '-' || c == '.':
continue
case c == '*' && i == len(name)-1:
continue
default:
except.Error("Invalid repo name %s", name)
return false
}
}
return true
}
func runAllScripts(imageID ImageIDType) (outMap map[string]interface{}, err error) {
//script name -> either byte array, or known types (e.g., ImageDataInfo)
outMap = make(map[string]interface{})
scripts := getScriptsToRun()
for _, script := range scripts {
//run script
output, err := script.Run(imageID)
if err != nil {
except.Error(err, ": Error in running script: ", script.Name())
continue //continue trying to run other scripts
}
//analyze script output
switch script.Name() {
case PKGEXTRACTSCRIPT:
imageDataInfo, err := parsePkgExtractOutput(output, imageID)
if err != nil {
except.Error(err, ": Error in parsing PkgExtractOuput")
return nil, err
}
outMap[script.Name()] = imageDataInfo
default:
//script name -> byte array
outMap[script.Name()] = output
}
}
return
}
func dockerImageID(regspec string, metadata *ImageMetadataInfo) (ID string, err error) {
matchRepo := string(metadata.Repo)
if regspec != config.DockerHub {
matchRepo = regspec + "/" + matchRepo
}
matchTag := string(metadata.Tag)
if strings.HasPrefix(matchRepo, "library/") {
matchRepo = strings.Replace(matchRepo, "library/", "", 1)
}
// verify the image ID of the pulled image matches the expected metadata.
imageMap, err := GetLocalImages(false, false)
if err != nil {
except.Error(err, ":unable to list local images")
return
}
for imageID, repotagSlice := range imageMap {
for _, repotag := range repotagSlice {
if string(repotag.Repo) == matchRepo && string(repotag.Tag) == matchTag {
ID = string(imageID)
return
}
}
}
err = errors.New("Failed to find local image ID for " + metadata.Repo + ":" + metadata.Tag)
except.Error(err)
return
}
// RegistryRequestWithToken queries a Docker Registry that uses v1 Token Auth, e.g., Docker Hub.
func RegistryRequestWithToken(client *http.Client, URL string, dockerToken string) (response []byte, e error) {
var req *http.Request
req, e = http.NewRequest("GET", URL, nil)
if e != nil {
except.Error(e)
return
}
req.Header.Set("Authorization", "Token "+dockerToken)
var r *http.Response
r, e = client.Do(req)
if e != nil {
except.Error(e)
return
}
defer r.Body.Close()
if r.StatusCode != 200 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
response, e = ioutil.ReadAll(r.Body)
if e != nil {
except.Error(e)
return
}
return
}
func v2GetMetadata(client *http.Client, repo, tag string) (metadata ImageMetadataInfo, e error) {
response, err := RegistryQueryV2(client, RegistryAPIURL+"/v2/"+repo+"/manifests/"+tag)
if err != nil {
except.Error(err)
if s, ok := err.(*HTTPStatusCodeError); ok {
except.Error("Skipping Repo: %s, tag lookup status code %d", string(repo), s.StatusCode)
e = err
}
return
}
metadata.Repo = repo
metadata.Tag = tag
metadata.Image = ""
// Recent versions of Docker daemon (1.8.3+?) have complex code that calculates
// the image ID from the V2 manifest.
// This seems to be in flux as Docker moves toward content-addressable images in 1.10+,
// and as the registry image manifest schema itself is still evolving.
// As a temporary measure until Docker converges to a more stable state, collector
// will calculate its own hash over the V2 manifest and use the calculated
// value to try to filter out images that have previously been processed.
// The Docker-calculated image ID will get added to the metadata struct
// after the image is pulled.
sum := sha256.Sum256(response)
metadata.ManifestHash = hex.EncodeToString(sum[:])
var m ManifestV2Schema1
b := bytes.NewBuffer(response)
if e = json.NewDecoder(b).Decode(&m); e != nil {
blog.Warn("Failed to parse manifest")
return
}
if m.SchemaVersion != 1 {
blog.Warn("Manifest schema version %d is not yet supported\n", m.SchemaVersion)
return
}
if len(m.History) == 0 {
e = errors.New("repo " + repo + ":" + tag + " no images found in history")
return
}
var image ImageStruct
if e = json.Unmarshal([]byte(m.History[0].V1Compatibility), &image); e != nil {
blog.Warn("Failed to parse ImageStruct")
return
}
var creationTime time.Time
if creationTime, e = time.Parse(time.RFC3339Nano, image.Created); e != nil {
blog.Warn("Failed to parse creation time")
return
}
metadata.Datetime = creationTime
metadata.Size = image.Size
metadata.Author = image.Author
metadata.Checksum = image.Checksum
metadata.Comment = image.Comment
metadata.Parent = image.Parent
return
}
// DockerAPI performs an HTTP GET,POST,DELETE operation to the Docker daemon.
func DockerAPI(tr *http.Transport, operation, apipath string, jsonString []byte,
XRegistryAuth string) (resp []byte, e error) {
switch operation {
case "GET", "POST", "DELETE":
break
default:
e = errors.New("Operation " + operation + " not supported")
return
}
// for unix socket, URL (host.domain) is needed but can be anything
var host string
HTTP := "http://"
if DockerProto == "unix" {
host = dummydomain
} else {
host = DockerAddr
if DockerTLSVerify {
HTTP = "https://"
}
}
URL := HTTP + host + apipath
blog.Info("DockerAPI %s", URL)
req, e := http.NewRequest(operation, URL, bytes.NewBuffer(jsonString))
if e != nil {
except.Error(e, ":DockerAPI failed to create http request")
return
}
req.Header.Add("Content-Type", "application/json")
if XRegistryAuth != "" {
req.Header.Add("X-Registry-Auth", XRegistryAuth)
}
//req.Header.Set("Authorization", "Bearer "+authToken)
client := &http.Client{Transport: tr, Timeout: DockerTimeout}
r, e := client.Do(req)
if e != nil {
except.Error(e, ":DockerAPI URL", URL, "client request failed")
return
}
defer r.Body.Close()
resp, e = ioutil.ReadAll(r.Body)
if e != nil {
except.Error(e, ":DockerAPI URL", URL, "invalid response body")
return
}
if r.StatusCode < 200 || r.StatusCode > 299 {
e = errors.New("DockerAPI URL: " + URL + " status code: " + strconv.Itoa(r.StatusCode) +
"error: " + string(resp))
return
}
return
}
// RemoveImages removes least recently pulled docker images from the local docker host.
func RemoveImages(PulledImages []ImageMetadataInfo) {
numRemoved := 0
imageMap, err := GetLocalImages(false, false)
if err != nil {
except.Error(err, ": RemoveImages unable to list local images")
}
for _, metadata := range PulledImages {
if strings.HasPrefix(metadata.Repo, "library/") {
metadata.Repo = strings.Replace(metadata.Repo, "library/", "", 1)
}
imageID := ImageIDType(metadata.Image)
if metadata.Image == "" {
// unknown image ID. Search the repotags for a match
var err error
imageID, err = imageMap.Image(RepoType(metadata.Repo), TagType(metadata.Tag))
if err != nil {
except.Error(err, ": RemoveImages unable to find image ID")
break
}
}
// Get all repo:tags associated with the image
repoTagSlice := imageMap.RepoTags(imageID)
if len(repoTagSlice) == 0 {
except.Error("RemoveImages unable to find expected repo:tag " + metadata.Repo +
":" + metadata.Tag + " for image ID=" + string(imageID))
except.Error("imageMap is %v", imageMap)
continue
}
for _, repotag := range repoTagSlice {
// basespec := RegistrySpec + "/" + string(t.Repo) + ":"
if ExcludeRepo[RepoType(repotag.Repo)] {
continue
}
apipath := "/images/" + string(repotag.Repo) + ":" + string(repotag.Tag)
blog.Info("RemoveImages %s", apipath)
config.BanyanUpdate("Remove", apipath)
_, err := DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
if err != nil {
except.Error(err, "RemoveImages Repo:Tag", repotag.Repo, repotag.Tag,
"image", metadata.Image)
}
numRemoved++
}
}
blog.Info("Number of repo/tags removed this time around: %d", numRemoved)
RemoveDanglingImages()
return
}
func copy(src, dest string) {
// Read all content of src to data
data, err := ioutil.ReadFile(src)
if err != nil {
except.Error(err, ": Error in reading from file: ", src)
return
}
// Write data to dest
err = ioutil.WriteFile(dest, data, 0755)
if err != nil {
except.Error(err, ": Error in writing to file: ", dest)
return
}
}
func jsonifyAndWriteToFile(filenamePath string, data interface{}) (err error) {
b, err := json.MarshalIndent(data, "", "\t")
if err != nil {
except.Error(err, ": Error in marshaling json")
return err
}
err = ioutil.WriteFile(filenamePath, b, 0644)
if err != nil {
except.Error(err, ": Error in writing to file: ", filenamePath)
return err
}
return nil
}
// LogsContainer makes a docker remote API call to get logs from a container.
func LogsContainer(containerID string) (output []byte, err error) {
apipath := "/containers/" + containerID + "/logs?stdout=1"
resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath)
return
}
blog.Debug("Response from docker remote API call for logs: " + string(resp))
for {
if len(resp) < 8 {
break
}
header := resp[0:8]
var size int32
buf := bytes.NewBuffer(header[4:8])
binary.Read(buf, binary.BigEndian, &size)
payload := resp[8:(8 + size)]
// blog.Info(string(frame))
resp = resp[(8 + size):]
if header[0] == uint8(1) {
// 1=stdout: return only the stdout log
output = append(output, payload...)
}
}
return
}
// getRepos queries the Docker registry for the list of the repositories it is currently hosting.
// However, if the user specified a list of repositories, then getRepos() just returns that list
// of specified repositories and does not query the Docker registry.
func getRepos() (repoSlice []RepoType, err error) {
if len(ReposToProcess) > 0 {
for repo := range ReposToProcess {
repoSlice = append(repoSlice, repo)
}
return
}
if *RegistryProto == "v2" {
except.Error("v2 registry search/catalog interface not yet supported in collector")
return
}
// a query with an empty query string returns all the repos
var client *http.Client
if *RegistryTLSNoVerify {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client = &http.Client{Transport: tr}
} else {
client = &http.Client{}
}
return registrySearchV1(client, "")
}
// getReposTokenAuthV1 validates the user-specified list of repositories against an index server, e.g., Docker Hub.
// It returns a list of IndexInfo structs with index info for each validated repository.
func getReposTokenAuthV1(repo RepoType, client *http.Client) (indexInfo IndexInfo, e error) {
_, _, BasicAuth, XRegistryAuth = GetRegistryURL()
URL := RegistryAPIURL + "/v1/repositories/" + string(repo) + "/images"
req, e := http.NewRequest("GET", URL, nil)
req.Header.Set("X-Docker-Token", "true")
if BasicAuth != "" {
req.Header.Set("Authorization", "Basic "+BasicAuth)
}
r, e := client.Do(req)
if e != nil {
except.Error(e, ":getReposTokenAuthV1 HTTP request failed")
return
}
defer r.Body.Close()
if r.StatusCode != 200 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
dockerToken := r.Header.Get("X-Docker-Token")
if dockerToken == "" {
e = errors.New("lookup error for repo " + string(repo))
return
}
registryURL := r.Header.Get("X-Docker-Endpoints")
arr := strings.Split(registryURL, ",")
if len(arr) == 0 {
registryURL = ""
e = errors.New("lookup error for repo " + string(repo))
return
}
registryURL = strings.TrimSpace(arr[0])
indexInfo = IndexInfo{Repo: repo, DockerToken: dockerToken, RegistryURL: registryURL}
return
}
// WriteImageAllData writes image (pkg and other) data into file
func (f *FileWriter) WriteImageAllData(outMapMap map[string]map[string]interface{}) {
blog.Info("Writing image (pkg and other) data into file...")
for imageID, scriptMap := range outMapMap {
for scriptName, out := range scriptMap {
scriptDir := f.dir + "/" + trimExtension(scriptName)
err := fsutil.CreateDirIfNotExist(scriptDir)
if err != nil {
except.Error(err, ": Error creating script dir: ", scriptDir)
continue
}
image := string(imageID)
if len(image) < 12 {
except.Warn("Weird...Haven't seen imageIDs so small -- possibly a test?")
} else {
image = string(imageID)[0:12]
}
filenamePath := scriptDir + "/" + image
if _, ok := out.([]byte); ok {
f.format = "txt"
filenamePath += "-miscdata"
} else {
// by default it is json. But f.format could get overwritten at any point
// in the for loop if the output type is []byte, hence the (re)assignment
f.format = "json"
// NOTE: If we start using json for output other than imageData, change this
filenamePath += "-pkgdata"
}
f.writeFileInFormat(filenamePath, &out)
}
}
return
}
func CreateDirIfNotExist(dir string) (err error) {
exists, err := DirExists(dir)
if err != nil {
except.Error(err, ": Error while querying dir: ", dir)
return err
}
if !exists {
err = os.MkdirAll(dir, 0755)
if err != nil {
except.Error(err, ": Error in creating dir: ", dir)
return err
}
}
return nil
}
// lookupMetadataTokenAuthV1 takes as input the imageID, and Docker Hub auth/index info,
// and it returns ImageMetadataInfo for that image by querying the indexed registry.
func lookupMetadataTokenAuthV1(imageID ImageIDType, client *http.Client, indexInfo IndexInfo) (
metadata ImageMetadataInfo, e error) {
blog.Info("Get Metadata for Image: %s", string(imageID))
URL := "https://" + indexInfo.RegistryURL + "/v1/images/" + string(imageID) + "/json"
response, e := RegistryRequestWithToken(client, URL, indexInfo.DockerToken)
if e != nil {
except.Error(e, "Unable to query metadata for image: "+string(imageID))
return
}
// log.Print("metadata query response: " + string(response))
var m ImageStruct
if e = json.Unmarshal(response, &m); e != nil {
return
}
var creationTime time.Time
metadata.Image = string(imageID)
if creationTime, e = time.Parse(time.RFC3339Nano, m.Created); e != nil {
return
}
metadata.Datetime = creationTime
metadata.Size = m.Size
metadata.Author = m.Author
metadata.Checksum = m.Checksum
metadata.Comment = m.Comment
metadata.Parent = m.Parent
return
}
func getTagsTokenAuthV1(repo RepoType, client *http.Client, indexInfo IndexInfo) (tagSlice []TagInfo, e error) {
tagSlice, e = lookupTagsTokenAuthV1(client, indexInfo)
if e != nil {
except.Error(e, ": Error in looking up tags in dockerhub")
}
return
}
// Run handles running of a script inside an image
func (sh ScriptInfo) Run(imageID ImageIDType) (b []byte, err error) {
jsonString, err := createCmd(imageID, sh.name, sh.staticBinary, sh.dirPath)
if err != nil {
except.Error(err, ": Error in creating command")
return
}
blog.Debug("Container spec: %s", string(jsonString))
containerID, err := CreateContainer(jsonString)
if err != nil {
except.Error(err, ": Error in creating container")
return
}
blog.Debug("New container ID: %s", containerID)
defer RemoveContainer(containerID)
jsonString, err = StartContainer(containerID)
if err != nil {
except.Error(err, ": Error in starting container")
return
}
blog.Debug("Response from StartContainer: %s", string(jsonString))
statusCode, err := WaitContainer(containerID)
if err != nil {
except.Error(err, ": Error in waiting for container to stop")
return
}
if statusCode != 0 {
err = errors.New("Bash script exit status: " + strconv.Itoa(statusCode))
return
}
b, err = LogsContainer(containerID)
if err != nil {
except.Error(err, ":Error in extracting output from container")
return
}
/*
_, err = removeContainer(containerID)
if err != nil {
except.Error(err, ":Error in removing container for image", containerID)
return
}
*/
return
}
// RemoveImageByID calls Docker to remove an image specified by ID.
func RemoveImageByID(image ImageIDType) (resp []byte, err error) {
apipath := "/images/" + string(image)
resp, err = DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
if err != nil {
except.Error(err, "RemoveImageByID")
return
}
return
}
func getMetadataTokenAuthV1(repotag TagInfo, metadataMap ImageToMetadataMap, client *http.Client,
indexInfo IndexInfo) (metadataSlice []ImageMetadataInfo, e error) {
// for each tag, generate the current Image Metadata Info
repo := repotag.Repo
tagmap := repotag.TagMap
for tag, imageID := range tagmap {
if metadataMap.Exists(imageID) {
continue
}
var metadata ImageMetadataInfo
metadata, e = lookupMetadataTokenAuthV1(imageID, client, indexInfo)
if e != nil {
if s, ok := e.(*HTTPStatusCodeError); ok {
except.Error("Registry returned HTTP status code %d, skipping %s:%s image %s",
s.StatusCode, string(repo), string(tag), string(imageID))
continue
}
// some other error (network broken?), so give up
except.Error(e, "Unable to lookup metadata for",
repo, ":", tag, string(imageID))
return
}
metadata.Repo = string(repo)
metadata.Tag = string(tag)
metadataMap.Insert(ImageIDType(metadata.Image), metadata)
}
for tag, imageID := range tagmap {
var curr ImageMetadataInfo
if metadataMap.Exists(imageID) {
// copy previous entry and fill in this repo/tag
curr, _ = metadataMap.Metadata(imageID)
curr.Repo = string(repo)
curr.Tag = string(tag)
metadataSlice = append(metadataSlice, curr)
} else {
e = errors.New("Missing metadata for image ID " + string(imageID))
return
}
}
return
}
func InspectImage(imageID string) (resp []byte, err error) {
apipath := "/images/" + imageID + "/json"
resp, err = DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
blog.Debug("Response from docker remote API call for inspect image " + imageID + " : \n" + string(resp))
return
}
func InspectContainer(containerID string) (containerSpec ContainerInspection, err error) {
apipath := "/containers/" + containerID + "/json"
resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
err = json.Unmarshal(resp, &containerSpec)
return
}
// listImages makes a docker remote API call to get a list of images
func listImages() (resp []byte, err error) {
apipath := "/images/json"
resp, err = DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
blog.Debug("Response from docker remote API call for list images: " + string(resp))
return
}
// RemoveContainer makes a docker remote API call to remove a container.
func RemoveContainer(containerID string) (resp []byte, err error) {
apipath := "/containers/" + containerID
resp, err = DockerAPI(DockerTransport, "DELETE", apipath, []byte{}, "")
if err != nil {
except.Error(err)
return
}
blog.Debug("Response from docker remote API call for remove: " + string(resp))
return
}
// StartContainer makes a docker remote API call to start a container.
func StartContainer(containerID string) (jsonOut []byte, err error) {
apipath := "/containers/" + containerID + "/start"
resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath)
return
}
blog.Debug("Response from docker remote API call for start: " + string(resp))
return
}
// WaitContainer makes a docker remote API call to wait for a container to finish running.
func WaitContainer(containerID string) (statusCode int, err error) {
apipath := "/containers/" + containerID + "/wait"
resp, err := DockerAPI(DockerTransport, "POST", apipath, []byte{}, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath)
return
}
blog.Debug("Response from docker remote API call for wait: " + string(resp))
var msg struct {
StatusCode int
}
err = json.Unmarshal(resp, &msg)
if err != nil {
except.Error(err, "waitContainer resp", string(resp))
return
}
blog.Info("Got StatusCode %d\n", msg.StatusCode)
statusCode = msg.StatusCode
return
}
// ListDanglingImages calls Docker to get the list of dangling images, and
// returns a list of their image IDs.
func ListDanglingImages() (imageList []ImageIDType, err error) {
apipath := `/images/json?filters={"dangling":["true"]}`
// apipath = "/images/json"
resp, err := DockerAPI(DockerTransport, "GET", apipath, []byte{}, "")
if err != nil {
except.Error(err, "ListDanglingImages")
return
}
var localImageList []LocalImageStruct
if err = json.Unmarshal(resp, &localImageList); err != nil {
except.Error(err, "ListDanglingImages JSON unmarshal")
return
}
for _, imInfo := range localImageList {
imageList = append(imageList, ImageIDType(imInfo.ID))
}
return
}
func (f *FileWriter) writeFileInFormat(filenamePath string, data interface{}) {
blog.Info("Writing " + filenamePath + "...")
switch f.format {
case "json":
err := jsonifyAndWriteToFile(filenamePath+".json", data)
if err != nil {
except.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
return
}
case "txt":
// what's passed in is ptr to interface{}. First get interface{} out of it and then
// typecast that to []byte
err := ioutil.WriteFile(filenamePath+".txt", (*(data.(*interface{}))).([]byte), 0644)
if err != nil {
except.Error(err, ": Error in writing to file: ", filenamePath)
return
}
default:
except.Warn("Currently only supporting json output to write to files")
}
}
func (f *FileWriter) appendFileInFormat(filenamePath string, data ImageMetadataAndAction) {
switch f.format {
case "json":
err := jsonifyAndAppendToFile(filenamePath+".json", data)
if err != nil {
except.Error(err, ": Error in writing json output into file: ", filenamePath+".json")
return
}
default:
except.Warn("Currently only supporting json output to write to files")
}
}
// CreateContainer makes a docker remote API call to create a container.
func CreateContainer(containerSpec []byte) (containerID string, err error) {
apipath := "/containers/create"
resp, err := DockerAPI(DockerTransport, "POST", apipath, containerSpec, "")
if err != nil {
except.Error(err, ": Error in Remote Docker API call: ", apipath, string(containerSpec))
return
}
blog.Debug("Response from docker remote API call for create: " + string(resp))
var msg struct {
Id string
Warnings string
}
err = json.Unmarshal(resp, &msg)
if err != nil {
except.Error(err, "createContainer resp", string(resp))
return
}
blog.Info("Got ID %s Warnings %s\n", msg.Id, msg.Warnings)
containerID = msg.Id
return
}