Esempio n. 1
0
func startAnalyzers(sourceDir string, images []string, dind bool) (containers []string, errs []error) {
	var wg sync.WaitGroup
	for id, fullImage := range images {
		wg.Add(1)
		go func(id int, image string) {
			analyzerContainer, port := getContainerAndAddress(image, id)
			if docker.ImageMatches(image, analyzerContainer) {
				glog.Infof("Reusing analyzer %v started at localhost:%d", image, port)
			} else {
				glog.Infof("Found no analyzer container (%v) to reuse for %v", analyzerContainer, image)
				// Analyzer is either running with the wrong image version, or not running
				// Stopping in case it's the first case
				result := docker.Stop(analyzerContainer, 0, true)
				if result.Err != nil {
					glog.Infof("Failed to stop %v (may not be running)", analyzerContainer)
				}
				result = docker.RunAnalyzer(image, analyzerContainer, sourceDir, localLogs, port, dind)
				if result.Err != nil {
					glog.Infof("Could not start %v at localhost:%d: %v, stderr: %v", image, port, result.Err.Error(), result.Stderr)
					errs = append(errs, result.Err)
				} else {
					glog.Infof("Analyzer %v started at localhost:%d", image, port)
					containers = append(containers, analyzerContainer)
				}
			}
			wg.Done()
		}(id, fullImage)
	}
	if len(images) > 0 {
		glog.Info("Waiting for dockerized analyzers to start up...")
		wg.Wait()
		glog.Info("Analyzers up")
	}
	return containers, errs
}
Esempio n. 2
0
// startShipshapeService ensures that there is a service started with the given image and
// attached analyzers that can analyze the directory at absRoot (an absolute path). If a
// service is not started up that can do this, it will shut down the existing one and start
// a new one.
// The methods returns the (ready) client, the relative path from the docker container's mapped
// volume to the absRoot that we are analyzing, and any errors from attempting to run the service.
// TODO(ciera): This *should* check the analyzers that are connected, but does not yet
// do so.
func startShipshapeService(image, absRoot string, analyzers []string, dind bool) (*client.Client, string, error) {
	glog.Infof("Starting shipshape...")
	container := "shipping_container"
	// subPath is the relatve path from the mapped volume on shipping container
	// to the directory we are analyzing (absRoot)
	isMapped, subPath := docker.MappedVolume(absRoot, container)
	// Stop and restart the container if:
	// 1: The container is not using the latest image OR
	// 2: The container is not mapped to the right directory OR
	// 3: The container is not linked to the right analyzer containers
	// Otherwise, use the existing container
	if !docker.ImageMatches(image, container) || !isMapped || !docker.ContainsLinks(container, analyzers) {
		glog.Infof("Restarting container with %s", image)
		stop(container, 0)
		result := docker.RunService(image, container, absRoot, localLogs, analyzers, dind)
		subPath = ""
		printStreams(result)
		if result.Err != nil {
			return nil, "", result.Err
		}
	}
	glog.Infof("Image %s running in service mode", image)
	c := client.NewHTTPClient("localhost:10007")
	return c, subPath, c.WaitUntilReady(10 * time.Second)
}
Esempio n. 3
0
func stop(container string, timeWait time.Duration) {
	glog.Infof("Stopping and removing %s", container)
	result := docker.Stop(container, timeWait, true)
	printStreams(result)
	if result.Err != nil {
		glog.Infof("Could not stop %s: %v", container, result.Err)
	} else {
		glog.Infoln("Removed.")
	}
}
Esempio n. 4
0
// RunAnalyzer runs the analyzer image with container analyzerContainer. It runs it at port (mapped
// to internal port 10005), binds the volumes for the workspacePath and logsPath, and gives the privileged
// if dind (docker-in-docker) is true.
func RunAnalyzer(image, analyzerContainer, workspacePath, logsPath string, port int, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(analyzerContainer) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{
		workspacePath: shipshapeWork,
		logsPath:      shipshapeLogs,
	}
	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(analyzerContainer, map[int]int{port: 10005}, volumeMap, nil, nil)...)
	args = append(args, "-d", image)

	glog.Infof("Running 'docker %v'\n", args)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}
Esempio n. 5
0
// RunService runs the shipshape service at image, as the container named container. It binds the
// shipshape workspace and logs appropriately. It starts with the third-party analyzers already
// running at analyzerContainers. The service is started with the privileged flag if dind (docker-in-docker)
// is true.
func RunService(image, container, workspacePath, logsPath string, analyzerContainers []string, dind bool) CommandResult {
	stdout := bytes.NewBuffer(nil)
	stderr := bytes.NewBuffer(nil)
	if len(container) == 0 {
		return CommandResult{"", "", errors.New("need to provide a name for the container")}
	}

	volumeMap := map[string]string{workspacePath: shipshapeWork, logsPath: shipshapeLogs}

	var locations []string
	for _, container := range analyzerContainers {
		locations = append(locations, fmt.Sprintf(`$%s_PORT_10005_TCP_ADDR:$%s_PORT_10005_TCP_PORT`, strings.ToUpper(container), strings.ToUpper(container)))
	}
	locations = append(locations, "localhost:10005", "localhost:10006", "localhost:10008")

	args := []string{"run"}
	if dind {
		args = append(args, "--privileged")
	}
	args = append(args, setupArgs(container, map[int]int{10007: 10007}, volumeMap, analyzerContainers, map[string]string{"START_SERVICE": "true", "ANALYZERS": strings.Join(locations, ",")})...)
	args = append(args, "-d", image)

	glog.Infof("Running 'docker %v'\n", args)

	cmd := exec.Command("docker", args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	err := cmd.Run()
	return CommandResult{stdout.String(), stderr.String(), err}
}
Esempio n. 6
0
func printStreams(result docker.CommandResult) {
	out := strings.TrimSpace(result.Stdout)
	err := strings.TrimSpace(result.Stderr)
	if len(out) > 0 {
		glog.Infof("stdout:\n%s\n", strings.TrimSpace(result.Stdout))
	}
	if len(err) > 0 {
		glog.Errorf("stderr:\n%s\n", strings.TrimSpace(result.Stderr))
	}
}
func (i *Invocation) StartService() error {
	_, paths, cleanup, err := i.startServices()
	// TODO(ciera): This is a problem right now. Since we call cleanup here,
	// it's going to immediately take down any third party analyzer, thus making the entire
	// start service step rather pointless when we use third party analyzers.
	defer cleanup()
	if err != nil {
		return fmt.Errorf("HTTP client did not become healthy: %v", err)
	}
	glog.Infof("Service started for %s", paths.absRoot)
	return nil
}
Esempio n. 8
0
func pull(image string) {
	if !docker.OutOfDate(image) {
		return
	}
	glog.Infof("Pulling image %s", image)
	result := docker.Pull(image)
	printStreams(result)
	if result.Err != nil {
		glog.Errorf("Error from pull: %v", result.Err)
		return
	}
	glog.Infoln("Pulling complete")
}
func pull(image string) {
	// If we are "local", use a local version and don't actually do a pull.
	// Also don't pull if we aren't out of date yet.
	if strings.HasSuffix(image, ":local") || !docker.OutOfDate(image) {
		return
	}
	glog.Infof("Pulling image %s", image)
	result := docker.Pull(image)
	printStreams(result)
	if result.Err != nil {
		glog.Errorf("Error from pull: %v", result.Err)
		return
	}
	glog.Infoln("Pulling complete")
}
Esempio n. 10
0
func (i *Invocation) ShowCategories() error {
	var c *client.Client
	var res *rpcpb.GetCategoryResponse

	// Run it on files
	c, _, cleanup, err := i.startServices()
	defer cleanup()
	if err != nil {
		return fmt.Errorf("HTTP client did not become healthy: %v", err)
	}
	glog.Infof("Calling to get the categories")
	err = c.Call("/ShipshapeService/GetCategory", &rpcpb.GetCategoryRequest{}, &res)
	if err != nil {
		fmt.Errorf("Could not get categories: %v", err)
		return err
	}

	fmt.Println(strings.Join(res.Category, "\n"))
	return nil
}
Esempio n. 11
0
func analyze(c *client.Client, req *rpcpb.ShipshapeRequest, originalDir string, handleResponse func(msg *rpcpb.ShipshapeResponse, directory string) error) (int, error) {
	var totalNotes = 0
	glog.Infof("Calling to the shipshape service with %v", req)
	rd := c.Stream("/ShipshapeService/Run", req)
	defer rd.Close()
	for {
		var msg rpcpb.ShipshapeResponse
		if err := rd.NextResult(&msg); err == io.EOF {
			break
		} else if err != nil {
			return 0, fmt.Errorf("received an error from calling run: %v", err.Error())
		}

		err := handleResponse(&msg, originalDir)
		if err != nil {
			return 0, fmt.Errorf("could not parse results: %v", err.Error())
		}
		totalNotes += numNotes(&msg)
	}
	return totalNotes, nil
}
Esempio n. 12
0
func (i *Invocation) Run() (int, error) {
	glog.Infof("Starting shipshape...")
	fs, err := os.Stat(i.options.File)
	if err != nil {
		return 0, fmt.Errorf("%s is not a valid file or directory\n", i.options.File)
	}

	origDir := i.options.File
	if !fs.IsDir() {
		origDir = filepath.Dir(i.options.File)
	}

	absRoot, err := filepath.Abs(origDir)
	if err != nil {
		return 0, fmt.Errorf("could not get absolute path for %s: %v\n", origDir, err)
	}

	if !docker.HasDocker() {
		return 0, fmt.Errorf("docker could not be found. Make sure you have docker installed.")
	}

	image := docker.FullImageName(i.options.Repo, image, i.options.Tag)
	glog.Infof("Starting shipshape using %s on %s", image, absRoot)

	// Create the request

	if len(i.options.TriggerCats) == 0 {
		glog.Infof("No categories provided. Will be using categories specified by the config file for the event %s", i.options.Event)
	}

	if len(i.options.ThirdPartyAnalyzers) == 0 {
		i.options.ThirdPartyAnalyzers, err = service.GlobalConfig(absRoot)
		if err != nil {
			glog.Infof("Could not get global config; using only the default analyzers: %v", err)
		}
	}

	// If we are not running in local mode, pull the latest copy
	// Notice this will use the local tag as a signal to not pull the
	// third-party analyzers either.
	if i.options.Tag != "local" {
		pull(image)
		pullAnalyzers(i.options.ThirdPartyAnalyzers)
	}

	// Put in this defer before calling run. Even if run fails, it can
	// still create the container.
	if !i.options.StayUp {
		// TODO(ciera): Rather than immediately sending a SIGKILL,
		// we should use the default 10 seconds and properly handle
		// SIGTERMs in the endpoint script.
		defer stop("shipping_container", 0)
		// Stop all the analyzers, even the ones that had trouble starting,
		// in case they did actually start
		for id, analyzerRepo := range i.options.ThirdPartyAnalyzers {
			container, _ := getContainerAndAddress(analyzerRepo, id)
			defer stop(container, 0)
		}
	}

	containers, errs := startAnalyzers(absRoot, i.options.ThirdPartyAnalyzers, i.options.Dind)
	for _, err := range errs {
		glog.Errorf("Could not start up third party analyzer: %v", err)
	}

	var c *client.Client
	var req *rpcpb.ShipshapeRequest
	var numNotes int

	// Run it on files
	relativeRoot := ""
	c, relativeRoot, err = startShipshapeService(image, absRoot, containers, i.options.Dind)
	if err != nil {
		return 0, fmt.Errorf("HTTP client did not become healthy: %v", err)
	}
	var files []string
	if !fs.IsDir() {
		files = []string{filepath.Base(i.options.File)}
	}
	req = createRequest(i.options.TriggerCats, files, i.options.Event, filepath.Join(workspace, relativeRoot), ctxpb.Stage_PRE_BUILD.Enum())
	glog.Infof("Calling with request %v", req)
	numNotes, err = analyze(c, req, origDir, i.options.HandleResponse)
	if err != nil {
		return numNotes, fmt.Errorf("error making service call: %v", err)
	}

	// If desired, generate compilation units with a kythe image
	if i.options.Build != "" {
		// TODO(ciera): Handle other build systems
		fullKytheImage := docker.FullImageName(i.options.Repo, kytheImage, i.options.Tag)
		if !i.options.LocalKythe {
			pull(fullKytheImage)
		}

		// TODO(emso): Add a check for an already running kythe container.
		// The below defer should stop the one started below but in case this
		// failed for some reason (or a kythe container was started in some other
		// way) the below run command will fail.
		defer stop("kythe", 10*time.Second)
		glog.Infof("Retrieving compilation units with %s", i.options.Build)

		result := docker.RunKythe(fullKytheImage, "kythe", absRoot, i.options.Build, i.options.Dind)
		if result.Err != nil {
			// kythe spews output, so only capture it if something went wrong.
			printStreams(result)
			return numNotes, fmt.Errorf("error from run: %v", result.Err)
		}
		glog.Infoln("CompilationUnits prepared")

		req.Stage = ctxpb.Stage_POST_BUILD.Enum()
		glog.Infof("Calling with request %v", req)
		numBuildNotes, err := analyze(c, req, origDir, i.options.HandleResponse)
		numNotes += numBuildNotes
		if err != nil {
			return numNotes, fmt.Errorf("error making service call: %v", err)
		}
	}
	if i.options.ResponsesDone != nil {
		if err := i.options.ResponsesDone(); err != nil {
			return numNotes, err
		}
	}

	glog.Infoln("End of Results.")
	return numNotes, nil
}
Esempio n. 13
0
func (i *Invocation) Run() (int, error) {
	var c *client.Client
	var req *rpcpb.ShipshapeRequest
	var numNotes int

	// Run it on files
	c, paths, cleanup, err := i.startServices()
	defer cleanup()
	if err != nil {
		return 0, err
	}
	var files []string
	if !paths.fs.IsDir() {
		files = []string{filepath.Base(i.options.File)}
	}
	if len(i.options.TriggerCats) == 0 {
		glog.Infof("No categories provided. Will be using categories specified by the config file for the event %s", i.options.Event)
	}
	req = createRequest(i.options.TriggerCats, files, i.options.Event, filepath.Join(workspace, paths.relativeRoot), ctxpb.Stage_PRE_BUILD.Enum())
	glog.Infof("Calling with request %v", req)
	numNotes, err = analyze(c, req, paths.origDir, i.options.HandleResponse)
	if err != nil {
		return numNotes, fmt.Errorf("error making service call: %v", err)
	}

	// If desired, generate compilation units with a kythe image
	if i.options.Build != "" {
		// TODO(ciera): Handle other build systems
		fullKytheImage := docker.FullImageName(i.options.Repo, kytheImage, i.options.Tag)
		if !i.options.LocalKythe {
			pull(fullKytheImage)
		}

		// Stop kythe if it is running otherwise we will fail when we start kythe below
		exists, err := docker.ContainerExists(fullKytheImage)
		if err != nil {
			return numNotes, fmt.Errorf("error making service call: %v", err)
		}
		if exists {
			stop(fullKytheImage, 0)
		}
		// Make sure we stop kythe after we are done
		defer stop("kythe", 10*time.Second)

		glog.Infof("Retrieving compilation units with %s", i.options.Build)
		result := docker.RunKythe(fullKytheImage, "kythe", paths.absRoot, i.options.Build, i.options.Dind)
		if result.Err != nil {
			// kythe spews output, so only capture it if something went wrong.
			printStreams(result)
			return numNotes, fmt.Errorf("error from run: %v", result.Err)
		}
		glog.Infoln("CompilationUnits prepared")

		req.Stage = ctxpb.Stage_POST_BUILD.Enum()
		glog.Infof("Calling with request %v", req)
		numBuildNotes, err := analyze(c, req, paths.origDir, i.options.HandleResponse)
		numNotes += numBuildNotes
		if err != nil {
			return numNotes, fmt.Errorf("error making service call: %v", err)
		}
	}
	if i.options.ResponsesDone != nil {
		if err := i.options.ResponsesDone(); err != nil {
			return numNotes, err
		}
	}

	glog.Infoln("End of Results.")
	return numNotes, nil
}
Esempio n. 14
0
func (i *Invocation) startServices() (*client.Client, Paths, func(), error) {
	var paths Paths
	var err error

	glog.Infof("Starting shipshape...")
	paths.fs, err = os.Stat(i.options.File)
	if err != nil {
		return nil, paths, func() {}, fmt.Errorf("%s is not a valid file or directory\n", i.options.File)
	}

	paths.origDir = i.options.File
	if !paths.fs.IsDir() {
		paths.origDir = filepath.Dir(i.options.File)
	}

	paths.absRoot, err = filepath.Abs(paths.origDir)
	if err != nil {
		return nil, paths, func() {}, fmt.Errorf("could not get absolute path for %s: %v\n", paths.origDir, err)
	}

	if !docker.HasDocker() {
		return nil, paths, func() {}, fmt.Errorf("docker could not be found. Make sure you have docker installed.")
	}

	image := docker.FullImageName(i.options.Repo, image, i.options.Tag)
	glog.Infof("Starting shipshape using %s on %s", image, paths.absRoot)

	if len(i.options.ThirdPartyAnalyzers) == 0 {
		i.options.ThirdPartyAnalyzers, err = service.GlobalConfig(paths.absRoot)
		if err != nil {
			glog.Infof("Could not get global config; using only the default analyzers: %v", err)
		}
	}

	pull(image)
	pullAnalyzers(i.options.ThirdPartyAnalyzers)

	// Create a cleanup function that will stop all the containers we started,
	// if that is desired.
	cleanup := func() {
		if !i.options.StayUp {
			// TODO(ciera): Rather than immediately sending a SIGKILL,
			// we should use the default 10 seconds and properly handle
			// SIGTERMs in the endpoint script.
			stop("shipping_container", 0)
		}
		// Stop all the analyzers, even the ones that had trouble starting,
		// in case they did actually start
		for id, analyzerRepo := range i.options.ThirdPartyAnalyzers {
			container, _ := getContainerAndAddress(analyzerRepo, id)
			stop(container, 0)
		}
	}

	containers, errs := startAnalyzers(paths.absRoot, i.options.ThirdPartyAnalyzers, i.options.Dind)
	for _, err := range errs {
		glog.Errorf("Could not start up third party analyzer: %v", err)
	}
	var c *client.Client
	c, paths.relativeRoot, err = startShipshapeService(image, paths.absRoot, containers, i.options.Dind)
	if err != nil {
		return nil, paths, cleanup, fmt.Errorf("HTTP client did not become healthy: %v", err)
	}
	return c, paths, cleanup, nil
}