// StartPod creates a new pod on a Kubernetes cluster and returns a buildlet client
// configured to speak to it.
func StartPod(ctx context.Context, kubeClient *kubernetes.Client, podName, builderType string, opts PodOpts) (*Client, error) {
conf, ok := dashboard.Builders[builderType]
if !ok || conf.KubeImage == "" {
return nil, fmt.Errorf("invalid builder type %q", builderType)
}
pod := &api.Pod{
TypeMeta: api.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: api.ObjectMeta{
Name: podName,
Labels: map[string]string{
"name": podName,
"type": builderType,
"role": "buildlet",
},
Annotations: map[string]string{},
},
Spec: api.PodSpec{
RestartPolicy: api.RestartPolicyNever,
Containers: []api.Container{
{
Name: "buildlet",
Image: imageID(opts.ImageRegistry, conf.KubeImage),
ImagePullPolicy: api.PullAlways,
Resources: api.ResourceRequirements{
Limits: api.ResourceList{
api.ResourceCPU: BuildletCPU,
api.ResourceMemory: BuildletMemory,
},
},
Command: []string{"/usr/local/bin/stage0"},
Ports: []api.ContainerPort{
{
ContainerPort: 80,
},
},
Env: []api.EnvVar{
{
Name: "IN_KUBERNETES",
Value: "1",
},
},
},
},
},
}
addEnv := func(name, value string) {
for i, _ := range pod.Spec.Containers {
pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, api.EnvVar{
Name: name,
Value: value,
})
}
}
// The buildlet-binary-url is the URL of the buildlet binary
// which the pods are configured to download at boot and run.
// This lets us/ update the buildlet more easily than
// rebuilding the whole pod image.
addEnv("META_BUILDLET_BINARY_URL", conf.BuildletBinaryURL())
addEnv("META_BUILDER_TYPE", builderType)
if !opts.TLS.IsZero() {
addEnv("META_TLS_CERT", opts.TLS.CertPEM)
addEnv("META_TLS_KEY", opts.TLS.KeyPEM)
addEnv("META_PASSWORD", opts.TLS.Password())
}
if opts.DeleteIn != 0 {
// In case the pod gets away from us (generally: if the
// coordinator dies while a build is running), then we
// set this annotation of when it should be killed so
// we can kill it later when the coordinator is
// restarted. The cleanUpOldPods goroutine loop handles
// that killing.
pod.ObjectMeta.Annotations["delete-at"] = fmt.Sprint(time.Now().Add(opts.DeleteIn).Unix())
}
status, err := kubeClient.RunPod(ctx, pod)
if err != nil {
return nil, fmt.Errorf("pod could not be created: %v", err)
}
condRun(opts.OnPodCreated)
// The new pod must be in Running phase. Possible phases are described at
// http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#pod-phase
if status.Phase != api.PodRunning {
return nil, fmt.Errorf("pod is in invalid state %q: %v", status.Phase, status.Message)
}
// Wait for the pod to boot and its buildlet to come up.
var buildletURL string
var ipPort string
if !opts.TLS.IsZero() {
buildletURL = "https://" + status.PodIP
ipPort = status.PodIP + ":443"
} else {
buildletURL = "http://" + status.PodIP
ipPort = status.PodIP + ":80"
}
condRun(opts.OnGotPodInfo)
impatientClient := &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
Dial: defaultDialer(),
DisableKeepAlives: true,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
c := make(chan error, 1)
go func() {
defer close(c)
try := 0
for {
try++
// Make sure pod is still running
podStatus, err := kubeClient.PodStatus(ctx, pod.Name)
if err != nil {
c <- fmt.Errorf("polling the buildlet pod for its status failed: %v", err)
return
}
if podStatus.Phase != api.PodRunning {
podLog, err := kubeClient.PodLog(ctx, pod.Name)
if err != nil {
log.Printf("failed to retrieve log for pod %q: %v", pod.Name, err)
c <- fmt.Errorf("buildlet pod left the Running phase and entered phase %q", podStatus.Phase)
return
}
log.Printf("log from pod %q: %v", pod.Name, podLog)
c <- fmt.Errorf("buildlet pod left the Running phase and entered phase %q", podStatus.Phase)
return
}
res, err := ctxhttp.Get(ctx, impatientClient, buildletURL)
if err != nil {
time.Sleep(1 * time.Second)
continue
}
res.Body.Close()
if res.StatusCode != 200 {
c <- fmt.Errorf("buildlet returned HTTP status code %d on try number %d", res.StatusCode, try)
}
return
}
}()
// Wait for the buildlet to respond to an HTTP request. If the timeout happens first, or
// if the buildlet pod leaves the running state, return an error.
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case err = <-c:
if err != nil {
return nil, err
}
return NewClient(ipPort, opts.TLS), nil
}
}
}
// StartPod creates a new pod on a Kubernetes cluster and returns a buildlet client
// configured to speak to it.
func StartPod(kubeClient *kubernetes.Client, podName, builderType string, opts PodOpts) (*Client, error) {
conf, ok := dashboard.Builders[builderType]
if !ok || conf.KubeImage == "" {
return nil, fmt.Errorf("invalid builder type %q", builderType)
}
pod := &api.Pod{
TypeMeta: api.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: api.ObjectMeta{
Name: podName,
Labels: map[string]string{
"name": podName,
"type": builderType,
"role": "buildlet",
},
},
Spec: api.PodSpec{
RestartPolicy: "Never",
Containers: []api.Container{
{
Name: "buildlet",
Image: imageID(opts.ImageRegistry, conf.KubeImage),
ImagePullPolicy: api.PullAlways,
Command: []string{"/usr/local/bin/stage0"},
Ports: []api.ContainerPort{
{
ContainerPort: 80,
},
},
Env: []api.EnvVar{
{
Name: "IN_KUBERNETES",
Value: "1",
},
},
},
},
},
}
addEnv := func(name, value string) {
for i, _ := range pod.Spec.Containers {
pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, api.EnvVar{
Name: name,
Value: value,
})
}
}
// The buildlet-binary-url is the URL of the buildlet binary
// which the pods are configured to download at boot and run.
// This lets us/ update the buildlet more easily than
// rebuilding the whole pod image.
addEnv("META_BUILDLET_BINARY_URL", conf.BuildletBinaryURL())
addEnv("META_BUILDER_TYPE", builderType)
if !opts.TLS.IsZero() {
addEnv("META_TLS_CERT", opts.TLS.CertPEM)
addEnv("META_TLS_KEY", opts.TLS.KeyPEM)
addEnv("META_PASSWORD", opts.TLS.Password())
}
if opts.DeleteIn != 0 {
// In case the pod gets away from us (generally: if the
// coordinator dies while a build is running), then we
// set this attribute of when it should be killed so
// we can kill it later when the coordinator is
// restarted. The cleanUpOldPods goroutine loop handles
// that killing.
addEnv("META_DELETE_AT", fmt.Sprint(time.Now().Add(opts.DeleteIn).Unix()))
}
status, err := kubeClient.Run(pod)
if err != nil {
return nil, fmt.Errorf("pod could not be created: %v", err)
}
// The new pod must be in Running phase. Possible phases are described at
// http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#pod-phase
if status.Phase != api.PodRunning {
return nil, fmt.Errorf("pod is in invalid state %q: %v", status.Phase, status.Message)
}
// Wait for the pod to boot and its buildlet to come up.
var buildletURL string
var ipPort string
if !opts.TLS.IsZero() {
buildletURL = "https://" + status.PodIP
ipPort = status.PodIP + ":443"
} else {
buildletURL = "http://" + status.PodIP
ipPort = status.PodIP + ":80"
}
condRun(opts.OnGotPodInfo)
const timeout = 3 * time.Minute
var alive bool
impatientClient := &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
Dial: defaultDialer(),
DisableKeepAlives: true,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
deadline := time.Now().Add(timeout)
try := 0
for time.Now().Before(deadline) {
try++
res, err := impatientClient.Get(buildletURL)
if err != nil {
time.Sleep(1 * time.Second)
continue
}
res.Body.Close()
if res.StatusCode != 200 {
return nil, fmt.Errorf("buildlet returned HTTP status code %d on try number %d", res.StatusCode, try)
}
alive = true
break
}
if !alive {
return nil, fmt.Errorf("buildlet didn't come up at %s in %v", buildletURL, timeout)
}
return NewClient(ipPort, opts.TLS), nil
}