// handleRun handles requests to run a command inside a container.
func (s *Server) handleRun(w http.ResponseWriter, req *http.Request) {
u, err := url.ParseRequestURI(req.RequestURI)
if err != nil {
s.error(w, err)
return
}
podNamespace, podID, uid, container, err := parseContainerCoordinates(u.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pod, ok := s.host.GetPodByName(podNamespace, podID)
if !ok {
http.Error(w, "Pod does not exist", http.StatusNotFound)
return
}
command := strings.Split(u.Query().Get("cmd"), " ")
data, err := s.host.RunInContainer(qingcontainer.GetPodFullName(pod), uid, container, command)
if err != nil {
s.error(w, err)
return
}
w.Header().Add("Content-type", "text/plain")
w.Write(data)
}
func (fmc *fakeMirrorClient) CreateMirrorPod(pod *api.Pod) error {
fmc.mirrorPodLock.Lock()
defer fmc.mirrorPodLock.Unlock()
podFullName := qingcontainer.GetPodFullName(pod)
fmc.mirrorPods.Insert(podFullName)
fmc.createCounts[podFullName]++
return nil
}
func TestNewStatus(t *testing.T) {
syncer := newTestStatusManager()
syncer.SetPodStatus(testPod, getRandomPodStatus())
verifyUpdates(t, syncer, 1)
status, _ := syncer.GetPodStatus(qingcontainer.GetPodFullName(testPod))
if status.StartTime.IsZero() {
t.Errorf("SetPodStatus did not set a proper start time value")
}
}
// isPodRunning returns true if all containers of a manifest are running.
func (kl *Qinglet) isPodRunning(pod *api.Pod, runningPod container.Pod) (bool, error) {
status, err := kl.containerRuntime.GetPodStatus(pod)
if err != nil {
glog.Infof("Failed to get the status of pod %q: %v", qingcontainer.GetPodFullName(pod), err)
return false, err
}
for _, st := range status.ContainerStatuses {
if st.State.Running == nil {
glog.Infof("Container %q not running: %#v", st.Name, st.State)
return false, nil
}
}
return true, nil
}
// If the UID belongs to a mirror pod, maps it to the UID of its static pod.
// Otherwise, return the original UID. All public-facing functions should
// perform this translation for UIDs because user may provide a mirror pod UID,
// which is not recognized by internal Qinglet functions.
func (pm *basicPodManager) TranslatePodUID(uid types.UID) types.UID {
if uid == "" {
return uid
}
pm.lock.RLock()
defer pm.lock.RUnlock()
if mirrorPod, ok := pm.mirrorPodByUID[uid]; ok {
podFullName := qingcontainer.GetPodFullName(mirrorPod)
if pod, ok := pm.podByFullName[podFullName]; ok {
return pod.UID
}
}
return uid
}
func TestChangedStatusKeepsStartTime(t *testing.T) {
syncer := newTestStatusManager()
now := util.Now()
firstStatus := getRandomPodStatus()
firstStatus.StartTime = &now
syncer.SetPodStatus(testPod, firstStatus)
syncer.SetPodStatus(testPod, getRandomPodStatus())
verifyUpdates(t, syncer, 2)
finalStatus, _ := syncer.GetPodStatus(qingcontainer.GetPodFullName(testPod))
if finalStatus.StartTime.IsZero() {
t.Errorf("StartTime should not be zero")
}
if !finalStatus.StartTime.Time.Equal(now.Time) {
t.Errorf("Expected %v, but got %v", now.Time, finalStatus.StartTime.Time)
}
}
func TestNewStatusPreservesPodStartTime(t *testing.T) {
syncer := newTestStatusManager()
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
UID: "12345678",
Name: "foo",
Namespace: "new",
},
Status: api.PodStatus{},
}
now := util.Now()
startTime := util.NewTime(now.Time.Add(-1 * time.Minute))
pod.Status.StartTime = &startTime
syncer.SetPodStatus(pod, getRandomPodStatus())
status, _ := syncer.GetPodStatus(qingcontainer.GetPodFullName(pod))
if !status.StartTime.Time.Equal(startTime.Time) {
t.Errorf("Unexpected start time, expected %v, actual %v", startTime, status.StartTime)
}
}
// probeReadiness probes and sets the readiness of a container.
// If the initial delay on the readiness probe has not passed, we set readiness to false.
func (pb *prober) probeReadiness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) {
var ready probe.Result
var output string
var err error
p := container.ReadinessProbe
if p == nil {
ready = probe.Success
} else if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
ready = probe.Failure
} else {
ready, output, err = pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}
ctrName := fmt.Sprintf("%s:%s", qingcontainer.GetPodFullName(pod), container.Name)
if err != nil || ready == probe.Failure {
// Readiness failed in one way or another.
pb.readinessManager.SetReadiness(containerID, false)
ref, ok := pb.refManager.GetRef(containerID)
if !ok {
glog.Warningf("No ref for pod '%v' - '%v'", containerID, container.Name)
}
if err != nil {
glog.V(1).Infof("readiness probe for %q errored: %v", ctrName, err)
if ok {
pb.recorder.Eventf(ref, "unhealthy", "Readiness probe errored: %v", err)
}
return
} else { // ready != probe.Success
glog.V(1).Infof("Readiness probe for %q failed (%v): %s", ctrName, ready, output)
if ok {
pb.recorder.Eventf(ref, "unhealthy", "Readiness probe failed: %s", output)
}
return
}
}
if ready == probe.Success {
pb.readinessManager.SetReadiness(containerID, true)
}
glog.V(3).Infof("Readiness probe for %q succeeded", ctrName)
}
func (s *statusManager) SetPodStatus(pod *api.Pod, status api.PodStatus) {
podFullName := qingcontainer.GetPodFullName(pod)
s.podStatusesLock.Lock()
defer s.podStatusesLock.Unlock()
oldStatus, found := s.podStatuses[podFullName]
// ensure that the start time does not change across updates.
if found && oldStatus.StartTime != nil {
status.StartTime = oldStatus.StartTime
}
// if the status has no start time, we need to set an initial time
// TODO(yujuhong): Consider setting StartTime when generating the pod
// status instead, which would allow statusManager to become a simple cache
// again.
if status.StartTime.IsZero() {
if pod.Status.StartTime.IsZero() {
// the pod did not have a previously recorded value so set to now
now := util.Now()
status.StartTime = &now
} else {
// the pod had a recorded value, but the qinglet restarted so we need to rebuild cache
// based on last observed value
status.StartTime = pod.Status.StartTime
}
}
// TODO: Holding a lock during blocking operations is dangerous. Refactor so this isn't necessary.
// The intent here is to prevent concurrent updates to a pod's status from
// clobbering each other so the phase of a pod progresses monotonically.
// Currently this routine is not called for the same pod from multiple
// workers and/or the qinglet but dropping the lock before sending the
// status down the channel feels like an easy way to get a bullet in foot.
if !found || !isStatusEqual(&oldStatus, &status) {
s.podStatuses[podFullName] = status
s.podStatusChannel <- podStatusSyncRequest{pod, status}
} else {
glog.V(3).Infof("Ignoring same pod status for %q - old: %+v new: %+v", podFullName, oldStatus, status)
}
}
func (pm *basicPodManager) setPods(newPods []*api.Pod) {
podByUID := make(map[types.UID]*api.Pod)
mirrorPodByUID := make(map[types.UID]*api.Pod)
podByFullName := make(map[string]*api.Pod)
mirrorPodByFullName := make(map[string]*api.Pod)
for _, pod := range newPods {
podFullName := qingcontainer.GetPodFullName(pod)
if isMirrorPod(pod) {
mirrorPodByUID[pod.UID] = pod
mirrorPodByFullName[podFullName] = pod
} else {
podByUID[pod.UID] = pod
podByFullName[podFullName] = pod
}
}
pm.podByUID = podByUID
pm.podByFullName = podByFullName
pm.mirrorPodByUID = mirrorPodByUID
pm.mirrorPodByFullName = mirrorPodByFullName
}
// probeLiveness probes the liveness of a container.
// If the initalDelay since container creation on liveness probe has not passed the probe will return probe.Success.
func (pb *prober) probeLiveness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
var live probe.Result
var output string
var err error
p := container.LivenessProbe
if p == nil {
return probe.Success, nil
}
if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
return probe.Success, nil
} else {
live, output, err = pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}
ctrName := fmt.Sprintf("%s:%s", qingcontainer.GetPodFullName(pod), container.Name)
if err != nil || live != probe.Success {
// Liveness failed in one way or another.
ref, ok := pb.refManager.GetRef(containerID)
if !ok {
glog.Warningf("No ref for pod %q - '%v'", containerID, container.Name)
}
if err != nil {
glog.V(1).Infof("Liveness probe for %q errored: %v", ctrName, err)
if ok {
pb.recorder.Eventf(ref, "unhealthy", "Liveness probe errored: %v", err)
}
return probe.Unknown, err
} else { // live != probe.Success
glog.V(1).Infof("Liveness probe for %q failed (%v): %s", ctrName, live, output)
if ok {
pb.recorder.Eventf(ref, "unhealthy", "Liveness probe failed: %s", output)
}
return live, nil
}
}
glog.V(3).Infof("Liveness probe for %q succeeded", ctrName)
return probe.Success, nil
}
// syncBatch syncs pods statuses with the apiserver.
func (s *statusManager) syncBatch() error {
if s.qingClient == nil {
glog.V(4).Infof("QingYuan client is nil, skipping pod status updates")
return nil
}
syncRequest := <-s.podStatusChannel
pod := syncRequest.pod
podFullName := qingcontainer.GetPodFullName(pod)
status := syncRequest.status
var err error
statusPod := &api.Pod{
ObjectMeta: pod.ObjectMeta,
}
// TODO: make me easier to express from client code
statusPod, err = s.qingClient.Pods(statusPod.Namespace).Get(statusPod.Name)
if err == nil {
statusPod.Status = status
_, err = s.qingClient.Pods(pod.Namespace).UpdateStatus(statusPod)
// TODO: handle conflict as a retry, make that easier too.
if err == nil {
glog.V(3).Infof("Status for pod %q updated successfully", pod.Name)
return nil
}
}
// We failed to update status. In order to make sure we retry next time
// we delete cached value. This may result in an additional update, but
// this is ok.
// Doing this synchronously will lead to a deadlock if the podStatusChannel
// is full, and the pod worker holding the lock is waiting on this method
// to clear the channel. Even if this delete never runs subsequent container
// changes on the node should trigger updates.
go s.DeletePodStatus(podFullName)
return fmt.Errorf("error updating status for pod %q: %v", pod.Name, err)
}
func (s *Server) handlePortForward(w http.ResponseWriter, req *http.Request) {
u, err := url.ParseRequestURI(req.RequestURI)
if err != nil {
s.error(w, err)
return
}
podNamespace, podID, uid, err := parsePodCoordinates(u.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pod, ok := s.host.GetPodByName(podNamespace, podID)
if !ok {
http.Error(w, "Pod does not exist", http.StatusNotFound)
return
}
streamChan := make(chan httpstream.Stream, 1)
upgrader := spdy.NewResponseUpgrader()
conn := upgrader.UpgradeResponse(w, req, func(stream httpstream.Stream) error {
portString := stream.Headers().Get(api.PortHeader)
port, err := strconv.ParseUint(portString, 10, 16)
if err != nil {
return fmt.Errorf("Unable to parse '%s' as a port: %v", portString, err)
}
if port < 1 {
return fmt.Errorf("Port '%d' must be greater than 0", port)
}
streamChan <- stream
return nil
})
if conn == nil {
return
}
defer conn.Close()
conn.SetIdleTimeout(s.host.StreamingConnectionIdleTimeout())
var dataStreamLock sync.Mutex
dataStreamChans := make(map[string]chan httpstream.Stream)
Loop:
for {
select {
case <-conn.CloseChan():
break Loop
case stream := <-streamChan:
streamType := stream.Headers().Get(api.StreamType)
port := stream.Headers().Get(api.PortHeader)
dataStreamLock.Lock()
switch streamType {
case "error":
ch := make(chan httpstream.Stream)
dataStreamChans[port] = ch
go waitForPortForwardDataStreamAndRun(qingcontainer.GetPodFullName(pod), uid, stream, ch, s.host)
case "data":
ch, ok := dataStreamChans[port]
if ok {
ch <- stream
delete(dataStreamChans, port)
} else {
glog.Errorf("Unable to locate data stream channel for port %s", port)
}
default:
glog.Errorf("streamType header must be 'error' or 'data', got: '%s'", streamType)
stream.Reset()
}
dataStreamLock.Unlock()
}
}
}
// serveStats implements stats logic.
func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) {
// Stats requests are in the following forms:
//
// /stats/ : Root container stats
// /stats/container/ : Non-QingYuan container stats (returns a map)
// /stats/<pod name>/<container name> : Stats for QingYuan pod/container
// /stats/<namespace>/<pod name>/<uid>/<container name> : Stats for QingYuan namespace/pod/uid/container
components := strings.Split(strings.TrimPrefix(path.Clean(req.URL.Path), "/"), "/")
var stats interface{}
var err error
var query StatsRequest
query.NumStats = 60
err = json.NewDecoder(req.Body).Decode(&query)
if err != nil && err != io.EOF {
s.error(w, err)
return
}
cadvisorRequest := cadvisorApi.ContainerInfoRequest{
NumStats: query.NumStats,
Start: query.Start,
End: query.End,
}
switch len(components) {
case 1:
// Root container stats.
var statsMap map[string]*cadvisorApi.ContainerInfo
statsMap, err = s.host.GetRawContainerInfo("/", &cadvisorRequest, false)
stats = statsMap["/"]
case 2:
// Non-QingYuan container stats.
if components[1] != "container" {
http.Error(w, fmt.Sprintf("unknown stats request type %q", components[1]), http.StatusNotFound)
return
}
containerName := path.Join("/", query.ContainerName)
stats, err = s.host.GetRawContainerInfo(containerName, &cadvisorRequest, query.Subcontainers)
case 3:
// Backward compatibility without uid information, does not support namespace
pod, ok := s.host.GetPodByName(api.NamespaceDefault, components[1])
if !ok {
http.Error(w, "Pod does not exist", http.StatusNotFound)
return
}
stats, err = s.host.GetContainerInfo(qingcontainer.GetPodFullName(pod), "", components[2], &cadvisorRequest)
case 5:
pod, ok := s.host.GetPodByName(components[1], components[2])
if !ok {
http.Error(w, "Pod does not exist", http.StatusNotFound)
return
}
stats, err = s.host.GetContainerInfo(qingcontainer.GetPodFullName(pod), types.UID(components[3]), components[4], &cadvisorRequest)
default:
http.Error(w, fmt.Sprintf("Unknown resource: %v", components), http.StatusNotFound)
return
}
switch err {
case nil:
break
case ErrContainerNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
return
default:
s.error(w, err)
return
}
if stats == nil {
fmt.Fprint(w, "{}")
return
}
data, err := json.Marshal(stats)
if err != nil {
s.error(w, err)
return
}
w.Header().Add("Content-type", "application/json")
w.Write(data)
return
}
// handleExec handles requests to run a command inside a container.
func (s *Server) handleExec(w http.ResponseWriter, req *http.Request) {
u, err := url.ParseRequestURI(req.RequestURI)
if err != nil {
s.error(w, err)
return
}
podNamespace, podID, uid, container, err := parseContainerCoordinates(u.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pod, ok := s.host.GetPodByName(podNamespace, podID)
if !ok {
http.Error(w, "Pod does not exist", http.StatusNotFound)
return
}
req.ParseForm()
// start at 1 for error stream
expectedStreams := 1
if req.FormValue(api.ExecStdinParam) == "1" {
expectedStreams++
}
if req.FormValue(api.ExecStdoutParam) == "1" {
expectedStreams++
}
tty := req.FormValue(api.ExecTTYParam) == "1"
if !tty && req.FormValue(api.ExecStderrParam) == "1" {
expectedStreams++
}
if expectedStreams == 1 {
http.Error(w, "You must specify at least 1 of stdin, stdout, stderr", http.StatusBadRequest)
return
}
streamCh := make(chan httpstream.Stream)
upgrader := spdy.NewResponseUpgrader()
conn := upgrader.UpgradeResponse(w, req, func(stream httpstream.Stream) error {
streamCh <- stream
return nil
})
// from this point on, we can no longer call methods on w
if conn == nil {
// The upgrader is responsible for notifying the client of any errors that
// occurred during upgrading. All we can do is return here at this point
// if we weren't successful in upgrading.
return
}
defer conn.Close()
conn.SetIdleTimeout(s.host.StreamingConnectionIdleTimeout())
// TODO make it configurable?
expired := time.NewTimer(streamCreationTimeout)
var errorStream, stdinStream, stdoutStream, stderrStream httpstream.Stream
receivedStreams := 0
WaitForStreams:
for {
select {
case stream := <-streamCh:
streamType := stream.Headers().Get(api.StreamType)
switch streamType {
case api.StreamTypeError:
errorStream = stream
defer errorStream.Reset()
receivedStreams++
case api.StreamTypeStdin:
stdinStream = stream
receivedStreams++
case api.StreamTypeStdout:
stdoutStream = stream
receivedStreams++
case api.StreamTypeStderr:
stderrStream = stream
receivedStreams++
default:
glog.Errorf("Unexpected stream type: '%s'", streamType)
}
if receivedStreams == expectedStreams {
break WaitForStreams
}
case <-expired.C:
// TODO find a way to return the error to the user. Maybe use a separate
// stream to report errors?
glog.Error("Timed out waiting for client to create streams")
return
}
}
if stdinStream != nil {
// close our half of the input stream, since we won't be writing to it
stdinStream.Close()
}
err = s.host.ExecInContainer(qingcontainer.GetPodFullName(pod), uid, container, u.Query()[api.ExecCommandParamm], stdinStream, stdoutStream, stderrStream, tty)
if err != nil {
msg := fmt.Sprintf("Error executing command in container: %v", err)
glog.Error(msg)
errorStream.Write([]byte(msg))
}
}
// makePodManifest transforms a qinglet pod spec to the rkt pod manifest.
// TODO(yifan): Use the RunContainerOptions generated by GenerateRunContainerOptions().
func (r *runtime) makePodManifest(pod *api.Pod) (*appcschema.PodManifest, error) {
var globalPortMappings []qingcontainer.PortMapping
manifest := appcschema.BlankPodManifest()
for _, c := range pod.Spec.Containers {
imgManifest, err := r.getImageManifest(c.Image)
if err != nil {
return nil, err
}
if imgManifest.App == nil {
return nil, fmt.Errorf("no app section in image manifest for image: %q", c.Image)
}
img, err := r.getImageByName(c.Image)
if err != nil {
return nil, err
}
hash, err := appctypes.NewHash(img.id)
if err != nil {
return nil, err
}
opts, err := r.generator.GenerateRunContainerOptions(pod, &c)
if err != nil {
return nil, err
}
globalPortMappings = append(globalPortMappings, opts.PortMappings...)
if err := setApp(imgManifest.App, &c, opts); err != nil {
return nil, err
}
manifest.Apps = append(manifest.Apps, appcschema.RuntimeApp{
// TODO(yifan): We should allow app name to be different with
// image name. See https://github.com/coreos/rkt/pull/640.
Name: imgManifest.Name,
Image: appcschema.RuntimeImage{ID: *hash},
App: imgManifest.App,
})
}
volumeMap, ok := r.volumeGetter.GetVolumes(pod.UID)
if !ok {
return nil, fmt.Errorf("cannot get the volumes for pod %q", qingcontainer.GetPodFullName(pod))
}
// Set global volumes.
for name, volume := range volumeMap {
volName, err := appctypes.NewACName(name)
if err != nil {
return nil, fmt.Errorf("cannot use the volume's name %q as ACName: %v", name, err)
}
manifest.Volumes = append(manifest.Volumes, appctypes.Volume{
Name: *volName,
Kind: "host",
Source: volume.GetPath(),
})
}
// Set global ports.
for _, port := range globalPortMappings {
name, err := appctypes.SanitizeACName(port.Name)
if err != nil {
return nil, fmt.Errorf("cannot use the port's name %q as ACName: %v", port.Name, err)
}
portName := appctypes.MustACName(name)
manifest.Ports = append(manifest.Ports, appctypes.ExposedPort{
Name: *portName,
HostPort: uint(port.HostPort),
})
}
// TODO(yifan): Set pod-level isolators once it's supported in qingyuan.
return manifest, nil
}
// handleContainerLogs handles containerLogs request against the Qinglet
func (s *Server) handleContainerLogs(w http.ResponseWriter, req *http.Request) {
defer req.Body.Close()
u, err := url.ParseRequestURI(req.RequestURI)
if err != nil {
s.error(w, err)
return
}
parts := strings.Split(u.Path, "/")
// req URI: /containerLogs/<podNamespace>/<podID>/<containerName>
var podNamespace, podID, containerName string
if len(parts) == 5 {
podNamespace = parts[2]
podID = parts[3]
containerName = parts[4]
} else {
http.Error(w, "Unexpected path for command running", http.StatusBadRequest)
return
}
if len(podID) == 0 {
http.Error(w, `{"message": "Missing podID."}`, http.StatusBadRequest)
return
}
if len(containerName) == 0 {
http.Error(w, `{"message": "Missing container name."}`, http.StatusBadRequest)
return
}
if len(podNamespace) == 0 {
http.Error(w, `{"message": "Missing podNamespace."}`, http.StatusBadRequest)
return
}
uriValues := u.Query()
follow, _ := strconv.ParseBool(uriValues.Get("follow"))
previous, _ := strconv.ParseBool(uriValues.Get("previous"))
tail := uriValues.Get("tail")
pod, ok := s.host.GetPodByName(podNamespace, podID)
if !ok {
http.Error(w, fmt.Sprintf("Pod %q does not exist", podID), http.StatusNotFound)
return
}
// Check if containerName is valid.
containerExists := false
for _, container := range pod.Spec.Containers {
if container.Name == containerName {
containerExists = true
}
}
if !containerExists {
http.Error(w, fmt.Sprintf("Container %q not found in Pod %q", containerName, podID), http.StatusNotFound)
return
}
if _, ok := w.(http.Flusher); !ok {
s.error(w, fmt.Errorf("unable to convert %v into http.Flusher", w))
return
}
fw := flushwriter.Wrap(w)
w.Header().Set("Transfer-Encoding", "chunked")
w.WriteHeader(http.StatusOK)
err = s.host.GetQingletContainerLogs(qingcontainer.GetPodFullName(pod), containerName, tail, follow, previous, fw, fw)
if err != nil {
s.error(w, err)
return
}
}
// SyncPod syncs the running pod to match the specified desired pod.
func (r *runtime) SyncPod(pod *api.Pod, runningPod qingcontainer.Pod, podStatus api.PodStatus, pullSecrets []api.Secret) error {
podFullName := qingcontainer.GetPodFullName(pod)
if len(runningPod.Containers) == 0 {
glog.V(4).Infof("Pod %q is not running, will start it", podFullName)
return r.RunPod(pod)
}
// Add references to all containers.
unidentifiedContainers := make(map[types.UID]*qingcontainer.Container)
for _, c := range runningPod.Containers {
unidentifiedContainers[c.ID] = c
}
restartPod := false
for _, container := range pod.Spec.Containers {
expectedHash := qingcontainer.HashContainer(&container)
c := runningPod.FindContainerByName(container.Name)
if c == nil {
if qingcontainer.ShouldContainerBeRestarted(&container, pod, &podStatus, r.readinessManager) {
glog.V(3).Infof("Container %+v is dead, but RestartPolicy says that we should restart it.", container)
// TODO(yifan): Containers in one pod are fate-sharing at this moment, see:
// https://github.com/appc/spec/issues/276.
restartPod = true
break
}
continue
}
// TODO(yifan): Take care of host network change.
containerChanged := c.Hash != 0 && c.Hash != expectedHash
if containerChanged {
glog.Infof("Pod %q container %q hash changed (%d vs %d), it will be killed and re-created.", podFullName, container.Name, c.Hash, expectedHash)
restartPod = true
break
}
result, err := r.prober.Probe(pod, podStatus, container, string(c.ID), c.Created)
// TODO(vmarmol): examine this logic.
if err == nil && result != probe.Success {
glog.Infof("Pod %q container %q is unhealthy (probe result: %v), it will be killed and re-created.", podFullName, container.Name, result)
restartPod = true
break
}
if err != nil {
glog.V(2).Infof("Probe container %q failed: %v", container.Name, err)
}
delete(unidentifiedContainers, c.ID)
}
// If there is any unidentified containers, restart the pod.
if len(unidentifiedContainers) > 0 {
restartPod = true
}
if restartPod {
// TODO(yifan): Handle network plugin.
if err := r.KillPod(runningPod); err != nil {
return err
}
if err := r.RunPod(pod); err != nil {
return err
}
}
return nil
}
// async continuation of LaunchTask
func (k *QingYuanExecutor) launchTask(driver bindings.ExecutorDriver, taskId string, pod *api.Pod) {
//HACK(jdef): cloned binding construction from k8s plugin/pkg/scheduler/scheduler.go
binding := &api.Binding{
ObjectMeta: api.ObjectMeta{
Namespace: pod.Namespace,
Name: pod.Name,
Annotations: make(map[string]string),
},
Target: api.ObjectReference{
Kind: "Node",
Name: pod.Annotations[meta.BindingHostKey],
},
}
// forward the annotations that the scheduler wants to apply
for k, v := range pod.Annotations {
binding.Annotations[k] = v
}
deleteTask := func() {
k.lock.Lock()
defer k.lock.Unlock()
delete(k.tasks, taskId)
k.resetSuicideWatch(driver)
}
log.Infof("Binding '%v/%v' to '%v' with annotations %+v...", pod.Namespace, pod.Name, binding.Target.Name, binding.Annotations)
ctx := api.WithNamespace(api.NewContext(), binding.Namespace)
// TODO(k8s): use Pods interface for binding once clusters are upgraded
// return b.Pods(binding.Namespace).Bind(binding)
err := k.client.Post().Namespace(api.NamespaceValue(ctx)).Resource("bindings").Body(binding).Do().Error()
if err != nil {
deleteTask()
k.sendStatus(driver, newStatus(mutil.NewTaskID(taskId), mesos.TaskState_TASK_FAILED,
messages.CreateBindingFailure))
return
}
podFullName := container.GetPodFullName(pod)
// allow a recently failed-over scheduler the chance to recover the task/pod binding:
// it may have failed and recovered before the apiserver is able to report the updated
// binding information. replays of this status event will signal to the scheduler that
// the apiserver should be up-to-date.
data, err := json.Marshal(api.PodStatusResult{
ObjectMeta: api.ObjectMeta{
Name: podFullName,
SelfLink: "/podstatusresult",
},
})
if err != nil {
deleteTask()
log.Errorf("failed to marshal pod status result: %v", err)
k.sendStatus(driver, newStatus(mutil.NewTaskID(taskId), mesos.TaskState_TASK_FAILED,
err.Error()))
return
}
k.lock.Lock()
defer k.lock.Unlock()
// Add the task.
task, found := k.tasks[taskId]
if !found {
log.V(1).Infof("task %v not found, probably killed: aborting launch, reporting lost", taskId)
k.reportLostTask(driver, taskId, messages.LaunchTaskFailed)
return
}
//TODO(jdef) check for duplicate pod name, if found send TASK_ERROR
// from here on, we need to delete containers associated with the task
// upon it going into a terminal state
task.podName = podFullName
k.pods[podFullName] = pod
// send the latest snapshot of the set of pods to the qinglet via the pod update channel.
// this results in the qinglet spinning up the new pod.
update := qinglet.PodUpdate{Op: qinglet.SET}
for _, p := range k.pods {
update.Pods = append(update.Pods, p)
}
k.updateChan <- update
statusUpdate := &mesos.TaskStatus{
TaskId: mutil.NewTaskID(taskId),
State: mesos.TaskState_TASK_STARTING.Enum(),
Message: proto.String(messages.CreateBindingSuccess),
Data: data,
}
k.sendStatus(driver, statusUpdate)
// Delay reporting 'task running' until container is up.
psf := podStatusFunc(func() (*api.PodStatus, error) {
status, err := k.podStatusFunc(k.kl, pod)
if err != nil {
return nil, err
}
status.Phase = qinglet.GetPhase(&pod.Spec, status.ContainerStatuses)
hostIP, err := k.kl.GetHostIP()
if err != nil {
log.Errorf("Cannot get host IP: %v", err)
} else {
status.HostIP = hostIP.String()
}
return status, nil
})
go k._launchTask(driver, taskId, podFullName, psf)
}