func (cli *HyperClient) HyperCmdAttach(args ...string) error {
var parser = gflag.NewParser(nil, gflag.Default)
parser.Usage = "attach CONTAINER\n\nAttach to the tty of a specified container in a pod"
args, err := parser.ParseArgs(args)
if err != nil {
if !strings.Contains(err.Error(), "Usage") {
return err
} else {
return nil
}
}
if len(args) == 0 {
return fmt.Errorf("Can not accept the 'attach' command without Container ID!")
}
var (
podId = args[0]
containerId = args[0]
tty bool
)
if strings.Contains(podId, "pod-") {
pod, err := cli.client.GetPodInfo(podId)
if err != nil {
return err
}
if len(pod.Spec.Containers) == 0 {
return fmt.Errorf("failed to get container from %s", podId)
}
c := pod.Spec.Containers[0]
containerId = c.ContainerID
tty = c.Tty
} else {
c, err := cli.client.GetContainerInfo(containerId)
if err != nil {
return err
}
podId = c.PodID
containerId = c.Container.ContainerID
tty = c.Container.Tty
}
if tty {
oldState, err := term.SetRawTerminal(cli.inFd)
if err != nil {
return err
}
defer term.RestoreTerminal(cli.inFd, oldState)
cli.monitorTtySize(containerId, "")
}
if err := cli.client.Attach(containerId, tty, cli.in, cli.out, cli.err); err != nil {
return err
}
return cli.client.GetExitCode(containerId, "")
}
func startContainer(context *cli.Context, container, address string, config *specs.Spec) int {
pid := os.Getpid()
r := &types.CreateContainerRequest{
Id: container,
BundlePath: context.String("bundle"),
Stdin: fmt.Sprintf("/proc/%d/fd/0", pid),
Stdout: fmt.Sprintf("/proc/%d/fd/1", pid),
Stderr: fmt.Sprintf("/proc/%d/fd/2", pid),
}
c := getClient(address)
timestamp := uint64(time.Now().Unix())
if _, err := c.CreateContainer(netcontext.Background(), r); err != nil {
fmt.Printf("error %v\n", err)
return -1
}
if config.Process.Terminal {
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
fmt.Printf("error %v\n", err)
return -1
}
defer term.RestoreTerminal(os.Stdin.Fd(), s)
monitorTtySize(c, container, "init")
}
if context.String("pid-file") != "" {
stateData, err := ioutil.ReadFile(filepath.Join(context.GlobalString("root"), container, stateJson))
if err != nil {
fmt.Printf("read state.json error %v\n", err)
return -1
}
var s specs.State
if err := json.Unmarshal(stateData, &s); err != nil {
fmt.Printf("unmarshal state.json error %v\n", err)
return -1
}
err = createPidFile(context.String("pid-file"), s.Pid)
if err != nil {
fmt.Printf("create pid-file error %v\n", err)
}
}
return waitForExit(c, timestamp, container, "init")
}
func runProcess(root, container string, config *specs.Process) int {
pid := os.Getpid()
process := fmt.Sprintf("p-%x", pid+0xabcdef) // uniq name
p := &types.AddProcessRequest{
Id: container,
Pid: process,
Args: config.Args,
Cwd: config.Cwd,
Terminal: config.Terminal,
Env: config.Env,
User: &types.User{
Uid: config.User.UID,
Gid: config.User.GID,
},
Stdin: fmt.Sprintf("/proc/%d/fd/0", pid),
Stdout: fmt.Sprintf("/proc/%d/fd/1", pid),
Stderr: fmt.Sprintf("/proc/%d/fd/2", pid),
}
c := getClient(filepath.Join(root, container, "namespace/namespaced.sock"))
evChan := containerEvents(c, container)
if _, err := c.AddProcess(netcontext.Background(), p); err != nil {
fmt.Printf("error %v\n", err)
return -1
}
if config.Terminal {
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
fmt.Printf("error %v\n", err)
return -1
}
defer term.RestoreTerminal(os.Stdin.Fd(), s)
monitorTtySize(c, container, process)
}
for e := range evChan {
if e.Type == "exit" && e.Pid == process {
return int(e.Status)
}
}
return -1
}
func runProcess(root, container string, config *specs.Process) int {
pid := os.Getpid()
process := fmt.Sprintf("p-%x", pid+0xabcdef) // uniq name
p := &types.AddProcessRequest{
Id: container,
Pid: process,
Args: config.Args,
Cwd: config.Cwd,
Terminal: config.Terminal,
Env: config.Env,
User: &types.User{
Uid: config.User.UID,
Gid: config.User.GID,
},
Stdin: fmt.Sprintf("/proc/%d/fd/0", pid),
Stdout: fmt.Sprintf("/proc/%d/fd/1", pid),
Stderr: fmt.Sprintf("/proc/%d/fd/2", pid),
}
c := getClient(filepath.Join(root, container, "namespace/namespaced.sock"))
timestamp := uint64(time.Now().Unix())
if _, err := c.AddProcess(netcontext.Background(), p); err != nil {
fmt.Printf("error %v\n", err)
return -1
}
if config.Terminal {
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
fmt.Printf("error %v\n", err)
return -1
}
defer term.RestoreTerminal(os.Stdin.Fd(), s)
monitorTtySize(c, container, process)
}
return waitForExit(c, timestamp, container, process)
}
func startContainer(context *cli.Context, container, address string, config *specs.Spec) int {
pid := os.Getpid()
r := &types.CreateContainerRequest{
Id: container,
BundlePath: context.String("bundle"),
Stdin: fmt.Sprintf("/proc/%d/fd/0", pid),
Stdout: fmt.Sprintf("/proc/%d/fd/1", pid),
Stderr: fmt.Sprintf("/proc/%d/fd/2", pid),
}
c := getClient(address)
evChan := containerEvents(c, container)
if _, err := c.CreateContainer(netcontext.Background(), r); err != nil {
fmt.Printf("error %v\n", err)
return -1
}
if config.Process.Terminal {
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
fmt.Printf("error %v\n", err)
return -1
}
defer term.RestoreTerminal(os.Stdin.Fd(), s)
monitorTtySize(c, container, "init")
}
var started bool
for e := range evChan {
if e.Type == "exit" && e.Pid == "init" {
fmt.Printf("get exit event before start event\n")
return int(e.Status)
}
if e.Type == "start-container" {
started = true
break
}
}
if !started {
fmt.Printf("failed to get the start event\n")
return -1
}
if context.String("pid-file") != "" {
stateData, err := ioutil.ReadFile(filepath.Join(context.GlobalString("root"), container, stateJson))
if err != nil {
fmt.Printf("read state.json error %v\n", err)
return -1
}
var s specs.State
if err := json.Unmarshal(stateData, &s); err != nil {
fmt.Printf("unmarshal state.json error %v\n", err)
return -1
}
err = createPidFile(context.String("pid-file"), s.Pid)
if err != nil {
fmt.Printf("create pid-file error %v\n", err)
}
}
for e := range evChan {
if e.Type == "exit" && e.Pid == "init" {
return int(e.Status)
}
}
return -1
}
func (cli *HyperClient) hijack(method, path string, setRawTerminal bool, in io.ReadCloser, stdout, stderr io.Writer, started chan io.Closer, data interface{}, hostname string) error {
defer func() {
if started != nil {
close(started)
}
}()
params, err := cli.encodeData(data)
if err != nil {
return err
}
req, err := http.NewRequest(method, fmt.Sprintf("/v%s%s", utils.APIVERSION, path), params)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Hyper-Client/"+utils.VERSION)
req.Header.Set("Content-Type", "text/plain")
req.Header.Set("Connection", "Upgrade")
req.Header.Set("Upgrade", "tcp")
req.Host = cli.addr
dial, err := cli.dial()
if err != nil {
return err
}
// When we set up a TCP connection for hijack, there could be long periods
// of inactivity (a long running command with no output) that in certain
// network setups may cause ECONNTIMEOUT, leaving the client in an unknown
// state. Setting TCP KeepAlive on the socket connection will prohibit
// ECONNTIMEOUT unless the socket connection truly is broken
if tcpConn, ok := dial.(*net.TCPConn); ok {
tcpConn.SetKeepAlive(true)
tcpConn.SetKeepAlivePeriod(30 * time.Second)
}
if err != nil {
if strings.Contains(err.Error(), "connection refused") {
return fmt.Errorf("Cannot connect to the Hyper daemon. Is 'hyperd' running on this host?")
}
return err
}
clientconn := httputil.NewClientConn(dial, nil)
defer clientconn.Close()
// Server hijacks the connection, error 'connection closed' expected
_, err = clientconn.Do(req)
if err != nil {
fmt.Printf("Client DO: %s\n", err.Error())
}
rwc, br := clientconn.Hijack()
defer rwc.Close()
if started != nil {
started <- rwc
}
var (
receiveStdout chan error
oldState *term.State
)
if in != nil && setRawTerminal {
// fmt.Printf("In the Raw Terminal!!!\n")
oldState, err = term.SetRawTerminal(cli.inFd)
if err != nil {
return err
}
defer term.RestoreTerminal(cli.inFd, oldState)
}
if stdout != nil || stderr != nil {
receiveStdout = promise.Go(func() (err error) {
defer func() {
if in != nil {
if setRawTerminal {
term.RestoreTerminal(cli.inFd, oldState)
}
}
}()
_, err = io.Copy(stdout, br)
// fmt.Printf("[hijack] End of stdout\n")
return err
})
}
go func() {
if in != nil {
io.Copy(rwc, in)
// fmt.Printf("[hijack] End of stdin\n")
}
if conn, ok := rwc.(interface {
CloseWrite() error
}); ok {
if err := conn.CloseWrite(); err != nil {
fmt.Printf("Couldn't send EOF: %s", err.Error())
}
}
}()
if stdout != nil || stderr != nil {
if err := <-receiveStdout; err != nil {
fmt.Printf("Error receiveStdout: %s\n", err.Error())
return err
}
}
return nil
}
func main() {
hypervisor.InterfaceCount = 0
var containerInfoList []*hypervisor.ContainerInfo
var roots []string
var containerId string
var err error
ocffile := flag.String("config", "", "ocf configure file")
kernel := flag.String("kernel", "", "hyper kernel")
initrd := flag.String("initrd", "", "hyper initrd")
vbox := flag.String("vbox", "", "vbox boot iso")
driver := flag.String("driver", "", "hypervisor driver")
flag.Parse()
if *ocffile == "" {
*ocffile = "config.json"
}
if _, err = os.Stat(*ocffile); os.IsNotExist(err) {
fmt.Printf("Please specify ocffile or put config.json under current working directory\n")
return
}
if *vbox == "" {
*vbox = "./vbox.iso"
}
if _, err = os.Stat(*vbox); err == nil {
*vbox, err = filepath.Abs(*vbox)
if err != nil {
fmt.Printf("Cannot get abs path for vbox: %s\n", err.Error())
return
}
}
if *kernel == "" {
*kernel = "./kernel"
}
if _, err = os.Stat(*kernel); err == nil {
*kernel, err = filepath.Abs(*kernel)
if err != nil {
fmt.Printf("Cannot get abs path for kernel: %s\n", err.Error())
return
}
}
if *initrd == "" {
*initrd = "./initrd.img"
}
if _, err = os.Stat(*initrd); err == nil {
*initrd, err = filepath.Abs(*initrd)
if err != nil {
fmt.Printf("Cannot get abs path for initrd: %s\n", err.Error())
return
}
}
if *driver == "" {
*driver = "kvm"
fmt.Printf("Use default hypervisor KVM\n")
}
if hypervisor.HDriver, err = driverloader.Probe(*driver); err != nil {
fmt.Printf("%s\n", err.Error())
return
}
podId := fmt.Sprintf("pod-%s", pod.RandStr(10, "alpha"))
vmId := fmt.Sprintf("vm-%s", pod.RandStr(10, "alpha"))
ocfData, err := ioutil.ReadFile(*ocffile)
if err != nil {
fmt.Printf("%s\n", err.Error())
return
}
userPod, err := pod.OCFConvert2Pod(ocfData)
if err != nil {
fmt.Printf("%s\n", err.Error())
return
}
mypod := hypervisor.NewPod(podId, userPod)
var (
cpu = 1
mem = 128
)
if userPod.Resource.Vcpu > 0 {
cpu = userPod.Resource.Vcpu
}
if userPod.Resource.Memory > 0 {
mem = userPod.Resource.Memory
}
b := &hypervisor.BootConfig{
Kernel: *kernel,
Initrd: *initrd,
Bios: "",
Cbfs: "",
Vbox: *vbox,
CPU: cpu,
Memory: mem,
}
vm := hypervisor.NewVm(vmId, cpu, mem, false, types.VM_KEEP_NONE)
err = vm.Launch(b)
if err != nil {
fmt.Printf("%s\n", err.Error())
return
}
sharedDir := path.Join(hypervisor.BaseDir, vm.Id, hypervisor.ShareDirTag)
for _, c := range userPod.Containers {
var root string
var err error
containerId = GenerateRandomID()
rootDir := path.Join(sharedDir, containerId)
os.MkdirAll(rootDir, 0755)
rootDir = path.Join(rootDir, "rootfs")
if !filepath.IsAbs(c.Image) {
root, err = filepath.Abs(c.Image)
if err != nil {
fmt.Printf("%s\n", err.Error())
return
}
} else {
root = c.Image
}
err = mount(root, rootDir)
if err != nil {
fmt.Printf("mount %s to %s failed: %s\n", root, rootDir, err.Error())
return
}
roots = append(roots, rootDir)
containerInfo := &hypervisor.ContainerInfo{
Id: containerId,
Rootfs: "rootfs",
Image: containerId,
Fstype: "dir",
}
containerInfoList = append(containerInfoList, containerInfo)
mypod.AddContainer(containerId, podId, "", []string{}, types.S_POD_CREATED)
}
qemuResponse := vm.StartPod(mypod, userPod, containerInfoList, nil)
if qemuResponse.Data == nil {
fmt.Printf("StartPod fail: QEMU response data is nil\n")
return
}
fmt.Printf("result: code %d %s\n", qemuResponse.Code, qemuResponse.Cause)
inFd, _ := term.GetFdInfo(os.Stdin)
outFd, isTerminalOut := term.GetFdInfo(os.Stdout)
oldState, err := term.SetRawTerminal(inFd)
if err != nil {
return
}
height, width := getTtySize(outFd, isTerminalOut)
winSize := &hypervisor.WindowSize{
Row: uint16(height),
Column: uint16(width),
}
tag := pod.RandStr(8, "alphanum")
monitorTtySize(vm, tag, outFd, isTerminalOut)
vm.Attach(os.Stdin, os.Stdout, tag, containerId, winSize)
qemuResponse = vm.StopPod(mypod, "yes")
term.RestoreTerminal(inFd, oldState)
for _, root := range roots {
umount(root)
}
if qemuResponse.Data == nil {
fmt.Printf("StopPod fail: QEMU response data is nil\n")
return
}
fmt.Printf("result: code %d %s\n", qemuResponse.Code, qemuResponse.Cause)
}
// hyperctl run [OPTIONS] image [COMMAND] [ARGS...]
func (cli *HyperClient) HyperCmdRun(args ...string) (err error) {
var opts struct {
PodFile string `short:"p" long:"podfile" value-name:"\"\"" description:"Create and Run a pod based on the pod file"`
K8s string `short:"k" long:"kubernetes" value-name:"\"\"" description:"Create and Run a pod based on the kubernetes pod file"`
Yaml bool `short:"y" long:"yaml" default:"false" default-mask:"-" description:"Create a pod based on Yaml file"`
Name string `long:"name" value-name:"\"\"" description:"Assign a name to the container"`
Attach bool `short:"a" long:"attach" default:"false" default-mask:"-" description:"(from podfile) Attach the stdin, stdout and stderr to the container"`
Detach bool `short:"d" long:"detach" default:"false" default-mask:"-" description:"(from cmdline) Not Attach the stdin, stdout and stderr to the container"`
Workdir string `long:"workdir" value-name:"\"\"" default-mask:"-" description:"Working directory inside the container"`
Tty bool `short:"t" long:"tty" default:"false" default-mask:"-" description:"the run command in tty, such as bash shell"`
Cpu int `long:"cpu" default:"1" value-name:"1" default-mask:"-" description:"CPU number for the VM"`
Memory int `long:"memory" default:"128" value-name:"128" default-mask:"-" description:"Memory size (MB) for the VM"`
Env []string `long:"env" value-name:"[]" default-mask:"-" description:"Set environment variables"`
EntryPoint string `long:"entrypoint" value-name:"\"\"" default-mask:"-" description:"Overwrite the default ENTRYPOINT of the image"`
RestartPolicy string `long:"restart" default:"never" value-name:"\"\"" default-mask:"-" description:"Restart policy to apply when a container exits (never, onFailure, always)"`
LogDriver string `long:"log-driver" value-name:"\"\"" description:"Logging driver for Pod"`
LogOpts []string `long:"log-opt" description:"Log driver options"`
Remove bool `long:"rm" default:"false" default-mask:"-" description:"Automatically remove the pod when it exits"`
Portmap []string `long:"publish" value-name:"[]" default-mask:"-" description:"Publish a container's port to the host, format: --publish [tcp/udp:]hostPort:containerPort"`
Labels []string `long:"label" value-name:"[]" default-mask:"-" description:"Add labels for Pod, format: --label key=value"`
Volumes []string `short:"v" long:"volume" value-name:"[]" default-mask:"-" description:"Mount host file/directory as a data file/volume, format: -v|--volume=[[hostDir:]containerDir[:options]]"`
}
var (
podId string
vmId string
podJson string
attach bool = false
)
var parser = gflag.NewParser(&opts, gflag.Default|gflag.IgnoreUnknown)
parser.Usage = "run [OPTIONS] IMAGE [COMMAND] [ARG...]\n\nCreate a pod, and launch a new VM to run the pod"
args, err = parser.ParseArgs(args)
if err != nil {
if !strings.Contains(err.Error(), "Usage") {
return err
} else {
return nil
}
}
if opts.PodFile != "" {
attach = opts.Attach
podJson, err = cli.JsonFromFile(opts.PodFile, opts.Yaml, false)
} else if opts.K8s != "" {
attach = opts.Attach
podJson, err = cli.JsonFromFile(opts.K8s, opts.Yaml, true)
} else {
if len(args) == 0 {
return fmt.Errorf("%s: \"run\" requires a minimum of 1 argument, please provide the image.", os.Args[0])
}
attach = !opts.Detach
podJson, err = cli.JsonFromCmdline(args, opts.Env, opts.Portmap, opts.LogDriver, opts.LogOpts,
opts.Name, opts.Workdir, opts.RestartPolicy, opts.Cpu, opts.Memory, opts.Tty, opts.Labels, opts.EntryPoint, opts.Volumes)
}
if err != nil {
return err
}
t1 := time.Now()
var (
spec pod.UserPod
code int
async = true
tty = false
)
json.Unmarshal([]byte(podJson), &spec)
vmId, err = cli.client.CreateVm(spec.Resource.Vcpu, spec.Resource.Memory, async)
if err != nil {
return
}
defer func() {
if err != nil {
cli.client.RmVm(vmId)
}
}()
podId, code, err = cli.client.CreatePod(&spec)
if err != nil {
if code == http.StatusNotFound {
err = cli.PullImages(&spec)
if err != nil {
return
}
podId, code, err = cli.client.CreatePod(&spec)
}
if err != nil {
return
}
}
if !attach {
fmt.Printf("POD id is %s\n", podId)
}
if opts.Remove {
defer func() {
rmerr := cli.client.RmPod(podId)
if rmerr != nil {
fmt.Fprintf(cli.out, "failed to rm pod, %v\n", rmerr)
}
}()
}
if attach {
if opts.PodFile == "" && opts.K8s == "" {
tty = opts.Tty
} else {
tty = spec.Tty || spec.Containers[0].Tty
}
if tty {
p, err := cli.client.GetPodInfo(podId)
if err == nil {
cli.monitorTtySize(p.Spec.Containers[0].ContainerID, "")
}
oldState, err := term.SetRawTerminal(cli.inFd)
if err != nil {
return err
}
defer term.RestoreTerminal(cli.inFd, oldState)
}
}
_, err = cli.client.StartPod(podId, vmId, attach, tty, cli.in, cli.out, cli.err)
if err != nil {
return
}
if !attach {
t2 := time.Now()
fmt.Printf("Time to run a POD is %d ms\n", (t2.UnixNano()-t1.UnixNano())/1000000)
}
return nil
}
func (cli *HyperClient) HyperCmdExec(args ...string) error {
var opts struct {
Attach bool `short:"a" long:"attach" default:"true" description:"attach current terminal to the stdio of command"`
Tty bool `short:"t" long:"tty" description:"Allocate a pseudo-TTY"`
}
var parser = gflag.NewParser(&opts, gflag.Default|gflag.IgnoreUnknown)
parser.Usage = "exec [OPTIONS] POD|CONTAINER COMMAND [ARGS...]\n\nRun a command in a container of a running pod"
args, err := parser.ParseArgs(args)
if err != nil {
if !strings.Contains(err.Error(), "Usage") {
return err
} else {
return nil
}
}
if len(args) == 0 {
return fmt.Errorf("Can not accept the 'exec' command without Container ID!")
}
if len(args) == 1 {
return fmt.Errorf("Can not accept the 'exec' command without command!")
}
var (
podName = args[0]
containerId string
)
if strings.Contains(podName, "pod-") {
containerId, err = cli.client.GetContainerByPod(podName)
if err != nil {
return err
}
} else {
containerId = args[0]
}
command, err := json.Marshal(args[1:])
if err != nil {
return err
}
execId, err := cli.client.CreateExec(containerId, command, opts.Tty)
if err != nil {
return err
}
if opts.Tty {
if err := cli.monitorTtySize(containerId, execId); err != nil {
fmt.Printf("Monitor tty size fail for %s!\n", podName)
}
oldState, err := term.SetRawTerminal(cli.inFd)
if err != nil {
return err
}
defer term.RestoreTerminal(cli.inFd, oldState)
}
err = cli.client.StartExec(containerId, execId, opts.Tty, cli.in, cli.out, cli.err)
if err != nil {
return err
}
return cli.client.GetExitCode(containerId, execId)
}