func dropContainer(c *model.Container) {
if g.Config().Debug {
log.Println("drop container:", c)
}
addr := fmt.Sprintf("http://%s:%d", c.Ip, g.Config().DockerPort)
client, err := docker.NewClient(addr)
if err != nil {
log.Println("docker.NewClient fail:", err)
return
}
err = client.RemoveContainer(docker.RemoveContainerOptions{ID: c.Id, Force: true})
if err != nil {
log.Println("docker.RemoveContainer fail:", err)
return
}
// remember to delete real state map item
sa, exists := g.RealState.GetSafeApp(c.AppName)
if exists {
sa.DeleteContainer(c)
}
}
func Start() {
http.HandleFunc("/health", healthHandler)
http.HandleFunc("/nodes", nodesHandler)
http.HandleFunc("/real", realStateHandler)
http.HandleFunc("/app/", appHandler)
addr := fmt.Sprintf("%s:%d", g.Config().Http.Addr, g.Config().Http.Port)
err := http.ListenAndServe(addr, nil)
if err != nil {
log.Fatalf("ListenAndServe %s fail: %s", addr, err)
}
}
func CheckStale() {
duration := time.Duration(g.Config().Interval) * time.Second
for {
time.Sleep(duration)
checkStale()
}
}
func SyncDomain() {
duration := time.Duration(g.Config().Interval) * time.Second
for {
syncDomain()
time.Sleep(duration)
}
}
func CompareState() {
duration := time.Duration(g.Config().Interval) * time.Second
time.Sleep(duration)
for {
time.Sleep(duration)
compareState()
}
}
func dropApp(appName string) {
if appName == "" {
return
}
if g.Config().Debug {
log.Println("drop app:", appName)
}
sa, _ := g.RealState.GetSafeApp(appName)
cs := sa.Containers()
for _, c := range cs {
dropContainer(c)
}
g.RealState.DeleteSafeApp(appName)
rc := g.RedisConnPool.Get()
defer rc.Close()
uriKey := fmt.Sprintf("%s%s.%s", g.Config().Redis.RsPrefix, appName, g.Config().Domain)
rc.Do("DEL", uriKey)
}
func Start() {
addr := fmt.Sprintf("%s:%d", g.Config().Rpc.Addr, g.Config().Rpc.Port)
tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
log.Fatalf("net.ResolveTCPAddr fail: %s", err)
}
listener, err := net.ListenTCP("tcp", tcpAddr)
if err != nil {
log.Fatalf("listen %s fail: %s", addr, err)
}
rpc.Register(new(NodeState))
for {
conn, err := listener.Accept()
if err != nil {
log.Printf("listener.Accept occur error: %s", err)
continue
}
go rpc.ServeConn(conn)
}
}
func createNewContainer(app *model.App, deployCnt int) {
if deployCnt == 0 {
return
}
if app.Status != model.AppStatus_Success {
if g.Config().Debug {
log.Printf("!!! App=%s Status = %d", app.Name, app.Status)
}
return
}
ip_count := g.ChooseNode(app, deployCnt)
if len(ip_count) == 0 {
log.Println("no node..zZ")
return
}
for ip, count := range ip_count {
for k := 0; k < count; k++ {
DockerRun(app, ip)
}
}
}
func checkStale() {
now := time.Now().Unix()
before := now - 3*int64(g.Config().Interval)
g.DeleteStaleNode(before)
g.RealState.DeleteStale(before)
}
func compareState() {
desiredState, err := getDesiredState()
if err != nil {
log.Println("[ERROR] get desired state fail:", err)
return
}
debug := g.Config().Debug
if debug {
log.Println("comparing......")
}
if len(desiredState) == 0 {
if debug {
log.Println("no desired app. do nothing")
}
// do nothing.
return
}
newAppSlice := []string{}
for name, app := range desiredState {
if !g.RealState.RealAppExists(name) {
if debug && app.InstanceCnt > 0 {
log.Println("[=-NEW-=]:", name)
}
newAppSlice = append(newAppSlice, name)
createNewContainer(app, app.InstanceCnt)
}
}
realNames := g.RealState.Keys()
for ii, name := range realNames {
if debug {
log.Printf("#%d: %s", ii, name)
}
if slice.ContainsString(newAppSlice, name) {
continue
}
app, exists := desiredState[name]
if !exists {
if debug {
log.Println("[=-DEL-=]:", name)
}
dropApp(name)
continue
}
sa, _ := g.RealState.GetSafeApp(name)
isOld, olds := sa.IsOldVersion(app.Image)
if isOld {
if len(olds) > 0 || app.InstanceCnt > 0 {
log.Println("[=-UPGRADE-=]")
}
// deploy new instances
createNewContainer(app, app.InstanceCnt)
// delete old instances
for _, c := range olds {
dropContainer(c)
}
continue
}
nowCnt := sa.ContainerCount()
if nowCnt < app.InstanceCnt {
if debug {
log.Printf("add:%d", app.InstanceCnt-nowCnt)
}
createNewContainer(app, app.InstanceCnt-nowCnt)
continue
}
if nowCnt > app.InstanceCnt {
if debug {
log.Printf("del:%d", nowCnt-app.InstanceCnt)
}
dropContainers(sa.Containers(), nowCnt-app.InstanceCnt)
}
}
}
func DockerRun(app *model.App, ip string) {
if g.Config().Debug {
log.Printf("create container. app:%s, ip:%s\n", app.Name, ip)
}
envVars, err := g.LoadEnvVarsOf(app.Name)
if err != nil {
log.Println("[ERROR] load env fail:", err)
return
}
envVars["APP_NAME"] = app.Name
envVars["HOST_IP"] = ip
if g.Config().Scribe.Ip != "" {
envVars["SCRIBE_IP"] = g.Config().Scribe.Ip
} else {
envVars["SCRIBE_IP"] = ip
}
envVars["SCRIBE_PORT"] = fmt.Sprintf("%d", g.Config().Scribe.Port)
addr := fmt.Sprintf("http://%s:%d", ip, g.Config().DockerPort)
client, err := docker.NewClient(addr)
if err != nil {
log.Println("[ERROR] docker.NewClient fail:", err)
return
}
opts := docker.CreateContainerOptions{
Config: &docker.Config{
Memory: int64(app.Memory * 1024 * 1024),
ExposedPorts: map[docker.Port]struct{}{
docker.Port("8080/tcp"): {},
},
Image: app.Image,
AttachStdin: false,
AttachStdout: false,
AttachStderr: false,
Env: BuildEnvArray(envVars),
},
}
container, err := client.CreateContainer(opts)
if err != nil {
if err == docker.ErrNoSuchImage {
repos, tag := ParseRepositoryTag(app.Image)
e := client.PullImage(docker.PullImageOptions{Repository: repos, Tag: tag}, docker.AuthConfiguration{})
if e != nil {
log.Println("[ERROR] pull image", app.Image, "fail:", e)
return
}
// retry
container, err = client.CreateContainer(opts)
if err != nil {
log.Println("[ERROR] retry create container fail:", err, "ip:", ip)
g.UpdateAppStatus(app, model.AppStatus_CreateContainerFail)
return
}
} else {
log.Println("[ERROR] create container fail:", err, "ip:", ip)
if err != nil && strings.Contains(err.Error(), "cannot connect") {
g.DeleteNode(ip)
g.RealState.DeleteByIp(ip)
return
}
g.UpdateAppStatus(app, model.AppStatus_CreateContainerFail)
return
}
}
err = client.StartContainer(container.ID, &docker.HostConfig{
PortBindings: map[docker.Port][]docker.PortBinding{
"8080/tcp": []docker.PortBinding{docker.PortBinding{}},
},
})
if err != nil {
log.Println("[ERROR] docker.StartContainer fail:", err)
g.UpdateAppStatus(app, model.AppStatus_StartContainerFail)
return
}
if g.Config().Debug {
log.Println("start container success:-)")
}
}
func syncDomain() {
_sql := "select domain, app_name from domain where app_id <> 0"
rows, err := g.DB.Query(_sql)
if err != nil {
log.Printf("[ERROR] exec %s fail: %s", _sql, err)
return
}
needUpdateRedis := false
for rows.Next() {
var domain, appName string
err = rows.Scan(&domain, &appName)
if err != nil {
log.Printf("[ERROR] %s scan fail: %s", _sql, err)
return
}
name, existent := Domains[domain]
if !existent || name != appName {
Domains[domain] = appName
DomainsToUpdate[domain] = true
needUpdateRedis = true
}
}
if !needUpdateRedis {
return
}
rc := g.RedisConnPool.Get()
defer rc.Close()
err = rc.Send("MULTI")
if err != nil {
log.Printf("[ERROR] rc.Do(\"MULTI\") fail: %v", err)
return
}
rsPrefix := g.Config().Redis.RsPrefix
cnamePrefix := g.Config().Redis.CNamePrefix
domain := g.Config().Domain
debug := g.Config().Debug
for d, toUp := range DomainsToUpdate {
if !toUp {
continue
}
uriKey := fmt.Sprintf("%s%s.%s", rsPrefix, Domains[d], domain)
cname := fmt.Sprintf("%s%s", cnamePrefix, d)
if debug {
log.Printf("[Redis] SET %s %s", cname, uriKey)
}
rc.Send("SET", cname, uriKey)
DomainsToUpdate[d] = false
}
_, err = rc.Do("EXEC")
if err != nil {
log.Printf("[ERROR] rc.Do(\"EXEC\") fail: %v", err)
}
}