func (e Env) ListMachineNames() ([]string, error) { logs.WithFields(e.fields).Debug("list machines") if inMemoryNames != nil { return inMemoryNames, nil } data, modification := ggn.Home.LoadMachinesCacheWithDate(e.name) if data == "" || modification.Add(12*time.Hour).Before(time.Now()) { logs.WithFields(e.fields).Debug("reloading list machines cache") datatmp, _, err := e.RunFleetCmdGetOutput("list-machines", "--fields=metadata", "--no-legend") if err != nil { return nil, errors.Annotate(err, "Cannot list-machines") } data = datatmp ggn.Home.SaveMachinesCache(e.name, data) } var names []string machines := strings.Split(data, "\n") for _, machine := range machines { metas := strings.Split(machine, ",") for _, meta := range metas { elem := strings.Split(meta, "=") if elem[0] == "name" { // TODO this is specific to blablacar's metadata ?? names = append(names, elem[1]) } } } inMemoryNames = names return names, nil }
func (s *Service) Lock(command string, ttl time.Duration, message string) { userAndHost := "[" + ggn.GetUserAndHost() + "] " message = userAndHost + message logs.WithFields(s.fields).WithField("ttl", ttl).WithField("message", message).Info("Locking") s.runHook(EARLY, command, "lock") defer s.runHook(LATE, command, "lock") kapi := s.env.EtcdClient() resp, err := kapi.Get(context.Background(), s.lockPath, nil) if cerr, ok := err.(*client.ClusterError); ok { logs.WithEF(cerr, s.fields).Fatal("Server error reading on fleet") } else if err != nil { _, err := kapi.Set(context.Background(), s.lockPath, message, &client.SetOptions{TTL: ttl}) if err != nil { logs.WithEF(cerr, s.fields).Fatal("Cannot write lock") } } else if strings.HasPrefix(resp.Node.Value, userAndHost) { _, err := kapi.Set(context.Background(), s.lockPath, message, &client.SetOptions{TTL: ttl}) if err != nil { logs.WithEF(cerr, s.fields).Fatal("Cannot write lock") } } else { logs.WithFields(s.fields).WithField("message", resp.Node.Value). WithField("ttl", resp.Node.TTLDuration().String()). Fatal("Service is already locked") } }
func (s *Service) Start(stopper <-chan struct{}, stopWait *sync.WaitGroup) { logs.WithFields(s.fields).Info("Starting service check") stopWait.Add(1) defer stopWait.Done() checkStopWait := &sync.WaitGroup{} statusChange := make(chan Check, 2) for checker := range s.typedCheckersWithStatus { go checker.Run(statusChange, stopper, checkStopWait) } for { select { case status := <-statusChange: logs.WithF(s.fields.WithField("status", status)).Debug("New status received") s.processCheckResult(status) case <-stopper: //TODO since stop is the same everywhere, statusChange chan may stay stuck on shutdown logs.WithFields(s.fields).Debug("Stop requested") checkStopWait.Wait() close(statusChange) if *s.SetServiceAsDownOnShutdown { wait := &sync.WaitGroup{} wait.Add(1) s.Disable(wait, false) wait.Wait() } for reporter := range s.typedReportersWithReported { reporter.Destroy() } return case <-time.After(time.Duration(s.ReportReplayInMilli) * time.Millisecond): s.reportAndTellIfAtLeastOneReported(false) } } }
func (s *Service) prepareNodesAsJsonMap() { if s.manifest.Nodes == nil || len(s.manifest.Nodes.([]interface{})) == 0 { logs.WithFields(s.fields).Warn("No nodes defined in service") return } tmpRes, err := utils.TransformYamlToJson(s.manifest.Nodes) var res []interface{} = tmpRes.([]interface{}) if err != nil { logs.WithEF(err, s.fields).Fatal("Cannot transform yaml to json") } if res[0].(map[string]interface{})[NODE_HOSTNAME].(string) == "*" { if len(res) > 1 { logs.WithFields(s.fields).Fatal("You cannot mix all nodes with single node. Yet ?") } newNodes := *new([]interface{}) machines, err := s.env.ListMachineNames() if err != nil { logs.WithEF(err, s.fields).Fatal("Cannot list machines to generate units") } for _, machine := range machines { node := utils.CopyMap(res[0].(map[string]interface{})) node[NODE_HOSTNAME] = machine newNodes = append(newNodes, node) } res = newNodes } s.nodesAsJsonMap = res }
func (u *Unit) Update(command string) error { if err := u.Service.Generate(); err != nil { logs.WithEF(err, u.Fields).Fatal("Generate failed") } logs.WithFields(u.Fields).Debug("Update") u.runHook(EARLY, command, "update") defer u.runHook(LATE, command, "update") u.Service.Lock(command, 1*time.Hour, "Update "+u.Name) defer u.Service.Unlock(command) same, err := u.IsLocalContentSameAsRemote() if err != nil { logs.WithEF(err, u.Fields).Warn("Cannot compare local and remote service") } if same { logs.WithFields(u.Fields).Info("Remote service is already up to date") if !u.IsRunning() { logs.WithFields(u.Fields).Info("But service is not running") } else if !BuildFlags.Force { return nil } } u.UpdateInside(command) return nil }
func (aci *Aci) prepareBuildAci() (string, error) { logs.WithFields(aci.fields).Debug("Preparing builder") if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil { return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path") } if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil { return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file") } capa, err := types.NewLinuxCapabilitiesRetainSet("all") if err != nil { return "", errs.WithEF(err, aci.fields, "Failed to create all capability retain Set") } allIsolator, err := capa.AsIsolator() if err != nil { return "", errs.WithEF(err, aci.fields, "Failed to prepare all retain set isolator") } aci.manifest.Aci.App.Isolators = types.Isolators([]types.Isolator{*allIsolator}) if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), BuildVersion); err != nil { return "", err } if err := aci.tarAci(aci.target + pathBuilder); err != nil { return "", err } logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt") hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci) if err != nil { return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed") } return hash, nil }
func (u *Unit) Diff(command string) { logs.WithFields(u.Fields).Debug("diff") if err := u.Service.Generate(); err != nil { logs.WithEF(err, u.Fields).Fatal("Generate failed") } u.runHook(EARLY, command, "diff") defer u.runHook(LATE, command, "diff") same, err := u.IsLocalContentSameAsRemote() if err != nil { logs.WithFields(u.Fields).Warn("Cannot read unit") } if !same { u.DisplayDiff() } }
func (aci *Aci) prepareBuildAci() (string, error) { logs.WithFields(aci.fields).Debug("Preparing builder") if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil { return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path") } if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil { return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file") } aci.manifest.Aci.App.Isolators = []common.Isolator{{Name: "os/linux/capabilities-retain-set", Value: common.LinuxCapabilitiesSetValue{Set: []types.LinuxCapability{"all"}}}} if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), dgrVersion); err != nil { return "", err } if err := aci.tarAci(aci.target + pathBuilder); err != nil { return "", err } logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt") hash, err := Home.Rkt.FetchInsecure(aci.target + pathBuilder + pathImageAci) if err != nil { return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed") } return hash, nil }
func (s *Service) Update() error { if err := s.Generate(); err != nil { return err } logs.WithFields(s.fields).Info("Updating service") s.Lock("service/update", 1*time.Hour, "Updating") defer s.Unlock("service/update") if s.manifest.ConcurrentUpdater > 1 && !BuildFlags.Yes { logs.WithFields(s.fields).Fatal("Update concurrently require -y") } s.concurrentUpdater(s.ListUnits()) return nil }
func (u *Unit) Start(command string) error { if u.IsRunning() { logs.WithFields(u.Fields).Info("Service is already running") return nil } if !u.IsLoaded() { logs.WithFields(u.Fields).Debug("unit is not loaded yet") if err := u.Service.Generate(); err != nil { logs.WithEF(err, u.Fields).Fatal("Generate failed") } u.Load(command) } else { logs.WithFields(u.Fields).Debug("unit is already loaded") } return u.runAction(command, "start") }
func (r *ReporterZookeeper) refresher() { for { select { case <-r.stopChecker: logs.WithFields(r.fields).Debug("Stop refresher requested") return default: time.Sleep(time.Duration(r.RefreshIntervalInMilli) * time.Millisecond) } logs.WithFields(r.fields).Debug("Refreshing report") if err := r.sendReportToZk(); err != nil { logs.WithEF(err, r.fields).Error("Failed to refresh status in zookeeper") } } }
func (aci *Aci) upload(name *common.ACFullname) error { if Home.Config.Push.Type == "maven" && name.DomainName() == "aci.blbl.cr" { // TODO this definitely need to be removed logs.WithF(aci.fields).Info("Uploading aci") if err := common.ExecCmd("curl", "-f", "-i", "-F", "r=releases", "-F", "hasPom=false", "-F", "e=aci", "-F", "g=com.blablacar.aci.linux.amd64", "-F", "p=aci", "-F", "v="+name.Version(), "-F", "a="+strings.Split(string(name.Name()), "/")[1], "-F", "file=@"+aci.target+pathImageGzAci, "-u", Home.Config.Push.Username+":"+Home.Config.Push.Password, Home.Config.Push.Url+"/service/local/artifact/maven/content"); err != nil { return errs.WithEF(err, aci.fields, "Failed to push aci") } } else { systemConf := Home.Config.Rkt.SystemConfig if systemConf == "" { systemConf = "/usr/lib/rkt" } localConf := Home.Config.Rkt.LocalConfig if localConf == "" { localConf = "/etc/rkt" } conf, err := config.GetConfigFrom(systemConf, localConf) if err != nil { return errs.WithEF(err, aci.fields, "Failed to get rkt configuration") } upload := Uploader{ Acipath: aci.target + pathImageGzAci, Ascpath: aci.target + pathImageGzAciAsc, Uri: aci.manifest.NameAndVersion.String(), Debug: false, SetHTTPHeaders: func(r *http.Request) { if r.URL == nil { return } headerer, ok := conf.AuthPerHost[r.URL.Host] if !ok { logs.WithFields(aci.fields).WithField("domain", r.URL.Host). Warn("No auth credential found in rkt configuration for this domain") return } header := headerer.Header() for k, v := range header { r.Header[k] = append(r.Header[k], v...) } }, } err = upload.Upload() if err != nil { return errs.WithEF(err, aci.fields, "Failed to upload aci") } } return nil }
func NewService(path string, name string, env Env) *Service { l := env.GetFields() hasTimer := false if _, err := os.Stat(path + "/" + name + PATH_UNIT_TIMER_TEMPLATE); err == nil { hasTimer = true } service := &Service{ units: map[string]*Unit{}, unitsMutex: &sync.Mutex{}, aciListMutex: &sync.Mutex{}, generatedMutex: &sync.Mutex{}, hasTimer: hasTimer, fields: l.WithField("service", name), path: path + "/" + name, Name: name, env: env, lockPath: "/ggn-lock/" + name + "/lock", } logs.WithFields(service.fields).Debug("New service") service.loadManifest() service.loadAttributes() service.prepareNodesAsJsonMap() return service }
func (e Env) Check() { e.Generate() logs.WithFields(e.fields).Debug("Running check") info := HookInfo{Command: "env/check", Action: "env/check"} e.RunEarlyHook(info) defer e.RunLateHook(info) e.concurrentChecker(e.ListServices()) // e.Generate() // units, _, err := e.RunFleetCmdGetOutput("-strict-host-key-checking=false", "list-unit-files", "-no-legend", "-fields", "unit") // if err != nil { // e.log.WithError(err).Fatal("Cannot list unit files") // } // // for _, unitName := range strings.Split(units, "\n") { // unitInfo := strings.Split(unitName, "_") // if len(unitInfo) != 3 { // e.log.WithField("unit", unitName).Warn("Unknown unit format for GGN") // continue // } // split := strings.Split(unitInfo[2], ".") // e.LoadService(unitInfo[1]).LoadUnit(split[0]).Check("env/check") // } }
func (c *CheckCommon) CommonRun(checker Checker, statusChange chan<- Check, stop <-chan struct{}, doneWait *sync.WaitGroup) { logs.WithF(c.fields).Info("Starting check") doneWait.Add(1) defer doneWait.Done() for { status := checker.Check() if logs.IsTraceEnabled() { logs.WithEF(status, c.fields).Trace("Check done") } if status != nil { logs.WithEF(status, c.fields).Debug("Failed check") } if status != nil && !c.service.NoMetrics { c.service.nerve.checkerFailureCount.WithLabelValues(c.service.Name, c.Host, strconv.Itoa(c.Port), c.Type).Inc() } c.saveStatus(status) current := c.stableStatus latest := c.latestStatuses if (latest[0] == nil && sameLastStatusCount(latest) >= c.Rise && (current == nil || *current != nil)) || (latest[0] != nil && sameLastStatusCount(latest) >= c.Fall && (current == nil || *current == nil)) { c.stableStatus = &status statusChange <- Check{checker, *c.stableStatus} } select { case <-stop: logs.WithFields(c.fields).Debug("Stopping check") return case <-time.After(time.Duration(c.CheckIntervalInMilli) * time.Millisecond): } } }
func (e Env) Fleetctl(args []string) { logs.WithFields(e.fields).Debug("Running fleetctl") err := e.RunFleetCmd(args...) if err != nil { logs.WithEF(err, e.fields).Error("Fleetctl command failed") } }
func (aci *Aci) prepareBuildAci() (string, error) { logs.WithFields(aci.fields).Debug("Preparing builder") if err := os.MkdirAll(aci.target+pathBuilder+common.PathRootfs, 0777); err != nil { return "", errs.WithEF(err, aci.fields.WithField("path", aci.target+pathBuilder), "Failed to create builder aci path") } if err := ioutil.WriteFile(aci.target+pathBuilder+common.PathRootfs+"/.keep", []byte(""), 0644); err != nil { return "", errs.WithEF(err, aci.fields.WithField("file", aci.target+pathBuilder+common.PathRootfs+"/.keep"), "Failed to write keep file") } if err := common.WriteAciManifest(aci.manifest, aci.target+pathBuilder+common.PathManifest, common.PrefixBuilder+aci.manifest.NameAndVersion.Name(), dgrVersion); err != nil { return "", err } if err := aci.tarAci(aci.target + pathBuilder); err != nil { return "", err } logs.WithF(aci.fields.WithField("path", aci.target+pathBuilder+pathImageAci)).Info("Importing build to rkt") hash, err := Home.Rkt.Fetch(aci.target + pathBuilder + pathImageAci) if err != nil { return "", errs.WithEF(err, aci.fields, "fetch of builder aci failed") } return hash, nil }
func (u *Unit) Restart(command string) error { logs.WithFields(u.Fields).Debug("restart") if u.Type == TYPE_SERVICE && u.Service.HasTimer() { logs.WithFields(u.Fields).Fatal("You cannot restart a service associated to a time") } u.runHook(EARLY, command, "restart") defer u.runHook(LATE, command, "restart") u.Service.Lock(command, 1*time.Hour, "Restart "+u.Name) defer u.Service.Unlock(command) u.Stop(command) time.Sleep(time.Second * 2) u.Start(command) return nil }
func (u *Unit) Status(command string) { logs.WithFields(u.Fields).Debug("status") u.runHook(EARLY, command, "status") defer u.runHook(LATE, command, "status") err := u.Service.GetEnv().RunFleetCmd("status", u.Filename) if err != nil { os.Exit(1) } }
func (u *Unit) Ssh(command string) { logs.WithFields(u.Fields).Debug("ssh") u.runHook(EARLY, command, "ssh") defer u.runHook(LATE, command, "ssh") err := u.Service.GetEnv().RunFleetCmd("ssh", u.Filename) if err != nil { logs.WithEF(err, u.Fields).Fatal("Failed to run status") } }
func (s *Service) Unlock(command string) { logs.WithFields(s.fields).Info("Unlocking") s.runHook(EARLY, command, "unlock") defer s.runHook(LATE, command, "unlock") kapi := s.env.EtcdClient() _, err := kapi.Delete(context.Background(), s.lockPath, nil) if cerr, ok := err.(*client.ClusterError); ok { logs.WithEF(cerr, s.fields).Fatal("Cannot unlock service") } }
func (e Env) Generate() { logs.WithFields(e.fields).Debug("Generating units") services := e.ListServices() for _, service := range services { service := e.LoadService(service) if err := service.Generate(); err != nil { logs.WithE(err).Error("Generate failed") } } }
func (h *HomeStruct) LoadMachinesCacheWithDate(env string) (string, time.Time) { logs.WithFields(h.fields).WithField("env", env).Debug("Loading list machines cache") info, err := os.Stat(h.Path + PATH_LIST_MACHINES_CACHE + "." + env) if err != nil { return "", time.Now() } content, err := ioutil.ReadFile(h.Path + PATH_LIST_MACHINES_CACHE + "." + env) if err != nil { return "", time.Now() } return string(content), info.ModTime() }
func (e *Env) loadAttributes() { files, err := utils.AttributeFiles(e.path + PATH_ATTRIBUTES) if err != nil { logs.WithEF(err, e.fields).WithField("path", e.path+PATH_ATTRIBUTES).Fatal("Cannot load attribute files") } files, err = e.addIncludeFiles(files) if err != nil { logs.WithEF(err, e.fields).WithField("path", e.path+PATH_ATTRIBUTES).Fatal("Cannot load include files") } e.attributes = attributes.MergeAttributesFiles(files) logs.WithFields(e.fields).WithField("attributes", e.attributes).Debug("Attributes loaded") }
func (e Env) runHook(path string, info HookInfo) { logs.WithFields(e.fields).WithField("path", path).WithField("info", info).Debug("Running hook") files, err := ioutil.ReadDir(e.path + PATH_HOOKS + path) if err != nil { logs.WithEF(err, e.fields).Debug("Cannot read hook directory") return } envs := map[string]string{} envs["ENV"] = e.name envs["COMMAND"] = info.Command if info.Unit != nil { envs["UNIT"] = info.Unit.GetName() } if info.Service != nil { envs["SERVICE"] = info.Service.GetName() } envs["WHO"] = ggn.GetUserAndHost() envs["ACTION"] = info.Action envs["ATTRIBUTES"] = info.Attributes envs["GGN_HOME_PATH"] = ggn.Home.Path for _, f := range files { if !f.IsDir() { hookFields := data.WithField("name", f.Name()) args := []string{e.path + PATH_HOOKS + path + "/" + f.Name()} for key, val := range envs { args = append([]string{key + "='" + strings.Replace(val, "'", "'\"'\"'", -1) + "'"}, args...) } logs.WithFields(hookFields).Debug("Running Hook") if err := common.ExecCmd("bash", "-c", strings.Join(args, " ")); err != nil { logs.WithFields(hookFields).Fatal("Hook status is failed") } } } }
func (u *Unit) Generate(tmpl *template.Templating) error { u.generatedMutex.Lock() defer u.generatedMutex.Unlock() if u.generated { return nil } logs.WithFields(u.Fields).Debug("Generate") data := u.GenerateAttributes() aciList, err := u.Service.PrepareAcis() if err != nil { return err } acis := "" for _, aci := range aciList { acis += aci + " " } data["aciList"] = aciList data["acis"] = acis out, err := json.Marshal(data) if err != nil { logs.WithEF(err, u.Fields).Panic("Cannot marshall attributes") } res := strings.Replace(string(out), "\\\"", "\\\\\\\"", -1) res = strings.Replace(res, "'", `\'`, -1) data["attributes"] = res data["attributesBase64"] = "base64," + base64.StdEncoding.EncodeToString([]byte(out)) data["environmentAttributes"], data["environmentAttributesVars"] = u.prepareEnvironmentAttributes(data["attributes"].(string), "ATTR_") data["environmentAttributesBase64"], data["environmentAttributesVarsBase64"] = u.prepareEnvironmentAttributes(data["attributesBase64"].(string), "ATTR_BASE64_") var b bytes.Buffer err = tmpl.Execute(&b, data) if err != nil { logs.WithEF(err, u.Fields).Error("Failed to run templating") } ok, err := utils.Exists(u.path) if !ok || err != nil { os.Mkdir(u.path, 0755) } err = ioutil.WriteFile(u.path+"/"+u.Filename, b.Bytes(), 0644) if err != nil { logs.WithEF(err, u.Fields).WithField("path", u.path+"/"+u.Filename).Error("Cannot writer unit") } u.generated = true return nil }
func (s *Service) loadAttributes() { attr := utils.CopyMap(s.env.GetAttributes()) files, err := utils.AttributeFiles(s.path + PATH_ATTRIBUTES) if err != nil { logs.WithEF(err, s.fields).WithField("path", s.path+PATH_ATTRIBUTES).Fatal("Cannot load Attributes files") } files, err = s.addIncludeFiles(files) if err != nil { logs.WithEF(err, s.fields).WithField("path", s.path+PATH_ATTRIBUTES).Fatal("Cannot load include files") } attr = attributes.MergeAttributesFilesForMap(attr, files) s.attributes = attr logs.WithFields(s.fields).WithField("attributes", s.attributes).Debug("Attributes loaded") }
func (u *Unit) runAction(command string, action string) error { if command == action { u.Service.Lock(command, 1*time.Hour, action+" "+u.Name) defer u.Service.Unlock(command) } logs.WithFields(u.Fields).Debug(action) u.runHook(EARLY, command, action) defer u.runHook(LATE, command, action) _, _, err := u.Service.GetEnv().RunFleetCmdGetOutput(action, u.unitPath) if err != nil { logs.WithEF(err, u.Fields).Error("Cannot " + action + " unit") return err } return nil }
func (u *Unit) Journal(command string, follow bool, lines int) { logs.WithFields(u.Fields).Debug("journal") u.runHook(EARLY, command, "journal") defer u.runHook(LATE, command, "journal") args := []string{"journal", "-lines", strconv.Itoa(lines)} if follow { args = append(args, "-f") } args = append(args, u.Filename) err := u.Service.GetEnv().RunFleetCmd(args...) if err != nil && !follow { logs.WithEF(err, u.Fields).Fatal("Failed to run journal") } }
func (s *Service) updateUnit(u Unit) { uField := s.fields.WithField("unit", u.Name) ask: for { same, err := u.IsLocalContentSameAsRemote() if err != nil { logs.WithEF(err, uField).Warn("Cannot compare local and remote service") } if same { logs.WithFields(uField).Info("Remote service is already up to date") if !u.IsRunning() { logs.WithFields(uField).Info("But service is not running") } else if !BuildFlags.All { return } } if BuildFlags.Yes { break ask } action := askToProcessService(u) switch action { case ACTION_DIFF: u.DisplayDiff() case ACTION_QUIT: logs.WithFields(uField).Debug("User want to quit") if globalUpdater == 0 { s.Unlock("service/update") } os.Exit(1) case ACTION_SKIP: logs.WithFields(uField).Debug("User skip this service") return case ACTION_YES: break ask default: logs.WithFields(uField).Fatal("Should not be here") } } if atomic.LoadUint32(&globalUpdater) == 0 { atomic.AddUint32(&globalUpdater, 1) s.runHook(EARLY, "service/update", "update") defer s.runHook(LATE, "service/update", "update") } else { atomic.AddUint32(&globalUpdater, 1) } logs.WithFields(uField).Info("Updating unit") u.UpdateInside("service/update") time.Sleep(time.Second * 2) }