func (p *HTTPReceiver) Handler(res gohttp.ResponseWriter, req *gohttp.Request) { var deliveries []spirit.Delivery var err error deliveryChan := make(chan spirit.Delivery) doneChan := make(chan bool) defer close(deliveryChan) defer close(doneChan) if deliveries, err = p.requestHandler(res, req, deliveryChan, doneChan); err != nil { spirit.Logger().WithField("actor", spirit.ActorReceiver). WithField("event", "call http request handler"). Errorln(err) return } deliveriesChan.Put(deliveryChan, deliveries...) if err := p.putter.Put(deliveries); err != nil { spirit.Logger().WithField("actor", spirit.ActorReceiver). WithField("event", "put deliveries"). Errorln(err) deliveriesChan.Delete(deliveries...) res.WriteHeader(gohttp.StatusInternalServerError) } else { // wait http request handler to finish process <-doneChan deliveriesChan.Delete(deliveries...) } }
func (p *HTTPReceiver) Start() (err error) { p.statusLocker.Lock() defer p.statusLocker.Unlock() spirit.Logger().WithField("actor", spirit.ActorReceiver). WithField("urn", "urn:spirit-contrib:receiver:http[base]"). WithField("event", "start"). Debugln("enter start") if p.status == spirit.StatusRunning { err = ErrHTTPReceiverAlreadyStarted return } go p.serve() p.status = spirit.StatusRunning spirit.Logger().WithField("actor", spirit.ActorReceiver). WithField("urn", "urn:spirit-contrib:receiver:http[base]"). WithField("event", "start"). Infoln("started") return }
func (p *Package) Get(update bool) (err error) { baseCMD := "go get " if verbosity > 0 { baseCMD = "go get -v " } cmd := baseCMD + p.URI if update { cmd = baseCMD + "-u " + p.URI } var out []byte if out, err = execCommand(cmd); err != nil { spirit.Logger().Errorln(string(out)) return } if p.Revision == "" { return } checkoutCMD := "git -C " + path.Join(p.gosrc, p.URI) + " checkout " + p.Revision if out, err = execCommand(checkoutCMD); err != nil { spirit.Logger().Errorln(string(out)) return } return }
func (p *MNSSender) Stop() (err error) { p.statusLocker.Lock() defer p.statusLocker.Unlock() spirit.Logger().WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "stop"). Debugln("enter stop") if p.status == spirit.StatusStopped { err = spirit.ErrSenderDidNotRunning return } p.terminaled <- true close(p.terminaled) spirit.Logger().WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "stop"). Infoln("stopped") return }
func (p *SpiritHelper) RunProject(createOpts CreateOptions, detach bool, envs []string, tmplArgs map[string]interface{}) (err error) { if err = p.BuildProject(createOpts, "main", tmplArgs); err != nil { return } if cmder, e := execute(path.Join(createOpts.ProjectPath, "main"), createOpts.ProjectPath, !detach, envs); e != nil { err = e return } else if !detach { startSigHandlers() go func() { cmder.Wait() close(subProcExited) close(interrupted) }() <-subProcExited <-interrupted } else { spirit.Logger().Infof("PID: %d\n", cmder.Process.Pid) } if createOpts.IsTempPath && !detach { err = os.RemoveAll(createOpts.ProjectPath) } return }
func main() { var err error defer func() { if err != nil { spirit.Logger().Error(err) os.Exit(128) } }() if configFile != "" { var fileData []byte if fileData, err = ioutil.ReadFile(configFile); err != nil { return } config = string(fileData) } if config == "" { err = fmt.Errorf("config is empty") return } spiritConf := spirit.SpiritConfig{} if envJsonKey != "" && os.Getenv(envJsonKey) != "" { envJson := env_json.NewEnvJson(envJsonKey, envJsonExt) if err = envJson.Unmarshal([]byte(config), &spiritConf); err != nil { return } } else { if err = json.Unmarshal([]byte(config), &spiritConf); err != nil { return } } if err = spiritConf.Validate(); err != nil { err = fmt.Errorf("spirit config validate failed, %s", err) return } var sp spirit.Spirit if sp, err = spirit.NewClassicSpirit(); err != nil { err = fmt.Errorf("create new classic spirit error, %s", err) return } if err = sp.Build(spiritConf); err != nil { err = fmt.Errorf("build classic spirit error, %s", err) return } var wg *sync.WaitGroup if wg, err = sp.Run(); err != nil { return } wg.Wait() }
func (p *HTTPReceiver) Stop() (err error) { p.statusLocker.Lock() defer p.statusLocker.Unlock() spirit.Logger().WithField("actor", spirit.ActorReceiver). WithField("urn", "urn:spirit-contrib:receiver:http[base]"). WithField("event", "stop"). Debugln("enter stop") if p.status == spirit.StatusStopped { return } spirit.Logger().WithField("actor", spirit.ActorReceiver). WithField("urn", "urn:spirit-contrib:receiver:http[base]"). WithField("event", "stop"). Infoln("stopped") return }
func (p *HTTPReceiver) serve() { var logger *log.Logger if p.conf.DisableLogger { logger = log.New(new(emptyWriter), "", 0) } else { logger = log.New(spirit.Logger().Writer(), "[http]", 0) } p.marti.Map(logger) p.marti.RunOnAddr(p.conf.Address) }
func (p *SpiritHelper) BuildProject(createOpts CreateOptions, name string, tmplArgs map[string]interface{}) (err error) { if err = p.CreateProject(createOpts, tmplArgs); err != nil { return } cmd := "go build -o " if verbosity > 0 { cmd = "go build -v -o " } var out []byte if out, err = execCommandWithDir(cmd+name+" "+path.Join(createOpts.ProjectPath, "main.go"), createOpts.ProjectPath); err != nil { spirit.Logger().Errorln(string(out)) return } return }
func (p *MNSWriter) Write(data []byte) (n int, err error) { resource := fmt.Sprintf("queues/%s/%s", p.conf.Queue, "messages") reqData := ali_mns.MessageSendRequest{ MessageBody: ali_mns.Base64Bytes(data), DelaySeconds: int64(p.conf.DelaySeconds), Priority: int64(p.conf.Priority), } var resp *http.Response if resp, err = p.client.Send(ali_mns.POST, nil, reqData, resource); err != nil { return } if resp != nil { defer resp.Body.Close() if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { err = ErrMNSWriteMessageError if body, e := ioutil.ReadAll(resp.Body); e == nil { spirit.Logger().WithField("actor", spirit.ActorWriter). WithField("urn", msnWriterURN). WithField("event", "write"). WithField("url", p.conf.URL). WithField("queue", p.conf.Queue). WithField("status_code", resp.StatusCode). WithField("response", string(body)). Debugln(err) } return } } return }
func upgrade(context *cli.Context) { verbosity = context.Int("verbosity") if !context.IsSet("verbosity") { if verbosity < 2 { verbosity = 2 } } spirit.Logger().Level = logrus.Level(verbosity) var err error defer func() { if err != nil { spirit.Logger().Error(err) os.Exit(128) } }() var out []byte cmd := "go get -u github.com/gogap/spirit-tool" if verbosity > 0 { cmd = "go get -v -u github.com/gogap/spirit-tool" } if out, err = execCommand(cmd); err != nil { spirit.Logger().Errorln(err) return } spirit.Logger().Infoln(out) cmd = "go install github.com/gogap/spirit-tool" if verbosity > 0 { cmd = "go install -v github.com/gogap/spirit-tool" } if out, err = execCommand(cmd); err != nil { spirit.Logger().Errorln(err) return } spirit.Logger().Infoln(out) return }
func (p *MNSSender) Start() (err error) { p.statusLocker.Lock() defer p.statusLocker.Unlock() spirit.Logger().WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "start"). Debugln("enter start") if p.status == spirit.StatusRunning { err = spirit.ErrSenderAlreadyRunning return } if p.getter == nil { err = spirit.ErrSenderDeliveryGetterIsNil return } p.terminaled = make(chan bool) p.status = spirit.StatusRunning go func() { for { if deliveries, err := p.getter.Get(); err != nil { spirit.Logger(). WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "get delivery from getter"). Errorln(err) } else { for _, delivery := range deliveries { if metadata := delivery.Metadata(); metadata != nil { flow := MNSFlowMetadata{} if err := metadata.Object(mnsSenderFlowMetadata, &flow); err != nil { spirit.Logger(). WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "convert metadata to mns_flow failed"). Errorln(err) continue } p.callFlow(delivery, &flow) } else { spirit.Logger(). WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "get metadata"). Warnln("metadata not exist") continue } } } select { case signal := <-p.terminaled: { if signal == true { spirit.Logger().WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "terminal"). Debugln("terminal singal received") return } } case <-time.After(time.Microsecond * 10): { continue } } } }() spirit.Logger().WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "start"). Infoln("started") return }
func (p *MNSSender) callFlow(delivery spirit.Delivery, flowMetadata *MNSFlowMetadata) { if flowMetadata == nil { return } sendDeliveryFunc := func(ignoreSendError bool, delivery spirit.Delivery, queueURN map[string]string, queues ...string) (err error) { if queues == nil || len(queues) == 0 { return } backupURN := delivery.URN() backupParallel := new(MNSParallelFlowMetadata) delivery.Payload().Context().Object(mnsSenderQueueParallelIdContext, backupParallel) // recover defer func() { delivery.SetURN(backupURN) delivery.Payload().SetContext(mnsSenderQueueParallelIdContext, backupParallel) }() for _, q := range queues { if q == "" { continue } parallelQueues := strings.Split(q, "||") parallelCount := len(parallelQueues) parallelQueuePid := "" if parallelCount > 0 { parallelQueuePid = xid.New().String() } for index, queue := range parallelQueues { // recover delivery.SetURN(backupURN) delivery.Payload().SetContext(mnsSenderQueueParallelIdContext, backupParallel) urn := "" if queueURN != nil { urn, _ = queueURN[queue] } buf := bytes.Buffer{} delivery.SetURN(urn) if parallelCount > 0 { delivery.Payload().SetContext(mnsSenderQueueParallelIdContext, MNSParallelFlowMetadata{Id: parallelQueuePid, Index: index, Count: parallelCount}) } if err = p.translator.Out(&buf, delivery); err != nil { spirit.Logger(). WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "translate delivery"). Errorln(err) return } else { reqData := ali_mns.MessageSendRequest{ MessageBody: ali_mns.Base64Bytes(buf.Bytes()), DelaySeconds: int64(p.conf.DelaySeconds), Priority: int64(p.conf.Priority), } client := p.getMNSClient(queue) if _, err = client.SendMessage(reqData); err != nil { spirit.Logger(). WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "send mns message"). Errorln(err) if ignoreSendError { err = nil continue } return } } } } return } currentQueueURN := map[string]string{} if err := delivery.Metadata().Object(mnsSenderQueueURNMetadata, ¤tQueueURN); err != nil { spirit.Logger(). WithField("actor", spirit.ActorSender). WithField("urn", mnsSenderURN). WithField("event", "get queue urn"). Errorln(err) } if delivery.Payload().LastError() != nil { // run error flow if flowMetadata.Error != nil && len(flowMetadata.Error) > 0 { queueURN := map[string]string{} for _, queue := range flowMetadata.Error { if urn, exist := currentQueueURN[queue]; exist { queueURN[queue] = urn } } delivery.SetMetadata(mnsSenderFlowMetadata, nil) sendDeliveryFunc(true, delivery, queueURN, flowMetadata.Error...) } } else { // run normal flow if flowMetadata.Normal != nil && len(flowMetadata.Normal) > 0 { nextQueue := flowMetadata.Normal[flowMetadata.CurrentFlowId] queueURN := map[string]string{} if len(flowMetadata.Normal) > flowMetadata.CurrentFlowId { tmpRemainQueues := flowMetadata.Normal[flowMetadata.CurrentFlowId:] remainQueues := []string{} for _, queue := range tmpRemainQueues { queues := strings.Split(queue, "||") remainQueues = append(remainQueues, queues...) } for _, queue := range remainQueues { if urn, exist := currentQueueURN[queue]; exist { queueURN[queue] = urn } } } flowMetadata.CurrentFlowId += 1 defer func() { flowMetadata.CurrentFlowId -= 1 }() delivery.SetMetadata(mnsSenderFlowMetadata, flowMetadata) sendDeliveryFunc(false, delivery, queueURN, nextQueue) } } }
func (p *SpiritHelper) CreateProject(createOpts CreateOptions, tmplArgs map[string]interface{}) (err error) { if err = createOpts.Validate(); err != nil { return } goSrc := path.Join(createOpts.GoPath, "src") if err = p.parse(goSrc, createOpts.Sources); err != nil { return } // download packages if createOpts.GetPackages { if err = p.GetPackages(goSrc, createOpts.PackagesRevision, createOpts.UpdatePackages); err != nil { return } } // make project dir projectPath := path.Join(goSrc, createOpts.ProjectPath) if path.IsAbs(createOpts.ProjectPath) { projectPath = createOpts.ProjectPath } if !createOpts.IsTempPath { if fi, e := os.Stat(projectPath); e != nil { if !strings.Contains(e.Error(), "no such file or directory") && !os.IsNotExist(e) { err = e return } } else if !fi.IsDir() { err = fmt.Errorf("your project path %s already exist, but it is not a directory", projectPath) return } else if createOpts.ForceWrite { spirit.Logger().Warnf("project path %s already exist, it will be overwrite", projectPath) } else { err = fmt.Errorf("your project path %s already exist", projectPath) return } } if err = os.MkdirAll(projectPath, os.FileMode(0755)); err != nil { if !os.IsNotExist(err) { return } else if !createOpts.ForceWrite { return } err = nil } // render code template tmplPathFmt := "github.com/gogap/spirit-tool/template/%s/main.go" tmplArgsPathFmt := "github.com/gogap/spirit-tool/template/%s/args.json" tmplPath := path.Join(goSrc, fmt.Sprintf(tmplPathFmt, createOpts.TemplateName)) spirit.Logger().Infof("using template of %s: %s", createOpts.TemplateName, tmplPath) tmplArgsPath := path.Join(goSrc, fmt.Sprintf(tmplArgsPathFmt, createOpts.TemplateName)) spirit.Logger().Infof("using template args of %s: %s", createOpts.TemplateName, tmplArgsPath) var tmpl *template.Template if tmpl, err = template.New("main.go").Option("missingkey=error").Delims("//<-", "->//").ParseFiles(tmplPath); err != nil { return } internalArgs := map[string]interface{}{} if argData, e := ioutil.ReadFile(tmplArgsPath); e == nil { if err = json.Unmarshal(argData, &internalArgs); err != nil { return } } if tmplArgs != nil { for k, v := range tmplArgs { internalArgs[k] = v } } buffer := &bytes.Buffer{} if err = tmpl.Execute(buffer, map[string]interface{}{ "create_options": createOpts, "packages": p.RefPackages, "config": p.configFile, "config_filename": p.configFileName, "create_time": time.Now(), "args": internalArgs}); err != nil { return } srcPath := path.Join(projectPath, "main.go") if err = ioutil.WriteFile(srcPath, buffer.Bytes(), os.FileMode(0644)); err != nil { return } confPath := path.Join(projectPath, p.configFileName) if err = ioutil.WriteFile(confPath, p.originalConfig, os.FileMode(0644)); err != nil { return } // format code for sort import packages order if _, err = execCommand("go fmt " + srcPath); err != nil { return } spirit.Logger().Infof("project created at %s\n", projectPath) return }
func create(context *cli.Context) { verbosity = context.Int("verbosity") if !context.IsSet("verbosity") { if verbosity < 2 { verbosity = 2 } } spirit.Logger().Level = logrus.Level(verbosity) var err error defer func() { if err != nil { spirit.Logger().Error(err) os.Exit(128) } }() goPath := context.String("gopath") projectPath := context.String("path") configFile := context.String("config") extSources := context.StringSlice("source") getPkg := context.Bool("get") updatePkg := context.Bool("update") strArgs := context.StringSlice("args") forceWrite := context.Bool("force") templateName := context.String("template") revConfig := context.String("rev") if goPath == "" { err = fmt.Errorf("could not get GOPATH") return } spirit.Logger().Infof("GOPATH: %s", goPath) if projectPath == "" { err = fmt.Errorf("please input your project path, like: github.com/your_orgs/project_name ") return } if configFile == "" { err = fmt.Errorf("please input config file") return } sources := []string{ path.Join(goPath, "src", "github.com/gogap/spirit-tool/source/offical.json"), path.Join(goPath, "src", "github.com/gogap/spirit-tool/source/third_party.json"), } sources = append(sources, extSources...) tmplArgs := map[string]interface{}{} for _, arg := range strArgs { arg = strings.TrimSpace(arg) if arg != "" { v := strings.Split(arg, "=") if len(v) != 2 { err = fmt.Errorf("the args to template format error, arg: %s", arg) return } tmplArgs[v[0]] = v[1] } } helper := SpiritHelper{} if err = helper.LoadSpiritConfig(configFile); err != nil { return } var rev map[string]string if revConfig != "" { loadKeyValueJSON(revConfig, &rev) } createOpts := CreateOptions{ TemplateName: templateName, GoPath: goPath, ProjectPath: projectPath, GetPackages: getPkg, UpdatePackages: updatePkg, ForceWrite: forceWrite, Sources: sources, PackagesRevision: nil, } if err = helper.CreateProject(createOpts, tmplArgs); err != nil { return } return }
func (p *JsonApiReceiver) requestHandler( res gohttp.ResponseWriter, req *gohttp.Request, deliveryChan <-chan spirit.Delivery, done chan<- bool, ) (deliveries []spirit.Delivery, err error) { var apiIds map[string]string // request to deliveries if deliveries, apiIds, err = p.toDeliveries(req); err != nil { var apiResponse APIResponse switch errCode := err.(type) { case errors.ErrCode: { apiResponse = APIResponse{ Code: errCode.Code(), ErrorId: errCode.Id(), ErrorNamespace: errCode.Namespace(), Message: errCode.Error(), Result: nil, } } default: e := ErrApiGenericError.New().Append(err) apiResponse = APIResponse{ Code: e.Code(), ErrorId: e.Id(), ErrorNamespace: e.Namespace(), Message: e.Error(), Result: nil, } } if data, e := json.Marshal(apiResponse); e != nil { spirit.Logger(). WithField("event", "to deliveries"). WithField("urn", p.URN()). WithField("name", p.Name()). Errorln(err) } else { p.writeResponse(data, res, req) } return } go func( count int, apiIds map[string]string, res gohttp.ResponseWriter, req *gohttp.Request, deliveryChan <-chan spirit.Delivery, done chan<- bool) { defer func() { // notify the main handler finished select { case done <- true: { } case <-time.After(time.Second * 3): { } } }() // get timeout duration timeout := time.Duration(p.conf.Timeout) * time.Millisecond if strTimeout := req.Header.Get(p.conf.HeaderDefines.TimeoutHeader); strTimeout != "" { if i, e := strconv.Atoi(strTimeout); e == nil { timeout = time.Duration(i) * time.Millisecond } } if timeout <= 0 { timeout = DefaultTimeout } apiResponse := map[string]APIResponse{} i := count // get deliveries label_timeout_or_finished: for i > 0 { select { case delivery := <-deliveryChan: { if api, exist := apiIds[delivery.Id()]; !exist { spirit.Logger(). WithField("urn", p.URN()). WithField("name", p.Name()). WithField("api", api). WithField("delivery_id", delivery.Id()). Errorln("api not exist in request while delivery response") } else { apiResponse[api] = p.deliveryToApiResponse(delivery) } i = i - 1 } case <-time.After(timeout): { break label_timeout_or_finished } } } // request timeout for _, api := range apiIds { if _, exist := apiResponse[api]; !exist { errCode := ErrRequestTimeout.New() apiResponse[api] = APIResponse{ Code: errCode.Code(), ErrorId: errCode.Id(), ErrorNamespace: errCode.Namespace(), Message: errCode.Error(), Result: nil, } } } // render deliveries to json response // normal response: {"code": 0, "message": "", "result": null} // error response: {"code": 212, "error_namespace": "xxxx", "message": "something wrong", "result": null} isMultiCall := req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "1" || req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "on" || req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "true" if renderedData, e := p.responseRenderer.Render(isMultiCall, apiResponse); e != nil { err := ErrRenderApiDataFailed.New(errors.Params{"err": e}) resp := APIResponse{ Code: err.Code(), ErrorId: err.Id(), ErrorNamespace: err.Namespace(), Message: err.Error(), Result: nil, } if errRespData, e := json.Marshal(resp); e != nil { strInternalErr := `{"code": 500, "message": "api server internal error", "result": null}` p.writeResponseWithStatusCode([]byte(strInternalErr), res, req, gohttp.StatusInternalServerError) } else { p.writeResponse(errRespData, res, req) } } else { p.writeResponse(renderedData, res, req) } return }(len(deliveries), apiIds, res, req, deliveryChan, done) return }
func (p *MNSReader) request() (err error) { if p.requesting || p.reading { return } defer func() { p.requesting = false if err != nil { p.reading = false p.tmpReader = nil } }() p.requesting = true resource := fmt.Sprintf("queues/%s/%s?numOfMessages=%d", p.conf.Queue, "messages", p.conf.MaxReadCount) if p.conf.PollingWaitSeconds > 0 { resource = fmt.Sprintf("queues/%s/%s?numOfMessages=%d&waitseconds=%d", p.conf.Queue, "messages", p.conf.MaxReadCount, p.conf.PollingWaitSeconds) } var resp *http.Response if resp, err = p.client.Send(ali_mns.GET, nil, nil, resource); err != nil { return } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNoContent { spirit.Logger().WithField("actor", spirit.ActorReader). WithField("urn", mnsReaderURN). WithField("url", p.conf.URL). WithField("queue", p.conf.Queue). WithField("status_code", resp.StatusCode). Debugln("no content") err = io.EOF return } mnsDecoder := ali_mns.NewAliMNSDecoder() errResp := ali_mns.ErrorMessageResponse{} if e := mnsDecoder.Decode(resp.Body, &errResp); e == nil { if errResp.Code != "MessageNotExist" { spirit.Logger().WithField("actor", spirit.ActorReader). WithField("urn", mnsReaderURN). WithField("event", "read"). WithField("url", p.conf.URL). WithField("queue", p.conf.Queue). WithField("status_code", resp.StatusCode). WithField("code", errResp.Code). WithField("host_id", errResp.HostId). WithField("request_id", errResp.RequestId). Errorln(errors.New(errResp.Message)) } else { err = io.EOF return } } else { spirit.Logger().WithField("actor", spirit.ActorReader). WithField("urn", mnsReaderURN). WithField("event", "decode mns error response"). WithField("url", p.conf.URL). WithField("queue", p.conf.Queue). WithField("status_code", resp.StatusCode). Errorln(e) } err = ErrAliMNSResponseError return } decoder := ali_mns.NewAliMNSDecoder() batchMessages := ali_mns.BatchMessageReceiveResponse{} if err = decoder.Decode(resp.Body, &batchMessages); err != nil { return } var buf bytes.Buffer for _, message := range batchMessages.Messages { if _, err = buf.Write(message.MessageBody); err != nil { return } } p.reading = true p.tmpReader = &buf return }