func (this *NfsStorage) Download() ([]byte, util.Error) { var ( bts []byte err error ) var messagesize int if this.Cat != nil { tran := this.Cat.NewTransaction(CATTITLE, "Nfs.Download") tran.AddData("path", this.Path) defer func() { if err != nil { tran.SetStatus(err) } else { util.LogEvent(this.Cat, "Size", util.GetImageSizeDistribution(messagesize), map[string]string{"size": strconv.Itoa(messagesize)}) tran.SetStatus("0") } tran.Complete() }() } bts, err = ioutil.ReadFile(this.Path) if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_NFSDOWNLOADERR, err.Error()) return []byte{}, util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_NFSDOWNLOADERR} } messagesize = len(bts) return bts, util.Error{} }
func (this *Channel) GetAll() (map[string]string, util.Error) { var err error if len(channels) < 1 || IsRefresh(getChannelsTime) { if this.Cat != nil { var err error tran := this.Cat.NewTransaction(DBTITLE, "Channel.GetAll") defer func() { if err != nil { tran.SetStatus(err) } else { tran.SetStatus("0") } tran.Complete() }() } o := orm.NewOrm() var res orm.Params _, err = o.Raw("SELECT name,code FROM channel").RowsToMap(&res, "name", "code") if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_GETCHANNEL, err.Error()) return nil, util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_GETCHANNEL} } m := make(map[string]string) for k, v := range res { m[k] = fmt.Sprintf("%v", v) } channels = m getChannelsTime = time.Now() } return channels, util.Error{} }
func (this ImageRequest) getStorageBySource(path string, isSource bool) (Storage, util.Error) { path = strings.Replace(path, "/", "\\", -1) var ( storagePath string storageType string ) var storage Storage if isNewUri(path) { imagename := util.Substr(path, 1, NEWIMAGENAMELENGTH) imgIndex := models.ImageIndex{Cat: this.Cat} if e := imgIndex.Parse(imagename); e.Err != nil { return nil, e } storagePath = imgIndex.StoragePath storageType = imgIndex.StorageType storage = CreateStorage(storagePath, storageType, this.Cat) if storage == nil { util.LogErrorEvent(this.Cat, ERRORTYPE_STORAGETYPENOSUPPORTE, util.JoinString("Can't supporte storagetype[", storageType, "]")) return nil, util.Error{IsNormal: false, Err: errors.New(util.JoinString("Can't supporte storagetype[", storageType, "]")), Type: ERRORTYPE_STORAGETYPENOSUPPORTE} } } else { storageType = STORAGETYPE_NFS if isFdfs(path) { storageType = STORAGETYPE_FDFS } storage = CreateStorage(path, storageType, this.Cat) if e := storage.ConvertFilePath(isSource); e.Err != nil { return nil, e } } return storage, util.Error{} }
func (this *FdfsStorage) initFdfsClient() util.Error { if fdfsClient == nil { lock <- 1 defer func() { <-lock }() if fdfsClient != nil { return util.Error{} } config := models.Config{Cat: this.Cat} fdfsdomain, e := config.GetFdfsDomain() if e.Err != nil { return e } fdfsport, e := config.GetFdfsPort() if e.Err != nil { return e } var err error fdfsClient, err = fdfs.NewFdfsClient([]string{fdfsdomain}, fdfsport) if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_FDFSCONNECTIONERR, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_FDFSCONNECTIONERR} } } return util.Error{} }
func (this *FdfsStorage) Download() ([]byte, util.Error) { if e := this.initFdfsClient(); e.Err != nil { return nil, e } var ( bts []byte err error result = util.Error{} ) var messagesize int if this.Cat != nil { tran := this.Cat.NewTransaction(CATTITLE, "Fdfs.Download") tran.AddData("path", this.Path) defer func() { if result.Err != nil && result.IsNormal { tran.SetStatus(result.Err) } else { util.LogEvent(this.Cat, "Size", util.GetImageSizeDistribution(messagesize), map[string]string{"size": strconv.Itoa(messagesize)}) tran.SetStatus("0") } tran.Complete() }() } bts, err = fdfsClient.DownloadToBuffer(this.Path, this.Cat) if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_FDFSDOWNLOADERR, err.Error()) result = util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_FDFSDOWNLOADERR} return []byte{}, result } messagesize = len(bts) return bts, result }
//fdfs ignore isDeleteAll func (this *FdfsStorage) Delete(isDeleteAll bool) util.Error { if e := this.initFdfsClient(); e.Err != nil { return e } var ( err error result = util.Error{} ) if this.Cat != nil { tran := this.Cat.NewTransaction(CATTITLE, "Fdfs.Delete") tran.AddData("path", this.Path) defer func() { if result.Err != nil && result.IsNormal { tran.SetStatus(result.Err) } else { tran.SetStatus("0") } tran.Complete() }() } err = fdfsClient.DeleteFile(this.Path) if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_FDFSDELETEERR, err.Error()) result = util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_FDFSDELETEERR} return result } return result }
func Parse(ctx context.Context) (*CommandArgument, error) { var ( err error commandArg *CommandArgument catVar = ctx.Value("cat").(cat.Cat) imagePath = ctx.Value("imagepath").(string) ) params, ok := generalArgPattern.FindStringSubmatchMap(imagePath) if ok { commandArg, err = makeGeneralArg(imagePath, params) } else { params, ok = sourceDigitalMarkArgPattern.FindStringSubmatchMap(imagePath) if ok { commandArg.digitalMarkOnSource = true commandArg, err = makeDigitalMarkOnSourceArg(params) } else { err = errors.New("Image.Command.ParseError") log.WithFields(log.Fields{"uri": imagePath}).Warn(err.Error()) util.LogErrorEvent(catVar, "URI.ParseError", "") } } return commandArg, err }
func (this *NfsStorage) Delete(isDeleteAll bool) util.Error { var ( cmd *exec.Cmd err error ) if this.Cat != nil { tran := this.Cat.NewTransaction(CATTITLE, "Nfs.Delete") tran.AddData("path", this.Path) defer func() { if err != nil { tran.SetStatus(err) } else { tran.SetStatus("0") } tran.Complete() }() } if !isDeleteAll { cmd = exec.Command("rm", this.Path) } else { cmd = exec.Command("rm", util.Substr(this.Path, 4, len(this.Path)-4)+"_*") } err = cmd.Run() if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_NFSDELETEERR, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_NFSDELETEERR} } return util.Error{} }
func (this *ImageIndex) GetStorage() util.Error { if this.Id < 1 || this.TableZone < 1 || this.PartitionKey < 1 { util.LogErrorEvent(this.Cat, ERRTYPE_GETIMAGEINDEX, "Parameter is Invalid") return util.Error{IsNormal: true, Err: errors.New("getimageindex parameters is invalid"), Type: ERRTYPE_GETIMAGEINDEX} } var result util.Error = util.Error{} if this.Cat != nil { tran := this.Cat.NewTransaction(DBTITLE, "ImageIndex.GetStorage") defer func() { if result.Err != nil { tran.SetStatus(result.Err) } else { tran.SetStatus("0") } tran.AddData("TabeZone", strconv.Itoa(this.TableZone)) tran.AddData("Id", strconv.FormatInt(this.Id, 10)) tran.AddData("PartitionKey", strconv.Itoa(int(this.PartitionKey))) tran.Complete() }() } o := orm.NewOrm() o.Using(getDBString(this.TableZone)) res := make(orm.Params) nums, err := o.Raw(util.JoinString("SELECT storagepath,storagetype FROM imageindex_", strconv.Itoa(this.TableZone), " WHERE id=? AND partitionKey=?"), this.Id, this.PartitionKey).RowsToMap(&res, "storagepath", "storagetype") if err != nil { util.LogErrorEvent(this.Cat, ERRTYPE_GETIMAGEINDEX, err.Error()) result = util.Error{IsNormal: false, Err: err, Type: ERRTYPE_GETIMAGEINDEX} return result } if nums < 1 { util.LogEvent(this.Cat, ERRTYPE_IMAGENAMEINVALID, "IdNoFound", nil) result = util.Error{IsNormal: true, Err: errors.New("image id is't exists"), Type: ERRTYPE_IMAGENAMEINVALID} return result } for k, v := range res { this.StoragePath = k this.StorageType = fmt.Sprintf("%v", v) } return result }
func EncodeRequest(Cat cat.Cat, bts []byte, req *request.Request) util.Error { soapTran := Cat.NewTransaction("SOAP", "EncodeRequest") result := util.Error{} defer func() { if result.Err != nil && !result.IsNormal { soapTran.SetStatus(result.Err) } else { soapTran.SetStatus(CATTRANSUCCESS) } soapTran.Complete() }() if err := soapparse.EncReq(bts, req); err != nil { util.LogErrorEvent(Cat, "SoapParseRequestError", err.Error()) result = util.Error{IsNormal: false, Err: err, Type: "SoapParseRequestError"} } return result }
func DecodeResponse(Cat cat.Cat, header *response.Header, resp interface{}) ([]byte, util.Error) { soapTran := Cat.NewTransaction("SOAP", "DecodeResponse") result := util.Error{} defer func() { if result.Err != nil && !result.IsNormal { soapTran.SetStatus(result.Err) } else { soapTran.SetStatus(CATTRANSUCCESS) } soapTran.Complete() }() content, err := soapparse.DecResp(header, resp) // if err != nil { util.LogErrorEvent(Cat, "SoapParseResponseError", err.Error()) result = util.Error{IsNormal: false, Err: err, Type: "SoapParseResponseError"} } return content, result }
func (w *ImageProcessWorker) Start(opt StartOpt) <-chan error { if !opt.NewHosting { opts := &web.ImageProcessOptions{ListenAddress: fmt.Sprintf("127.0.0.1:%d", w.Port)} webHandler := web.NewImageProcessWebHandler(opts) util.LogEvent(cat.Instance(), "Reboot", fmt.Sprintf("%s:%d", util.LocalIP(), w.Port), nil) go webHandler.Run() return webHandler.ListenError() } else { cmd := exec.Command("go", "run", "imgsvrd.go", "-process.worker", "-port", strconv.Itoa(w.Port)) err := cmd.Start() if err != nil { log.Errorf("start worker=%d failed. error=%s", w.Port, err.Error()) util.LogErrorEvent(cat.Instance(), "DaemonProcess.StartWorkerError", err.Error()) w.startErrCh <- err } } return w.startErrCh }
func (m *Master) killDeadWorkerAndStartNew(w *Worker) bool { isDead := w.isDead() if isDead { log.Debugf("find worker=%d is dead, prepared to restart", w.host.GetPort()) err := killPort(w.host.GetPort()) if err != nil { log.WithFields(log.Fields{ "port": w.host.GetPort(), "type": "DaemonProcess.KillProcessError", }).Error(err.Error()) util.LogErrorEvent(cat.Instance(), "DaemonProcess.KillProcessError", err.Error()) } else { log.Infof("kill worker=%d ok.", w.host.GetPort()) } w.host.Start(StartOpt{NewHosting: true}) log.Debugf("restart worker=%d ok.", w.host.GetPort()) w.healthy() } return isDead }
func (m *Master) checkWorkerHealth(w *Worker) { val, err := doHttpCall(util.JoinString( "http://127.0.0.1:", strconv.Itoa(w.host.GetPort()), "/heartbeat/")) if err != nil { w.sick() log.WithFields(log.Fields{ "port": w.host.GetPort(), "type": "WorkerProcess.HeartbeatError", }).Error(err.Error()) util.LogErrorEvent(cat.Instance(), "WorkerProcess.HeartbeatError", err.Error()) } else { w.healthy() url, err := url.Parse(string(val)) if err != nil { log.Errorf("parse heartbeat from worker=%d failed.", w.host.GetPort()) } else { w.heartbeatInfo = heartbeatValues{mapValues: nil, urlValues: url.Query()} } } }
func (this *Config) GetConfigs() (map[string]map[string]string, util.Error) { if len(config) < 1 || isRefresh(getConfigTime) { var err error if this.Cat != nil { tran := this.Cat.NewTransaction(DBTITLE, "Config.GetConfigs") defer func() { if err != nil { tran.SetStatus(err) } else { tran.SetStatus("0") } tran.Complete() }() } o := orm.NewOrm() var list []orm.Params _, err = o.Raw("SELECT channelCode,`key`,value FROM config").Values(&list) if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_GETCONFIGS, err.Error()) return nil, util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_GETCONFIGS} } m := make(map[string]map[string]string) for _, v := range list { channelCode := v["channelCode"].(string) key := v["key"].(string) value := v["value"].(string) _, exists := m[channelCode] if exists { m[channelCode][key] = value } else { childmap := make(map[string]string) childmap[key] = value m[channelCode] = childmap } } config = m getConfigTime = time.Now() } return config, util.Error{} }
func (this *ImageIndex) Delete() util.Error { var err error if this.Cat != nil { tran := this.Cat.NewTransaction(DBTITLE, "ImageIndex.Delete") defer func() { if err != nil { tran.SetStatus(err) } else { tran.SetStatus("0") } tran.Complete() }() } o := orm.NewOrm() o.Using(getDBString(this.TableZone)) _, err = o.Raw("DELETE FROM `imageindex_"+strconv.Itoa(this.TableZone)+"` WHERE id = ? AND partitionKey = ?", this.Id, this.PartitionKey).Exec() if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_DELETEIMAGEINDEX, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_DELETEIMAGEINDEX} } return util.Error{} }
func (this *FdfsStorage) Upload(bts []byte, fileExt string) (string, util.Error) { if e := this.initFdfsClient(); e.Err != nil { return "", e } config := models.Config{Cat: this.Cat} groups, e := config.GetGroups() if e.Err != nil { return "", e } if uploadcount == 99999999 { uploadcount = 0 } i := uploadcount % len(groups) uploadcount = 0 g := groups[i] var result util.Error = util.Error{} if this.Cat != nil { tran := this.Cat.NewTransaction(CATTITLE, "Fdfs.Upload") util.LogEvent(this.Cat, "Size", util.GetImageSizeDistribution(len(bts)), map[string]string{"size": strconv.Itoa(len(bts))}) defer func() { if result.Err != nil && result.IsNormal { tran.SetStatus(result.Err) } else { tran.SetStatus("0") } tran.Complete() }() } path, err := fdfsClient.UploadByBuffer(g, bts, fileExt) if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_FDFSUPLOADERR, err.Error()) result = util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_FDFSUPLOADERR} return "", result } return path, result }
func (this ImageRequest) Save(r *request.SaveRequest) (response.SaveResponse, util.Error) { r.TargetFormat = strings.ToLower(r.TargetFormat) r.Channel = strings.ToLower(r.Channel) util.LogEvent(this.Cat, "Save", r.Channel, nil) this.checkPlanID(r) if err := this.checkSaveRequest(r); err.Err != nil { return response.SaveResponse{}, err } if err := this.checkSaveCheckItem(r); err.Err != nil { return response.SaveResponse{}, err } storage, storageType := NewStorage(this.Cat) path, e := storage.Upload(r.FileBytes, r.TargetFormat) if e.Err != nil { return response.SaveResponse{}, e } tableZone := sharding() channel := models.Channel{Cat: this.Cat} imgIndex := models.ImageIndex{ChannelCode: channel.GetChannelCode(r.Channel), StoragePath: path, StorageType: storageType, TableZone: tableZone, Cat: this.Cat} plan := "" if r.Process.AnyTypes != nil && len(r.Process.AnyTypes) > 0 { bts, err := r.Process.MarshalJSON() if err != nil { util.LogErrorEvent(this.Cat, ERRORTYPE_MARSHALJSON, err.Error()) return response.SaveResponse{}, util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_MARSHALJSON} } plan = string(bts) } if e := imgIndex.SaveToDB(plan); e.Err != nil { return response.SaveResponse{}, e } uri := imgIndex.GetImageName() return response.SaveResponse{CheckPass: true, OriginalPath: uri, TargetPath: uri}, util.Error{} }
func (this *ImageIndex) SaveToDB(plan string) util.Error { var ( res sql.Result err error id int64 ) if this.Cat != nil { tran := this.Cat.NewTransaction(DBTITLE, util.JoinString("ImageIndex_"+strconv.Itoa(this.TableZone)+".Insert")) defer func() { if err != nil { tran.SetStatus(err) } else { tran.SetStatus("0") } tran.Complete() }() } o := orm.NewOrm() o.Using(getDBString(this.TableZone)) o.Begin() partitionKey := util.GetPartitionKey(time.Now()) res, err = o.Raw("INSERT INTO `imageindex_"+strconv.Itoa(this.TableZone)+"` (`channelCode`,`storagePath`,`storageType`,`profile`,`createtime`,`partitionKey`)VALUES(?,?,?,?,NOW(),?)", this.ChannelCode, this.StoragePath, this.StorageType, this.Profile, partitionKey).Exec() if err != nil { o.Rollback() util.LogErrorEvent(this.Cat, ERRORTYPE_INSERTIMAGEINDEX, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_INSERTIMAGEINDEX} } id, err = res.LastInsertId() if err != nil { o.Rollback() util.LogErrorEvent(this.Cat, ERRORTYPE_INSERTIMAGEINDEX, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_INSERTIMAGEINDEX} } if plan != "" { if this.Cat != nil { tran := this.Cat.NewTransaction(DBTITLE, util.JoinString("ImagePlan_"+strconv.Itoa(this.TableZone)+".Insert")) defer func() { if err != nil { tran.SetStatus(err) } else { tran.SetStatus("0") } tran.Complete() }() } res, err = o.Raw("INSERT INTO `imageplan_"+strconv.Itoa(this.TableZone)+"`(imgId,plan,partitionKey)VALUES(?,?,?)", id, plan, partitionKey).Exec() if err != nil { o.Rollback() util.LogErrorEvent(this.Cat, ERRORTYPE_INSERTIMAGEPLAN, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_INSERTIMAGEPLAN} } if _, err = res.RowsAffected(); err != nil { o.Rollback() util.LogErrorEvent(this.Cat, ERRORTYPE_INSERTIMAGEPLAN, err.Error()) return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_INSERTIMAGEPLAN} } } if err = o.Commit(); err != nil { return util.Error{IsNormal: false, Err: err, Type: ERRORTYPE_INSERTIMAGEINDEX} } this.Id = id this.PartitionKey = partitionKey return util.Error{} }
func (handler *ImageWS) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() w.Header().Set("Connection", "keep-alive") w.Header().Set("Content-Type", "text/xml") Cat := cat.Instance() tran := Cat.NewTransaction("URL", "ImageWs.asmx") util.LogEvent(Cat, "URL", "URL.Client", map[string]string{ "clientip": util.GetClientIP(r), "serverip": util.GetIP(), "proto": r.Proto, "referer": r.Referer(), //"agent": request.UserAgent(), }) var result util.Error defer func() { if p := recover(); p != nil { Cat.LogPanic(p) } if result.Err != nil && !result.IsNormal { tran.SetStatus(result.Err) } else { tran.SetStatus("0") } tran.Complete() }() bts, err := ioutil.ReadAll(r.Body) if err != nil { util.LogErrorEvent(Cat, "RequestReadError", err.Error()) result = util.Error{IsNormal: false, Err: err, Type: "RequestReadError"} msg := []byte(err.Error()) writeResponse(w, msg) return } req := request.Request{} result = EncodeRequest(Cat, bts, &req) if result.Err != nil && !result.IsNormal { writeResponse(w, []byte(result.Type)) return } var ( resp interface{} header *response.Header e util.Error imgRequest = business.ImageRequest{Cat: Cat} ) requestTran := Cat.NewTransaction("Request", strings.Replace(req.Header.RequestType, "Arch.Base.ImageWS.", "", -1)) defer func() { if p := recover(); p != nil { Cat.LogPanic(p) } if result.Err != nil && !result.IsNormal { requestTran.SetStatus(result.Err) } else { requestTran.SetStatus(CATTRANSUCCESS) } requestTran.Complete() }() switch req.Header.RequestType { case REQUESTTYPE_SAVEIMAGE: resp, e = imgRequest.Save(&req.SaveRequest) case REQUESTTYPE_LOADIMAGE: resp, e = imgRequest.Download(&req.LoadImgRequest) case REQUESTTYPE_LOADZIP: resp, e = imgRequest.DownloadZip(&req.LoadZipRequest) case REQUESTTYPE_DELETEIMAGE: resp, e = imgRequest.Delete(&req.DeleteRequest) default: util.LogErrorEvent(Cat, "RequestTypeInvalid", req.Header.RequestType) e = util.Error{IsNormal: true, Err: errors.New("requesttype is invalid!"), Type: "RequestTypeInvalid"} } if e.Err != nil { result = e header = createFailHeader(req.Header, fmt.Sprintf("%v", e.Err)) } else { header = transformHeader(req.Header, RESULTCODE_SUCCESS, "") } content, e := DecodeResponse(Cat, header, resp) if e.Err != nil { result = e writeResponse(w, []byte(result.Err.(error).Error())) } else { writeResponse(w, content) } }
func (handler *LogoWS) ServeHTTP(w http.ResponseWriter, r *http.Request) { Cat := cat.Instance() w.Header().Set("Connection", "keep-alive") w.Header().Set("Content-Type", "text/xml") tran := Cat.NewTransaction("URL", "LogoWs.asmx") util.LogEvent(Cat, "URL", "URL.Client", map[string]string{ "clientip": util.GetClientIP(r), "serverip": util.GetIP(), "proto": r.Proto, "referer": r.Referer(), //"agent": request.UserAgent(), }) var result util.Error defer func() { if p := recover(); p != nil { Cat.LogPanic(p) } if result.Err != nil && !result.IsNormal { tran.SetStatus(result.Err) } else { tran.SetStatus("0") } tran.Complete() }() bts, err := ioutil.ReadAll(r.Body) if err != nil { util.LogErrorEvent(Cat, "RequestReadError", err.Error()) result = util.Error{IsNormal: false, Err: err, Type: "RequestReadError"} return } req := request.Request{} result = EncodeRequest(Cat, bts, &req) if result.Err != nil && !result.IsNormal { //writeResponse(w, []byte(result.Type)) return } var ( resp interface{} header *response.Header e util.Error logoRequest = business.LogoRequest{ ImageRequest: business.ImageRequest{ Cat: Cat, }, FontSizes: []int{14, 16, 18, 20}, } ) requestTran := Cat.NewTransaction("Request", "SaveLogo") defer func() { if p := recover(); p != nil { Cat.LogPanic(p) } if result.Err != nil && !result.IsNormal { requestTran.SetStatus(result.Err) } else { requestTran.SetStatus("0") } requestTran.Complete() }() resp, result = logoRequest.Save(&req.SaveRequest) if result.Err != nil { header = createFailHeader(req.Header, fmt.Sprintf("%v", result.Err)) } else { header = transformHeader(req.Header, RESULTCODE_SUCCESS, "") } content, e := DecodeResponse(Cat, header, resp) if e.Err != nil { result = e writeResponse(w, []byte(result.Err.(error).Error())) } else { writeResponse(w, content) } //content, err := soapparse.DecResp(header, resp) // //if err != nil { // return //} //writeResponse(w, content) }