func CycleHandleImage() {
defer func() {
if r := recover(); r != nil {
log.WithFields(log.Fields{
"type": "HandleImagePanic",
}).Error(fmt.Sprintf("%v", r))
LogErrorEvent(CatInstance, "HandleImagePanic", fmt.Sprintf("%v", r))
go CycleHandleImage()
}
}()
for {
status := true
//get a task from task chan
task := <-taskChan
if task.GetCanceled() {
continue
}
chain := task.chain
image := task.inImg
if err := chainProcImg(task.CatInstance, chain, image); err != nil {
log.WithFields(log.Fields{
"type": "ProcessError",
}).Error(err.Error())
LogErrorEvent(task.CatInstance, "ProcessError", err.Error())
status = false
}
task.rspChan <- status
}
}
//log error with logging fields uri
func logErrWithUri(uri string, errMsg string, errLevel string) {
switch errLevel {
case "errorLevel":
log.WithFields(log.Fields{
"uri": uri,
}).Error(errMsg)
default:
log.WithFields(log.Fields{
"uri": uri,
}).Warn(errMsg)
}
}
func (this *HostProcessor) monitorWorkerProcesses() {
defer func() {
if p := recover(); p != nil {
log.WithFields(log.Fields{
"type": "DaemonProcess.MonitorPanic",
}).Error(fmt.Sprintf("%v", p))
LogErrorEvent(CatInstance, "DaemonProcess.MonitorPanic", fmt.Sprintf("%v", p))
this.monitorWorkerProcesses()
}
}()
time.Sleep(8 * time.Second) //sleep ,wait sub process run
for {
time.Sleep(2 * time.Second)
log.Debug("monitor......")
for port, countor := range ports {
if port == "" {
continue
}
if countor > 2 {
log.WithFields(log.Fields{
"port": port,
}).Debug("restart port")
err := KillProcessByPort(port)
if err != nil {
log.WithFields(log.Fields{
"port": port,
"type": "DaemonProcess.KillProcessError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "DaemonProcess.KillProcessError", err.Error())
}
this.startWorkerProcess(port)
ports[port] = 0
} else {
_, err := GetHttp(JoinString("http://127.0.0.1:", port, "/heartbeat/"))
if err != nil {
ports[port] = ports[port] + 1
log.WithFields(log.Fields{
"port": port,
"type": "WorkerProcess.HeartbeatError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "WorkerProcess.HeartbeatError", err.Error())
} else {
ports[port] = 0
}
}
}
}
}
func (this *HostProcessor) listenHeartbeat() {
log.WithFields(log.Fields{
"hostPort": hostPort,
}).Debug("listen and serve port")
//start server
http.HandleFunc("/heartbeat/", this.heartbeatHandler)
if err := http.ListenAndServe(":"+hostPort, nil); err != nil {
log.WithFields(log.Fields{
"hostPort": hostPort,
"type": "DaemonProcess.ListenAndServeError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "DaemonProcess.ListenAndServeError", err.Error())
os.Exit(1)
}
}
func (this *QualityProcessor) Process(img *img4g.Image) error {
log.WithFields(log.Fields{
"quality": this.Quality,
}).Debug("process quality")
err := img.SetCompressionQuality(this.Quality)
return err
}
func (this *FormatProcessor) Process(img *img4g.Image) error {
log.WithFields(log.Fields{
"format": this.Format,
}).Debug("process format")
err := img.SetFormat(this.Format)
return err
}
func (this *HostProcessor) ModifyNginxconf() {
this.computePorts()
if this.NginxPath != "" {
if err := ModifyNginxconf(this.NginxPath, this.NginxPort, ports); err != nil {
log.WithFields(log.Fields{
"nginxPath": this.NginxPath,
"nginxPort": this.NginxPort,
"type": "Daemon.ModifyNginxError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "Daemon.ModifyNginxError", err.Error())
return
}
if err := RestartNginx(this.NginxPath); err != nil {
log.WithFields(log.Fields{
"nginxPath": this.NginxPath,
"nginxPort": this.NginxPort,
"type": "Daemon.RestartNginxError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "Daemon.RestartNginxError", err.Error())
return
}
}
}
func (this *HostProcessor) startWorkerProcess(port string) {
cmd := exec.Command("go", "run", "imgsvrd.go", "-s", port, hostPort)
err := cmd.Start()
if err != nil {
log.WithFields(log.Fields{
"port": port,
"hostPort": hostPort,
"type": "DaemonProcess.StartWorkerError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "DaemonProcess.StartWorkerError", err.Error())
return
}
return
}
func (this *HostProcessor) heartbeatHandler(w http.ResponseWriter, request *http.Request) {
port := request.FormValue("port")
var value string = "0"
if port != "" {
portstats[port] = request.Form
value = "1"
}
log.WithFields(log.Fields{
"port": port,
}).Debug("get heartbeat from port")
w.Header().Set("Connection", "keep-alive")
a := []byte(value)
w.Header().Set("Content-Length", strconv.Itoa(len(a)))
w.Write(a)
}
func chainProcImg(catinstance cat.Cat, chain *proc.ProcessorChain, img *img4g.Image) (err error) {
defer func() {
if r := recover(); r != nil {
log.WithFields(log.Fields{
"type": "ProcessImage.Panic",
}).Error(fmt.Sprintf("%v", r))
LogErrorEvent(catinstance, "ProcessImage.Panic", fmt.Sprintf("%v", r))
}
}()
defer img.DestoryWand()
if err = img.CreateWand(); err != nil {
return
}
if err = chain.Process(img); err != nil {
return
}
err = img.WriteImageBlob()
return
}
func TestLogstashFormatter(t *testing.T) {
assert := assert.New(t)
lf := LogstashFormatter{Type: "abc"}
fields := logrus.Fields{
"message": "def",
"level": "ijk",
"type": "lmn",
"one": 1,
"pi": 3.14,
"bool": true,
}
entry := logrus.WithFields(fields)
entry.Message = "msg"
entry.Level = logrus.InfoLevel
b, _ := lf.Format(entry)
var data map[string]interface{}
dec := json.NewDecoder(bytes.NewReader(b))
dec.UseNumber()
dec.Decode(&data)
// base fields
assert.Equal(json.Number("1"), data["@version"])
assert.NotEmpty(data["@timestamp"])
assert.Equal("abc", data["type"])
assert.Equal("msg", data["message"])
assert.Equal("info", data["level"])
// substituted fields
assert.Equal("def", data["fields.message"])
assert.Equal("ijk", data["fields.level"])
assert.Equal("lmn", data["fields.type"])
// formats
assert.Equal(json.Number("1"), data["one"])
assert.Equal(json.Number("3.14"), data["pi"])
assert.Equal(true, data["bool"])
}
func (this *HostProcessor) Run() {
hostPort = strconv.Itoa(this.Port)
threadcount := strconv.Itoa(this.ThreadCount)
log.WithFields(log.Fields{
"hostPort": hostPort,
"threadCount": threadcount,
"nginxPath": this.NginxPath,
"nginxPort": this.NginxPort,
}).Debug("run host process")
defer func() {
if p := recover(); p != nil {
log.WithFields(log.Fields{
"hostPort": hostPort,
"threadCount": threadcount,
"type": "DaemonProcess.RunPanic",
}).Error(fmt.Sprintf("%v", p))
LogErrorEvent(CatInstance, "DaemonProcess.RunPanic", fmt.Sprintf("%v", p))
}
os.Exit(2)
}()
func() {
Cat := cat.Instance()
tran := Cat.NewTransaction("System", Reboot)
defer func() {
tran.SetStatus("0")
tran.Complete()
}()
LogEvent(Cat, Reboot, JoinString(GetIP(), ":", hostPort), nil)
}()
this.computePorts()
if this.NginxPath != "" {
if err := ModifyNginxconf(this.NginxPath, this.NginxPort, ports); err != nil {
log.WithFields(log.Fields{
"nginxPath": this.NginxPath,
"nginxPort": this.NginxPort,
"type": "DaemonProcess.ModifyNginxError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "DaemonProcess.ModifyNginxError", err.Error())
return
}
if err := RestartNginx(this.NginxPath); err != nil {
log.WithFields(log.Fields{
"nginxPath": this.NginxPath,
"type": "DaemonProcess.RestartNginxError",
}).Error(err.Error())
LogErrorEvent(CatInstance, "DaemonProcess.RestartNginxError", err.Error())
return
}
}
portstats = make(map[string]url.Values)
for p, _ := range ports {
this.startWorkerProcess(p)
log.WithFields(log.Fields{
"port": p,
}).Debug("run subproccess on port")
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill)
go this.monitorWorkerProcesses()
go this.sendCatHeartBeat()
go this.listenHeartbeat()
<-c //Interupt signal coming
os.Exit(0)
}
func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
Cat := cat.Instance()
handler.ChainBuilder = &ProcChainBuilder{Cat}
uri := request.URL.String()
tran := Cat.NewTransaction("URL", getShortUri(uri))
var (
err error
isSuccess bool = true
)
defer func() {
p := recover()
if p != nil {
logErrWithUri(uri, fmt.Sprintf("%v", p), "errorLevel")
Cat.LogPanic(p)
tran.SetStatus(p)
}
if isSuccess {
tran.SetStatus("0")
tran.Complete()
} else {
tran.SetStatus(err)
tran.Complete()
}
if p != nil || err != nil {
http.Error(writer, http.StatusText(404), 404)
}
}()
LogEvent(Cat, "URL", "URL.Client", map[string]string{
"clientip": GetClientIP(request),
"serverip": GetIP(),
"proto": request.Proto,
"referer": request.Referer(),
//"agent": request.UserAgent(),
})
LogEvent(Cat, "URL", "URL.Method", map[string]string{
"Http": request.Method + " " + uri,
})
LogEvent(Cat, "UpstreamProcess", JoinString(GetIP(), ":", WorkerPort), nil)
isDigimarkUrl := false
params, ok1 := legalUrl.FindStringSubmatchMap(uri)
if !ok1 {
params, ok1 = digimarkUrl.FindStringSubmatchMap(uri)
if ok1 {
isDigimarkUrl = true
} else {
err = errors.New("URI.ParseError")
logErrWithUri(uri, err.Error(), "warnLevel")
LogErrorEvent(Cat, "URI.ParseError", "")
return
}
}
//parse storage from url parameters
store, storagetype, err1 := FindStorage(params, Cat)
if err1 != nil {
err = errors.New("Storage.ParseError")
logErrWithUri(uri, err1.Error(), "warnLevel")
LogErrorEvent(Cat, "Storage.ParseError", err1.Error())
return
}
//parse handlers chain from url parameters
var chain *proc.ProcessorChain = nil
var buildErr *buildError = nil
if isDigimarkUrl == true {
chain, buildErr = handler.ChainBuilder.DigimarkProcChain(params)
} else {
chain, buildErr = handler.ChainBuilder.Build(params)
}
if buildErr != nil {
err = errors.New(buildErr.Type())
logErrWithUri(uri, buildErr.Error(), "warnLevel")
LogErrorEvent(Cat, buildErr.Type(), buildErr.Error())
return
}
//download image from storage
var bts []byte
func() {
type storageError interface {
Error() string
Normal() bool //is normal error?
Type() string //error type
}
var err1 error
getimagetran := Cat.NewTransaction("Storage", storagetype)
defer func() {
if err1 != nil {
logErrWithUri(uri, err1.Error(), "errorLevel")
e, ok := err1.(storageError)
if ok && e.Normal() {
err = errors.New(fmt.Sprintf("%v.%v", storagetype, e.Type()))
LogErrorEvent(Cat, fmt.Sprintf("%v.%v", storagetype, e.Type()), e.Error())
} else {
err = errors.New(storagetype + ".UnExpectedError")
LogErrorEvent(Cat, err.Error(), err1.Error())
isSuccess = false
}
} else if len(bts) == 0 {
err = errors.New(storagetype + ".ImgLenZero")
LogErrorEvent(Cat, err.Error(), "recv image length is 0")
logErrWithUri(uri, "recv image length is 0", "warnLevel")
}
if isSuccess {
getimagetran.SetStatus("0")
} else {
getimagetran.SetStatus(err)
}
getimagetran.Complete()
}()
bts, err1 = store.GetImage()
}()
if err != nil {
return
}
size := len(bts)
sizestr := strconv.Itoa(size)
tran.AddData("size", sizestr)
Cat.LogEvent("Size", GetImageSizeDistribution(size))
log.WithFields(log.Fields{
"size": size,
"uri": uri,
}).Debug("recv image length")
format, _ := params["ext"]
img := &img4g.Image{Blob: bts, Format: format, Cat: Cat}
rspChan := make(chan bool, 1)
task := &nepheleTask{inImg: img, chain: chain, rspChan: rspChan, CatInstance: Cat, canceled: false}
taskChan <- task
select {
case ok := <-rspChan:
if !ok {
err = errors.New("ProcessError")
isSuccess = false
logErrWithUri(uri, err.Error(), "errorLevel")
return
}
case <-time.After(time.Second * 5):
task.SetCanceled()
err = errors.New("ProcessTimeout")
logErrWithUri(uri, err.Error(), "errorLevel")
isSuccess = false
LogErrorEvent(Cat, "ProcessTimeout", "")
return
}
writer.Header().Set("Content-Type", "image/"+format)
writer.Header().Set("Content-Length", strconv.Itoa(len(img.Blob)))
writer.Header().Set("Last-Modified", "2015/1/1 01:01:01")
log.WithFields(log.Fields{
"size": size,
"uri": uri,
}).Debug("final image size")
if _, err1 = writer.Write(img.Blob); err1 != nil {
logErrWithUri(uri, err1.Error(), "errorLevel")
err = errors.New("Response.WriteError")
LogErrorEvent(Cat, "Response.Writeerror", err1.Error())
isSuccess = false
}
}