Ejemplo n.º 1
0
func CycleHandleImage() {
	defer func() {
		if r := recover(); r != nil {
			log.WithFields(log.Fields{
				"type": "HandleImagePanic",
			}).Error(fmt.Sprintf("%v", r))
			LogErrorEvent(CatInstance, "HandleImagePanic", fmt.Sprintf("%v", r))
			go CycleHandleImage()
		}
	}()

	for {
		status := true
		//get a task from task chan
		task := <-taskChan
		if task.GetCanceled() {
			continue
		}
		chain := task.chain
		image := task.inImg
		if err := chainProcImg(task.CatInstance, chain, image); err != nil {
			log.WithFields(log.Fields{
				"type": "ProcessError",
			}).Error(err.Error())
			LogErrorEvent(task.CatInstance, "ProcessError", err.Error())
			status = false
		}
		task.rspChan <- status
	}
}
Ejemplo n.º 2
0
//log error with logging fields uri
func logErrWithUri(uri string, errMsg string, errLevel string) {
	switch errLevel {
	case "errorLevel":
		log.WithFields(log.Fields{
			"uri": uri,
		}).Error(errMsg)
	default:
		log.WithFields(log.Fields{
			"uri": uri,
		}).Warn(errMsg)
	}
}
Ejemplo n.º 3
0
func (this *HostProcessor) monitorWorkerProcesses() {
	defer func() {
		if p := recover(); p != nil {
			log.WithFields(log.Fields{
				"type": "DaemonProcess.MonitorPanic",
			}).Error(fmt.Sprintf("%v", p))
			LogErrorEvent(CatInstance, "DaemonProcess.MonitorPanic", fmt.Sprintf("%v", p))
			this.monitorWorkerProcesses()
		}
	}()
	time.Sleep(8 * time.Second) //sleep ,wait sub process run
	for {
		time.Sleep(2 * time.Second)
		log.Debug("monitor......")
		for port, countor := range ports {
			if port == "" {
				continue
			}

			if countor > 2 {
				log.WithFields(log.Fields{
					"port": port,
				}).Debug("restart port")
				err := KillProcessByPort(port)
				if err != nil {
					log.WithFields(log.Fields{
						"port": port,
						"type": "DaemonProcess.KillProcessError",
					}).Error(err.Error())
					LogErrorEvent(CatInstance, "DaemonProcess.KillProcessError", err.Error())
				}
				this.startWorkerProcess(port)
				ports[port] = 0
			} else {
				_, err := GetHttp(JoinString("http://127.0.0.1:", port, "/heartbeat/"))
				if err != nil {
					ports[port] = ports[port] + 1
					log.WithFields(log.Fields{
						"port": port,
						"type": "WorkerProcess.HeartbeatError",
					}).Error(err.Error())
					LogErrorEvent(CatInstance, "WorkerProcess.HeartbeatError", err.Error())
				} else {
					ports[port] = 0
				}
			}
		}
	}
}
Ejemplo n.º 4
0
func (this *HostProcessor) listenHeartbeat() {
	log.WithFields(log.Fields{
		"hostPort": hostPort,
	}).Debug("listen and serve port")
	//start server
	http.HandleFunc("/heartbeat/", this.heartbeatHandler)
	if err := http.ListenAndServe(":"+hostPort, nil); err != nil {
		log.WithFields(log.Fields{
			"hostPort": hostPort,
			"type":     "DaemonProcess.ListenAndServeError",
		}).Error(err.Error())
		LogErrorEvent(CatInstance, "DaemonProcess.ListenAndServeError", err.Error())
		os.Exit(1)
	}
}
Ejemplo n.º 5
0
func (this *QualityProcessor) Process(img *img4g.Image) error {
	log.WithFields(log.Fields{
		"quality": this.Quality,
	}).Debug("process quality")
	err := img.SetCompressionQuality(this.Quality)
	return err
}
Ejemplo n.º 6
0
func (this *FormatProcessor) Process(img *img4g.Image) error {
	log.WithFields(log.Fields{
		"format": this.Format,
	}).Debug("process format")
	err := img.SetFormat(this.Format)
	return err
}
Ejemplo n.º 7
0
func (this *HostProcessor) ModifyNginxconf() {
	this.computePorts()
	if this.NginxPath != "" {
		if err := ModifyNginxconf(this.NginxPath, this.NginxPort, ports); err != nil {
			log.WithFields(log.Fields{
				"nginxPath": this.NginxPath,
				"nginxPort": this.NginxPort,
				"type":      "Daemon.ModifyNginxError",
			}).Error(err.Error())
			LogErrorEvent(CatInstance, "Daemon.ModifyNginxError", err.Error())
			return
		}
		if err := RestartNginx(this.NginxPath); err != nil {
			log.WithFields(log.Fields{
				"nginxPath": this.NginxPath,
				"nginxPort": this.NginxPort,
				"type":      "Daemon.RestartNginxError",
			}).Error(err.Error())
			LogErrorEvent(CatInstance, "Daemon.RestartNginxError", err.Error())
			return
		}
	}
}
Ejemplo n.º 8
0
func (this *HostProcessor) startWorkerProcess(port string) {
	cmd := exec.Command("go", "run", "imgsvrd.go", "-s", port, hostPort)
	err := cmd.Start()
	if err != nil {
		log.WithFields(log.Fields{
			"port":     port,
			"hostPort": hostPort,
			"type":     "DaemonProcess.StartWorkerError",
		}).Error(err.Error())
		LogErrorEvent(CatInstance, "DaemonProcess.StartWorkerError", err.Error())
		return
	}
	return
}
Ejemplo n.º 9
0
func (this *HostProcessor) heartbeatHandler(w http.ResponseWriter, request *http.Request) {
	port := request.FormValue("port")
	var value string = "0"
	if port != "" {
		portstats[port] = request.Form
		value = "1"
	}
	log.WithFields(log.Fields{
		"port": port,
	}).Debug("get heartbeat from port")

	w.Header().Set("Connection", "keep-alive")
	a := []byte(value)
	w.Header().Set("Content-Length", strconv.Itoa(len(a)))
	w.Write(a)
}
Ejemplo n.º 10
0
func chainProcImg(catinstance cat.Cat, chain *proc.ProcessorChain, img *img4g.Image) (err error) {
	defer func() {
		if r := recover(); r != nil {
			log.WithFields(log.Fields{
				"type": "ProcessImage.Panic",
			}).Error(fmt.Sprintf("%v", r))
			LogErrorEvent(catinstance, "ProcessImage.Panic", fmt.Sprintf("%v", r))
		}
	}()
	defer img.DestoryWand()
	if err = img.CreateWand(); err != nil {
		return
	}
	if err = chain.Process(img); err != nil {
		return
	}
	err = img.WriteImageBlob()
	return
}
Ejemplo n.º 11
0
func TestLogstashFormatter(t *testing.T) {
	assert := assert.New(t)

	lf := LogstashFormatter{Type: "abc"}

	fields := logrus.Fields{
		"message": "def",
		"level":   "ijk",
		"type":    "lmn",
		"one":     1,
		"pi":      3.14,
		"bool":    true,
	}

	entry := logrus.WithFields(fields)
	entry.Message = "msg"
	entry.Level = logrus.InfoLevel

	b, _ := lf.Format(entry)

	var data map[string]interface{}
	dec := json.NewDecoder(bytes.NewReader(b))
	dec.UseNumber()
	dec.Decode(&data)

	// base fields
	assert.Equal(json.Number("1"), data["@version"])
	assert.NotEmpty(data["@timestamp"])
	assert.Equal("abc", data["type"])
	assert.Equal("msg", data["message"])
	assert.Equal("info", data["level"])

	// substituted fields
	assert.Equal("def", data["fields.message"])
	assert.Equal("ijk", data["fields.level"])
	assert.Equal("lmn", data["fields.type"])

	// formats
	assert.Equal(json.Number("1"), data["one"])
	assert.Equal(json.Number("3.14"), data["pi"])
	assert.Equal(true, data["bool"])
}
Ejemplo n.º 12
0
func (this *HostProcessor) Run() {
	hostPort = strconv.Itoa(this.Port)
	threadcount := strconv.Itoa(this.ThreadCount)
	log.WithFields(log.Fields{
		"hostPort":    hostPort,
		"threadCount": threadcount,
		"nginxPath":   this.NginxPath,
		"nginxPort":   this.NginxPort,
	}).Debug("run host process")

	defer func() {
		if p := recover(); p != nil {
			log.WithFields(log.Fields{
				"hostPort":    hostPort,
				"threadCount": threadcount,
				"type":        "DaemonProcess.RunPanic",
			}).Error(fmt.Sprintf("%v", p))
			LogErrorEvent(CatInstance, "DaemonProcess.RunPanic", fmt.Sprintf("%v", p))
		}
		os.Exit(2)
	}()
	func() {
		Cat := cat.Instance()
		tran := Cat.NewTransaction("System", Reboot)
		defer func() {
			tran.SetStatus("0")
			tran.Complete()
		}()
		LogEvent(Cat, Reboot, JoinString(GetIP(), ":", hostPort), nil)
	}()

	this.computePorts()
	if this.NginxPath != "" {
		if err := ModifyNginxconf(this.NginxPath, this.NginxPort, ports); err != nil {
			log.WithFields(log.Fields{
				"nginxPath": this.NginxPath,
				"nginxPort": this.NginxPort,
				"type":      "DaemonProcess.ModifyNginxError",
			}).Error(err.Error())
			LogErrorEvent(CatInstance, "DaemonProcess.ModifyNginxError", err.Error())
			return
		}
		if err := RestartNginx(this.NginxPath); err != nil {
			log.WithFields(log.Fields{
				"nginxPath": this.NginxPath,
				"type":      "DaemonProcess.RestartNginxError",
			}).Error(err.Error())
			LogErrorEvent(CatInstance, "DaemonProcess.RestartNginxError", err.Error())
			return
		}
	}

	portstats = make(map[string]url.Values)
	for p, _ := range ports {
		this.startWorkerProcess(p)
		log.WithFields(log.Fields{
			"port": p,
		}).Debug("run subproccess on port")
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	go this.monitorWorkerProcesses()
	go this.sendCatHeartBeat()
	go this.listenHeartbeat()
	<-c //Interupt signal coming
	os.Exit(0)
}
Ejemplo n.º 13
0
func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	Cat := cat.Instance()
	handler.ChainBuilder = &ProcChainBuilder{Cat}
	uri := request.URL.String()
	tran := Cat.NewTransaction("URL", getShortUri(uri))
	var (
		err       error
		isSuccess bool = true
	)
	defer func() {
		p := recover()
		if p != nil {
			logErrWithUri(uri, fmt.Sprintf("%v", p), "errorLevel")
			Cat.LogPanic(p)
			tran.SetStatus(p)
		}

		if isSuccess {
			tran.SetStatus("0")
			tran.Complete()
		} else {
			tran.SetStatus(err)
			tran.Complete()
		}
		if p != nil || err != nil {
			http.Error(writer, http.StatusText(404), 404)
		}
	}()

	LogEvent(Cat, "URL", "URL.Client", map[string]string{
		"clientip": GetClientIP(request),
		"serverip": GetIP(),
		"proto":    request.Proto,
		"referer":  request.Referer(),
		//"agent":    request.UserAgent(),
	})

	LogEvent(Cat, "URL", "URL.Method", map[string]string{
		"Http": request.Method + " " + uri,
	})

	LogEvent(Cat, "UpstreamProcess", JoinString(GetIP(), ":", WorkerPort), nil)

	isDigimarkUrl := false
	params, ok1 := legalUrl.FindStringSubmatchMap(uri)
	if !ok1 {
		params, ok1 = digimarkUrl.FindStringSubmatchMap(uri)
		if ok1 {
			isDigimarkUrl = true
		} else {
			err = errors.New("URI.ParseError")
			logErrWithUri(uri, err.Error(), "warnLevel")
			LogErrorEvent(Cat, "URI.ParseError", "")
			return
		}
	}

	//parse storage from url parameters
	store, storagetype, err1 := FindStorage(params, Cat)
	if err1 != nil {
		err = errors.New("Storage.ParseError")
		logErrWithUri(uri, err1.Error(), "warnLevel")
		LogErrorEvent(Cat, "Storage.ParseError", err1.Error())
		return
	}
	//parse handlers chain from url parameters
	var chain *proc.ProcessorChain = nil
	var buildErr *buildError = nil
	if isDigimarkUrl == true {
		chain, buildErr = handler.ChainBuilder.DigimarkProcChain(params)
	} else {
		chain, buildErr = handler.ChainBuilder.Build(params)
	}
	if buildErr != nil {
		err = errors.New(buildErr.Type())
		logErrWithUri(uri, buildErr.Error(), "warnLevel")
		LogErrorEvent(Cat, buildErr.Type(), buildErr.Error())
		return
	}
	//download image from storage
	var bts []byte
	func() {
		type storageError interface {
			Error() string
			Normal() bool //is normal error?
			Type() string //error type
		}

		var err1 error
		getimagetran := Cat.NewTransaction("Storage", storagetype)
		defer func() {
			if err1 != nil {
				logErrWithUri(uri, err1.Error(), "errorLevel")
				e, ok := err1.(storageError)
				if ok && e.Normal() {
					err = errors.New(fmt.Sprintf("%v.%v", storagetype, e.Type()))
					LogErrorEvent(Cat, fmt.Sprintf("%v.%v", storagetype, e.Type()), e.Error())
				} else {
					err = errors.New(storagetype + ".UnExpectedError")
					LogErrorEvent(Cat, err.Error(), err1.Error())
					isSuccess = false
				}
			} else if len(bts) == 0 {
				err = errors.New(storagetype + ".ImgLenZero")
				LogErrorEvent(Cat, err.Error(), "recv image length is 0")
				logErrWithUri(uri, "recv image length is 0", "warnLevel")
			}
			if isSuccess {
				getimagetran.SetStatus("0")
			} else {
				getimagetran.SetStatus(err)
			}
			getimagetran.Complete()
		}()
		bts, err1 = store.GetImage()
	}()
	if err != nil {
		return
	}
	size := len(bts)
	sizestr := strconv.Itoa(size)
	tran.AddData("size", sizestr)
	Cat.LogEvent("Size", GetImageSizeDistribution(size))

	log.WithFields(log.Fields{
		"size": size,
		"uri":  uri,
	}).Debug("recv image length")
	format, _ := params["ext"]
	img := &img4g.Image{Blob: bts, Format: format, Cat: Cat}

	rspChan := make(chan bool, 1)
	task := &nepheleTask{inImg: img, chain: chain, rspChan: rspChan, CatInstance: Cat, canceled: false}
	taskChan <- task

	select {
	case ok := <-rspChan:
		if !ok {
			err = errors.New("ProcessError")
			isSuccess = false
			logErrWithUri(uri, err.Error(), "errorLevel")
			return
		}
	case <-time.After(time.Second * 5):
		task.SetCanceled()
		err = errors.New("ProcessTimeout")
		logErrWithUri(uri, err.Error(), "errorLevel")
		isSuccess = false
		LogErrorEvent(Cat, "ProcessTimeout", "")
		return
	}

	writer.Header().Set("Content-Type", "image/"+format)
	writer.Header().Set("Content-Length", strconv.Itoa(len(img.Blob)))
	writer.Header().Set("Last-Modified", "2015/1/1 01:01:01")
	log.WithFields(log.Fields{
		"size": size,
		"uri":  uri,
	}).Debug("final image size")
	if _, err1 = writer.Write(img.Blob); err1 != nil {
		logErrWithUri(uri, err1.Error(), "errorLevel")
		err = errors.New("Response.WriteError")
		LogErrorEvent(Cat, "Response.Writeerror", err1.Error())
		isSuccess = false
	}
}