Example #1
0
func (p *ScaleProcessor) Process(img *img4g.Image) error {
	log.Debug("process scale")
	var err error
	tran := cat.Instance().NewTransaction("Command", "Scale")
	defer func() {
		tran.SetStatus(err)
		tran.Complete()
	}()
	err = img.Resize(p.Width, p.Height)
	return err
}
Example #2
0
func TestDownloadToBuffer(t *testing.T) {
	trackerIp := []string{"10.2.25.31"}
	trackerPort := "22122"
	fdfsClient, err := NewFdfsClient(trackerIp, trackerPort)
	if err != nil {
		t.Error(err)
	}
	catInstance := cat.Instance()
	b, err := fdfsClient.DownloadToBuffer("group1/M00/18/91/CgIZH1RyormAP7xTAAA1U2y_hqk858.jpg", catInstance)
	if err != nil {
		t.Error(err)
	}
	ioutil.WriteFile("download.jpg", b, os.ModePerm)

}
Example #3
0
func (this *HostProcessor) Run() {
	hostPort = strconv.Itoa(this.Port)
	threadcount := strconv.Itoa(this.ThreadCount)
	log.WithFields(log.Fields{
		"hostPort":    hostPort,
		"threadCount": threadcount,
		"nginxPath":   this.NginxPath,
		"nginxPort":   this.NginxPort,
	}).Debug("run host process")

	defer func() {
		if p := recover(); p != nil {
			log.WithFields(log.Fields{
				"hostPort":    hostPort,
				"threadCount": threadcount,
				"type":        "DaemonProcess.RunPanic",
			}).Error(fmt.Sprintf("%v", p))
			LogErrorEvent(CatInstance, "DaemonProcess.RunPanic", fmt.Sprintf("%v", p))
		}
		os.Exit(2)
	}()
	func() {
		Cat := cat.Instance()
		tran := Cat.NewTransaction("System", Reboot)
		defer func() {
			tran.SetStatus("0")
			tran.Complete()
		}()
		LogEvent(Cat, Reboot, JoinString(GetIP(), ":", hostPort), nil)
	}()

	this.computePorts()
	if this.NginxPath != "" {
		if err := ModifyNginxconf(this.NginxPath, this.NginxPort, ports); err != nil {
			log.WithFields(log.Fields{
				"nginxPath": this.NginxPath,
				"nginxPort": this.NginxPort,
				"type":      "DaemonProcess.ModifyNginxError",
			}).Error(err.Error())
			LogErrorEvent(CatInstance, "DaemonProcess.ModifyNginxError", err.Error())
			return
		}
		if err := RestartNginx(this.NginxPath); err != nil {
			log.WithFields(log.Fields{
				"nginxPath": this.NginxPath,
				"type":      "DaemonProcess.RestartNginxError",
			}).Error(err.Error())
			LogErrorEvent(CatInstance, "DaemonProcess.RestartNginxError", err.Error())
			return
		}
	}

	portstats = make(map[string]url.Values)
	for p, _ := range ports {
		this.startWorkerProcess(p)
		log.WithFields(log.Fields{
			"port": p,
		}).Debug("run subproccess on port")
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	go this.monitorWorkerProcesses()
	go this.sendCatHeartBeat()
	go this.listenHeartbeat()
	<-c //Interupt signal coming
	os.Exit(0)
}
Example #4
0
func (this *HostProcessor) sendCatHeartBeat() {
	defer func() {
		if err := recover(); err != nil {
			log.Error(fmt.Sprintf("%v", err))
			this.sendCatHeartBeat()
		}
	}()
	ip := GetIP()
	second := time.Now().Second()
	if second < 29 {
		sleep := time.Duration((29 - second) * 1000000000)
		time.Sleep(sleep)
	}

	catinstance := cat.Instance()
	for {
		log.Debug("send cat heartbeat")
		stats1 := GetStatus()
		data := url.Values{}
		data.Add("port", hostPort)
		for k, v := range stats1 {
			data.Add(k, v)
		}
		portstats[hostPort] = data

		tran := catinstance.NewTransaction("System", "Status")
		h := catinstance.NewHeartbeat("HeartBeat", ip)
		for _, heart := range portstats {
			if heart == nil {
				continue
			}
			port := heart.Get("port")
			if port == "" {
				continue
			}
			h.Set("System", JoinString("Alloc_", port), heart.Get("Alloc"))
			h.Set("System", JoinString("TotalAlloc_", port), heart.Get("TotalAlloc"))
			h.Set("System", JoinString("Sys_", port), heart.Get("Sys"))
			h.Set("System", JoinString("Mallocs_", port), heart.Get("Mallocs"))
			h.Set("System", JoinString("Frees_", port), heart.Get("Frees"))
			h.Set("System", JoinString("OtherSys_", port), heart.Get("OtherSys"))
			h.Set("System", JoinString("PauseNs_", port), heart.Get("PauseNs"))

			h.Set("HeapUsage", JoinString("HeapAlloc_", port), heart.Get("HeapAlloc"))
			h.Set("HeapUsage", JoinString("HeapSys_", port), heart.Get("HeapSys"))
			h.Set("HeapUsage", JoinString("HeapIdle_", port), heart.Get("HeapIdle"))
			h.Set("HeapUsage", JoinString("HeapInuse_", port), heart.Get("HeapInuse"))
			h.Set("HeapUsage", JoinString("HeapReleased_", port), heart.Get("HeapReleased"))
			h.Set("HeapUsage", JoinString("HeapObjects_", port), heart.Get("HeapObjects"))

			h.Set("GC", JoinString("NextGC_", port), heart.Get("NextGC"))
			h.Set("GC", JoinString("LastGC_", port), heart.Get("LastGC"))
			h.Set("GC", JoinString("NumGC_", port), heart.Get("NumGC"))
			portstats[port] = nil
		}
		h.SetStatus("0")
		h.Complete()
		tran.SetStatus("0")
		tran.Complete()
		second = time.Now().Second()
		sleep := time.Duration((90 - second) * 1000000000)
		time.Sleep(sleep)
	}
}
Example #5
0
func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	Cat := cat.Instance()
	handler.ChainBuilder = &ProcChainBuilder{Cat}
	uri := request.URL.String()
	tran := Cat.NewTransaction("URL", getShortUri(uri))
	var (
		err       error
		isSuccess bool = true
	)
	defer func() {
		p := recover()
		if p != nil {
			logErrWithUri(uri, fmt.Sprintf("%v", p), "errorLevel")
			Cat.LogPanic(p)
			tran.SetStatus(p)
		}

		if isSuccess {
			tran.SetStatus("0")
			tran.Complete()
		} else {
			tran.SetStatus(err)
			tran.Complete()
		}
		if p != nil || err != nil {
			http.Error(writer, http.StatusText(404), 404)
		}
	}()

	LogEvent(Cat, "URL", "URL.Client", map[string]string{
		"clientip": GetClientIP(request),
		"serverip": GetIP(),
		"proto":    request.Proto,
		"referer":  request.Referer(),
		//"agent":    request.UserAgent(),
	})

	LogEvent(Cat, "URL", "URL.Method", map[string]string{
		"Http": request.Method + " " + uri,
	})

	LogEvent(Cat, "UpstreamProcess", JoinString(GetIP(), ":", WorkerPort), nil)

	isDigimarkUrl := false
	params, ok1 := legalUrl.FindStringSubmatchMap(uri)
	if !ok1 {
		params, ok1 = digimarkUrl.FindStringSubmatchMap(uri)
		if ok1 {
			isDigimarkUrl = true
		} else {
			err = errors.New("URI.ParseError")
			logErrWithUri(uri, err.Error(), "warnLevel")
			LogErrorEvent(Cat, "URI.ParseError", "")
			return
		}
	}

	//parse storage from url parameters
	store, storagetype, err1 := FindStorage(params, Cat)
	if err1 != nil {
		err = errors.New("Storage.ParseError")
		logErrWithUri(uri, err1.Error(), "warnLevel")
		LogErrorEvent(Cat, "Storage.ParseError", err1.Error())
		return
	}
	//parse handlers chain from url parameters
	var chain *proc.ProcessorChain = nil
	var buildErr *buildError = nil
	if isDigimarkUrl == true {
		chain, buildErr = handler.ChainBuilder.DigimarkProcChain(params)
	} else {
		chain, buildErr = handler.ChainBuilder.Build(params)
	}
	if buildErr != nil {
		err = errors.New(buildErr.Type())
		logErrWithUri(uri, buildErr.Error(), "warnLevel")
		LogErrorEvent(Cat, buildErr.Type(), buildErr.Error())
		return
	}
	//download image from storage
	var bts []byte
	func() {
		type storageError interface {
			Error() string
			Normal() bool //is normal error?
			Type() string //error type
		}

		var err1 error
		getimagetran := Cat.NewTransaction("Storage", storagetype)
		defer func() {
			if err1 != nil {
				logErrWithUri(uri, err1.Error(), "errorLevel")
				e, ok := err1.(storageError)
				if ok && e.Normal() {
					err = errors.New(fmt.Sprintf("%v.%v", storagetype, e.Type()))
					LogErrorEvent(Cat, fmt.Sprintf("%v.%v", storagetype, e.Type()), e.Error())
				} else {
					err = errors.New(storagetype + ".UnExpectedError")
					LogErrorEvent(Cat, err.Error(), err1.Error())
					isSuccess = false
				}
			} else if len(bts) == 0 {
				err = errors.New(storagetype + ".ImgLenZero")
				LogErrorEvent(Cat, err.Error(), "recv image length is 0")
				logErrWithUri(uri, "recv image length is 0", "warnLevel")
			}
			if isSuccess {
				getimagetran.SetStatus("0")
			} else {
				getimagetran.SetStatus(err)
			}
			getimagetran.Complete()
		}()
		bts, err1 = store.GetImage()
	}()
	if err != nil {
		return
	}
	size := len(bts)
	sizestr := strconv.Itoa(size)
	tran.AddData("size", sizestr)
	Cat.LogEvent("Size", GetImageSizeDistribution(size))

	log.WithFields(log.Fields{
		"size": size,
		"uri":  uri,
	}).Debug("recv image length")
	format, _ := params["ext"]
	img := &img4g.Image{Blob: bts, Format: format, Cat: Cat}

	rspChan := make(chan bool, 1)
	task := &nepheleTask{inImg: img, chain: chain, rspChan: rspChan, CatInstance: Cat, canceled: false}
	taskChan <- task

	select {
	case ok := <-rspChan:
		if !ok {
			err = errors.New("ProcessError")
			isSuccess = false
			logErrWithUri(uri, err.Error(), "errorLevel")
			return
		}
	case <-time.After(time.Second * 5):
		task.SetCanceled()
		err = errors.New("ProcessTimeout")
		logErrWithUri(uri, err.Error(), "errorLevel")
		isSuccess = false
		LogErrorEvent(Cat, "ProcessTimeout", "")
		return
	}

	writer.Header().Set("Content-Type", "image/"+format)
	writer.Header().Set("Content-Length", strconv.Itoa(len(img.Blob)))
	writer.Header().Set("Last-Modified", "2015/1/1 01:01:01")
	log.WithFields(log.Fields{
		"size": size,
		"uri":  uri,
	}).Debug("final image size")
	if _, err1 = writer.Write(img.Blob); err1 != nil {
		logErrWithUri(uri, err1.Error(), "errorLevel")
		err = errors.New("Response.WriteError")
		LogErrorEvent(Cat, "Response.Writeerror", err1.Error())
		isSuccess = false
	}
}
Example #6
0
func init() {
	util.InitCat()
	CatInstance = cat.Instance()
}