Exemple #1
0
//run service
func (this *CsvInput) Run() {
	log.Infoln("CsvInput is starting")
	log.Infof("open file %s", this.Config.FileName)

	g := float64(this.Config.Period)
	if bts, err := ioutil.ReadFile(this.Config.FileName); err == nil {
		strs := strings.Split(string(bts), "\n")
		for common.IsRun {
			beginget := time.Now()
			mod := this.TIME.Minute() % 3
			//			log.Infof("w count is %d", mod+1)
			for i := 0; i <= mod; i++ {
				row := new(common.DataRows)
				row.Time = time.Now().Format(this.Config.TimeFormat)
				row.RowMap = this.RowMap
				for _, str := range strs {
					if this.Config.OutputColumn == nil {
						log.Infoln("output column is nil")
						break
					}
					cols := strings.Split(str, ",")
					var r []interface{}
					for _, c := range cols {
						if d, err := strconv.ParseFloat(c, 10); err == nil {
							r = append(r, d)
						} else {
							r = append(r, 0)
						}
					}
					row.Rows = append(row.Rows, r)
				}
				this.Output <- row
				//log.Infof("push row is %v", row.Time)
			}
			//计算下一次运行的时间

			this.TIME = this.TIME.Add(time.Minute * time.Duration(g))
			log.Infof("next time is %v", this.TIME)
			sec := time.Since(beginget).Seconds()

			//计算下一次运行的时间
			if sec < g*60 { //如果一个处理周期内处理完,就延时,没有处理完就立即执行,如果正式运行期间还需要进行时间纠正
				log.Infof("sleep seconds %f", (g*60)-sec)
				time.Sleep(time.Second * (time.Duration(g*60 - sec))) //sleep one minute-sec
			}
		}

	} else {
		log.Error(err.Error())
	}
	close(this.Output)
}
Exemple #2
0
//处理单元入口
func GetData(dp *common.DataParam, t time.Time) *common.OutputParam {
	if dp.TimeRange == 1 && len(dp.LimitId) == 1 { //单网元,单时间点,用于指标详细
		log.Infoln("开始处理过程,单网元,单时间点,用于指标详细")
		return GetColumnsData(dp, t)
	} else if dp.TimeRange > 1 && len(dp.LimitId) == 1 { //单网元,多时间点,用于走势图
		log.Infoln("开始处理过程,单网元,多时间点,用于走势图")
		return GetTimeLineData(dp, t)
	} else if dp.TimeRange == 1 && (len(dp.LimitId) > 1 || len(dp.LimitNodeId) > 0) { //单时间点,多网元,用于排行榜
		log.Infoln("开始处理过程,单时间点,多网元,用于排行榜")
		return GetOrderData(dp, t)
	} else { //多时间点,多网元,暂不支持
		log.Error("开始处理过程,不被支持的过程", dp.ToString())
		return nil
	}
}
Exemple #3
0
//运行计算
func (this *TimeCache) Run() {
	log.Infoln("cache is started", this.Id)
	period := 1
	if p, ok := Get(FormatKey("System", "Period", this.Id)); ok {
		period = compute.AsInt(p)
	}
	timeFormat := common.GetTimeFormat(period)

	for rs := range this.input { //接收数据
		//log.Infof("time changed ", rs.Time)
		currTime, err := time.Parse(timeFormat, rs.Time)
		if err != nil {
			log.Error("cache time format is error", err.Error())
			currTime = time.Now()
		}
		cacheItem, ok := this.caches[rs.Time] //这个时间点是否已记录
		if !ok {
			cacheItem = &TimeCacheItem{Cache: make(map[string]interface{})}
			this.caches[rs.Time] = cacheItem
			cacheItem.ExpiredTime = time.Now().Add(time.Minute * common.StoreMinute)
		}

		for _, vs := range rs.Rows { //分析每一条数据
			row := rs.CreateDataRow(vs)
			key := row.GetKey(this.KeyName)
			//log.Infoln("add to cache", key)
			this.lock.Lock()
			if r, ok := cacheItem.Cache[key]; ok { //如果同类数据已存在,就进行数据合并
				row.Merge(r.(*common.DataRow).Row)
			}
			cacheItem.Cache[key] = row
			this.lock.Unlock()
			//log.Infoln(key, row.Row)
			//	log.Info("key=%s", key)
			go saveToStore(FormatStoreKey(this.Id, key), row, period*common.StoreCount) //保存到外部存贮,并保存粒度*保存个数分钟
		}

		//通知外到有新数据到来
		if TimeChanged != nil {
			TimeChanged(&common.TimeMessage{Key: this.Id, Time: currTime})
		}
	}
}
Exemple #4
0
func (this *InputFace) InitColumn(c *config.Expression, k []string) {
	this.RowMap = new(common.DataRowMap)
	if c == nil {
		log.Infoln("output column is nil")
		return
	}
	i := 0
	this.RowMap.Key = make(map[string]int)
	if c.Sum != nil {
		this.RowMap.Sum = make(map[string]int)
		for _, v := range c.Sum {
			this.RowMap.Sum[v] = i
			this.RowMap.Key[v] = i
			i = i + 1
		}
	}
	if c.Max != nil {
		this.RowMap.Max = make(map[string]int)
		for _, v := range c.Max {
			this.RowMap.Key[v] = i
			this.RowMap.Max[v] = i
			i = i + 1
		}
	}
	if c.Min != nil {
		this.RowMap.Min = make(map[string]int)
		for _, v := range c.Min {
			this.RowMap.Key[v] = i
			this.RowMap.Min[v] = i
			i = i + 1
		}
	}

	if k != nil {
		for _, v := range k {
			this.RowMap.Key[v] = i
			i = i + 1
		}
	}
}
Exemple #5
0
//run service
func (this *TestInput) Run() {
	log.Infoln("TestInput is starting")
	g := float64(this.Config.Period)
	for common.IsRun {
		beginget := time.Now()
		for i := 0; i < 60; i++ {
			row := new(common.DataRows)
			row.RowMap = this.RowMap
			if this.Config.OutputColumn == nil {
				log.Infoln("output column is nil")
				break
			}
			rnd := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
			for j := 0; j < 100000; j++ {
				var r []interface{}
				if this.Config.OutputColumn.Sum != nil {
					for k := 0; k < len(this.Config.OutputColumn.Sum); k++ {
						r = append(r, rnd.Float32())
					}
				}
				if this.Config.OutputColumn.Max != nil {
					for k := 0; k < len(this.Config.OutputColumn.Max); k++ {
						r = append(r, rnd.Float32())
					}
				}
				if this.Config.OutputColumn.Min != nil {
					for k := 0; k < len(this.Config.OutputColumn.Min); k++ {
						r = append(r, rnd.Float32())
					}
				}

				if this.Config.OutputKey != nil {
					for _, k := range this.Config.OutputKey {
						var v interface{}
						if k == this.Config.TimeKey {
							v = time.Now()
						} else {
							v = i * j
						}
						r = append(r, v)
					}
				}
				row.Rows = append(row.Rows, r)
			}

			row.Time = this.TIME.Format(this.Config.TimeFormat)

			//log.Infof("testinput 正在输出数据...%v", row.Time)
			this.Output <- row

		}
		//计算下一次运行的时间

		this.TIME = this.TIME.Add(time.Minute * time.Duration(g))
		log.Infof("next time is %v", this.TIME)
		//计算时间间隔
		sec := time.Since(beginget).Seconds()

		//计算下一次运行的时间
		if sec < g*60 { //如果一个处理周期内处理完,就延时,没有处理完就立即执行,如果正式运行期间还需要进行时间纠正
			log.Infof("sleep seconds %f", (g*60)-sec)
			time.Sleep(time.Second * (time.Duration(g*60 - sec))) //sleep one minute-sec
		}

	}
	close(this.Output)
}
Exemple #6
0
//通知所有客户端有时间消息
func SendTimeMessageToAll(tm *common.TimeMessage) {
	log.Infoln("send message is ", tm.Time, tm.Key, len(onlineClient))
	for _, c := range onlineClient {
		go c.Processing(tm)
	}
}