Example #1
0
// 针对1个号段,对指定日期内的所有表,执行create或者drop操作
func execute(user string, passwd string, port string, uin int, host string,
	dates []string, sqlTemp map[string]string) {

	dbname := fmt.Sprintf("petLog_%d", uin)
	dns := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=gbk", user, passwd, host, port, dbname)
	conn, err := sql.Open("mysql", dns)
	if err != nil {
		util.AgentWarn("logAdmin, sql.Open")
		log.Panicf("ERR: sql.Open, %s, %s", dns, err)
	}
	for _, date := range dates {

		for _, temp := range sqlTemp {
			sqlstr := fmt.Sprintf(temp, uin, date)
			stmt, err := conn.Prepare(sqlstr)
			if err != nil {
				util.AgentWarn("logAdmin, conn.Prepare")
				log.Panicf("ERR, conn.Prepare, %s, sql=%s", err, sqlstr)
			}
			_, err = stmt.Exec()
			if err != nil {
				util.AgentWarn("logAdmin, stmt.Exec")
				log.Panicf("ERR, stmt.Exec, %s, sql=%s", err, sqlstr)
			}
			log.Printf("INF, host=%s, sql=%s", host, sqlstr)
			stmt.Close()
		}
	}
	log.Printf("INF: execute, host=%s, uin=%d", host, uin)
	conn.Close()
}
Example #2
0
// 检查表是否成功创建,如果失败,产生Agent告警
func checkTbl(ctx *cli.Context) {
	tomorrow := time.Now().AddDate(0, 0, 1)
	datestr := tomorrow.Format("20060102")

	jsonconf := prod.NewJsonConf(ctx.String("json"))

	table := ctx.String("table")
	// 最后1个号段的db
	dbname := "petLog_99"
	user := jsonconf.Default_db_user
	passwd := jsonconf.Default_db_pw
	port := jsonconf.Default_db_port
	// 配置文件中最后一台机器
	host := jsonconf.Route_list[len(jsonconf.Route_list)-1].Db_host

	dns := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=gbk", user, passwd, host, port, dbname)
	conn, err := sql.Open("mysql", dns)
	if err != nil {
		log.Panicf("ERR: sql.Open, %s, %s", dns, err)
	}

	expectTable := fmt.Sprintf("%s_%s", table, datestr)
	sqlstr := fmt.Sprintf("select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s'",
		dbname, expectTable)
	rows, err := conn.Query(sqlstr)
	if err != nil {
		util.AgentWarn(fmt.Sprintf("logAdmin, check, conn.Query, %s, %s, %s", dbname, table, datestr))
		log.Panic("ERR: db.Query, %s", err)
	}

	log.Printf("INF: host=%s, sql=%s", host, sqlstr)
	result := false
	for rows.Next() {
		var tablename string
		err = rows.Scan(&tablename)
		if err != nil {
			log.Panic("ERR: rows.Scan, %s", err)
			break
		}
		if tablename == expectTable {
			result = true
			break
		}
	}
	if !result {
		util.AgentWarn("logAdmin, check, fail to create tomorrow table")
	}
	conn.Close()
}
Example #3
0
func (p *DBPool) Start(jsonconf *prod.JsonConf) {
	user := jsonconf.Default_db_user
	passwd := jsonconf.Default_db_pw
	port := jsonconf.Default_db_port

	for _, v := range jsonconf.Route_list {
		host := v.Db_host
		for i := v.UinStartInt; i <= v.UinEndInt; i++ {
			dbname := fmt.Sprintf("petLog_%d", i)
			dns := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=gbk", user, passwd, host, port, dbname)
			c, err := sql.Open("mysql", dns)
			if err != nil {
				log.Fatal("ERR, sql.Open, %s, %s", dns, err)
			}

			if p.ping(dbname, c) {
				log.Printf("INF: db '%s' ping ok", dbname)
				p.conns[i] = c
			} else {
				log.Fatal("ERR, PingDB, dns=%s", dns)
			}
		}
	}

	go func() {
		for {
			it, ok := <-p.InputChan
			if !ok {
				log.Println("INF, dbpool loop break")
				// 正常停止
				break
			}

			if !p.dump {
				res, err := p.conns[it.dbIdx].Exec(it.sql)
				if err != nil {
					errstr := fmt.Sprintf("ERR: logcons, conn.Exec, %s, db=%d, sql=%s", err, it.dbIdx, it.sql)
					log.Printf(errstr)
					util.AgentWarn(errstr)
					p.dump = true
					p.FailChan <- true
					p.dump2file(it.dbIdx, it.sql)
					continue
				}

				rows, _ := res.RowsAffected()
				log.Printf("INF: db=%d, table=%s, RowsAffected=%d", it.dbIdx, it.table, rows)
			} else {
				p.dump2file(it.dbIdx, it.sql)
			}
		}
	}()
}
Example #4
0
// 读取1个日志文件
func (s *ProdServer) readOne(file string, now *time.Time, count *int) {
	f, err := os.Open(file)
	if err != nil {
		log.Printf("WAR: fail to open file: %s", file)
		return
	}
	defer f.Close()

	reader := bufio.NewReader(f)
	rows := 0
	validLog := 0
	// 各日志的数量
	stat := make(map[int]int)

	locate, _ := time.LoadLocation("Asia/Shanghai")

	for {
	LoopNextLine:

		line, err := reader.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				log.Printf("WAR: fail to read file, %s, %s", file, err.Error())
			}
			break
		}

		line = strings.TrimSuffix(line, "\n")
		rows += 1
		parts := strings.Split(line, "|")
		// 至少有10个部分
		if len(parts) < 10 {
			continue
		}
		// 第1个是时间戳
		// 判断是否在时间范围内
		pt2 := strings.Split(parts[0], ".")
		if len(pt2) != 2 {
			log.Printf("WAR: invalid date, %s:%d, %s", file, rows, parts[0])
			continue
		}

		// 本地时间parse
		const layout = "2006-01-02 15:04:05"
		t, err := time.Parse(layout, pt2[0])
		if err != nil {
			log.Printf("WAR: parse date, %s:%d, %s", file, rows, parts[0])
			continue
		}
		// parse转换默认是UTC, 这个接口有点awful
		t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), locate)

		if t.Before(s.lastRun) || t.After(*now) {
			//log.Printf("WAR: time check, %s:%d, %s, %s, %s", file, rows, t, s.lastRun, *now)
			continue
		}

		// 判断logid是否合法
		logid, err := strconv.Atoi(parts[2])
		if err != nil {
			log.Printf("WAR: fail to get logid, %s:%d", file, rows)
			continue
		}
		if !s.isValidLogid(logid) {
			continue
		}

		// 辅助定位问题
		modName := parts[6]

		// 检查每行日志,是否满足表的字段的要求
		conf := s.LogConfByID(logid)
		for _, field := range conf.Field_list {
			if field.FieldNumInt > len(parts) {
				log.Printf("WAR: field invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
				goto LoopNextLine
			}
			idx := field.FieldNumInt - 1

			if field.TypeInt == FIELD_TYPE_INT {
				_, err := strconv.Atoi(parts[idx])
				if err != nil {
					log.Printf("WAR: int invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
					goto LoopNextLine
				}
			} else if field.TypeInt == FIELD_TYPE_UINT {
				_, err := strconv.ParseUint(parts[idx], 10, 32)
				if err != nil {
					log.Printf("WAR: uint invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
					goto LoopNextLine
				}
			} else if field.TypeInt == FIELD_TYPE_DATETIME {
				// 约定:datetime类型, 位置只能在第1个字段
				if idx != 0 {
					log.Printf("WAR: datetime invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
					goto LoopNextLine
				}
			}
		}

		// 计数+1
		validLog++
		tablePre := s.jsonconf.LogidSet[logid]
		err = s.w.Publish(tablePre, []byte(line))
		if err != nil {
			util.AgentWarn(fmt.Sprintf("WAR: nsqd publish, %s", err))
			log.Fatal("WAR: nsq publish, %s", err)
		}
		stat[logid]++
	}
	*count = *count + validLog
	log.Printf("INF: file=%s, logs=%d, %v", file, validLog, stat)

	return
}