// 针对1个号段，对指定日期内的所有表，执行create或者drop操作
func execute(user string, passwd string, port string, uin int, host string,
dates []string, sqlTemp map[string]string) {
dbname := fmt.Sprintf("petLog_%d", uin)
dns := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=gbk", user, passwd, host, port, dbname)
conn, err := sql.Open("mysql", dns)
if err != nil {
util.AgentWarn("logAdmin, sql.Open")
log.Panicf("ERR: sql.Open, %s, %s", dns, err)
}
for _, date := range dates {
for _, temp := range sqlTemp {
sqlstr := fmt.Sprintf(temp, uin, date)
stmt, err := conn.Prepare(sqlstr)
if err != nil {
util.AgentWarn("logAdmin, conn.Prepare")
log.Panicf("ERR, conn.Prepare, %s, sql=%s", err, sqlstr)
}
_, err = stmt.Exec()
if err != nil {
util.AgentWarn("logAdmin, stmt.Exec")
log.Panicf("ERR, stmt.Exec, %s, sql=%s", err, sqlstr)
}
log.Printf("INF, host=%s, sql=%s", host, sqlstr)
stmt.Close()
}
}
log.Printf("INF: execute, host=%s, uin=%d", host, uin)
conn.Close()
}
// 检查表是否成功创建，如果失败，产生Agent告警
func checkTbl(ctx *cli.Context) {
tomorrow := time.Now().AddDate(0, 0, 1)
datestr := tomorrow.Format("20060102")
jsonconf := prod.NewJsonConf(ctx.String("json"))
table := ctx.String("table")
// 最后1个号段的db
dbname := "petLog_99"
user := jsonconf.Default_db_user
passwd := jsonconf.Default_db_pw
port := jsonconf.Default_db_port
// 配置文件中最后一台机器
host := jsonconf.Route_list[len(jsonconf.Route_list)-1].Db_host
dns := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=gbk", user, passwd, host, port, dbname)
conn, err := sql.Open("mysql", dns)
if err != nil {
log.Panicf("ERR: sql.Open, %s, %s", dns, err)
}
expectTable := fmt.Sprintf("%s_%s", table, datestr)
sqlstr := fmt.Sprintf("select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s'",
dbname, expectTable)
rows, err := conn.Query(sqlstr)
if err != nil {
util.AgentWarn(fmt.Sprintf("logAdmin, check, conn.Query, %s, %s, %s", dbname, table, datestr))
log.Panic("ERR: db.Query, %s", err)
}
log.Printf("INF: host=%s, sql=%s", host, sqlstr)
result := false
for rows.Next() {
var tablename string
err = rows.Scan(&tablename)
if err != nil {
log.Panic("ERR: rows.Scan, %s", err)
break
}
if tablename == expectTable {
result = true
break
}
}
if !result {
util.AgentWarn("logAdmin, check, fail to create tomorrow table")
}
conn.Close()
}
func (p *DBPool) Start(jsonconf *prod.JsonConf) {
user := jsonconf.Default_db_user
passwd := jsonconf.Default_db_pw
port := jsonconf.Default_db_port
for _, v := range jsonconf.Route_list {
host := v.Db_host
for i := v.UinStartInt; i <= v.UinEndInt; i++ {
dbname := fmt.Sprintf("petLog_%d", i)
dns := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=gbk", user, passwd, host, port, dbname)
c, err := sql.Open("mysql", dns)
if err != nil {
log.Fatal("ERR, sql.Open, %s, %s", dns, err)
}
if p.ping(dbname, c) {
log.Printf("INF: db '%s' ping ok", dbname)
p.conns[i] = c
} else {
log.Fatal("ERR, PingDB, dns=%s", dns)
}
}
}
go func() {
for {
it, ok := <-p.InputChan
if !ok {
log.Println("INF, dbpool loop break")
// 正常停止
break
}
if !p.dump {
res, err := p.conns[it.dbIdx].Exec(it.sql)
if err != nil {
errstr := fmt.Sprintf("ERR: logcons, conn.Exec, %s, db=%d, sql=%s", err, it.dbIdx, it.sql)
log.Printf(errstr)
util.AgentWarn(errstr)
p.dump = true
p.FailChan <- true
p.dump2file(it.dbIdx, it.sql)
continue
}
rows, _ := res.RowsAffected()
log.Printf("INF: db=%d, table=%s, RowsAffected=%d", it.dbIdx, it.table, rows)
} else {
p.dump2file(it.dbIdx, it.sql)
}
}
}()
}
// 读取1个日志文件
func (s *ProdServer) readOne(file string, now *time.Time, count *int) {
f, err := os.Open(file)
if err != nil {
log.Printf("WAR: fail to open file: %s", file)
return
}
defer f.Close()
reader := bufio.NewReader(f)
rows := 0
validLog := 0
// 各日志的数量
stat := make(map[int]int)
locate, _ := time.LoadLocation("Asia/Shanghai")
for {
LoopNextLine:
line, err := reader.ReadString('\n')
if err != nil {
if err != io.EOF {
log.Printf("WAR: fail to read file, %s, %s", file, err.Error())
}
break
}
line = strings.TrimSuffix(line, "\n")
rows += 1
parts := strings.Split(line, "|")
// 至少有10个部分
if len(parts) < 10 {
continue
}
// 第1个是时间戳
// 判断是否在时间范围内
pt2 := strings.Split(parts[0], ".")
if len(pt2) != 2 {
log.Printf("WAR: invalid date, %s:%d, %s", file, rows, parts[0])
continue
}
// 本地时间parse
const layout = "2006-01-02 15:04:05"
t, err := time.Parse(layout, pt2[0])
if err != nil {
log.Printf("WAR: parse date, %s:%d, %s", file, rows, parts[0])
continue
}
// parse转换默认是UTC, 这个接口有点awful
t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(),
t.Second(), t.Nanosecond(), locate)
if t.Before(s.lastRun) || t.After(*now) {
//log.Printf("WAR: time check, %s:%d, %s, %s, %s", file, rows, t, s.lastRun, *now)
continue
}
// 判断logid是否合法
logid, err := strconv.Atoi(parts[2])
if err != nil {
log.Printf("WAR: fail to get logid, %s:%d", file, rows)
continue
}
if !s.isValidLogid(logid) {
continue
}
// 辅助定位问题
modName := parts[6]
// 检查每行日志，是否满足表的字段的要求
conf := s.LogConfByID(logid)
for _, field := range conf.Field_list {
if field.FieldNumInt > len(parts) {
log.Printf("WAR: field invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
goto LoopNextLine
}
idx := field.FieldNumInt - 1
if field.TypeInt == FIELD_TYPE_INT {
_, err := strconv.Atoi(parts[idx])
if err != nil {
log.Printf("WAR: int invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
goto LoopNextLine
}
} else if field.TypeInt == FIELD_TYPE_UINT {
_, err := strconv.ParseUint(parts[idx], 10, 32)
if err != nil {
log.Printf("WAR: uint invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
goto LoopNextLine
}
} else if field.TypeInt == FIELD_TYPE_DATETIME {
// 约定：datetime类型, 位置只能在第1个字段
if idx != 0 {
log.Printf("WAR: datetime invalid, %s:%d, field=%d, logid=%d, modName=%s", file, rows, field.FieldNumInt, logid, modName)
goto LoopNextLine
}
}
}
// 计数+1
validLog++
tablePre := s.jsonconf.LogidSet[logid]
err = s.w.Publish(tablePre, []byte(line))
if err != nil {
util.AgentWarn(fmt.Sprintf("WAR: nsqd publish, %s", err))
log.Fatal("WAR: nsq publish, %s", err)
}
stat[logid]++
}
*count = *count + validLog
log.Printf("INF: file=%s, logs=%d, %v", file, validLog, stat)
return
}