/*将数据写入数据库*/
func Writer(dbIni *DBini, xmlInfo *XmlInfo) {
logger.Info("***************************")
logger.Info("Writer begin >>>>>>>>>>>>>>>>")
if len(xmlInfo.Datas) == 0 {
logger.Info("Writer end >>>>>>>>>>>>>>>>")
return
}
stmt, err := dbIni.T_Db.db.Prepare(xmlInfo.InsertSQL)
defer stmt.Close()
if err != nil {
logger.Error(err)
return
}
colsLength := len(xmlInfo.Columns)
c := 0
arrIds := make([]string, len(xmlInfo.Datas))
for _, v := range xmlInfo.Datas {
scans := make([]interface{}, colsLength+2)
for i := range xmlInfo.Columns {
scans[i] = v[xmlInfo.Columns[i]]
}
scans[colsLength] = dbIni.SysId
scans[colsLength+1] = dbIni.SysName
arrIds[c] = v[xmlInfo.Columns[0]]
stmt.Exec(scans...)
c++
}
xmlInfo.Datas = nil
xmlInfo.Ids = arrIds
logger.Info("Writer end >>>>>>>>>>>>>>>>")
}
/*复制文件 如果存在则不再复制*/
func copyFile(src, dest string) error {
srcFile, e := os.Open(src)
if e != nil {
logger.Error("copyFile:", e.Error())
return e
}
defer srcFile.Close()
df, e := os.Open(dest)
defer df.Close()
if e != nil && os.IsNotExist(e) {
logger.Info("desIniFile:", dest, "file not exist!")
destFile, e := os.Create(dest)
if e != nil {
logger.Error("copyFile destFile:", e.Error())
return e
}
defer destFile.Close()
io.Copy(destFile, srcFile)
logger.Info("copy file from :", src, " to ", dest)
} else {
logger.Info("desIniFile:", dest, "已存在")
}
return nil
}
/*执行数据同步读取直接写入 single data 单次执行可变的业务数据 */
func (xmlInfo *XmlInfo) RunSingleDataPipe(dbIni *DBini) {
logger.Info("All run begin >>>>>>>>>>>>>>>>")
logger.Info("dbIni.F_DbType:", dbIni.F_DbType, "dbIni.T_DbType:", dbIni.T_DbType)
if !StartFlag {
return
}
defer func() {
if err := recover(); err != nil {
logger.Error("查询数据库失败", err)
}
}()
f_pwd, _ := Base64Dec(dbIni.F_Password)
t_pwd, _ := Base64Dec(dbIni.T_Password)
dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
dbIni.T_Db.Db_open(dbIni.T_DbType, dbIni.T_User, t_pwd, dbIni.T_Ip, dbIni.T_Port, dbIni.T_Dbname)
//关闭数据库
defer dbIni.F_Db.db.Close()
defer dbIni.T_Db.db.Close()
if dbIni.F_Db.db == nil || dbIni.T_Db.db == nil {
logger.Error("无法链接两方数据库")
panic("无法链接两方数据库")
return
}
DataSinglePipe(dbIni, xmlInfo)
Updater(dbIni, xmlInfo)
ExecProc(dbIni, xmlInfo)
logger.Info("All run end >>>>>>>>>>>>>>>>")
time.Sleep(5 * time.Second)
}
/*判断是否需要添加标记位，如需要添加标记位*/
func Alter(dbIni *DBini, xmlInfo *XmlInfo) {
if len(xmlInfo.CheckColumnSQL) > 0 {
defer func() {
if err := recover(); err != nil {
logger.Error("数据库执行失败", err)
}
}()
f_pwd, _ := Base64Dec(dbIni.F_Password)
dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
defer dbIni.F_Db.db.Close()
query, err := dbIni.F_Db.db.Query(strings.Replace(xmlInfo.CheckColumnSQL, "@[dbname]", dbIni.F_Dbname, -1)) //查询数据库
defer query.Close()
if err != nil {
logger.Error("查询数据库失败", err.Error())
panic(err)
return
}
testQuery, err := dbIni.F_Db.db.Query("select 1")
if err != nil {
logger.Error("数据库连接异常,无法查询测试数据")
panic(err)
return
} else {
if !testQuery.Next() {
logger.Error("数据库连接异常,无法遍历测试数据")
panic(err)
return
}
defer testQuery.Close()
}
if query.Next() {
logger.Info(xmlInfo.Remark, "经检测，标记位已添加")
xmlInfo.AlterFlag = true
} else {
stmt, err := dbIni.F_Db.db.Prepare(xmlInfo.AlterSQL)
defer stmt.Close()
if err != nil {
logger.Error(err)
return
}
stmt.Exec()
logger.Info(xmlInfo.Remark, "在源数据库中添加成功")
updateStmt, err := dbIni.F_Db.db.Prepare(xmlInfo.UpdateFlagSQL)
defer updateStmt.Close()
if err != nil {
logger.Error(err)
return
}
updateStmt.Exec()
logger.Info(xmlInfo.Remark, "在源数据库中更新成功")
xmlInfo.AlterFlag = true
}
} else {
logger.Info(xmlInfo.Remark, "无需检测标记位")
xmlInfo.AlterFlag = true
}
}
/*启动组织机构同步服务*/
func (configinfo *ConfigInfo) StartOrgSync() {
logger.Info(" StartOrgSync is run... ")
var orgVersionUrl = "http://" + configinfo.core_ip + ":" + configinfo.core_port + configinfo.orgversion_url
var allOrgUrl = "http://" + configinfo.core_ip + ":" + configinfo.core_port + configinfo.allorg_url
logger.Info("orgVersionUrl start >>>>>>>>>>>>>>>>", orgVersionUrl)
logger.Info("allOrgUrl start >>>>>>>>>>>>>>>>", allOrgUrl)
//发起链接访问，获取返回的json数据
for {
var return_orgs []byte
var return_version []byte
var tmpOrgVersion string = ""
if response, err := http.Get(orgVersionUrl); err != nil {
logger.Error(" err = ", err)
} else {
defer response.Body.Close()
if return_version, err = ioutil.ReadAll(response.Body); err != nil {
logger.Error(" err = ", err)
}
tmpOrgVersion = string(return_version)
}
logger.Info(" return_version = ", tmpOrgVersion)
if len(OrgVersion) == 0 || (len(tmpOrgVersion) > 0 && !strings.EqualFold(tmpOrgVersion, OrgVersion)) {
OrgVersion = tmpOrgVersion
//重新加载组织机构
if response, err := http.Get(allOrgUrl); err != nil {
logger.Error(" err = ", err)
} else {
defer response.Body.Close()
if return_orgs, err = ioutil.ReadAll(response.Body); err != nil {
logger.Error(" err = ", err)
}
}
var orgInfos []OrgInfo
err := json.Unmarshal(return_orgs, &orgInfos)
if err != nil {
logger.Error(" err = ", err)
}
if len(orgInfos) > 0 {
for k, _ := range OrgMap {
delete(OrgMap, k)
}
for k, _ := range OrgNameMap {
delete(OrgNameMap, k)
}
for i := range orgInfos {
org := &orgInfos[i]
OrgMap[org.Org_path] = org.Id
OrgMap[org.Name] = org.Id
OrgNameMap[org.Id] = org.Name
}
}
}
time.Sleep(Time_interval * time.Second)
}
}
/*主入口*/
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
//指定是否控制台打印，默认为true
logger.SetConsole(true)
//指定日志级别 ALL，DEBUG，INFO，WARN，ERROR，FATAL，OFF 级别由低到高
logger.SetLevel(logger.INFO)
logger.SetRollingFile("./log", "datapostlog.log", 100, 50, logger.MB)
if beego.AppConfig.String("runmode") == "dev" {
orm.Debug = true
}
//支持异构版本升级
pipe.Upgrade()
time.Sleep(3 * time.Second)
//配置文件初始化
err := pipe.InitFile()
if err != nil {
logger.Error("InitFile():", err)
}
//注册运行服务
ServiceHandle(cSvcName, cSvcName, cSvcName, func() {
//初始化运行参数
pipe.InitConfigMap()
//运行参数管理UI服务
beego.Router("/", &pipe.IndexInfo{}, "*:Index")
beego.Router("/index/list", &pipe.IndexInfo{}, "*:List")
beego.Router("/index/edit", &pipe.IndexInfo{}, "*:Edit")
beego.Router("/index/test", &pipe.IndexInfo{}, "*:Test")
beego.Router("/index/help", &pipe.IndexInfo{}, "*:Help")
beego.Router("/index/treedata", &pipe.IndexInfo{}, "*:TreeData")
//启动数据同步
go pipe.StartAllSys()
//启动https服务
go pipe.Https_start()
//启动参数管理UI 服务
beego.Run()
logger.Info("DaemonName=", cSvcName, "Daemon started.")
for {
time.Sleep(time.Hour)
}
}, func() {
Stop()
logger.Info("DaemonName=", cSvcName, "Daemon stoped.")
})
}
func (p *Workshop) GC() {
runtime.ReadMemStats(&(p.ms))
alloc := p.ms.Alloc / 1024 / 1024
logger.Info("memAlloc: ", alloc, "M heapAlloc: ", p.ms.HeapAlloc/1024/1024, "M, stackAlloc: ", p.ms.StackInuse/1024/1024, "M")
if alloc >= p.memLimit {
debug.FreeOSMemory()
runtime.ReadMemStats(&(p.ms))
alloc = p.ms.Alloc / 1024 / 1024
logger.Info("after GC memAlloc: ", alloc, "M heapAlloc: ", p.ms.HeapAlloc/1024/1024, "M, stackAlloc: ", p.ms.StackInuse/1024/1024, "M")
}
}
func (protocol *TransProtocolComm) Decode_pack(data []byte) (*Packet, int) {
log.Info("Enter Decode_pack")
decodeLen := 0
pack := &Packet{}
dataLen := len(data)
offset := 0
headLen := binary.Size(pack._header)
//1.get packhader
//= not cause out of memory so >= is needed
for offset = 0; dataLen-offset >= headLen; offset++ {
pack._header._ver = int16(binary.BigEndian.Uint16(data[offset : offset+2]))
pack._header._seq = int32(binary.BigEndian.Uint32(data[offset+2 : offset+6]))
pack._header._type = int16(binary.BigEndian.Uint16(data[offset+6 : offset+8]))
pack._header._bodyLen = int32(binary.BigEndian.Uint32(data[offset+8 : offset+12]))
//get pack header successful
if int16(VER) == pack._header._ver && pack._header._bodyLen < int32(MAX_PACK_SIZE) {
break
}
}
//2.check is a complete Packet
if dataLen-offset < headLen || dataLen-offset < int(pack._header._bodyLen) {
//can't find VER in data Slice or is not a complete pack.
log.Error("[PACK_NOT_COMPLETE],pack bodyLen is", pack._header._bodyLen)
return nil, 0
}
//3.get a complete pack
pack._data = make([]byte, pack._header._bodyLen)
copy(pack._data, data[offset:])
decodeLen = offset + headLen + int(pack._header._bodyLen)
return pack, decodeLen
}
/*启动所有配置信息*/
func StartAllSys() {
dirs := strings.Split(ConfigMap["transferes"], ",")
nosqlDir := ConfigMap["nosql"]
logger.Info("nosqlDir >>>>>>>>>>>>>>>>", nosqlDir)
StartFlag = true
//nosql目录
if len(nosqlDir) > 0 {
noSqlDbIni := GetSysIni(nosqlDir)
noSqlDbIni.RegisterSys()
go noSqlDbIni.StartNoSqlSys()
}
for i := range dirs {
dirName := dirs[i]
if len(dirName) > 0 {
sysIni := GetSysIni(dirName)
if i == 0 {
go sysIni.StdOrgSync()
}
//非nosql目录
sysIni.RegisterSys()
go sysIni.StartSys()
}
}
for {
time.Sleep(2 * time.Second)
}
}
func log(i int) {
logger.Debug("Debug>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Info("Info>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Warn("Warn>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Error("Error>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Fatal("Fatal>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
}
func StopWorkShop() error {
WSChan_ <- CMD_EXIT
StopWorkShop_LOOP:
for {
select {
case ret := <-WSReportChan_:
if ret == RET_EXIT_SUCCESS {
logger.Info("workshop exit success")
} else {
logger.Info("workshop exit fail")
}
}
break StopWorkShop_LOOP
}
return nil
}
/*判断是否需要查询数据*/
func Check(dbIni *DBini, xmlInfo *XmlInfo) {
if strings.EqualFold(xmlInfo.IsValData, "true") {
xmlInfo.CheckFlag = true
return
}
if len(xmlInfo.CheckRowSQL) > 0 {
defer func() {
if err := recover(); err != nil {
logger.Error("查询数据库失败", err)
}
}()
f_pwd, _ := Base64Dec(dbIni.F_Password)
dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
defer dbIni.F_Db.db.Close()
query, err := dbIni.F_Db.db.Query(strings.Replace(xmlInfo.CheckRowSQL, "@[dbname]", dbIni.F_Dbname, -1)) //查询数据库
defer query.Close()
if err != nil {
logger.Error("查询数据库失败", err.Error())
xmlInfo.CheckFlag = false
panic(err)
return
}
cols, _ := query.Columns()
values := make([][]byte, len(cols)) //values是每个列的值
scans := make([]interface{}, len(cols)) //query.Scan的参数 用len(cols)定住当次查询的长度
for i := range values { //让每一行数据都填充到[][]byte里面
scans[i] = &values[i]
}
dbRows := 0
if query.Next() {
//判断是否与上次的数据一致，如果不一致则查询数据库，否则不查询
if err := query.Scan(scans...); err != nil { //query.Scan查询出来的不定长值放到scans[i] = &values[i],也就是每行都放在values里
logger.Error(err)
xmlInfo.CheckFlag = false
return
}
for _, v := range values {
intV, err := strconv.Atoi(string(v))
if err != nil {
xmlInfo.CheckFlag = false
} else {
dbRows = intV
}
}
if dbRows != 0 && dbRows != xmlInfo.RowNum { //获取reader 如何获取到数据则置为0，如果获取不到了则赋予全局GobalNum
xmlInfo.RowNum = dbRows
xmlInfo.GobalNum = dbRows
xmlInfo.CheckFlag = true
} else {
xmlInfo.CheckFlag = false
}
} else {
xmlInfo.CheckFlag = true
}
} else {
logger.Info(xmlInfo.Remark, "无需查询系统库行记录是否变化")
xmlInfo.CheckFlag = true
}
}
/*启动*/
func (sys *SysIni) StartSys() {
fmt.Println("SysIni is start................")
StartFlag = true
if StartFlag { //判断标记位并增加标记位
for i := range sys.xmlInfoes {
Alter(&sys.dbini, &sys.xmlInfoes[i])
logger.Info("Alter >>>>>>>>>>>>>>>>:", i)
}
}
logger.Info("StartSys >>>>>>>>>>>>>>>>goto for ")
//单次同步
c := cron.New()
c.Start()
defer c.Stop()
fmt.Println("singletimer>>>>", sys.configInfo.singletimer)
//单次同步
for j := range sys.singleXmlInfoes {
go func(sys *SysIni, j int) {
singleXmlInfo := sys.singleXmlInfoes[j]
c.AddFunc(sys.configInfo.singletimer,
func() {
logger.Info("single[", singleXmlInfo.FileName, "] job is run ......................")
singleXmlInfo.RunSingleDataPipe(&sys.dbini)
},
singleXmlInfo.FileName)
}(sys, j)
}
//实时同步
for StartFlag {
//边读边写，更新
for i := range sys.xmlInfoes {
//logger.Info("sys.xmlInfoes[i].NotExistData >>>>>>>>>>>>>>>>", sys.xmlInfoes[i].Remark, sys.xmlInfoes[i].NotExistData)
if sys.xmlInfoes[i].NotExistData {
Check(&sys.dbini, &sys.xmlInfoes[i])
}
if sys.xmlInfoes[i].CheckFlag {
sys.xmlInfoes[i].RunDataPipe(&sys.dbini)
}
}
time.Sleep(2 * time.Second)
}
}
/*执行数据同步*/
func (xmlInfo *XmlInfo) Run(dbIni *DBini) {
logger.Info("All run begin >>>>>>>>>>>>>>>>")
f_pwd, _ := Base64Dec(dbIni.F_Password)
t_pwd, _ := Base64Dec(dbIni.T_Password)
dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
dbIni.T_Db.Db_open(dbIni.T_DbType, dbIni.T_User, t_pwd, dbIni.T_Ip, dbIni.T_Port, dbIni.T_Dbname)
//关闭数据库
defer dbIni.F_Db.db.Close()
defer dbIni.T_Db.db.Close()
if dbIni.F_Db.db == nil || dbIni.T_Db.db == nil {
logger.Error("无法链接两方数据库")
return
}
Reader(dbIni, xmlInfo)
Writer(dbIni, xmlInfo)
Updater(dbIni, xmlInfo)
logger.Info("All run end >>>>>>>>>>>>>>>>")
time.Sleep(5 * time.Second)
}
/*访问数据库获取数据*/
func Reader(dbIni *DBini, xmlInfo *XmlInfo) {
logger.Info("***************************")
logger.Info("Reader begin >>>>>>>>>>>>>>>>")
query, err := dbIni.F_Db.db.Query(xmlInfo.DataSQL) //查询数据库
if err != nil {
logger.Error("查询数据库失败", err.Error())
return
}
defer query.Close()
cols, _ := query.Columns()
values := make([][]byte, len(cols)) //values是每个列的值
scans := make([]interface{}, len(cols)) //query.Scan的参数 用len(cols)定住当次查询的长度
for i := range values { //让每一行数据都填充到[][]byte里面
scans[i] = &values[i]
}
results := make(map[int]map[string]string) //最后得到的map
i := 0
for query.Next() { //循环
if err := query.Scan(scans...); err != nil { //query.Scan查询出来的不定长值放到scans[i] = &values[i],也就是每行都放在values里
logger.Error(err)
return
}
row := make(map[string]string) //每行数据
for k, v := range values {
key := cols[k]
row[key] = string(v)
}
results[i] = row //装入结果集中
i++
}
if i == 0 {
xmlInfo.RowNum = xmlInfo.GobalNum
} else {
xmlInfo.RowNum = 0
}
xmlInfo.Datas = results
xmlInfo.Columns = cols
logger.Info("Reader end >>>>>>>>>>>>>>>>")
}
//indexinfo to ini
func IndexInfoToIni(this *IndexInfo, cfg *goconfig.Config) {
f_dbname, _ := cfg.Get("dbconfig::f_dbname")
t_dbname, _ := cfg.Get("dbconfig::t_dbname")
f_dbtype, _ := cfg.Get("dbconfig::f_dbtype")
f_driver, _ := cfg.Get("dbconfig::f_driver")
t_dbtype, _ := cfg.Get("dbconfig::t_dbtype")
t_driver, _ := cfg.Get("dbconfig::t_driver")
var f_dburl string = ""
if strings.EqualFold(f_dbtype, "mysql") {
f_dburl_arr := []string{"jdbc:mysql://", this.F_ip, ":", this.F_port, "/", f_dbname, "?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
f_dburl = strings.Join(f_dburl_arr, "")
} else if strings.EqualFold(f_dbtype, "mssql") {
f_dburl_arr := []string{"jdbc:microsoft:sqlserver://", this.F_ip, ":", this.F_port, ";", "DatabaseName=", f_dbname}
f_dburl = strings.Join(f_dburl_arr, "")
}
t_dburl_arr := []string{"jdbc:mysql://", this.T_ip, ":", this.T_port, "/", t_dbname, "?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
var t_dburl = strings.Join(t_dburl_arr, "")
cfg.Set("dbconfig", "f_dbtype", f_dbtype)
cfg.Set("dbconfig", "f_driver", f_driver)
cfg.Set("dbconfig", "f_dburl", f_dburl)
cfg.Set("dbconfig", "f_ip", this.F_ip)
cfg.Set("dbconfig", "f_port", this.F_port)
cfg.Set("dbconfig", "f_dbname", f_dbname)
cfg.Set("dbconfig", "f_user", this.F_username)
cfg.Set("dbconfig", "f_password", this.F_password)
cfg.Set("dbconfig", "t_dbtype", t_dbtype)
cfg.Set("dbconfig", "t_driver", t_driver)
cfg.Set("dbconfig", "t_dburl", t_dburl)
cfg.Set("dbconfig", "t_ip", this.T_ip)
cfg.Set("dbconfig", "t_port", this.T_port)
cfg.Set("dbconfig", "t_dbname", t_dbname)
cfg.Set("dbconfig", "t_user", this.T_username)
cfg.Set("dbconfig", "t_password", this.T_password)
sysid, _ := cfg.Get("dbconfig::sysid")
systype, _ := cfg.Get("dbconfig::systype")
sysname, _ := cfg.Get("dbconfig::sysname")
company, _ := cfg.Get("dbconfig::company")
platform, err := cfg.Get("dbconfig::platform")
logger.Info("platform===========================", platform, sysname, err)
cfg.Set("dbconfig", "sysid", sysid)
cfg.Set("dbconfig", "systype", systype)
cfg.Set("dbconfig", "sysname", sysname)
cfg.Set("dbconfig", "company", company)
cfg.Set("dbconfig", "platform", platform)
}
func main() {
if len(os.Args) < 3 {
fmt.Fprintf(os.Stderr, "Usage: $0 -c config_file [-t]\n")
os.Exit(1)
}
flag.StringVar(&configFile_, "c", "conf.json", "config file in json")
flag.BoolVar(&isTest, "t", false, "test mode")
flag.Parse()
// SIGHUP to reload configFile, SIGTERM/SIGINT to exit the process
sigChan := make(chan os.Signal, 10)
signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
// start worshop for the first time
if err := StartNewWorkShop(isTest); err != nil {
fmt.Fprintf(os.Stderr, "workshop init fail [%+v]\n", err)
os.Exit(1)
}
if isTest {
os.Exit(0)
}
// loop. wait for signals
MAIN_LOOP:
for {
select {
case sig := <-sigChan:
fmt.Fprintf(os.Stderr, "receive sig [%+v]\n", sig)
// logger.Error("receive sig ", sig)
if sig == syscall.SIGHUP {
// wait for workshop to do some clean-up
StopWorkShop()
ticker_.Stop()
} else if sig == syscall.SIGTERM || sig == syscall.SIGINT {
StopWorkShop()
ticker_.Stop()
break MAIN_LOOP
} else {
logger.Error("main routine receive unexpected signal [", sig, "], this should NEVER happen")
os.Exit(1)
}
case <-ticker_.C:
WSChan_ <- CMD_TICK
logger.Debug("main_routine tick")
}
}
logger.Info("the qtask application exit")
}
/*回写标记位*/
func Updater(dbIni *DBini, xmlInfo *XmlInfo) {
logger.Info("***************************")
logger.Info("Updater begin >>>>>>>>>>>>>>>>")
defer func() {
if err := recover(); err != nil {
logger.Error("数据库执行失败", err)
}
}()
if len(xmlInfo.SuccessSQL) > 0 {
var buffer bytes.Buffer
idsLength := len(xmlInfo.Ids)
idsLength--
for i := range xmlInfo.Ids {
if i == idsLength {
buffer.WriteString("'")
buffer.WriteString(xmlInfo.Ids[i])
buffer.WriteString("'")
} else {
buffer.WriteString("'")
buffer.WriteString(xmlInfo.Ids[i])
buffer.WriteString("',")
}
}
if len(buffer.String()) > 0 {
stmt, err := dbIni.F_Db.db.Prepare(strings.Replace(xmlInfo.SuccessSQL, "@[id]", buffer.String(), -1))
defer stmt.Close()
if err != nil {
logger.Error(err)
panic(err)
return
}
stmt.Exec()
}
}
logger.Info("Updater end >>>>>>>>>>>>>>>>")
}
func (p *Workshop) closeSubRoutines(ctrlChan chan int, reportChan chan int, n int) {
p.sendCtrlInfo(ctrlChan, CMD_EXIT, n)
closeSuccess := 0
i := 0
for i < n {
select {
case ret := <-reportChan:
i++
if ret == RET_EXIT_SUCCESS {
closeSuccess++
}
}
}
logger.Info("close ", closeSuccess, "/", n, " subRoutines")
}
/*已匹配组织机构信息*/
func (sys *SysIni) StdOrgSync() {
logger.Info("StdOrgSync start >>>>>>>>>>>>>>>>")
defer func() {
if err := recover(); err != nil {
logger.Error("查询数据库失败", err)
}
}()
for {
dbIni := &sys.dbini
t_pwd, _ := Base64Dec(dbIni.T_Password)
dbIni.T_Db.Db_open(dbIni.T_DbType, dbIni.T_User, t_pwd, dbIni.T_Ip, dbIni.T_Port, dbIni.T_Dbname)
defer dbIni.T_Db.db.Close()
query, err := dbIni.T_Db.db.Query(MapQuerySQL) //查询数据库
if err != nil {
logger.Error("查询数据库失败", err.Error())
panic(err)
return
}
defer query.Close()
for k, _ := range StdOrgMap {
delete(StdOrgMap, k)
}
cols, _ := query.Columns()
colsLength := len(cols)
values := make([][]byte, colsLength)
scans := make([]interface{}, colsLength)
for i := range values {
scans[i] = &values[i]
}
for query.Next() { //循环
if err := query.Scan(scans...); err != nil {
logger.Error(err)
return
}
row := make(map[string]string) //每行数据
for k, v := range values {
key := cols[k]
row[key] = string(v)
}
StdOrgMap[row["map_deptname"]] = row["std_deptid"]
}
time.Sleep(Time_interval * time.Second)
}
}
//indexinfo to ini
func IndexInfoToIni(this *IndexInfo, cfg *goconfig.Config) {
f_dburl_arr := []string{"jdbc:mysql://", this.F_ip, ":", this.F_port, "/mid?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
var f_dburl = strings.Join(f_dburl_arr, "")
t_dburl_arr := []string{"jdbc:mysql://", this.T_ip, ":", this.T_port, "/rptdb?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
var t_dburl = strings.Join(t_dburl_arr, "")
f_dbname, _ := cfg.Get("dbconfig::f_dbname")
t_dbname, _ := cfg.Get("dbconfig::t_dbname")
f_dbtype, _ := cfg.Get("dbconfig::f_dbtype")
f_driver, _ := cfg.Get("dbconfig::f_driver")
t_dbtype, _ := cfg.Get("dbconfig::t_dbtype")
t_driver, _ := cfg.Get("dbconfig::t_driver")
cfg.Set("dbconfig", "f_dbtype", f_dbtype)
cfg.Set("dbconfig", "f_driver", f_driver)
cfg.Set("dbconfig", "f_dburl", f_dburl)
cfg.Set("dbconfig", "f_ip", this.F_ip)
cfg.Set("dbconfig", "f_port", this.F_port)
cfg.Set("dbconfig", "f_dbname", f_dbname)
cfg.Set("dbconfig", "f_user", this.F_username)
cfg.Set("dbconfig", "f_password", this.F_password)
cfg.Set("dbconfig", "t_dbtype", t_dbtype)
cfg.Set("dbconfig", "t_driver", t_driver)
cfg.Set("dbconfig", "t_dburl", t_dburl)
cfg.Set("dbconfig", "t_ip", this.T_ip)
cfg.Set("dbconfig", "t_port", this.T_port)
cfg.Set("dbconfig", "t_dbname", t_dbname)
cfg.Set("dbconfig", "t_user", this.T_username)
cfg.Set("dbconfig", "t_password", this.T_password)
sysid, _ := cfg.Get("dbconfig::sysid")
systype, _ := cfg.Get("dbconfig::systype")
sysname, _ := cfg.Get("dbconfig::sysname")
company, _ := cfg.Get("dbconfig::company")
platform, _ := cfg.Get("dbconfig::platform")
logger.Info("platform===========================", platform, sysname)
cfg.Set("dbconfig", "sysid", sysid)
cfg.Set("dbconfig", "systype", systype)
cfg.Set("dbconfig", "sysname", sysname)
cfg.Set("dbconfig", "company", company)
cfg.Set("dbconfig", "platform", platform)
}
func (p *Workshop) Run() {
LOOP:
for {
select {
case cmd := <-p.upCtrlChan:
if cmd == CMD_EXIT {
close(p.msgChan) // will send an empty msg to its receivers
p.closeSubRoutines(p.PRCtrlChan, p.PRReportChan, p.processorNum)
p.closeSubRoutines(p.CLCtrlChan, p.CLReportChan, p.collectorNum)
break LOOP
} else if cmd == CMD_TICK {
// logger.Debug("run_routine tick")
p.sendCtrlInfo(p.PRCtrlChan, cmd, p.processorNum)
p.sendCtrlInfo(p.CLCtrlChan, cmd, p.collectorNum)
p.cronTick++
if p.cronTick > p.cronTickInterval {
p.Tick()
p.cronTick = 0
}
} else {
logger.Error(" WorkShop receive a un-expected cmd ", cmd)
}
default:
messages, err := p.provider.GetNextMsg()
if err != nil {
logger.Error("src provider get msg err occur with ", len(messages), " msgs, err: ", err)
}
if messages == nil || len(messages) == 0 {
continue
}
for _, v := range messages {
if len(v) > 0 {
logger.Info("emit!")
p.msgChan <- string(v)
}
}
logger.Debug("main routine receive ", len(messages), " msgs from srcProvider")
p.processedLine += len(messages)
p.provider.Ack()
}
}
p.provider.Destory()
p.upReportChan <- RET_EXIT_SUCCESS
}
/*启动*/
func (sys *SysIni) StartNoSqlSys() {
fmt.Println("StartNoSqlSys is start................")
//单次同步
c := cron.New()
c.Start()
defer c.Stop()
for j := range sys.singleXmlInfoes {
singleXmlInfo := sys.singleXmlInfoes[j]
c.AddFunc(sys.configInfo.singletimer,
func() {
logger.Info("single[", singleXmlInfo.FileName, "] job is run ......................")
NoSqlSync(&sys.dbini, &sys.singleXmlInfoes[j])
},
singleXmlInfo.FileName)
}
for {
time.Sleep(2 * time.Second)
}
}
func (sock *TcpSocket) to_write() {
var pack *packet.Packet
for ESTABLISHED == sock._socketState {
pack = <-sock._writerChannel
if pack == nil {
log.Error("get pack error")
continue
}
data := pack.Serialize()
n, err := sock._conn.Write(data)
if err != nil {
if io.ErrShortWrite != err {
log.Error("write err close--", n)
sock.closeSock()
break
}
}
log.Info("send data success")
}
}
//编辑
func (this *IndexInfo) Edit() {
var (
config = new(IndexInfo)
err error
ret Ret
flag bool
)
if err = json.Unmarshal([]byte(this.GetString("json")), &config); err == nil {
err = config.SaveSingleConf()
}
if err != nil {
logger.Error(err)
ret.Code, ret.Msg = 1, err.Error()
flag = false
logger.Info("Edit", err.Error())
} else {
ret.Code = 0
flag = true
}
this.Data["json"] = &ret
this.ServeJson(flag)
}
func (p *PrintProcessor) Process(msg string) error {
logger.Info("receive: ", msg)
strArr := strings.Split(msg, "\n")
if len(strArr) <= 0 {
return nil
}
for _, str := range strArr {
str = strings.TrimSpace(str)
if str == "" {
continue
}
index := p.getCollectorIndex(len(p.itemChans))
p.itemChans[index] <- Item{
Category: p.category,
Id: "",
Content: "",
RawMsg: str,
}
}
return nil
}
/*执行数据处理*/
func ExecProc(dbIni *DBini, xmlInfo *XmlInfo) {
proname := xmlInfo.Proname
if strings.EqualFold(xmlInfo.Isexecpro, "true") && len(proname) > 0 {
var buffer bytes.Buffer
buffer.WriteString("call ")
buffer.WriteString(proname)
buffer.WriteString("()")
defer func() {
if err := recover(); err != nil {
logger.Error("查询数据库失败", err)
}
}()
logger.Info("执行存储过程 >>>>>>>>>>>>>>>>", buffer.String())
stmt, err := dbIni.T_Db.db.Prepare(buffer.String())
defer stmt.Close()
if err != nil {
logger.Error(err)
panic(err)
return
}
stmt.Exec()
}
}
func (p *Workshop) Init(ctx *sj.Json, upCtrlChan chan int, upReportChan chan int, isTest bool) error {
p.ctx = ctx
p.upCtrlChan = upCtrlChan
p.upReportChan = upReportChan
p.processedLine = 0
var err error
if err = p.initLog(isTest); err != nil {
return err
}
logger.Debug("workshop init log success")
if err = p.initSrcProvider(); err != nil {
return err
}
logger.Debug("workshop init srcProvider success")
p.cronTickInterval = ctx.Get("main").Get("cron_tick_interval").MustInt()
p.memLimit = ctx.Get("main").Get("mem_limit").MustUint64()
p.PRCtrlChan = make(chan int, 1000) // 1000 is enough
p.PRReportChan = make(chan int)
p.CLCtrlChan = make(chan int, 1000) // 1000 is enough
p.CLReportChan = make(chan int)
if err = p.initChannel(isTest); err != nil {
return err
}
logger.Debug("workshop init channel and subroutine success")
logger.Info("start... config:\n[", *p.ctx, "]\n")
if isTest {
p.provider.Destory()
}
logger.SetConsole(false)
return nil
}
/**
* 边读边写边更新 节省服务器内存开销
* 中间库的rptdb数据库逻辑 根据查询sql获取i、d状态的数据，逐行插入到rptdb中，插入时判断如果状态为i则insert，状态为d则执行delete语句。
* 最后更新标记位，该数据记录已读取。
*/
func DataSinglePipe(dbIni *DBini, xmlInfo *XmlInfo) {
logger.Info("***************************")
logger.Info("Reader begin >>>>>>>>>>>>>>>>")
var MatchOrgMap = make(map[string]string)
var NoneOrgMap = make(map[string]string)
var begtime string = NowTime()
var logErr error
defer func() {
if err := recover(); err != nil {
logger.Error("查询数据库失败", err)
}
}()
query, err := dbIni.F_Db.db.Query(xmlInfo.DataSQL) //查询数据库
if err != nil {
logger.Error("查询数据库失败", err.Error())
panic(err.Error())
return
}
defer query.Close()
stmt, err := dbIni.T_Db.db.Prepare(xmlInfo.InsertSQL)
defer stmt.Close()
if err != nil {
logger.Error(err)
return
}
deleteByIdStmt, err := dbIni.T_Db.db.Prepare(xmlInfo.DeleteByIDSQL)
defer deleteByIdStmt.Close()
if err != nil {
logger.Error(err)
return
}
cols, _ := query.Columns()
colsLength := len(cols)
values := make([][]byte, colsLength)
scans := make([]interface{}, colsLength)
for i := range values {
scans[i] = &values[i]
}
results := make(map[int]map[string]string)
idsMap := make(map[int]string)
i := 0
for query.Next() { //循环
if err := query.Scan(scans...); err != nil {
logger.Error(err)
logErr = err
return
}
row := make(map[string]string) //每行数据
for k, v := range values {
key := cols[k]
row[key] = string(v)
}
results[i] = row //装入结果集中
wscans := make([]interface{}, colsLength+2)
deleteByIdScans := make([]interface{}, 1)
for j := range cols {
wscans[j] = row[cols[j]]
}
//执行组织机构匹配
if len(xmlInfo.MatchOrgName) > 0 {
deptname := row[xmlInfo.MatchOrgName]
//就近原则 当前批次查找
v, ok := MatchOrgMap[deptname]
if ok {
wscans[colsLength] = v
wscans[colsLength+1] = OrgNameMap[v]
} else {
stdorgid, stdorgname := MatchOrg(deptname)
if strings.EqualFold(stdorgid, noneid) { //未匹配上
NoneOrgMap[deptname] = noneid
wscans[colsLength] = noneid
wscans[colsLength+1] = nonename
} else { //匹配上了
MatchOrgMap[deptname] = stdorgid
wscans[colsLength] = stdorgid
wscans[colsLength+1] = stdorgname
}
}
} else {
wscans[colsLength] = ""
wscans[colsLength+1] = ""
}
//判断是否为d 只删除不插入新数据
deleteByIdScans[0] = row[cols[0]]
deleteByIdStmt.Exec(deleteByIdScans...)
if !strings.EqualFold(row["oper_flag"], "d") {
r, err := stmt.Exec(wscans...)
if err != nil {
logErr = err
logger.Error("#DataSinglePipe装入结果集中,执行insert:", r, err)
}
}
idsMap[i] = row[cols[0]]
i++
}
//insert stdorgmap
insertStdOrgMap(dbIni, NoneOrgMap)
//判断是否有数据
if i == 0 {
xmlInfo.NotExistData = true
} else {
xmlInfo.NotExistData = false
//insert data log
DataLog(dbIni, xmlInfo, begtime, logErr)
}
mapLength := len(idsMap)
if mapLength > 0 {
arrIds := make([]string, mapLength)
for k, v := range idsMap {
arrIds[k] = v
}
xmlInfo.Ids = arrIds
}
idsMap = nil
logger.Info("Reader end >>>>>>>>>>>>>>>>")
}
func (reactor *ComReactor) OnRead(sock *socket.TcpSocket, pack *packet.Packet) {
data := pack.GetData()
log.Info("Recv:", data)
time.Sleep(1)
sock.Write(data)
}