Exemple #1
0
/*将数据写入数据库*/
func Writer(dbIni *DBini, xmlInfo *XmlInfo) {
	logger.Info("***************************")
	logger.Info("Writer begin >>>>>>>>>>>>>>>>")
	if len(xmlInfo.Datas) == 0 {
		logger.Info("Writer end >>>>>>>>>>>>>>>>")
		return
	}
	stmt, err := dbIni.T_Db.db.Prepare(xmlInfo.InsertSQL)
	defer stmt.Close()
	if err != nil {
		logger.Error(err)
		return
	}
	colsLength := len(xmlInfo.Columns)
	c := 0
	arrIds := make([]string, len(xmlInfo.Datas))
	for _, v := range xmlInfo.Datas {
		scans := make([]interface{}, colsLength+2)
		for i := range xmlInfo.Columns {
			scans[i] = v[xmlInfo.Columns[i]]
		}
		scans[colsLength] = dbIni.SysId
		scans[colsLength+1] = dbIni.SysName
		arrIds[c] = v[xmlInfo.Columns[0]]
		stmt.Exec(scans...)
		c++
	}
	xmlInfo.Datas = nil
	xmlInfo.Ids = arrIds
	logger.Info("Writer end >>>>>>>>>>>>>>>>")
}
Exemple #2
0
/*复制文件 如果存在则不再复制*/
func copyFile(src, dest string) error {
	srcFile, e := os.Open(src)
	if e != nil {
		logger.Error("copyFile:", e.Error())
		return e
	}
	defer srcFile.Close()

	df, e := os.Open(dest)
	defer df.Close()
	if e != nil && os.IsNotExist(e) {
		logger.Info("desIniFile:", dest, "file not exist!")
		destFile, e := os.Create(dest)
		if e != nil {
			logger.Error("copyFile destFile:", e.Error())
			return e
		}
		defer destFile.Close()
		io.Copy(destFile, srcFile)
		logger.Info("copy file from :", src, " to ", dest)
	} else {
		logger.Info("desIniFile:", dest, "已存在")
	}

	return nil
}
Exemple #3
0
/*执行数据同步读取直接写入 single data 单次执行可变的业务数据 */
func (xmlInfo *XmlInfo) RunSingleDataPipe(dbIni *DBini) {
	logger.Info("All run begin >>>>>>>>>>>>>>>>")
	logger.Info("dbIni.F_DbType:", dbIni.F_DbType, "dbIni.T_DbType:", dbIni.T_DbType)
	if !StartFlag {
		return
	}
	defer func() {
		if err := recover(); err != nil {
			logger.Error("查询数据库失败", err)
		}
	}()
	f_pwd, _ := Base64Dec(dbIni.F_Password)
	t_pwd, _ := Base64Dec(dbIni.T_Password)
	dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
	dbIni.T_Db.Db_open(dbIni.T_DbType, dbIni.T_User, t_pwd, dbIni.T_Ip, dbIni.T_Port, dbIni.T_Dbname)
	//关闭数据库
	defer dbIni.F_Db.db.Close()
	defer dbIni.T_Db.db.Close()

	if dbIni.F_Db.db == nil || dbIni.T_Db.db == nil {
		logger.Error("无法链接两方数据库")
		panic("无法链接两方数据库")
		return
	}
	DataSinglePipe(dbIni, xmlInfo)
	Updater(dbIni, xmlInfo)
	ExecProc(dbIni, xmlInfo)
	logger.Info("All run end >>>>>>>>>>>>>>>>")
	time.Sleep(5 * time.Second)
}
Exemple #4
0
/*判断是否需要添加标记位,如需要添加标记位*/
func Alter(dbIni *DBini, xmlInfo *XmlInfo) {
	if len(xmlInfo.CheckColumnSQL) > 0 {
		defer func() {
			if err := recover(); err != nil {
				logger.Error("数据库执行失败", err)
			}
		}()
		f_pwd, _ := Base64Dec(dbIni.F_Password)
		dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
		defer dbIni.F_Db.db.Close()
		query, err := dbIni.F_Db.db.Query(strings.Replace(xmlInfo.CheckColumnSQL, "@[dbname]", dbIni.F_Dbname, -1)) //查询数据库
		defer query.Close()
		if err != nil {
			logger.Error("查询数据库失败", err.Error())
			panic(err)
			return
		}
		testQuery, err := dbIni.F_Db.db.Query("select 1")
		if err != nil {
			logger.Error("数据库连接异常,无法查询测试数据")
			panic(err)
			return
		} else {
			if !testQuery.Next() {
				logger.Error("数据库连接异常,无法遍历测试数据")
				panic(err)
				return
			}
			defer testQuery.Close()
		}
		if query.Next() {
			logger.Info(xmlInfo.Remark, "经检测,标记位已添加")
			xmlInfo.AlterFlag = true
		} else {
			stmt, err := dbIni.F_Db.db.Prepare(xmlInfo.AlterSQL)
			defer stmt.Close()
			if err != nil {
				logger.Error(err)
				return
			}
			stmt.Exec()
			logger.Info(xmlInfo.Remark, "在源数据库中添加成功")
			updateStmt, err := dbIni.F_Db.db.Prepare(xmlInfo.UpdateFlagSQL)
			defer updateStmt.Close()
			if err != nil {
				logger.Error(err)
				return
			}
			updateStmt.Exec()
			logger.Info(xmlInfo.Remark, "在源数据库中更新成功")
			xmlInfo.AlterFlag = true
		}
	} else {
		logger.Info(xmlInfo.Remark, "无需检测标记位")
		xmlInfo.AlterFlag = true
	}
}
Exemple #5
0
/*启动组织机构同步服务*/
func (configinfo *ConfigInfo) StartOrgSync() {
	logger.Info(" StartOrgSync is run... ")
	var orgVersionUrl = "http://" + configinfo.core_ip + ":" + configinfo.core_port + configinfo.orgversion_url
	var allOrgUrl = "http://" + configinfo.core_ip + ":" + configinfo.core_port + configinfo.allorg_url
	logger.Info("orgVersionUrl start >>>>>>>>>>>>>>>>", orgVersionUrl)
	logger.Info("allOrgUrl start >>>>>>>>>>>>>>>>", allOrgUrl)
	//发起链接访问,获取返回的json数据
	for {
		var return_orgs []byte
		var return_version []byte
		var tmpOrgVersion string = ""
		if response, err := http.Get(orgVersionUrl); err != nil {
			logger.Error(" err = ", err)
		} else {
			defer response.Body.Close()
			if return_version, err = ioutil.ReadAll(response.Body); err != nil {
				logger.Error(" err = ", err)
			}
			tmpOrgVersion = string(return_version)
		}
		logger.Info(" return_version = ", tmpOrgVersion)
		if len(OrgVersion) == 0 || (len(tmpOrgVersion) > 0 && !strings.EqualFold(tmpOrgVersion, OrgVersion)) {
			OrgVersion = tmpOrgVersion
			//重新加载组织机构
			if response, err := http.Get(allOrgUrl); err != nil {
				logger.Error(" err = ", err)
			} else {
				defer response.Body.Close()
				if return_orgs, err = ioutil.ReadAll(response.Body); err != nil {
					logger.Error(" err = ", err)
				}
			}
			var orgInfos []OrgInfo
			err := json.Unmarshal(return_orgs, &orgInfos)
			if err != nil {
				logger.Error(" err = ", err)
			}
			if len(orgInfos) > 0 {
				for k, _ := range OrgMap {
					delete(OrgMap, k)
				}
				for k, _ := range OrgNameMap {
					delete(OrgNameMap, k)
				}
				for i := range orgInfos {
					org := &orgInfos[i]
					OrgMap[org.Org_path] = org.Id
					OrgMap[org.Name] = org.Id
					OrgNameMap[org.Id] = org.Name
				}
			}
		}
		time.Sleep(Time_interval * time.Second)
	}
}
Exemple #6
0
/*主入口*/
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	//指定是否控制台打印,默认为true
	logger.SetConsole(true)

	//指定日志级别  ALL,DEBUG,INFO,WARN,ERROR,FATAL,OFF 级别由低到高
	logger.SetLevel(logger.INFO)
	logger.SetRollingFile("./log", "datapostlog.log", 100, 50, logger.MB)

	if beego.AppConfig.String("runmode") == "dev" {
		orm.Debug = true
	}

	//支持异构版本升级
	pipe.Upgrade()
	time.Sleep(3 * time.Second)

	//配置文件初始化
	err := pipe.InitFile()
	if err != nil {
		logger.Error("InitFile():", err)
	}

	//注册运行服务
	ServiceHandle(cSvcName, cSvcName, cSvcName, func() {
		//初始化运行参数
		pipe.InitConfigMap()

		//运行参数管理UI服务
		beego.Router("/", &pipe.IndexInfo{}, "*:Index")
		beego.Router("/index/list", &pipe.IndexInfo{}, "*:List")
		beego.Router("/index/edit", &pipe.IndexInfo{}, "*:Edit")
		beego.Router("/index/test", &pipe.IndexInfo{}, "*:Test")
		beego.Router("/index/help", &pipe.IndexInfo{}, "*:Help")
		beego.Router("/index/treedata", &pipe.IndexInfo{}, "*:TreeData")

		//启动数据同步
		go pipe.StartAllSys()

		//启动https服务
		go pipe.Https_start()

		//启动参数管理UI 服务
		beego.Run()
		logger.Info("DaemonName=", cSvcName, "Daemon started.")
		for {
			time.Sleep(time.Hour)
		}
	}, func() {
		Stop()
		logger.Info("DaemonName=", cSvcName, "Daemon stoped.")
	})
}
Exemple #7
0
func (p *Workshop) GC() {
	runtime.ReadMemStats(&(p.ms))
	alloc := p.ms.Alloc / 1024 / 1024
	logger.Info("memAlloc: ", alloc, "M heapAlloc: ", p.ms.HeapAlloc/1024/1024, "M, stackAlloc: ", p.ms.StackInuse/1024/1024, "M")
	if alloc >= p.memLimit {
		debug.FreeOSMemory()
		runtime.ReadMemStats(&(p.ms))
		alloc = p.ms.Alloc / 1024 / 1024
		logger.Info("after GC  memAlloc: ", alloc, "M heapAlloc: ", p.ms.HeapAlloc/1024/1024, "M, stackAlloc: ", p.ms.StackInuse/1024/1024, "M")
	}
}
Exemple #8
0
func (protocol *TransProtocolComm) Decode_pack(data []byte) (*Packet, int) {
	log.Info("Enter Decode_pack")
	decodeLen := 0
	pack := &Packet{}
	dataLen := len(data)
	offset := 0
	headLen := binary.Size(pack._header)
	//1.get packhader
	//= not cause out of memory so >= is needed
	for offset = 0; dataLen-offset >= headLen; offset++ {
		pack._header._ver = int16(binary.BigEndian.Uint16(data[offset : offset+2]))
		pack._header._seq = int32(binary.BigEndian.Uint32(data[offset+2 : offset+6]))
		pack._header._type = int16(binary.BigEndian.Uint16(data[offset+6 : offset+8]))
		pack._header._bodyLen = int32(binary.BigEndian.Uint32(data[offset+8 : offset+12]))
		//get pack header successful
		if int16(VER) == pack._header._ver && pack._header._bodyLen < int32(MAX_PACK_SIZE) {
			break
		}
	}
	//2.check is a complete Packet
	if dataLen-offset < headLen || dataLen-offset < int(pack._header._bodyLen) {
		//can't find VER in data Slice or is not a complete pack.
		log.Error("[PACK_NOT_COMPLETE],pack bodyLen is", pack._header._bodyLen)
		return nil, 0
	}

	//3.get a complete pack
	pack._data = make([]byte, pack._header._bodyLen)
	copy(pack._data, data[offset:])
	decodeLen = offset + headLen + int(pack._header._bodyLen)
	return pack, decodeLen
}
Exemple #9
0
/*启动所有配置信息*/
func StartAllSys() {
	dirs := strings.Split(ConfigMap["transferes"], ",")
	nosqlDir := ConfigMap["nosql"]
	logger.Info("nosqlDir  >>>>>>>>>>>>>>>>", nosqlDir)
	StartFlag = true
	//nosql目录
	if len(nosqlDir) > 0 {
		noSqlDbIni := GetSysIni(nosqlDir)
		noSqlDbIni.RegisterSys()
		go noSqlDbIni.StartNoSqlSys()
	}
	for i := range dirs {
		dirName := dirs[i]
		if len(dirName) > 0 {
			sysIni := GetSysIni(dirName)
			if i == 0 {
				go sysIni.StdOrgSync()
			}
			//非nosql目录
			sysIni.RegisterSys()
			go sysIni.StartSys()
		}
	}

	for {
		time.Sleep(2 * time.Second)
	}
}
Exemple #10
0
func log(i int) {
	logger.Debug("Debug>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
	logger.Info("Info>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
	logger.Warn("Warn>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
	logger.Error("Error>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
	logger.Fatal("Fatal>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
}
Exemple #11
0
func StopWorkShop() error {
	WSChan_ <- CMD_EXIT
StopWorkShop_LOOP:
	for {
		select {
		case ret := <-WSReportChan_:
			if ret == RET_EXIT_SUCCESS {
				logger.Info("workshop exit success")
			} else {
				logger.Info("workshop exit fail")
			}
		}
		break StopWorkShop_LOOP
	}
	return nil
}
Exemple #12
0
/*判断是否需要查询数据*/
func Check(dbIni *DBini, xmlInfo *XmlInfo) {
	if strings.EqualFold(xmlInfo.IsValData, "true") {
		xmlInfo.CheckFlag = true
		return
	}
	if len(xmlInfo.CheckRowSQL) > 0 {
		defer func() {
			if err := recover(); err != nil {
				logger.Error("查询数据库失败", err)
			}
		}()
		f_pwd, _ := Base64Dec(dbIni.F_Password)
		dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
		defer dbIni.F_Db.db.Close()
		query, err := dbIni.F_Db.db.Query(strings.Replace(xmlInfo.CheckRowSQL, "@[dbname]", dbIni.F_Dbname, -1)) //查询数据库
		defer query.Close()

		if err != nil {
			logger.Error("查询数据库失败", err.Error())
			xmlInfo.CheckFlag = false
			panic(err)
			return
		}
		cols, _ := query.Columns()
		values := make([][]byte, len(cols))     //values是每个列的值
		scans := make([]interface{}, len(cols)) //query.Scan的参数 用len(cols)定住当次查询的长度
		for i := range values {                 //让每一行数据都填充到[][]byte里面
			scans[i] = &values[i]
		}
		dbRows := 0
		if query.Next() {
			//判断是否与上次的数据一致,如果不一致则查询数据库,否则不查询
			if err := query.Scan(scans...); err != nil { //query.Scan查询出来的不定长值放到scans[i] = &values[i],也就是每行都放在values里
				logger.Error(err)
				xmlInfo.CheckFlag = false
				return
			}
			for _, v := range values {
				intV, err := strconv.Atoi(string(v))
				if err != nil {
					xmlInfo.CheckFlag = false
				} else {
					dbRows = intV
				}
			}
			if dbRows != 0 && dbRows != xmlInfo.RowNum { //获取reader 如何获取到数据则置为0,如果获取不到了则赋予全局GobalNum
				xmlInfo.RowNum = dbRows
				xmlInfo.GobalNum = dbRows
				xmlInfo.CheckFlag = true
			} else {
				xmlInfo.CheckFlag = false
			}
		} else {
			xmlInfo.CheckFlag = true
		}
	} else {
		logger.Info(xmlInfo.Remark, "无需查询系统库行记录是否变化")
		xmlInfo.CheckFlag = true
	}
}
Exemple #13
0
/*启动*/
func (sys *SysIni) StartSys() {
	fmt.Println("SysIni is start................")
	StartFlag = true
	if StartFlag { //判断标记位并增加标记位
		for i := range sys.xmlInfoes {
			Alter(&sys.dbini, &sys.xmlInfoes[i])
			logger.Info("Alter >>>>>>>>>>>>>>>>:", i)
		}
	}
	logger.Info("StartSys >>>>>>>>>>>>>>>>goto for ")
	//单次同步
	c := cron.New()
	c.Start()
	defer c.Stop()
	fmt.Println("singletimer>>>>", sys.configInfo.singletimer)

	//单次同步
	for j := range sys.singleXmlInfoes {
		go func(sys *SysIni, j int) {
			singleXmlInfo := sys.singleXmlInfoes[j]
			c.AddFunc(sys.configInfo.singletimer,
				func() {
					logger.Info("single[", singleXmlInfo.FileName, "] job is run ......................")
					singleXmlInfo.RunSingleDataPipe(&sys.dbini)
				},
				singleXmlInfo.FileName)
		}(sys, j)
	}
	//实时同步
	for StartFlag {
		//边读边写,更新
		for i := range sys.xmlInfoes {
			//logger.Info("sys.xmlInfoes[i].NotExistData >>>>>>>>>>>>>>>>", sys.xmlInfoes[i].Remark, sys.xmlInfoes[i].NotExistData)
			if sys.xmlInfoes[i].NotExistData {
				Check(&sys.dbini, &sys.xmlInfoes[i])
			}
			if sys.xmlInfoes[i].CheckFlag {
				sys.xmlInfoes[i].RunDataPipe(&sys.dbini)
			}
		}
		time.Sleep(2 * time.Second)
	}
}
Exemple #14
0
/*执行数据同步*/
func (xmlInfo *XmlInfo) Run(dbIni *DBini) {
	logger.Info("All run begin >>>>>>>>>>>>>>>>")
	f_pwd, _ := Base64Dec(dbIni.F_Password)
	t_pwd, _ := Base64Dec(dbIni.T_Password)
	dbIni.F_Db.Db_open(dbIni.F_DbType, dbIni.F_User, f_pwd, dbIni.F_Ip, dbIni.F_Port, dbIni.F_Dbname)
	dbIni.T_Db.Db_open(dbIni.T_DbType, dbIni.T_User, t_pwd, dbIni.T_Ip, dbIni.T_Port, dbIni.T_Dbname)
	//关闭数据库
	defer dbIni.F_Db.db.Close()
	defer dbIni.T_Db.db.Close()

	if dbIni.F_Db.db == nil || dbIni.T_Db.db == nil {
		logger.Error("无法链接两方数据库")
		return
	}
	Reader(dbIni, xmlInfo)
	Writer(dbIni, xmlInfo)
	Updater(dbIni, xmlInfo)
	logger.Info("All run end >>>>>>>>>>>>>>>>")
	time.Sleep(5 * time.Second)
}
Exemple #15
0
/*访问数据库获取数据*/
func Reader(dbIni *DBini, xmlInfo *XmlInfo) {
	logger.Info("***************************")
	logger.Info("Reader begin >>>>>>>>>>>>>>>>")
	query, err := dbIni.F_Db.db.Query(xmlInfo.DataSQL) //查询数据库
	if err != nil {
		logger.Error("查询数据库失败", err.Error())
		return
	}
	defer query.Close()
	cols, _ := query.Columns()
	values := make([][]byte, len(cols))     //values是每个列的值
	scans := make([]interface{}, len(cols)) //query.Scan的参数 用len(cols)定住当次查询的长度
	for i := range values {                 //让每一行数据都填充到[][]byte里面
		scans[i] = &values[i]
	}
	results := make(map[int]map[string]string) //最后得到的map
	i := 0
	for query.Next() { //循环
		if err := query.Scan(scans...); err != nil { //query.Scan查询出来的不定长值放到scans[i] = &values[i],也就是每行都放在values里
			logger.Error(err)
			return
		}
		row := make(map[string]string) //每行数据
		for k, v := range values {
			key := cols[k]
			row[key] = string(v)
		}
		results[i] = row //装入结果集中
		i++
	}
	if i == 0 {
		xmlInfo.RowNum = xmlInfo.GobalNum
	} else {
		xmlInfo.RowNum = 0
	}
	xmlInfo.Datas = results
	xmlInfo.Columns = cols
	logger.Info("Reader end >>>>>>>>>>>>>>>>")
}
Exemple #16
0
//indexinfo to ini
func IndexInfoToIni(this *IndexInfo, cfg *goconfig.Config) {
	f_dbname, _ := cfg.Get("dbconfig::f_dbname")
	t_dbname, _ := cfg.Get("dbconfig::t_dbname")

	f_dbtype, _ := cfg.Get("dbconfig::f_dbtype")
	f_driver, _ := cfg.Get("dbconfig::f_driver")
	t_dbtype, _ := cfg.Get("dbconfig::t_dbtype")
	t_driver, _ := cfg.Get("dbconfig::t_driver")

	var f_dburl string = ""
	if strings.EqualFold(f_dbtype, "mysql") {
		f_dburl_arr := []string{"jdbc:mysql://", this.F_ip, ":", this.F_port, "/", f_dbname, "?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
		f_dburl = strings.Join(f_dburl_arr, "")
	} else if strings.EqualFold(f_dbtype, "mssql") {
		f_dburl_arr := []string{"jdbc:microsoft:sqlserver://", this.F_ip, ":", this.F_port, ";", "DatabaseName=", f_dbname}
		f_dburl = strings.Join(f_dburl_arr, "")
	}
	t_dburl_arr := []string{"jdbc:mysql://", this.T_ip, ":", this.T_port, "/", t_dbname, "?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
	var t_dburl = strings.Join(t_dburl_arr, "")

	cfg.Set("dbconfig", "f_dbtype", f_dbtype)
	cfg.Set("dbconfig", "f_driver", f_driver)
	cfg.Set("dbconfig", "f_dburl", f_dburl)
	cfg.Set("dbconfig", "f_ip", this.F_ip)
	cfg.Set("dbconfig", "f_port", this.F_port)
	cfg.Set("dbconfig", "f_dbname", f_dbname)
	cfg.Set("dbconfig", "f_user", this.F_username)
	cfg.Set("dbconfig", "f_password", this.F_password)

	cfg.Set("dbconfig", "t_dbtype", t_dbtype)
	cfg.Set("dbconfig", "t_driver", t_driver)
	cfg.Set("dbconfig", "t_dburl", t_dburl)
	cfg.Set("dbconfig", "t_ip", this.T_ip)
	cfg.Set("dbconfig", "t_port", this.T_port)
	cfg.Set("dbconfig", "t_dbname", t_dbname)
	cfg.Set("dbconfig", "t_user", this.T_username)
	cfg.Set("dbconfig", "t_password", this.T_password)

	sysid, _ := cfg.Get("dbconfig::sysid")
	systype, _ := cfg.Get("dbconfig::systype")
	sysname, _ := cfg.Get("dbconfig::sysname")
	company, _ := cfg.Get("dbconfig::company")
	platform, err := cfg.Get("dbconfig::platform")
	logger.Info("platform===========================", platform, sysname, err)
	cfg.Set("dbconfig", "sysid", sysid)
	cfg.Set("dbconfig", "systype", systype)
	cfg.Set("dbconfig", "sysname", sysname)
	cfg.Set("dbconfig", "company", company)
	cfg.Set("dbconfig", "platform", platform)

}
Exemple #17
0
func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: $0 -c config_file [-t]\n")
		os.Exit(1)
	}

	flag.StringVar(&configFile_, "c", "conf.json", "config file in json")
	flag.BoolVar(&isTest, "t", false, "test mode")
	flag.Parse()

	// SIGHUP to reload configFile, SIGTERM/SIGINT to exit the process
	sigChan := make(chan os.Signal, 10)
	signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)

	// start worshop for the first time
	if err := StartNewWorkShop(isTest); err != nil {
		fmt.Fprintf(os.Stderr, "workshop init fail [%+v]\n", err)
		os.Exit(1)
	}

	if isTest {
		os.Exit(0)
	}
	// loop. wait for signals
MAIN_LOOP:
	for {
		select {
		case sig := <-sigChan:
			fmt.Fprintf(os.Stderr, "receive sig [%+v]\n", sig)
			//			logger.Error("receive sig ", sig)
			if sig == syscall.SIGHUP {
				// wait for workshop to do some clean-up
				StopWorkShop()
				ticker_.Stop()
			} else if sig == syscall.SIGTERM || sig == syscall.SIGINT {
				StopWorkShop()
				ticker_.Stop()
				break MAIN_LOOP
			} else {
				logger.Error("main routine receive unexpected signal [", sig, "], this should NEVER happen")
				os.Exit(1)
			}
		case <-ticker_.C:
			WSChan_ <- CMD_TICK
			logger.Debug("main_routine tick")
		}
	}
	logger.Info("the qtask application exit")
}
Exemple #18
0
/*回写标记位*/
func Updater(dbIni *DBini, xmlInfo *XmlInfo) {
	logger.Info("***************************")
	logger.Info("Updater begin >>>>>>>>>>>>>>>>")
	defer func() {
		if err := recover(); err != nil {
			logger.Error("数据库执行失败", err)
		}
	}()
	if len(xmlInfo.SuccessSQL) > 0 {
		var buffer bytes.Buffer
		idsLength := len(xmlInfo.Ids)
		idsLength--
		for i := range xmlInfo.Ids {
			if i == idsLength {
				buffer.WriteString("'")
				buffer.WriteString(xmlInfo.Ids[i])
				buffer.WriteString("'")
			} else {
				buffer.WriteString("'")
				buffer.WriteString(xmlInfo.Ids[i])
				buffer.WriteString("',")
			}
		}
		if len(buffer.String()) > 0 {
			stmt, err := dbIni.F_Db.db.Prepare(strings.Replace(xmlInfo.SuccessSQL, "@[id]", buffer.String(), -1))
			defer stmt.Close()
			if err != nil {
				logger.Error(err)
				panic(err)
				return
			}
			stmt.Exec()
		}
	}
	logger.Info("Updater end >>>>>>>>>>>>>>>>")
}
Exemple #19
0
func (p *Workshop) closeSubRoutines(ctrlChan chan int, reportChan chan int, n int) {
	p.sendCtrlInfo(ctrlChan, CMD_EXIT, n)

	closeSuccess := 0
	i := 0
	for i < n {
		select {
		case ret := <-reportChan:
			i++
			if ret == RET_EXIT_SUCCESS {
				closeSuccess++
			}
		}
	}
	logger.Info("close  ", closeSuccess, "/", n, "  subRoutines")
}
Exemple #20
0
/*已匹配组织机构信息*/
func (sys *SysIni) StdOrgSync() {
	logger.Info("StdOrgSync start >>>>>>>>>>>>>>>>")
	defer func() {
		if err := recover(); err != nil {
			logger.Error("查询数据库失败", err)
		}
	}()
	for {
		dbIni := &sys.dbini
		t_pwd, _ := Base64Dec(dbIni.T_Password)
		dbIni.T_Db.Db_open(dbIni.T_DbType, dbIni.T_User, t_pwd, dbIni.T_Ip, dbIni.T_Port, dbIni.T_Dbname)
		defer dbIni.T_Db.db.Close()
		query, err := dbIni.T_Db.db.Query(MapQuerySQL) //查询数据库
		if err != nil {
			logger.Error("查询数据库失败", err.Error())
			panic(err)
			return
		}
		defer query.Close()
		for k, _ := range StdOrgMap {
			delete(StdOrgMap, k)
		}

		cols, _ := query.Columns()
		colsLength := len(cols)
		values := make([][]byte, colsLength)
		scans := make([]interface{}, colsLength)
		for i := range values {
			scans[i] = &values[i]
		}
		for query.Next() { //循环
			if err := query.Scan(scans...); err != nil {
				logger.Error(err)
				return
			}
			row := make(map[string]string) //每行数据
			for k, v := range values {
				key := cols[k]
				row[key] = string(v)
			}
			StdOrgMap[row["map_deptname"]] = row["std_deptid"]
		}
		time.Sleep(Time_interval * time.Second)
	}
}
Exemple #21
0
//indexinfo to ini
func IndexInfoToIni(this *IndexInfo, cfg *goconfig.Config) {
	f_dburl_arr := []string{"jdbc:mysql://", this.F_ip, ":", this.F_port, "/mid?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
	var f_dburl = strings.Join(f_dburl_arr, "")

	t_dburl_arr := []string{"jdbc:mysql://", this.T_ip, ":", this.T_port, "/rptdb?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true"}
	var t_dburl = strings.Join(t_dburl_arr, "")
	f_dbname, _ := cfg.Get("dbconfig::f_dbname")
	t_dbname, _ := cfg.Get("dbconfig::t_dbname")

	f_dbtype, _ := cfg.Get("dbconfig::f_dbtype")
	f_driver, _ := cfg.Get("dbconfig::f_driver")
	t_dbtype, _ := cfg.Get("dbconfig::t_dbtype")
	t_driver, _ := cfg.Get("dbconfig::t_driver")
	cfg.Set("dbconfig", "f_dbtype", f_dbtype)
	cfg.Set("dbconfig", "f_driver", f_driver)
	cfg.Set("dbconfig", "f_dburl", f_dburl)
	cfg.Set("dbconfig", "f_ip", this.F_ip)
	cfg.Set("dbconfig", "f_port", this.F_port)
	cfg.Set("dbconfig", "f_dbname", f_dbname)
	cfg.Set("dbconfig", "f_user", this.F_username)
	cfg.Set("dbconfig", "f_password", this.F_password)

	cfg.Set("dbconfig", "t_dbtype", t_dbtype)
	cfg.Set("dbconfig", "t_driver", t_driver)
	cfg.Set("dbconfig", "t_dburl", t_dburl)
	cfg.Set("dbconfig", "t_ip", this.T_ip)
	cfg.Set("dbconfig", "t_port", this.T_port)
	cfg.Set("dbconfig", "t_dbname", t_dbname)
	cfg.Set("dbconfig", "t_user", this.T_username)
	cfg.Set("dbconfig", "t_password", this.T_password)

	sysid, _ := cfg.Get("dbconfig::sysid")
	systype, _ := cfg.Get("dbconfig::systype")
	sysname, _ := cfg.Get("dbconfig::sysname")
	company, _ := cfg.Get("dbconfig::company")
	platform, _ := cfg.Get("dbconfig::platform")
	logger.Info("platform===========================", platform, sysname)
	cfg.Set("dbconfig", "sysid", sysid)
	cfg.Set("dbconfig", "systype", systype)
	cfg.Set("dbconfig", "sysname", sysname)
	cfg.Set("dbconfig", "company", company)
	cfg.Set("dbconfig", "platform", platform)

}
Exemple #22
0
func (p *Workshop) Run() {
LOOP:
	for {
		select {
		case cmd := <-p.upCtrlChan:
			if cmd == CMD_EXIT {
				close(p.msgChan) // will send an empty msg to its receivers
				p.closeSubRoutines(p.PRCtrlChan, p.PRReportChan, p.processorNum)
				p.closeSubRoutines(p.CLCtrlChan, p.CLReportChan, p.collectorNum)
				break LOOP
			} else if cmd == CMD_TICK {
				//				logger.Debug("run_routine tick")
				p.sendCtrlInfo(p.PRCtrlChan, cmd, p.processorNum)
				p.sendCtrlInfo(p.CLCtrlChan, cmd, p.collectorNum)
				p.cronTick++
				if p.cronTick > p.cronTickInterval {
					p.Tick()
					p.cronTick = 0
				}
			} else {
				logger.Error(" WorkShop receive a un-expected cmd ", cmd)
			}
		default:
			messages, err := p.provider.GetNextMsg()
			if err != nil {
				logger.Error("src provider get msg err occur with ", len(messages), " msgs, err: ", err)
			}
			if messages == nil || len(messages) == 0 {
				continue
			}
			for _, v := range messages {
				if len(v) > 0 {
					logger.Info("emit!")
					p.msgChan <- string(v)
				}
			}
			logger.Debug("main routine receive ", len(messages), " msgs from srcProvider")
			p.processedLine += len(messages)
			p.provider.Ack()
		}
	}
	p.provider.Destory()
	p.upReportChan <- RET_EXIT_SUCCESS
}
Exemple #23
0
/*启动*/
func (sys *SysIni) StartNoSqlSys() {
	fmt.Println("StartNoSqlSys is start................")
	//单次同步
	c := cron.New()
	c.Start()
	defer c.Stop()
	for j := range sys.singleXmlInfoes {
		singleXmlInfo := sys.singleXmlInfoes[j]
		c.AddFunc(sys.configInfo.singletimer,
			func() {
				logger.Info("single[", singleXmlInfo.FileName, "] job is run ......................")
				NoSqlSync(&sys.dbini, &sys.singleXmlInfoes[j])
			},
			singleXmlInfo.FileName)
	}
	for {
		time.Sleep(2 * time.Second)
	}
}
Exemple #24
0
func (sock *TcpSocket) to_write() {
	var pack *packet.Packet
	for ESTABLISHED == sock._socketState {
		pack = <-sock._writerChannel
		if pack == nil {
			log.Error("get pack error")
			continue
		}
		data := pack.Serialize()
		n, err := sock._conn.Write(data)
		if err != nil {
			if io.ErrShortWrite != err {
				log.Error("write err close--", n)
				sock.closeSock()
				break
			}
		}
		log.Info("send data success")
	}
}
Exemple #25
0
//编辑
func (this *IndexInfo) Edit() {
	var (
		config = new(IndexInfo)
		err    error
		ret    Ret
		flag   bool
	)
	if err = json.Unmarshal([]byte(this.GetString("json")), &config); err == nil {
		err = config.SaveSingleConf()
	}
	if err != nil {
		logger.Error(err)
		ret.Code, ret.Msg = 1, err.Error()
		flag = false
		logger.Info("Edit", err.Error())
	} else {
		ret.Code = 0
		flag = true
	}
	this.Data["json"] = &ret
	this.ServeJson(flag)
}
Exemple #26
0
func (p *PrintProcessor) Process(msg string) error {
	logger.Info("receive: ", msg)
	strArr := strings.Split(msg, "\n")
	if len(strArr) <= 0 {
		return nil
	}

	for _, str := range strArr {
		str = strings.TrimSpace(str)
		if str == "" {
			continue
		}
		index := p.getCollectorIndex(len(p.itemChans))
		p.itemChans[index] <- Item{
			Category: p.category,
			Id:       "",
			Content:  "",
			RawMsg:   str,
		}
	}
	return nil
}
Exemple #27
0
/*执行数据处理*/
func ExecProc(dbIni *DBini, xmlInfo *XmlInfo) {
	proname := xmlInfo.Proname
	if strings.EqualFold(xmlInfo.Isexecpro, "true") && len(proname) > 0 {
		var buffer bytes.Buffer
		buffer.WriteString("call ")
		buffer.WriteString(proname)
		buffer.WriteString("()")
		defer func() {
			if err := recover(); err != nil {
				logger.Error("查询数据库失败", err)
			}
		}()
		logger.Info("执行存储过程  >>>>>>>>>>>>>>>>", buffer.String())
		stmt, err := dbIni.T_Db.db.Prepare(buffer.String())
		defer stmt.Close()
		if err != nil {
			logger.Error(err)
			panic(err)
			return
		}
		stmt.Exec()
	}
}
Exemple #28
0
func (p *Workshop) Init(ctx *sj.Json, upCtrlChan chan int, upReportChan chan int, isTest bool) error {
	p.ctx = ctx
	p.upCtrlChan = upCtrlChan
	p.upReportChan = upReportChan

	p.processedLine = 0
	var err error
	if err = p.initLog(isTest); err != nil {
		return err
	}
	logger.Debug("workshop init log success")

	if err = p.initSrcProvider(); err != nil {
		return err
	}
	logger.Debug("workshop init srcProvider success")

	p.cronTickInterval = ctx.Get("main").Get("cron_tick_interval").MustInt()
	p.memLimit = ctx.Get("main").Get("mem_limit").MustUint64()
	p.PRCtrlChan = make(chan int, 1000) // 1000 is enough
	p.PRReportChan = make(chan int)
	p.CLCtrlChan = make(chan int, 1000) // 1000 is enough
	p.CLReportChan = make(chan int)

	if err = p.initChannel(isTest); err != nil {
		return err
	}
	logger.Debug("workshop init channel and subroutine success")

	logger.Info("start...  config:\n[", *p.ctx, "]\n")
	if isTest {
		p.provider.Destory()
	}
	logger.SetConsole(false)
	return nil
}
Exemple #29
0
/**
 * 边读边写边更新 节省服务器内存开销
 * 中间库的rptdb数据库逻辑 根据查询sql获取i、d状态的数据,逐行插入到rptdb中,插入时判断如果状态为i则insert,状态为d则执行delete语句。
 * 最后更新标记位,该数据记录已读取。
 */
func DataSinglePipe(dbIni *DBini, xmlInfo *XmlInfo) {
	logger.Info("***************************")
	logger.Info("Reader begin >>>>>>>>>>>>>>>>")
	var MatchOrgMap = make(map[string]string)
	var NoneOrgMap = make(map[string]string)
	var begtime string = NowTime()
	var logErr error

	defer func() {
		if err := recover(); err != nil {
			logger.Error("查询数据库失败", err)
		}
	}()

	query, err := dbIni.F_Db.db.Query(xmlInfo.DataSQL) //查询数据库
	if err != nil {
		logger.Error("查询数据库失败", err.Error())
		panic(err.Error())
		return
	}
	defer query.Close()
	stmt, err := dbIni.T_Db.db.Prepare(xmlInfo.InsertSQL)
	defer stmt.Close()
	if err != nil {
		logger.Error(err)
		return
	}

	deleteByIdStmt, err := dbIni.T_Db.db.Prepare(xmlInfo.DeleteByIDSQL)
	defer deleteByIdStmt.Close()
	if err != nil {
		logger.Error(err)
		return
	}
	cols, _ := query.Columns()
	colsLength := len(cols)
	values := make([][]byte, colsLength)
	scans := make([]interface{}, colsLength)
	for i := range values {
		scans[i] = &values[i]
	}
	results := make(map[int]map[string]string)
	idsMap := make(map[int]string)
	i := 0
	for query.Next() { //循环
		if err := query.Scan(scans...); err != nil {
			logger.Error(err)
			logErr = err
			return
		}
		row := make(map[string]string) //每行数据
		for k, v := range values {
			key := cols[k]
			row[key] = string(v)
		}
		results[i] = row //装入结果集中
		wscans := make([]interface{}, colsLength+2)
		deleteByIdScans := make([]interface{}, 1)
		for j := range cols {
			wscans[j] = row[cols[j]]
		}
		//执行组织机构匹配
		if len(xmlInfo.MatchOrgName) > 0 {
			deptname := row[xmlInfo.MatchOrgName]

			//就近原则 当前批次查找
			v, ok := MatchOrgMap[deptname]
			if ok {
				wscans[colsLength] = v
				wscans[colsLength+1] = OrgNameMap[v]
			} else {
				stdorgid, stdorgname := MatchOrg(deptname)
				if strings.EqualFold(stdorgid, noneid) { //未匹配上
					NoneOrgMap[deptname] = noneid
					wscans[colsLength] = noneid
					wscans[colsLength+1] = nonename
				} else { //匹配上了
					MatchOrgMap[deptname] = stdorgid
					wscans[colsLength] = stdorgid
					wscans[colsLength+1] = stdorgname
				}
			}
		} else {
			wscans[colsLength] = ""
			wscans[colsLength+1] = ""
		}

		//判断是否为d 只删除不插入新数据
		deleteByIdScans[0] = row[cols[0]]
		deleteByIdStmt.Exec(deleteByIdScans...)
		if !strings.EqualFold(row["oper_flag"], "d") {
			r, err := stmt.Exec(wscans...)
			if err != nil {
				logErr = err
				logger.Error("#DataSinglePipe装入结果集中,执行insert:", r, err)
			}
		}

		idsMap[i] = row[cols[0]]
		i++
	}

	//insert stdorgmap
	insertStdOrgMap(dbIni, NoneOrgMap)
	//判断是否有数据
	if i == 0 {
		xmlInfo.NotExistData = true
	} else {
		xmlInfo.NotExistData = false
		//insert data log
		DataLog(dbIni, xmlInfo, begtime, logErr)
	}
	mapLength := len(idsMap)
	if mapLength > 0 {
		arrIds := make([]string, mapLength)
		for k, v := range idsMap {
			arrIds[k] = v
		}
		xmlInfo.Ids = arrIds
	}
	idsMap = nil
	logger.Info("Reader end >>>>>>>>>>>>>>>>")
}
Exemple #30
0
func (reactor *ComReactor) OnRead(sock *socket.TcpSocket, pack *packet.Packet) {
	data := pack.GetData()
	log.Info("Recv:", data)
	time.Sleep(1)
	sock.Write(data)
}