func SaveTraceInfo(orgcode string, data []map[string]string) { log.Println("SaveTraceInfo") session := db.GetDBSession() defer db.ReleaseDBSession(session) c := session.DB("detector").C("detector_report") bulk := c.Bulk() for i, fields := range data { //log.Println(fields) mac := fields["MAC"] mac = filterMac(mac) ap_mac := fields["ACCESS_AP_MAC"] ap_mac = filterMac(ap_mac) lng, err1 := strconv.ParseFloat(fields["COLLECTION_EQUIPMENT_LONGITUDE"], 64) lat, err2 := strconv.ParseFloat(fields["COLLECTION_EQUIPMENT_LATITUDE"], 64) time, err3 := strconv.Atoi(fields["CAPTURE_TIME"]) if err1 != nil || err2 != nil || err3 != nil { continue } lng, lat = data_file.Bd09towgs84(lng, lat) log.Println(i, ": mac", mac, "ap_mac", ap_mac, "lng", lng, "lat", lat, "time", time) if saveToDB { bulk.Insert(bson.M{"ap_mac": ap_mac, "device_mac": mac, "longitude": lng, "latitude": lat, "org_code": orgcode, "time": uint32(time)}) } } bulk.Run() }
func SaveBehaviorLog(orgcode string, data []map[string]string) { log.Println("SaveLog") session := db.GetDBSession() defer db.ReleaseDBSession(session) c := session.DB("person_info").C("behavior_log") bulk := c.Bulk() for i, fields := range data { log.Println(i, fields) mac := fields["MAC"] mac = filterMac(mac) IpInt, err4 := strconv.ParseInt(fields["DST_IP"], 10, 64) Ip := inet_ntoa(IpInt) port := fields["DST_PORT"] lng, err1 := strconv.ParseFloat(fields["LONGITUDE"], 64) lat, err2 := strconv.ParseFloat(fields["LATITUDE"], 64) if orgcode == "779852855" && err2 != nil { lat, err2 = strconv.ParseFloat(fields["LAITTUDE"], 64) } time, err3 := strconv.Atoi(fields["CAPTURE_TIME"]) if err1 != nil || err2 != nil || err3 != nil || err4 != nil { log.Println("error:", err1, err2, err3, err4) continue } lng, lat = data_file.Bd09towgs84(lng, lat) if saveToDB { //c.Insert(fields) bulk.Insert(bson.M{"mac": mac, "dst_ip": Ip, "dst_port": port, "longitude": lng, "latitude": lat, "org_code": orgcode, "time": uint32(time)}) } } bulk.Run() }
func ExportTrace() { session := db.GetDBSession() defer db.ReleaseDBSession(session) traceArr := make([]TraceDBInfo, 0) err := session.DB("detector").C("detector_report").Find(bson.M{"org_code": bson.M{"$ne": "555400905"}}).Sort("-_id").Limit(1000).All(&traceArr) if err != nil { log.Println(err) return } outArr := make([]TraceInfo, 0) for _, e := range traceArr { detectorDBInfo := DetectorDBInfo{} session.DB("detector").C("detector_info").FindId(e.ApMac).One(&detectorDBInfo) ApMac := strings.ToUpper(e.ApMac[len(e.ApMac)-12:]) var trace TraceInfo trace.MAC = FormatMac(e.DeviceMac) trace.TYPE = 2 trace.START_TIME = e.Time trace.BSSID = FormatMac(ApMac) trace.XPOINT = strconv.FormatFloat(e.Longitude, 'f', 6, 64) trace.YPOINT = strconv.FormatFloat(e.Latitude, 'f', 6, 64) trace.DEVMAC = FormatMac(ApMac) trace.DEVICENUM = OrgCode + ApMac trace.SERVICECODE = ServiceCodePrefix + fmt.Sprintf("%06d", detectorDBInfo.No) outArr = append(outArr, trace) } jsonString, err := json.Marshal(outArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "001") }
func ExportAPTrace() { session := db.GetDBSession() defer db.ReleaseDBSession(session) detectorArr := make([]DetectorDBInfo, 0) err := session.DB("detector").C("detector_info").Find(bson.M{"company": "02"}).All(&detectorArr) if err != nil { log.Println(err) return } outArr := make([]TraceInfo, 0) for _, e := range detectorArr { if len(e.Mac) < 12 { continue } service := protocol.ServiceInfo{} err := session.DB("platform").C("service").Find(bson.M{"id": e.OrgCode + "_" + e.NetbarWacode}).One(&service) if err != nil { continue } no, _ := strconv.Atoi(service.NO) service.SERVICE_CODE = service.NETBAR_WACODE[:8] + fmt.Sprintf("%06d", no) traceArr := make([]TraceDBInfo, 0) err = session.DB("detector").C("detector_report").Find(bson.M{"ap_mac": e.Mac}).Sort("-time").Limit(1).All(&traceArr) if err != nil { log.Println(err) return } for _, e := range traceArr { detectorDBInfo := DetectorDBInfo{} session.DB("detector").C("detector_info").FindId(e.ApMac).One(&detectorDBInfo) ApMac := strings.ToUpper(e.ApMac[len(e.ApMac)-12:]) var trace TraceInfo trace.MAC = FormatMac(e.DeviceMac) trace.TYPE = 2 trace.START_TIME = e.Time trace.BSSID = FormatMac(ApMac) trace.XPOINT = strconv.FormatFloat(e.Longitude, 'f', 6, 64) trace.YPOINT = strconv.FormatFloat(e.Latitude, 'f', 6, 64) trace.DEVMAC = FormatMac(ApMac) trace.DEVICENUM = OrgCode + ApMac trace.SERVICECODE = service.SERVICE_CODE outArr = append(outArr, trace) } } jsonString, err := json.Marshal(outArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "001") }
func ExportDetectorInfo() { session := db.GetDBSession() defer db.ReleaseDBSession(session) detectorArr := make([]DetectorDBInfo, 0) err := session.DB("detector").C("detector_info").Find(bson.M{"company": "01"}).All(&detectorArr) if err != nil { log.Println(err) return } outArr := make([]DetectorInfo, 0) outServiceArr := make([]ServiceInfo, 0) for _, e := range detectorArr { if len(e.Mac) < 12 { continue } service := ExportService(e.No, e.Longitude, e.Latitude) outServiceArr = append(outServiceArr, service) Mac := strings.ToUpper(e.Mac[len(e.Mac)-12:]) var detector DetectorInfo detector.MAC = FormatMac(Mac) detector.EQUIPMENT_NUM = OrgCode + Mac detector.EQUIPMENT_NAME = "广晟通信_梅州_" + Mac[6:] detector.SECURITY_FACTORY_ORGCODE = OrgCode detector.SERVICE_CODE = service.SERVICE_CODE detector.PROVINCE_CODE = "440000" detector.CITY_CODE = "441400" detector.AREA_CODE = service.AREA_CODE detector.EQUIPMENT_TYPE = "10" detector.LATITUDE = strconv.FormatFloat(e.Latitude, 'f', 6, 64) detector.LONGITUDE = strconv.FormatFloat(e.Longitude, 'f', 6, 64) detector.CREATE_TIME = "2016-07-03 12:32:00" detector.LAST_CONNECT_TIME = time.Now().Format("2006-01-02 15:04:05") detector.WDA_VERSION = "1.10" detector.FIRMWARE_VERSION = "1.0" detector.COLLECTION_RADIUS = 150 detector.UPLOAD_TIME_INTERVAL = 60 detector.CREATER = "黄工" outArr = append(outArr, detector) } jsonString, err := json.Marshal(outArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "010") jsonString, err = json.Marshal(outServiceArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "008") }
func SaveFeature(waitgroup *sync.WaitGroup, f1 Feature, f2 Feature) error { if f1.Type == "" || f2.Type == "" { log.Println("SaveFeature:type is empty, orgCode:" + f1.OrgCode) waitgroup.Done() return errors.New("type is empty") } if strings.Contains(f1.Type, " ") { log.Println("SaveFeature:type contains space, orgCode:" + f1.OrgCode + ", type:" + f1.Type) f1.Type = strings.Replace(f1.Type, " ", "", -1) } if strings.Contains(f2.Type, " ") { log.Println("SaveFeature:type contains space, orgCode:" + f2.OrgCode + ", type:" + f2.Type) f2.Type = strings.Replace(f2.Type, " ", "", -1) } session := db.GetDBSession() defer db.ReleaseDBSession(session) c := session.DB("feature").C("feature_set") f1Obj := bson.M{} f2Obj := bson.M{} SaveFeatureLocker.Lock() err1 := c.Find(bson.M{"feature.value": f1.Value, "feature.type": f1.Type}).One(f1Obj) err2 := c.Find(bson.M{"feature.value": f2.Value, "feature.type": f2.Type}).One(f2Obj) if err1 == nil && err2 == nil { if f1Obj["_id"].(string) != f2Obj["_id"].(string) { //merge feature2, ok := f2Obj["feature"] if ok { c.UpdateId(f1Obj["_id"].(string), bson.M{"$pushAll": bson.M{"feature": feature2.([]interface{})}}) c.RemoveId(f2Obj["_id"].(string)) } } } else if err1 == nil && err2 != nil { //push f2 in f1 c.UpdateId(f1Obj["_id"].(string), bson.M{"$push": bson.M{"feature": bson.M{"type": f2.Type, "value": f2.Value, "org_code": f2.OrgCode, "time": f2.Time}}}) } else if err1 != nil && err2 == nil { //push f1 in f2 c.UpdateId(f2Obj["_id"].(string), bson.M{"$push": bson.M{"feature": bson.M{"type": f1.Type, "value": f1.Value, "org_code": f1.OrgCode, "time": f1.Time}}}) } else { //insert new featureArr := []bson.M{bson.M{"type": f1.Type, "value": f1.Value, "org_code": f1.OrgCode, "time": f1.Time}, bson.M{"type": f2.Type, "value": f2.Value, "org_code": f2.OrgCode, "time": f2.Time}} c.Insert(bson.M{"_id": bson.NewObjectId().Hex(), "feature": featureArr}) } SaveFeatureV2(session, f1, f2) SaveFeatureLocker.Unlock() waitgroup.Done() return nil }
func ExportAPInfo() { session := db.GetDBSession() defer db.ReleaseDBSession(session) detectorArr := make([]DetectorDBInfo, 0) err := session.DB("detector").C("detector_info").Find(bson.M{"company": "02"}).All(&detectorArr) if err != nil { log.Println(err) return } outArr := make([]DetectorInfo, 0) for _, e := range detectorArr { if len(e.Mac) < 12 { continue } service := protocol.ServiceInfo{} err := session.DB("platform").C("service").Find(bson.M{"id": e.OrgCode + "_" + e.NetbarWacode}).One(&service) if err != nil { continue } no, _ := strconv.Atoi(service.NO) service.SERVICE_CODE = service.NETBAR_WACODE[:8] + fmt.Sprintf("%06d", no) Mac := strings.ToUpper(e.Mac[len(e.Mac)-12:]) var detector DetectorInfo detector.MAC = FormatMac(Mac) detector.EQUIPMENT_NUM = OrgCode + Mac detector.EQUIPMENT_NAME = service.SERVICE_NAME + Mac[6:] detector.SECURITY_FACTORY_ORGCODE = OrgCode detector.SERVICE_CODE = service.SERVICE_CODE detector.PROVINCE_CODE = service.PROVINCE_CODE detector.CITY_CODE = service.CITY_CODE detector.AREA_CODE = service.AREA_CODE detector.EQUIPMENT_TYPE = "00" detector.LATITUDE = strconv.FormatFloat(e.Latitude, 'f', 6, 64) detector.LONGITUDE = strconv.FormatFloat(e.Longitude, 'f', 6, 64) detector.CREATE_TIME = "2016-07-03 12:32:00" detector.LAST_CONNECT_TIME = time.Now().Format("2006-01-02 15:04:05") detector.WDA_VERSION = "1.10" detector.FIRMWARE_VERSION = "1.0" detector.COLLECTION_RADIUS = 150 detector.UPLOAD_TIME_INTERVAL = 60 detector.CREATER = service.PERSON_NAME outArr = append(outArr, detector) } jsonString, err := json.Marshal(outArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "010") }
func ExportDeviceInfo() { rawData := make([]RawData, 0) session := db.GetDBSession() defer db.ReleaseDBSession(session) err := session.DB("3rd_data").C("raw_data").Find(bson.M{"org_code": "555400905", "type": "WA_SOURCE_FJ_0001"}).Sort("-_id").Limit(10).All(&rawData) if err != nil { log.Println(err) return } outArr := make([]protocol.DeviceInfo, 0) for _, data := range rawData { fields := make(map[string]string) for _, e := range data.Fields { fields[e.Key] = e.Value } wc := fields["G020004"] service := protocol.ServiceInfo{} err := session.DB("platform").C("service").Find(bson.M{"id": data.OrgCode + "_" + wc}).One(&service) if err != nil { continue } no, _ := strconv.Atoi(service.NO) service.SERVICE_CODE = service.NETBAR_WACODE[:8] + fmt.Sprintf("%06d", no) deviceInfo := protocol.DeviceInfo{} deviceInfo.SERVICE_CODE = service.SERVICE_CODE tmp, _ := strconv.ParseUint(fields["H010015"], 10, 32) deviceInfo.ONLINE_TIME = uint32(tmp) deviceInfo.NET_ENDING_NAME = "" tmp, _ = strconv.ParseUint(fields["F020001"], 10, 32) deviceInfo.NET_ENDING_IP = uint32(tmp) deviceInfo.NET_ENDING_MAC = fields["C040002"] deviceInfo.SESSION_ID = fields["H010013"] deviceInfo.AP_MAC = fields["F030011"] deviceInfo.AP_NUM = data.OrgCode + filterMac(deviceInfo.AP_MAC) deviceInfo.COMPANY_ID = data.OrgCode outArr = append(outArr, deviceInfo) } jsonString, err := json.Marshal(outArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "005") }
func CreateServiceNo() (int, error) { session := db.GetDBSession() defer db.ReleaseDBSession(session) change := mgo.Change{ Update: bson.M{"$inc": bson.M{"value": 1}}, ReturnNew: true, Upsert: true, } doc := bson.M{} _, err := session.DB("platform").C("ids").Find(bson.M{"_id": "service_no"}).Apply(change, &doc) if err == nil { no := int(db.GetNumber(doc, "value")) return no, nil } return -1, err }
func SaveRawData(data *data_file.BCPFile) { session := db.GetDBSession() defer db.ReleaseDBSession(session) c := session.DB("3rd_data").C("raw_data") bulk := c.Bulk() orgCode := data.Meta.OrgCode dataType := data.Meta.DataType for _, fields := range data.RawData { RawData := RawData{} RawData.OrgCode = orgCode RawData.Type = dataType RawData.Fields = fields bulk.Insert(RawData) } bulk.Run() }
func ExportServiceFromDB(onlyStatus bool) { session := db.GetDBSession() defer db.ReleaseDBSession(session) serviceArr := make([]protocol.ServiceInfo, 0) outServiceStatusArr := make([]ServiceStatus, 0) err := session.DB("platform").C("service").Find(bson.M{}).All(&serviceArr) if err != nil { log.Println(err) return } for i := range serviceArr { service := &serviceArr[i] no, _ := strconv.Atoi(service.NO) service.SERVICE_CODE = service.NETBAR_WACODE[:8] + fmt.Sprintf("%06d", no) serviceStatus := ServiceStatus{} serviceStatus.SERVICE_CODE = service.SERVICE_CODE serviceStatus.SERVICE_ONLINE_STATUS = 1 serviceStatus.DATA_ONLINE_STATUS = 1 serviceStatus.EQUIPMENT_RUNNING_STATUS = 1 serviceStatus.ACTIVE_PC = 0 serviceStatus.REPORT_PC = 0 serviceStatus.ONLINE_PERSON = 0 serviceStatus.VITRUAL_NUM = 0 serviceStatus.EXIT_IP = "0.0.0.0" serviceStatus.UPDATE_TIME = time.Now().Format("2006-01-02 15:04:05") outServiceStatusArr = append(outServiceStatusArr, serviceStatus) } if !onlyStatus { jsonString, err := json.Marshal(serviceArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "008") } jsonString, err := json.Marshal(outServiceStatusArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "009") }
func SaveServiceInfo(data *data_file.BCPFile) { session := db.GetDBSession() defer db.ReleaseDBSession(session) orgCode := data.Meta.OrgCode for _, fields := range data.KeyFields { serviceInfo := protocol.ServiceInfo{} serviceInfo.NETBAR_WACODE = fields["G020004"] serviceInfo.ID = orgCode + "_" + serviceInfo.NETBAR_WACODE serviceInfo.SERVICE_NAME = fields["F040002"] serviceInfo.PRINCIPAL = fields["E020001"] serviceInfo.PERSON_NAME = fields["E020001"] serviceInfo.PERSON_TEL = fields["B070003"] serviceInfo.BUSINESS_NATURE = fields["E010007"] serviceInfo.STATUS = 1 serviceInfo.SERVICE_TYPE = 9 serviceInfo.PROVINCE_CODE = serviceInfo.NETBAR_WACODE[:2] + "0000" serviceInfo.CITY_CODE = serviceInfo.NETBAR_WACODE[:4] + "00" serviceInfo.AREA_CODE = serviceInfo.NETBAR_WACODE[:6] serviceInfo.ADDRESS = fields["G020017"] serviceInfo.ORG_CODE = orgCode lng, _ := strconv.ParseFloat(fields["F010016"], 64) lat, _ := strconv.ParseFloat(fields["F010017"], 64) lng, lat = data_file.Bd09towgs84(lng, lat) serviceInfo.XPOINT = fmt.Sprintf("%.6f", lng) serviceInfo.YPOINT = fmt.Sprintf("%.6f", lat) serviceInfo.CREATE_TIME = "2016-07-02 00:00:00" serviceInfo.CAP_TYPE = "2" serviceInfoOld := protocol.ServiceInfo{} err := session.DB("platform").C("service").Find(bson.M{"id": serviceInfo.ID}).One(&serviceInfoOld) if err != nil { no, err := CreateServiceNo() serviceInfo.NO = strconv.Itoa(no) if err != nil { continue } session.DB("platform").C("service").Insert(serviceInfo) } else { serviceInfo.NO = serviceInfoOld.NO session.DB("platform").C("service").Update(bson.M{"id": serviceInfo.ID}, serviceInfo) } } }
func ExportServiceStatus() { session := db.GetDBSession() defer db.ReleaseDBSession(session) detectorArr := make([]DetectorDBInfo, 0) err := session.DB("detector").C("detector_info").Find(bson.M{"company": "01"}).All(&detectorArr) if err != nil { log.Println(err) return } outServiceStatusArr := make([]ServiceStatus, 0) for _, e := range detectorArr { if len(e.Mac) < 12 { continue } serviceStatus := ServiceStatus{} serviceStatus.SERVICE_CODE = ServiceCodePrefix + fmt.Sprintf("%06d", e.No) serviceStatus.SERVICE_ONLINE_STATUS = 1 if e.Latitude != 0 { serviceStatus.DATA_ONLINE_STATUS = 1 serviceStatus.EQUIPMENT_RUNNING_STATUS = 1 } else { serviceStatus.DATA_ONLINE_STATUS = 2 serviceStatus.EQUIPMENT_RUNNING_STATUS = 2 } serviceStatus.ACTIVE_PC = 0 serviceStatus.REPORT_PC = 0 serviceStatus.ONLINE_PERSON = 0 serviceStatus.VITRUAL_NUM = 0 serviceStatus.EXIT_IP = "0.0.0.0" serviceStatus.UPDATE_TIME = time.Now().Format("2006-01-02 15:04:05") outServiceStatusArr = append(outServiceStatusArr, serviceStatus) } jsonString, err := json.Marshal(outServiceStatusArr) if err != nil { return } log.Print(string(jsonString)) SaveFile(string(jsonString), "009") }
func UpdateApData(orgcode string, data []map[string]string) { session := db.GetDBSession() defer db.ReleaseDBSession(session) c := session.DB("detector").C("detector_info") for i, fields := range data { mac := fields["AP_MAC"] mac = filterMac(mac) netbar_wacode := fields["NETBAR_WACODE"] lng, err1 := strconv.ParseFloat(fields["LONGITUDE"], 64) lat, err2 := strconv.ParseFloat(fields["LATITUDE"], 64) if orgcode == "779852855" && err2 != nil { lat, err2 = strconv.ParseFloat(fields["LAITTUDE"], 64) } if err1 != nil || err2 != nil { continue } lng, lat = data_file.Bd09towgs84(lng, lat) log.Println(i, ": mac", mac, "lng", lng, "lat", lat) if saveToDB { c.UpsertId(mac, bson.M{"_id": mac, "longitude": lng, "latitude": lat, "last_active_time": uint32(time.Now().Unix()), "company": "02", "org_code": orgcode, "netbar_wacode": netbar_wacode}) } } }
func StatsFeatureNum(orgCode string) { session := db.GetDBSession() defer db.ReleaseDBSession(session) count, _ := session.DB("feature").C("feature").Find(bson.M{"org_code": orgCode}).Count() fmt.Println("feature_num:", count) }
func StatsDeviceLogNum(orgCode string) { session := db.GetDBSession() defer db.ReleaseDBSession(session) count, _ := session.DB("person_info").C("behavior_log").Find(bson.M{"org_code": orgCode}).Count() fmt.Println("log_num:", count) }
func StatsDeviceFeatureNum(orgCode string) { session := db.GetDBSession() defer db.ReleaseDBSession(session) count, _ := session.DB("person_info").C("mac").Find(bson.M{"org_code": orgCode}).Count() fmt.Println("device_feature_num:", count) }
func StatsDetectorNum(orgCode string) { session := db.GetDBSession() defer db.ReleaseDBSession(session) count, _ := session.DB("detector").C("detector_info").Find(bson.M{"org_code": orgCode}).Count() fmt.Println("detector_num:", count) }
func SaveDeviceInfo(orgcode string, data []map[string]string) { session := db.GetDBSession() defer db.ReleaseDBSession(session) c := session.DB("person_info").C("mac") bulk := c.Bulk() var waitgroup sync.WaitGroup for i, fields := range data { mac := fields["MAC"] mac = filterMac(mac) ap_mac := fields["AP_MAC"] ap_mac = filterMac(ap_mac) authType := fields["AUTH_TYPE"] authAccount := fields["AUTH_ACCOUNT"] time, err3 := strconv.Atoi(fields["START_TIME"]) if err3 != nil { continue } log.Println(i, ": mac", mac, "ap_mac", ap_mac, "auth_type", authType, "account", authAccount, "time", time) if saveToDB { f1 := data_import.Feature{} f2 := data_import.Feature{} f1.Type = "1020002" f1.Value = mac f1.OrgCode = orgcode f1.Time = uint32(time) if authType == "1020004" { bulk.Upsert(bson.M{"mac": mac, "phone": authAccount}, bson.M{"mac": mac, "phone": authAccount, "org_code": orgcode, "time": uint32(time)}) } else if authType == "1029999" { if isPhoneNo(authAccount) { bulk.Upsert(bson.M{"mac": mac, "phone": authAccount}, bson.M{"mac": mac, "phone": authAccount, "org_code": orgcode, "time": uint32(time)}) } } if authType == "1029999" { if isPhoneNo(authAccount) { f2.Type = "1020004" f2.Value = authAccount f2.OrgCode = orgcode f2.Time = uint32(time) } else { continue } } else { f2.Type = authType if f2.Type == "1020002" { f2.Value = filterMac(authAccount) } else { f2.Value = authAccount } f2.OrgCode = orgcode f2.Time = uint32(time) } waitgroup.Add(1) go data_import.SaveFeature(&waitgroup, f1, f2) } } bulk.Run() waitgroup.Wait() }