func integrityVerify(filename string) bool {
iplist := iputil.GetDetectedIpInfoSlice(filename)
bIntegrity := true
value := iplist[0]
fmt.Println(value)
for i, ipMap := range iplist {
if i == 0 {
continue
}
if ipMap["end"] == "" {
ipMap["end"] = ipMap["ip"]
ipMap["len"] = "1"
}
testip1 := iputil.InetAtonInt(value["end"])
testip2 := iputil.InetAtonInt(ipMap["ip"])
if isSequentail(testip1, testip2) == false {
ip11 := iputil.InetAtonInt(value["end"]) + 1
ip22 := iputil.InetAtonInt(ipMap["ip"]) - 1
sip1 := iputil.InetNtoaStr(ip11)
sip2 := iputil.InetNtoaStr(ip22)
fmt.Println(sip1 + "|" + sip2)
bIntegrity = false
}
value = ipMap
}
return bIntegrity
}
func stillNeedDetectIPNum(filename string) int {
fp, err := os.Open(filename)
if err != nil {
fmt.Println("failed")
return 0
}
defer fp.Close()
br := bufio.NewReader(fp)
var count int64 = 0
for {
line, e := br.ReadString('\n')
if e != nil {
fmt.Println("read end of file")
break
}
line = strings.TrimSuffix(line, "\n")
ipip := strings.Split(line, "|")
ip1 := ipip[0]
ip2 := ipip[1]
count += iputil.InetAtonInt(ip2) - iputil.InetAtonInt(ip1) + 1
}
//fmt.Printf("still has %d ips", count)
return int(count)
}
func SaveSameNetwork(startip, endip string, ipinfoMap interface{}, fileFP *os.File) {
ipmap := ipinfoMap.(map[string]string)
if ipmap == nil {
url := fmt.Sprintf("%s%s", taobaoURL, endip)
ipmap, _ = iputil.ParseUrlToMap(url)
}
lens := iputil.InetAtonInt(endip) - iputil.InetAtonInt(startip) + 1
result := iputil.KeyinfoFormatToOutput(ipmap)
fileFP.WriteString(startip + "|" + endip + "|" + strconv.Itoa(int(lens)) + "|" + result + "\n")
fileFP.Sync()
}
func detect_ip(result chan string, hk *temp_data, newip string) {
head:
startIpInfo, e1 := iputil.ParseUrlToMap(hk.url)
endIpInfo, e2 := iputil.ParseUrlToMap(hk.endIPUrl)
if startIpInfo["country_id"] == "" {
println("NULL VALUE", startIpInfo)
time.Sleep(1 * 1e9)
goto head
}
if e2 && e1 && startIpInfo["country_id"] == endIpInfo["country_id"] && startIpInfo["isp_id"] == endIpInfo["isp_id"] && startIpInfo["city_id"] == endIpInfo["city_id"] {
fmt.Println(startIpInfo)
info := iputil.KeyinfoFormatToOutput(startIpInfo)
address := fmt.Sprintf("%s|%s|%d|%s\n", startIpInfo["ip"], newip, (1 + iputil.InetAtonInt(newip) - iputil.InetAtonInt(startIpInfo["ip"])), info)
fmt.Printf("!!!ok!!!", address)
result <- address
} else {
info1 := iputil.KeyinfoFormatToOutput(startIpInfo) + "|" + newip
info1 = fmt.Sprintf("%s|lineno:%d\n", info1, hk.fileno)
info2 := iputil.KeyinfoFormatToOutput(endIpInfo) + "\n"
fmt.Printf("$$$", info1)
fmt.Printf("$$$", info2)
hk.fp.WriteString(info1)
hk.fp.WriteString(info2)
hk.fp.Sync()
result <- "NOTSAME"
}
}
func ipCheck(filename string) bool {
fp, err := os.Open(filename)
if err != nil {
fmt.Println("open failed")
return false
}
defer fp.Close()
br := bufio.NewReader(fp)
i := 0
var ip string
var lens int
var lastline string
checkResult := true
for {
line, e := br.ReadString('\n')
if e != nil {
fmt.Println("end of file")
break
}
info := strings.Split(line, "|")
if i == 0 {
ip = info[0]
lens, _ = strconv.Atoi(info[1])
lastline = line
i++
continue
}
nextip := info[0]
nextlens, _ := strconv.Atoi(info[1])
if iputil.InetAtonInt(ip)+int64(lens) == iputil.InetAtonInt(nextip) {
ip = nextip
lens = nextlens
} else {
fmt.Printf(lastline)
fmt.Printf(line)
fmt.Println("----------")
checkResult = false
ip = nextip
lens = nextlens
}
lastline = line
}
return checkResult
}
func CalcuAndSplit(startip, endip string, ipinfoMap map[string]interface{}, resultFP, middleresultFP *os.File) {
defer func() {
if r := recover(); r != nil {
fmt.Println("PPPPPPPPPPPPPPP get panic", r)
}
}()
var startipMap map[string]string
var endipMap map[string]string
fmt.Println("-------------------------------------------------")
fmt.Println("startip|endip|" + startip + "|" + endip)
fmt.Println("-------------------------------------------------")
info1, b1 := ipinfoMap[startip]
if b1 == false {
url1 := fmt.Sprintf("%s%s", taobaoURL, startip)
startipMap, _ = iputil.ParseUrlToMap(url1)
} else {
startipMap = info1.(map[string]string)
}
ipinfoMap[startip] = startipMap
result1 := iputil.KeyinfoFormatToOutput(startipMap)
middleresultFP.WriteString(startip + "|" + startip + "|1|" + result1 + "\n")
if startip == endip {
resultFP.WriteString(startip + "|" + endip + "\n")
return
}
info2, b2 := ipinfoMap[endip]
if b2 == false {
url2 := fmt.Sprintf("%s%s", taobaoURL, endip)
endipMap, _ = iputil.ParseUrlToMap(url2)
} else {
endipMap = info2.(map[string]string)
}
ipinfoMap[endip] = endipMap
result2 := iputil.KeyinfoFormatToOutput(endipMap)
middleresultFP.WriteString(endip + "|" + endip + "|1|" + result2 + "\n")
ip1 := iputil.InetAtonInt(startip)
ip2 := iputil.InetAtonInt(endip)
for ip1 < ip2 {
m := (ip1 + ip2) / 2
ip1_str := iputil.InetNtoaStr(ip1)
ip2_str := iputil.InetNtoaStr(ip2)
mip := iputil.InetNtoaStr(m)
mip_rfirst := iputil.InetNtoaStr(m + 1)
fmt.Println("start|middle-ip|end", ip1_str, mip, ip2_str)
url1 := fmt.Sprintf("%s%s", taobaoURL, mip)
url2 := fmt.Sprintf("%s%s", taobaoURL, mip_rfirst)
var mipinfo1, mipinfo2 map[string]string
mipInfo1, exist1 := ipinfoMap[mip]
if exist1 == false {
mipinfo1, _ = iputil.ParseUrlToMap(url1)
ipinfoMap[mip] = mipinfo1
} else {
mipinfo1 = mipInfo1.(map[string]string)
}
mipInfo2, exist2 := ipinfoMap[mip_rfirst]
if exist2 == false {
mipinfo2, _ = iputil.ParseUrlToMap(url2)
ipinfoMap[mip_rfirst] = mipinfo2
} else {
mipinfo2 = mipInfo2.(map[string]string)
}
/*store middle detect result*/
result1 := iputil.KeyinfoFormatToOutput(mipinfo1)
result2 := iputil.KeyinfoFormatToOutput(mipinfo2)
middleresultFP.WriteString(mip + "|" + mip + "|1|" + result1 + "\n")
middleresultFP.WriteString(mip_rfirst + "|" + mip_rfirst + "|1|" + result2 + "\n")
middleresultFP.Sync()
var finded string
finded = QualifiedIpAtLevel("country", mipinfo1, startipMap, endipMap)
fmt.Println("country finded:", finded)
switch finded {
case goon:
finded = QualifiedIpAtLevel("isp", mipinfo1, startipMap, endipMap)
fmt.Println("isp finded:", finded)
switch finded {
case goon:
finded = QualifiedIpAtLevel("region", mipinfo1, startipMap, endipMap)
fmt.Println("province finded:", finded)
switch finded {
case goon:
fmt.Println("this is same network")
SaveSameNetwork(ip1_str, ip2_str, ipinfoMap[ip1_str], resultFP)
return
case leftmove:
ip1 = m + 1
SaveSameNetwork(ip1_str, mip, ipinfoMap[ip1_str], resultFP)
case rightmove:
ip2 = m
SaveSameNetwork(mip_rfirst, ip2_str, ipinfoMap[mip_rfirst], resultFP)
case morenetwork:
CalcuAndSplit(ip1_str, mip, ipinfoMap, resultFP, middleresultFP)
CalcuAndSplit(mip_rfirst, ip2_str, ipinfoMap, resultFP, middleresultFP)
return
}
case leftmove:
SaveSameNetwork(ip1_str, mip, ipinfoMap[ip1_str], resultFP)
ip1 = m + 1
case rightmove:
SaveSameNetwork(mip_rfirst, ip2_str, ipinfoMap[mip_rfirst], resultFP)
ip2 = m
case morenetwork:
CalcuAndSplit(ip1_str, mip, ipinfoMap, resultFP, middleresultFP)
CalcuAndSplit(mip_rfirst, ip2_str, ipinfoMap, resultFP, middleresultFP)
return
}
case leftmove:
SaveSameNetwork(ip1_str, mip, ipinfoMap[ip1_str], resultFP)
ip1 = m + 1
case rightmove:
SaveSameNetwork(mip_rfirst, ip2_str, ipinfoMap[mip_rfirst], resultFP)
ip2 = m
case morenetwork:
CalcuAndSplit(ip1_str, mip, ipinfoMap, resultFP, middleresultFP)
CalcuAndSplit(mip_rfirst, ip2_str, ipinfoMap, resultFP, middleresultFP)
return
}
}
}
func (obj ByIP) Less(i, j int) bool {
len1 := iputil.InetAtonInt(obj[i]["ip"])
len2 := iputil.InetAtonInt(obj[j]["ip"])
return len1 < len2
}
func MergeIP(filename, mergedFile, breakFile string) bool {
mergeFP, err := os.Create(mergedFile)
if err != nil {
fmt.Println("open file failed")
return false
}
defer mergeFP.Close()
breakFP, err := os.Create(breakFile)
if err != nil {
fmt.Println("open file failed")
return false
}
defer breakFP.Close()
iplist := iputil.GetDetectedIpInfoSlice(filename)
bIntegrity := true
current := iplist[0]
for i, ipMap := range iplist {
if i == 0 {
continue
}
if ipMap["end"] == "" {
ipMap["end"] = ipMap["ip"]
ipMap["len"] = "1"
}
testip1 := iputil.InetAtonInt(current["end"])
testip2 := iputil.InetAtonInt(ipMap["ip"])
if testip1 == testip2 {
if EqualOfTwoNetwork(current, ipMap) == true {
current = ipMap
} else {
fmt.Println("ERROR CURR:", current)
fmt.Println("ERROR IPMAP:", ipMap)
}
current = ipMap
} else if testip1+1 < testip2 {
newgInfo := iputil.AllKeyinfoFormatToOutput(current)
mergeFP.WriteString(newgInfo + "\n")
ip11 := iputil.InetAtonInt(current["end"]) + 1
ip22 := iputil.InetAtonInt(ipMap["ip"]) - 1
sip1 := iputil.InetNtoaStr(ip11)
sip2 := iputil.InetNtoaStr(ip22)
breakFP.WriteString(sip1 + "|" + sip2 + "\n")
bIntegrity = false
current = ipMap
} else if testip1+1 > testip2 {
current = ipMap
} else if testip1+1 == testip2 {
if EqualOfTwoNetwork(current, ipMap) == true {
//info1 := iputil.AllKeyinfoFormatToOutput(current)
//info2 := iputil.AllKeyinfoFormatToOutput(ipMap)
//fmt.Println(info1)
//fmt.Println(info2)
current["end"] = ipMap["end"]
l1, _ := strconv.Atoi(current["len"])
l2, _ := strconv.Atoi(ipMap["len"])
current["len"] = strconv.Itoa(l1 + l2)
} else {
newgInfo := iputil.AllKeyinfoFormatToOutput(current)
mergeFP.WriteString(newgInfo + "\n")
//fmt.Println("11111 not equal start!!!")
//fmt.Println(current)
//fmt.Println(ipMap)
//fmt.Println("11111 not equal end!!!")
current = ipMap
}
}
}
return bIntegrity
}
