示例#1
0
文件: dnsapi.go 项目: aarpy/whcrawler
// NewDNSCache is a great class
func NewDNSCache(dnsServer string, dnsConcurency int, dnsRetryTime string, groupCacheSize int64, redisHost string) api.DNSCache {
	log.Info("Creating new DNS Cache Resolver")

	resolveMgr := resolve.NewResolver(dnsServer, dnsConcurency, 120, dnsRetryTime, true, false)

	log.Info("Creating new DNS Cache Manager")

	cacheMgr := cache.NewCache(groupCacheSize, redisHost, func(request *api.ValueRequest) {

		log.WithField("domain", request.Key).Info("DnsApi:ResolverRequest")

		resolverRequest := api.NewValueRequest(request.Key)
		resolveMgr.Resolve(resolverRequest)
		resolverResponse := <-resolverRequest.Response

		log.WithFields(log.Fields{
			"domain": request.Key,
			"IP":     resolverResponse.Value,
		}).Info("DnsApi:ResolverRequest:Done")

		// Request from DNS
		request.Response <- resolverResponse
	})

	return &dnsCacheMgr{cache: cacheMgr, resolver: resolveMgr}
}
示例#2
0
// NewGroupCache function
func NewGroupCache(cacheSize int64, redisGetFunc GetFunc) Cache {
	return &groupCacheMgr{
		group: gc.NewGroup(singleGroupName, cacheSize, gc.GetterFunc(func(ctx gc.Context, key string, dest gc.Sink) error {
			log.WithField("domain", key).Info("GroupCache:RedisRequest")

			redisRequest := api.NewValueRequest(key)
			redisGetFunc(redisRequest)
			redisResponse := <-redisRequest.Response

			log.WithField("domain", key).Info("GroupCache:RedisRequest:Done")
			return dest.SetString(redisResponse.Value)
		}))}
}
示例#3
0
func (c *redisCache) GetValue(request *api.ValueRequest) {
	log.WithField("domain", request.Key).Info("RedisCache:GetValue:Start")

	// Get from Redis
	value, err := c.client.Get(request.Key).Result()

	log.WithFields(log.Fields{
		"domain": request.Key,
		"value":  value,
		"err":    err,
	}).Info("RedisCache:GetValue:GetComplete")

	// Value found without error or empty
	if err == nil {
		// Notify the calling group cache
		request.Response <- api.NewValueResponse(value, nil)
		return
	}

	log.WithField("domain", request.Key).Info("RedisCache:GetValue:CheckResolver")

	// Request Resolver
	resolverRequest := api.NewValueRequest(request.Key)
	c.resolverFunc(resolverRequest)
	resolverResponse := <-resolverRequest.Response

	log.WithFields(log.Fields{
		"domain": request.Key,
		"value":  value,
	}).Info("RedisCache:GetValue:FromResolver")

	// Save it to Redis irrespectively to ensure no requests are sent to Resolver
	c.client.Set(request.Key, resolverResponse.Value, 0)

	log.WithField("domain", request.Key).Info("RedisCache:GetValue:SetComplete")

	// Notify the calling group cache
	request.Response <- resolverResponse

	log.WithField("domain", request.Key).Info("RedisCache:GetValue:Done")
}
示例#4
0
文件: dnsapi.go 项目: aarpy/whcrawler
func (d *dnsCacheMgr) GetIP(domainName string, getIPFunc api.GetIPFunc) {

	log.WithField("domain", domainName).Info("DnsApi:GetIP:Start")

	request := api.NewValueRequest(domainName)

	d.cache.GetValue(request)

	select {
	case response, ok := <-request.Response:
		if ok {
			log.WithFields(log.Fields{
				"domain": domainName,
				"value":  response.Value,
				"err":    response.Err}).Info("DnsApi:GetIP:Response")

			ipStrings := strings.Split(response.Value, " ")
			ipNumbers := make([]net.IP, len(ipStrings))
			if response.Value != "" {
				for i, ipString := range ipStrings {
					ipNumbers[i] = net.ParseIP(ipString)
				}
			}

			getIPFunc(ipNumbers, nil)
		} else {
			log.WithField("domain", domainName).Info("DnsApi:GetIP:ChannelClosed")

			// channel was closed
			getIPFunc(nil, errors.New("Channel closed"))
		}
		break
	case <-time.After(time.Duration(10 * time.Second)):
		log.WithField("domain", domainName).Info("DnsApi:GetIP:ChannelTimeout")
		// channel failed to return in time
		getIPFunc(nil, errors.New("Timeout occurred"))
		break
	}

	log.WithField("domain", domainName).Info("DnsApi:GetIP:Done")
}