コード例 #1
0
ファイル: groupcache.go プロジェクト: aarpy/whcrawler
func (c *groupCacheMgr) GetValue(request *api.ValueRequest) {
	log.WithField("domain", request.Key).Info("GroupCache:GetValue")

	var value string
	err := c.group.Get(request, request.Key, gc.StringSink(&value))

	log.WithFields(log.Fields{
		"domain": request.Key,
		"value":  value,
	}).Info("GroupCache:GetValue:Get")

	// send response to client
	request.Response <- api.NewValueResponse(value, err)
	close(request.Response)

	log.WithField("domain", request.Key).Info("GroupCache:GetValue:Done")
}
コード例 #2
0
ファイル: rediscache.go プロジェクト: aarpy/whcrawler
func (c *redisCache) GetValue(request *api.ValueRequest) {
	log.WithField("domain", request.Key).Info("RedisCache:GetValue:Start")

	// Get from Redis
	value, err := c.client.Get(request.Key).Result()

	log.WithFields(log.Fields{
		"domain": request.Key,
		"value":  value,
		"err":    err,
	}).Info("RedisCache:GetValue:GetComplete")

	// Value found without error or empty
	if err == nil {
		// Notify the calling group cache
		request.Response <- api.NewValueResponse(value, nil)
		return
	}

	log.WithField("domain", request.Key).Info("RedisCache:GetValue:CheckResolver")

	// Request Resolver
	resolverRequest := api.NewValueRequest(request.Key)
	c.resolverFunc(resolverRequest)
	resolverResponse := <-resolverRequest.Response

	log.WithFields(log.Fields{
		"domain": request.Key,
		"value":  value,
	}).Info("RedisCache:GetValue:FromResolver")

	// Save it to Redis irrespectively to ensure no requests are sent to Resolver
	c.client.Set(request.Key, resolverResponse.Value, 0)

	log.WithField("domain", request.Key).Info("RedisCache:GetValue:SetComplete")

	// Notify the calling group cache
	request.Response <- resolverResponse

	log.WithField("domain", request.Key).Info("RedisCache:GetValue:Done")
}
コード例 #3
0
ファイル: resolve.go プロジェクト: aarpy/whcrawler
func doMapGaurd(requests <-chan *api.ValueRequest,
	domainSlotAvailable chan<- bool,
	timeoutRegister chan<- *domainRecord,
	timeoutExpired <-chan *domainRecord,
	tryResolving chan<- *domainRecord,
	resolved <-chan *domainAnswer,
	verbose bool) (int, float64) {

	m := make(map[uint16]*domainRecord)

	done := false

	sumTries := 0
	domainCount := 0

	for done == false || len(m) > 0 {
		select {
		case domain := <-requests:
			fmt.Fprintf(os.Stdout, "Found domain: %s\n", domain)
			var id uint16
			for {
				id = uint16(rand.Int())
				if id != 0 && m[id] == nil {
					break
				}
			}
			dr := &domainRecord{id, domain, time.Now(), 1}
			m[id] = dr
			if verbose {
				fmt.Fprintf(os.Stderr, "0x%04x resolving %s\n", id, domain)
			}
			timeoutRegister <- dr
			tryResolving <- dr

		case dr := <-timeoutExpired:
			if m[dr.id] == dr {
				dr.resend++
				dr.timeout = time.Now()
				if verbose {
					fmt.Fprintf(os.Stderr, "0x%04x resend (try:%d) %s\n", dr.id,
						dr.resend, dr.request.Key)
				}
				timeoutRegister <- dr
				tryResolving <- dr
			}

		case da := <-resolved:
			if m[da.id] != nil {
				dr := m[da.id]
				if dr.request.Key != da.domain {
					if verbose {
						fmt.Fprintf(os.Stderr, "0x%04x error, unrecognized domain: %s != %s\n",
							da.id, dr.request.Key, da.domain)
					}
					break
				}

				if verbose {
					fmt.Fprintf(os.Stderr, "0x%04x resolved %s\n",
						dr.id, dr.request.Key)
				}

				s := make([]string, 0, 16)
				for _, ip := range da.ips {
					s = append(s, ip.String())
				}
				sort.Sort(sort.StringSlice(s))

				// without trailing dot
				dr.request.Key = dr.request.Key[:len(dr.request.Key)-1]
				ips := strings.Join(s, " ")
				fmt.Printf("%s, %s\n", dr.request.Key, ips)

				dr.request.Response <- api.NewValueResponse(ips, nil)

				sumTries += dr.resend
				domainCount++

				delete(m, dr.id)
				domainSlotAvailable <- true
			}
		}
	}
	return domainCount, float64(sumTries) / float64(domainCount)
}