Example #1
0
func main() {
	page_count := 1
	download_count := 0

	for {
		offset := page_count * 50
		param := map[string]string{
			"AppId":        appid,
			"Version":      "2.2",
			"Market":       "ja-JP",
			"Sources":      "Image",
			"Image.Count":  strconv.Itoa(50),
			"Image.Offset": strconv.Itoa(offset),
			"Adult":        "off",
			"Query":        "おっぱい",
		}

		var sr *json_root
		res, err := http.Get(get_request_uri(param))
		if err != nil {
			break
		}
		reader := bufio.NewReader(res.Body)
		line, err := reader.ReadBytes('\n')
		if err == nil {
			break
		}
		json.Unmarshal(line, &sr)

		for i := 0; i < len(sr.SearchResponse.Image.Results); i++ {
			result := sr.SearchResponse.Image.Results[i]
			if regexp.MustCompile(".jpg$").FindString(result.MediaUrl) == "" {
				continue
			}
			download_count++

			filename := md5hex(encode_utf8(result.MediaUrl)) + ".jpg"
			filepath := dir + filename

			if _, err := os.Stat(filepath); err == nil {
				continue
			}
			fmt.Printf("%d : Download... %s\n", download_count, result.MediaUrl)

			res, err := http.Get(result.MediaUrl)
			if err != nil {
				runtime.Goexit()
			}
			data, err := ioutil.ReadAll(res.Body)
			if err != nil {
				runtime.Goexit()
			}
			if regexp.MustCompile("^image").FindString(http.DetectContentType(data)) != "" {
				ioutil.WriteFile(filepath, data, 0666)
			}
		}
		page_count++
	}
}
Example #2
0
func main() {
	// This works
	r, _, err := http.Get("https://www.google.com")
	if err != nil {
		log.Exit(err)
	}
	log.Println(r)

	// This doesn't
	r, _, err = http.Get("https://streaming.campfirenow.com")
	if err != nil {
		log.Exit(err)
	}
	log.Println(r)
}
Example #3
0
func testQuestions() {
	r, _, err := http.Get("http://api.stackoverflow.com/1.0/questions?key=change_me&answers=true")
	if err != nil {
		fmt.Println(err.String())
		return
	}

	d, err := gzip.NewReader(r.Body)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	var qr gostack.QuestionsResult
	b, _ := ioutil.ReadAll(d)

	err = json.Unmarshal(b, &qr)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	fmt.Println(qr)

}
Example #4
0
func (req *Googl) get(url string, params ...map[string]string) (string, os.Error) {
	if !strings.Contains(url, "://goo.gl/") {
		return "", os.EINVAL
	}

	req_url := GOOGL_V1 + "?shortUrl=" + url
	if 0 < len(params) {
		for i := 0; i < len(params); i++ {
			req_url += "&" + toQuery(params[i])
		}
	}

	if "" != req.Key {
		req_url += "&key=" + req.Key
	}

	res, _, err := http.Get(req_url)
	defer res.Body.Close()
	if err != nil {
		return "", os.Error(err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	return string(body), nil
}
Example #5
0
func (s *TestHTTPServer) Start() {
	if s.started {
		return
	}
	s.started = true

	s.request = make(chan *http.Request, 64)
	s.response = make(chan *testResponse, 64)
	s.pending = make(chan bool, 64)

	url, _ := url.Parse(s.URL)
	go http.ListenAndServe(url.Host, s)

	s.PrepareResponse(202, nil, "Nothing.")
	fmt.Fprintf(os.Stderr, "\nWaiting for the fake server to be up...")
	for {
		resp, err := http.Get(s.URL)
		if err == nil && resp.StatusCode == 202 {
			break
		}

		time.Sleep(1e8)
	}

	fmt.Fprintf(os.Stderr, "Done\n")
	s.WaitRequest()
}
Example #6
0
func DiscoverXml(id string) (*string, os.Error) {
	resp, _, err := http.Get(id)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	parser := xml.NewParser(resp.Body)
	inURI := false
	for {
		t, err := parser.Token()
		if err != nil {
			return nil, err
		}
		switch tt := t.(type) {
		case xml.StartElement:
			if tt.Name.Local == "URI" {
				inURI = true
			}
		case xml.CharData:
			if inURI {
				s := string([]byte(tt))
				return &s, nil
			}
		}
	}
	return nil, &DiscoveryError{str: "URI not found"}
}
Example #7
0
func (c *Client) StatusesRetweets(id uint64, count int) (t []Tweet) {
	var params string
	var tweets []Tweet

	if id == 0 {
		return nil
	}

	if count != 0 {
		params = addParam(params, "count", fmt.Sprintf("%d", count))
	}

	url := c.makeAuthURL(statusesRetweets+fmt.Sprintf("/%d", id), params)
	res, _, err := http.Get(url)
	if err != nil {
		return nil
	}

	if res.Status != "200 OK" {
		return nil
	}

	reader := bufio.NewReader(res.Body)
	line, _ := reader.ReadString(0)

	print(line + "\n")
	json.Unmarshal(line, &tweets)

	return tweets
}
Example #8
0
File: sns.go Project: supr/sns
func (sns *SNS) query(topic *Topic, message *Message, params map[string]string, resp interface{}) os.Error {
	params["Timestamp"] = time.UTC().Format(time.RFC3339)
	url_, err := url.Parse(sns.Region.SNSEndpoint)
	if err != nil {
		return err
	}

	sign(sns.Auth, "GET", "/", params, url_.Host)
	url_.RawQuery = multimap(params).Encode()
	r, err := http.Get(url_.String())
	if err != nil {
		return err
	}
	defer r.Body.Close()

	//dump, _ := http.DumpResponse(r, true)
	//println("DUMP:\n", string(dump))
	//return nil

	if r.StatusCode != 200 {
		return buildError(r)
	}
	err = xml.Unmarshal(r.Body, resp)
	return err
}
Example #9
0
// scrape scrapes the given URL and saves it to disk
func scrape(url string, done chan bool) {
	// Notify main when we're done right after we return
	defer func() { done <- true }() // Anonymous functions ftw!

	fmt.Printf("Scraping %s...\n", url)
	defer fmt.Printf("Finished scraping %s\n", url)

	// Don't make the user type "http://" for every freaking URL!
	if !strings.Contains(url, "://") {
		url = "http://" + url
	}

	// Download website contents
	req, err := http.Get(url)
	if somethingBroke(err) {
		return
	}

	// Save contents to variable
	contents, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()
	if somethingBroke(err) {
		return
	}

	// Write contents to disk. TODO: Store URL, text data in a DB
	url = strings.Replace(url, "/", "___", -1)
	filename := fmt.Sprintf("%v-%v", url, time.Seconds())
	err = ioutil.WriteFile(SCRAPES_DIR+filename, contents, 0644)
	if somethingBroke(err) {
		return
	}

	return
}
Example #10
0
func fetchUrlToFile(url, filename string, expectedSize int64) bool {
	fi, statErr := os.Stat(filename)
	if statErr == nil &&
		(expectedSize == -1 && fi.Size > 0 ||
			expectedSize == fi.Size) {
		// TODO: re-fetch mode?
		return true
	}

	netop := NewNetworkOperation()
	defer netop.Done()

	res, _, err := http.Get(url)
	if err != nil {
		addError(fmt.Sprintf("Error fetching %s: %v", url, err))
		return false
	}
	defer res.Body.Close()

	fileBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		addError(fmt.Sprintf("Error reading XML from %s: %v", url, err))
		return false
	}

	err = ioutil.WriteFile(filename, fileBytes, 0600)
	if err != nil {
		addError(fmt.Sprintf("Error writing file %s: %v", filename, err))
		return false
	}
	return true
}
func randomQuote() string {
	// Simulates slow connection; shows off `defer`, goroutine asynchronicity
	defer func() { time.Sleep(2e9) }()

	// Scrape site
	url := "http://subfusion.net/cgi-bin/quote.pl?quote=cookie&number=1"
	req, err := http.Get(url)
	checkError(err)

	// Create regex
	re, err := regexp.Compile("<body><br><br><b><hr><br>(.*)<br><br><hr><br>")
	checkError(err)

	// Read HTML
	html, err := ioutil.ReadAll(req.Body)
	req.Body.Close()
	checkError(err)

	// Parse out quote
	quote := re.FindString(string(html))
	quote = strings.Replace(quote, "<body><br><br><b><hr><br>", "", -1)
	quote = strings.Replace(quote, "<br><br><hr><br>", "", -1)

	return quote
}
Example #12
0
File: wtf.go Project: J5lx/luminous
func tf(c *http.Conn, b string, e string, n int, y int, s string) int {

	var qs string
	var ntf int = 0

	if len(s) < 1 {
		return ntf
	}

	if s[0] == '#' && len(s) > 1 {
		qs = s
	} else {
		qs = "from:" + s
	}
	r, _, err := http.Get(fmt.Sprintf(queryURI,
		qformat, http.URLEscape(qs), b, e, n))
	if err == nil {
		if r.StatusCode == http.StatusOK {
			ntf = readjson(c, r.Body, b, e, y)
		} else {
			fmt.Printf("Twitter is unable to search for %s (%s)\n", s, r.Status)
		}
		r.Body.Close()
	} else {
		fmt.Printf("%v\n", err)
	}
	return ntf
}
Example #13
0
func Crawl(url string) (result string, finalUrl string, err os.Error) {
	resp, finalUrl, err := http.Get(url)
	if err == nil {
		fmt.Println("Status: " + resp.Status)
		fmt.Printf("Close: %b\n", resp.Close)
		body := resp.Body
		defer body.Close()

		buf := make([]byte, BufSize)
		var n int
		for {
			n, err = body.Read(buf)
			if n != 0 {
				result += string(buf[0:n])
			}
			if err != nil {
				break
			}
		}
	}

	if err == os.EOF {
		err = nil
	}
	return result, finalUrl, err
}
Example #14
0
func parseChat(msg string, m *ircbot.Message) (reply string) {
	if config.Ignores[m.GetSender()] {
		return ""
	}

	if msg == "!help acm-bot" {
		return help(nil)
	}

	if matches := urlRegex.FindAllStringSubmatch(msg, -1); matches != nil {
		for _, m := range matches {

			response, finalURL, err := http.Get(m[0])

			if err != nil {
				errors.Printf("%s - Fetch failed: %s\n", m[0], err.String())
			} else if finalURL != m[0] || config.TitleWhitelist[m[1]] {
				if t := getTitle(response.Body); t != "" {
					info.Println("Fetched: " + m[0])
					reply += fmt.Sprintf("Title:%s\n", t)
				}
			}
		}
	}

	recordSighting(m)

	return
}
Example #15
0
func testBadges() {
	r, _, err := http.Get("http://api.stackoverflow.com/1.0/badges?key=change_me")
	if err != nil {
		fmt.Println(err.String())
		return
	}

	d, err := gzip.NewReader(r.Body)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	var br gostack.BadgesResult
	b, _ := ioutil.ReadAll(d)

	err = json.Unmarshal(b, &br)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	fmt.Println(br)

}
Example #16
0
func GetChunk(chunkX, chunkY int) (*ChunkInfo, os.Error) {
	address := fmt.Sprintf("http://%s:%s/a/r?cy=%d&cx=%d", *ip, *port, chunkY, chunkX)
	fmt.Println("Making request " + address)
	r, _, err := http.Get(address)
	if err != nil {
		log.Panic(err)
	}
	defer r.Body.Close()
	content, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Panic(err)
	}
	info := &ChunkInfo{chunkX, chunkY, &ChunkData{}}
	err = json.Unmarshal(content, &info.Data)
	if err != nil {
		fmt.Println(err.String())
	}
	for y := 0; y < 16; y++ {
		for x := 0; x < 16; x++ {
			if len(info.Data[y][x]) == 0 {
				info.Data[y][x] = append(info.Data[y][x], "0")
			} else if info.Data[y][x][0] == "" {
				info.Data[y][x][0] = "0"
			}
		}
	}
	fmt.Println(info.Data)
	return info, err
}
Example #17
0
func (site *Site) GetRooms() {
	url := site.CampfireUrl("/rooms.xml")

	parsed_url := ParseURL(url)

	//fmt.Printf("Going to request URL: %s\n", parsedUrl.String())
	response, err := http.Get(parsed_url.String())
	if err != nil {
		log.Fatal(err)
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		log.Printf("Status: %s\n", response.Status)
		log.Fatal("Could not list rooms")
	}

	parser := xml.NewParser(response.Body)
	rooms := Rooms{Room: nil}
	err = parser.Unmarshal(&rooms, nil)
	if err != nil {
		log.Fatal("Error unmarshalling xml:", err)
	}
	site.Rooms = rooms.Room
	fmt.Println("Rooms", rooms)
}
Example #18
0
// Adds common parameters to the "params" map, signs the request,
// adds the signature to the "params" map and sends the request
// to the server.  It then unmarshals the response in to the "resp"
// parameter using xml.Unmarshal()
func (mt *MTurk) query(params map[string]string, operation string, resp interface{}) os.Error {
	service := MTURK_SERVICE
	timestamp := time.UTC().Format(TIMESTAMP_FORMAT)

	params["AWSAccessKeyId"] = mt.Auth.AccessKey
	params["Service"] = service
	params["Timestamp"] = timestamp
	params["Operation"] = operation

	// make a copy
	url := *mt.URL

	sign(mt.Auth, service, operation, timestamp, params)
	url.RawQuery = multimap(params).Encode()
	r, err := http.Get(url.String())
	if err != nil {
		return err
	}
	dump, _ := http.DumpResponse(r, true)
	println("DUMP:\n", string(dump))
	if r.StatusCode != 200 {
		return os.NewError(fmt.Sprintf("%d: unexpected status code", r.StatusCode))
	}
	err = xml.Unmarshal(r.Body, resp)
	r.Body.Close()
	return err
}
Example #19
0
File: dcb.go Project: gaxxx/funx
func getDcbValue(period int) (string, os.Error) {
	res, _, err := http.Get(fmt.Sprintf("http://kaijiang.zhcw.com/zhcw/html/ssq/detail_%d.html", period))
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	js, _ := ioutil.ReadAll(res.Body)
	s := string(js)
	i := strings.Index(s, "中奖号码")
	if i != -1 {
		s := string(s[i:])
		b := strings.Index(s, `<li`)
		e := strings.Index(s, `</ul>`)
		if b > e {
			return "", os.NewError("parse err\n")
		}
		xmlStr := string(s[b : e-1])
		var l Xml
		xmlStr = `<?xml version="1.0" encoding="UTF-8"?><a xmlns="http://domain">` + xmlStr + "</a>"
		fmt.Printf("%s\n", xmlStr)
		if err := xml.Unmarshal(strings.NewReader(xmlStr), &l); err != nil {
			return "", err
		} else {
			var str string
			for _, v := range l.Li {
				str += fmt.Sprintf("%2d|", v)
			}
			return str, nil
		}
	} else {
		return "", os.NewError("404\n")
	}
	return "", nil
}
Example #20
0
func runTest(test testRecord, j int, webaddr string, t *testing.T) {
	var response *http.Response
	var err os.Error
	defer func() { done <- j }()
	url := "http://" + webaddr + test.URL
	if response, _, err = http.Get(url); err != nil {
		t.Error(err)
	}
	if response.StatusCode != test.StatusCode {
		t.Error(j, webaddr, test.URL, "Response had wrong status code:", response.StatusCode)
	}
	if len(test.BodyPrefix) > 0 {
		prefix := make([]byte, len(test.BodyPrefix))
		if n, err := response.Body.Read(prefix); err == nil {
			p := string(prefix[0:n])
			if p != test.BodyPrefix {
				t.Error(j, webaddr, test.URL, "Bad body, expected prefix:", test.BodyPrefix, "got:", p)
			}
		} else {
			t.Error(j, webaddr, test.URL, "Error reading response.Body:", err)
		}
	}
	if test.Headers != nil {
		for _, hdr := range test.Headers {
			if v := response.GetHeader(hdr.Key); v != hdr.Val {
				t.Error(j, webaddr, test.URL, "Header value in response:", strconv.Quote(v), "did not match", strconv.Quote(hdr.Val))
			}
		}
	}
}
Example #21
0
func (c *Client) RetweetedOfMe(sinceId uint64, maxId uint64, count uint, page uint) (t []Tweet) {
	var params string
	var tweets []Tweet

	if sinceId != 0 {
		params = addParam(params, "since_id", fmt.Sprintf("%d", sinceId))
	}
	if maxId != 0 {
		params = addParam(params, "max_id", fmt.Sprintf("%d", maxId))
	}
	if count != 0 {
		params = addParam(params, "count", fmt.Sprintf("%d", count))
	}
	if page != 0 {
		params = addParam(params, "page", fmt.Sprintf("%d", page))
	}

	url := c.makeAuthURL(retweetedOfMe, params)
	res, _, err := http.Get(url)
	if err != nil {
		return nil
	}

	if res.Status != "200 OK" {
		return nil
	}

	reader := bufio.NewReader(res.Body)
	line, _ := reader.ReadString(0)

	json.Unmarshal(line, &tweets)

	return tweets
}
Example #22
0
func BenchmarkStaticFileOverHTTPWithMultiplex(b *testing.B) {
	b.StopTimer()
	var C = 50 // number of simultaneous clients
	http.Handle("/static/", http.FileServer("_test/", "/static"))
	if err := createStaticTestFile(); err != nil {
		log.Print("Failed to create test file:", err)
		return
	}
	weblisten, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Print("net.Listen error:", err)
		return
	}
	url := "http://" + weblisten.Addr().String() + "/static/fcgi_test.html"
	go http.Serve(weblisten, nil)

	// allow this many simultaneous connections to the webserver
	start := make(chan bool, C)
	for i := 0; i < C; i++ {
		start <- true
	}
	done := make(chan bool, b.N) // for syncing all the multiplex goroutines
	b.StartTimer()
	log.Print("Loop starting...", b.N)
	for i := 0; i < b.N; i++ {
		go func(index int) {
			<-start
			defer func() {
				done <- true
				start <- true
			}()
			response, _, err := http.Get(url)
			if err != nil {
				log.Print("http.Get error:", err)
			}
			if response == nil {
				log.Print("Nil response.")
				return
			}
			if response.StatusCode != 200 {
				log.Print("Bad response status:", response.StatusCode)
				return
			}
			if response != nil {
				body, err := ioutil.ReadAll(response.Body)
				if err != nil {
					log.Print("ioutil.ReadAll error:", err)
					return
				}
				b.SetBytes(int64(len(body)))
				response.Body.Close()
			}
		}(i)
	}
	for i := 0; i < b.N; i++ {
		<-done
	}
	weblisten.Close()
	removeStaticTestFile()
}
Example #23
0
func (c *Client) StatusesFollowers(userId uint64, screenName string, cursor int) (u []User) {
	var params string
	var users []User

	if userId != 0 {
		params = addParam(params, "user_id", fmt.Sprintf("%d", userId))
	}
	if screenName != "" {
		params = addParam(params, "screen_name", screenName)
	}
	if cursor != 0 {
		params = addParam(params, "cursor", fmt.Sprintf("%d", cursor))
	}

	url := c.makeAuthURL(statusesFollowers, params)
	res, _, err := http.Get(url)
	if err != nil {
		return nil
	}

	if res.Status != "200 OK" {
		return nil
	}

	reader := bufio.NewReader(res.Body)
	line, _ := reader.ReadString(0)

	json.Unmarshal(line, &users)

	return users

}
Example #24
0
// home handles requests to the home page.
func home(req *web.Request) {
	token, err := credentials(req, "tok")
	if err != nil {
		homeLoggedOut(req)
		return
	}
	param := make(web.Values)
	url := "http://api.twitter.com/1/statuses/home_timeline.json"
	oauthClient.SignParam(token, "GET", url, param)
	url = url + "?" + param.FormEncodedString()
	resp, err := http.Get(url)
	if err != nil {
		req.Error(web.StatusInternalServerError, err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		req.Error(web.StatusInternalServerError, os.NewError(fmt.Sprint("Status ", resp.StatusCode)))
		return
	}
	var d interface{}
	err = json.NewDecoder(resp.Body).Decode(&d)
	if err != nil {
		req.Error(web.StatusInternalServerError, err)
		return
	}
	homeTempl.Execute(req.Respond(web.StatusOK, web.HeaderContentType, web.ContentTypeHTML), d)
}
Example #25
0
func DiscoverHtml(id string) (*string, os.Error) {
	resp, _, err := http.Get(id)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	tokenizer := html.NewTokenizer(resp.Body)
	for {
		tt := tokenizer.Next()
		switch tt {
		case html.ErrorToken:
			log.Println("Error: ", tokenizer.Error())
			return nil, tokenizer.Error()
		case html.StartTagToken, html.EndTagToken:
			tk := tokenizer.Token()
			if tk.Data == "link" {
				ok := false
				for _, attr := range tk.Attr {
					if attr.Key == "rel" && attr.Val == "openid2.provider" {
						log.Println(tk.String())
						ok = true
					} else if attr.Key == "href" && ok {
						return &attr.Val, nil
					}
				}
			}
		}
	}
	return nil, &DiscoveryError{str: "provider not found"}
}
Example #26
0
//Retrieve the gist of the error logs pointed to by the URL
func getBuildLog(url string) string {
	response, err := http.Get(url)
	if url == "" {
		return ""
	}

	defer response.Body.Close()

	b, err := ioutil.ReadAll(response.Body)
	check(err)

	//only the last part of the log is relevant
	e := len(b) - 1
	s := 0
	if e > lastBytes {
		s = e - lastBytes
	}

	//drop anything after the error message
	res := strings.SplitN(string(b[s:e]), endOfLog, 2)

	//keep only last 200 lines
	split := strings.Split(res[0], "\n")

	e = len(split) - 1
	s = 0
	if e > 200 {
		s = e - 200
	}

	return strings.Join(split[s:e], "\n")
}
Example #27
0
func shorten(long string) (short string) {
	key := "R_e659dbb5514e34edc3540a7c95b0041b"
	login := "******"

	long = url.QueryEscape(long)

	url_ := fmt.Sprintf("http://api.bit.ly/v3/shorten?login=%s&apiKey=%s&longUrl=%s&format=json", login, key, long)
	r, err := http.Get(url_)
	defer r.Body.Close()

	if err != nil {
		return "Error connecting to bit.ly"
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "Error reading bit.ly response"
	}

	var j map[string]interface{}

	err = json.Unmarshal(b, &j)
	if err != nil {
		return "Unable to shorten URL."
	}

	var data map[string]interface{} = j["data"].(map[string]interface{})

	return data["url"].(string)
}
Example #28
0
func ExpandURL(shortUrl string) (expandedUrl string, err os.Error) {
	param := http.EncodeQuery(map[string][]string{"shortUrl": {shortUrl}})
	res, _, err := http.Get("https://www.googleapis.com/urlshortener/v1/url?" + param)
	if err != nil {
		return
	}
	if res.StatusCode != 200 {
		err = os.NewError("failed to post")
		return
	}
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	var decbuf bytes.Buffer
	decbuf.Write(b)
	dec := json.NewDecoder(&decbuf)
	var out map[string]interface{}
	err = dec.Decode(&out)
	if err != nil {
		return
	}
	expandedUrl = out["longUrl"].(string)
	return
}
Example #29
0
func remoteSearch(query string) (res *http.Response, err os.Error) {
	search := "/search?f=text&q=" + url.QueryEscape(query)

	// list of addresses to try
	var addrs []string
	if *serverAddr != "" {
		// explicit server address - only try this one
		addrs = []string{*serverAddr}
	} else {
		addrs = []string{
			defaultAddr,
			"golang.org",
		}
	}

	// remote search
	for _, addr := range addrs {
		url := "http://" + addr + search
		res, err = http.Get(url)
		if err == nil && res.StatusCode == http.StatusOK {
			break
		}
	}

	if err == nil && res.StatusCode != http.StatusOK {
		err = os.NewError(res.Status)
	}

	return
}
Example #30
0
func Fetcher(queue *list.List, limit int) chan *WikiArticle {
	articles := make(chan *WikiArticle)
	go func() {
		for i := 0; i < limit; i++ {
			for queue.Front() == nil {
				runtime.Gosched()
			}
			// url := q.PopFront()
			article := queue.Front().Value.(*WikiArticle)
			queue.Remove(queue.Front())

			r, url, e := http.Get(article.url)
			if e != nil {
				println("Failed:", article.url)
				continue
			}
			println("Fetched:", article.depth, url)

			buf := bytes.NewBufferString("")
			io.Copy(buf, r.Body)
			article.content = buf.String()
			r.Body.Close()
			articles <- article
		}
		close(articles)
	}()
	return articles
}