Example #1
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ltime|log.Lshortfile), *logLevel)

	stream := make(chan []byte, 1000)
	done := make(chan bool)

	client := httpstream.NewBasicAuthClient(*user, *pwd, httpstream.OnlyTweetsFilter(func(line []byte) {
		stream <- line
	}))

	keywords := strings.Split("android,golang,zeromq,javascript", ",")
	err := client.Filter([]int64{14230524, 783214}, keywords, []string{"en"}, false, done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {

		go func() {
			ct := 0
			for tw := range stream {
				println(string(tw))
				ct++
				if ct > 10 {
					done <- true
				}
			}
		}()
		_ = <-done
	}

}
Example #2
0
func main() {

	flag.Parse()

	// make a go channel for
	stream := make(chan []byte, 200)
	done := make(chan bool)

	// set the logger and log level
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ltime|log.Lshortfile), *logLevel)

	// the stream listener effectively operates in one "thread"
	client := httpstream.NewBasicAuthClient("", "", func(line []byte) {
		//println(string(line))
		stream <- line
	})
	client.MaxWait = 20

	err := client.Connect(customUrl, nil, done)
	if err != nil {
		println(err.Error())
	} else {
		go func() {
			for line := range stream {

				println(string(line))
			}
		}()

		_ = <-done
	}

}
Example #3
0
func main() {

	flag.Parse()
	// set the logger and log level
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)

	stream := make(chan []byte)
	done := make(chan bool)

	client := httpstream.NewBasicAuthClient(*user, *pwd, func(line []byte) {
		stream <- line
	})
	//err := client.Track([]string{"bieber,iphone,mac,android,ios,lady gaga,dancing,sick,game,when,why,where,how,who"}, stream)
	// this opens a go routine that is effectively thread 1
	err := client.Sample(done)
	if err != nil {
		println(err.Error())
	}
	// 2nd thread
	go func() {
		for {
			line := <-stream
			println()
			HandleLine(1, line)
		}
	}()
	// 3rd thread
	for {
		line := <-stream
		HandleLine(2, line)
	}
}
Example #4
0
// Connect to twitter streaming api and send lines to be written.
func writeTweetStream(w io.Writer) {
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), "info")
	stream := make(chan []byte, 1000)
	done := make(chan bool)
	client := httpstream.NewBasicAuthClient(vafanConf.twitter.user, vafanConf.twitter.password, func(line []byte) {
		stream <- line
	})
	err := client.Filter([]int64{twSaulHowardID, twConvictFilmsID}, []string{"brightonwok", "brighton wok", "convictfilms", "convict films"}, false, done)
	// this opens a go routine that is effectively thread 1
	// err := client.Sample(done)
	if err != nil {
		println(err.Error())
	}
	// 2nd thread
	go func() {
		for {
			line := <-stream
			writeTweet(w, line)
		}
	}()
	// 3rd thread
	for {
		line := <-stream
		writeTweet(w, line)
	}
}
Example #5
0
func fetchTwitter() {
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
	stream := make(chan []byte, 1000)
	done := make(chan bool)

	client := httpstream.NewBasicAuthClient(*user, *pwd, httpstream.OnlyTweetsFilter(func(line []byte) {
		stream <- line
	}))

	err := client.Sample(done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {

		go func() {
			ct := 0
			for tw := range stream {
				println(string(tw))
				// heavy lifting
				ct++
				if ct > 1 {
					done <- true
				}
			}
		}()
		_ = <-done
	}
}
Example #6
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
	ct := 0

	// make a go channel for sending from listener to processor
	// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
	stream := make(chan []byte,1000) 
	done := make(chan bool)

	// the stream listener effectively operates in one "thread"/goroutine
	// as the httpstream Client processes inside a go routine it opens
	// That includes the handler func we pass in here
	client := httpstream.NewBasicAuthClient(*user, *pwd, func(line []byte) {
		stream <- line
		// although you can do heavy lifting here, it means you are doing all
		// your work in the same thread as the http streaming/listener
		// by using a go channel, you can send the work to a 
		// different thread/goroutine
	})

	//err := client.Track([]string{"eat,iphone,mac,android,ios,burger"}, stream)
	err := client.Sample(done)
	if err != nil {
		httpstream.Log(ERROR, (err.Error())
	} else {

		go func() {
			// while this could be in a different "thread(s)"
			for tw := range stream {
				
				println(string(tw))
				// heavy lifting
				ct ++
				if ct > 10 {
					os.Exit(0)
				}
			}
		}()
		_ = <- done
	}

}
Example #7
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)

	// Connectin to the local mongo database
	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}
	// We defer the session closing so we don't forget later
	defer session.Close()
	t := time.Now()
	collectionName := fmt.Sprintf("%d-%d-%d_%d:%d_%s", t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute(), *collection)
	fmt.Println("Collection:", collectionName)
	// Creating DB "twitter" and Collection "tweets"
	c := session.DB("mundial").C(collectionName)
	// Optional. Switch the session to a monotonic behavior. session.SetMode(mgo.Monotonic, true)

	// make a go channel for sending from listener to processor
	// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
	stream := make(chan []byte, 1000)
	done := make(chan bool)

	httpstream.OauthCon = oauth.NewConsumer(
		*consumerKey,
		*consumerSecret,
		oauth.ServiceProvider{
			RequestTokenUrl:   "http://api.twitter.com/oauth/request_token",
			AuthorizeTokenUrl: "https://api.twitter.com/oauth/authorize",
			AccessTokenUrl:    "https://api.twitter.com/oauth/access_token",
		})

	at := oauth.AccessToken{
		Token:  *ot,
		Secret: *osec,
	}

	// Creamos el cliente OAuth
	client := httpstream.NewOAuthClient(&at, httpstream.OnlyTweetsFilter(func(line []byte) { stream <- line }))
	fmt.Println("Cliente OAuth creado")
	// find list of userids we are going to search for
	userIds := make([]int64, 0)
	for _, userId := range strings.Split(*users, ",") {
		if id, err := strconv.ParseInt(userId, 10, 64); err == nil {
			userIds = append(userIds, id)
		}
	}
	var keywords []string
	if search != nil && len(*search) > 0 {
		keywords = strings.Split(*search, ",")
	}
	//err := client.Filter(userIds, keywords, []string{"es"}, nil, false, done)
	//var locats []string
	// locats := make([]string, 4)
	// locats[0] = "-180"
	// locats[1] = "-90"
	// locats[2] = "180"
	// locats[3] = "90"
	//err = client.Filter(userIds, keywords, nil, locats, false, done)
	err = client.Filter(userIds, keywords, nil, nil, false, done)
	//err := client.Filter(nil, keywords, nil, nil, false, done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {
		fmt.Println("Filter realizado, esperando datos")
		go func() {
			// while this could be in a different "thread(s)"
			ct := 0
			//var tweet twittertypes.Tweet
			for tw := range stream {
				//println("Tweet", ct, "capturado")
				tweet := new(twittertypes.Tweet)
				json.Unmarshal(tw, &tweet)
				tweetstamp, err := time.Parse(time.RubyDate, tweet.Created_at)
				tweet.Epoch = tweetstamp.Unix()
				err = c.Insert(tweet)
				if err != nil {
					panic(err)
				}
				// heavy lifting
				ct++
				if ct >= *maxCt {
					done <- true
				}
			}
		}()
		_ = <-done
	}
}
Example #8
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)

	// make a go channel for sending from listener to processor
	// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
	stream := make(chan []byte, 1000)
	done := make(chan bool)

	httpstream.OauthCon = &oauth.OAuthConsumer{
		Service:          "twitter",
		RequestTokenURL:  "http://twitter.com/oauth/request_token",
		AccessTokenURL:   "http://twitter.com/oauth/access_token",
		AuthorizationURL: "http://twitter.com/oauth/authorize",
		ConsumerKey:      *ck,
		ConsumerSecret:   *cs,
		CallBackURL:      "oob",
		UserAgent:        "go/httpstream",
	}

	//at := goauthcon.GetAccessToken(rt.Token, pin)
	at := oauth.AccessToken{Id: "",
		Token:    *ot,
		Secret:   *osec,
		UserRef:  *user,
		Verifier: "",
		Service:  "twitter",
	}
	// the stream listener effectively operates in one "thread"/goroutine
	// as the httpstream Client processes inside a go routine it opens
	// That includes the handler func we pass in here
	client := httpstream.NewOAuthClient(&at, httpstream.OnlyTweetsFilter(func(line []byte) {
		stream <- line
		// although you can do heavy lifting here, it means you are doing all
		// your work in the same thread as the http streaming/listener
		// by using a go channel, you can send the work to a
		// different thread/goroutine
	}))

	keywords := strings.Split("android,golang,zeromq,javascript", ",")
	err := client.Filter([]int64{14230524, 783214}, keywords, false, done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {

		go func() {
			// while this could be in a different "thread(s)"
			ct := 0
			for tw := range stream {
				println(string(tw))
				// heavy lifting
				ct++
				if ct > 10 {
					done <- true
				}
			}
		}()
		_ = <-done
	}

}
Example #9
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)

	// make a go channel for sending from listener to processor
	// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
	stream := make(chan []byte, 1000)
	done := make(chan bool)

	httpstream.OauthCon = oauth.NewConsumer(
		*consumerKey,
		*consumerSecret,
		oauth.ServiceProvider{
			RequestTokenUrl:   "http://api.twitter.com/oauth/request_token",
			AuthorizeTokenUrl: "https://api.twitter.com/oauth/authorize",
			AccessTokenUrl:    "https://api.twitter.com/oauth/access_token",
		})

	at := oauth.AccessToken{
		Token:  *ot,
		Secret: *osec,
	}
	// the stream listener effectively operates in one "thread"/goroutine
	// as the httpstream Client processes inside a go routine it opens
	// That includes the handler func we pass in here
	client := httpstream.NewOAuthClient(&at, httpstream.OnlyTweetsFilter(func(line []byte) {
		stream <- line
		// although you can do heavy lifting here, it means you are doing all
		// your work in the same thread as the http streaming/listener
		// by using a go channel, you can send the work to a
		// different thread/goroutine
	}))

	// find list of userids we are going to search for
	userIds := make([]int64, 0)
	for _, userId := range strings.Split(*users, ",") {
		if id, err := strconv.ParseInt(userId, 10, 64); err == nil {
			userIds = append(userIds, id)
		}
	}
	var keywords []string
	if search != nil && len(*search) > 0 {
		keywords = strings.Split(*search, ",")
	}
	err := client.Filter(userIds, keywords, []string{"en"}, nil, false, done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {

		go func() {
			// while this could be in a different "thread(s)"
			ct := 0
			for tw := range stream {
				println(string(tw))
				// heavy lifting
				ct++
				if ct > *maxCt {
					done <- true
				}
			}
		}()
		_ = <-done
	}

}