func main() {
flag.Parse()
httpstream.SetLogger(log.New(os.Stdout, "", log.Ltime|log.Lshortfile), *logLevel)
stream := make(chan []byte, 1000)
done := make(chan bool)
client := httpstream.NewBasicAuthClient(*user, *pwd, httpstream.OnlyTweetsFilter(func(line []byte) {
stream <- line
}))
keywords := strings.Split("android,golang,zeromq,javascript", ",")
err := client.Filter([]int64{14230524, 783214}, keywords, []string{"en"}, false, done)
if err != nil {
httpstream.Log(httpstream.ERROR, err.Error())
} else {
go func() {
ct := 0
for tw := range stream {
println(string(tw))
ct++
if ct > 10 {
done <- true
}
}
}()
_ = <-done
}
}
func main() {
flag.Parse()
// make a go channel for
stream := make(chan []byte, 200)
done := make(chan bool)
// set the logger and log level
httpstream.SetLogger(log.New(os.Stdout, "", log.Ltime|log.Lshortfile), *logLevel)
// the stream listener effectively operates in one "thread"
client := httpstream.NewBasicAuthClient("", "", func(line []byte) {
//println(string(line))
stream <- line
})
client.MaxWait = 20
err := client.Connect(customUrl, nil, done)
if err != nil {
println(err.Error())
} else {
go func() {
for line := range stream {
println(string(line))
}
}()
_ = <-done
}
}
func main() {
flag.Parse()
// set the logger and log level
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
stream := make(chan []byte)
done := make(chan bool)
client := httpstream.NewBasicAuthClient(*user, *pwd, func(line []byte) {
stream <- line
})
//err := client.Track([]string{"bieber,iphone,mac,android,ios,lady gaga,dancing,sick,game,when,why,where,how,who"}, stream)
// this opens a go routine that is effectively thread 1
err := client.Sample(done)
if err != nil {
println(err.Error())
}
// 2nd thread
go func() {
for {
line := <-stream
println()
HandleLine(1, line)
}
}()
// 3rd thread
for {
line := <-stream
HandleLine(2, line)
}
}
// Connect to twitter streaming api and send lines to be written.
func writeTweetStream(w io.Writer) {
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), "info")
stream := make(chan []byte, 1000)
done := make(chan bool)
client := httpstream.NewBasicAuthClient(vafanConf.twitter.user, vafanConf.twitter.password, func(line []byte) {
stream <- line
})
err := client.Filter([]int64{twSaulHowardID, twConvictFilmsID}, []string{"brightonwok", "brighton wok", "convictfilms", "convict films"}, false, done)
// this opens a go routine that is effectively thread 1
// err := client.Sample(done)
if err != nil {
println(err.Error())
}
// 2nd thread
go func() {
for {
line := <-stream
writeTweet(w, line)
}
}()
// 3rd thread
for {
line := <-stream
writeTweet(w, line)
}
}
func fetchTwitter() {
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
stream := make(chan []byte, 1000)
done := make(chan bool)
client := httpstream.NewBasicAuthClient(*user, *pwd, httpstream.OnlyTweetsFilter(func(line []byte) {
stream <- line
}))
err := client.Sample(done)
if err != nil {
httpstream.Log(httpstream.ERROR, err.Error())
} else {
go func() {
ct := 0
for tw := range stream {
println(string(tw))
// heavy lifting
ct++
if ct > 1 {
done <- true
}
}
}()
_ = <-done
}
}
func main() {
flag.Parse()
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
ct := 0
// make a go channel for sending from listener to processor
// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
stream := make(chan []byte,1000)
done := make(chan bool)
// the stream listener effectively operates in one "thread"/goroutine
// as the httpstream Client processes inside a go routine it opens
// That includes the handler func we pass in here
client := httpstream.NewBasicAuthClient(*user, *pwd, func(line []byte) {
stream <- line
// although you can do heavy lifting here, it means you are doing all
// your work in the same thread as the http streaming/listener
// by using a go channel, you can send the work to a
// different thread/goroutine
})
//err := client.Track([]string{"eat,iphone,mac,android,ios,burger"}, stream)
err := client.Sample(done)
if err != nil {
httpstream.Log(ERROR, (err.Error())
} else {
go func() {
// while this could be in a different "thread(s)"
for tw := range stream {
println(string(tw))
// heavy lifting
ct ++
if ct > 10 {
os.Exit(0)
}
}
}()
_ = <- done
}
}
func main() {
flag.Parse()
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
// Connectin to the local mongo database
session, err := mgo.Dial("localhost")
if err != nil {
panic(err)
}
// We defer the session closing so we don't forget later
defer session.Close()
t := time.Now()
collectionName := fmt.Sprintf("%d-%d-%d_%d:%d_%s", t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute(), *collection)
fmt.Println("Collection:", collectionName)
// Creating DB "twitter" and Collection "tweets"
c := session.DB("mundial").C(collectionName)
// Optional. Switch the session to a monotonic behavior. session.SetMode(mgo.Monotonic, true)
// make a go channel for sending from listener to processor
// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
stream := make(chan []byte, 1000)
done := make(chan bool)
httpstream.OauthCon = oauth.NewConsumer(
*consumerKey,
*consumerSecret,
oauth.ServiceProvider{
RequestTokenUrl: "http://api.twitter.com/oauth/request_token",
AuthorizeTokenUrl: "https://api.twitter.com/oauth/authorize",
AccessTokenUrl: "https://api.twitter.com/oauth/access_token",
})
at := oauth.AccessToken{
Token: *ot,
Secret: *osec,
}
// Creamos el cliente OAuth
client := httpstream.NewOAuthClient(&at, httpstream.OnlyTweetsFilter(func(line []byte) { stream <- line }))
fmt.Println("Cliente OAuth creado")
// find list of userids we are going to search for
userIds := make([]int64, 0)
for _, userId := range strings.Split(*users, ",") {
if id, err := strconv.ParseInt(userId, 10, 64); err == nil {
userIds = append(userIds, id)
}
}
var keywords []string
if search != nil && len(*search) > 0 {
keywords = strings.Split(*search, ",")
}
//err := client.Filter(userIds, keywords, []string{"es"}, nil, false, done)
//var locats []string
// locats := make([]string, 4)
// locats[0] = "-180"
// locats[1] = "-90"
// locats[2] = "180"
// locats[3] = "90"
//err = client.Filter(userIds, keywords, nil, locats, false, done)
err = client.Filter(userIds, keywords, nil, nil, false, done)
//err := client.Filter(nil, keywords, nil, nil, false, done)
if err != nil {
httpstream.Log(httpstream.ERROR, err.Error())
} else {
fmt.Println("Filter realizado, esperando datos")
go func() {
// while this could be in a different "thread(s)"
ct := 0
//var tweet twittertypes.Tweet
for tw := range stream {
//println("Tweet", ct, "capturado")
tweet := new(twittertypes.Tweet)
json.Unmarshal(tw, &tweet)
tweetstamp, err := time.Parse(time.RubyDate, tweet.Created_at)
tweet.Epoch = tweetstamp.Unix()
err = c.Insert(tweet)
if err != nil {
panic(err)
}
// heavy lifting
ct++
if ct >= *maxCt {
done <- true
}
}
}()
_ = <-done
}
}
func main() {
flag.Parse()
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
// make a go channel for sending from listener to processor
// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
stream := make(chan []byte, 1000)
done := make(chan bool)
httpstream.OauthCon = &oauth.OAuthConsumer{
Service: "twitter",
RequestTokenURL: "http://twitter.com/oauth/request_token",
AccessTokenURL: "http://twitter.com/oauth/access_token",
AuthorizationURL: "http://twitter.com/oauth/authorize",
ConsumerKey: *ck,
ConsumerSecret: *cs,
CallBackURL: "oob",
UserAgent: "go/httpstream",
}
//at := goauthcon.GetAccessToken(rt.Token, pin)
at := oauth.AccessToken{Id: "",
Token: *ot,
Secret: *osec,
UserRef: *user,
Verifier: "",
Service: "twitter",
}
// the stream listener effectively operates in one "thread"/goroutine
// as the httpstream Client processes inside a go routine it opens
// That includes the handler func we pass in here
client := httpstream.NewOAuthClient(&at, httpstream.OnlyTweetsFilter(func(line []byte) {
stream <- line
// although you can do heavy lifting here, it means you are doing all
// your work in the same thread as the http streaming/listener
// by using a go channel, you can send the work to a
// different thread/goroutine
}))
keywords := strings.Split("android,golang,zeromq,javascript", ",")
err := client.Filter([]int64{14230524, 783214}, keywords, false, done)
if err != nil {
httpstream.Log(httpstream.ERROR, err.Error())
} else {
go func() {
// while this could be in a different "thread(s)"
ct := 0
for tw := range stream {
println(string(tw))
// heavy lifting
ct++
if ct > 10 {
done <- true
}
}
}()
_ = <-done
}
}
func main() {
flag.Parse()
httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
// make a go channel for sending from listener to processor
// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
stream := make(chan []byte, 1000)
done := make(chan bool)
httpstream.OauthCon = oauth.NewConsumer(
*consumerKey,
*consumerSecret,
oauth.ServiceProvider{
RequestTokenUrl: "http://api.twitter.com/oauth/request_token",
AuthorizeTokenUrl: "https://api.twitter.com/oauth/authorize",
AccessTokenUrl: "https://api.twitter.com/oauth/access_token",
})
at := oauth.AccessToken{
Token: *ot,
Secret: *osec,
}
// the stream listener effectively operates in one "thread"/goroutine
// as the httpstream Client processes inside a go routine it opens
// That includes the handler func we pass in here
client := httpstream.NewOAuthClient(&at, httpstream.OnlyTweetsFilter(func(line []byte) {
stream <- line
// although you can do heavy lifting here, it means you are doing all
// your work in the same thread as the http streaming/listener
// by using a go channel, you can send the work to a
// different thread/goroutine
}))
// find list of userids we are going to search for
userIds := make([]int64, 0)
for _, userId := range strings.Split(*users, ",") {
if id, err := strconv.ParseInt(userId, 10, 64); err == nil {
userIds = append(userIds, id)
}
}
var keywords []string
if search != nil && len(*search) > 0 {
keywords = strings.Split(*search, ",")
}
err := client.Filter(userIds, keywords, []string{"en"}, nil, false, done)
if err != nil {
httpstream.Log(httpstream.ERROR, err.Error())
} else {
go func() {
// while this could be in a different "thread(s)"
ct := 0
for tw := range stream {
println(string(tw))
// heavy lifting
ct++
if ct > *maxCt {
done <- true
}
}
}()
_ = <-done
}
}