Example #1
0
func main() {

	flag.Parse()

	// make a go channel for
	stream := make(chan []byte, 200)
	done := make(chan bool)

	// set the logger and log level
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ltime|log.Lshortfile), *logLevel)

	// the stream listener effectively operates in one "thread"
	client := httpstream.NewBasicAuthClient("", "", func(line []byte) {
		//println(string(line))
		stream <- line
	})
	client.MaxWait = 20

	err := client.Connect(customUrl, nil, done)
	if err != nil {
		println(err.Error())
	} else {
		go func() {
			for line := range stream {

				println(string(line))
			}
		}()

		_ = <-done
	}

}
Example #2
0
func main() {

	flag.Parse()
	// set the logger and log level
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)

	stream := make(chan []byte)
	done := make(chan bool)

	client := httpstream.NewBasicAuthClient(*user, *pwd, func(line []byte) {
		stream <- line
	})
	//err := client.Track([]string{"bieber,iphone,mac,android,ios,lady gaga,dancing,sick,game,when,why,where,how,who"}, stream)
	// this opens a go routine that is effectively thread 1
	err := client.Sample(done)
	if err != nil {
		println(err.Error())
	}
	// 2nd thread
	go func() {
		for {
			line := <-stream
			println()
			HandleLine(1, line)
		}
	}()
	// 3rd thread
	for {
		line := <-stream
		HandleLine(2, line)
	}
}
Example #3
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ltime|log.Lshortfile), *logLevel)

	stream := make(chan []byte, 1000)
	done := make(chan bool)

	client := httpstream.NewBasicAuthClient(*user, *pwd, httpstream.OnlyTweetsFilter(func(line []byte) {
		stream <- line
	}))

	keywords := strings.Split("android,golang,zeromq,javascript", ",")
	err := client.Filter([]int64{14230524, 783214}, keywords, []string{"en"}, false, done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {

		go func() {
			ct := 0
			for tw := range stream {
				println(string(tw))
				ct++
				if ct > 10 {
					done <- true
				}
			}
		}()
		_ = <-done
	}

}
Example #4
0
// Connect to twitter streaming api and send lines to be written.
func writeTweetStream(w io.Writer) {
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), "info")
	stream := make(chan []byte, 1000)
	done := make(chan bool)
	client := httpstream.NewBasicAuthClient(vafanConf.twitter.user, vafanConf.twitter.password, func(line []byte) {
		stream <- line
	})
	err := client.Filter([]int64{twSaulHowardID, twConvictFilmsID}, []string{"brightonwok", "brighton wok", "convictfilms", "convict films"}, false, done)
	// this opens a go routine that is effectively thread 1
	// err := client.Sample(done)
	if err != nil {
		println(err.Error())
	}
	// 2nd thread
	go func() {
		for {
			line := <-stream
			writeTweet(w, line)
		}
	}()
	// 3rd thread
	for {
		line := <-stream
		writeTweet(w, line)
	}
}
Example #5
0
func fetchTwitter() {
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
	stream := make(chan []byte, 1000)
	done := make(chan bool)

	client := httpstream.NewBasicAuthClient(*user, *pwd, httpstream.OnlyTweetsFilter(func(line []byte) {
		stream <- line
	}))

	err := client.Sample(done)
	if err != nil {
		httpstream.Log(httpstream.ERROR, err.Error())
	} else {

		go func() {
			ct := 0
			for tw := range stream {
				println(string(tw))
				// heavy lifting
				ct++
				if ct > 1 {
					done <- true
				}
			}
		}()
		_ = <-done
	}
}
Example #6
0
func main() {

	flag.Parse()
	httpstream.SetLogger(log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), *logLevel)
	ct := 0

	// make a go channel for sending from listener to processor
	// we buffer it, to help ensure we aren't backing up twitter or else they cut us off
	stream := make(chan []byte,1000) 
	done := make(chan bool)

	// the stream listener effectively operates in one "thread"/goroutine
	// as the httpstream Client processes inside a go routine it opens
	// That includes the handler func we pass in here
	client := httpstream.NewBasicAuthClient(*user, *pwd, func(line []byte) {
		stream <- line
		// although you can do heavy lifting here, it means you are doing all
		// your work in the same thread as the http streaming/listener
		// by using a go channel, you can send the work to a 
		// different thread/goroutine
	})

	//err := client.Track([]string{"eat,iphone,mac,android,ios,burger"}, stream)
	err := client.Sample(done)
	if err != nil {
		httpstream.Log(ERROR, (err.Error())
	} else {

		go func() {
			// while this could be in a different "thread(s)"
			for tw := range stream {
				
				println(string(tw))
				// heavy lifting
				ct ++
				if ct > 10 {
					os.Exit(0)
				}
			}
		}()
		_ = <- done
	}

}
Example #7
0
func runTwitterStream() {
	// stream Twitter
	stream := make(chan []byte)
	client := httpstream.NewBasicAuthClient(username, password, func(line []byte) {
		stream <- line
	})
	tracks := strings.Split(*track, ",")

	err := client.Track(tracks)
	if err != nil {
		println(err.Error())
	}
	fmt.Fprintf(os.Stderr, "track = %v\n", *track)

	// process the tweets
	for {
		tw := <-stream
		if !*printOnly {
			process(string(tw))
		} else {
			fmt.Println(string(tw))
		}
	}
}
Example #8
0
func main() {
	flag.Parse()

	if len(*twitterUsername) == 0 || len(*twitterPassword) == 0 || len(*streamSpigotHostname) == 0 || len(*streamSpigotSecret) == 0 {
		flag.Usage()
		return
	}

	baseUrl := fmt.Sprintf("http://%s/bird-feeder/pinger/", *streamSpigotHostname)
	followingUrl := fmt.Sprintf("%sfollowing?secret=%s", baseUrl, url.QueryEscape(*streamSpigotSecret))
	pingUrl := baseUrl + "ping"

	var followingUserIds []int64
	var followingUserIdMap map[int64]bool

	stream := make(chan []byte)
	done := make(chan bool)
	updateFollowingListTick := time.Tick(followingListUpdateIntervalNanosec)

	client := httpstream.NewBasicAuthClient(*twitterUsername, *twitterPassword, func(line []byte) {
		stream <- line
	})

	updateFollowingList := func() {
		followingUserIds, followingUserIdMap = getFollowingList(followingUrl)

		fmt.Printf("Tracking updates for %d users...\n", len(followingUserIds))

		client.Close()
		err := client.Filter(followingUserIds, nil, false, done)
		if err != nil {
			fmt.Println(err)
		}
	}

	updateFollowingList()

	for {
		select {
		case <-updateFollowingListTick:
			updateFollowingList()
		case <-done:
			fmt.Printf("Client says it's done")
		case line := <-stream:
			switch {
			case bytes.HasPrefix(line, []byte(`{"event":`)):
				var event httpstream.Event
				json.Unmarshal(line, &event)
			case bytes.HasPrefix(line, []byte(`{"friends":`)):
				var friends httpstream.FriendList
				json.Unmarshal(line, &friends)
			default:
				tweet := httpstream.Tweet{}
				json.Unmarshal(line, &tweet)
				if tweet.User != nil && tweet.User.Id != nil {
					fmt.Printf("%s: %s\n", tweet.User.ScreenName, tweet.Text)

					userId := int64(*tweet.User.Id)

					// We ignore tweets that come from users that we're not following (the
					// Streaming API will also notify when tweets of theirs are retweeted or
					// replied to).
					if _, inMap := followingUserIdMap[userId]; inMap {
						// Similarly, we ignore tweets that are in reply to users that aren't
						// being followed. This will have false negatives: if user A follows X
						// and user B follows X and Z, a reply by X to Z will cause both A and
						// B's streams to get pinged, even though A won't actually see that
						// status. However, that should be rare.
						if tweet.In_reply_to_user_id != nil {
							if in_reply_to_user_id := int64(*tweet.In_reply_to_user_id); in_reply_to_user_id != 0 {
								if _, inMap := followingUserIdMap[in_reply_to_user_id]; !inMap {
									continue
								}
							}
						}

						go pingUser(userId, int64(*tweet.Id), pingUrl)
					}
				} else {
					fmt.Printf("No tweet?")
				}
			}
		}
	}
}