func GenerateData(configurationFile string) {
if configurationFile == "" {
panic("I can't work without a configuration file")
}
log.Info("Loading config")
conf, err := config.LoadConfig(configurationFile)
if err != nil {
panic(err)
}
//api := GetTwitter(&conf)
db := nutz.NewStorage(configurationFile+".db", 0600, nil)
mygraph := Graph{Nodes: []Node{}, Links: []Link{}}
innercount := 0
nodecount := 0
group := 0
for _, account := range conf.TwitterAccounts {
tweets := db.Get(account, "tweets")
from := db.Get(account, "from")
retweets := db.Get(account, "retweets")
unique_mentions := db.Get(account, "unique_mentions")
total_mentions := db.Get(account, "total_mentions")
followers := db.Get(account, "followers")
following := db.Get(account, "following")
followers_followed := db.Get(account, "followers_followed")
mentions_to_followed := db.Get(account, "mentions_to_followed")
log.Info("Account: " + account)
log.Info("from: " + string(from.Data))
log.Info("Tweets: " + string(tweets.Data))
log.Info("retweets: " + string(retweets.Data))
log.Info("unique_mentions: " + string(unique_mentions.Data))
log.Info("total_mentions: " + string(total_mentions.Data))
log.Info("followers: " + string(followers.Data))
log.Info("following: " + string(following.Data))
log.Info("followers_followed: " + string(followers_followed.Data))
log.Info("mentions_to_followed: " + string(mentions_to_followed.Data))
myUniqueMentions := db.GetAll(account, "map_unique_mentions").DataList
nUniqueMentions, _ := strconv.Atoi(string(unique_mentions.Data))
nMentions_to_followed, _ := strconv.Atoi(string(mentions_to_followed.Data))
nTweets, _ := strconv.Atoi(string(tweets.Data))
nReTweets, _ := strconv.Atoi(string(retweets.Data))
om := OutsideMentions(nUniqueMentions, nMentions_to_followed)
apt := AnswerPeopleTax(nUniqueMentions, nMentions_to_followed, nTweets, nReTweets)
if math.IsNaN(float64(om)) {
om = float32(0.01)
}
if math.IsNaN(float64(apt)) {
apt = float32(0.01)
}
// fmt.Println("\tDemocracy tax: " + FloatToString(di))
fmt.Println("\tOutside of circle mentions: " + FloatToString(om))
fmt.Println("\t of answering to external people: " + FloatToString(apt))
mygraph.Nodes = append(mygraph.Nodes, Node{Name: account, Group: group, Thickness: om, Size: apt})
for k, v := range myUniqueMentions {
//id, _ := strconv.ParseInt(k, 10, 64)
//User, _ := api.GetUsersShowById(id, nil)
//log.Info("[" + User.ScreenName + "]:" + string(v))
// now you can put User.ScreeName in the name of the node
weight, _ := strconv.Atoi(string(v))
mygraph.Nodes = append(mygraph.Nodes, Node{Name: string(k), Group: group, Thickness: 0.01, Size: 0.01})
mygraph.Links = append(mygraph.Links, Link{Source: innercount, Target: nodecount, Value: weight})
innercount++
}
innercount++
nodecount = innercount
group++
}
fileJson, _ := json.MarshalIndent(mygraph, "", " ")
err = ioutil.WriteFile(configurationFile+".output", fileJson, 0644)
if err != nil {
log.Info("WriteFileJson ERROR: " + err.Error())
}
}
func GatherData(configurationFile string) {
if configurationFile == "" {
panic("I can't work without a configuration file")
}
log.Info("Loading config")
conf, err := config.LoadConfig(configurationFile)
if err != nil {
panic(err)
}
myTweets := make(map[string]timelinesTweets)
api := GetTwitter(&conf)
db := nutz.NewStorage(configurationFile+".db", 0600, nil)
retweetRegex, _ := regexp.Compile(`^RT`) // detecting retweets
for _, account := range conf.TwitterAccounts {
log.Info("-== Timeline for Account: %#v ==-\n", account)
myTweets[account] = GetTimelines(api, account, conf.FetchFrom)
log.Info("-== END TIMELINE for %#v ==-\n", account)
}
log.Info("Analyzing && collecting data")
for i := range myTweets {
var retweets int
var mymentions int
var mentions struct {
Name string
Indices []int
Screen_name string
Id int64
Id_str string
}
var myUniqueMentions map[int64]int
myUniqueMentions = make(map[int64]int)
fmt.Println("-== Account: " + i + " ==-")
fmt.Println("\tTweets: " + strconv.Itoa(len(myTweets[i])))
for _, t := range myTweets[i] {
// detecting retweets
if retweetRegex.MatchString(t.Text) == true {
retweets++
} else {
//detecting mentions outside retweets
for _, mentions = range t.Entities.User_mentions {
mymentions++
if t.InReplyToUserID != 0 { //we are interested only in replies
myUniqueMentions[mentions.Id]++
}
}
}
}
if conf.FetchFollow == true {
log.Info("-== GetFollowers for Account: %#v ==-\n", i)
Followers := GetFollowers(api, i)
log.Info("-== GetFollowing for Account: %#v ==-\n", i)
Following := GetFollowing(api, i)
log.Info("-== End getting Following/Followers for Account: %#v ==-\n", i)
var Corrispective []int64
var MentionsWithCorrispective []int64
for _, i := range Following {
if _, ok := myUniqueMentions[i]; ok {
MentionsWithCorrispective = append(MentionsWithCorrispective, i)
}
if utils.IntInSlice(i, Followers) == true {
Corrispective = append(Corrispective, i)
}
}
fmt.Println("\tFollowers: " + strconv.Itoa(len(Followers)))
fmt.Println("\tFollowing: " + strconv.Itoa(len(Following)))
fmt.Println("\tFollowers && Following: " + strconv.Itoa(len(Corrispective)))
fmt.Println("\tBetween mentions, those are whom the user is following: " + strconv.Itoa(len(MentionsWithCorrispective)))
//di := DemocracyIndex(len(myUniqueMentions), len(MentionsWithCorrispective), len(myTweets[i]), retweets)
om := OutsideMentions(len(myUniqueMentions), len(MentionsWithCorrispective))
apt := AnswerPeopleTax(len(myUniqueMentions), len(MentionsWithCorrispective), len(myTweets[i]), retweets)
// fmt.Println("\tDemocracy tax: " + FloatToString(di))
fmt.Println("\tOutside of circle mentions: " + FloatToString(om))
fmt.Println("\t of answering to external people: " + FloatToString(apt))
db.Create(i, "followers", []byte(strconv.Itoa(len(Followers))))
db.Create(i, "following", []byte(strconv.Itoa(len(Following))))
db.Create(i, "followers_followed", []byte(strconv.Itoa(len(Corrispective))))
db.Create(i, "mentions_to_followed", []byte(strconv.Itoa(len(MentionsWithCorrispective))))
}
fmt.Println("\tof wich, there are " + strconv.Itoa(retweets) + " retweets")
fmt.Println("\tof wich, there are " + strconv.Itoa(len(myUniqueMentions)) + " unique mentions (not in retweets)")
fmt.Println("\tof wich, there are " + strconv.Itoa(mymentions) + " total mentions (not in retweets)")
db.Create(i, "from", []byte(conf.Date))
db.Create(i, "tweets", []byte(strconv.Itoa(len(myTweets[i]))))
db.Create(i, "retweets", []byte(strconv.Itoa(retweets)))
db.Create(i, "unique_mentions", []byte(strconv.Itoa(len(myUniqueMentions))))
db.Create(i, "total_mentions", []byte(strconv.Itoa(mymentions)))
for k, v := range myUniqueMentions {
db.Create(i, strconv.FormatInt(k, 10), []byte(strconv.Itoa(v)), "map_unique_mentions")
}
// Visualize example:
//http://bl.ocks.org/mbostock/4062045
// Circle size is defined by it's radius (r) : .attr("r", 5)
// TOOlTIP: http://bl.ocks.org/Caged/6476579
}
}
