Esempio n. 1
0
// Sets the hypermedia response "_links" section with all of the routes we have defined for territories.
func setTerritoryLinks(self string) *config.HypermediaResource {
	res := config.NewHypermediaResource()
	res.Links["territory:list"] = config.HypermediaLink{
		Href: "/territory/list",
	}
	res.Links["territory:count"] = config.HypermediaLink{
		Href: "/territory/count/{territory}/{series}/{field}{?from,to,network,fieldValue}",
	}
	res.Links["territory:timeseries-count"] = config.HypermediaLink{
		Href: "/territory/timeseries/count/{territory}/{series}/{field}{?from,to,network,fieldValue}",
	}
	res.Links["territory:aggregate"] = config.HypermediaLink{
		Href: "/territory/aggregate/{territory}/{series}{?from,to,network,fields}",
	}
	res.Links["territory:timeseries-aggregate"] = config.HypermediaLink{
		Href: "/territory/timeseries/aggregate/{territory}/{series}{?from,to,network,fields,resolution}",
	}
	res.Links["territory:messages"] = config.HypermediaLink{
		Href: "/territory/messages/{territory}{?from,to,limit,skip,network,lang,country,geohash,gender,questions}",
	}
	res.Links["territory:top-images"] = config.HypermediaLink{
		Href: "/territory/top/images/{territory}/{series}{?from,to,network}",
	}
	res.Links["territory:top-videos"] = config.HypermediaLink{
		Href: "/territory/top/videos/{territory}/{series}{?from,to,network}",
	}
	res.Links["territory:top-audio"] = config.HypermediaLink{
		Href: "/territory/top/audio/{territory}/{series}{?from,to,network}",
	}
	res.Links["territory:top-links"] = config.HypermediaLink{
		Href: "/territory/top/links/{territory}/{series}{?from,to,network}",
	}
	res.Links["territory:top-locations"] = config.HypermediaLink{
		Href: "/territory/top/locations/{territory}/{series}{?from,to,network}",
	}
	res.Links["territory:top-keywords"] = config.HypermediaLink{
		Href: "/territory/top/keywords/{territory}/{series}{?from,to,network}",
	}
	res.Links["territory:top-hashtags"] = config.HypermediaLink{
		Href: "/territory/top/hashtags/{territory}/{series}{?from,to,network}",
	}

	selfedRes := config.NewHypermediaResource()
	for link, _ := range res.Links {
		if link == self {
			selfedRes.Links["self"] = res.Links[link]
		} else {
			selfedRes.Links[link] = res.Links[link]
		}
	}
	return selfedRes
}
Esempio n. 2
0
// Retrieves information to provide a summary about a give URL, specifically articles/blog posts.
// TODO: Make this more robust (more details, videos, etc.). Some of this may eventually also go into the harvest.
// TODO: Likely fork this package and add in some of the things I did for Virality Score in order to get even more data.
func LinkDetails(w rest.ResponseWriter, r *rest.Request) {
	res := config.NewHypermediaResource()
	res.Links["self"] = config.HypermediaLink{
		Href: "/link/details{?url}",
	}

	queryParams := r.URL.Query()
	if len(queryParams["url"]) > 0 {
		g := goose.New()
		article := g.ExtractFromUrl(queryParams["url"][0])
		log.Println(article)

		res.Data["title"] = article.Title
		res.Data["published"] = article.PublishDate
		res.Data["favicon"] = article.MetaFavicon
		res.Data["domain"] = article.Domain
		res.Data["description"] = article.MetaDescription
		res.Data["keywords"] = article.MetaKeywords
		res.Data["content"] = article.CleanedText
		res.Data["url"] = article.FinalUrl
		res.Data["image"] = article.TopImage
		res.Data["movies"] = article.Movies
		res.Success()
	}

	w.WriteJson(res.End())
}
Esempio n. 3
0
// Reloads the configuration from the available file on disk
func ReloadSocialHarvestConfig(w rest.ResponseWriter, r *rest.Request) {
	res := config.NewHypermediaResource()
	res.Links["self"] = config.HypermediaLink{
		Href: "/config/reload{?original}",
	}
	res.Links["read"] = config.HypermediaLink{
		Href: "/config/read",
	}
	res.Links["write"] = config.HypermediaLink{
		Href: "/config/write",
	}

	// If the original configuration (passed via --conf flag or the default social-harvest-conf.json) is desired, pass true to setConfig() by looking
	// for an "?original=true" in the route. By default, it will look for an updated config in the "sh-data" directory.
	queryParams := r.URL.Query()
	original := false
	if len(queryParams["original"]) > 0 {
		original = true
		res.Meta.Message = "Original configuration loaded."
	}
	setConfig(original)

	// Return the updated config
	res.Data["config"] = socialHarvest.Config.Harvest
	res.Success()
	w.WriteJson(res.End())
}
Esempio n. 4
0
// API: Shows the harvest schedule as currently configured
func ShowSchedule(w rest.ResponseWriter, r *rest.Request) {
	res := config.NewHypermediaResource()
	res.AddCurie("schedule", "/docs/rels/{rel}", true)

	res.Links["self"] = config.HypermediaLink{
		Href: "/schedule/read",
	}
	res.Links["schedule:add"] = config.HypermediaLink{
		Href: "/schedule/add",
	}
	res.Links["schedule:delete"] = config.HypermediaLink{
		Href:      "/schedule/delete/{id}",
		Templated: true,
	}

	jobs := []map[string]interface{}{}
	for _, item := range socialHarvest.Schedule.Cron.Entries() {
		m := make(map[string]interface{})
		m["id"] = item.Id
		m["name"] = item.Name
		m["next"] = item.Next
		m["prev"] = item.Prev
		m["job"] = getFunctionName(item.Job)
		jobs = append(jobs, m)
	}
	res.Data["totalJobs"] = len(jobs)
	res.Data["jobs"] = jobs

	res.Success()
	w.WriteJson(res.End("There are " + strconv.Itoa(len(jobs)) + " jobs scheduled."))
}
Esempio n. 5
0
// Sets the hypermedia response "_links" section with all of the routes we have defined for territories.
func setTerritoryLinks(self string) *config.HypermediaResource {
	res := config.NewHypermediaResource()
	res.Links["territory:list"] = config.HypermediaLink{
		Href: "/territory/list",
	}

	selfedRes := config.NewHypermediaResource()
	for link, _ := range res.Links {
		if link == self {
			selfedRes.Links["self"] = res.Links[link]
		} else {
			selfedRes.Links[link] = res.Links[link]
		}
	}
	return selfedRes
}
Esempio n. 6
0
// Writes a new JSON configuration file into "sh-data" (a reload should be called afer this unless there's an error) so that the original is preserved
func WriteSocialHarvestConfig(w rest.ResponseWriter, r *rest.Request) {
	res := config.NewHypermediaResource()
	res.Links["self"] = config.HypermediaLink{
		Href: "/config/write",
	}
	res.Links["read"] = config.HypermediaLink{
		Href: "/config/read",
	}
	res.Links["reload"] = config.HypermediaLink{
		Href: "/config/reload{?original}",
	}

	// TODO: Take JSON from request and create new SocialHarvestConf struct with it.
	// Then save to disk in "sh-data" path.
	// Validate it? Aside from being able to convert it to a struct... Make sure it has a certain number of fields?

	var c = config.SocialHarvestConf{}
	err := r.DecodeJsonPayload(&c)
	if err != nil {
		//rest.Error(w, err.Error(), http.StatusInternalServerError)
		//return
		res.Meta.Message = "Invalid configuration."
		w.WriteJson(res.End())
	}
	if config.SaveConfig(c) {
		res.Success()
	}

	w.WriteJson(res.End())
}
Esempio n. 7
0
// API: Shows the current harvester configuration
func ShowSocialHarvestConfig(w rest.ResponseWriter, r *rest.Request) {
	res := config.NewHypermediaResource()
	res.Links["self"] = config.HypermediaLink{
		Href: "/config/read",
	}
	res.Links["reload"] = config.HypermediaLink{
		Href: "/config/reload{?original}",
	}
	res.Links["write"] = config.HypermediaLink{
		Href: "/config/write",
	}
	res.Data["config"] = socialHarvest.Config.Harvest
	res.Success()
	w.WriteJson(res.End())
}
Esempio n. 8
0
// Returns information about the currently configured database, if it's reachable, etc.
func DatabaseInfo(w rest.ResponseWriter, r *rest.Request) {
	res := config.NewHypermediaResource()
	res.Links["database:info"] = config.HypermediaLink{
		Href: "/database/info",
	}

	if socialHarvest.Database.Postgres != nil {
		res.Data["type"] = "postgres"
		// SELECT * FROM has_database_privilege('username', 'database', 'connect');
		// var r struct {
		// 	hasAccess string `db:"has_database_privilege" json:"has_database_privilege"`
		// }
		//err := socialHarvest.Database.Postgres.Get(&r, "SELECT * FROM has_database_privilege("+socialHarvest.Config.Database.User+", "+socialHarvest.Config.Database.Database+", 'connect')")
		//res.Data["r"] = r
		//res.Data["err"] = err
		res.Data["hasAccess"] = socialHarvest.Database.HasAccess()
	}

	res.Data["configuredType"] = socialHarvest.Config.Database.Type

	res.Success()
	w.WriteJson(res.End())
}