コード例 #1
0
ファイル: api_configs.go プロジェクト: speedland/service
func setupAPIConfigs(app *server.App) {
	var API = app.API()
	// --------------------------------------------------------------------------
	// Sever Config
	API.GET("/configs/",
		server.Handler(func(req *wcg.Request) response.Response {
			return response.NewJSONResponse(configs.GetAll(req))
		}))

	API.GET("/configs/:key.json",
		server.Handler(func(req *wcg.Request) response.Response {
			val := configs.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				return response.NewJSONResponse(val)
			}
		}),
	)

	API.PUT("/configs/:key.json",
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("value").Required()
		}),
		server.Handler(func(req *wcg.Request) response.Response {
			val := configs.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				val.Value = req.Form("value")
				configs.PutMulti(req, val)
				return response.NewJSONResponse(val)
			}
		}))
}
コード例 #2
0
ファイル: api_gates.go プロジェクト: speedland/service
func setupAPIGates(app *server.App) {
	var API = app.API()

	// --------------------------------------------------------------------------
	// Gate
	API.GET("/gates/",
		server.Handler(func(req *wcg.Request) response.Response {
			return response.NewJSONResponse(gates.GetAll(req))
		}))

	API.GET("/gates/:key.json",
		server.Handler(func(req *wcg.Request) response.Response {
			val := gates.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				return response.NewJSONResponse(val)
			}
		}))

	API.PUT("/gates/:key.json",
		middleware.ParseForm(nil),
		server.Handler(func(req *wcg.Request) response.Response {
			val := gates.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				val.UIDs = req.HTTPRequest().Form["uids"]
				gates.PutMulti(req, val)
				return response.NewJSONResponse(val)
			}
		}))
}
コード例 #3
0
ファイル: api_members.go プロジェクト: speedland/service
func setupAPIMembers(app *server.App) {
	var API = app.API()
	API.GET("/:artist/members/",
		handlers.ArtistHandler(
			func(req *wcg.Request, artist *hplink.Artist) response.Response {
				memberQuery := Member.Query().Order("Index").Filter("ArtistKey=", artist.Key).Cache(fmt.Sprintf(ckAllMembersTemplate, artist.Key))
				return middleware.EntityAll(memberQuery)(req)
			},
		))

	API.GET("/:artist/members/:member.json",
		handlers.MemberHandler(
			func(req *wcg.Request, member *hplink.Member) response.Response {
				return response.NewJSONResponse(member)
			},
		),
	)

	API.PUT("/:artist/members/:member.json",
		handlers.AdminGate,
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("color").Required()
		}),
		handlers.MemberHandler(
			func(req *wcg.Request, member *hplink.Member) response.Response {
				if err := Member.UpdateEntityFromForm(member, req.HTTPRequest().Form); err != nil {
					req.Logger.Warnf("UpdateEntityFromForm returns and error: %#v", err)
					return response.APIBadRequest(middleware.ErrInvalidFormParameters)
				}
				Member.Put().Cache(ckAllMembersTemplate).MustUpdate(req, member)
				return response.NewJSONResponse(member)
			},
		),
	)
}
コード例 #4
0
ファイル: entity.go プロジェクト: speedland/service
func EntityDelete(del *entities.Delete, keyParam string) server.Handler {
	return server.Handler(
		func(req *wcg.Request) response.Response {
			del.Key(req.Param(keyParam)).MustCommit(req)
			return response.NewJSONResponse(true)
		})
}
コード例 #5
0
ファイル: parallel.go プロジェクト: speedland/service
func Parallel(handlers map[string]server.Handler) server.Handler {
	return server.Handler(
		func(req *wcg.Request) response.Response {
			var status = make(map[string]int)
			var data = make(map[string]interface{})
			var mutex sync.Mutex
			var wg sync.WaitGroup
			for k, q := range handlers {
				wg.Add(1)
				go func(k string, h server.Handler) {
					defer wg.Done()
					v := h.Handle(req)
					if vv, ok := v.(*response.JSONResponse); ok {
						mutex.Lock()
						defer mutex.Unlock()
						status[k] = vv.StatusCode
						data[k] = vv.Data
					}
				}(k, q)
			}
			wg.Wait()
			for k, code := range status {
				// return non 2xx response
				if code > 0 && (code < 200 || code >= 300) {
					return response.NewJSONResponseWithStatus(data[k], code)
				}
			}
			return response.NewJSONResponse(data)
		})
}
コード例 #6
0
ファイル: api_stats.go プロジェクト: speedland/service
func setupAPIStats(app *server.App) {
	var API = app.API()

	// --------------------------------------------------------------------------
	// Stats
	API.GET("/stats/",
		server.Handler(func(req *wcg.Request) response.Response {
			stats, _ := runtime.Stats(gae.NewContext(req))
			json := map[string]interface{}{
				"version":     lib.Commit,
				"timestamp":   lib.Timestamp,
				"stats":       stats,
				"environment": lib.CurrentEnvironment(),
				"envvars":     lib.GetEnvVars(),
			}
			if _, dsStats, err := entities.DatastoreStat.Get().Key("total_entities_usage").One(req); err != nil {
				json["datastore"] = dsStats
			}

			// use array response for API convention.
			return response.NewJSONResponse(
				[](map[string]interface{}){
					json,
				},
			)
		}))
}
コード例 #7
0
ファイル: entity.go プロジェクト: speedland/service
func EntityTail(query *entities.Query) server.Handler {
	return server.Handler(
		func(req *wcg.Request) response.Response {
			result := query.MustExecute(req)
			return response.NewJSONResponse(result.Tail())
		})
}
コード例 #8
0
ファイル: api_users.go プロジェクト: speedland/service
func setupAPIUsers(app *server.App) {
	var API = app.API()

	API.GET("/me/",
		server.Handler(func(req *wcg.Request) response.Response {
			if req.User == nil || request.ByGuest(req) {
				return response.NewJSONResponse(nil)
			}
			return response.NewJSONResponse(req.User)
		}))

	API.POST("/me/gates/",
		server.Handler(func(req *wcg.Request) response.Response {
			keys := strings.Split(req.Form("keys"), ",")
			result := gates.EvalKeys(req, keys...)
			return response.NewJSONResponse(result)
		}))
}
コード例 #9
0
ファイル: entity.go プロジェクト: speedland/service
func EntityGet(get *entities.Get, keyParam string) server.Handler {
	return server.Handler(
		func(req *wcg.Request) response.Response {
			_, one := get.Key(req.Param(keyParam)).MustOne(req)
			if one == nil {
				return response.APINotFound
			}
			return response.NewJSONResponse(one)
		})
}
コード例 #10
0
ファイル: api_configs.go プロジェクト: speedland/service
func setupAPIConfigs(app *server.App) {
	var API = app.API()
	API.GET("/configs/:userid/messenger.json",
		middleware.Gate("family"),
		validateUserID,
		requireMessengeOptIn,
		server.Handler(
			func(req *wcg.Request) response.Response {
				_, one := MessengerNotification.Get().Key(req.Param("userid")).UseDefaultIfNil(true).Cache(true).MustOne(req)
				return response.NewJSONResponse(one)
			},
		))

	API.PUT("/configs/:userid/messenger.json",
		middleware.Gate("family"),
		validateUserID,
		requireMessengeOptIn,
		middleware.ParseForm(nil),
		middleware.EntityPutOrCreate(MessengerNotification.Put(), "userid"))

	API.POST("/configs/:userid/messenger/notify/",
		middleware.Gate("family"),
		validateUserID,
		requireMessengeOptIn,
		server.Handler(
			func(req *wcg.Request) response.Response {
				content, err := createDailyNotificationContent(req)
				if err != nil {
					return response.InternalServerError(req, err)
				}
				err = messenger.NewMessengerClient(req).SendText(content.title + "\n" + strings.Join(content.lines, "\n"))
				if err != nil {
					req.Logger.Errorf("Failed to notify a message to messenger: %v", err)
				}
				return response.NewJSONResponse(map[string]interface{}{
					"ok": true,
				})
			},
		))

}
コード例 #11
0
ファイル: cron.go プロジェクト: speedland/service
// Process implements wcg.Handler#Process
func (h *Cron) Process(res *wcg.Response, req *wcg.Request) {
	req.Logger.Infof("[Cron] %s (%s) - start", h.Path, h.Time)
	for _, handler := range h.handlers {
		if err := handler(req); err != nil {
			req.Logger.Fatalf("[Cron] %s (%s) - error: %v", h.Path, h.Description, err)
			response.NewJSONResponseWithStatus(err, 500).Render(res, req)
			return
		}
	}
	req.Logger.Infof("[Cron] %s (%s) - finish", h.Path, h.Description)
	response.NewJSONResponse(true).Render(res, req)
	return
}
コード例 #12
0
ファイル: parallel_test.go プロジェクト: speedland/service
func Test_Parallel(t *testing.T) {
	assert := gaetest.NewAssert(t)
	router := wcg.NewRouter()
	funA := server.Handler(func(req *wcg.Request) response.Response {
		return response.NewJSONResponse(true)
	})
	funB := server.Handler(func(req *wcg.Request) response.Response {
		return response.NewJSONResponse(false)
	})

	router.GET("/*", Parallel(map[string]server.Handler{
		"a": funA,
		"b": funB,
	}))
	req := ts.GET("/a/")
	res := req.RouteTo(router)

	var got map[string]bool
	assert.HTTPStatus(200, res)
	assert.JSONResponse(&got, res)
	assert.OK(got["a"])
	assert.Not(got["b"])
}
コード例 #13
0
ファイル: api_artists.go プロジェクト: speedland/service
func setupAPIArtist(app *server.App) {
	var API = app.API()
	API.GET("/",
		server.Handler(func(req *wcg.Request) response.Response {
			pArtists := entities.Artist.Query().Order("Index").Cache(ckAllArtits).MustExecute(req)
			artist := pArtists.Data.([]hplink.Artist)
			if req.Query("w") == "members" {
				iterator.MustParallelSlice(artist, func(i int, a *hplink.Artist) error {
					pMembers := entities.Member.Query().Order("Index").Filter("ArtistKey=", a.Key).Cache(fmt.Sprintf(ckAllMembersTemplate, a.Key)).MustExecute(req)
					artist[i].Members = pMembers.Data.([]hplink.Member)
					return nil
				})
			}
			return response.NewJSONResponse(artist)
		}),
	)

	API.GET("/:artist/",
		middleware.EntityGet(entities.Artist.Get().Cache(true), "artist"))
}
コード例 #14
0
ファイル: api_stats.go プロジェクト: speedland/service
func execStatsQuery(req *wcg.Request, query *entities.Query) response.Response {
	now := lib.Now()
	until := wcg.ParseDateTimeOr(req.Query("until"), now)
	if until.After(now) {
		until = now
	}
	sinceDefault := until.Add(StatsQueryDefaultTimeWindow)
	sinceMin := until.Add(StatsQueryMaxTimeWindow)
	since := wcg.ParseDateTimeOr(req.Query("since"), until.Add(-24*time.Hour))
	if since.After(until) {
		since = sinceDefault
	} else if sinceMin.After(since) {
		since = sinceMin
	}
	query = query.Filter("Timestamp >=", since).Filter("Timestamp <", until)
	if req.Query("device") != "" {
		query = query.Filter("Device =", req.Query("device"))
	}
	return response.NewJSONResponse(query.MustExecute(req).Data)
}
コード例 #15
0
ファイル: api.go プロジェクト: speedland/service
			req,
			fmt.Errorf(
				"token mismatch: hub.verify_token(%q) != facebook_messenger_verify_token(%q)",
				req.Query("hub.verify_token"), token,
			),
		)
	}
	return response.NewTextResponse(req.Query("hub.challenge"))
})

// WebhookHandler is a server.Handler to respond setup HTTP call from a Facebook server
var WebHookHandler = server.Handler(func(req *wcg.Request) response.Response {
	messages, perr := parseMessage(req.HTTPRequest().Body)
	if perr != nil {
		req.Logger.Warnf("Server could not parse incomming messenger hook message: %s", perr)
		return response.NewJSONResponse(true)
	}
	for _, msg := range messages {
		switch msg.(type) {
		case *MessageAuthentication:
			if err := performAuthentication(req, msg.(*MessageAuthentication)); err != nil {
				req.Logger.Warnf("Could not perform messenger authentication...: %v", err)
				return response.NewJSONResponse(true)
			}
			for _, options := range handlers {
				if options.HandleAuthentication != nil {
					if err := options.HandleAuthentication(req, msg.(*MessageAuthentication)); err != nil {
						req.Logger.Warnf("Could not handle messenger authentication...: %v", err)
						return response.NewJSONResponse(true)
					}
				}
コード例 #16
0
ファイル: api_iepg_records.go プロジェクト: speedland/service
func setupAPIIEPGRecords(app *server.App) {
	var API = app.API()

	API.GET("/iepg/records/",
		middleware.EntityQuery(recentRecordsQuery()))

	API.GET("/iepg/records/:id.json",
		middleware.EntityGet(IEPG.Get().Cache(true), "id"))

	API.POST("/iepg/records/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("program_title").Required()
			v.Field("start_at").Required()
			v.Field("end_at").Required()
			v.Field("category").Required()
			v.Field("cid").Required()
			v.Field("sid").Required()
		}),
		middleware.EntityPost(
			IEPG.Put().
				SetField("Source", pt.IEPGSourceUser).
				Cache(_CacheRecentIEPGRecords),
		),
	)

	API.PUT("/iepg/records/:id.json",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("program_title").Required()
			v.Field("start_at").Required()
			v.Field("end_at").Required()
			v.Field("category").Required()
			v.Field("cid").Required()
			v.Field("sid").Required()
		}),
		requireIEPGOwner("id"),
		middleware.EntityPut(
			IEPG.Put().
				SetField("Source", pt.IEPGSourceUser).
				Cache(_CacheRecentIEPGRecords),
			"id",
		),
	)

	API.POST("/iepg/records/:id/notification.json",
		middleware.Gate("family"),
		getIEPG("id", func(req *wcg.Request, iepg *pt.IEPG) response.Response {
			var msg string
			var configs []pt.MessengerNotification
			var err error
			t := req.Form("type")
			switch t {
			case "start":
				msg = fmt.Sprintf("[%s] 録画を開始しました。", iepg.ProgramTitle)
				configs, err = getMessengerNotificationOptInUsers(req, "OnStart")
				break
			case "end":
				msg = fmt.Sprintf("[%s] 録画を終了しました。", iepg.ProgramTitle)
				configs, err = getMessengerNotificationOptInUsers(req, "OnEnd")
				break
			default:
				return response.BadRequest(req, fmt.Errorf("`type` should be specified and must be one of 'start', 'end' (actual=%s)", t))
			}
			if err != nil {
				return response.InternalServerError(req, err)
			}
			if req.Form("message") != "" {
				msg = fmt.Sprintf("%s\n%s", msg, req.Form("message"))
			}

			for _, cfg := range configs {
				client := messenger.NewMessengerClient(req)
				client.UserID = cfg.UserID
				err = client.SendText(msg)
				if err != nil {
					req.Logger.Warnf("Failed to send a messenger message: %v", err)
				}
			}
			return response.NewJSONResponse(true)
		}),
	)

	API.DELETE("/iepg/records/:id.json",
		middleware.Gate("family"),
		requireIEPGOwner("id"),
		middleware.EntityDelete(IEPG.Delete().Cache(_CacheRecentIEPGRecords), "id"))

	// optin or optout for auto collected IEPGs
	API.PUT("/iepg/records/:id/opt-in/",
		middleware.Gate("family"),
		getIEPG("id", func(req *wcg.Request, iepg *pt.IEPG) response.Response {
			err := iepg.SetOptIn(req.Form("opt_in") == "1" || req.Form("opt_in") == "true")
			if err != nil {
				return response.BadRequest(req, fmt.Errorf("Could not configure opt-in field on the user created IEPG"))
			}
			IEPG.Put().Key(req.Param("id")).Cache(_CacheRecentIEPGRecords).MustUpdate(req, iepg)
			return response.NewJSONResponse(map[string]bool{
				"ok":      true,
				"opt_in":  iepg.OptIn,
				"opt_out": iepg.OptOut,
			})
		}),
	)
}
コード例 #17
0
func setupAPICrawlerSettings(app *server.App) {
	var API = app.API()

	var urlValidator = server.Handler(func(req *wcg.Request) response.Response {
		_, err := url.Parse(req.Param("url"))
		if err != nil {
			return response.APINotFound
		}
		return nil
	})

	API.GET("/crawlersettings/:type/",
		middleware.EntityAll(CrawlerSettings.Query().Filter("Type=", request.Value(func(req *wcg.Request) interface{} {
			t, _ := hplink.ParseCrawlerSettingsType(req.Param("type"))
			if t == hplink.CrawlerSettingsTypeUnknown {
				return entities.FilterValueSkip
			}
			return t
		}))),
	)

	API.GET("/crawlersettings/ameblo/stats/",
		server.Handler(func(req *wcg.Request) response.Response {
			type post struct {
				URL string    `json:"url"`
				At  time.Time `json:"at"`
			}
			type stats struct {
				URL               string `json:"url"`
				FirstPost         *post  `json:"first_post,omitempty"`
				LastPost          *post  `json:"last_post,omitempty"`
				TotalPosts        int    `json:"total_posts"`
				CrawledPosts      int    `json:"crawled_posts"`
				ImageFailurePosts int    `json:"image_failure_posts"`
			}
			p := CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req)
			settings := p.Data.([]hplink.CrawlerSettings)
			s := make([]stats, len(settings))
			if err := iterator.ParallelSlice(settings, func(i int, v *hplink.CrawlerSettings) error {
				s[i].URL = v.URL
				s[i].TotalPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).MustCount(req)
				if s[i].TotalPosts > 0 {
					s[i].CrawledPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("IsContentsCrawled=", true).MustCount(req)
					s[i].ImageFailurePosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("Images.Height=", 0).MustCount(req)
					pf := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("PostAt").Limit(1).MustExecute(req)
					pl := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("-PostAt").Limit(1).MustExecute(req)
					first := pf.Head().(*hplink.AmebloPost)
					last := pl.Head().(*hplink.AmebloPost)
					s[i].FirstPost = &post{
						URL: first.URL, At: first.PostAt,
					}
					s[i].LastPost = &post{
						URL: last.URL, At: last.PostAt,
					}
					return nil
				}
				return nil
			}); err != nil {
				panic(err)
			}
			return response.NewJSONResponse(s)
		}),
	)

	API.GET("/crawlersettings/:url.json",
		middleware.EntityGet(CrawlerSettings.Get(), "url"),
	)

	API.PUT("/crawlersettings/:url.json",
		urlValidator,
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("artist_key").Required()
		}),
		middleware.EntityPutOrCreate(
			CrawlerSettings.Put(),
			"url",
		),
	)

	API.DELETE("/crawlersettings/:url.json",
		urlValidator,
		middleware.EntityDelete(CrawlerSettings.Delete(), "url"),
	)
}
コード例 #18
0
ファイル: api_stats.go プロジェクト: speedland/service
func setupAPIStats(app *server.App) {
	var API = app.API()

	API.GET("/stats/servers/",
		middleware.EntityAll(Server.Query()))

	API.GET("/stats/nasdisks/",
		middleware.EntityAll(NASDisk.Query()))

	API.GET("/stats/servers/:server/:stats/",
		server.Handler(
			func(req *wcg.Request) response.Response {
				key, _ := Server.Get().Key(req.Param("server")).Cache(true).MustOne(req)
				if key == nil {
					req.Logger.Debugf("server %q is not found.", req.Param("server"))
					return response.NotFound(req)
				}
				var kind *entities.Kind
				switch req.Param("stats") {
				case "system":
					kind = SystemStats
					break
				case "cpu":
					kind = CPUStats
					break
				case "memory":
					kind = MemoryStats
					break
				case "disk":
					kind = DiskStats
					break
				case "filesystem":
					kind = FileSystemStats
					break
				case "network":
					kind = NetworkStats
					break
				default:
					req.Logger.Debugf("metric %q is invalid.", req.Param("stats"))
					return response.NotFound(req)
				}
				return execStatsQuery(req, kind.Query().Ancestor(key))
			},
		))

	API.POST("/stats/metrics/",
		middleware.APITokenOnly(),
		server.Handler(
			func(req *wcg.Request) response.Response {
				ent, err := SystemMetric.CreateEntitiesFromJSON(req.HTTPRequest().Body)
				if err != nil {
					return response.BadRequest(req, err)
				}
				collection := home.NewStatsFromMetrics(ent.([]*home.SystemMetric))
				if len(collection) == 0 {
					return response.BadRequest(req, fmt.Errorf("No stats are collected."))
				}
				if collection[0].System.ServerName == "" {
					return response.BadRequest(req, fmt.Errorf("server_name is missing"))
				}
				for _, stats := range collection {
					key := getServerKey(req, stats.System.ServerName)
					id := fmt.Sprintf("%s.%d", key.StringID(), stats.Timestamp.Unix())
					stats.System.ID = id
					stats.CPU.ID = id
					stats.Memory.ID = id
					for _, v := range stats.Disks {
						v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
					}
					for _, v := range stats.Networks {
						v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
					}
					SystemStats.Put().Parent(key).MustUpdate(req, stats.System)
					CPUStats.Put().Parent(key).MustUpdate(req, stats.CPU)
					MemoryStats.Put().Parent(key).MustUpdate(req, stats.Memory)
					DiskStats.PutMulti().Parent(key).MustUpdate(req, stats.Disks)
					NetworkStats.PutMulti().Parent(key).MustUpdate(req, stats.Networks)
				}
				return response.NewJSONResponse(true)
			},
		))

	API.POST("/stats/filesystem/",
		middleware.APITokenOnly(),
		server.Handler(
			func(req *wcg.Request) response.Response {
				ent, err := FileSystemStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
				if err != nil {
					return response.BadRequest(req, err)
				}
				filesystems := ent.([]*home.FileSystemStats)
				if filesystems[0].ServerName == "" {
					return response.BadRequest(req, fmt.Errorf("server_name is missing"))
				}
				key := getServerKey(req, filesystems[0].ServerName)
				for _, fs := range filesystems {
					fs.ID = fmt.Sprintf("%s.%s.%d", fs.ServerName, fs.Name, fs.Timestamp.Unix())
				}
				FileSystemStats.PutMulti().Parent(key).MustUpdate(req, filesystems)
				return response.NewJSONResponse(true)
			},
		))

	API.POST("/stats/smart/",
		middleware.APITokenOnly(),
		server.Handler(
			func(req *wcg.Request) response.Response {
				ent, err := SMARTStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
				if err != nil {
					return response.BadRequest(req, err)
				}
				smartstats := ent.([]*home.SMARTStats)
				if smartstats[0].Serial == "" {
					return response.BadRequest(req, fmt.Errorf("serial is missing"))
				}
				key := getNASDiskKey(req, smartstats[0].Serial)
				for _, smart := range smartstats {
					smart.ID = fmt.Sprintf("%s.%d", smart.Serial, smart.Timestamp.Unix())
				}
				SMARTStats.PutMulti().Parent(key).MustUpdate(req, smartstats)
				return response.NewJSONResponse(true)
			},
		))
}
コード例 #19
0
ファイル: api_datastore.go プロジェクト: speedland/service
func setupAPIDatastore(app *server.App) {
	var API = app.API()

	if lib.IsOnLocalGAE() {
		API.GET("/datastore/cleanup/:kind.json",
			server.Handler(func(req *wcg.Request) response.Response {
				k := entities.FindKind(req.Param("kind"), req.Query("ns"))
				if k == nil {
					return response.NotFound(req)
				}
				num := k.DeleteQuery().MustCommit(req)
				return response.NewJSONResponse(map[string]int{
					"entities": num,
				})
			}),
		)
	}

	API.GET("/datastore/export/:kind.json",
		middleware.APITokenOnly(),
		server.Handler(func(req *wcg.Request) response.Response {
			k := entities.FindKind(req.Param("kind"), req.Query("ns"))
			if k == nil {
				return response.NotFound(req)
			}
			// TODO: export with gzip
			return middleware.EntityStreaming(
				k.Query().Order(fmt.Sprintf("-%s", k.TimestampFieldName)).Limit(
					wcg.ParseInt(req.Query("limit"), 100, -1, math.MaxInt32),
				),
				true,
			).Handle(req)
		}))

	API.POST("/datastore/import/:kind.json",
		middleware.APITokenOnly(),
		server.Handler(func(req *wcg.Request) response.Response {
			k := entities.FindKind(req.Param("kind"), req.Query("ns"))
			if k == nil {
				return response.NotFound(req)
			}
			var line = 0
			var total = 0
			var errors = make([]*entityImportError, 0)
			var list = make([]interface{}, 0, 100)
			var keys = make([]*datastore.Key, 0, 100)
			var buff []byte
			var err error
			reader := bufio.NewReaderSize(req.HTTPRequest().Body, 4096)
			for err != io.EOF {
				buff, err = reader.ReadBytes('\n')
				line += 1
				if err != nil && err != io.EOF {
					return response.InternalServerError(req, err)
				}
				buff = bytes.TrimSpace(buff)
				if len(buff) == 0 { // skip empty row
					continue
				}
				var row entityImportRow
				if err2 := json.Unmarshal(buff, &row); err2 != nil {
					errors = append(errors, &entityImportError{
						Line:  line,
						Error: fmt.Sprintf("%v - (row: %q)", err2, buff),
					})
					continue
				}
				ent, err2 := k.NewEntityFromJSON([]byte(row.Entity))
				if err2 != nil {
				}
				total = total + 1
				keys = append(keys, buildKey(req, row.Key))
				list = append(list, ent)
				if len(list) == 100 {
					k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
					list = make([]interface{}, 0, 100)
					keys = make([]*datastore.Key, 0, 100)
				}
			}
			if len(list) > 0 {
				k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
			}
			return response.NewJSONResponse(&entityImportResult{
				Total:  total,
				Lines:  line - 1,
				Errors: errors,
			})
		}))
}
コード例 #20
0
func setupAPIIEPGKeywords(app *server.App) {
	var API = app.API()

	API.GET("/iepg/keywords/",
		middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))

	API.POST("/iepg/keywords/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("keyword").Required()
			v.Field("category").Required()
			v.IntField("scope").Required()
		}),
		middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))

	API.DELETE("/iepg/keywords/:id.json",
		middleware.Gate("family"),
		middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))

	API.POST("/iepg/keywords/preview/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("keyword").Required()
			v.IntField("scope").Required()
		}),
		server.Handler(
			func(req *wcg.Request) response.Response {
				list, err := crawlKeyword(req, &pt.IEPGKeyword{
					Keyword: req.Form("keyword"),
					Scope:   pt.ParseIEPGCrawlerScope(req.Form("scope")),
				})
				if err != nil {
					return response.InternalServerError(req, err)
				}
				return response.NewJSONResponse(map[string]interface{}{
					"hits": list,
				})
			},
		))

	var AsyncAPI = app.AsyncAPI()

	keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
	keywordCrawlTask.Queue.Rate = "1/s"
	keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
	keywordCrawlTask.OnTrigger(middleware.Gate("family"))
	keywordCrawlTask.OnMonitor(middleware.Gate("family"))
	keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
		func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
			data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
			if data == nil {
				return nil, nil
			}
			var keywords = data.([]pt.IEPGKeyword)
			req.Logger.Infof("Crawling %d keywords", len(keywords))
			for _, key := range keywords {
				list, err := crawlKeyword(req, &key)
				if err != nil {
					return nil, err
				}
				IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
			}
			return nil, nil
		},
	))
	keywordCrawlTask.ScheduleCron(
		"Crawl IEPG keywords",
		"every 1 hours",
	)
}
コード例 #21
0
ファイル: async_api.go プロジェクト: speedland/service
// Define endpoints to process an async task.
func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig {
	if !strings.HasSuffix(path, "/") {
		panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path))
	}
	api := helper.app.API()
	// /path/to/something/ => api-path-to-something
	p := api.Path(path)
	name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters
	name = strings.Replace(name, "/", "-", -1)          // replace '/' with '-'
	config := &AsyncAPIConfig{
		Queue: helper.app.Queue().DefinePush(name),
		app:   helper.app,
		path:  path,
	}

	// Endpoint that trigger the task.
	api.POST(path, Handler(
		func(req *wcg.Request) response.Response {
			for _, h := range config.triggerHandlers {
				resp := h.Handle(req)
				if resp != nil {
					req.Logger.Debugf("Async Task is not triggered.")
					return resp
				}
			}
			task := &models.AsyncAPITask{}
			task.ID = wcg.NewUUID()
			task.Path = path
			task.Query = req.HTTPRequest().URL.RawQuery
			task.Status = models.AsyncAPIStatusReady
			entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task)
			// Push a task
			taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode())
			if err := config.Queue.PushTask(req, taskPath, nil); err != nil {
				return response.InternalServerError(req, err)
			}
			req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath)
			return response.NewJSONResponseWithStatus(
				AsyncTaskTriggerResponse{task.ID},
				201,
			)
		},
	))

	// Endpoint that executes the task in background
	// This endpoint must be called by PushQueue
	api.POST(path+":taskid.json", Handler(
		func(req *wcg.Request) response.Response {
			if !request.IsTask(req) {
				req.Logger.Warnf(
					"Non system call to the async task endpoint: UserAgent=%s, IP=%s",
					req.HTTPRequest().UserAgent(),
					req.HTTPRequest().RemoteAddr,
				)
				return response.APIOK
			}

			_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
			if one == nil {
				req.Logger.Warnf(
					"Task %q not found: UserAgent=%s, IP=%s",
					req.Param("taskid"),
					req.HTTPRequest().UserAgent(),
					req.HTTPRequest().RemoteAddr,
				)
				return response.APIOK
			}
			task := one.(*models.AsyncAPITask)
			logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name))
			if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning {
				logger.Warnf("Task is not ready: Status=%s", task.Status)
				return response.APIOK
			}

			if task.Status == models.AsyncAPIStatusReady {
				task.StartAt = lib.Now()
				task.Status = models.AsyncAPIStatusRunning
				entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
			}

			var err error
			var progress *models.AsyncAPITaskProgress
			var resp response.Response
			func() {
				defer func() {
					if x := recover(); x != nil {
						logger.Errorf("Unhandle error recovered from the task: %v", x)
						err = fmt.Errorf("%v", x)
					}
				}()
				progress, err = config.processHandler(req, task)
			}()

			if progress != nil {
				task.Progress = append(task.Progress, *progress)
			}
			if progress != nil && progress.Next != nil {
				// update the entity (Progress field)
				entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)

				// Push a next request into the queue
				// This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code)
				// The test environment uses the response JSON to call the next URL and that response is used only for this purpose.
				// If the code cannot push task for the next request, we should stop the task as failure.
				logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next)
				if !req.IsTest() {
					err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil)
				}
				if err == nil {
					return response.NewJSONResponse(progress.Next)
				}
				// if an error occurrs by PushTask, just stopping the task and update the datastore as failure.
			}
			// finished the task
			task.FinishAt = lib.Now()
			tt := task.FinishAt.Sub(task.StartAt)
			if err == nil {
				task.Status = models.AsyncAPIStatusSuccess
				logger.Infof("Task finished successfully (time: %s)", tt.String())
				resp = response.APIOK
			} else {
				task.Error = err.Error()
				task.Status = models.AsyncAPIStatusFailure
				logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error)
				resp = response.APIInternalServerError
			}
			entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
			return resp
		},
	))

	// Endpoint that returns the task status.
	api.GET(path+":taskid.json", Handler(
		func(req *wcg.Request) response.Response {
			for _, h := range config.triggerHandlers {
				resp := h.Handle(req)
				if resp != nil {
					return resp
				}
			}
			_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
			if one == nil {
				return response.NotFound(req)
			}
			task := one.(*models.AsyncAPITask)
			return response.NewJSONResponse(
				AsyncTaskMonitorResponse{
					ID:       task.ID,
					Status:   task.Status,
					Progress: task.LastProgress(),
				},
			)
		},
	))

	return config
}