func gaeDevCommand(_ *command.Args, opts *gaeDevOptions) error {
log.Debugf("starting App Engine development server - press Control+C to stop")
var buildArgs []string
resources, err := makeAppAssets(buildArgs)
if err != nil {
return err
}
go watchAppResources(buildArgs, resources)
serveCmd, err := startServe(buildArgs, opts)
if err != nil {
return err
}
errCh := make(chan error, 1)
go func() {
errCh <- serveCmd.Wait()
}()
ch := make(chan os.Signal, 1)
signal.Notify(ch, os.Interrupt)
select {
case <-ch:
log.Debugf("exiting")
<-errCh
log.Debugf("exited")
case err := <-errCh:
return err
}
return nil
}
func (t *Template) replaceExtendTag(name string, treeMap map[string]*parse.Tree, from string) error {
var err error
hasExtend := false
var loc string
for _, v := range treeMap {
templateutil.WalkTree(v, func(n, p parse.Node) {
if err != nil {
return
}
if templateutil.IsPseudoFunction(n, "extend") {
if hasExtend {
loc2, _ := v.ErrorContext(n)
err = fmt.Errorf("multiple {{ extend }} tags in %q, %s and %s", name, loc, loc2)
return
}
hasExtend = true
loc, _ = v.ErrorContext(n)
var repl parse.Node
if from == "" {
// empty
log.Debugf("removing {{ extend }} from %q", name)
repl = &parse.TextNode{
NodeType: parse.NodeText,
Pos: n.Position(),
}
} else {
log.Debugf("extending %q at %s with %q", name, loc, from)
repl = templateutil.TemplateNode(from, n.Position())
}
err = templateutil.ReplaceNode(n, p, repl)
}
})
}
return err
}
func sendCommand(serverAddr string, args []string) (bool, error) {
scheme, addr, err := parseServerAddr(serverAddr)
if err != nil {
return false, err
}
conn, err := net.Dial(scheme, addr)
if err != nil {
return false, err
}
defer conn.Close()
if err := encodeArgs(conn, args); err != nil {
return false, err
}
log.Debugf("sent command %s", args)
closed := false
ch := make(chan os.Signal, 1)
signal.Notify(ch, os.Interrupt)
defer signal.Stop(ch)
done := make(chan struct{}, 1)
defer func() {
done <- struct{}{}
}()
go func() {
select {
case <-ch:
closed = true
conn.Close()
case <-done:
}
}()
ok := true
for {
r, s, err := decodeResponse(conn)
if err != nil {
if closed {
return ok, nil
}
return ok, err
}
log.Debugf("received response %d", r)
switch r {
case respEnd:
return ok, nil
case respOk:
fmt.Print(s)
case respErr:
ok = false
fmt.Fprint(os.Stderr, s)
default:
return false, fmt.Errorf("invalid response type %d", r)
}
}
return ok, nil
}
func (app *App) importAssets(included *includedApp) error {
im := included.app.assetsManager
if !app.shouldImportAssets() {
im.SetPrefix(included.prefix + im.Prefix())
return nil
}
m := app.assetsManager
prefix := strings.ToLower(included.name)
renames := make(map[string]string)
err := vfs.Walk(im.VFS(), "/", func(fs vfs.VFS, p string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if p != "" && p[0] == '/' {
p = p[1:]
}
log.Debugf("will import asset %v from app %s", p, included.name)
src, err := im.Load(p)
if err != nil {
return err
}
defer src.Close()
seeker, err := assets.Seeker(src)
if err != nil {
return err
}
sum := hashutil.Fnv32a(seeker)
nonExt := p[:len(p)-len(path.Ext(p))]
dest := path.Join(prefix, nonExt+".gen."+sum+path.Ext(p))
renames[p] = dest
log.Debugf("importing asset %q as %q", p, dest)
if m.Has(dest) {
return nil
}
f, err := m.Create(dest, true)
if err != nil {
return err
}
defer f.Close()
if _, err := seeker.Seek(0, os.SEEK_SET); err != nil {
return err
}
if _, err := io.Copy(f, seeker); err != nil {
return err
}
return nil
})
if err != nil {
return err
}
included.renames = renames
return nil
}
func extractGoType(messages messageMap, fset *token.FileSet, f *ast.File, typ string) error {
// for castings
calls, err := astutil.Calls(fset, f, typ)
if err != nil {
return err
}
for _, c := range calls {
if len(c.Args) > 0 {
lit, pos := astutil.StringLiteral(fset, c.Args[0])
if pos == nil {
p := fset.Position(c.Pos())
log.Debugf("Skipping cast to %s (%v) - not a literal", typ, p)
continue
}
comment := comments(fset, f, pos)
if err := messages.AddString(&astutil.String{Value: lit, Position: pos}, comment); err != nil {
return err
}
}
}
strings, err := astutil.Strings(fset, f, typ)
if err != nil {
return err
}
for _, s := range strings {
comment := comments(fset, f, s.Position)
if err := messages.AddString(s, comment); err != nil {
return err
}
}
return nil
}
func extractGoMessages(messages messageMap, path string, opts *ExtractOptions) error {
log.Debugf("Extracting messages from Go file %s", path)
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
if err != nil {
return fmt.Errorf("error parsing go file %s: %s", path, err)
}
if opts != nil {
for _, v := range opts.Functions {
if v.Template {
continue
}
if err := extractGoFunc(messages, fset, f, v); err != nil {
return err
}
}
for _, v := range opts.Types {
if err := extractGoType(messages, fset, f, v); err != nil {
return err
}
}
for _, v := range opts.TagFields {
if err := extractGoTagField(messages, fset, f, v); err != nil {
return err
}
}
}
return nil
}
func bakeCommand(_ *command.Args, opts *bakeOptions) error {
extensions := []string{".html", ".css", ".js"}
if opts.Dir == "" {
return errors.New("dir can't be empty")
}
if opts.Name == "" {
base := filepath.Base(opts.Dir)
if opts.VFS {
opts.Name = base + "FS"
} else {
opts.Name = base + "Data"
}
}
if opts.Out == "" {
opts.Out = filepath.Base(opts.Dir) + "_baked.go"
}
// go ignores files starting with _
opts.Out = strings.TrimLeft(opts.Out, "_")
extensions = append(extensions, strings.Split(opts.Extensions, ",")...)
var buf bytes.Buffer
odir := filepath.Dir(opts.Out)
p, err := build.ImportDir(odir, 0)
if err == nil {
buf.WriteString(fmt.Sprintf("package %s\n", p.Name))
}
buf.WriteString(genutil.AutogenString())
if err := writeBakedFSCode(&buf, opts, extensions); err != nil {
return err
}
if err := genutil.WriteAutogen(opts.Out, buf.Bytes()); err != nil {
return err
}
log.Debugf("Assets written to %s (%d bytes)", opts.Out, buf.Len())
return nil
}
func (t *Template) Compile() error {
if err := t.noCompiled("can't compile"); err != nil {
return err
}
if err := t.preparePlugins(); err != nil {
return err
}
for _, v := range t.referencedTemplates() {
if _, ok := t.trees[v]; !ok {
log.Debugf("adding missing template %q as empty", v)
tree := compileTree("")
t.AddParseTree(v, tree)
}
}
if err := t.prepareAssets(); err != nil {
return err
}
prog, err := newProgram(t)
if err != nil {
return err
}
prog.debugDump()
t.prog = prog
return nil
}
func compileMessagesCommand(_ *command.Args, opts *compileMessagesOptions) error {
var poFiles []string
err := filepath.Walk(opts.Messages, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.ToLower(filepath.Ext(path)) == ".po" {
log.Debugf("compiling po file %s", path)
poFiles = append(poFiles, path)
}
return nil
})
if err != nil {
return err
}
pos := make([]*po.Po, len(poFiles))
for ii, v := range poFiles {
p, err := po.ParseFile(v)
if err != nil {
return err
}
pos[ii] = p
}
copts := &messages.CompileOptions{DefaultContext: opts.Context}
return messages.Compile(opts.Out, pos, copts)
}
func makeAppAssets(buildArgs []string) ([]string, error) {
log.Debugf("compiling app assets")
if err := runBuild(buildArgs); err != nil {
return nil, err
}
p, err := appPath()
if err != nil {
return nil, err
}
defer os.Remove(p)
err = runCmd(exec.Command(p, "-log-debug=false", "make-assets"))
if err != nil {
return nil, err
}
var buf bytes.Buffer
cmd := exec.Command(p, "_print-resources")
cmd.Stdout = &buf
if err := cmd.Run(); err != nil {
return nil, err
}
var m map[string]string
if err := json.Unmarshal(buf.Bytes(), &m); err != nil {
return nil, err
}
values := make([]string, 0, len(m))
for _, v := range m {
values = append(values, v)
}
return values, nil
}
func updatePackage(e *doc.Environment, pkg string) error {
if strings.HasSuffix(pkg, "/") {
pkg += "..."
}
goBin := "go"
env := make(map[string]string)
for _, v := range os.Environ() {
if eq := strings.Index(v, "="); eq >= 0 {
env[v[:eq]] = v[eq+1:]
}
}
if goRoot := e.Context.GOROOT; goRoot != "" {
goBin = filepath.Join(goRoot, "bin", "go")
env["GOROOT"] = goRoot
}
if goPath := e.Context.GOPATH; goPath != "" {
env["GOPATH"] = goPath
}
cmd := exec.Command(goBin, "get", "-u", "-v", pkg)
cmdEnv := make([]string, 0, len(env))
for k, v := range env {
cmdEnv = append(cmdEnv, k+"="+v)
}
cmd.Env = cmdEnv
log.Debugf("Updating package %s", pkg)
if log.Level() == log.LDebug {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
}
return cmd.Run()
}
func (d *Driver) Initialize(ms []driver.Model) error {
// Create tables
for _, v := range ms {
tbl, err := d.makeTable(v)
if err != nil {
return err
}
existingTbl, err := d.backend.Inspect(d.db, v)
if err != nil {
return err
}
if existingTbl != nil {
err = d.mergeTable(v, existingTbl, tbl)
} else {
if len(tbl.Fields) == 0 {
log.Debugf("Skipping collection %s (model %v) because it has no fields", v.Table, v)
continue
}
// Table does not exists, create it
err = d.createTable(v, tbl)
}
if err != nil {
return err
}
}
// Create indexes
for _, v := range ms {
if err := d.createIndexes(v); err != nil {
return err
}
}
return nil
}
func (f *fileWriter) Open(name string) error {
dir, err := os.Stat(f.dir)
if err != nil || !dir.IsDir() {
os.Remove(f.dir)
// Make logs directory
if err := os.MkdirAll(f.dir, 0755); err != nil {
return err
}
}
if f.f != nil {
if err := f.Close(); err != nil {
return err
}
}
f.name = name
logPath := f.logPath(0)
w, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
return fmt.Errorf("error opening log file %s: %s", logPath, err)
}
log.Debugf("opened log file %s", name)
info, err := w.Stat()
if err != nil {
w.Close()
return err
}
f.size = uint64(info.Size())
f.f = w
return nil
}
// Remove deletes the file with the given id.
func (s *Blobstore) Remove(id string) error {
s.drv.Remove(s.metaName(id))
err := s.drv.Remove(id)
if err == nil {
log.Debugf("[BLOBSTORE]: Removed %v", id)
}
return err
}
func (app *App) handleError(ctx *Context, err interface{}) bool {
if gerr, ok := err.(Error); ok {
log.Debugf("HTTP error: %s (%d)", gerr.Error(), gerr.StatusCode())
app.handleHTTPError(ctx, gerr.Error(), gerr.StatusCode())
return true
}
return false
}
func (g *Governator) startServer() error {
q := newQuit()
scheme, addr, err := parseServerAddr(g.ServerAddr)
if err != nil {
return err
}
if scheme == "unix" {
os.Remove(addr)
}
server, err := net.Listen(scheme, addr)
if err != nil {
return err
}
if scheme == "unix" {
if gid := getGroupId(AppName); gid >= 0 {
os.Chown(addr, 0, gid)
os.Chmod(addr, 0775)
}
}
conns := make(chan net.Conn, 10)
go func() {
for {
conn, err := server.Accept()
if err != nil {
log.Errorf("error accepting connection: %s", err)
}
log.Debugf("new connection %s", conn.RemoteAddr())
conns <- conn
}
}()
go func() {
for {
select {
case <-q.stop:
if scheme == "unix" {
os.Remove(addr)
}
q.sendStopped()
return
case conn := <-conns:
go func() {
if err := g.serveConn(conn); err != nil {
log.Errorf("error serving connection: %s", err)
}
}()
}
}
}()
g.mu.Lock()
defer g.mu.Unlock()
g.quits = append(g.quits, q)
return nil
}
func (f *fileWriter) rotate() error {
if err := f.Close(); err != nil {
return err
}
last := []string{f.logPath(f.count - 1), f.compressedLogPath(f.count - 1)}
for _, v := range last {
if fileExists(v) {
log.Debugf("removing %s", v)
if err := os.Remove(v); err != nil {
return err
}
}
}
var compress []string
for ii := f.count - 2; ii >= 0; ii-- {
ccur := f.compressedLogPath(ii)
if fileExists(ccur) {
to := f.compressedLogPath(ii + 1)
log.Debugf("moving %s to %s", ccur, to)
if err := os.Rename(ccur, to); err != nil {
return err
}
continue
}
cur := f.logPath(ii)
if fileExists(cur) {
to := f.logPath(ii + 1)
log.Debugf("moving %s to %s", cur, to)
if err := os.Rename(cur, to); err != nil {
return err
}
compress = append(compress, to)
}
}
for _, v := range compress {
f.compressFile(v)
}
return f.Open(f.name)
}
func Compile(m *Manager, name string, typ Type, opts Options) (string, error) {
ext := path.Ext(name)
compiler := compilers[typ][strings.ToLower(ext)]
if compiler == nil {
return name, nil
}
f, err := m.Load(name)
if err != nil {
return "", err
}
defer f.Close()
seeker, err := Seeker(f)
fnv := hashutil.Fnv32a(seeker)
out := fmt.Sprintf("%s.gen.%s.%s", name, fnv, typ.Ext())
if o, _ := m.Load(out); o != nil {
o.Close()
log.Debugf("%s already compiled to %s", name, out)
return out, nil
}
seeker.Seek(0, 0)
var buf bytes.Buffer
log.Debugf("compiling %s to %s", name, out)
if err := compiler.Compile(&buf, seeker, opts); err != nil {
return "", err
}
w, err := m.Create(out, true)
if err != nil {
return "", err
}
if _, err := io.Copy(w, bytes.NewReader(buf.Bytes())); err != nil {
w.Close()
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
return out, nil
}
func appendScriptFallback(m *Manager, script *Asset, assets *[]*Asset, fallback string) error {
if fallback != "" {
if _, ok := script.Attributes["async"]; ok {
log.Debugf("skipping fallback for async script %s", script.Name)
return nil
}
fb, err := scriptFallback(m, script, fallback)
if err != nil {
return err
}
*assets = append(*assets, fb)
}
return nil
}
func (f *fileWriter) compressFile(name string) {
log.Debugf("will compress %s", name)
var wg sync.WaitGroup
wg.Add(1)
go func() {
if err := compressFile(name); err != nil {
log.Errorf("error compressing %s: %s", name, err)
}
wg.Done()
}()
if f.waitCompress {
wg.Wait()
}
}
func (app *App) chainTemplate(t *Template, included *includedApp) (*Template, error) {
log.Debugf("chaining template %s", t.tmpl.Name())
container, name, err := app.loadContainerTemplate(included)
if err != nil {
return nil, err
}
if err := app.rewriteAssets(t.tmpl, included); err != nil {
return nil, err
}
if err := container.tmpl.InsertTemplate(t.tmpl, name); err != nil {
return nil, err
}
return container, nil
}
func (s *Service) updateConfig(cfg *Config) {
if reflect.DeepEqual(s.Config, cfg) {
// there were changes to the file which don't affect the conf
return
}
log.Debugf("changed service %s's configuration", s.Name())
start := false
if s.State == StateStarted {
start = s.Stop() == nil
}
s.Config = cfg
if start {
s.Start()
}
}
func ArticleHandler(ctx *app.Context) {
slug := ctx.IndexValue(0)
var art *article.Article
articles := getArticles(ctx)
for _, v := range articles {
if v.Slug() == slug {
art = v
break
}
}
if art == nil {
for _, v := range articles {
for _, s := range v.AllSlugs() {
if s == slug {
ctx.MustRedirectReverse(true, ctx.HandlerName(), s)
return
}
}
}
ctx.NotFound("article not found")
return
}
fs := vfs.Memory()
filename := path.Base(art.Filename)
if filename == "" {
filename = "article.html"
}
if err := vfs.WriteFile(fs, filename, art.Text, 0644); err != nil {
panic(err)
}
log.Debugf("loading article %s", articleId(art))
tmpl, err := app.LoadTemplate(ctx.App(), fs, nil, filename)
if err != nil {
panic(err)
}
var buf bytes.Buffer
if err := tmpl.ExecuteTo(&buf, ctx, nil); err != nil {
panic(err)
}
body := buf.String()
data := map[string]interface{}{
"Article": art,
"Title": art.Title(),
"Body": template.HTML(body),
}
ctx.MustExecute("article.html", data)
}
// LoadTemplate loads a template using the template
// loader and the asset manager assocciated with
// this app
func (app *App) LoadTemplate(name string) (*Template, error) {
app.templatesMutex.RLock()
tmpl := app.templatesCache[name]
app.templatesMutex.RUnlock()
if tmpl == nil {
var err error
log.Debugf("Loading root template %s", name)
if profile.On && profile.Profiling() {
defer profile.Start("template").Note("load", name).End()
}
tmpl, err = app.loadTemplate(app.templatesFS, app.assetsManager, name)
if err != nil {
return nil, err
}
var funcs []*template.Func
for _, v := range app.included {
funcs = append(funcs, &template.Func{
Name: v.assetFuncName(),
Fn: v.assetFunc(tmpl.tmpl),
Traits: template.FuncTraitPure,
})
}
tmpl.tmpl.Funcs(funcs)
for _, v := range app.templatePlugins {
if err := tmpl.tmpl.AddPlugin(v); err != nil {
return nil, fmt.Errorf("error adding plugin %q: %s", v.Template.Root(), err)
}
}
if profile.On {
if profilePlugin != nil {
tmpl.tmpl.AddPlugin(profilePlugin)
}
}
if err := tmpl.prepare(); err != nil {
return nil, err
}
if !app.cfg.TemplateDebug {
app.templatesMutex.Lock()
if app.templatesCache == nil {
app.templatesCache = make(map[string]*Template)
}
app.templatesCache[name] = tmpl
app.templatesMutex.Unlock()
}
}
return tmpl, nil
}
// Bake writes the data for a VFS generated from dir to the given
// io.Writer. The extensions argument can be used to limit the
// files included in the VFS by their extension. If empty, all
// files are included.
func Bake(w io.Writer, dir string, extensions []string) error {
fs, err := vfs.FS(dir)
if err != nil {
return err
}
if len(extensions) > 0 {
// Clone the fs and remove files not matching the extension
exts := make(map[string]bool)
for _, v := range extensions {
if v == "" {
continue
}
if v[0] != '.' {
v = "." + v
}
exts[strings.ToLower(v)] = true
}
mem := vfs.Memory()
if err := vfs.Clone(mem, fs); err != nil {
return err
}
err := vfs.Walk(mem, "/", func(fs vfs.VFS, p string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if !exts[strings.ToLower(path.Ext(p))] {
if err := fs.Remove(p); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
fs = mem
}
vfs.Walk(fs, "/", func(_ vfs.VFS, p string, info os.FileInfo, err error) error {
if !info.IsDir() {
log.Debugf("baking %s", p)
}
return nil
})
return vfs.WriteTarGzip(w, fs)
}
// Create a local fallback for the given script, downloading it if
// necessary
func scriptFallback(m *Manager, script *Asset, fallback string) (*Asset, error) {
fallbackName := script.Name
if !m.Has(fallbackName) {
var scriptURL string
if urlutil.IsURL(fallbackName) {
scriptURL = fallbackName
fallbackName = "asset.gen." + hashutil.Adler32(fallbackName) + "." + path.Base(fallbackName)
} else {
cdn, _, err := Cdn(fallbackName)
if err != nil {
return nil, err
}
scriptURL = cdn
}
if !m.Has(fallbackName) {
u, err := url.Parse(scriptURL)
if err != nil {
return nil, err
}
if u.Scheme == "" {
u.Scheme = "http"
}
log.Debugf("fetching local fallback for %s to %s", u, fallbackName)
resp, err := http.Get(u.String())
if err != nil {
return nil, err
}
defer resp.Body.Close()
w, err := m.Create(fallbackName, true)
if err != nil {
return nil, err
}
defer w.Close()
if _, err := io.Copy(w, resp.Body); err != nil {
return nil, err
}
}
}
return &Asset{
Name: fallbackName,
Position: Bottom,
Type: TypeOther,
HTML: fmt.Sprintf("<script>%s || document.write('<scr'+'ipt src=\"%s\"><\\/scr'+'ipt>')</script>", fallback, m.URL(fallbackName)),
}, nil
}
// CreateId works like Create, but uses the given id rather than generating
// a new one. If a file with the same id already exists, it's overwritten.
func (s *Blobstore) CreateId(id string) (*WFile, error) {
if strings.HasSuffix(id, metaSuffix) {
return nil, fmt.Errorf("invalid id %s, can't end with .meta", id)
}
if len(id) < minIdLength {
return nil, fmt.Errorf("id is too short (%d characters), minimum length is %d", len(id), minIdLength)
}
w, err := s.drv.Create(id)
if err != nil {
return nil, err
}
log.Debugf("[BLOBSTORE]: Created %v", id)
return &WFile{
id: id,
file: w,
dataHash: newHash(),
store: s,
}, nil
}
// Emit emits this signal, calling all registered listeners. Note
// that this function should only be called from a typed Signal
// wrapping it. See the package documentation for examples.
func (s *Signal) Emit(data interface{}) {
s.mu.RLock()
if c := len(s.listeners); c > 0 {
listeners := make([]*listener, len(s.listeners))
copy(listeners, s.listeners)
// Don't hold the lock while invoking the
// listeners, otherwise calling Remove()
// from a handler would cause a deadlock.
s.mu.RUnlock()
if s.name != "" {
log.Debugf("emitting signal %v (%d listeners)", s.name, c)
}
for _, v := range listeners {
v.Handler(data)
}
} else {
s.mu.RUnlock()
}
}
func (g *Governator) Run() error {
g.mu.Lock()
if g.quit != nil {
g.mu.Unlock()
return errors.New("governator already running")
}
g.quit = newQuit()
g.mu.Unlock()
go g.monitor.Run()
if g.configDir != "" {
if err := g.startWatching(); err != nil {
log.Errorf("error watching %s, configuration won't be automatically updated: %s", g.servicesDir(), err)
}
}
if g.ServerAddr != "" {
if err := g.startServer(); err != nil {
log.Errorf("error starting server, can't receive remote commands: %s", err)
}
}
g.startServices(nil)
g.quit.waitForStop()
g.mu.Lock()
for _, q := range g.quits {
q.sendStop()
}
// Wait for goroutines to exit cleanly
for _, q := range g.quits {
q.waitForStopped()
}
g.quits = nil
g.mu.Unlock()
// Release the lock for stopServices
g.stopServices(nil)
g.mu.Lock()
defer g.mu.Unlock()
g.monitor.quit.sendStop()
g.monitor.quit.waitForStopped()
log.Debugf("daemon exiting")
g.quit.sendStopped()
g.quit = nil
return nil
}
func compressFile(name string) error {
r, err := os.Open(name)
if err != nil {
return err
}
defer r.Close()
w, err := os.OpenFile(name+".gz", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer w.Close()
gw := gzip.NewWriter(w)
defer gw.Close()
if _, err := io.Copy(gw, r); err != nil {
return err
}
os.Remove(name)
log.Debugf("compressed %s to %s", r.Name(), w.Name())
return nil
}