func (s *Source) loadBackPANSource() error {
log.Info("Loading BackPAN index: backpan-index")
file, err := os.Open("backpan-index")
if err != nil {
log.Warn(err.Error())
return nil
}
index, err := ioutil.ReadAll(file)
file.Close()
if err != nil {
log.Fatal(err)
}
for _, p := range strings.Split(string(index), "\n") {
if !strings.HasPrefix(p, "authors/id/") {
continue
}
//log.Printf("Parsing: %s\n", p)
m := s.ModuleFromBackPANIndex(p)
if m != nil {
s.ModuleList[m.Name+"-"+m.Version] = m
}
}
log.Printf("Found %d packages for source: %s", len(s.ModuleList), s)
return nil
}
func WatchConfig(configWrapper iConfigWrapper, configPath string) {
watcherForConfigFile, err := fsnotify.NewWatcher()
CheckError(err)
go func() {
for {
select {
case e := <-watcherForConfigFile.Event:
if lastConfigModifiedTime.Add(1 * time.Second).After(time.Now()) {
continue
}
lastConfigModifiedTime = time.Now()
// log.Warn("Config '%s' modified, but NOT RELOADING YET.", e.Name)
log.Warn("Config '%s' modified, now reloading (most of the) config settings.", e.Name)
configWrapper.SetConfig(loadFile(configPath))
configWrapper.ConfigReloaded()
break
case err := <-watcherForConfigFile.Error:
log.Error("Error: %+v", err)
break
}
}
}()
err = watcherForConfigFile.Watch(configPath)
CheckError(err)
}
// ApplyCmds applies a map of known commands and their values.
func ApplyCmds(cmds map[string]string) {
for cmd, val := range cmds {
switch cmd {
case "volume":
v, err := strconv.ParseFloat(val, 32)
if err != nil {
log.Warn("Config parse error: %v", err)
continue
}
SetVolume(v)
case "brightness":
v, err := strconv.ParseFloat(val, 32)
if err != nil {
log.Warn("Config parse error: %v", err)
continue
}
SetBrightness(v)
}
}
}
func (s *Source) loadCPANSource() error {
log.Info("Loading CPAN index: %s", s.Index)
res, err := http.Get(s.Index)
if err != nil {
log.Warn("Error loading index: %s", err)
return nil
}
// TODO optional gzip
r, err := gzip.NewReader(res.Body)
if err != nil {
log.Warn(err.Error())
b, _ := ioutil.ReadAll(res.Body)
log.Info("%s", string(b))
return nil
}
packages, err := ioutil.ReadAll(r)
res.Body.Close()
if err != nil {
log.Warn(err)
return nil
}
foundnl := false
for _, p := range strings.Split(string(packages), "\n") {
if !foundnl && len(p) == 0 {
foundnl = true
continue
}
if !foundnl || len(p) == 0 {
continue
}
m := s.ModuleFromCPANIndex(p)
s.ModuleList[m.Name] = m
}
log.Info("Found %d packages for source: %s", len(s.ModuleList), s)
return nil
}
func Create(assetLoader func(string) ([]byte, error)) *Config {
if assetLoader == nil {
log.Warn("No asset loader provided; content loading will fail")
assetLoader = func(asset string) ([]byte, error) {
log.Warn("Attempted to load asset but no asset loader provided: %s", asset)
return nil, errors.New("Not found")
}
}
config := &Config{
Listen: ":7050",
AssetLoader: assetLoader,
Cache: make(map[string]interface{}),
Events: &events.Emitter{},
LeftDelim: "{{",
RightDelim: "}}",
}
config.env()
config.flags()
return config
}
func (ws *Workspace) ExecFunction(task *Task, name string, args ...string) string {
log.Info("Executing function %s: %s", name, args)
var fn *Function
if f, ok := ws.Functions[name]; ok {
fn = f
} else if f, ok := GlobalWorkspace.Functions[name]; ok {
fn = f
} else {
log.Warn("Function not found: %s", name)
return ""
}
argmap := make(map[string]string)
for i, arg := range fn.Args {
argmap[arg] = args[i]
}
for k, v := range argmap {
log.Info("argmap: %s => %s", k, v)
for t, m := range task.Metadata {
log.Info("meta: %s => %s", t, m)
v = strings.Replace(v, "$"+t, m, -1)
}
argmap[k] = v
}
c := fn.Command
for k, v := range argmap {
log.Info("ARG: %s => %s", k, v)
c = strings.Replace(c, k, v, -1)
}
var funcEnvironment map[string]string
if ws.InheritEnvironment {
funcEnvironment = ws.Environment
} else if GlobalWorkspace.InheritEnvironment {
funcEnvironment = GlobalWorkspace.Environment
} else {
funcEnvironment = make(map[string]string)
}
tsk := NewTask(nil, "Function$"+name, fn.Executor, c, funcEnvironment, false, "", "", make(map[string]string), "")
ch := tsk.Start()
<-ch
return tsk.TaskRuns[0].StdoutBuf.String()
}
func (s *Source) Find(d *Dependency) (*Module, error) {
log.Debug("Finding dependency: %s", d)
switch s.Type {
case "SmartPAN":
log.Debug("=> Using SmartPAN source")
url := s.URL
if !strings.HasSuffix(s.URL, "/") {
url += "/"
}
url += "where/" + d.Name + "/" + d.Modifier + d.Version
log.Info("Query: %s", url)
res, err := http.Get(url)
if err != nil {
log.Error("Error querying SmartPAN: %s", err.Error())
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
log.Trace("Got response: %s", string(body))
if res.StatusCode != http.StatusOK {
log.Info("Module not found in SmartPAN: %s", d.Name)
return nil, nil
}
var v *WhereOutput
if err = json.Unmarshal(body, &v); err != nil {
log.Error("Error parsing JSON: %s", err.Error())
return nil, err
}
log.Trace("Found module %s", v.Module)
if len(v.Versions) == 0 {
log.Info("Found module but no versions returned")
return nil, nil
}
var lv *VersionOutput
for _, ver := range v.Versions {
if ver.Version == v.Latest {
log.Info("Using latest version of %s: %f", v.Module, ver.Version)
lv = ver
break
}
}
if lv == nil {
log.Info("Couldn't find latest version, selecting first available")
lv = v.Versions[0]
}
return &Module{
Name: d.Name,
Version: fmt.Sprintf("%f", lv.Version),
Source: s,
Url: lv.URL,
}, nil
case "CPAN":
log.Debug("=> Using CPAN source")
if mod, ok := s.ModuleList[d.Name]; ok {
log.Trace("=> Found in source: %s", mod)
if d.Matches(mod) {
log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
return mod, nil
}
log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
return nil, nil
}
case "BackPAN":
log.Debug("=> Using BackPAN source")
// TODO better version matching - new backpan index?
if mod, ok := s.ModuleList[d.Name+"-"+d.Version]; ok {
log.Trace("=> Found in source: %s", mod)
if d.Matches(mod) {
log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
return mod, nil
}
log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
return nil, nil
}
case "MetaCPAN":
log.Debug("=> Using MetaCPAN source")
var sout, serr bytes.Buffer
var cpanm_args string = fmt.Sprintf("-L %s --info %s~\"%s%s\"", config.InstallDir, d.Name, d.Modifier, d.Version)
cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
if err != nil {
log.Error("Failed to get absolute path of gopan cache directory: %s", err)
return nil, err
}
log.Trace("About to exec: cpanm %s", cpanm_args)
os.Setenv("CPANM_INFO_ARGS", cpanm_args)
os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
cmd := exec.Command("bash", "-c", `eval cpanm $CPANM_INFO_ARGS`)
cmd.Stdout = &sout
cmd.Stderr = &serr
if err := cmd.Run(); err != nil {
log.Error("cpanm %s: %s,\n%s\n", cpanm_args, err, serr.String())
return nil, nil
}
if 0 == len(sout.String()) {
log.Warn("No author/module from cpanm")
return nil, nil
}
author_module := strings.TrimRight(sout.String(), "\n")
mematches := metacpanRe.FindStringSubmatch(author_module)
if nil == mematches {
log.Error("Match failed for: %s", author_module)
return nil, nil
}
log.Trace("Resolved: %s", author_module)
for _, mesource := range config.MetaSources {
meurl := fmt.Sprintf("authors/id/%s/%s/%s",
mematches[1][0:1],
mematches[1][0:2],
mematches[0])
archive_url := fmt.Sprintf("%s/%s", mesource.URL, meurl)
log.Trace("Checking: " + archive_url)
resp, err := http.Head(archive_url)
if err != nil {
log.Trace(err)
continue
}
log.Trace("HEAD status code: %d", resp.StatusCode)
if 200 == resp.StatusCode {
// No module/version check since 'cpanm --info' may resolve to
// archive and version that may not match source
return &Module{
Name: mematches[2],
Version: mematches[3],
Source: mesource,
Url: meurl,
}, nil
}
}
log.Error("Could not get archive URL via 'cpanm %s'", cpanm_args)
return nil, nil
default:
log.Error("Unrecognised source type: %s", s.Type)
return nil, errors.New(fmt.Sprintf("Unrecognised source: %s", s))
}
log.Trace("=> Not found in source")
return nil, nil
}
func main() {
log.Logger().SetAppender(NewAppender())
global := "websysd.json"
flag.StringVar(&global, "global", global, "global environment configuration")
workspaces := make([]string, 0)
flag.Var((*AppendSliceValue)(&workspaces), "workspace", "websysd workspace file (can be specified multiple times), defaults to './workspace.json'")
// Create our Gotcha application
var app = gotcha.Create(Asset)
if len(workspaces) == 0 {
workspaces = append(workspaces, "./workspace.json")
}
LoadConfig(global, workspaces)
GlobalWorkspace = NewWorkspace(GlobalConfigWorkspace.Name, GlobalConfigWorkspace.Environment, make(map[string]map[string][]string), GlobalConfigWorkspace.InheritEnvironment)
for fn, args := range GlobalConfigWorkspace.Functions {
log.Info("=> Creating global function: %s", fn)
GlobalWorkspace.Functions[fn] = &Function{
Name: fn,
Args: args.Args,
Command: args.Command,
Executor: args.Executor,
}
}
if GlobalWorkspace.InheritEnvironment {
log.Info("=> Inheriting process environment into global workspace")
for _, k := range os.Environ() {
p := strings.SplitN(k, "=", 2)
if strings.TrimSpace(p[0]) == "" {
log.Warn("Skipping empty environment key")
continue
}
log.Info(" %s = %s", p[0], p[1])
// TODO variable subst for current env vars
if _, ok := GlobalWorkspace.Environment[p[0]]; !ok {
GlobalWorkspace.Environment[p[0]] = p[1]
}
}
}
for _, ws := range ConfigWorkspaces {
log.Info("=> Creating workspace: %s", ws.Name)
var workspace *Workspace
if wks, ok := Workspaces[ws.Name]; ok {
log.Warn("Workspace %s already exists, merging tasks and environment")
workspace = wks
} else {
workspace = NewWorkspace(ws.Name, ws.Environment, ws.Columns, ws.InheritEnvironment)
Workspaces[ws.Name] = workspace
}
workspace.IsLocked = ws.IsLocked
if workspace.InheritEnvironment && !GlobalWorkspace.InheritEnvironment {
log.Info("=> Inheriting process environment into workspace")
for _, k := range os.Environ() {
p := strings.SplitN(k, "=", 2)
if strings.TrimSpace(p[0]) == "" {
log.Warn("Skipping empty environment key")
continue
}
log.Info(" %s = %s", p[0], p[1])
// TODO variable subst for current env vars
if _, ok := workspace.Environment[p[0]]; !ok {
workspace.Environment[p[0]] = p[1]
}
}
}
for fn, args := range ws.Functions {
log.Info("=> Creating workspace function: %s", fn)
workspace.Functions[fn] = &Function{
Name: fn,
Args: args.Args,
Command: args.Command,
Executor: args.Executor,
}
}
for _, t := range ws.Tasks {
log.Info("=> Creating task: %s", t.Name)
if _, ok := workspace.Tasks[t.Name]; ok {
log.Warn("Task %s already exists, overwriting")
}
env := make(map[string]string)
for k, v := range GlobalWorkspace.Environment {
env[k] = v
}
for k, v := range ws.Environment {
env[k] = v
}
for k, v := range t.Environment {
env[k] = v
}
task := NewTask(workspace, t.Name, t.Executor, t.Command, env, t.Service, t.Stdout, t.Stderr, t.Metadata, t.Pwd)
workspace.Tasks[t.Name] = task
}
}
// Get the router
r := app.Router
// Create some routes
r.Get("/", list_workspaces)
r.Get("/favicon.ico", r.Static("assets/favicon.ico"))
r.Get("/log", show_log)
r.Get("/workspace/(?P<workspace>[^/]+)", list_tasks)
// Serve static content (but really use a CDN)
r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))
r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/start", startTask)
r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/stop", stopTask)
r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/enable", enableServiceTask)
r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/disable", disableServiceTask)
r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)", taskHistory)
r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)", taskRun)
r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stdout", taskRunStdout)
r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stderr", taskRunStderr)
// Start our application
app.Start()
defer func() {
for _, ws := range Workspaces {
for _, t := range ws.Tasks {
if t.ActiveTask != nil && t.ActiveTask.Cmd != nil && t.ActiveTask.Cmd.Process != nil {
t.ActiveTask.Cmd.Process.Kill()
}
}
}
}()
<-make(chan int)
}
func main() {
configure()
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
indexes = make(map[string]map[string]*gopan.Source)
if !config.NoCache {
indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
}
if config.NoCache || config.Update {
for _, s := range config.Sources {
b := strings.SplitN(s, "=", 2)
if len(b) < 2 {
log.Error("Expected Name=URL pair, got: %s", s)
return
}
if idx, ok := indexes[config.Index][b[0]]; ok {
log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1])
idx.URL = b[1]
} else {
indexes[config.Index][b[0]] = &gopan.Source{
Name: b[0],
URL: b[1],
Authors: make(map[string]*gopan.Author, 0),
}
}
}
if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN {
log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN")
config.CPAN = true
config.BackPAN = true
}
if config.CPAN {
if _, ok := indexes[config.Index]["CPAN"]; !ok {
log.Debug("Adding CPAN index")
indexes[config.Index]["CPAN"] = gopan.CPANSource()
} else {
log.Debug("CPAN index already exists")
}
}
if config.BackPAN {
if _, ok := indexes[config.Index]["BackPAN"]; !ok {
log.Debug("Adding BackPAN index")
indexes[config.Index]["BackPAN"] = gopan.BackPANSource()
} else {
log.Debug("BackPAN index already exists")
}
}
log.Info("Using sources:")
for fname, _ := range indexes {
log.Info("From %s", fname)
for _, source := range indexes[fname] {
log.Info("=> %s", source.String())
}
}
newAuthors := getAuthors()
newPackages := getPackages()
os.MkdirAll(config.CacheDir, 0777)
if !config.NoCache {
gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index])
}
if config.Update {
log.Info("Found %d new packages by %d new authors", newAuthors, newPackages)
}
}
nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes)
log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc)
if !config.NoMirror {
mirrorPan()
}
}
func (r *Response) EventStream() chan []byte {
c := make(chan []byte)
r.IsEventStream = true
r.Headers.Add("Content-Type", "text/event-stream")
r.Headers.Add("Cache-Control", "no-cache")
r.Headers.Add("Connection", "keep-alive")
//r.Write([]byte("\n\n"))
r.Send()
hj, ok := r.writer.(nethttp.Hijacker)
if !ok {
log.Warn("Connection unsuitable for hijack")
return nil
}
conn, bufrw, err := hj.Hijack()
if err != nil {
log.Warn("Connection hijack failed")
return nil
}
r.esBufrw = bufrw
r.esConn = conn
go func() {
for b := range c {
if len(b) == 0 {
log.Trace("Event stream ended")
r.esConn.Close()
break
}
lines := strings.Split(string(b), "\n")
data := ""
for _, l := range lines {
data += "data: " + l + "\n"
}
data += "\n"
sz := len(data) + 1
log.Info("Event stream message is %d bytes", sz)
size := fmt.Sprintf("%X", sz)
r.esBufrw.Write([]byte(size + "\r\n"))
lines = strings.Split(data, "\n")
for _, ln := range lines {
r.esBufrw.Write([]byte(ln + "\n"))
}
_, err := r.esBufrw.Write([]byte("\r\n"))
if err != nil {
log.Error("Error writing to connection: %s\n", err)
r.esConn.Close()
break
}
err = r.esBufrw.Flush()
if err != nil {
log.Error("Error flushing buffer: %s\n", err)
r.esConn.Close()
break
}
}
}()
return c
}