Example #1
0
func configure() {
	loglevel := "INFO"
	flag.StringVar(&loglevel, "loglevel", "INFO", "Log level")

	cachedir := ".gopancache"
	flag.StringVar(&cachedir, "cachedir", ".gopancache", "GoPAN cache directory")

	index := "index"
	flag.StringVar(&index, "index", "index", "Name of the primary GoPAN index file")

	bind := ":7050"
	flag.StringVar(&bind, "bind", ":7050", "Interface to bind to")

	cpan := "cpan_index.gz"
	flag.StringVar(&cpan, "cpan", "cpan_index.gz", "Name of CPAN index to support readthrough")

	backpan := "backpan_index.gz"
	flag.StringVar(&backpan, "backpan", "backpan_index.gz", "Name of BackPAN index to support readthrough")

	remote := "http://localhost:7050"
	flag.StringVar(&remote, "remote", "http://localhost:7050", "Remote host and port to connect to")

	loglayout := "[%d] [%p] %m"
	flag.StringVar(&loglayout, "loglayout", loglayout, "Log layout (for github.com/ian-kent/go-log pattern layout)")

	indexes := make([]string, 0)
	flag.Var((*gopan.AppendSliceValue)(&indexes), "s-index", "Secondary indexes to load (can be provided multiple times)")

	testdeps := true
	flag.BoolVar(&testdeps, "test-deps", true, "Test dependencies on startup")

	flag.Parse()

	config = &Config{
		LogLevel:         loglevel,
		CacheDir:         cachedir,
		Index:            index,
		Bind:             bind,
		Indexes:          indexes,
		CanUpdate:        false,
		LatestRelease:    "0.0",
		CurrentRelease:   "0.0", // set in main.go so its in one place
		UpdateURL:        "",
		ImportAvailable:  false,
		CPANIndex:        cpan,
		HasCPANIndex:     false,
		CPANIndexDate:    "",
		BackPANIndex:     backpan,
		HasBackPANIndex:  false,
		BackPANIndexDate: "",
		BackPANStatus:    "Unavailable",
		CPANStatus:       "Unavailable",
		RemoteHost:       remote,
		TestDeps:         testdeps,
	}

	// log level and layout
	log.Logger().Appender().SetLayout(layout.Pattern(loglayout))
	log.Logger().SetLevel(log.Stol(loglevel))
}
Example #2
0
File: main.go Project: sogko/gotcha
func main() {
	// Create our Gotcha application
	var app = gotcha.Create(Asset)

	// Set the logger output pattern
	log.Logger().Appender().SetLayout(layout.Pattern("[%d] [%p] %m"))
	log.Logger().SetLevel(log.Stol("TRACE"))

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", example)
	r.Post("/", examplepost)
	r.Get("/foo", example2)
	r.Get("/bar", example3)
	r.Get("/stream", streamed)
	r.Get("/err", err)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// Listen to some events
	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		n := 0
		c, ok := session.Request.Cookies["test"]
		if ok {
			n, _ = strconv.Atoi(c.Value)
		}
		session.Stash["test"] = n
		log.Printf("Got BeforeHandler event! n = %d", n)
		next()
	})
	app.On(events.AfterHandler, func(session *http.Session, next func()) {
		n := session.Stash["test"].(int) + 1
		session.Response.Cookies.Set(&nethttp.Cookie{
			Name:  "test",
			Value: strconv.Itoa(n),
		})
		log.Println("Got AfterHandler event!")
		next()
	})
	app.On(events.AfterResponse, func(session *http.Session, next func()) {
		log.Println("Got AfterResponse event!")
		next()
	})

	// Start our application
	app.Start()

	<-make(chan int)
}
func New(logFileName, prefix string, isDevMode bool) Logger {
	l := golog.Logger()

	if isDevMode {
		l.SetLevel(levels.DEBUG)
	} else {
		l.SetLevel(levels.INFO)
	}

	layoutToUse := layout.Pattern(getPrefixWithSpace(prefix) + "[%p] %m") //level/priority, message

	rollingFileAppender := appenders.RollingFile(logFileName, true)
	rollingFileAppender.MaxBackupIndex = 5
	rollingFileAppender.MaxFileSize = 20 * 1024 * 1024 // 20 MB
	rollingFileAppender.SetLayout(layoutToUse)

	consoleAppender := appenders.Console()
	consoleAppender.SetLayout(layoutToUse)
	l.SetAppender(
		Multiple( //appenders.Multiple( ONCE PULL REQUEST OF ABOVE IS IN
			layoutToUse,
			rollingFileAppender,
			consoleAppender,
		))

	return &logger{l}
}
Example #4
0
File: main.go Project: djui/necd
func init() {
	logger := log.Logger()
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		logger.Appender().SetLayout(layout.Pattern("%p: %m"))
	} else {
		logger.Appender().SetLayout(layout.Pattern("%d %p: %m"))
	}
	logger.SetLevel(levels.DEBUG)
}
func New(svcLogger service.Logger, prefix string, isDevMode bool) Logger {
	l := golog.Logger()

	if isDevMode {
		l.SetLevel(levels.DEBUG)
	} else {
		l.SetLevel(levels.INFO)
	}

	layoutToUse := layout.Pattern(getPrefixWithSpace(prefix) + "[%p] %m") //level/priority, message

	serviceAppender1 := newAppender(layoutToUse, svcLogger)
	l.SetAppender(serviceAppender1)

	return &logger{l}
}
Example #6
0
func getLogger() logger.Logger {
	logger := log.Logger()

	layoutToUse := layout.Pattern("%d [%p] %m") //date, level/priority, message

	rollingFileAppender := appenders.RollingFile("rolling-log.log", true)
	rollingFileAppender.MaxBackupIndex = 5
	rollingFileAppender.MaxFileSize = 20 * 1024 * 1024 // 20 MB
	rollingFileAppender.SetLayout(layoutToUse)

	consoleAppender := appenders.Console()
	consoleAppender.SetLayout(layoutToUse)
	logger.SetAppender(
		Multiple( //appenders.Multiple( ONCE PULL REQUEST OF ABOVE IS IN
			layoutToUse,
			rollingFileAppender,
			consoleAppender,
		))

	return logger
}
Example #7
0
func main() {
	log.Logger().SetAppender(NewAppender())

	global := "websysd.json"
	flag.StringVar(&global, "global", global, "global environment configuration")

	workspaces := make([]string, 0)
	flag.Var((*AppendSliceValue)(&workspaces), "workspace", "websysd workspace file (can be specified multiple times), defaults to './workspace.json'")

	// Create our Gotcha application
	var app = gotcha.Create(Asset)

	if len(workspaces) == 0 {
		workspaces = append(workspaces, "./workspace.json")
	}

	LoadConfig(global, workspaces)

	GlobalWorkspace = NewWorkspace(GlobalConfigWorkspace.Name, GlobalConfigWorkspace.Environment, make(map[string]map[string][]string), GlobalConfigWorkspace.InheritEnvironment)
	for fn, args := range GlobalConfigWorkspace.Functions {
		log.Info("=> Creating global function: %s", fn)
		GlobalWorkspace.Functions[fn] = &Function{
			Name:     fn,
			Args:     args.Args,
			Command:  args.Command,
			Executor: args.Executor,
		}
	}

	if GlobalWorkspace.InheritEnvironment {
		log.Info("=> Inheriting process environment into global workspace")
		for _, k := range os.Environ() {
			p := strings.SplitN(k, "=", 2)
			if strings.TrimSpace(p[0]) == "" {
				log.Warn("Skipping empty environment key")
				continue
			}
			log.Info("  %s = %s", p[0], p[1])
			// TODO variable subst for current env vars
			if _, ok := GlobalWorkspace.Environment[p[0]]; !ok {
				GlobalWorkspace.Environment[p[0]] = p[1]
			}
		}
	}

	for _, ws := range ConfigWorkspaces {
		log.Info("=> Creating workspace: %s", ws.Name)

		var workspace *Workspace
		if wks, ok := Workspaces[ws.Name]; ok {
			log.Warn("Workspace %s already exists, merging tasks and environment")
			workspace = wks
		} else {
			workspace = NewWorkspace(ws.Name, ws.Environment, ws.Columns, ws.InheritEnvironment)
			Workspaces[ws.Name] = workspace
		}

		workspace.IsLocked = ws.IsLocked

		if workspace.InheritEnvironment && !GlobalWorkspace.InheritEnvironment {
			log.Info("=> Inheriting process environment into workspace")
			for _, k := range os.Environ() {
				p := strings.SplitN(k, "=", 2)
				if strings.TrimSpace(p[0]) == "" {
					log.Warn("Skipping empty environment key")
					continue
				}
				log.Info("  %s = %s", p[0], p[1])
				// TODO variable subst for current env vars
				if _, ok := workspace.Environment[p[0]]; !ok {
					workspace.Environment[p[0]] = p[1]
				}
			}
		}

		for fn, args := range ws.Functions {
			log.Info("=> Creating workspace function: %s", fn)
			workspace.Functions[fn] = &Function{
				Name:     fn,
				Args:     args.Args,
				Command:  args.Command,
				Executor: args.Executor,
			}
		}

		for _, t := range ws.Tasks {
			log.Info("=> Creating task: %s", t.Name)

			if _, ok := workspace.Tasks[t.Name]; ok {
				log.Warn("Task %s already exists, overwriting")
			}

			env := make(map[string]string)
			for k, v := range GlobalWorkspace.Environment {
				env[k] = v
			}
			for k, v := range ws.Environment {
				env[k] = v
			}
			for k, v := range t.Environment {
				env[k] = v
			}

			task := NewTask(workspace, t.Name, t.Executor, t.Command, env, t.Service, t.Stdout, t.Stderr, t.Metadata, t.Pwd)
			workspace.Tasks[t.Name] = task
		}
	}

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", list_workspaces)
	r.Get("/favicon.ico", r.Static("assets/favicon.ico"))
	r.Get("/log", show_log)
	r.Get("/workspace/(?P<workspace>[^/]+)", list_tasks)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/start", startTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/stop", stopTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/enable", enableServiceTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/disable", disableServiceTask)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)", taskHistory)

	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)", taskRun)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stdout", taskRunStdout)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stderr", taskRunStderr)

	// Start our application
	app.Start()

	defer func() {
		for _, ws := range Workspaces {
			for _, t := range ws.Tasks {
				if t.ActiveTask != nil && t.ActiveTask.Cmd != nil && t.ActiveTask.Cmd.Process != nil {
					t.ActiveTask.Cmd.Process.Kill()
				}
			}
		}
	}()

	<-make(chan int)
}
Example #8
0
func main() {
	configure()

	indexes = make(map[string]map[string]*gopan.Source)
	indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	// FIXME inefficient
	_, _, tpkg, _ := gopan.CountIndex(indexes)

	npkg := 0
	nmod := 0

	var pc = func() float64 {
		return float64(nmod) / float64(tpkg) * 100
	}

	log.Info("Writing packages index file")

	out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
	if err != nil {
		log.Error("Error creating packages index: %s", err.Error())
		return
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, idx := range indexes[fname] {
			log.Debug("Index: %s", idx)
			out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
			for _, auth := range idx.Authors {
				log.Debug("Author %s", auth)
				out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
				for _, pkg := range auth.Packages {
					out.Write([]byte("  " + pkg.Name + " => " + pkg.URL + "\n"))
					log.Debug("Package: %s", pkg)

					if !config.Flatten {
						if len(pkg.Provides) == 0 {
							// TODO better handling of filenames
							modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")

							tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name

							if _, err := os.Stat(tgzpath); err != nil {
								log.Error("File not found: %s", tgzpath)
								return
							}

							extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
							dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name

							log.Trace("=> tgzpath: %s", tgzpath)
							log.Trace(" > extpath: %s", extpath)
							log.Trace(" > dirpath: %s", dirpath)

							// Only index packages if they don't already exist
							if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
								log.Error("Error retrieving package list: %s", err)
								continue
							}
						}

						npkg += len(pkg.Provides)
						nmod += 1

						for p, pk := range pkg.Provides {
							out.Write([]byte("   " + p + " (" + pk.Version + "): " + pk.File + "\n"))
						}

						if nmod > 0 && nmod%100 == 0 {
							log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
						}

					}
				}
			}
		}
	}

	out.Close()

	log.Info("Found %d packages from %d modules", npkg, nmod)
}
Example #9
0
func main() {
	configure()

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	indexes = make(map[string]map[string]*gopan.Source)
	if !config.NoCache {
		indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
	}

	if config.NoCache || config.Update {
		for _, s := range config.Sources {
			b := strings.SplitN(s, "=", 2)
			if len(b) < 2 {
				log.Error("Expected Name=URL pair, got: %s", s)
				return
			}

			if idx, ok := indexes[config.Index][b[0]]; ok {
				log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1])
				idx.URL = b[1]
			} else {
				indexes[config.Index][b[0]] = &gopan.Source{
					Name:    b[0],
					URL:     b[1],
					Authors: make(map[string]*gopan.Author, 0),
				}
			}
		}

		if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN {
			log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN")
			config.CPAN = true
			config.BackPAN = true
		}

		if config.CPAN {
			if _, ok := indexes[config.Index]["CPAN"]; !ok {
				log.Debug("Adding CPAN index")
				indexes[config.Index]["CPAN"] = gopan.CPANSource()
			} else {
				log.Debug("CPAN index already exists")
			}
		}

		if config.BackPAN {
			if _, ok := indexes[config.Index]["BackPAN"]; !ok {
				log.Debug("Adding BackPAN index")
				indexes[config.Index]["BackPAN"] = gopan.BackPANSource()
			} else {
				log.Debug("BackPAN index already exists")
			}
		}

		log.Info("Using sources:")
		for fname, _ := range indexes {
			log.Info("From %s", fname)
			for _, source := range indexes[fname] {
				log.Info("=> %s", source.String())
			}
		}

		newAuthors := getAuthors()
		newPackages := getPackages()

		os.MkdirAll(config.CacheDir, 0777)

		if !config.NoCache {
			gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index])
		}

		if config.Update {
			log.Info("Found %d new packages by %d new authors", newAuthors, newPackages)
		}
	}

	nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes)
	log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc)

	if !config.NoMirror {
		mirrorPan()
	}
}
Example #10
0
func Configure() *Config {
	conf := DefaultConfig()

	cpan := make([]string, 0)
	flag.Var((*gopan.AppendSliceValue)(&cpan), "cpan", "A CPAN mirror (can be specified multiple times)")

	backpan := make([]string, 0)
	flag.Var((*gopan.AppendSliceValue)(&backpan), "backpan", "A BackPAN mirror (can be specified multiple times)")

	smart := make([]string, 0)
	flag.Var((*gopan.AppendSliceValue)(&smart), "smart", "A SmartPAN mirror (can be specified multiple times)")

	test := make([]string, 0)
	flag.Var((*gopan.AppendSliceValue)(&test), "test", "A module to install with testing (can be specified multiple times)")

	tests := false
	flag.BoolVar(&tests, "tests", false, "Enables all tests during installation phase")

	nocpan := false
	flag.BoolVar(&nocpan, "nocpan", false, "Disables the default CPAN source")

	nobackpan := false
	flag.BoolVar(&nobackpan, "nobackpan", false, "Disables the default BackCPAN source")

	cpanfile := "cpanfile"
	flag.StringVar(&cpanfile, "cpanfile", "cpanfile", "The cpanfile to install")

	numcpus := runtime.NumCPU()
	flag.IntVar(&numcpus, "cpus", numcpus, "The number of CPUs to use, defaults to "+strconv.Itoa(numcpus)+" for your environment")

	nodepdump := false
	flag.BoolVar(&nodepdump, "nodepdump", nodepdump, "Disables dumping resolved dependencies listing")

	noinstall := false
	flag.BoolVar(&noinstall, "noinstall", noinstall, "Disables installation phase")

	metacpan := false
	flag.BoolVar(&metacpan, "metacpan", metacpan, "Enable resolving source via MetaCPAN")

	loglevel := "INFO"
	flag.StringVar(&loglevel, "loglevel", loglevel, "Log output level (ERROR, INFO, WARN, DEBUG, TRACE)")

	loglayout := "[%d] [%p] %m"
	flag.StringVar(&loglayout, "loglayout", loglayout, "Log layout (for github.com/ian-kent/go-log pattern layout)")

	cachedir := "./.gopancache"
	flag.StringVar(&cachedir, "cachedir", cachedir, "Cache directory for CPAN modules")

	installdir := "./local"
	flag.StringVar(&installdir, "installdir", installdir, "Install directory for CPAN modules")

	flag.Parse()

	if nocpan || nobackpan {
		conf.Sources = DefaultSources(!nocpan, !nobackpan)
	}

	// parse cpan mirrors
	for _, url := range cpan {
		mirror := strings.TrimSuffix(url, "/")
		m := NewSource("CPAN", mirror+"/modules/02packages.details.txt.gz", mirror)
		conf.Sources = append(conf.Sources, m)
	}

	// parse backpan mirrors
	for _, url := range backpan {
		mirror := strings.TrimSuffix(url, "/")
		m := NewSource("BackPAN", mirror+"/backpan-index", mirror) // FIXME
		conf.Sources = append(conf.Sources, m)
	}

	// parse smartpan mirrors
	for _, url := range smart {
		mirror := strings.TrimSuffix(url, "/")
		m := NewSource("SmartPAN", mirror, mirror)
		conf.Sources = append(conf.Sources, m)
	}

	// resolve via metacpan
	conf.MetaCPAN = metacpan
	if metacpan {
		m := NewSource("MetaCPAN", "", "")
		conf.Sources = append(conf.Sources, m)
		c := NewMetaSource("cpan", "", "http://www.cpan.org", m.ModuleList)
		conf.MetaSources = append(conf.MetaSources, c)
		b := NewMetaSource("backpan", "", "http://backpan.perl.org", m.ModuleList)
		conf.MetaSources = append(conf.MetaSources, b)
	}

	sort.Sort(sortSources(conf.Sources))

	// parse notest
	for _, n := range test {
		conf.Test.Modules[n] = true
	}
	conf.Test.Global = tests

	// parse cpanfile
	conf.CPANFile = cpanfile

	// numcpus
	conf.CPUs = numcpus

	// nodepdeump
	conf.NoDepdump = nodepdump

	// noinstall
	conf.NoInstall = noinstall

	// install dir
	conf.InstallDir = installdir

	// cache dir
	conf.CacheDir = cachedir

	// log level and layout
	log.Logger().Appender().SetLayout(layout.Pattern(loglayout))
	log.Logger().SetLevel(log.Stol(loglevel))

	// create cache dir
	os.MkdirAll(conf.CacheDir, 0777)

	config = conf

	return conf
}
Example #11
0
func main() {
	configure()

	args := flag.Args()

	if len(args) > 0 && args[0] == "init" {
		log.Info("Initialising SmartPAN")

		log.Info("=> Installing Perl dependencies")

		// FIXME most of this is repeated from getpan/main.go
		cfg := getpan.DefaultConfig()
		cfg.CacheDir = config.CacheDir

		for _, source := range cfg.Sources {
			if err := source.Load(); err != nil {
				log.Error("Error loading sources: %s", err)
				os.Exit(1)
				return
			}
		}

		deps := &getpan.DependencyList{
			Dependencies: make([]*getpan.Dependency, 0),
		}

		d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "")
		d2, _ := getpan.DependencyFromString("JSON::XS", "")
		deps.AddDependency(d1)
		deps.AddDependency(d2)

		if err := deps.Resolve(); err != nil {
			log.Error("Error resolving dependencies: %s", err)
			os.Exit(1)
			return
		}

		_, err := deps.Install()
		if err != nil {
			log.Error("Error installing dependencies: %s", err)
			os.Exit(2)
			return
		}

		log.Info("   - Installed %d modules", deps.UniqueInstalled())

		log.Info("SmartPAN initialisation complete")

		return
	}

	if config.TestDeps {
		perldeps := gopan.TestPerlDeps()
		perldeps.Dump()
		if !perldeps.Ok {
			log.Error("Required perl dependencies are missing")
			os.Exit(1)
			return
		}
	}

	if len(args) > 0 && args[0] == "import" {
		if len(args) < 4 {
			log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX")
			return
		}

		fname := args[1]
		log.Info("Importing module from %s", fname)
		log.Info("Author ID: %s", args[2])
		log.Info("Index    : %s", args[3])

		extraParams := map[string]string{
			"importinto": args[3],
			"authorid":   args[2],
			"newindex":   "",
			"cpanmirror": "",
			"importurl":  "",
			"fromdir":    "",
		}

		if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
			log.Info("URL: %s", fname)

			extraParams["importurl"] = fname

			request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams)
			if err != nil {
				log.Error("Create request error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)

			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		} else {
			fname = strings.TrimPrefix(fname, "file://")
			log.Info("File: %s", fname)

			if _, err := os.Stat(fname); err != nil {
				log.Error("File not found: %s", err.Error())
				return
			}

			request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname)
			if err != nil {
				log.Error("Create upload error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)
			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		}

		return
	}

	config.CurrentRelease = CurrentRelease

	var wg sync.WaitGroup

	load_index = func(index string, file string) {
		indexes[index] = gopan.LoadIndex(file)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()

		indexes = make(map[string]map[string]*gopan.Source)

		// Load CPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil {
			config.HasCPANIndex = true
			config.CPANIndexDate = fi.ModTime().String()
			config.CPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
				config.CPANStatus = "Loaded"
			}()
		}

		// Load BackPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil {
			config.HasBackPANIndex = true
			config.BackPANIndexDate = fi.ModTime().String()
			config.BackPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
				config.BackPANStatus = "Loaded"
			}()
		}

		// Load our secondary indexes
		for _, idx := range config.Indexes {
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(idx, config.CacheDir+"/"+idx)
			}()
		}

		// Load our primary index (this is the only index written back to)
		wg.Add(1)
		go func() {
			defer wg.Done()
			load_index(config.Index, config.CacheDir+"/"+config.Index)
		}()
	}()

	update_indexes = func() {
		wg.Wait()
		wg.Add(1)
		go func() {
			wg.Wait()
			config.ImportAvailable = true

			nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
			// TODO should probably be in the index - needs to udpate when index changes
			summary = &Summary{nsrc, nauth, npkg, nprov}

			// Do this now so changing the level doesn't interfere with index load
			log.Logger().SetLevel(log.Stol(config.LogLevel))
		}()
		defer wg.Done()
		// Create in-memory indexes for UI/search etc
		for fname, _ := range indexes {
			for idn, idx := range indexes[fname] {
				mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
				for _, auth := range idx.Authors {
					// author name
					if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
						mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
						mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth

					// wildcards
					if _, ok := mapped[idx.Name]["*"]; !ok {
						mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
						mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name]["*"]["**"][auth.Name] = auth

					// combos
					if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
						mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
						mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
					mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth

					for _, pkg := range auth.Packages {
						filemap[pkg.AuthorURL()] = idn
						for _, prov := range pkg.Provides {
							parts := strings.Split(prov.Name, "::")
							log.Trace("PACKAGE: %s", prov.Name)

							if _, ok := packages[parts[0]]; !ok {
								packages[parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if _, ok := idxpackages[idx.Name]; !ok {
								idxpackages[idx.Name] = make(map[string]*PkgSpace)
							}
							if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
								idxpackages[idx.Name][parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if len(parts) == 1 {
								packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
								packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
								idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
							} else {
								packages[parts[0]].Populate(parts[1:], prov)
								idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
							}
						}
					}
				}
			}
		}
	}
	go update_indexes()

	// Get latest SmartPAN version
	go func() {
		res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases")
		if err != nil {
			log.Error("Error getting latest version: %s", err.Error())
			return
		}
		defer res.Body.Close()
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Error("Error reading stream: %s", err.Error())
			return
		}

		var r Releases
		if err = json.Unmarshal(b, &r); err != nil {
			log.Error("Error unmarshalling JSON: %s", err.Error())
			return
		}

		log.Info("Current release: %s", config.CurrentRelease)
		rel := strings.TrimPrefix(r[0].TagName, "v")
		log.Info("Latest release: %s", rel)
		config.LatestRelease = rel
		config.UpdateURL = r[0].URL

		if config.CurrentRelease < rel {
			config.CanUpdate = true
			log.Info("Your version of SmartPAN can be updated.")
		}
	}()

	// Create our Gotcha application
	var app = gotcha.Create(Asset)
	app.Config.Listen = config.Bind

	summary = &Summary{0, 0, 0, 0}

	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		session.Stash["summary"] = summary
		session.Stash["config"] = config

		next()
	})

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", search)
	r.Post("/", search)

	r.Get("/help", help)
	r.Get("/settings", settings)
	r.Get("/browse", browse)

	r.Get("/import", import1)
	r.Post("/import", import1)

	r.Get("/import/(?P<jobid>[^/]+)", import2)
	r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)

	r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// JSON endpoints
	r.Get("/where/(?P<module>[^/]+)/?", where)
	r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)

	// Put these last so they only match /{repo} if nothing else matches
	r.Get("/(?P<repo>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
	r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
	r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)

	// Start our application
	app.Start()

	<-make(chan int)
}