Example #1
0
// HTTP logging handler
func LoggingHandler(inner http.Handler, log *logging.Logger) http.Handler{
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request){
		sw := MakeLogger(w)
		inner.ServeHTTP(sw, r)
		log.Info(buildCommonLogLine(r, *r.URL, time.Now(), sw.Status(), sw.Size()))
	})
}
Example #2
0
File: get.go Project: pulcy/pulsar
// Get executes a `go get` with a cache support.
func Get(log *log.Logger, flags *GetFlags) error {
	// Check GOPATH
	if gopath == "" {
		return maskAny(errors.New("Specify GOPATH"))
	}
	gopathDir := strings.Split(gopath, string(os.PathListSeparator))[0]

	// Get cache dir
	cachedir, cacheIsValid, err := cache.Dir(flags.Package, cacheValid)
	if err != nil {
		return maskAny(err)
	}

	if !cacheIsValid {
		// Cache has become invalid
		log.Info(updating("Refreshing cache of %s"), flags.Package)
		// Execute `go get` towards the cache directory
		if err := runGoGet(log, flags.Package, cachedir); err != nil {
			return maskAny(err)
		}
	}

	// Sync with local gopath
	if err := os.MkdirAll(gopathDir, 0777); err != nil {
		return maskAny(err)
	}
	if err := util.ExecPrintError(nil, "rsync", "-a", filepath.Join(cachedir, srcDir), gopathDir); err != nil {
		return maskAny(err)
	}

	return nil
}
func NewRateLimiter(cfg config.RateLimiting, red *redis.Pool, logger *logging.Logger) (RateLimitingMiddleware, error) {
	t := new(RedisSimpleRateThrottler)
	t.burstSize = int64(cfg.Burst)
	t.requestsPerSecond = int64(cfg.RequestsPerSecond)
	t.redisPool = red
	t.logger = logger

	if w, err := time.ParseDuration(cfg.Window); err != nil {
		return nil, err
	} else {
		t.window = w
	}

	logger.Info("Initialize rate limiter (burst size %d)", t.burstSize)

	return t, nil
}
Example #4
0
// Get executes a `go get` with a cache support.
func Vendor(log *log.Logger, flags *VendorFlags) error {
	// Get cache dir
	cachedir, _, err := cache.Dir(flags.Package, time.Millisecond)
	if err != nil {
		return maskAny(err)
	}

	// Cache has become invalid
	log.Info(updating("Fetching %s"), flags.Package)
	// Execute `go get` towards the cache directory
	if err := runGoGet(log, flags.Package, cachedir); err != nil {
		return maskAny(err)
	}

	// Sync with vendor dir
	if err := os.MkdirAll(flags.VendorDir, 0777); err != nil {
		return maskAny(err)
	}
	if err := util.ExecPrintError(nil, "rsync", "--exclude", ".git", "-a", filepath.Join(cachedir, srcDir)+"/", flags.VendorDir); err != nil {
		return maskAny(err)
	}

	return nil
}
Example #5
0
func Release(log *log.Logger, flags *Flags) error {
	// Detect environment
	hasMakefile := false
	isDev := flags.ReleaseType == "dev"
	if _, err := os.Stat(makefileFile); err == nil {
		hasMakefile = true
		log.Info("Found %s", makefileFile)
	}

	hasGruntfile := false
	if _, err := os.Stat(gruntfileFile); err == nil {
		hasGruntfile = true
		log.Info("Found %s", gruntfileFile)
	}

	hasDockerfile := false
	if _, err := os.Stat(dockerfileFile); err == nil {
		hasDockerfile = true
		log.Info("Found %s", dockerfileFile)
	}

	// Read the current version and name
	info, err := GetProjectInfo()
	if err != nil {
		return maskAny(err)
	}

	log.Info("Found old version %s", info.Version)
	version, err := semver.NewVersion(info.Version)
	if err != nil {
		return maskAny(err)
	}

	// Check repository state
	if !isDev {
		if err := checkRepoClean(log, info.GitBranch); err != nil {
			return maskAny(err)
		}
	}

	// Bump version
	switch flags.ReleaseType {
	case "major":
		version.Major++
		version.Minor = 0
		version.Patch = 0
	case "minor":
		version.Minor++
		version.Patch = 0
	case "patch":
		version.Patch++
	case "dev":
		// Do not change version
	default:
		return errgo.Newf("Unknown release type %s", flags.ReleaseType)
	}
	version.Metadata = ""

	// Write new release version
	if !isDev {
		if err := writeVersion(log, version.String(), info.Manifests, info.GradleConfigFile, false); err != nil {
			return maskAny(err)
		}
	}

	// Build project
	if hasGruntfile && !info.NoGrunt {
		if _, err := os.Stat(nodeModulesFolder); os.IsNotExist(err) {
			log.Info("Folder %s not found", nodeModulesFolder)
			if err := util.ExecPrintError(log, "npm", "install"); err != nil {
				return maskAny(err)
			}
		}
		if err := util.ExecPrintError(log, "grunt", "build-release"); err != nil {
			return maskAny(err)
		}
	}
	if hasMakefile {
		// Clean first
		if !isDev {
			if err := util.ExecPrintError(log, "make", info.Targets.CleanTarget); err != nil {
				return maskAny(err)
			}
		}
		// Now build
		makeArgs := []string{}
		if info.Targets.ReleaseTarget != "" {
			makeArgs = append(makeArgs, info.Targets.ReleaseTarget)
		}
		if err := util.ExecPrintError(log, "make", makeArgs...); err != nil {
			return maskAny(err)
		}
	}

	if hasDockerfile {
		// Build docker images
		tagVersion := version.String()
		if isDev {
			tagVersion = strings.Replace(time.Now().Format("2006-01-02-15-04-05"), "-", "", -1)
		}
		imageAndVersion := fmt.Sprintf("%s:%s", info.Image, tagVersion)
		imageAndMajorVersion := fmt.Sprintf("%s:%d", info.Image, version.Major)
		imageAndMinorVersion := fmt.Sprintf("%s:%d.%d", info.Image, version.Major, version.Minor)
		imageAndLatest := fmt.Sprintf("%s:latest", info.Image)
		buildTag := path.Join(info.Namespace, imageAndVersion)
		buildLatestTag := path.Join(info.Namespace, imageAndLatest)
		buildMajorVersionTag := path.Join(info.Namespace, imageAndMajorVersion)
		buildMinorVersionTag := path.Join(info.Namespace, imageAndMinorVersion)
		if err := util.ExecPrintError(log, "docker", "build", "--tag", buildTag, "."); err != nil {
			return maskAny(err)
		}
		if info.TagLatest {
			util.ExecSilent(log, "docker", "rmi", buildLatestTag)
			if err := util.ExecPrintError(log, "docker", "tag", buildTag, buildLatestTag); err != nil {
				return maskAny(err)
			}
		}
		if info.TagMajorVersion && !isDev {
			util.ExecSilent(log, "docker", "rmi", buildMajorVersionTag)
			if err := util.ExecPrintError(log, "docker", "tag", buildTag, buildMajorVersionTag); err != nil {
				return maskAny(err)
			}
		}
		if info.TagMinorVersion && !isDev {
			util.ExecSilent(log, "docker", "rmi", buildMinorVersionTag)
			if err := util.ExecPrintError(log, "docker", "tag", buildTag, buildMinorVersionTag); err != nil {
				return maskAny(err)
			}
		}
		registry := flags.DockerRegistry
		if info.Registry != "" {
			registry = info.Registry
		}
		namespace := info.Namespace
		if registry != "" || namespace != "" {
			// Push image to registry
			if err := docker.Push(log, imageAndVersion, registry, namespace); err != nil {
				return maskAny(err)
			}
			if info.TagLatest {
				// Push latest image to registry
				if err := docker.Push(log, imageAndLatest, registry, namespace); err != nil {
					return maskAny(err)
				}
			}
			if info.TagMajorVersion && !isDev {
				// Push major version image to registry
				if err := docker.Push(log, imageAndMajorVersion, registry, namespace); err != nil {
					return maskAny(err)
				}
			}
			if info.TagMinorVersion && !isDev {
				// Push minor version image to registry
				if err := docker.Push(log, imageAndMinorVersion, registry, namespace); err != nil {
					return maskAny(err)
				}
			}
		}
	}

	// Build succeeded, re-write new release version and commit
	if !isDev {
		if err := writeVersion(log, version.String(), info.Manifests, info.GradleConfigFile, true); err != nil {
			return maskAny(err)
		}

		// Tag version
		if err := git.Tag(log, version.String()); err != nil {
			return maskAny(err)
		}

		// Create github release (if needed)
		if err := createGithubRelease(log, version.String(), *info); err != nil {
			return maskAny(err)
		}

		// Update version to "+git" working version
		version.Metadata = "git"

		// Write new release version
		if err := writeVersion(log, version.String(), info.Manifests, info.GradleConfigFile, true); err != nil {
			return maskAny(err)
		}

		// Push changes
		if err := git.Push(log, "origin", false); err != nil {
			return maskAny(err)
		}

		// Push tags
		if err := git.Push(log, "origin", true); err != nil {
			return maskAny(err)
		}
	}

	return nil
}
Example #6
0
File: get.go Project: pulcy/pulsar
// Get ensures that flags.Folder contains an up to date copy of flags.RepoUrl checked out to flags.Version.
func Get(log *log.Logger, flags *Flags) error {
	// Get cache dir
	cachedir, cacheIsValid, err := cache.Dir(flags.RepoUrl, 0)
	if err != nil {
		return maskAny(err)
	}

	// Expand folder
	flags.Folder, err = filepath.Abs(flags.Folder)
	if err != nil {
		return maskAny(err)
	}

	// Get current folder
	wd, _ := os.Getwd()

	linked := false
	if flags.AllowLink {
		if info, err := util.ParseVCSURL(flags.RepoUrl); err == nil {
			siblingPath := filepath.Join(filepath.Dir(wd), info.Name)
			if _, err := os.Stat(siblingPath); err == nil {
				//log.Infof("Sibling folder %s exists", siblingPath)
				util.ExecuteInDir(siblingPath, func() error {
					remote, err := git.GetRemoteOriginUrl(nil)
					if err != nil {
						return maskAny(err)
					}
					if remote == flags.RepoUrl {
						if relPath, err := filepath.Rel(filepath.Dir(flags.Folder), siblingPath); err == nil {
							if err := os.Symlink(relPath, flags.Folder); err == nil {
								log.Infof("Linked -> %s", siblingPath)
								linked = true

								if vendorDir, err := golang.GetVendorDir(siblingPath); err != nil {
									return maskAny(err)
								} else {
									// Flatten sibling in copy-only mode
									if err := golang.Flatten(log, &golang.FlattenFlags{
										VendorDir: vendorDir,
										NoRemove:  true,
									}); err != nil {
										return maskAny(err)
									}
								}

							}
						}
					}
					return nil
				})
			}
		}
	}
	if linked {
		return nil
	}

	// Fill cache if needed
	cloned := false
	if !cacheIsValid {
		// Clone repo into cachedir
		if err := git.Clone(log, flags.RepoUrl, cachedir); err != nil {
			return maskAny(err)
		}
		cloned = true
	}

	// Make sure a clone exists
	_, err = os.Stat(flags.Folder)
	if os.IsNotExist(err) {
		// Sync into target folder
		if err := os.MkdirAll(flags.Folder, 0777); err != nil {
			return maskAny(err)
		}
		if err := util.ExecPrintError(nil, "rsync", "-a", appendDirSep(cachedir), appendDirSep(flags.Folder)); err != nil {
			return maskAny(err)
		}
	}
	// Change dir to folder
	if err := os.Chdir(flags.Folder); err != nil {
		return maskAny(err)
	}
	// Specific version needed?
	if flags.Version == "" {
		// Get latest version
		if !cloned {
			localCommit, err := git.GetLatestLocalCommit(nil, flags.Folder, defaultGetBranch, false)
			if err != nil {
				return maskAny(err)
			}
			remoteCommit, err := git.GetLatestRemoteCommit(nil, flags.RepoUrl, defaultGetBranch)
			if err != nil {
				return maskAny(err)
			}
			if localCommit != remoteCommit {
				if err := git.Pull(log, "origin"); err != nil {
					return maskAny(err)
				}
			} else {
				log.Info(allGood("%s is up to date\n"), makeRel(wd, flags.Folder))
			}
		}
	} else {
		// Get latest (local) version
		localVersion, err := git.GetLatestTag(nil, flags.Folder)
		if err != nil {
			return maskAny(err)
		}
		if localVersion != flags.Version {
			// Checkout requested version
			if cloned {
				log.Info(updating("Checking out version %s in %s.\n"), flags.Version, makeRel(wd, flags.Folder))
			} else {
				log.Info(updating("Found version %s, wanted %s. Updating %s now.\n"), localVersion, flags.Version, makeRel(wd, flags.Folder))
			}
			// Fetch latest changes
			if err := git.Fetch(log, "origin"); err != nil {
				return maskAny(err)
			}
			if err := git.FetchTags(log, "origin"); err != nil {
				return maskAny(err)
			}
			// Checkout intended version
			if err := git.Checkout(log, flags.Version); err != nil {
				return maskAny(err)
			}
		} else {
			log.Info(allGood("Found correct version. No changes needed in %s\n"), makeRel(wd, flags.Folder))
		}
		// Get latest remote version
		remoteVersion, err := git.GetLatestRemoteTag(nil, flags.RepoUrl)
		if err != nil {
			return maskAny(err)
		}
		if remoteVersion != flags.Version {
			log.Warning(attention("Update available for %s: '%s' => '%s'\n"), makeRel(wd, flags.Folder), flags.Version, remoteVersion)
		}
	}
	return nil
}
Example #7
0
func buildDispatcher(
	startup *StartupConfig,
	cfg *config.Configuration,
	consul *api.Client,
	handler *proxy.ProxyHandler,
	rpool *redis.Pool,
	logger *logging.Logger,
	lastIndex uint64,
) (dispatcher.Dispatcher, uint64, error) {
	var disp dispatcher.Dispatcher
	var err error
	var meta *api.QueryMeta
	var configs api.KVPairs
	var localCfg config.Configuration = *cfg
	var appCfgs map[string]config.Application = make(map[string]config.Application)

	dispLogger := logging.MustGetLogger("dispatch")

	switch startup.DispatchingMode {
	case "path":
		disp, err = dispatcher.NewPathBasedDispatcher(&localCfg, dispLogger, handler)
	case "host":
		disp, err = dispatcher.NewHostBasedDispatcher(&localCfg, dispLogger, handler)
	default:
		err = fmt.Errorf("unsupported dispatching mode: '%s'", startup.DispatchingMode)
	}

	if err != nil {
		return nil, 0, fmt.Errorf("error while creating proxy builder: %s", err)
	}

	applicationConfigBase := startup.ConsulBaseKey + "/applications"
	queryOpts := api.QueryOptions{
		WaitIndex: lastIndex,
		WaitTime:  30 * time.Minute,
	}

	logger.Info("loading gateway config from KV %s", startup.ConsulBaseKey)
	configs, meta, err = consul.KV().List(startup.ConsulBaseKey, &queryOpts)
	if err != nil {
		return nil, 0, err
	}

	for _, cfgKVPair := range configs {
		logger.Debug("found KV pair with key '%s'", cfgKVPair.Key)

		switch strings.TrimPrefix(startup.ConsulBaseKey+"/", cfgKVPair.Key) {
		case "authentication":
			if err := json.Unmarshal(cfgKVPair.Value, &localCfg.Authentication); err != nil {
				return nil, meta.LastIndex, fmt.Errorf("JSON error on consul KV pair '%s': %s", cfgKVPair.Key, err)
			}
		case "rate_limiting":
			if err := json.Unmarshal(cfgKVPair.Value, &localCfg.RateLimiting); err != nil {
				return nil, meta.LastIndex, fmt.Errorf("JSON error on consul KV pair '%s': %s", cfgKVPair.Key, err)
			}
		}

		if strings.HasPrefix(cfgKVPair.Key, applicationConfigBase) {
			var appCfg config.Application

			if err := json.Unmarshal(cfgKVPair.Value, &appCfg); err != nil {
				return nil, meta.LastIndex, fmt.Errorf("JSON error on consul KV pair '%s': %s", cfgKVPair.Key, err)
			}

			name := strings.TrimPrefix(cfgKVPair.Key, applicationConfigBase+"/")
			appCfgs[name] = appCfg
		}
	}

	authHandler, err := auth.NewAuthDecorator(&localCfg.Authentication, rpool, logging.MustGetLogger("auth"), startup.UiDir)
	if err != nil {
		return nil, meta.LastIndex, err
	}

	rlim, err := ratelimit.NewRateLimiter(localCfg.RateLimiting, rpool, logging.MustGetLogger("ratelimiter"))
	if err != nil {
		logger.Fatal(fmt.Sprintf("error while configuring rate limiting: %s", err))
	}

	cch := cache.NewCache(4096)

	// Order is important here! Behaviours will be called in LIFO order;
	// behaviours that are added last will be called first!
	disp.AddBehaviour(dispatcher.NewCachingBehaviour(cch))
	disp.AddBehaviour(dispatcher.NewAuthenticationBehaviour(authHandler))
	disp.AddBehaviour(dispatcher.NewRatelimitBehaviour(rlim))

	for name, appCfg := range appCfgs {
		logger.Info("registering application '%s' from Consul", name)
		if err := disp.RegisterApplication(name, appCfg); err != nil {
			return nil, meta.LastIndex, err
		}
	}

	for name, appCfg := range localCfg.Applications {
		logger.Info("registering application '%s' from local config", name)
		if err := disp.RegisterApplication(name, appCfg); err != nil {
			return nil, meta.LastIndex, err
		}
	}

	if err = disp.Initialize(); err != nil {
		return nil, meta.LastIndex, err
	}

	return disp, meta.LastIndex, nil
}