func (s *Source) extract() (err error) {
var cmd *exec.Cmd
if strings.HasSuffix(s.Path, ".tar") {
cmd = exec.Command("tar", "--no-same-owner", "-xf", s.Path)
} else if strings.HasSuffix(s.Path, ".zip") {
cmd = exec.Command("unzip", s.Path)
} else {
split := strings.Split(s.Path, ".")
if len(split) > 2 && split[len(split)-2] == "tar" {
cmd = exec.Command("tar", "--no-same-owner", "-xf", s.Path)
} else {
return
}
}
cmd.Dir = s.Output
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
err = &GetError{
errors.Wrapf(err, "builder: Failed to extract source '%s'",
s.Source),
}
return
}
return
}
func (p *Project) Init() (err error) {
p.MirrorRoot = filepath.Join(p.Root, "mirror")
p.BuildRoot = filepath.Join(p.Root, "mirror.tmp")
p.confPath = filepath.Join(p.Root, "pacur.json")
exists, err := utils.Exists(p.confPath)
if err != nil {
return
}
if exists {
dataByt, e := utils.ReadFile(p.confPath)
if e != nil {
err = e
return
}
data := conf{}
err = json.Unmarshal(dataByt, &data)
if err != nil {
err = &ParseError{
errors.Wrapf(err,
"project: Failed to parse project conf '%s'", p.confPath),
}
return
}
p.Name = data.Name
} else {
p.Name = "pacur"
}
return
}
func Page(q PageQuery) (*PageResponse, error) {
if q.ID == 0 {
return nil, errors.New("a page id is required")
}
resp, err := http.Get(q.urlString())
if err != nil {
return nil, errors.Wrap(err, "could not get http response")
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, errors.Wrapf(errors.New(resp.Status), "StatusCode: %d; URL: %s", resp.StatusCode, q.urlString())
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "could not read http response body")
}
var response PageResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, errors.Wrap(err, "could not unmarshal http response")
}
return &response, nil
}
func Project() (err error) {
path, err := os.Getwd()
if err != nil {
err = &FileError{
errors.Wrapf(err, "cmd: Failed to get working directory"),
}
return
}
proj := &project.Project{
Root: path,
}
err = proj.Init()
if err != nil {
return
}
cmd := flag.Arg(1)
switch cmd {
case "init":
err = proj.InitProject()
case "build":
err = proj.Build(flag.Arg(2))
case "repo":
err = proj.Repo(flag.Arg(2))
default:
err = &UnknownCommand{
errors.Newf("cmd: Unknown cmd '%s'", cmd),
}
}
return
}
func ExecInput(dir, input, name string, arg ...string) (err error) {
cmd := exec.Command(name, arg...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
stdin, err := cmd.StdinPipe()
if err != nil {
err = &ExecError{
errors.Wrapf(err, "utils: Failed to get stdin in exec '%s'", name),
}
return
}
defer stdin.Close()
if dir != "" {
cmd.Dir = dir
}
err = cmd.Start()
if err != nil {
err = &ExecError{
errors.Wrapf(err, "utils: Failed to exec '%s'", name),
}
return
}
_, err = io.WriteString(stdin, input)
if err != nil {
err = &ExecError{
errors.Wrapf(err, "utils: Failed to write stdin in exec '%s'",
name),
}
return
}
err = cmd.Wait()
if err != nil {
err = &ExecError{
errors.Wrapf(err, "utils: Failed to exec '%s'", name),
}
return
}
return
}
func (c *ShardedClient) connectionError(shard int, err error) error {
if err == nil {
return errors.Newf(
"Connection unavailable for memcache shard %d", shard)
}
return errors.Wrapf(
err,
"Connection unavailable for memcache shard %d", shard)
}
func request(url string, tokens []string) ([]byte, error) {
if len(tokens) == 0 {
return nil, errors.New("a NOAA token is required to draw data from the API. Request one here: http://www.ncdc.noaa.gov/cdo-web/token.")
}
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, errors.Wrapf(err, "could not build request for url[%s]", url)
}
req.Header.Set("token", tokens[0])
response, err := client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, "could not do request for url[%s]", url)
}
defer response.Body.Close()
// "status" : "429", "message" : "This token has reached its temporary request limit of 1000 per day."
// Rotate through tokens to try and find another.
if response.StatusCode == 429 {
if len(tokens) == 1 {
return nil, errors.New("All NOAA API tokens have reached their request limit for today. Add more or come back tomorrow.")
}
tokens = tokens[1:]
return request(url, tokens)
}
if response.StatusCode != 200 {
return nil, errors.Newf("NOAA API response statusCode is %d", response.StatusCode)
}
return ioutil.ReadAll(response.Body)
}
func RemoveAll(path string) (err error) {
err = os.RemoveAll(path)
if err != nil {
err = &WriteError{
errors.Wrapf(err, "utils: Failed to remove '%s'", path),
}
return
}
return
}
func Chmod(path string, perm os.FileMode) (err error) {
err = os.Chmod(path, perm)
if err != nil {
err = &WriteError{
errors.Wrapf(err, "utils: Failed to chmod '%s'", path),
}
return
}
return
}
func Open(path string) (file *os.File, err error) {
file, err = os.Open(path)
if err != nil {
err = &ReadError{
errors.Wrapf(err, "utils: Failed to open file '%s'", path),
}
return
}
return
}
func ReadDir(path string) (items []os.FileInfo, err error) {
items, err = ioutil.ReadDir(path)
if err != nil {
err = &ReadError{
errors.Wrapf(err, "utils: Failed to read dir '%s'", path),
}
return
}
return
}
func ReadFile(path string) (data []byte, err error) {
data, err = ioutil.ReadFile(path)
if err != nil {
err = &ReadError{
errors.Wrapf(err, "utils: Failed to read file '%s'", path),
}
return
}
return
}
func MkdirAll(path string) (err error) {
err = os.MkdirAll(path, 0755)
if err != nil {
err = &constants.WriteError{
errors.Wrapf(err, "utils: Failed to create '%s'", path),
}
return
}
return
}
func Create(path string) (file *os.File, err error) {
file, err = os.Create(path)
if err != nil {
err = &constants.WriteError{
errors.Wrapf(err, "utils: Failed to create '%s'", path),
}
return
}
return
}
func Read(path string) (data string, err error) {
dataByt, err := ioutil.ReadFile(path)
if err != nil {
err = &constants.ReadError{
errors.Wrapf(err, "utils: Failed to read '%s'", path),
}
return
}
data = string(dataByt)
return
}
func (r *Redhat) getRpmPath() (path string, err error) {
archs, err := ioutil.ReadDir(r.rpmsDir)
if err != nil {
err = &BuildError{
errors.Wrapf(err, "redhat: Failed to find rpms arch from '%s'",
r.rpmsDir),
}
return
}
if len(archs) < 1 {
err = &BuildError{
errors.Newf("redhat: Failed to find rpm arch from '%s'",
r.rpmsDir),
}
return
}
archPath := filepath.Join(r.rpmsDir, archs[0].Name())
rpms, err := ioutil.ReadDir(archPath)
if err != nil {
err = &BuildError{
errors.Wrapf(err, "redhat: Failed to find rpms from '%s'",
r.rpmsDir),
}
return
}
if len(rpms) < 1 {
err = &BuildError{
errors.Newf("redhat: Failed to find rpm from '%s'"),
}
return
}
path = filepath.Join(archPath, rpms[0].Name())
return
}
func (s *SearchQuery) request(page int) error {
select {
case <-s.tmb.Dying():
return nil
default:
}
resp, err := http.Get(s.url(page))
if err != nil {
return errors.Wrap(err, "could not get http response")
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.Wrapf(errors.New(resp.Status), "StatusCode: %d; URL: %s", resp.StatusCode, s.url(page))
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "could not read http response body")
}
var response results
if err := json.Unmarshal(body, &response); err != nil {
return errors.Wrap(err, "could not unmarshal http response")
}
for i := range response.Results {
s.ch <- response.Results[i]
}
if page > 1 {
return nil
}
// If this the first page, schedule the remaining requests
totalRequests := math.Ceil(response.TotalResults / response.ItemsPerPage)
var wg sync.WaitGroup
for i := 2; i <= int(totalRequests); i++ {
func(count int) {
wg.Add(1)
go s.tmb.Go(func() error {
defer wg.Done()
return s.request(count)
})
}(i)
}
wg.Wait()
return nil
}
func Exists(path string) (exists bool, err error) {
_, err = os.Stat(path)
if os.IsNotExist(err) {
err = nil
} else if err != nil {
err = errortypes.ReadError{
errors.Wrapf(err, "utils: Failed to stat file '%s'", path),
}
return
} else {
exists = true
}
return
}
func HttpGet(url, output string) (err error) {
cmd := exec.Command("wget", url, "-O", output)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
err = &HttpError{
errors.Wrapf(err, "utils: Failed to get '%s'", url),
}
return
}
return
}
func ExecSilent(dir, name string, args ...string) (err error) {
cmd := exec.Command(name, args...)
if dir != "" {
cmd.Dir = dir
}
_, err = cmd.CombinedOutput()
if err != nil {
err = &constants.ExecError{
errors.Wrapf(err, "utils: Failed to exec %s %s", name, args),
}
return
}
return
}
func Exists(path string) (exists bool, err error) {
_, err = os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
err = nil
} else {
err = &ExistsError{
errors.Wrapf(err, "utils: Exists check error for '%s'", path),
}
}
} else {
exists = true
}
return
}
func Write(path string, data string) (err error) {
file, err := Create(path)
if err != nil {
return
}
defer file.Close()
_, err = file.WriteString(data)
if err != nil {
err = &constants.WriteError{
errors.Wrapf(err, "utils: Failed to write to file '%s'", path),
}
return
}
return
}
func Rsync(source, dest string) (err error) {
cmd := exec.Command("rsync", "-a", "-A", "-X",
source+string(os.PathSeparator),
dest+string(os.PathSeparator))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
err = &CopyError{
errors.Wrapf(err, "utils: Failed to rsync '%s' to '%s'", source,
dest),
}
return
}
return
}
func GetDirSize(path string) (size int, err error) {
output, err := ExecOutput("", "du", "-c", "-s", path)
if err != nil {
return
}
split := strings.Fields(output)
size, err = strconv.Atoi(split[len(split)-2])
if err != nil {
err = &ReadError{
errors.Wrapf(err, "utils: Failed to get dir size '%s'", path),
}
return
}
return
}
func ExistsMakeDir(path string) (err error) {
_, err = os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
err = MkdirAll(path)
if err != nil {
return
}
} else {
err = &WriteError{
errors.Wrapf(err, "utils: Failed to stat '%s'", path),
}
}
return
}
return
}
func ExecOutput(dir, name string, arg ...string) (output string, err error) {
cmd := exec.Command(name, arg...)
cmd.Stderr = os.Stderr
if dir != "" {
cmd.Dir = dir
}
outputByt, err := cmd.Output()
if err != nil {
err = &ExecError{
errors.Wrapf(err, "utils: Failed to exec '%s'", name),
}
return
}
output = string(outputByt)
return
}
func Exec(dir, name string, args ...string) (err error) {
cmd := exec.Command(name, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if dir != "" {
cmd.Dir = dir
}
err = cmd.Run()
if err != nil {
err = &constants.ExecError{
errors.Wrapf(err, "utils: Failed to exec %s %s", name, args),
}
return
}
return
}
// This gets an active resource from the resource pool. Note that the
// resourceLocation argument is ignroed (The handles are associated to the
// resource location provided by the first Register call).
func (p *SimpleResourcePool) Get(unused string) (ManagedHandle, error) {
activeCount := atomic.AddInt32(p.numActive, 1)
if p.options.MaxActiveHandles > 0 &&
activeCount > p.options.MaxActiveHandles {
atomic.AddInt32(p.numActive, -1)
return nil, errors.Newf(
"Too many handles to %s",
p.location)
}
if h := p.getIdleHandle(); h != nil {
return h, nil
}
p.mutex.Lock()
defer p.mutex.Unlock()
if p.location == "" {
atomic.AddInt32(p.numActive, -1)
return nil, errors.Newf(
"Resource location is not set for SimpleResourcePool")
}
if p.isLameDuck {
atomic.AddInt32(p.numActive, -1)
return nil, errors.Newf(
"Lame duck resource pool cannot return handles to %s",
p.location)
}
handle, err := p.options.Open(p.location)
if err != nil {
atomic.AddInt32(p.numActive, -1)
return nil, errors.Wrapf(
err,
"Failed to open resource handle: %s",
p.location)
}
return NewManagedHandle(p.location, handle, p, p.options), nil
}
func (r *Redhat) copy() (err error) {
archs, err := ioutil.ReadDir(r.rpmsDir)
if err != nil {
err = &BuildError{
errors.Wrapf(err, "redhat: Failed to find rpms from '%s'",
r.rpmsDir),
}
return
}
for _, arch := range archs {
err = utils.CopyFiles(filepath.Join(r.rpmsDir, arch.Name()),
r.Pack.Home, false)
if err != nil {
return
}
}
return
}
func FindMatch(path, match string) (matches []string, err error) {
files, err := ioutil.ReadDir(path)
if err != nil {
err = &ReadError{
errors.Wrapf(err, "utils: Failed to read dir '%s'", path),
}
return
}
for _, file := range files {
if file.IsDir() {
continue
}
if strings.Contains(file.Name(), match) {
matches = append(matches, filepath.Join(path, file.Name()))
}
}
return
}