func PostAsyncFile(url, path, fileKey, pathToFile string) (*http.Response, error) {
completeURL := parseURL(url, path)
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
// this step is very important
fileWriter, err := bodyWriter.CreateFormFile(fileKey, pathToFile)
if err != nil {
return nil, lxerrors.New("error writing to buffer", err)
}
// open file handle
fh, err := os.Open(pathToFile)
if err != nil {
return nil, lxerrors.New("error opening file", err)
}
//iocopy
_, err = io.Copy(fileWriter, fh)
if err != nil {
return nil, lxerrors.New("error copying file to form", err)
}
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
resp, err := http.Post(completeURL, contentType, bodyBuf)
if err != nil {
return resp, lxerrors.New("error performing post", err)
}
return resp, nil
}
func Respond(res http.ResponseWriter, message interface{}) error {
switch message.(type) {
case string:
messageString := message.(string)
data := []byte(messageString)
_, err := res.Write(data)
if err != nil {
return lxerrors.New("writing data", err)
}
return nil
case error:
responseError := message.(error)
http.Error(res, responseError.Error(), http.StatusInternalServerError)
return nil
}
data, err := json.Marshal(message)
if err != nil {
return lxerrors.New("marshalling message to json", err)
}
_, err = res.Write(data)
if err != nil {
return lxerrors.New("writing data", err)
}
return nil
}
func postData(url string, path string, headers map[string]string, data []byte) (*http.Response, []byte, error) {
completeURL := parseURL(url, path)
request, err := http.NewRequest("POST", completeURL, bytes.NewReader(data))
if err != nil {
return nil, emptyBytes, lxerrors.New("error generating post request", err)
}
if auth != nil {
request.SetBasicAuth(auth.username, auth.password)
auth = nil
}
for key, value := range headers {
request.Header.Add(key, value)
}
resp, err := newClient().c.Do(request)
if err != nil {
return resp, emptyBytes, lxerrors.New("error performing post request", err)
}
respBytes, err := ioutil.ReadAll(resp.Body)
if resp.Body != nil {
defer resp.Body.Close()
}
if err != nil {
return resp, emptyBytes, lxerrors.New("error reading post response", err)
}
return resp, respBytes, nil
}
func deleteAsyncWithRetries(url string, path string, headers map[string]string, retries int) (*http.Response, error) {
resp, err := func() (*http.Response, error) {
completeURL := parseURL(url, path)
request, err := http.NewRequest("DELETE", completeURL, nil)
if err != nil {
return nil, lxerrors.New("error generating delete request", err)
}
if auth != nil {
request.SetBasicAuth(auth.username, auth.password)
auth = nil
}
for key, value := range headers {
request.Header.Add(key, value)
}
resp, err := newClient().c.Do(request)
if err != nil {
return resp, lxerrors.New("error performing delete request", err)
}
return resp, nil
}()
if err != nil && retries > 0 {
return deleteAsyncWithRetries(url, path, headers, retries-1)
}
return resp, err
}
func getWithRetries(url string, path string, headers map[string]string, retries int) (*http.Response, []byte, error) {
resp, respBytes, err := func() (*http.Response, []byte, error) {
completeURL := parseURL(url, path)
request, err := http.NewRequest("GET", completeURL, nil)
if err != nil {
return nil, emptyBytes, lxerrors.New("error generating get request", err)
}
if auth != nil {
request.SetBasicAuth(auth.username, auth.password)
auth = nil
}
for key, value := range headers {
request.Header.Add(key, value)
}
resp, err := newClient().c.Do(request)
if err != nil {
return resp, emptyBytes, lxerrors.New("error performing get request", err)
}
respBytes, err := ioutil.ReadAll(resp.Body)
if resp.Body != nil {
defer resp.Body.Close()
}
if err != nil {
return resp, emptyBytes, lxerrors.New("error reading get response", err)
}
return resp, respBytes, nil
}()
if err != nil && retries > 0 {
return getWithRetries(url, path, headers, retries-1)
}
return resp, respBytes, err
}
func Get(key string) (string, error) {
m.Lock()
defer m.Unlock()
key = prefixKey(key)
kapi := client.NewKeysAPI(c)
resp, err := kapi.Get(context.Background(), key, nil)
if err != nil {
return "", lxerrors.New("getting key/val", err)
}
if resp.Node.Dir {
return "", lxerrors.New("get used on a dir", err)
}
return resp.Node.Value, nil
}
func Rm(key string) error {
m.Lock()
defer m.Unlock()
key = prefixKey(key)
kapi := client.NewKeysAPI(c)
resp, err := kapi.Delete(context.Background(), key, nil)
if err != nil {
return lxerrors.New("deleting key/val pair", err)
}
if resp.Node.Key != key {
return lxerrors.New("removed pair does not have expected key", nil)
}
return nil
}
func Set(key string, value string) error {
m.Lock()
defer m.Unlock()
key = prefixKey(key)
kapi := client.NewKeysAPI(c)
resp, err := kapi.Set(context.Background(), key, value, nil)
if err != nil {
return lxerrors.New("setting key/val pair", err)
}
if resp.Node.Key != key || resp.Node.Value != value {
fmt.Printf("key was %s, value was %s", resp.Node.Key, resp.Node.Value)
return lxerrors.New("key/value pair not set as expected", nil)
}
return nil
}
func postAsyncPB(url string, path string, headers map[string]string, pb proto.Message) (*http.Response, error) {
data, err := proto.Marshal(pb)
if err != nil {
return nil, lxerrors.New("could not proto.Marshal mesasge", err)
}
return postAsyncData(url, path, headers, data)
}
func GetLocalIp() (net.IP, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, err
}
// handle err
for _, i := range ifaces {
if i.Name == "eth1" {
addrs, err := i.Addrs()
if err != nil {
return nil, err
}
// handle err
for _, addr := range addrs {
switch v := addr.(type) {
case *net.IPNet:
return v.IP, nil
case *net.IPAddr:
return v.IP, nil
}
// process IP address
}
}
}
return nil, lxerrors.New("Could not find IP in network interfaces", nil)
}
func Mkdir(dir string) error {
m.Lock()
defer m.Unlock()
dir = prefixKey(dir)
kapi := client.NewKeysAPI(c)
opts := &client.SetOptions{
Dir: true,
}
resp, err := kapi.Set(context.Background(), dir, "", opts)
if err != nil {
return lxerrors.New("making directory", err)
}
if resp.Node.Key != dir || !resp.Node.Dir {
return lxerrors.New("directory not created as expected", nil)
}
return nil
}
func Rmdir(dir string, recursive bool) error {
m.Lock()
defer m.Unlock()
dir = prefixKey(dir)
kapi := client.NewKeysAPI(c)
opts := &client.DeleteOptions{
Dir: true,
Recursive: recursive,
}
resp, err := kapi.Delete(context.Background(), dir, opts)
if err != nil {
return lxerrors.New("removing directory", err)
}
if resp.Node.Key != dir || !resp.Node.Dir {
return lxerrors.New("directory not created as expected", nil)
}
return nil
}
func UntarFileToDirectory(targetDirectory string, sourceTar multipart.File, header *multipart.FileHeader) (int64, error) {
savedTar, err := os.OpenFile(targetDirectory+filepath.Base(header.Filename), os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
return 0, lxerrors.New("creating empty file for copying to", err)
}
defer savedTar.Close()
bytesWritten, err := io.Copy(savedTar, sourceTar)
if err != nil {
return 0, lxerrors.New("copying uploaded file to disk", err)
}
err = Untar(savedTar.Name(), targetDirectory)
if err != nil {
err = UntarNogzip(savedTar.Name(), targetDirectory)
if err != nil {
return 0, lxerrors.New("untarring saved tar", err)
}
}
return bytesWritten, nil
}
func postAsyncBuffer(url string, path string, headers map[string]string, reader io.Reader) (*http.Response, error) {
completeURL := parseURL(url, path)
request, err := http.NewRequest("POST", completeURL, reader)
if err != nil {
return nil, lxerrors.New("error generating post request", err)
}
for key, value := range headers {
request.Header.Add(key, value)
}
if auth != nil {
request.SetBasicAuth(auth.username, auth.password)
auth = nil
}
resp, err := newClient().c.Do(request)
if err != nil {
return resp, lxerrors.New("error performing post request", err)
}
return resp, nil
}
func GetSubdirectories(dir string) ([]string, error) {
m.Lock()
defer m.Unlock()
dir = prefixKey(dir)
kapi := client.NewKeysAPI(c)
resp, err := kapi.Get(context.Background(), dir, nil)
if err != nil {
return []string{}, lxerrors.New("getting key/vals for dir", err)
}
if !resp.Node.Dir {
return []string{}, lxerrors.New("ls used on a non-dir key", err)
}
result := []string{}
for _, node := range resp.Node.Nodes {
if node.Dir {
result = append(result, node.Key)
} //ignore keys
}
return result, nil
}
func GetKeys(dir string) (map[string]string, error) {
m.Lock()
defer m.Unlock()
dir = prefixKey(dir)
kapi := client.NewKeysAPI(c)
resp, err := kapi.Get(context.Background(), dir, nil)
if err != nil {
return map[string]string{}, lxerrors.New("getting key/vals for dir", err)
}
if !resp.Node.Dir {
return map[string]string{}, lxerrors.New("ls used on a non-dir key", err)
}
result := make(map[string]string)
for _, node := range resp.Node.Nodes {
if !node.Dir {
result[node.Key] = node.Value
} //ignore directories
}
return result, nil
}
func PostWithUnmarshal(url string, path string, headers map[string]string, message, jsonObject interface{}) (*http.Response, []byte, error) {
resp, body, err := Post(url, path, headers, message)
if err != nil {
return resp, body, err
}
err = json.Unmarshal(body, jsonObject)
if err != nil {
err = lxerrors.New("could not unmarshal body into jsonObject", err)
}
return resp, body, err
}
func RunCommand(args ...string) (string, error) {
if len(args) < 1 {
return "", lxerrors.New("must provide path to command", nil)
}
cmd := exec.Command(args...)
stdErr, err := cmd.StderrPipe()
if err != nil {
return "", lxerrors.New("piping stderr of command", err)
}
errBuf := new(bytes.Buffer)
_, err = errBuf.ReadFrom(stdErr)
if err != nil {
return "", lxerrors.New("reading stdout from pipe", err)
}
out, err := cmd.Output()
if err != nil {
return "", lxerrors.New("command exited with error: "+errBuf.String(), err)
}
return out, nil
}
func Init(etcdEndpoints []string) error {
cfg := client.Config{
Endpoints: etcdEndpoints,
Transport: client.DefaultTransport,
HeaderTimeoutPerRequest: time.Second,
}
var err error
c, err = client.New(cfg)
if err != nil {
return lxerrors.New("initialize etcd", err)
}
m = &sync.Mutex{}
return nil
}
func UntarNogzip(src, dest string) error {
tarPath, err := exec.LookPath("tar")
if err != nil {
return lxerrors.New("tar not found in path", nil)
}
err = os.MkdirAll(dest, 0755)
if err != nil {
return err
}
command := exec.Command(tarPath, "pxf", src, "-C", dest)
command.Stdout = os.Stdout
command.Stderr = os.Stderr
return command.Run()
}
func postAsyncWithRetries(url string, path string, headers map[string]string, message interface{}, retries int) (*http.Response, error) {
resp, err := func() (*http.Response, error) {
switch message.(type) {
case proto.Message:
return postAsyncPB(url, path, headers, message.(proto.Message))
case io.Reader:
return postAsyncBuffer(url, path, headers, message.(io.Reader))
default:
_, err := json.Marshal(message)
if err != nil {
return nil, lxerrors.New("message was not of expected type `json` or `protobuf`", err)
}
return postAsyncJson(url, path, headers, message)
}
}()
if err != nil && retries > 0 {
return postAsyncWithRetries(url, path, headers, message, retries-1)
}
return resp, err
}
func PostFile(url, path, fileKey, pathToFile string) (*http.Response, []byte, error) {
completeURL := parseURL(url, path)
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
// this step is very important
fileWriter, err := bodyWriter.CreateFormFile(fileKey, pathToFile)
if err != nil {
return nil, emptyBytes, lxerrors.New("error writing to buffer", err)
}
// open file handle
fh, err := os.Open(pathToFile)
if err != nil {
return nil, emptyBytes, lxerrors.New("error opening file", err)
}
//iocopy
_, err = io.Copy(fileWriter, fh)
if err != nil {
return nil, emptyBytes, lxerrors.New("error copying file to form", err)
}
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
request, err := http.NewRequest("POST", completeURL, bodyBuf)
if err != nil {
return nil, emptyBytes, lxerrors.New("error generating post request", err)
}
if auth != nil {
request.SetBasicAuth(auth.username, auth.password)
auth = nil
}
request.Header.Set("Content-type", contentType)
resp, err := newClient().c.Do(request)
if err != nil {
return resp, emptyBytes, lxerrors.New("error performing post request", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return resp, body, lxerrors.New("reading response body", err)
}
return resp, body, nil
}
func Tee(file *os.File, buf *bytes.Buffer) error {
r, w, err := os.Pipe()
if err != nil {
return lxerrors.New("creating pipe", err)
}
stdout := file
file = w
multi := io.MultiWriter(stdout, bufio.NewWriter(buf))
reader := bufio.NewReader(r)
go func() {
for {
line, err := reader.ReadBytes('\n')
if err != nil {
return
}
_, err = multi.Write(append(line, byte('\n')))
if err != nil {
return
}
}
}()
return nil
}