func jobPurger(couchbaseAddress string) {
client, _ := couchbase.Connect(couchbaseAddress)
pool, _ := client.GetPool("default")
params := map[string]interface{}{
"reduce": false,
"include_docs": true,
"stale": "false",
}
bucket := "server"
mc, err := memcached.Connect("tcp", "127.0.0.1:11210")
_, err = mc.Auth(bucket, "")
if err != nil {
log.Fatal(err)
}
// query bucket
b, err := pool.GetBucket(bucket)
if err != nil {
log.Fatalf("Error reading bucket: %v", err)
}
vr, err := b.View(bucket, "data_by_build", params)
if err != nil {
log.Println(err)
}
for _, row := range vr.Rows {
vals := row.Value.([]interface{})
jobUrl := vals[5].(string)
resp, _ := http.Head(jobUrl)
if resp == nil {
continue
}
if resp.StatusCode == 404 {
// make sure 404 is not because jenkins is down
parsedUrl, _ := url.Parse(jobUrl)
resp, _ = http.Head(parsedUrl.Scheme + "://" + parsedUrl.Host)
if resp != nil && resp.StatusCode != 200 {
log.Println("Jenkins down! skipping: " + jobUrl)
continue
}
log.Println("Purging: " + jobUrl)
id := row.ID
// TODO: using dirty workaround until keys are in correct vbuckets
_, err = mc.Get(0, id)
if err == nil {
_, err = mc.Del(0, id)
if err != nil {
log.Fatal(err)
}
}
}
}
}
func getOnlineFile(fileUrl string, fileName string) string {
finalPath := filepath.Join(getWorkingDirectoryAbsolutePath(), fileName)
// HEAD request to see if file exist on server
resp, err := http.Head(fileUrl)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
// If whole file does not exist
if resp.StatusCode == 404 {
resp, err = http.Head(fileUrl + `.part0`)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
// If small parts do not exist
if resp.StatusCode == 404 {
log.Fatalln(`File`, fileUrl, `not found`)
// Else, it must be big file
} else {
// Download parts
for i := 0; resp.StatusCode != 404; i++ {
pathToWrite := filepath.Join(getWorkingDirectoryAbsolutePath(), fileName+`.part`+strconv.Itoa(i))
downloadIfBadFile(fileUrl+`.part`+strconv.Itoa(i), pathToWrite, fileName+`.part`+strconv.Itoa(i))
resp, err = http.Head(fileUrl + `.part` + strconv.Itoa(i+1))
if err != nil {
log.Fatalln(`Error getting part`, strconv.Itoa(i)+`:`, err)
}
defer resp.Body.Close()
}
// Reconstruct
chunker.Reconstruct(finalPath)
}
// Else, download like normal
} else {
pathToWrite := finalPath
downloadIfBadFile(fileUrl, pathToWrite, fileName)
}
return finalPath
}
func patchExists(a, b release) (bool, error) {
res, err := http.Head(s3PatchURL + patchFilename(a, b))
if err != nil {
return false, err
}
return res.StatusCode == 200, nil
}
func (srv *Server) wait(timeout <-chan time.Time) error {
var (
lastError error
)
url := fmt.Sprintf("http://%s/", srv.addr)
for {
select {
case <-timeout:
return E_TIMEOUT
default:
if srv.state != nil {
return srv.state
}
response, err := http.Head(url)
if err == nil {
if response != nil && response.Body != nil {
response.Body.Close()
}
return nil
}
lastError = err
time.Sleep(100 * time.Millisecond)
}
}
return lastError
}
// Poll executes an HTTP HEAD request for the resource url
// and returns health status and a detail string;
// TODO: if HTTP status is 403 or 404 returns an os.Error (NewError)
func (svc *HTTPService) Poll() (monitor.Health, string, error) {
resp, err := http.Head(svc.URL)
if err != nil {
return monitor.Critical, err.Error(), nil
}
return monitor.Ok, resp.Status, nil
}
func checkStatus(urlName string) {
url, found := res[urlName]
if !found {
fmt.Println("\nUnknown URL name:", urlName)
printUrlNames()
fmt.Println()
return
}
// Just want the response header. Don't want the body content.
// Use HEAD method instead of GET.
resp, err := http.Head(url)
if err != nil {
fmt.Println("Error:", err)
return
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
fmt.Println(urlName, ":", url, "is up.")
case http.StatusServiceUnavailable:
fmt.Println(urlName, ":", url, "is down.")
default:
fmt.Println(urlName, ":", url, " return statusCode ", resp.StatusCode)
}
}
// Download fetches an image from the Internet.
// Defaults:
// - dest os.Getwd()
func Download(URL string, dest string) error {
_, err := url.Parse(URL)
if err != nil {
return fmt.Errorf("not a valid URL: %+v", err)
}
res, err := http.Head(URL)
if err != nil {
return fmt.Errorf("url unavailable: %s", URL)
}
if res.StatusCode != http.StatusOK {
return fmt.Errorf("non-200 code from url: %s", URL)
}
res, err = http.Get(URL)
if err != nil {
return fmt.Errorf("couldn't download the file: %+v", err)
}
defer res.Body.Close()
if dest == "" {
tok := strings.Split(URL, "/")
dest = tok[len(tok)-1]
}
if _, err := os.Stat(dest); !os.IsNotExist(err) {
return fmt.Errorf("file already exists: %s", dest)
}
w, err := os.Create(dest)
if err != nil {
return fmt.Errorf("couldn't create file: %s", dest)
}
defer w.Close()
_, err = io.Copy(w, res.Body)
return err
}
func (c *Config) restartLoop() {
if c.RestartPolicy == RestartNever {
return
}
url := "https://storage.googleapis.com/" + c.BinaryBucket + "/" + c.binaryObject()
var lastEtag string
for {
res, err := http.Head(url + "?" + fmt.Sprint(time.Now().Unix()))
if err != nil {
log.Printf("Warning: %v", err)
time.Sleep(15 * time.Second)
continue
}
etag := res.Header.Get("Etag")
if etag == "" {
log.Printf("Warning, no ETag in response: %v", res)
time.Sleep(15 * time.Second)
continue
}
if lastEtag != "" && etag != lastEtag {
log.Printf("Binary updated; restarting.")
// TODO: more graceful restart, letting systemd own the network connections.
// Then we can finish up requests here.
os.Exit(0)
}
lastEtag = etag
time.Sleep(15 * time.Second)
}
}
// useData will read until the chan is closed but it will read items concurrently and the
// work is controlled with a loop acting as a pool and a WaitGroup to ensure it all finishes before we return.
func useData() {
defer close(done)
var wg sync.WaitGroup
concurrencyRate := 10 // in the wild you'd use a config variable for this
for i := 0; i <= concurrencyRate; i++ {
fmt.Println("Worker ", i)
wg.Add(1)
go func() {
defer wg.Done()
for p := range providerChan {
func() {
fmt.Printf("Read from chan: %q, %q\n", p.name, p.url)
resp, err := http.Head(p.url)
if err != nil {
fmt.Printf("Error making head request for %q: %v\n", p.url, err)
return
}
defer resp.Body.Close()
fmt.Printf("Processing Data: %q\t%v\n", p.name, resp)
}()
}
}()
}
wg.Wait()
}
func GetCode(url string) {
res, err := http.Head(url)
if err != nil {
fmt.Println(err)
}
fmt.Println(res.StatusCode)
}
func (self *InputConfig) SendRequest() (data string, err error) {
var (
res *http.Response
raw []byte
)
switch self.Method {
case "HEAD":
res, err = http.Head(self.Url)
case "GET":
res, err = http.Get(self.Url)
default:
err = errors.New("Unknown method")
}
if err != nil {
return
}
defer res.Body.Close()
if raw, err = ioutil.ReadAll(res.Body); err != nil {
return
}
data = string(raw)
data = strings.TrimSpace(data)
return
}
// 获取下载文件信息
func download_file_info(url string) (download_file *DownloadFile, err error) {
head_resp, err := http.Head(url)
if err != nil {
return nil, err
}
download_file = &DownloadFile{}
download_file.OriginUrl = url
if ranges := head_resp.Header.Get("Accept-Ranges"); len(strings.Trim(ranges, " ")) > 0 {
log.Println("==>", ranges)
download_file.AcceptRanges = true
}
download_file.ContentLength = float64(head_resp.ContentLength)
sections := strings.Split(url, "/")
download_file.FileName = sections[len(sections)-1]
if download_file.ContentLength > TB {
download_file.Sizer = fmt.Sprintf("%.2f%s", download_file.ContentLength/TB, "TB")
} else if download_file.ContentLength > GB {
download_file.Sizer = fmt.Sprintf("%.2f%s", download_file.ContentLength/GB, "GB")
} else if download_file.ContentLength > MB {
download_file.Sizer = fmt.Sprintf("%.2f%s", download_file.ContentLength/MB, "MB")
} else if download_file.ContentLength > KB {
download_file.Sizer = fmt.Sprintf("%.2f%s", download_file.ContentLength/KB, "KB")
} else {
download_file.Sizer = fmt.Sprintf("%.2f%s", download_file.ContentLength, "B")
}
return
}
func (r *clientImpl) BlobLength(name string, dgst digest.Digest) (int, error) {
blobURL, err := r.ub.BuildBlobURL(name, dgst)
if err != nil {
return -1, err
}
response, err := http.Head(blobURL)
if err != nil {
return -1, err
}
defer response.Body.Close()
// TODO(bbland): handle other status codes, like 5xx errors
switch {
case response.StatusCode == http.StatusOK:
lengthHeader := response.Header.Get("Content-Length")
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return -1, err
}
return int(length), nil
case response.StatusCode == http.StatusNotFound:
return -1, nil
case response.StatusCode >= 400 && response.StatusCode < 500:
var errs v2.Errors
decoder := json.NewDecoder(response.Body)
err = decoder.Decode(&errs)
if err != nil {
return -1, err
}
return -1, &errs
default:
return -1, &UnexpectedHTTPStatusError{Status: response.Status}
}
}
func (rss *SourceRss) hasNewStories() (has bool) {
response, err := http.Head(rss.url)
if err != nil {
fmt.Printf("There was an error running the HEAD command on url:\n%s", rss.url)
fmt.Printf("Assuming new stories were available\n")
return true
}
// Try and figure out the mtime
mtime := response.Header.Get("Last-Modified")
var parsed_time int64
if len(mtime) > 0 {
parsed_time, err = parse_time(mtime)
if err != nil {
fmt.Printf("There was an error parsing the timestamp: %s\n", mtime)
}
fmt.Printf("Last crawled = %d\n", rss.lastCrawled)
fmt.Printf("Updated = %d (%s)\n", parsed_time, mtime)
} else {
fmt.Printf("There was omething wrong with Last-Modified")
}
// If there are changes, yes
if rss.lastCrawled < parsed_time {
return true
}
return false
}
func hasChangedSince(url string, lastModified time.Time) (bool, time.Time, error) {
res, err := http.Head(url)
if err != nil {
return false, time.Time{}, fmt.Errorf("HEAD request failed (%s)", err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
return false, time.Time{}, fmt.Errorf("HEAD request response %d", res.StatusCode)
}
header := res.Header.Get("Last-Modified")
t, err := time.Parse(time.RFC1123, header)
if err != nil {
return false, time.Time{}, fmt.Errorf("Failed to parse Last-Modified header (%s)", err)
}
if t.After(lastModified) {
return true, t, nil
}
return false, t, nil
}
func FetchHead(url string) (bool, CheckInfo) {
var info CheckInfo
var err error
var resp *http.Response
var s string
for i := 0; i < 2; i++ {
if i == 0 {
resp, err = http.Head(url)
s = "Head"
} else {
resp, err = http.Get(url)
s = "Get"
}
if err == nil && resp.StatusCode == 200 {
break
}
}
if err != nil {
info = CheckInfo{Status: Error | FetchableUrl | NoDataatUrlError, Position: -1, Context: fmt.Sprintf("%s URL kann nicht aufgelöst werden: %s (%s)", url, err, s)}
} else if sc := resp.StatusCode; sc != 200 {
info = CheckInfo{Status: Error | FetchableUrl | NoDataatUrlError, Position: -1, Context: fmt.Sprintf("%s liefert nicht-OK Status-Code '%d' (%s)", url, sc, s)}
} else {
info = CheckInfo{Status: Info | FetchableUrl | FetchSuccess, Position: -1, Context: url}
}
return (info.Status & (Info | FetchSuccess)) == (Info | FetchSuccess), info
}
func getHeaders(url string) (http.Header, error) {
resp, err := http.Head(url)
if err != nil {
return nil, err
}
return resp.Header, nil
}
func serverUp(ts *httptest.Server) func() bool {
return func() bool {
url := fmt.Sprintf("http://%s", ts.Listener.Addr())
resp, _ := http.Head(url)
return resp != nil && resp.StatusCode == http.StatusOK
}
}
func ping(domainList []string) {
for i := range domainList {
fmt.Println("Pinging", domainList[i])
wg.Add(1)
go func(domain string) {
url := fmt.Sprintf("http://%s", domain)
res, err := http.Head(url)
if err == nil {
if res.StatusCode == 200 {
fmt.Println("Site is up")
} else {
fmt.Println("Site is down:", res.Status)
go alert(domain, res.Status)
}
} else {
fmt.Println("Site didn't respond:", err)
go alert(domain, err.Error())
}
wg.Done()
}(domainList[i])
wg.Wait()
}
}
// TestHeadResponses verifies that responses to HEAD requests don't
// declare that they're chunking in their response headers, aren't
// allowed to produce output, and don't set a Content-Type since
// the real type of the body data cannot be inferred.
func TestHeadResponses(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte("Ignored body"))
if err != http.ErrBodyNotAllowed {
t.Errorf("on Write, expected ErrBodyNotAllowed, got %v", err)
}
// Also exercise the ReaderFrom path
_, err = io.Copy(w, strings.NewReader("Ignored body"))
if err != http.ErrBodyNotAllowed {
t.Errorf("on Copy, expected ErrBodyNotAllowed, got %v", err)
}
}))
defer ts.Close()
res, err := http.Head(ts.URL)
if err != nil {
t.Error(err)
}
if len(res.TransferEncoding) > 0 {
t.Errorf("expected no TransferEncoding; got %v", res.TransferEncoding)
}
ct := res.Header.Get("Content-Type")
if ct != "" {
t.Errorf("expected no Content-Type; got %s", ct)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Error(err)
}
if len(body) > 0 {
t.Errorf("got unexpected body %q", string(body))
}
}
func serveHeadAttachment(w http.ResponseWriter, r *http.Request) {
attid := mux.Vars(r)["attid"]
bugid := mux.Vars(r)["bugid"]
me := whoami(r)
if _, err := getBugOrDisplayErr(bugid, me, w, r); err != nil {
return
}
att := Attachment{}
err := db.Get(attid, &att)
if err != nil {
showError(w, r, err.Error(), 500)
return
}
if att.Type != "attachment" {
showError(w, r, "not an attachment", 500)
return
}
res, err := http.Head(att.Url)
if err != nil {
showError(w, r, err.Error(), 500)
return
}
defer res.Body.Close()
if res.StatusCode != 200 {
showError(w, r, res.Status, 500)
return
}
w.Header().Set("Content-Type", res.Header.Get("Content-Type"))
w.Header().Set("Content-Length", fmt.Sprintf("%v", res.ContentLength))
w.Header().Set("Content-Disposition", "attachment")
}
//Decide whether connection is alive
func isConnected(url string) bool {
_, err := http.Head(url)
if err != nil {
return false
}
return true
}
// TestURLFor checks that the URLFor method functions properly, but only if it
// is implemented
func (suite *DriverSuite) TestURLFor(c *check.C) {
filename := randomPath(32)
contents := randomContents(32)
defer suite.StorageDriver.Delete(firstPart(filename))
err := suite.StorageDriver.PutContent(filename, contents)
c.Assert(err, check.IsNil)
url, err := suite.StorageDriver.URLFor(filename, nil)
if err == storagedriver.ErrUnsupportedMethod {
return
}
c.Assert(err, check.IsNil)
response, err := http.Get(url)
c.Assert(err, check.IsNil)
defer response.Body.Close()
read, err := ioutil.ReadAll(response.Body)
c.Assert(err, check.IsNil)
c.Assert(read, check.DeepEquals, contents)
url, err = suite.StorageDriver.URLFor(filename, map[string]interface{}{"method": "HEAD"})
if err == storagedriver.ErrUnsupportedMethod {
return
}
c.Assert(err, check.IsNil)
response, err = http.Head(url)
c.Assert(response.StatusCode, check.Equals, 200)
c.Assert(response.ContentLength, check.Equals, int64(32))
}
func (c *client) getStats(serverUrl, remotePath string) (stats *Stats, returnErr error) {
defer CatchPanicAsError(&returnErr)
resp, err := http.Head(serverUrl + "?path=" + url.QueryEscape(remotePath))
if err != nil {
return nil, err
}
defer resp.Body.Close()
err = c.checkServerResponse(resp)
if err != nil {
return nil, err
}
if exists := resp.Header.Get("EXISTS"); exists == "" {
return nil, fmt.Errorf("Could not find 'EXISTS' header")
} else if exists == "0" {
return &Stats{}, nil
}
if isDir := resp.Header.Get("IS_DIR"); isDir == "" {
return nil, fmt.Errorf("Could not find 'IS_DIR' header")
} else {
return &Stats{
Exists: true,
IsDir: isDir == "1",
}, nil
}
}
func badHTTPHead() {
res, err := http.Head("http://foo.com")
defer res.Body.Close() // ERROR "using res before checking for errors"
if err != nil {
log.Fatal(err)
}
}
func main() {
//res, _ := http.Head("http://localhost/rand.txt"); // 187 MB file of random numbers per line
res, _ := http.Head("http://download.firefox.com.cn/releases/firefox/40.0/zh-CN/Firefox-latest.dmg")
maps := res.Header
length, _ := strconv.Atoi(maps["Content-Length"][0]) // Get the content length from the header request
limit := 10 // 10 Go-routines for the process so each downloads 18.7MB
len_sub := length / limit // Bytes for each Go-routine
diff := length % limit // Get the remaining for the last request
body := make([]string, 11) // Make up a temporary array to hold the data to be written to the file
for i := 0; i < limit; i++ {
wg.Add(1)
min := len_sub * i // Min range
max := len_sub * (i + 1) // Max range
if i == limit-1 {
max += diff // Add the remaining bytes in the last request
}
go func(min int, max int, i int) {
client := &http.Client{}
req, _ := http.NewRequest("GET", "http://download.firefox.com.cn/releases/firefox/40.0/zh-CN/Firefox-latest.dmg", nil)
range_header := "bytes=" + strconv.Itoa(min) + "-" + strconv.Itoa(max-1) // Add the data for the Range header of the form "bytes=0-100"
req.Header.Add("Range", range_header)
resp, _ := client.Do(req)
defer resp.Body.Close()
reader, _ := ioutil.ReadAll(resp.Body)
body[i] = string(reader)
ioutil.WriteFile("./"+strconv.Itoa(i), []byte(string(body[i])), 0x777) // Write to the file i as a byte array
wg.Done()
//ioutil.WriteFile("new_oct.png", []byte(string(body)), 0x777)
}(min, max, i)
}
wg.Wait()
}
func DiscoverHttps(endpoint *Endpoint) (*m.SuggestedMonitor, error) {
host := endpoint.Host
path := "/"
if endpoint.URL != nil && endpoint.URL.Scheme == "https" {
host = endpoint.URL.Host
path = endpoint.URL.Path
}
resp, err := http.Head(fmt.Sprintf("https://%s%s", host, path))
if err != nil {
return nil, err
}
requestUrl := resp.Request.URL
hostParts := strings.Split(requestUrl.Host, ":")
varHost := hostParts[0]
varPort := "443"
if len(hostParts) > 1 {
varPort = hostParts[1]
}
settings := []m.MonitorSettingDTO{
{Variable: "host", Value: varHost},
{Variable: "port", Value: varPort},
{Variable: "path", Value: requestUrl.Path},
}
return &m.SuggestedMonitor{MonitorTypeId: 2, Settings: settings}, nil
}
// Retry wrapper for http://golang.org/pkg/net/http/#Head where attempts is the number of http calls made (one plus number of retries).
func (httpRetryClient *Client) Head(url string) (resp *http.Response, attempts int, err error) {
return httpRetryClient.Retry(func() (*http.Response, error, error) {
resp, err := http.Head(url)
// assume all errors should result in a retry
return resp, err, nil
})
}
func downloadFile(url string, id int64, ext string) (string, error) {
// Create a local file to download to:
img_name := strconv.FormatInt(id, 10)
store_path := path.Join(store_folder(), img_name+ext)
fmt.Printf("Downloading %s to %s\n", url, store_path)
local_file, err := os.OpenFile(store_path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
// File already exists:
if os.IsExist(err) {
fmt.Printf("File exists...\n")
var fi os.FileInfo
var head_rsp *http.Response
// Check file size:
fi, err = os.Stat(store_path)
file_size := fi.Size()
fmt.Printf("File size = %d\n", file_size)
// HEAD to get download file size:
head_rsp, err = http.Head(url)
if err == nil {
fmt.Printf("HEAD reports size = %d\n", head_rsp.ContentLength)
if head_rsp.ContentLength > 0 {
if file_size == head_rsp.ContentLength {
fmt.Println("Same file size; skip")
// File is same size; do nothing:
return store_path, nil
}
}
// File is not same size; redownload:
fmt.Println("Redownload")
local_file, err = os.OpenFile(store_path, os.O_RDWR|os.O_TRUNC|os.O_EXCL, 0600)
}
}
if err != nil {
return "", err
}
defer local_file.Close()
// Do a HTTP GET to fetch the image:
img_rsp, err := http.Get(url)
if err != nil {
return "", err
}
defer img_rsp.Body.Close()
// Download file:
_, err = io.Copy(local_file, img_rsp.Body)
if err != nil {
return "", err
}
return local_file.Name(), nil
}
// Upload the log l, inform the websocket ws, and using the waitgroup wg
func upload(l Log, ws *websocket.Conn, wg *sync.WaitGroup) {
defer wg.Done()
path := fmt.Sprintf("%s/games/%s/%s/", upload_url, l.Uploader, l.Key)
// first check that the game hasn't already been uploaded
resp, err := http.Head(path)
if err != nil {
fmt.Printf("head failed: %#v\n", err)
} else if resp.StatusCode == http.StatusOK {
l.Status = "Skipped"
l.Reason = "Already Uploaded"
send(ws, l)
fmt.Printf("Already uploaded %s/%s -- skipping\n", l.Uploader, l.Key)
return
} else {
fmt.Printf("head failed: %#v\n", resp)
}
// now we're sending the log, we have to compress the log lines, and
// gob encode it -- for simplicity
var y bytes.Buffer
gz := gzip.NewWriter(&y)
if _, err := gz.Write(l.data.Bytes()); err != nil {
panic(err)
}
if err := gz.Close(); err != nil {
panic(err)
}
l.Data = y.Bytes()
var x bytes.Buffer
enc := gob.NewEncoder(&x)
err = enc.Encode(l)
if err != nil {
panic(err)
}
// use v1 of the upload spec -- this says what information is included in the log
path = fmt.Sprintf("%sv1", path)
resp, err = http.Post(path, "appliation/octet-stream", &x)
// tell the user what happened to the upload
if err != nil {
l.Status = "Failed"
l.Reason = fmt.Sprintf("Error contacting server: %s", err)
} else if resp.StatusCode != http.StatusAccepted {
b, _ := ioutil.ReadAll(resp.Body)
l.Status = "Failed"
l.Reason = fmt.Sprintf("Server returned: %d (%s)", resp.StatusCode, b)
} else {
l.Status = "Success"
l.Reason = fmt.Sprintf("View at: %s/games/%s/%s/", upload_url, l.Uploader, l.Key)
}
send(ws, l)
}
