// FileFetcher's Fetch() implementation
func (this *fileFetcherExtender) Fetch(u *url.URL, userAgent string, headRequest bool) (*http.Response, error) {
var res *http.Response = new(http.Response)
var req *http.Request
var e error
if req, e = http.NewRequest("GET", u.String(), nil); e != nil {
panic(e)
}
// Prepare the pseudo-request
req.Header.Add("User-Agent", userAgent)
// Open the file specified as path in u, relative to testdata/[host]/
f, e := os.Open(path.Join(FileFetcherBasePath, u.Host, u.Path))
if e != nil {
// Treat errors as 404s - file not found
res.Status = "404 Not Found"
res.StatusCode = 404
} else {
res.Status = "200 OK"
res.StatusCode = 200
res.Body = f
}
res.Request = req
return res, e
}
func (c *URLCache) Restore(res *http.Response) {
res.Status = c.CachedResponse.Status
res.StatusCode = c.CachedResponse.StatusCode
res.Header = c.CachedResponse.Header
res.ContentLength = c.CachedResponse.ContentLength
res.TransferEncoding = c.CachedResponse.TransferEncoding
res.Body = &ClosableBuffer{bytes.NewReader(c.CachedBody)}
res.Header.Set(CachedHeader, CachedHeaderVal)
res.Header.Set(CachedMD5Header, c.MD5)
}
// ModifyResponse sets the status code to 400 Bad Request if a loop was
// detected in the request.
func (m *viaModifier) ModifyResponse(res *http.Response) error {
ctx := martian.NewContext(res.Request)
if err, _ := ctx.Get(viaLoopKey); err != nil {
res.StatusCode = 400
res.Status = http.StatusText(400)
return err.(error)
}
return nil
}
// ModifyResponse runs resmod.ModifyResponse.
//
// If an error is returned from resmod.ModifyResponse it is returned.
func (m *Modifier) ModifyResponse(res *http.Response) error {
ctx := martian.NewContext(res.Request)
actx := auth.FromContext(ctx)
err := m.resmod.ModifyResponse(res)
if actx.Error() != nil {
res.StatusCode = 403
res.Status = http.StatusText(403)
}
return err
}
func (f *EtagFilter) FilterResponse(request *falcore.Request, res *http.Response) {
request.CurrentStage.Status = 1 // Skipped (default)
if if_none_match := request.HttpRequest.Header.Get("If-None-Match"); if_none_match != "" {
if res.StatusCode == 200 && res.Header.Get("Etag") == if_none_match {
res.StatusCode = 304
res.Status = "304 Not Modified"
res.Body.Close()
res.Body = nil
res.ContentLength = 0
request.CurrentStage.Status = 0 // Success
}
}
}
func (res *HTTPResponseEvent) ToResponse(body bool) *http.Response {
raw := new(http.Response)
raw.Header = res.Headers
raw.ProtoMajor = 1
raw.ProtoMinor = 1
raw.StatusCode = int(res.StatusCode)
raw.Status = http.StatusText(raw.StatusCode)
raw.TransferEncoding = res.Headers["TransferEncoding"]
if body {
raw.Body = NewHTTPBody(res.GetContentLength(), res.Content)
}
return raw
}
func TestResponse(t *testing.T) {
under := new(http.Response)
under.Status = "the status message"
under.StatusCode = 392
under.Header = make(http.Header)
under.Header.Set("Blah", "blah blah")
under.Body = ioutil.NopCloser(bytes.NewBufferString("some content"))
var resp runtime.ClientResponse = response{under}
assert.EqualValues(t, under.StatusCode, resp.Code())
assert.Equal(t, under.Status, resp.Message())
assert.Equal(t, "blah blah", resp.GetHeader("blah"))
assert.Equal(t, under.Body, resp.Body())
}
// NewNotification returns a notification response with a specific body content.
func NewNotification(body *bytes.Buffer) *http.Response {
resp := new(http.Response)
resp.Status = "200 OK"
resp.StatusCode = http.StatusOK
resp.ProtoMajor = 1
resp.ProtoMinor = 0
resp.Body = ioutil.NopCloser(body)
resp.ContentLength = int64(body.Len())
resp.Header = map[string][]string{}
resp.Header.Set("Content-Type", HTTPContentTypeHAPJson)
// Will be ignored unfortunately and won't be fixed https://github.com/golang/go/issues/9304
// Make sure to call FixProtocolSpecifier() instead
resp.Proto = "EVENT/1.0"
return resp
}
func (r *response) Response() *http.Response {
if r.Data == nil {
r.Data = new(bytes.Buffer)
}
out := new(http.Response)
out.Status = fmt.Sprintf("%d %s", r.StatusCode, http.StatusText(r.StatusCode))
out.StatusCode = r.StatusCode
out.Proto = "HTTP/1.1"
out.ProtoMajor = 1
out.ProtoMinor = 1
out.Header = r.Header
out.Body = &readCloser{r.Data}
out.ContentLength = int64(r.Data.Len())
out.TransferEncoding = nil
out.Close = true
out.Trailer = make(http.Header)
out.Request = r.Request
return out
}
func (p proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
r.Body.Close()
conn, _, err := w.(http.Hijacker).Hijack()
if err != nil {
log.Println("Failed to hijack connection in ProxyConnections.", err)
return
}
defer conn.Close()
if _, ok := conn.(*tls.Conn); !ok {
log.Println("Recieved a non-TLS connection in ProxyConnections.")
return
}
// Send the connection accepted response.
res := new(http.Response)
res.Status = "200 Connection Established"
res.StatusCode = http.StatusOK
res.Proto = "HTTP/1.1"
res.ProtoMajor = 1
res.ProtoMinor = 1
if err = res.Write(conn); err != nil {
log.Println("Failed to send connection established message in ProxyConnections.", err)
return
}
client, err := NewClientConn(conn, nil, 3, 1)
if err != nil {
log.Println("Error creating SPDY connection in ProxyConnections.", err)
return
}
go client.Run()
// Call user code.
p.ProxyConnHandle(client)
client.Close()
}
func resUnmarshal(p []byte) (string, *http.Response, error) {
var handle string
var res *http.Response
unzipped, err := unzip(p)
if err != nil {
return handle, res, err.(*Error).escalate(errResUnmarshal)
}
in := new(response)
in.Header = http.Header(make(map[string][]string))
if err = json.Unmarshal(unzipped, in); err != nil {
return handle, res, newError(errResUnmarshalJSON, err.Error())
}
handle = in.Handle
res = new(http.Response)
res.Body = &body{bytes.NewBuffer(in.Body)}
res.ContentLength = int64(len(in.Body))
res.Header = in.Header
res.StatusCode = in.Code
res.Status = http.StatusText(in.Code)
return handle, res, nil
}
// New returns an event response for a characteristic from an accessory.
func New(a *accessory.Accessory, c *characteristic.Characteristic) (*http.Response, error) {
body, err := Body(a, c)
if err != nil {
return nil, err
}
resp := new(http.Response)
resp.Status = "200 OK"
resp.StatusCode = http.StatusOK
resp.ProtoMajor = 1
resp.ProtoMinor = 0
resp.Body = ioutil.NopCloser(body)
resp.ContentLength = int64(body.Len())
resp.Header = map[string][]string{}
resp.Header.Set("Content-Type", netio.HTTPContentTypeHAPJson)
// (brutella) Not sure if Date header must be set
// resp.Header.Set("Date", netio.CurrentRFC1123Date())
// Will be ignored unfortunately and won't be fixed https://github.com/golang/go/issues/9304
// Make sure to call FixProtocolSpecifier() instead
resp.Proto = "EVENT/1.0"
return resp, nil
}
func (c *conn) largefetch(w *bufio.Writer, r *http.Request, first *http.Response) error {
pos := 0
length := 0
step := 100000
log.Printf("largefetch begin: %s", r.URL.Path)
var err error
if first == nil {
first, _, pos, length, err = c.rangeRoundTrip(r, 0, 1000000)
if err != nil {
r.Body.Close()
return fmt.Errorf("conn.largefetch(first roundtrip)>%s", err)
}
defer first.Body.Close()
} else {
contentRange := first.Header.Get("Content-Range")
if contentRange == "" {
r.Body.Close()
return errors.New("conn.largefetch: empty content range")
}
m := rangeresp.FindStringSubmatch(contentRange)
if len(m) != 4 {
r.Body.Close()
return errors.New("conn.largefetch: invalid content range")
}
pos, err = strconv.Atoi(m[2])
if err != nil {
r.Body.Close()
return fmt.Errorf("conn.largefetch(convert pos)>%s", err)
}
length, err = strconv.Atoi(m[3])
if err != nil {
r.Body.Close()
return fmt.Errorf("conn.largefetch(convert length)>%s", err)
}
}
first.Header.Del("Content-Range")
//first.ContentLength = int64(end - start + 1)
first.Header.Set("Content-Length", strconv.Itoa(length))
first.StatusCode = 200
first.Status = http.StatusText(200)
err = writeResponse(w, first)
if err != nil {
return fmt.Errorf("conn.largefetch>%s", err)
}
err = w.Flush()
if err != nil {
return fmt.Errorf("conn.largefetch(w.Flush)>%s", err)
}
if length-pos-1 == 0 {
return nil
}
var seq sequencer
task := make(chan int)
errChan := make(chan error)
for i := 0; i < 30; i++ {
go func() {
defer print("end--------")
for n := range task {
start_in := pos + 1 + step*n
end_in := pos + step*(n+1)
if end_in > length-1 {
end_in = length - 1
}
var start_out, end_out int
var resp *http.Response
var err error
for i := 0; i < 3; i++ {
resp, start_out, end_out, _, err = c.rangeRoundTrip(r, start_in, end_in)
if err == nil {
break
}
}
if err != nil {
errChan <- fmt.Errorf("conn.largefetch.routine>%s", err)
seq.Close()
return
}
if start_out != start_in || end_out != end_in {
errChan <- errors.New("conn.largefetch.routine: error range returned")
}
ok := seq.Start(uint(n))
if !ok { // seq closed
return
}
_, err = io.Copy(w, resp.Body)
resp.Body.Close()
if err != nil {
errChan <- fmt.Errorf("conn.largefetch.routine(send conts body)>%s", err)
seq.Close()
return
}
err = w.Flush()
if err != nil {
errChan <- fmt.Errorf("conn.largefetch.routine(conts flush)>%s", err)
seq.Close()
return
}
seq.End(uint(n))
errChan <- nil
}
}()
}
var nTask int
OUT2:
for i := 0; i < (length-pos-2)/step+1; i++ {
OUT1:
for {
select {
case task <- i:
nTask++
break OUT1
case err0 := <-errChan:
if err0 != nil {
err = fmt.Errorf("largefetch>%s", err0)
close(task)
nTask--
break OUT2
}
nTask--
}
}
}
for nTask > 0 {
err0 := <-errChan
if err0 != nil && err == nil {
err = fmt.Errorf("largefetch>%s", err0)
}
nTask--
}
close(errChan)
r.Body.Close()
log.Printf("largefetch end: %s", r.URL.Path)
return err
}
func (h *TestHandler) PrepareResponse(resp *http.Response) error {
resp.Status = "201"
return nil
}
func toShadowD(w icap.ResponseWriter, req *icap.Request) {
local_debug := false
if strings.Contains(req.URL.RawQuery, "debug=1") {
local_debug = true
}
h := w.Header()
h.Set("ISTag", ISTag)
h.Set("Service", "Shadower ICAP to WAF Connector")
if *debug {
fmt.Fprintln(os.Stderr, "Printing the full ICAP request")
fmt.Fprintln(os.Stderr, req)
fmt.Fprintln(os.Stderr, req.Request)
fmt.Fprintln(os.Stderr, req.Response)
}
switch req.Method {
case "OPTIONS":
h.Set("Methods", "REQMOD")
h.Set("Options-TTL", "1800")
h.Set("Allow", "204, 206")
h.Set("Preview", "0")
h.Set("Transfer-Preview", "*")
h.Set("Max-Connections", *maxConnections)
h.Set("X-Include", "X-Client-Ip, X-Authenticated-Groups, X-Authenticated-User, X-Subscriber-Id")
w.WriteHeader(200, nil, false)
case "REQMOD":
modified := false
nullBody := false
allow206 := false
allow204 := false
xclientip := false
if _, allow204Exists := req.Header["Allow"]; allow204Exists {
if strings.Contains(req.Header["Allow"][0], "204") {
allow204 = true
}
}
if _, allow206Exists := req.Header["Allow"]; allow206Exists {
if strings.Contains(req.Header["Allow"][0], "206") {
allow206 = true
}
}
if _, xclientipExists := req.Header["X-Client-Ip"]; xclientipExists {
if len(req.Header["X-Client-Ip"][0]) > 1 {
xclientip = true
}
}
if _, encapsulationExists := req.Header["Encapsulated"]; encapsulationExists {
if strings.Contains(req.Header["Encapsulated"][0], "null-body=") {
nullBody = true
}
}
if *debug || local_debug {
for k, v := range req.Header {
fmt.Fprintln(os.Stderr, "The ICAP headers:")
fmt.Fprintln(os.Stderr, "key size:", len(req.Header[k]))
fmt.Fprintln(os.Stderr, "key:", k, "value:", v)
}
}
_, _, _, _ = nullBody, allow206, modified, allow204
if xclientip {
req.Request.RemoteAddr = req.Header["X-Client-Ip"][0]
}
if wrongMethod(req) {
if *debug {
fmt.Println("This request has a", req.Request.Method, "method which is not being analyzed")
}
w.WriteHeader(204, nil, false)
return
}
if *debug || local_debug {
for k, v := range req.Request.Header {
fmt.Fprintln(os.Stderr, "key:", k, "value:", v)
}
}
// Send the request to ShadowD
// If an attack(5,6) was declared then send a custom 500 page
// If OK then send a 204 back
var resStatus = 1
shodowdres, err := shadowServer.SendToShadowd(req.Request)
newmap := make(map[string]interface{})
err = json.Unmarshal([]byte(shodowdres), &newmap)
if err != nil {
panic(err)
}
switch int(newmap["status"].(float64)) {
case shadowd.STATUS_OK:
if *debug || local_debug {
fmt.Println("Request reported, OK")
}
w.WriteHeader(204, nil, false)
return
case shadowd.STATUS_BAD_REQUEST:
resStatus = 400
case shadowd.STATUS_BAD_SIGNATURE:
resStatus = 503
case shadowd.STATUS_BAD_JSON:
resStatus = 504
case shadowd.STATUS_ATTACK:
resStatus = 505
case shadowd.STATUS_CRITICAL_ATTACK:
resStatus = 506
default:
resStatus = 500
}
resp := new(http.Response)
resp.Status = "Internal Server Error"
resp.StatusCode = resStatus
resp.Proto = req.Request.Proto
resp.ProtoMajor = req.Request.ProtoMajor
resp.ProtoMinor = req.Request.ProtoMinor
resp.Request = req.Request
myHeaderMap := make(map[string][]string)
resp.Header = myHeaderMap
resp.Header.Set("X-Ngtech-Proxy", "Shadower")
resp.Header.Set("X-Shadower", strconv.Itoa(resStatus))
resp.Header.Set("Content-Type", "text/html")
resp.Header.Set("Content-Length", strconv.Itoa(len(internalerrorpage)))
w.WriteHeader(200, resp, true)
io.WriteString(w, internalerrorpage)
return
if *debug {
fmt.Println("end of the line 204 response!.. Shouldn't happen.")
}
w.WriteHeader(204, nil, false)
return
case "RESPMOD":
w.WriteHeader(204, nil, false)
return
default:
w.WriteHeader(405, nil, false)
if *debug || local_debug {
fmt.Fprintln(os.Stderr, "Invalid request method")
}
}
}
// ReplyWithStatus helper to write the http.Response status code and text.
func ReplyWithStatus(res *http.Response, code int) {
res.StatusCode = code
res.Status = strconv.Itoa(code) + " " + http.StatusText(code)
}
func (c *conn) unpackResponse(resp *http.Response) (*http.Response, error) {
// read response
bodyr := bufio.NewReader(resp.Body)
compressed, err := bodyr.ReadByte()
if err != nil {
resp.Body.Close()
return nil, fmt.Errorf("conn.unpackResponse(bodyr.ReadByte)>%s", err)
}
var status, lenHeaderEncoded, lenContent uint32
var bodyReader io.ReadCloser
if compressed == '1' {
bodyReader, err = zlib.NewReader(bodyr)
if err != nil {
resp.Body.Close()
return nil, fmt.Errorf("conn.unpackResponse(zlib.NewReader)>%s", err)
}
bodyReader = &closeWrap{closer: resp.Body, ReadCloser: bodyReader}
} else {
bodyReader = &closeWrap{closer: resp.Body, ReadCloser: ioutil.NopCloser(bodyr)}
}
// deal with header
err = binary.Read(bodyReader, binary.BigEndian, &status)
if err != nil {
bodyReader.Close()
return nil, fmt.Errorf("conn.unpackResponse(read status)>%s", err)
}
err = binary.Read(bodyReader, binary.BigEndian, &lenHeaderEncoded)
if err != nil {
bodyReader.Close()
return nil, fmt.Errorf("conn.unpackResponse(read lenHeaderEncoded)>%s", err)
}
err = binary.Read(bodyReader, binary.BigEndian, &lenContent)
if err != nil {
bodyReader.Close()
return nil, fmt.Errorf("conn.unpackResponse(read lenContent)>%s", err)
}
response := new(http.Response)
response.StatusCode = int(status)
response.Status = http.StatusText(int(status))
response.ProtoMajor = 1
response.ProtoMinor = 0
bHeaderEncoded := make([]byte, lenHeaderEncoded)
_, err = io.ReadFull(bodyReader, bHeaderEncoded)
if err != nil {
bodyReader.Close()
return nil, fmt.Errorf("conn.unpackResponse(read header)>%s", err)
}
response.Header = make(http.Header)
for _, h := range strings.Split(string(bHeaderEncoded), "&") {
kv := strings.SplitN(h, "=", 2)
if len(kv) != 2 {
continue
}
value, err := hex.DecodeString(kv[1])
if err != nil {
bodyReader.Close()
return nil, fmt.Errorf("conn.unpackResponse(hex.DecodeString(kv[1]))>%s", err)
}
if strings.Title(kv[0]) == "Set-Cookie" {
for _, cookie := range strings.Split(string(value), "\r\nSet-Cookie: ") {
response.Header.Add("Set-Cookie", cookie)
}
} else {
response.Header.Add(strings.Title(kv[0]), string(value))
}
}
response.Header.Set("Content-Length", strconv.Itoa(int(lenContent)))
response.Body = bodyReader
return response, nil
}
func (rt *RoundTripper) readResponse(c *e3x.Channel) (*http.Response, error) {
var (
r = newClientPacketReadCloser(c)
resp *http.Response
err error
)
{ // read the header
var (
headLenData [2]byte
headLen uint16
headerData []byte
header map[string]interface{}
)
_, err = io.ReadFull(r, headLenData[:])
if err != nil {
if err == io.EOF {
return nil, io.ErrUnexpectedEOF
}
return nil, err
}
headLen = binary.BigEndian.Uint16(headLenData[:])
headerData = make([]byte, headLen)
_, err = io.ReadFull(r, headerData)
if err != nil {
if err == io.EOF {
return nil, io.ErrUnexpectedEOF
}
return nil, err
}
err = json.Unmarshal(headerData, &header)
if err != nil {
return nil, err
}
resp = &http.Response{}
if v, p := header[":status"]; p && v != nil {
var i int
switch w := v.(type) {
case int:
i = w
case int64:
i = int(w)
case float32:
i = int(w)
case float64:
i = int(w)
}
if i > 0 {
resp.StatusCode = i
resp.Status = http.StatusText(i)
}
delete(header, ":status")
}
if resp.StatusCode == 0 {
return nil, &http.ProtocolError{"missing `status` header"}
}
resp.Header = make(http.Header, len(header))
resp.Proto = "1.1"
resp.ProtoMajor = 1
resp.ProtoMinor = 1
for k, v := range header {
if s, ok := v.(string); !ok || s == "" {
if k != "status" {
resp.Header.Set(http.CanonicalHeaderKey(k), s)
}
}
}
}
{ // set the body
resp.Body = r
}
return resp, nil
}
// ModifyResponse overwrites the status text and code on an HTTP response and
// returns nil.
func (s *statusModifier) ModifyResponse(res *http.Response) error {
res.StatusCode = s.statusCode
res.Status = http.StatusText(s.statusCode)
return nil
}
// RoundTrip takes a Request and returns a Response
//
// If there is a fresh Response already in cache, then it will be returned without connecting to
// the server.
//
// If there is a stale Response, then any validators it contains will be set on the new request
// to give the server a chance to respond with NotModified. If this happens, then the cached Response
// will be returned.
func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
req = cloneRequest(req)
cacheKey := cacheKey(req)
cacheableMethod := req.Method == "GET" || req.Method == "HEAD"
var cachedResp *http.Response
if cacheableMethod {
cachedResp, err = CachedResponse(t.Cache, req)
} else {
// Need to invalidate an existing value
t.Cache.Delete(cacheKey)
}
transport := t.Transport
if transport == nil {
transport = http.DefaultTransport
}
if cachedResp != nil && err == nil && cacheableMethod && req.Header.Get("range") == "" {
if t.MarkCachedResponses {
cachedResp.Header.Set(XFromCache, "1")
}
if varyMatches(cachedResp, req) {
// Can only use cached value if the new request doesn't Vary significantly
freshness := getFreshness(cachedResp.Header, req.Header)
if freshness == fresh {
return cachedResp, nil
}
if freshness == stale {
// Add validators if caller hasn't already done so
etag := cachedResp.Header.Get("etag")
if etag != "" && req.Header.Get("etag") == "" {
req.Header.Set("if-none-match", etag)
}
lastModified := cachedResp.Header.Get("last-modified")
if lastModified != "" && req.Header.Get("last-modified") == "" {
req.Header.Set("if-modified-since", lastModified)
}
}
}
resp, err = transport.RoundTrip(req)
if err == nil && req.Method == "GET" && resp.StatusCode == http.StatusNotModified {
// Replace the 304 response with the one from cache, but update with some new headers
headersToMerge := getHopByHopHeaders(resp)
for _, headerKey := range headersToMerge {
cachedResp.Header.Set(headerKey, resp.Header.Get(headerKey))
}
cachedResp.Status = http.StatusText(http.StatusOK)
cachedResp.StatusCode = http.StatusOK
resp = cachedResp
} else {
if err != nil || resp.StatusCode != http.StatusOK {
t.Cache.Delete(cacheKey)
}
if err != nil {
return nil, err
}
}
} else {
reqCacheControl := parseCacheControl(req.Header)
if _, ok := reqCacheControl["only-if-cached"]; ok {
resp = newGatewayTimeoutResponse(req)
} else {
resp, err = transport.RoundTrip(req)
if err != nil {
return nil, err
}
}
}
reqCacheControl := parseCacheControl(req.Header)
respCacheControl := parseCacheControl(resp.Header)
if canStore(reqCacheControl, respCacheControl) {
vary := resp.Header.Get("Vary")
for _, varyKey := range strings.Split(vary, ",") {
varyKey = http.CanonicalHeaderKey(strings.Trim(varyKey, " "))
fakeHeader := "X-Varied-" + varyKey
reqValue := req.Header.Get(varyKey)
if reqValue != "" {
resp.Header.Set(fakeHeader, reqValue)
}
}
respBytes, err := httputil.DumpResponse(resp, true)
if err == nil {
t.Cache.Set(cacheKey, respBytes)
}
} else {
t.Cache.Delete(cacheKey)
}
return resp, nil
}
// ReadResponse reads an HTTP response. The header is taken from h,
// which must include the SPDY-specific fields starting with ':'.
// If r is not nil, the body will be read from r. If t is not nil,
// the trailer will be taken from t after the body is finished.
func ReadResponse(h, t http.Header, r io.Reader, req *http.Request) (*http.Response, error) {
for _, s := range badRespHeaderFields {
if _, ok := h[s]; ok {
return nil, &badStringError{"invalid header field", s}
}
}
var err error
resp := new(http.Response)
resp.Request = req
resp.Close = true
resp.Header = make(http.Header)
copyHeader(resp.Header, h)
f := strings.SplitN(h.Get(":status"), " ", 2)
var s string
if len(f) > 1 {
s = f[1]
}
resp.Status = f[0] + " " + s
resp.StatusCode, err = strconv.Atoi(f[0])
if err != nil {
return nil, &badStringError{"malformed HTTP status code", f[0]}
}
resp.Proto = h.Get(":version")
var ok bool
resp.ProtoMajor, resp.ProtoMinor, ok = http.ParseHTTPVersion(resp.Proto)
if !ok {
return nil, &badStringError{"malformed HTTP version", resp.Proto}
}
realLength, err := fixLength(true, resp.StatusCode, req.Method, resp.Header)
if err != nil {
return nil, err
}
if req.Method == "HEAD" {
if n, err := parseContentLength(h.Get("Content-Length")); err != nil {
return nil, err
} else {
resp.ContentLength = n
}
} else {
resp.ContentLength = realLength
}
switch {
case realLength == 0:
r = eofReader
case realLength > 0:
if r == nil {
// TODO(kr): return error
}
r = io.LimitReader(r, realLength)
}
if r == nil {
r = eofReader
}
body := &body{r: r}
resp.Body = body
if t != nil {
body.hdr = resp
body.trailer = t
}
return resp, nil
}
func filterByUrl(w icap.ResponseWriter, req *icap.Request) {
h := w.Header()
h.Set("ISTag", ISTag)
h.Set("Service", "SquidBlocker filter ICAP service")
if *debug {
fmt.Fprintln(os.Stderr, "Printing the full ICAP request")
fmt.Fprintln(os.Stderr, req)
fmt.Fprintln(os.Stderr, req.Request)
}
switch req.Method {
case "OPTIONS":
h.Set("Methods", "REQMOD, RESPMOD")
h.Set("Options-TTL", "1800")
h.Set("Allow", "204")
h.Set("Preview", "0")
h.Set("Transfer-Preview", "*")
h.Set("Max-Connections", "4000")
h.Set("X-Include", "X-Client-IP, X-Authenticated-Groups, X-Authenticated-User, X-Subscriber-Id")
w.WriteHeader(200, nil, false)
case "REQMOD":
// Check if the method is either OPTIONS\GET\POST\PUT etc
// Also to analyse the request stucutre to verify what is the current one used
// based on the RFC section at: http://tools.ietf.org/html/rfc7230#section-5.3
// Treat the CONNECT method in a special way due to the fact that it cannot actually be modified.
checkhost := ""
port := "0"
answer := *defaultAnswer
var err error
if *debug {
fmt.Fprintln(os.Stderr, "Default CASE. Request to host: "+req.Request.URL.Host+", Request Method: "+req.Request.Method)
fmt.Fprintln(os.Stderr, "The full url from the ICAP client request: "+req.Request.URL.String())
}
checkhost, port, err = net.SplitHostPort(req.Request.URL.Host)
if err != nil {
_ = err
checkhost = req.Request.URL.Host
}
if port != "0" {
if *debug {
fmt.Fprintln(os.Stderr, "Rquest with port: "+port)
}
}
if req.Request.Method == "CONNECT" && len(checkhost) > 0 && port != "0" {
answer = check_tcp(checkhost, port)
} else {
answer = check(req.Request.URL.String())
}
if *debug {
fmt.Fprintln(os.Stderr, "ERRlog: reporting answer size => "+strconv.Itoa(len(answer)))
fmt.Fprintln(os.Stderr, "ERRlog: reporitng answer => "+answer+", for =>"+req.Request.URL.String())
}
// The next part comes to make sure that a DUNO respnse will be handled as the default answer/action
if strings.HasPrefix(answer, "DUNO") {
answer = *defaultAnswer + " rate=100 default_answer=yes"
if *debug {
fmt.Fprintln(os.Stderr, "ERRlog: reporting answer startsWith => \"DUNO\", taking default action")
if len(*defaultAnswer) > 0 {
fmt.Fprintln(os.Stderr, req.Request.URL.String()+" "+*defaultAnswer+" rate=40 default_answer=yes")
} else {
fmt.Fprintln(os.Stderr, req.Request.URL.String()+" OK state=DUNO")
}
}
}
if strings.HasPrefix(answer, "OK") {
if *debug {
fmt.Fprintln(os.Stderr, "OK response and sending 204 back")
}
w.WriteHeader(204, nil, false)
return
}
if strings.HasPrefix(answer, "ERR") {
if *debug {
fmt.Fprintln(os.Stderr, "ERR response and sending 307 redirection back")
}
resp := new(http.Response)
resp.Status = "307 SquidBlocker this url has been filtered!"
resp.StatusCode = 307
resp.Proto = "HTTP/1.1"
resp.ProtoMajor = 1
resp.ProtoMinor = 1
myMap := make(map[string][]string)
//What if it is a connect request
myMap["Location"] = append(myMap["Location"], *block_page+"?url="+url.QueryEscape(req.Request.URL.String()))
resp.Header = myMap
//resp.Body = ioutil.NopCloser(bytes.NewBufferString(body))
//resp.ContentLength = int64(len(body))
resp.Request = req.Request
w.WriteHeader(200, resp, true)
return
}
if *debug {
fmt.Fprintln(os.Stderr, "Unknown asnwer and scenario, not adapting the request")
}
w.WriteHeader(204, nil, false)
return
case "RESPMOD":
w.WriteHeader(204, nil, false)
default:
w.WriteHeader(405, nil, false)
if *debug {
fmt.Fprintln(os.Stderr, "Invalid request method")
}
}
}
// RoundTrip takes a Request and returns a Response
//
// If there is a fresh Response already in cache, then it will be returned without connecting to
// the server.
//
// If there is a stale Response, then any validators it contains will be set on the new request
// to give the server a chance to respond with NotModified. If this happens, then the cached Response
// will be returned.
func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
cacheKey := cacheKey(req)
cacheableMethod := req.Method == "GET" || req.Method == "HEAD"
var cachedResp *http.Response
if cacheableMethod {
cachedResp, err = CachedResponse(t.Cache, req)
} else {
// Need to invalidate an existing value
t.Cache.Delete(cacheKey)
}
transport := t.Transport
if transport == nil {
transport = http.DefaultTransport
}
if cachedResp != nil && err == nil && cacheableMethod && req.Header.Get("range") == "" {
if t.MarkCachedResponses {
cachedResp.Header.Set(XFromCache, "1")
}
if varyMatches(cachedResp, req) {
// Can only use cached value if the new request doesn't Vary significantly
freshness := getFreshness(cachedResp.Header, req.Header)
if freshness == fresh {
return cachedResp, nil
}
if freshness == stale {
var req2 *http.Request
// Add validators if caller hasn't already done so
etag := cachedResp.Header.Get("etag")
if etag != "" && req.Header.Get("etag") == "" {
req2 = cloneRequest(req)
req2.Header.Set("if-none-match", etag)
}
lastModified := cachedResp.Header.Get("last-modified")
if lastModified != "" && req.Header.Get("last-modified") == "" {
if req2 == nil {
req2 = cloneRequest(req)
}
req2.Header.Set("if-modified-since", lastModified)
}
if req2 != nil {
// Associate original request with cloned request so we can refer to
// it in CancelRequest()
t.setModReq(req, req2)
req = req2
defer func() {
// Release req/clone mapping on error
if err != nil {
t.setModReq(req, nil)
}
if resp != nil {
// Release req/clone mapping on body close/EOF
resp.Body = &onEOFReader{
rc: resp.Body,
fn: func() { t.setModReq(req, nil) },
}
}
}()
}
}
}
resp, err = transport.RoundTrip(req)
if err == nil && req.Method == "GET" && resp.StatusCode == http.StatusNotModified {
// Replace the 304 response with the one from cache, but update with some new headers
endToEndHeaders := getEndToEndHeaders(resp.Header)
for _, header := range endToEndHeaders {
cachedResp.Header[header] = resp.Header[header]
}
cachedResp.Status = fmt.Sprintf("%d %s", http.StatusOK, http.StatusText(http.StatusOK))
cachedResp.StatusCode = http.StatusOK
resp = cachedResp
} else if (err != nil || (cachedResp != nil && resp.StatusCode >= 500)) &&
req.Method == "GET" && canStaleOnError(cachedResp.Header, req.Header) {
// In case of transport failure and stale-if-error activated, returns cached content
// when available
cachedResp.Status = fmt.Sprintf("%d %s", http.StatusOK, http.StatusText(http.StatusOK))
cachedResp.StatusCode = http.StatusOK
return cachedResp, nil
} else {
if err != nil || resp.StatusCode != http.StatusOK {
t.Cache.Delete(cacheKey)
}
if err != nil {
if t.ResponseErrorFunc != nil {
return t.ResponseErrorFunc(resp, err, req)
}
return nil, err
}
}
} else {
reqCacheControl := parseCacheControl(req.Header)
if _, ok := reqCacheControl["only-if-cached"]; ok {
resp = newGatewayTimeoutResponse(req)
} else {
resp, err = transport.RoundTrip(req)
if err != nil {
if t.ResponseErrorFunc != nil {
return t.ResponseErrorFunc(resp, err, req)
}
return nil, err
}
}
}
reqCacheControl := parseCacheControl(req.Header)
respCacheControl := parseCacheControl(resp.Header)
if canStore(reqCacheControl, respCacheControl) {
for _, varyKey := range headerAllCommaSepValues(resp.Header, "vary") {
varyKey = http.CanonicalHeaderKey(varyKey)
fakeHeader := "X-Varied-" + varyKey
reqValue := req.Header.Get(varyKey)
if reqValue != "" {
resp.Header.Set(fakeHeader, reqValue)
}
}
respBytes, err := httputil.DumpResponse(resp, true)
if err == nil {
t.Cache.Set(cacheKey, respBytes)
}
} else {
t.Cache.Delete(cacheKey)
}
return resp, nil
}
// ModifyResponse overwrites the status text and code on an HTTP response and
// returns nil.
func (s *statusModifier) ModifyResponse(res *http.Response) error {
res.StatusCode = s.statusCode
res.Status = fmt.Sprintf("%d %s", s.statusCode, http.StatusText(s.statusCode))
return nil
}
func (r *Response) Response() *http.Response {
out := new(http.Response)
r.headerM.Lock()
out.Status = fmt.Sprintf("%d %s", r.StatusCode, http.StatusText(r.StatusCode))
out.StatusCode = r.StatusCode
out.Header = r.Header
r.headerM.Unlock()
out.Proto = "HTTP/1.1"
out.ProtoMajor = 1
out.ProtoMinor = 1
r.dataM.Lock()
if r.data == nil {
out.Body = &ReadCloser{new(bytes.Buffer)}
} else if unrequestedGzip(r) {
// User-agents MUST support gzip compression.
// Regardless of the Accept-Encoding sent by the user-agent, the server may
// always send content encoded with gzip or deflate encoding.
r.data.Prep()
out.Header.Del("Content-Encoding")
out.Header.Del("Content-Length")
out.ContentLength = -1
out.Body = &gzipReader{body: r.data}
} else {
r.data.Prep()
out.Body = r.data
out.ContentLength = r.data.written
}
r.dataM.Unlock()
out.TransferEncoding = nil
out.Close = true
out.Trailer = make(http.Header)
out.Request = r.Request
return out
}