diff --git a/imageproxy.go b/imageproxy.go
index b61f730..a7fdc55 100644
--- a/imageproxy.go
+++ b/imageproxy.go
@@ -315,12 +315,13 @@ func (t *TransformingTransport) RoundTrip(req *http.Request) (*http.Response, er
 		return nil, err
 	}
 
+	defer resp.Body.Close()
+
 	if should304(req, resp) {
 		// bare 304 response, full response will be used from cache
 		return &http.Response{StatusCode: http.StatusNotModified}, nil
 	}
 
-	defer resp.Body.Close()
 	b, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err