diff --git a/imageproxy.go b/imageproxy.go
index cb4724b..ce671db 100644
--- a/imageproxy.go
+++ b/imageproxy.go
@@ -197,7 +197,7 @@ func (p *Proxy) serveImage(w http.ResponseWriter, r *http.Request) {
 				}
 				return errTooManyRedirects
 			}
-			if hostMatches(p.DenyHosts, newreq.URL) || (len(p.AllowHosts) > 0 && !hostMatches(p.AllowHosts, newreq.URL)) {
+			if hostMatches(p.DenyHosts, newreq.URL) {
 				http.Error(w, msgNotAllowedInRedirect, http.StatusForbidden)
 				return errNotAllowed
 			}