PHRAS-3588 implement http request quota by type (#4564)

* PHRAS-3588 manage http request limits by verbs

* PHRAS-3588 change limit method

* re-introduce burst parameters

* PHRAS-3588 add activation boolean

* applying auto-documentation format for env var

.env

@@ -224,6 +224,41 @@ GATEWAY_DENIED_IPS=
 # @run
 GATEWAY_USERS=
 
+
+
+# Manage http incoming request limits by verbs
+# this feature is based on ip adresses and need PHRASEANET_TRUSTED_PROXIES
+# defined to get real_ip
+# READ is for GET and HEAD requests
+# WRITE is for POST, PUT, DELETE and PATCH requests
+# Enabling the requests Limit
+# @run
+HTTP_REQUEST_LIMITS=false
+
+# (m) For Exemple 16,000 IP addresses takes 1 megabyte, so our zone can store about 160,000 addresses.
+# @run
+HTTP_READ_REQUEST_LIMIT_MEMORY=10
+
+# (r/s) Sets the maximum request rate. By default here the rate cannot exceed 10 requests per second
+# @run
+HTTP_READ_REQUEST_LIMIT_RATE=100
+
+# The burst parameter defines how many requests a client can make in excess of the rate specified
+# @run
+HTTP_READ_REQUEST_LIMIT_BURST=20
+
+# (m) For Exemple 16,000 IP addresses takes 1 megabyte, so our zone can store about 160,000 addresses.
+# @run
+HTTP_WRITE_REQUEST_LIMIT_MEMORY=10
+
+# (r/s) Sets the maximum request rate. By default here the rate cannot exceed 10 requests per second
+# @run
+HTTP_WRITE_REQUEST_LIMIT_RATE=100
+
+# The burst parameter defines how many requests a client can make in excess of the rate specified
+# @run
+HTTP_WRITE_REQUEST_LIMIT_BURST=20
+
 # https and reverse proxy (on/off)
 # set to on in the case : https behind a proxy
 # @run

docker-compose.yml

@@ -36,6 +36,14 @@ services:
     - GATEWAY_DENIED_IPS 
     - GATEWAY_USERS
     - GATEWAY_CSP
+    - HTTP_REQUEST_LIMITS
+    - HTTP_READ_REQUEST_LIMIT_MEMORY
+    - HTTP_READ_REQUEST_LIMIT_RATE
+    - HTTP_WRITE_REQUEST_LIMIT_MEMORY
+    - HTTP_WRITE_REQUEST_LIMIT_RATE
+    - HTTP_READ_REQUEST_LIMIT_BURST
+    - HTTP_WRITE_REQUEST_LIMIT_BURST
+
     ports:
       - ${PHRASEANET_APP_PORT}:80
     networks:

docker/nginx/root/entrypoint.sh

@@ -35,7 +35,15 @@ else
   envsubst < "/securitycontentpolicies.sample.conf" > /etc/nginx/conf.d/securitycontentpolicies.conf
 fi
 
-cat /nginx.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_SEND_TIMEOUT/$GATEWAY_SEND_TIMEOUT/g"  | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_PROXY_TIMEOUT/$GATEWAY_PROXY_TIMEOUT/g"  | sed "s/\$NEW_TARGET/$NEW_TARGET/g"  | sed "s/\$NEW_RESOLVER/$NEW_RESOLVER/g" | sed "s/\$GATEWAY_FASTCGI_HTTPS/$GATEWAY_FASTCGI_HTTPS/g" > /etc/nginx/conf.d/default.conf
+
+if [[ $HTTP_REQUEST_LIMITS && $HTTP_REQUEST_LIMITS = true ]] && [[ ! -z $HTTP_READ_REQUEST_LIMIT_MEMORY || ! -z $HTTP_READ_REQUEST_LIMIT_RATE || ! -z $HTTP_READ_REQUEST_LIMIT_BURST || ! -z $HTTP_WRITE_REQUEST_LIMIT_MEMORY || ! -z $HTTP_WRITE_REQUEST_LIMIT_RATE || ! -z $HTTP_WRITE_REQUEST_LIMIT_BURST ]]; then
+  echo "HTTP_REQUEST_LIMITS is $HTTP_REQUEST_LIMITS"
+  cat /nginx.request_limits.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_SEND_TIMEOUT/$GATEWAY_SEND_TIMEOUT/g"  | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_PROXY_TIMEOUT/$GATEWAY_PROXY_TIMEOUT/g"  | sed "s/\$NEW_TARGET/$NEW_TARGET/g"  | sed "s/\$NEW_RESOLVER/$NEW_RESOLVER/g" | sed "s/\$GATEWAY_FASTCGI_HTTPS/$GATEWAY_FASTCGI_HTTPS/g" | sed "s/\$HTTP_READ_REQUEST_LIMIT_MEMORY/$HTTP_READ_REQUEST_LIMIT_MEMORY/g"  | sed "s/\$HTTP_READ_REQUEST_LIMIT_RATE/$HTTP_READ_REQUEST_LIMIT_RATE/g" | sed "s/\$HTTP_WRITE_REQUEST_LIMIT_MEMORY/$HTTP_WRITE_REQUEST_LIMIT_MEMORY/g" | sed "s/\$HTTP_WRITE_REQUEST_LIMIT_RATE/$HTTP_WRITE_REQUEST_LIMIT_RATE/g" | sed "s/\$HTTP_READ_REQUEST_LIMIT_BURST/$HTTP_READ_REQUEST_LIMIT_BURST/g"| sed "s/\$HTTP_WRITE_REQUEST_LIMIT_BURST/$HTTP_WRITE_REQUEST_LIMIT_BURST/g" > /etc/nginx/conf.d/default.conf
+else
+  echo "HTTP_REQUEST_LIMITS is $HTTP_REQUEST_LIMITS or not defined"
+  cat /nginx.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_SEND_TIMEOUT/$GATEWAY_SEND_TIMEOUT/g"  | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_PROXY_TIMEOUT/$GATEWAY_PROXY_TIMEOUT/g"  | sed "s/\$NEW_TARGET/$NEW_TARGET/g"  | sed "s/\$NEW_RESOLVER/$NEW_RESOLVER/g" | sed "s/\$GATEWAY_FASTCGI_HTTPS/$GATEWAY_FASTCGI_HTTPS/g" > /etc/nginx/conf.d/default.conf
+fi
+
 cat /fastcgi_timeout.conf  | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" > /etc/nginx/fastcgi_extended_params
 
 echo `date +"%Y-%m-%d %H:%M:%S"` " - Setting for real_ip_from using Trusted Proxies"

docker/nginx/root/nginx.conf.sample

@@ -1,10 +1,12 @@
+
 send_timeout $GATEWAY_SEND_TIMEOUT;
 keepalive_timeout           $GATEWAY_SEND_TIMEOUT;
 proxy_connect_timeout       $GATEWAY_PROXY_TIMEOUT;
 proxy_send_timeout          $GATEWAY_PROXY_TIMEOUT;
 client_header_timeout       $GATEWAY_SEND_TIMEOUT;
 client_body_timeout         $GATEWAY_SEND_TIMEOUT;
 fastcgi_read_timeout        $GATEWAY_FASTCGI_TIMEOUT;
+
 resolver $NEW_RESOLVER;
 
 upstream backend {
@@ -36,7 +38,6 @@ server {
         if (-f /var/alchemy/Phraseanet/datas/nginx/maintenance.html) {
             return 503;
         }
-
         # First attempt to serve request as file, then
         # as directory, then fall back to index.html
         try_files $uri $uri/ @rewriteapp;

docker/nginx/root/nginx.request_limits.conf.sample

@@ -0,0 +1,95 @@
+
+send_timeout $GATEWAY_SEND_TIMEOUT;
+keepalive_timeout           $GATEWAY_SEND_TIMEOUT;
+proxy_connect_timeout       $GATEWAY_PROXY_TIMEOUT;
+proxy_send_timeout          $GATEWAY_PROXY_TIMEOUT;
+client_header_timeout       $GATEWAY_SEND_TIMEOUT;
+client_body_timeout         $GATEWAY_SEND_TIMEOUT;
+fastcgi_read_timeout        $GATEWAY_FASTCGI_TIMEOUT;
+
+map $request_method $postlimit {
+  default         "";
+  POST            $binary_remote_addr;
+}
+
+map $request_method $getlimit {
+  default         "";
+  GET            $binary_remote_addr;
+}
+
+limit_req_status 429;
+limit_req_zone $getlimit zone=readlimitsbyip:$HTTP_READ_REQUEST_LIMIT_MEMORYm rate=$HTTP_READ_REQUEST_LIMIT_RATEr/s;
+limit_req_zone $postlimit zone=writelimitsbyip:$HTTP_WRITE_REQUEST_LIMIT_MEMORYm rate=$HTTP_WRITE_REQUEST_LIMIT_RATEr/s;
+resolver $NEW_RESOLVER;
+
+upstream backend {
+    server phraseanet:9000;
+}
+
+#upstream samlsp {
+#    server phraseanet-saml-sp:8080;
+#}
+
+server {
+    listen       80;
+    root         /var/alchemy/Phraseanet/www;
+
+    index        index.php;
+    client_max_body_size $MAX_BODY_SIZE;
+
+    location /api {
+        if (-f /var/alchemy/Phraseanet/datas/nginx/maintenance.html) {
+            return 503;
+        }
+        rewrite ^(.*)$ /api.php/$1 last;
+    }
+
+    location / {
+
+        error_page 503 = @maintenance;
+        recursive_error_pages on;
+        if (-f /var/alchemy/Phraseanet/datas/nginx/maintenance.html) {
+            return 503;
+        }
+        # First attempt to serve request as file, then
+        # as directory, then fall back to index.html
+        try_files $uri $uri/ @rewriteapp;
+        limit_req zone=readlimitsbyip burst=$HTTP_READ_REQUEST_LIMIT_BURST nodelay;
+        limit_req zone=writelimitsbyip burst=$HTTP_WRITE_REQUEST_LIMIT_BURST nodelay;
+    }
+
+    location @rewriteapp {
+        rewrite ^(.*)$ /index.php/$1 last;
+    }
+
+    # PHP scripts -> PHP-FPM server listening on 127.0.0.1:9000
+    location ~ ^/(index|index_dev|api)\.php(/|$) {
+        fastcgi_pass   backend;
+        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
+        include        fastcgi_params;
+        $GATEWAY_FASTCGI_HTTPS
+        include        restrictions;
+    }
+
+    location ~ ^/(status|ping)$ {
+        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        fastcgi_index index.php;
+        include fastcgi_params;
+        include fastcgi_extended_params;
+        fastcgi_pass backend;
+    }
+
+    location  /simplesaml/ {
+        proxy_redirect off;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        set $target $NEW_TARGET:8080;
+        proxy_pass http://$target;
+
+    }
+
+    location @maintenance {
+        root /var/alchemy/Phraseanet/datas/nginx/;
+        try_files $uri /maintenance.html;
+    }
+}