Skip to content

Commit

Permalink
improve: remove ip ratelimiting, set limits more tolerable
Browse files Browse the repository at this point in the history
  • Loading branch information
@bobhageman
bobhageman committed Sep 6, 2023
1 parent 1161e6f commit bf65c51
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 153 deletions.
14 changes: 2 additions & 12 deletions src/main/java/foundation/privacybydesign/email/EmailRestApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public class EmailRestApi {
private static final String ERR_INVALID_TOKEN = "error:invalid-token";
private static final String ERR_INVALID_LANG = "error:invalid-language";
private static final String OK_RESPONSE = "OK"; // value doesn't really matter
private static final String PROXY_IP_HEADER = "X-Real-IP";
private static final String ERR_RATE_LIMITED = "error:ratelimit";

private EmailTokens signer;
Expand Down Expand Up @@ -67,12 +66,8 @@ public Response sendEmail(@Context HttpServletRequest req,
}

try {
String ip = req.getHeader(PROXY_IP_HEADER);
if (ip == null) {
ip = req.getRemoteAddr();
}

long retryAfter = rateLimiter.rateLimited(ip, email);
long retryAfter = rateLimiter.rateLimited(email);
if (retryAfter > 0) {
// 429 Too Many Requests
// https://tools.ietf.org/html/rfc6585#section-4
Expand Down Expand Up @@ -128,12 +123,7 @@ public Response sendEmailToken(@Context HttpServletRequest req,
return Response.status(Response.Status.BAD_REQUEST).entity(ERR_ADDRESS_MALFORMED).build();
}

String ip = req.getHeader(PROXY_IP_HEADER);
if (ip == null) {
ip = req.getRemoteAddr();
}

long retryAfter = rateLimiter.rateLimited(ip, emailAddress);
long retryAfter = rateLimiter.rateLimited(emailAddress);
if (retryAfter > 0) {
// 429 Too Many Requests
// https://tools.ietf.org/html/rfc6585#section-4
Expand Down
83 changes: 20 additions & 63 deletions src/main/java/foundation/privacybydesign/email/ratelimit/MemoryRateLimit.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@
*
* How it works:
* How much budget a user has, is expressed in a timestamp. The timestamp is
* initially some period in the past, but with every usage (countIP and
* countPhone) this timestamp is incremented. For IP address counting, this
* is a fixed amount (currently 10 seconds), but for phone numbers this
* initially some period in the past, but with every usage (countEmail)
* this timestamp is incremented. For e-mail addresses this
* amount is exponential.
*
* An algorithm with a similar goal is the Token Bucket algorithm. This
Expand All @@ -23,16 +22,12 @@ public class MemoryRateLimit extends RateLimit {
private static final long MINUTE = SECOND * 60;
private static final long HOUR = MINUTE * 60;
private static final long DAY = HOUR * 24;
private static final int IP_TIMEOUT = 10 * 1000; // timeout in seconds
private static final int IP_TRIES = 3; // number of tries on first visit

private static MemoryRateLimit instance;

private final Map<String, Long> ipLimits;
private final Map<String, Limit> emailLimits;

public MemoryRateLimit() {
ipLimits = new ConcurrentHashMap<>();
emailLimits = new ConcurrentHashMap<>();
}

Expand All @@ -43,53 +38,17 @@ public static MemoryRateLimit getInstance() {
return instance;
}

private long startLimitIP(long now) {
return now - IP_TIMEOUT*IP_TRIES;
}

@Override
protected synchronized long nextTryIP(String ip, long now) {
// Allow at most 1 try in each period (TIMEOUT), but kick in only
// after 3 tries. Thus while the user can do only 1 try per period
// over longer periods, the initial budget is 3 periods.
long limit = 0; // First try - last try was "long in the past".
if (ipLimits.containsKey(ip)) {
// Ah, there was a request before.
limit = ipLimits.get(ip);
}

long startLimit = startLimitIP(now);
if (limit < startLimit) {
// First visit or previous visit was long ago.
// Act like the last try was 3 periods ago.
limit = startLimit;
}

// Add a period to the current limit.
limit += IP_TIMEOUT;
return limit;
}

@Override
protected synchronized void countIP(String ip, long now) {
long nextTry = nextTryIP(ip, now);
if (nextTry > now) {
throw new IllegalStateException("counting rate limit while over the limit");
}
ipLimits.put(ip, nextTry);
}

// Is the user over the rate limit per e-mail address?
@Override
protected synchronized long nextTryEmail(String email, long now) {
// Rate limiter durations (sort-of logarithmic):
// 1 10 second
// 2 5 minute
// 3 1 hour
// 4 24 hour
// 5+ 1 per day
// Keep log 5 days for proper limiting.

// 1 10 seconds
// 2 1 minute
// 3 5 minutes
// 4 1 hour
// 5 12 hours
// 6+ 2 per day
// Keep log 2 days for proper limiting.
Limit limit = emailLimits.get(email);
if (limit == null) {
limit = new Limit(now);
Expand All @@ -103,14 +62,17 @@ protected synchronized long nextTryEmail(String email, long now) {
case 1: // try 2: allowed after 10 seconds
nextTry = limit.timestamp + 10 * SECOND;
break;
case 2: // try 3: allowed after 5 minutes
case 2: // try 3: allowed after 1 minute
nextTry = limit.timestamp + MINUTE;
break;
case 3: // try 4: allowed after 5 minutes
nextTry = limit.timestamp + 5 * MINUTE;
break;
case 3: // try 4: allowed after 3 hours
nextTry = limit.timestamp + 3 * HOUR;
case 4: // try 5: allowed after 1 hour
nextTry = limit.timestamp + HOUR;
break;
case 4: // try 5: allowed after 24 hours
nextTry = limit.timestamp + 24 * HOUR;
case 5: // try 6: allowed after 12 hours
nextTry = limit.timestamp + 12 * HOUR;
break;
default:
throw new IllegalStateException("invalid tries count");
Expand All @@ -127,8 +89,8 @@ protected synchronized void countEmail(String email, long now) {
if (nextTry > now) {
throw new IllegalStateException("counting rate limit while over the limit");
}
limit.tries = Math.min(limit.tries+1, 5); // add 1, max at 5
// If the last usage was e.g. ≥2 days ago, we should allow them 2 tries
limit.tries = Math.min(limit.tries+1, 6); // add 1, max at 6
// If the last usage was e.g. ≥2 days ago, we should allow them 2
// extra tries this day.
long lastTryDaysAgo = (now-limit.timestamp)/DAY;
long bonusTries = limit.tries - lastTryDaysAgo;
Expand All @@ -141,13 +103,8 @@ protected synchronized void countEmail(String email, long now) {
public void periodicCleanup() {
long now = System.currentTimeMillis();
// Use enhanced for loop, because an iterator makes sure concurrency issues cannot occur.
for (Map.Entry<String, Long> entry : ipLimits.entrySet()) {
if (entry.getValue() < startLimitIP(now)) {
ipLimits.remove(entry.getKey());
}
}
for (Map.Entry<String, Limit> entry : emailLimits.entrySet()) {
if (entry.getValue().timestamp < now - 5*DAY) {
if (entry.getValue().timestamp < now - 2*DAY) {
emailLimits.remove(entry.getKey());
}
}
Expand Down
54 changes: 6 additions & 48 deletions src/main/java/foundation/privacybydesign/email/ratelimit/RateLimit.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,74 +3,32 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.InetAddress;
import java.net.UnknownHostException;

/**
* Base class for rate limiting. Subclasses provide storage methods (memory
* for easier debugging and database for production).
*/
public abstract class RateLimit {
private static Logger logger = LoggerFactory.getLogger(RateLimit.class);

/** Take an IP address and an e-mail address and rate limit them.
* @param remoteAddr IP address (IPv4 or IPv6 in any format)
/** Take an e-mail address and rate limit it.
* @param email e-mail address
* @return the number of milliseconds that the client should wait - 0 if
* it shouldn't wait.
*/
public long rateLimited(String remoteAddr, String email) {
String addr = getAddressPrefix(remoteAddr);
public long rateLimited(String email) {
long now = System.currentTimeMillis();
long ipRetryAfter = nextTryIP(addr, now);
long phoneRetryAfter = nextTryEmail(email, now);
long retryAfter = Math.max(ipRetryAfter, phoneRetryAfter);
long retryAfter = nextTryEmail(email, now);

if (retryAfter > now) {
logger.warn("Denying request from {}: rate limit (ip and/or email) exceeded", addr);
logger.warn("Denying request: email rate limit email exceeded");
// Don't count this request if it has been denied.
return retryAfter - now;
}
countIP(addr, now);

countEmail(email, now);
return 0;
}

/** Insert an IP address (IPv4 or IPv6) and get a canonicalized version.
* For IPv6, also truncate to /56 (recommended residential block).
*
* This is a public method to ease testing.
*/
public static String getAddressPrefix(String remoteAddr) {
byte[] rawAddr;
try {
InetAddress addr = InetAddress.getByName(remoteAddr);
rawAddr = addr.getAddress();
} catch (UnknownHostException e) {
// Shouldn't be possible - we're using IP addresses here, not
// host names.
throw new RuntimeException("host name lookup on IP address?");
}
if (rawAddr.length == 4) { // IPv4
// take the whole IP address
} else if (rawAddr.length == 16) { // IPv6
// Use only the first /56 bytes, set the rest to 0.
for (int i=7; i<16; i++) {
rawAddr[i] = 0;
}
} else {
// I hope this will never happen.
throw new RuntimeException("no IPv4 or IPv6?");
}
try {
return InetAddress.getByAddress(rawAddr).getHostAddress();
} catch (UnknownHostException e) {
// Should Not Happen™
throw new RuntimeException("host name lookup on IP address?");
}
}

protected abstract long nextTryIP(String ip, long now);
protected abstract long nextTryEmail(String email, long now);
protected abstract void countIP(String ip, long now);
protected abstract void countEmail(String email, long now);
}
39 changes: 9 additions & 30 deletions src/test/java/foundation/privacybydesign/email/RateLimitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,45 +2,24 @@

import org.junit.Test;

import foundation.privacybydesign.email.ratelimit.RateLimit;
import foundation.privacybydesign.email.ratelimit.MemoryRateLimit;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotEquals;

/**
* Test whether the canonicalization algorithm for IP addresses works well.
* Test whether the rate limiter works as expected.
*/
public class RateLimitTest {
@Test
public void testIPv4Address() {
assertEquals("1.2.3.4",
RateLimit.getAddressPrefix("1.2.3.4"));
}

@Test
public void testIPv6AddressLocal() {
assertEquals("0:0:0:0:0:0:0:0",
RateLimit.getAddressPrefix("0:0:0:0:0:0:0:1"));
}
public void testRateLimit() {

@Test
public void testIPv6AddressRegular() {
assertEquals("2a00:123:4567:8900:0:0:0:0",
RateLimit.getAddressPrefix("2a00:0123:4567:89ab:cdef:fedc:ba98:7654"));
}
var rateLimit = MemoryRateLimit.getInstance();
var email = "[email protected]";

@Test
public void testIPv6Equals() {
String addr1 = RateLimit.getAddressPrefix("2a00:0123:4567:89ab:cdef:fedc:ba98:7654");
String addr2 = RateLimit.getAddressPrefix("2a00:0123:4567:89ac:cdef:fedc:ba98:7654");
assertTrue("IPv6 addresses should match", addr1.equals(addr2));
}

@Test
public void testIPv6Unequals() {
String addr1 = RateLimit.getAddressPrefix("2a00:0123:4567:89ab:cdef:fedc:ba98:7654");
String addr2 = RateLimit.getAddressPrefix("2a00:0123:4567:88ab:cdef:fedc:ba98:7654");
assertFalse("IPv6 addresses should not match", addr1.equals(addr2));
assertEquals("not rate limited", 0, rateLimit.rateLimited(email));

assertNotEquals("rate limited", 0, rateLimit.rateLimited(email));
}
}

0 comments on commit bf65c51

Please sign in to comment.