browsertrix: Remove page limit

The time and size limit should be enough and this is restricting large crawls too much.
nla · Nov 21, 2024 · d7cbd63 · d7cbd63
1 parent b1b3e8b
commit d7cbd63
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 10 deletions.
diff --git a/gatherer/src/pandas/gatherer/heritrix/BrowsertrixConfig.java b/gatherer/src/pandas/gatherer/heritrix/BrowsertrixConfig.java
@@ -7,7 +7,6 @@
 @ConfigurationProperties(prefix = "browsertrix")
 public class BrowsertrixConfig {
     private String podmanOptions;
-    private int pageLimit = 1000;
     private String userAgentSuffix = "nla.gov.au_bot (National Library of Australia Legal Deposit Request; +http://www.nla.gov.au/legal-deposit/request)";
     private int workers = 4;
 
@@ -24,14 +23,6 @@ public void setPodmanOptions(String podmanOptions) {
         this.podmanOptions = podmanOptions;
     }
 
-    public int getPageLimit() {
-        return pageLimit;
-    }
-
-    public void setPageLimit(int pageLimit) {
-        this.pageLimit = pageLimit;
-    }
-
     public String getUserAgentSuffix() {
         return userAgentSuffix;
     }

diff --git a/gatherer/src/pandas/gatherer/heritrix/BrowsertrixGatherer.java b/gatherer/src/pandas/gatherer/heritrix/BrowsertrixGatherer.java
@@ -82,7 +82,6 @@ public void gather(Instance instance) throws Exception {
                 "--generatecdx",
                 "--logging", "none",
                 "--saveState", "always",
-                "--limit", String.valueOf(config.getPageLimit()),
                 "--depth", String.valueOf(depth)));
 
         if (scope != null && scope.isIncludeSubdomains()) {