Added some parallel-queues sheets.

Radtoo · Dec 9, 2019 · 035fc00 · 035fc00
1 parent 1451097
commit 035fc00
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 1 deletion.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -111,7 +111,7 @@ services:
 
   # Web page rendering service
   webrender:
-    image: ukwa/webrender-api:1.0.7
+    image: ukwa/webrender-api:1.0.10
     environment: # DOCKER_HOST could be used to run rendering on a dedicated service.
       - "WARCPROX=warcprox:8000"
       - "DOCKER_NETWORK=ukwa-heritrix_default" # warcprox network, to attach renderer containers to

diff --git a/jobs/frequent/crawler-beans.cxml b/jobs/frequent/crawler-beans.cxml
@@ -817,6 +817,7 @@
 		<property name="forceQueueAssignment" value="" />
 		<property name="deferToPrevious" value="true" />
 		<property name="parallelQueues" value="1" />
+        <property name="parallelQueuesRandomAssignment" value="true"/>
 	</bean>
 
 	<!-- URI PRECEDENCE POLICY -->

diff --git a/jobs/frequent/sheets.xml b/jobs/frequent/sheets.xml
@@ -451,4 +451,30 @@
       </property>
    </bean>
 
+   <!-- Sheets to support running parallel queue for some hosts -->
+   <bean id="parallel-queues-2" class="org.archive.spring.Sheet">
+      <property name="map">
+         <map>
+            <entry key="queueAssignmentPolicy.parallelQueues" value="2" />
+            <entry key="queueAssignmentPolicy.parallelQueuesRandomAssignment" value="true" />
+         </map>
+      </property>
+   </bean>
+   <bean id="parallel-queues-4" class="org.archive.spring.Sheet">
+      <property name="map">
+         <map>
+            <entry key="queueAssignmentPolicy.parallelQueues" value="4" />
+            <entry key="queueAssignmentPolicy.parallelQueuesRandomAssignment" value="true" />
+         </map>
+      </property>
+   </bean>
+   <bean id="parallel-queues-8" class="org.archive.spring.Sheet">
+      <property name="map">
+         <map>
+            <entry key="queueAssignmentPolicy.parallelQueues" value="8" />
+            <entry key="queueAssignmentPolicy.parallelQueuesRandomAssignment" value="true" />
+         </map>
+      </property>
+   </bean>
+
 </beans>