Adding support for using precomputed files to more requests (#58)

* Added time-map * Added time-map-fast * Added time-filter * Added time-filter-proto * Formatting and naming * Removing support for unique requests percentage * README + float to int * more README * fix
traveltime-dev · May 7, 2024 · 5eda78f · 5eda78f
1 parent d08a67e
commit 5eda78f
Show file tree

Hide file tree

Showing 7 changed files with 198 additions and 70 deletions.
diff --git a/README.md b/README.md
@@ -22,10 +22,11 @@ docker run
     -e COUNTRY=gb //optional
     -e COORDINATES=-8,13 // optional, this will overwrite COUNTRY env var
     -e TRANSPORTATION='driving+ferry' //optional
+    -e DATA_PATH='../precomputed/origins.csv' // optional, this will read coordinates from a file instead of randomly generating them
     -e TRAVEL_TIME=7200 //optional
     -e RPM=60 // optional
     -e TEST_DURATION=3 //optional, benchmark duration in minutes (not including warmup)
-    -e UNIQUE_REQUESTS=2 // optional float, percentage of requests that should be unique
+    -e UNIQUE_REQUESTS=100 //optional int, the number of unique requests that should be generated
     -ti igeolise/traveltime-k6-benchmarks:latest k6 run scripts/time-map.js
 ```
 
@@ -39,12 +40,13 @@ docker run
     -e COUNTRY=gb //optional
     -e COORDINATES=-8,13 // optional, this will overwrite COUNTRY env var
     -e TRANSPORTATION='driving+ferry' //optional
+    -e DATA_PATH='../precomputed/origins.csv' // optional, this will read coordinates from a file instead of randomly generating them
     -e TRAVEL_TIME=7200 //optional
     -e LEVEL_OF_DETAILS=2 // optional
     -e RPM=60 // optional
     -e TEST_DURATION=3 //optional, benchmark duration in minutes (not including warmup)
     -e ARRIVAL_TIME_PERIOD='weekday_morning' //optional
-    -e UNIQUE_REQUESTS=2 // optional float, percentage of requests that should be unique
+    -e UNIQUE_REQUESTS=100 //optional int, the number of unique requests that should be generated
     -ti igeolise/traveltime-k6-benchmarks:latest k6 run scripts/time-map.js
 ```
 
@@ -58,12 +60,13 @@ docker run
     -e COUNTRY=gb //optional
     -e COORDINATES=-8,13 // optional, this will overwrite COUNTRY env var
     -e TRANSPORTATION='driving+ferry' //optional
+    -e DATA_PATH='../precomputed/origins.csv' // optional, this will read coordinates from a file instead of randomly generating them
     -e TRAVEL_TIME=7200 //optional
     -e DESTINATIONS=50 // optional
     -e RANGE=600 //optional
     -e RPM=60 // optional
     -e TEST_DURATION=3 //optional, benchmark duration in minutes (not including warmup)
-    -e UNIQUE_REQUESTS=2 // optional float, percentage of requests that should be unique
+    -e UNIQUE_REQUESTS=100 //optional int, the number of unique requests that should be generated
     -ti igeolise/traveltime-k6-benchmarks:latest k6 run scripts/time-filter.js
 ```
 
@@ -80,7 +83,7 @@ docker run
     -e DATA_PATH='../precomputed/routes.csv' // optional, this will read coordinates from a file instead of randomly generating them
     -e RPM=60 // optional
     -e TEST_DURATION=3 //optional, benchmark duration in minutes (not including warmup)
-    -e UNIQUE_REQUESTS=2 // optional float, percentage of requests that should be unique
+    -e UNIQUE_REQUESTS=100 //optional int, the number of unique requests that should be generated
     -ti igeolise/traveltime-k6-benchmarks:latest k6 run scripts/routes.js
 ```
 
@@ -94,12 +97,13 @@ docker run
     -e MANY_TO_ONE // optional
     -e HOST=proto.api.traveltimeapp.com // optional
     -e TRANSPORTATION=driving+ferry // optional
+    -e DATA_PATH='../precomputed/origins.csv' // optional, this will read coordinates from a file instead of randomly generating them
     -e COORDINATES=-8,13 // optional, this will overwrite COUNTRY env var
     -e COUNTRY=uk // optional, but mandatory if COORDINATES are specified
     -e RPM=60 // optional
     -e TEST_DURATION=3 //optional, benchmark duration in minutes (not including warmup)
     -e TRAVEL_TIME=7200 // optional
-    -e UNIQUE_REQUESTS=2 // optional float, percentage of requests that should be unique
+    -e UNIQUE_REQUESTS=100 //optional int, the number of unique requests that should be generated
     -e DISABLE_DECODING="true" // optional flag, skips proto response decoding if set to "true"
     -ti igeolise/traveltime-k6-benchmarks:latest k6 run scripts/time-filter-proto.js
 ```

diff --git a/precomputed/origins.csv b/precomputed/origins.csv
@@ -0,0 +1,3 @@
+lat,lng
+48.75653369999993,9.216932199999986
+53.39511600000001,8.708477599999993
diff --git a/scripts/routes.js b/scripts/routes.js
@@ -16,9 +16,7 @@ import {
   deleteOneScenarioMetrics,
   oneScenarioReport,
   getCountryCoordinates,
-  randomIndex,
-  rpm,
-  durationInMinutes
+  randomIndex
 } from './common.js'
 
 export const options = {
@@ -43,8 +41,7 @@ export function setup () {
   const countryCoords = getCountryCoordinates(countryCode, __ENV.COORDINATES)
   const url = `https://${host}/v4/routes`
   const transportation = __ENV.TRANSPORTATION || 'driving+ferry'
-  const uniqueRequestsPercentage = parseFloat(__ENV.UNIQUE_REQUESTS || 2)
-  const uniqueRequestsAmount = Math.ceil((rpm * durationInMinutes) * (uniqueRequestsPercentage / 100))
+  const uniqueRequestsAmount = parseInt(__ENV.UNIQUE_REQUESTS || 100)
 
   const dateTime = new Date().toISOString()
 

diff --git a/scripts/time-filter-proto.js b/scripts/time-filter-proto.js
@@ -1,4 +1,5 @@
 import { getCurrentStageIndex } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js'
+import papaparse from 'https://jslib.k6.io/papaparse/5.1.1/index.js'
 import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.3/index.js'
 import http from 'k6/http'
 import protobuf from 'k6/x/protobuf'
@@ -16,9 +17,7 @@ import {
   setThresholdsForScenarios,
   summaryTrendStats,
   getProtoCountryCoordinates,
-  randomIndex,
-  rpm,
-  durationInMinutes
+  randomIndex
 } from './common.js'
 
 export const options = {
@@ -33,6 +32,8 @@ export const options = {
 setThresholdsForScenarios(options)
 randomSeed(__ENV.SEED || 1234567)
 
+const precomputedDataFile = __ENV.DATA_PATH ? open(__ENV.DATA_PATH) : undefined
+
 export function setup () {
   const serviceImage = __ENV.SERVICE_IMAGE || 'unknown'
   const mapDate = __ENV.MAP_DATE || 'unknown'
@@ -48,8 +49,7 @@ export function setup () {
   const country = envCountry || 'uk'
   const query = __ENV.QUERY || `api/v2/${countryCode(country)}/time-filter/fast/${transportation}`
   const isManyToOne = __ENV.MANY_TO_ONE !== undefined
-  const uniqueRequestsPercentage = parseFloat(__ENV.UNIQUE_REQUESTS || 2)
-  const uniqueRequestsAmount = Math.ceil((rpm * durationInMinutes) * (uniqueRequestsPercentage / 100))
+  const uniqueRequestsAmount = parseInt(__ENV.UNIQUE_REQUESTS || 100)
   const disableBodyDecoding = __ENV.DISABLE_DECODING === 'true'
 
   const url = `${protocol}://${appId}:${apiKey}@${host}/${query}`
@@ -65,9 +65,9 @@ export function setup () {
     }
   }
 
-  console.log('The amount of requests generated: ' + uniqueRequestsAmount)
-
-  const requestBodies = generateRequestBodies(uniqueRequestsAmount, destinationsAmount, countryCoords, transportation, travelTime, isManyToOne)
+  const requestBodies = precomputedDataFile
+    ? readRequestsBodies(destinationsAmount, transportation, travelTime, isManyToOne, precomputedDataFile)
+    : generateRequestBodies(uniqueRequestsAmount, destinationsAmount, countryCoords, transportation, travelTime, isManyToOne)
 
   return { url, requestBodies, params, disableBodyDecoding }
 }
@@ -133,7 +133,7 @@ function countryCode (country) {
 
 function generateBody (destinationsAmount, coord, transportation, travelTime, isManyToOne) {
   const diff = 0.005
-  const originLocation = generateRandomCoordinate(coord.lat, coord.lng, diff)
+  const originLocation = coord
   const destinations = generateDestinations(destinationsAmount, originLocation, diff)
   if (isManyToOne) {
     return JSON.stringify({
@@ -160,8 +160,43 @@ function generateBody (destinationsAmount, coord, transportation, travelTime, is
   }
 }
 
-function generateRequestBodies (count, destinationsAmount, coord, transportation, travelTime, isManyToOne) {
-  return Array.from({ length: count }, () => generateBody(
-    destinationsAmount, coord, transportation, travelTime, isManyToOne
-  ))
+function readRequestsBodies (destinationsAmount, transportation, travelTime, isManyToOne, precomputedDataFile) {
+  const data = papaparse
+    .parse(precomputedDataFile, { header: true, skipEmptyLines: true })
+    .data
+    .map(origins =>
+      generateBody(
+        destinationsAmount,
+        { lat: parseFloat(origins.lat), lng: parseFloat(origins.lng) },
+        transportation,
+        travelTime,
+        isManyToOne
+      )
+    )
+  console.log('The amount of requests read: ' + data.length)
+  return data
+}
+
+function generateRequestBodies (
+  count,
+  destinationsAmount,
+  coords,
+  transportation,
+  travelTime,
+  isManyToOne
+) {
+  console.log('The amount of requests generated: ' + count)
+  const diff = 0.005
+
+  return Array
+    .from(
+      { length: count },
+      () => generateBody(
+        destinationsAmount,
+        generateRandomCoordinate(coords.lat, coords.lng, diff),
+        transportation,
+        travelTime,
+        isManyToOne
+      )
+    )
 }
diff --git a/scripts/time-filter.js b/scripts/time-filter.js
@@ -1,4 +1,5 @@
 import { getCurrentStageIndex } from 'https://jslib.k6.io/k6-utils/1.3.0/index.js'
+import papaparse from 'https://jslib.k6.io/papaparse/5.1.1/index.js'
 import {
   textSummary
 } from 'https://jslib.k6.io/k6-summary/0.0.3/index.js'
@@ -15,9 +16,7 @@ import {
   deleteOneScenarioMetrics,
   summaryTrendStats,
   getCountryCoordinates,
-  randomIndex,
-  rpm,
-  durationInMinutes
+  randomIndex
 } from './common.js'
 
 export const options = {
@@ -32,6 +31,8 @@ export const options = {
 setThresholdsForScenarios(options)
 randomSeed(__ENV.SEED || 1234567)
 
+const precomputedDataFile = __ENV.DATA_PATH ? open(__ENV.DATA_PATH) : undefined
+
 export function setup () {
   const appId = __ENV.APP_ID
   const apiKey = __ENV.API_KEY
@@ -43,8 +44,7 @@ export function setup () {
   const travelTime = parseInt(__ENV.TRAVEL_TIME || 1900)
   const destinationsAmount = parseInt(__ENV.DESTINATIONS || 50)
   const rangeWidth = __ENV.RANGE || 0
-  const uniqueRequestsPercentage = parseFloat(__ENV.UNIQUE_REQUESTS || 2)
-  const uniqueRequestsAmount = Math.ceil((rpm * durationInMinutes) * (uniqueRequestsPercentage / 100))
+  const uniqueRequestsAmount = parseInt(__ENV.UNIQUE_REQUESTS || 2)
   const rangeSettings = {
     enabled: rangeWidth !== 0,
     max_results: 3,
@@ -60,9 +60,9 @@ export function setup () {
     }
   }
 
-  console.log('The amount of requests generated: ' + uniqueRequestsAmount)
-
-  const requestBodies = generateRequestBodies(uniqueRequestsAmount, travelTime, transportation, destinationsAmount, rangeSettings, countryCoords, dateTime)
+  const requestBodies = precomputedDataFile
+    ? readRequestsBodies(travelTime, transportation, destinationsAmount, rangeSettings, dateTime, precomputedDataFile)
+    : generateRequestBodies(uniqueRequestsAmount, travelTime, transportation, destinationsAmount, rangeSettings, countryCoords, dateTime)
 
   return { url, requestBodies, params }
 }
@@ -98,22 +98,25 @@ function generateBody (
   transportation,
   destinationsAmount,
   rangeSettings,
-  countryCoords,
+  coords,
   dateTime
 ) {
-  const coordinates = countryCoords
+  const originLocation = {
+    id: 'destination1',
+    coords: { lat: coords.lat, lng: coords.lng }
+  }
 
-  const destinations = Array.from({
-    length: destinationsAmount
-  }, (_, i) => ({
-    id: `destination${i + 1}`,
-    coords: generateRandomCoordinate(coordinates.lat, coordinates.lng, 0.005)
+  const randomDestinations = Array.from({ length: destinationsAmount }, (_, i) => ({
+    id: `destination${i + 2}`,
+    coords: generateRandomCoordinate(coords.lat, coords.lng, 0.005)
   }))
 
+  const allLocations = [originLocation, ...randomDestinations]
+
   const departureSearches = [{
     id: 'Time filter benchmark',
     departure_location_id: 'destination1',
-    arrival_location_ids: destinations.map(destination => destination.id),
+    arrival_location_ids: randomDestinations.map(destination => destination.id),
     departure_time: dateTime,
     travel_time: travelTime,
     properties: [
@@ -126,11 +129,29 @@ function generateBody (
   }]
 
   return JSON.stringify({
-    locations: destinations,
+    locations: allLocations,
     departure_searches: departureSearches
   })
 }
 
+function readRequestsBodies (travelTime, transportation, destinationsAmount, rangeSettings, dateTime, precomputedDataFile) {
+  const data = papaparse
+    .parse(precomputedDataFile, { header: true, skipEmptyLines: true })
+    .data
+    .map(origins =>
+      generateBody(
+        travelTime,
+        transportation,
+        destinationsAmount,
+        rangeSettings,
+        { lat: parseFloat(origins.lat), lng: parseFloat(origins.lng) },
+        dateTime
+      )
+    )
+  console.log('The amount of requests read: ' + data.length)
+  return data
+}
+
 function generateRequestBodies (
   count,
   travelTime,
@@ -140,7 +161,19 @@ function generateRequestBodies (
   countryCoords,
   dateTime
 ) {
-  return Array.from({ length: count }, () => generateBody(
-    travelTime, transportation, destinationsAmount, rangeSettings, countryCoords, dateTime
-  ))
+  console.log('The amount of requests generated: ' + count)
+  const diff = 0.005
+
+  return Array
+    .from(
+      { length: count },
+      () => generateBody(
+        travelTime,
+        transportation,
+        destinationsAmount,
+        rangeSettings,
+        generateRandomCoordinate(countryCoords.lat, countryCoords.lng, diff),
+        dateTime
+      )
+    )
 }