Etro Rate Limit Fix (#58)

* rewrote etro import view to prevent rate limit hits

* added new test to ensure etro import supports relic weapons

* do bump

* added new entry to changelog about the etro fixes

* linting

backend/api/tests/test_etro.py

@@ -44,11 +44,41 @@ def test_import(self):
             'necklace': Gear.objects.get(name='Augmented Radiant Host').pk,
             'bracelet': Gear.objects.get(name='Augmented Radiant Host').pk,
             'right_ring': Gear.objects.get(name='Augmented Radiant Host').pk,
-            'min_il': 600,
+            'min_il': 560,
             'max_il': 605,
         }
         self.assertDictEqual(response.json(), expected)
 
+    def test_import_with_relic(self):
+        """
+        Test an import of a gearset with a custom relic
+        """
+        url = reverse('api:etro_import', kwargs={'id': '2745b09f-4023-40e6-93e2-72c652143182'})
+        user = self._get_user()
+        self.client.force_authenticate(user)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # Build an expected data packet
+        expected = {
+            'job_id': 'DRG',
+            'mainhand': Gear.objects.get(name='Majestic Manderville').pk,
+            'offhand': Gear.objects.get(name='Majestic Manderville').pk,
+            'head': Gear.objects.get(name='Ascension', has_armour=True).pk,
+            'body': Gear.objects.get(name='Ascension', has_armour=True).pk,
+            'hands': Gear.objects.get(name='Ascension', has_armour=True).pk,
+            'earrings': Gear.objects.get(name='Ascension', has_armour=True).pk,
+            'left_ring': Gear.objects.get(name='Ascension', has_armour=True).pk,
+            'legs': Gear.objects.get(name='Ascension', has_accessories=True).pk,
+            'feet': Gear.objects.get(name='Ascension', has_accessories=True).pk,
+            'necklace': Gear.objects.get(name='Ascension', has_accessories=True).pk,
+            'bracelet': Gear.objects.get(name='Ascension', has_accessories=True).pk,
+            'right_ring': Gear.objects.get(name='Ascension', has_accessories=True).pk,
+            'min_il': 645,
+            'max_il': 665,
+        }
+        self.assertDictEqual(response.json(), expected)
+
     def test_import_400(self):
         """
         Send a request with an invalid ID, check we get a proper error

backend/api/views/etro.py

@@ -2,7 +2,7 @@
 Given an etro id, convert it into a format that uses Savage Aim ids
 """
 # stdlib
-from typing import Dict, Set
+from typing import Dict
 # lib
 import coreapi
 import jellyfish
@@ -53,62 +53,63 @@ def _get_gear_id(gear_selection: Dict[str, str], item_name: str) -> str:
 
     def get(self, request: Request, id: str) -> Response:
         """
-        Return a list of Characters belonging to a certain User
+        Given an Etro Gearset ID, load the equipment from it
         """
         # Instantiate a Client instance for CoreAPI
         client = coreapi.Client()
         schema = client.get('https://etro.gg/api/docs/')
 
         # First things first, attempt to read the gearset
         try:
-            response = client.action(schema, ['gearsets', 'read'], params={'id': id})
+            gearset = client.action(schema, ['gearsets', 'read'], params={'id': id})
         except coreapi.exceptions.ErrorMessage as e:
             return Response({'message': e.error.title}, status=400)
-        job_id = response['jobAbbrev']
 
-        # Loop through each slot of the etro gearset, fetch the name and item level and store it in a dict
-        gear_details: Dict[str, str] = {}
-        item_levels: Set[int] = set()
-        min_il = float('inf')
-        max_il = float('-inf')
+        job_id = gearset['jobAbbrev']
+        min_il = gearset['minItemLevel']
+        max_il = gearset['maxItemLevel']
 
+        # Retrieve a list of all gear within the item level bracket for the job
+        params = {job_id: True, 'minItemLevel': min_il, 'maxItemLevel': max_il}
+        equipment = client.action(schema, ['equipment', 'list'], params=params)
+        # Map IDs to names
+        etro_map = {
+            item['id']: item['name'] for item in equipment
+        }
+
+        # Loop through the gear slots of the gear set and get the names for each
+        gear_names: Dict[str, str] = {}
         for etro_slot, sa_slot in SLOT_MAP.items():
-            gear_id = response[etro_slot]
+            gear_id = gearset[etro_slot]
             if gear_id is None:
                 continue
-            item_response = client.action(schema, ['equipment', 'read'], params={'id': gear_id})
-            gear_details[sa_slot] = item_response['name']
+            gear_names[sa_slot] = etro_map[gear_id]
 
-            # item level stuff
-            il = item_response['itemLevel']
-            item_levels.add(il)
-            if il < min_il:
-                min_il = il
-            if il > max_il:
-                max_il = il
-
-        # Get the names of all the gear with the specified Item Levels
-        gear_names = Gear.objects.filter(item_level__in=item_levels).values('name', 'id')
+        # Check for relic weapons
+        if gearset['weapon'] is None and 'weapon' in gearset['relics']:
+            relic_id = gearset['relics']['weapon']
+            relic = client.action(schema, ['relic', 'read'], params={'id': relic_id})
+            gear_names['mainhand'] = relic['baseItem']['name']
 
+        # Turn the names into SA gear ids
+        sa_gear = Gear.objects.filter(item_level__gte=min_il, item_level__lte=max_il).values('name', 'id')
         response = {
             'job_id': job_id,
+            'min_il': min_il,
+            'max_il': max_il,
         }
 
-        # Loop through the slots one final time, and get the gear id for that slot
-        for slot, item_name in gear_details.items():
+        # Loop through each gear slot and fetch the id based off the name
+        for slot, item_name in gear_names.items():
             if slot in ARMOUR_SLOTS:
-                response[slot] = self._get_gear_id(gear_names.filter(has_armour=True), item_name)
+                response[slot] = self._get_gear_id(sa_gear.filter(has_armour=True), item_name)
             elif slot in ACCESSORY_SLOTS:
-                response[slot] = self._get_gear_id(gear_names.filter(has_accessories=True), item_name)
+                response[slot] = self._get_gear_id(sa_gear.filter(has_accessories=True), item_name)
             else:
-                response[slot] = self._get_gear_id(gear_names.filter(has_weapon=True), item_name)
+                response[slot] = self._get_gear_id(sa_gear.filter(has_weapon=True), item_name)
 
         # Check for offhand
         if job_id != 'PLD':
             response['offhand'] = response['mainhand']
 
-        # Also add item level status
-        response['min_il'] = min_il
-        response['max_il'] = max_il
-
         return Response(response)

backend/backend/settings_live.py

@@ -178,7 +178,7 @@ def sampler(context):
     # If you wish to associate users to errors (assuming you are using
     # django.contrib.auth) you may enable sending PII data.
     send_default_pii=True,
-    release='savageaim@20230719',
+    release='savageaim@20230719.2',
 )
 
 # Channels

frontend/.env

@@ -1 +1 @@
-VUE_APP_VERSION="20230719"
+VUE_APP_VERSION="20230719.2"

frontend/src/components/modals/changelog.vue

@@ -16,6 +16,9 @@
       <div class="divider"><i class="material-icons icon">expand_more</i> FFXIV Patch 6.45 <i class="material-icons icon">expand_more</i></div>
       <p>Added Majestic Manderville Weapons, iL 645.</p>
 
+      <div class="divider"><i class="material-icons icon">expand_more</i> Fixes <i class="material-icons icon">expand_more</i></div>
+      <p>Rewrote Etro Import endpoint to prevent rate limit hits. Had the unplanned effect of speeding up the request as well.</p>
+
     </div>
   </div>
 </template>

frontend/src/main.ts

@@ -27,7 +27,7 @@ Sentry.init({
   Vue,
   dsn: 'https://[email protected]/6180221',
   logErrors: true,
-  release: 'savageaim@20230719',
+  release: 'savageaim@20230719.2',
 })
 
 new Vue({