Loot api 2 (#59)

* minor changes to sentry sampler

* wrote a v2 of the loot gear fetch algorithm

* loot page will now 50/50 pull from old and new api

* bump up trace rate for loot while testing

* bump and changelog for this minor release

* update requirements for security

backend/api/models/bis_list.py

@@ -8,6 +8,38 @@
 from django.db.models import Q
 
 
+class BISListManager(models.Manager):
+
+    def with_all_relations(self):
+        return self.select_related(
+            'bis_body',
+            'bis_bracelet',
+            'bis_earrings',
+            'bis_feet',
+            'bis_hands',
+            'bis_head',
+            'bis_left_ring',
+            'bis_legs',
+            'bis_mainhand',
+            'bis_necklace',
+            'bis_offhand',
+            'bis_right_ring',
+            'current_body',
+            'current_bracelet',
+            'current_earrings',
+            'current_feet',
+            'current_hands',
+            'current_head',
+            'current_left_ring',
+            'current_legs',
+            'current_mainhand',
+            'current_necklace',
+            'current_offhand',
+            'current_right_ring',
+            'job',
+        )
+
+
 class BISList(models.Model):
     bis_body = models.ForeignKey('Gear', on_delete=models.CASCADE, related_name='bis_body_set')
     bis_bracelet = models.ForeignKey('Gear', on_delete=models.CASCADE, related_name='bis_bracelet_set')
@@ -39,6 +71,8 @@ class BISList(models.Model):
     name = models.CharField(max_length=64, default='')
     owner = models.ForeignKey('Character', on_delete=models.CASCADE, related_name='bis_lists')
 
+    objects = BISListManager()
+
     class Meta:
         ordering = ['-job__role', 'job__ordering', 'name']
 

backend/api/urls.py

@@ -25,6 +25,7 @@
 
     # Loot
     path('team/<str:team_id>/loot/', views.LootCollection.as_view(), name='loot_collection'),
+    path('team/<str:team_id>/loot_v2/', views.LootCollectionV2.as_view(), name='loot_collection_v2'),
     path('team/<str:team_id>/loot/bis/', views.LootWithBIS.as_view(), name='loot_with_bis'),
 
     # Notifications

backend/api/views/__init__.py

@@ -3,7 +3,7 @@
 from .etro import EtroImport
 from .gear import GearCollection, ItemLevels
 from .job import JobCollection
-from .loot import LootCollection, LootWithBIS
+from .loot import LootCollection, LootWithBIS, LootCollectionV2
 from .notification import NotificationCollection, NotificationResource
 from .team import TeamCollection, TeamResource, TeamInvite
 from .team_member import TeamMemberResource, TeamMemberPermissionsResource
@@ -30,6 +30,7 @@
 
     'LootCollection',
     'LootWithBIS',
+    'LootCollectionV2',
 
     'NotificationCollection',
     'NotificationResource',

backend/api/views/loot.py

@@ -44,6 +44,11 @@ class LootCollection(APIView):
         'bracelet',
     ]
     # Offhand and Rings cannot be automated due to special handling
+    NON_AUTOMATED_SLOTS = [
+        'ring',
+        'tome-accessory-augment',
+        'tome-armour-augment'
+    ]
 
     # This could probably be done nice but /shrug for now
     def _get_gear_data(self, obj: Team) -> Dict[str, List[Dict[str, str]]]:
@@ -394,3 +399,142 @@ def post(self, request: Request, team_id: str) -> Response:
         self._send_to_team(team, {'type': 'loot', 'id': str(team.id)})
 
         return Response({'id': loot.pk}, status=201)
+
+
+class LootCollectionV2(LootCollection):
+    """
+    An alternative version of the Loot Collection with a new get_gear_data function
+
+    I want to randomly test both versions with real data and ensure that this one is as superior as I've found in my tests
+    """
+
+    def _get_gear_data(self, obj: Team) -> Dict[str, List[Dict[str, str]]]:
+        """
+        An attempt at improving the original loot response with less DB hits + waiting required.
+        """
+        # Set up the whole response dictionary at once
+        response = {slot: {'need': [], 'greed': []} for slot in self.AUTOMATED_SLOTS}
+        for slot in self.NON_AUTOMATED_SLOTS:
+            response[slot] = {'need': [], 'greed': []}
+
+        # Maintain a mapping of character greed lists
+        greed_lists = {}
+
+        raid_gear_name = obj.tier.raid_gear_name
+
+        # Loop through every member of the team.
+        for member in obj.members.all():
+            greed_lists[member.id] = {}
+            # Loop through the member's greed BIS Lists
+            for bis_list in BISList.objects.with_all_relations().filter(owner=member.character):
+                for slot in self.AUTOMATED_SLOTS:
+                    greed_lists[member.id].setdefault(slot, [])
+
+                    list_is_need = bis_list.id == member.bis_list_id
+                    bis = getattr(bis_list, f'bis_{slot}')
+                    current = getattr(bis_list, f'current_{slot}')
+
+                    if bis.name == raid_gear_name and current.name != raid_gear_name:
+                        if list_is_need:
+                            response[slot]['need'].append({
+                                'member_id': member.id,
+                                'character_name': member.character.display_name,
+                                'current_gear_name': current.name,
+                                'current_gear_il': current.item_level,
+                                'job_icon_name': bis_list.job.id,
+                                'job_role': bis_list.job.role,
+                            })
+                        else:
+                            greed_lists[member.id][slot].append({
+                                'bis_list_name': bis_list.display_name,
+                                'bis_list_id': bis_list.id,
+                                'current_gear_name': current.name,
+                                'current_gear_il': current.item_level,
+                                'job_icon_name': bis_list.job.id,
+                                'job_role': bis_list.job.role,
+                            })
+
+                # For non-automated slots we have to do stuff more manually
+                slot = 'ring'
+                greed_lists[member.id].setdefault(slot, [])
+
+                bis_ring_check = [
+                    bis_list.bis_right_ring.name == raid_gear_name,
+                    bis_list.bis_left_ring.name == raid_gear_name,
+                ]
+                current_ring_check = [
+                    bis_list.current_right_ring.name != raid_gear_name,
+                    bis_list.current_left_ring.name != raid_gear_name,
+                ]
+                if any(bis_ring_check) and all(current_ring_check):
+                    if list_is_need:
+                        response[slot]['need'].append({
+                            'member_id': member.id,
+                            'character_name': member.character.display_name,
+                            'current_gear_name': current.name,
+                            'current_gear_il': current.item_level,
+                            'job_icon_name': bis_list.job.id,
+                            'job_role': bis_list.job.role,
+                        })
+                    else:
+                        greed_lists[member.id][slot].append({
+                            'bis_list_name': bis_list.display_name,
+                            'bis_list_id': bis_list.id,
+                            'current_gear_name': current.name,
+                            'current_gear_il': current.item_level,
+                            'job_icon_name': bis_list.job.id,
+                            'job_role': bis_list.job.role,
+                        })
+
+                slot = 'tome-accessory-augment'
+                greed_lists[member.id].setdefault(slot, [])
+                required = bis_list.accessory_augments_required(obj.tier.tome_gear_name)
+                if required > 0:
+                    if list_is_need:
+                        response[slot]['need'].append({
+                            'member_id': member.id,
+                            'character_name': member.character.display_name,
+                            'job_icon_name': member.bis_list.job.id,
+                            'job_role': member.bis_list.job.role,
+                            'requires': required,
+                        })
+                    else:
+                        greed_lists[member.id][slot].append({
+                            'bis_list_name': bis_list.display_name,
+                            'bis_list_id': bis_list.id,
+                            'job_icon_name': bis_list.job.id,
+                            'job_role': bis_list.job.role,
+                            'requires': required,
+                        })
+
+                slot = 'tome-armour-augment'
+                greed_lists[member.id].setdefault(slot, [])
+                required = bis_list.armour_augments_required(obj.tier.tome_gear_name)
+                if required > 0:
+                    if list_is_need:
+                        response[slot]['need'].append({
+                            'member_id': member.id,
+                            'character_name': member.character.display_name,
+                            'job_icon_name': member.bis_list.job.id,
+                            'job_role': member.bis_list.job.role,
+                            'requires': required,
+                        })
+                    else:
+                        greed_lists[member.id][slot].append({
+                            'bis_list_name': bis_list.display_name,
+                            'bis_list_id': bis_list.id,
+                            'job_icon_name': bis_list.job.id,
+                            'job_role': bis_list.job.role,
+                            'requires': required,
+                        })
+
+        # Lastly, consolidate the two storages of items into one
+        for member in obj.members.all():
+            for slot in greed_lists[member.id]:
+                response[slot]['greed'].append({
+                    'member_id': member.id,
+                    'character_name': member.character.display_name,
+                    'greed_lists': greed_lists[member.id][slot],
+                })
+
+        return response

backend/backend/settings_live.py

@@ -155,6 +155,7 @@
 USE_L10N = True
 USE_TZ = True
 
+
 # Sentry for errors as well
 def sampler(context):
     # Always inherit parent context
@@ -163,12 +164,17 @@ def sampler(context):
     path = context.get('wsgi_environ', {}).get('PATH_INFO', '').lower()
     # I want to trace loot requests since I need to know what's going on in the loot requests so I can improve them
     if 'loot' in path:
-        return 0.5
+        return 0.75
     elif 'notifications' in path:
         # Notifications spans are not important
         return 0
+    # Also ignore celery tasks
+    if 'celery' in context['transaction_context']['op']:
+        return 0
+
     # Anything else I'm not too bothered by
-    return 0.05
+    return 0.1
+
 
 sentry_sdk.init(
     dsn=environ['SENTRY_DSN'],
@@ -178,7 +184,7 @@ def sampler(context):
     # If you wish to associate users to errors (assuming you are using
     # django.contrib.auth) you may enable sending PII data.
     send_default_pii=True,
-    release='savageaim@20230719.2',
+    release='savageaim@20230727',
 )
 
 # Channels

backend/requirements.txt

@@ -8,7 +8,7 @@ Automat==20.2.0
 beautifulsoup4==4.10.0
 billiard==3.6.4.0
 celery==5.2.3
-certifi==2022.12.7
+certifi==2023.7.22
 cffi==1.15.0
 channels==3.0.4
 channels-redis==3.4.0
@@ -26,8 +26,7 @@ defusedxml==0.7.1
 Deprecated==1.2.13
 Django==3.2.20
 django-allauth==0.47.0
-django-rest-framework==0.1.0
-djangorestframework==3.13.1
+djangorestframework==3.14.0
 gunicorn==20.1.0
 hiredis==2.0.0
 hyperlink==21.0.0

frontend/.env

@@ -1 +1 @@
-VUE_APP_VERSION="20230719.2"
+VUE_APP_VERSION="20230727"

frontend/src/components/modals/changelog.vue

@@ -13,11 +13,11 @@
     <div class="card-content content">
       <h2 class="has-text-primary subtitle">{{ version }}</h2>
 
-      <div class="divider"><i class="material-icons icon">expand_more</i> FFXIV Patch 6.45 <i class="material-icons icon">expand_more</i></div>
-      <p>Added Majestic Manderville Weapons, iL 645.</p>
-
-      <div class="divider"><i class="material-icons icon">expand_more</i> Fixes <i class="material-icons icon">expand_more</i></div>
-      <p>Rewrote Etro Import endpoint to prevent rate limit hits. Had the unplanned effect of speeding up the request as well.</p>
+      <div class="divider"><i class="material-icons icon">expand_more</i> Updates <i class="material-icons icon">expand_more</i></div>
+      <p>Wrote a new algorithm for fetching information for the Loot Manager page.</p>
+      <p>Until the new Loot Manager update comes out sometime next month, I've set up the Loot Manager to randomly use either of the algorithms (50/50 split).</p>
+      <p>The new one appears to be faster and less demanding in my own testing, but this kind of thing is best tested with real data.</p>
+      <p>As always, if there are any issues, please report them on the Discord!</p>
 
     </div>
   </div>

frontend/src/main.ts

@@ -27,7 +27,7 @@ Sentry.init({
   Vue,
   dsn: 'https://[email protected]/6180221',
   logErrors: true,
-  release: 'savageaim@20230719.2',
+  release: 'savageaim@20230727',
 })
 
 new Vue({

frontend/src/views/team/loot.vue

@@ -169,6 +169,10 @@ export default class TeamLoot extends TeamViewMixin {
     return `/backend/api/team/${this.teamId}/loot/`
   }
 
+  get v2Url(): string {
+    return `/backend/api/team/${this.teamId}/loot_v2/`
+  }
+
   async created(): Promise<void> {
     await this.fetchData(false)
     document.title = `Loot Tracker - ${this.team.name} - Savage Aim`
@@ -177,7 +181,11 @@ export default class TeamLoot extends TeamViewMixin {
   async fetchData(reload: boolean): Promise<void> {
     // Load the loot data from the API
     try {
-      const response = await fetch(this.url)
+      // Pick a URL at random, 50% odds each time
+      let response
+      if (Math.random() < 0.5) response = await fetch(this.url)
+      else response = await fetch(this.v2Url)
+
       if (response.ok) {
         // Parse the JSON and save it in instance variables
         const content = (await response.json()) as LootResponse