feat: support special score and score precision (#98)

* feat: add special score and score precision support

In this [commit](d466616aca), NTOJ-Judge supports checker overwriting the score and state.
Therefore, we need to update our backend to support this feature.

* feat: add cms special score support

In this PR, we add support for special score styles in CMS (groupmin).
We can precisely control the score in each subtask.
Multiply the subtask score by a decimal between 0 and 1.

* ci: fix pip no such option: --break-system-packages

* ci: fix postgresql authentication failed

* fix: race condition on challenge dispatching
This is a hotfix, reported by @blameazu

* impr: catch more specific exception `InvalidOperation` instead of `DecimalException`

* impr: simplify the problem of uploading by using a loop

* refactor: extract the duplicate code and loop through the add groups and test cases instead of hardcoding

* refactor(test): use a base URL to replace all hardcoded URLs

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ permissions:
 
 jobs:
   e2etest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v5
@@ -26,9 +26,10 @@ jobs:
           echo deb [arch=amd64 signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | sudo tee /etc/apt/sources.list.d/postgresql.list
           sudo apt update -y
           sudo apt install -f -y postgresql-16 postgresql-client-16 redis dos2unix
-          sudo sed -i 's/peer/trust/' /etc/postgresql/16/main/pg_hba.conf
-          sudo service postgresql start
-          sudo service redis-server start
+          sudo sed -i 's/peer/trust/g' /etc/postgresql/16/main/pg_hba.conf
+          sudo sed -i 's/scram-sha-256/trust/g' /etc/postgresql/16/main/pg_hba.conf
+          sudo service postgresql restart
+          sudo service redis-server restart
 
       - name: Install Coverage
         run: |

migration/20241012000000_special_score.py

@@ -0,0 +1,94 @@
+class NewStatus:
+    STATE_AC = 1
+    STATE_PC = 2
+    STATE_WA = 3
+    STATE_RE = 4
+    STATE_RESIG = 5
+    STATE_TLE = 6
+    STATE_MLE = 7
+    STATE_OLE = 8
+    STATE_CE = 9
+    STATE_CLE = 10
+    STATE_ERR = 11
+    STATE_SJE = 12
+    STATE_JUDGE = 100
+    STATE_NOTSTARTED = 101
+
+OldStatusMapper = {
+    1: NewStatus.STATE_AC,
+    2: NewStatus.STATE_WA,
+    3: NewStatus.STATE_RE,
+    9: NewStatus.STATE_RESIG,
+    4: NewStatus.STATE_TLE,
+    5: NewStatus.STATE_MLE,
+    6: NewStatus.STATE_CE,
+    10: NewStatus.STATE_CLE,
+    7: NewStatus.STATE_ERR,
+    8: NewStatus.STATE_OLE,
+    100: NewStatus.STATE_JUDGE,
+    101: NewStatus.STATE_NOTSTARTED,
+}
+
+async def dochange(db, rs):
+
+    res = await db.fetch('SELECT chal_id, test_idx, state FROM test;')
+    for test in res:
+        chal_id, test_idx, old_state = test['chal_id'], test['test_idx'], test['state']
+        await db.execute('UPDATE test SET state = $1 WHERE chal_id = $2 AND test_idx = $3',
+                         OldStatusMapper[old_state], chal_id, test_idx)
+        await db.execute(f'SELECT update_challenge_state({chal_id});')
+
+    await db.execute('ALTER TABLE test ADD COLUMN rate NUMERIC(10, 3);')
+    await db.execute('ALTER TABLE problem ADD COLUMN rate_precision INTEGER DEFAULT 0;')
+    await db.execute('ALTER TABLE challenge_state ALTER COLUMN rate TYPE NUMERIC(10, 3);')
+
+    await rs.delete('rate')
+
+    await db.execute(
+    '''
+    CREATE OR REPLACE FUNCTION update_challenge_state(p_chal_id INTEGER)
+    RETURNS VOID AS $$
+    BEGIN
+        WITH challenge_summary AS (
+            SELECT
+                t.chal_id,
+                MAX(t.state) AS max_state,
+                SUM(t.runtime) AS total_runtime,
+                SUM(t.memory) AS total_memory,
+                SUM(
+                    CASE
+                        WHEN (t.state = 1 OR t.state = 2) AND t.rate IS NOT NULL THEN t.rate -- special score
+                        WHEN t.state = 1 AND t.rate IS NULL THEN tvr.rate -- default score
+                        ELSE 0
+                    END
+                ) AS total_rate
+            FROM test t
+            LEFT JOIN test_valid_rate tvr ON t.pro_id = tvr.pro_id AND t.test_idx = tvr.test_idx
+            WHERE t.chal_id = p_chal_id
+            GROUP BY t.chal_id
+        )
+        INSERT INTO challenge_state (chal_id, state, runtime, memory, rate)
+        SELECT
+            chal_id,
+            max_state,
+            total_runtime,
+            total_memory,
+            total_rate
+        FROM challenge_summary
+        ON CONFLICT (chal_id) DO UPDATE
+        SET
+            state = EXCLUDED.state,
+            runtime = EXCLUDED.runtime,
+            memory = EXCLUDED.memory,
+            rate = EXCLUDED.rate
+        WHERE
+            challenge_state.state != EXCLUDED.state OR
+            challenge_state.runtime != EXCLUDED.runtime OR
+            challenge_state.memory != EXCLUDED.memory OR
+            challenge_state.rate != EXCLUDED.rate;
+
+        RETURN;
+    END;
+    $$ LANGUAGE plpgsql;
+    '''
+    )

src/handlers/chal.py

@@ -1,4 +1,5 @@
 import asyncio
+import decimal
 import json
 
 import tornado.web
@@ -132,6 +133,11 @@ async def get(self, chal_id):
         await self.render('chal', pro=pro, chal=chal, rechal=rechal)
         return
 
+class _Encoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, decimal.Decimal):
+            return str(o)
+        return super().default(o)
 
 class ChalListNewChalHandler(WebSocketSubHandler):
     async def listen_challistnewchal(self):
@@ -156,7 +162,7 @@ async def listen_challiststate(self):
             chal_id = int(msg['data'])
             if self.first_chal_id <= chal_id <= self.last_chal_id:
                 _, new_state = await ChalService.inst.get_single_chal_state_in_list(chal_id, self.acct)
-                await self.write_message(json.dumps(new_state))
+                await self.write_message(json.dumps(new_state, cls=_Encoder))
 
     async def open(self):
         self.first_chal_id = -1
@@ -189,7 +195,7 @@ async def listen_chalstate(self):
 
             if int(msg['data']) == self.chal_id:
                 _, chal_states = await ChalService.inst.get_chal_state(self.chal_id)
-                await self.write_message(json.dumps(chal_states))
+                await self.write_message(json.dumps(chal_states, cls=_Encoder))
 
     async def open(self):
         self.chal_id = -1

src/handlers/contests/manage/pro.py

@@ -90,7 +90,7 @@ async def post(self):
                 self.error('Ejudge')
                 return
 
-            err, pro = await ProService.inst.get_pro(pro_id, self.acct)
+            err, pro = await ProService.inst.get_pro(pro_id, self.acct, is_contest=True)
             if err:
                 self.error(err)
                 return

src/handlers/contests/scoreboard.py

@@ -25,7 +25,7 @@ def default(self, o):
             return f"{minutes}:{seconds:02}"
 
         elif isinstance(o, Decimal):
-            return int(o)
+            return float(o)
 
         else:
             return json.JSONEncoder.default(self, o)
@@ -36,6 +36,9 @@ def _encoder(self, obj):
         if isinstance(obj, datetime.datetime):
             return obj.timestamp()
 
+        elif isinstance(obj, Decimal):
+            return str(obj)
+
         return obj
 
     @reqenv
@@ -97,6 +100,7 @@ async def post(self):
                 for pro_score in s[pro_id].values():
                     pro_score['timestamp'] = datetime.datetime.fromtimestamp(pro_score['timestamp']).replace(
                         tzinfo=UTC8)
+                    pro_score['score'] = Decimal(pro_score['score'])
 
             if is_ended:
                 await self.rs.expire(cache_name, time=60 * 60)

src/handlers/manage/pro.py

@@ -775,6 +775,11 @@ async def post(self, page=None):
                 tags = self.get_argument('tags')
                 allow_submit = self.get_argument('allow_submit') == "true"
                 # NOTE: test config
+                rate_precision = int(self.get_argument('rate_precision'))
+                if rate_precision > ProConst.RATE_PRECISION_MAX or rate_precision < ProConst.RATE_PRECISION_MIN:
+                    self.error('Eparam')
+                    return
+
                 is_makefile = self.get_argument('is_makefile') == "true"
                 check_type = int(self.get_argument('check_type'))
 
@@ -811,6 +816,7 @@ async def post(self, page=None):
                 if check_type == ProConst.CHECKER_IOREDIR:
                     chalmeta = json.dumps(chalmeta)
 
+                pro['testm_conf']['rate_precision'] = rate_precision
                 await ProService.inst.update_test_config(pro_id, pro['testm_conf'])
                 await LogService.inst.add_log(
                     f"{self.acct.name} has sent a request to update the problem #{pro_id}", 'manage.pro.update.pro',
@@ -822,6 +828,7 @@ async def post(self, page=None):
                         'is_makefile': is_makefile,
                         'chalmeta': chalmeta,
                         'check_type': check_type,
+                        'rate_precision': rate_precision,
                     }
                 )
                 if err:

src/handlers/rank.py

@@ -4,6 +4,7 @@
 
 from handlers.base import RequestHandler, reqenv
 from services.user import UserConst, UserService, Account
+from services.chal import ChalConst
 
 
 class ProRankHandler(RequestHandler):
@@ -26,26 +27,33 @@ async def get(self, pro_id):
 
         async with self.db.acquire() as con:
             result = await con.fetch(
-                '''
+                f'''
                 SELECT *
                 FROM (
                 SELECT DISTINCT ON ("challenge"."acct_id")
-                "challenge"."chal_id",
-                "challenge"."acct_id",
-                "challenge"."timestamp",
-                "account"."name" AS "acct_name",
-                "challenge_state"."runtime",
-                "challenge_state"."memory"
-                FROM "challenge"
-                INNER JOIN "account"
-                ON "challenge"."acct_id"="account"."acct_id"
-                INNER JOIN "challenge_state"
-                ON "challenge"."chal_id"="challenge_state"."chal_id"
-                WHERE "challenge"."pro_id"= $1
-                AND "challenge_state"."state"=1
-                ORDER BY "challenge"."acct_id" ASC,
-                "challenge_state"."runtime" ASC, "challenge_state"."memory" ASC,
-                "challenge"."timestamp" ASC, "challenge"."acct_id" ASC
+                        "challenge"."chal_id",
+                        "challenge"."acct_id",
+                        "challenge"."timestamp",
+                        "account"."name" AS "acct_name",
+                        "challenge_state"."runtime",
+                        "challenge_state"."memory",
+                        ROUND("challenge_state"."rate", "problem"."rate_precision")
+
+                    FROM "challenge"
+                    INNER JOIN "account"
+                    ON "challenge"."acct_id"="account"."acct_id"
+
+                    INNER JOIN "challenge_state"
+                    ON "challenge"."chal_id"="challenge_state"."chal_id"
+
+                    INNER JOIN "problem"
+                    ON "challenge"."pro_id" = $1
+
+                    WHERE "challenge_state"."state"={ChalConst.STATE_AC}
+
+                    ORDER BY "challenge"."acct_id" ASC, "challenge_state"."rate" ASC,
+                    "challenge_state"."runtime" ASC, "challenge_state"."memory" ASC,
+                    "challenge"."timestamp" ASC
                 ) temp
                 ORDER BY "runtime" ASC, "memory" ASC,
                 "timestamp" ASC, "acct_id" ASC OFFSET $2 LIMIT $3;
@@ -73,7 +81,7 @@ async def get(self, pro_id):
             total_cnt = total_cnt[0]['count']
 
         chal_list = []
-        for rank, (chal_id, acct_id, timestamp, acct_name, runtime, memory) in enumerate(result):
+        for rank, (chal_id, acct_id, timestamp, acct_name, runtime, memory, rate) in enumerate(result):
             chal_list.append(
                 {
                     'rank': rank + pageoff + 1,
@@ -82,6 +90,7 @@ async def get(self, pro_id):
                     'acct_name': acct_name,
                     'runtime': int(runtime),
                     'memory': int(memory),
+                    'rate': rate,
                     'timestamp': timestamp.astimezone(tz),
                 }
             )

src/handlers/submit.py

@@ -133,7 +133,7 @@ async def post(self):
 
                 pro_id = chal['pro_id']
                 comp_type = chal['comp_type']
-                err, pro = await ProService.inst.get_pro(pro_id, self.acct)
+                err, pro = await ProService.inst.get_pro(pro_id, self.acct, is_contest=self.contest is not None)
                 if err:
                     self.finish(err)
                     return