Detect invalid logs and mark in DB (#57)

* start implementation

* finish error impl, set is_valid

* add force-parse parameter

README.md

@@ -24,6 +24,12 @@ When the server is running, you can POST two log files to the "/parseGame" route
 curl -X POST -F 'logs[]=@logs/L0526012.log' -F 'logs[]=@logs/L0526013.log' http://127.0.0.1:3000/parseGame
 ```
 
+If the log fails validation, you can add a "force" form body parameter with any non-null value (e.g., "on") to force-parse the log files anyway.
+
+```
+curl -X POST -F force=on -F 'logs[]=@logs/L0526012.log' -F 'logs[]=@logs/L0526013.log' http://127.0.0.1:3000/parseGame
+```
+
 ### Immediate to-do:
 
 - [ ] Sort summary view by # kills (or some other score metric)
@@ -42,7 +48,7 @@ Plan:
         - ngnix and node.js process should run in non-privileged context
     - Use [jQuery POST](https://api.jquery.com/jquery.post/) and [this tutorial](https://attacomsian.com/blog/xhr-node-file-upload) to do front-end call
 
-    - [ ] Set up reverse proxy for hampalyzer script and shove under /api/
+    - [x] Set up reverse proxy for hampalyzer script and shove under /api/
     - [x] Move "frontend" code to /var/www/app.hampalyzer.com
     - [x] Remove manual file parsing
     - [x] Force uploaded files to be local
@@ -57,7 +63,7 @@ Plan:
 
 - [ ] Handle player disconnects if they are in the middle of carrying flag (flag time / flag status)
 - [ ] If a player only plays one of two rounds, player stats doesn't format correctly (e.g., stats show in rd2 even though they only played rd1)
-- [ ] Classes may not be assigned correctly, e.g. http://app.hampalyzer.com/parsedlogs/Inhouse-2021-Jun-6-02-22/ (hamp rd2)
+- [x] Classes may not be assigned correctly, e.g. http://app.hampalyzer.com/parsedlogs/Inhouse-2021-Jun-6-02-22/ (hamp rd2)
 
 ## Building / Running
 

db-schema.md

@@ -0,0 +1,72 @@
+EVENT table (table is new)
+
+| name            | type     | values  | notes                      |
+|-----------------|----------|---------|----------------------------|
+| eventId         |          |         | auto-increment, NOT NULL   |
+| logId           |          |         | log table id ref, NOT NULL |
+| isFirstLog      | bool     |         | default true               |
+| eventType       | enum     | 0-71    | NOT NULL                   |
+| rawLine         | string   |         | NOT NULL                   |
+| lineNumber      | number   |         | NOT NULL                   |
+| timestamp       | datetime |         | NOT NULL                   |
+| gameTime        | number   | seconds | NOT NULL                   |
+| extraData       | string   |         | (prefer json format?)      |
+| playerFrom      |          |         | player table id ref        |
+| playerFromClass | short    | 0-9     |                            |
+| playerTo        |          |         | player table id ref        |
+| playerToClass   | short    | 0-9     |                            |
+| withWeapon      | short    | 0-39    |                            |
+| playerFromFlag  | bool     |         | default false              |
+| playerToFlag    | bool     |         | default false              |
+
+
+PLAYER table (table is new)
+
+| name            | type     | desc    | notes                      |
+|-----------------|----------|---------|----------------------------|
+| playerId        |          |         | auto-increment, NOT NULL   |
+| playerName      | string   |         | NOT NULL                   |
+| playerAlias     | string   |         |                            |
+| steamId         | number   |         | see [SteamID doc](https://developer.valvesoftware.com/wiki/SteamID) |
+
+
+LOGS table (* are new columns)
+
+| name            | type     | values   | notes                      |
+|-----------------|----------|----------|----------------------------|
+| * logId         |          |          | auto-increment, NOT NULL   |
+| parsedlog       | string   |          | output URI slug            |
+| log_file1       | string   |          | matches name in uploads/   |
+| log_file2       | string   |          | matches name in uploads/   |
+| date_parsed     | datetime |          | initial upload time        |
+| date_match      | datetime |          | reported in local time     |
+| map             | string   |          | can be "<multiple>"        |
+| server          | string   |          |                            |
+| num_players     | int      |          |                            |
+| * is_valid      | bool     |          | default false              |
+
+
+PARSEDLOGS table (table is new)
+
+| name            | type         | values   | notes                      |
+|-----------------|--------------|----------|----------------------------|
+| logId           |              |          | auto-increment, NOT NULL   |
+| jsonSummary     | varchar(MAX) |          | full json                  |
+
+
+MAPLOCATIONS table (table is new)
+
+| name            | type         | values   | notes                      |
+|-----------------|--------------|----------|----------------------------|
+| locationId      |              |          | auto-increment, NOT NULL   |
+| map             | string       |          | NOT NULL, add index        |
+| name            | string       |          | user-provided name/callout |
+| geom            | geometry     |          | will be POLYGON Z, typ.    |
+
+SELECT name
+  FROM mapLocations
+WHERE ST_3DIntersects(
+    geom,
+    'POINT Z($x, $y, $z)'::geometry
+)
+LIMIT 1;

frontend/index.html

@@ -48,6 +48,10 @@ <h2>Hampalyzer &mdash; TFC Statistics</h2>
                             <div class="invalid-feedback">You must select one or two .log files.</div>
                         </div>
                     </div>
+                    <div class="form-group form-check">
+                        <input type="checkbox" class="form-check-input" name="force" id="force">
+                        <label class="form-check-label" for="force"><abbr title="will skip validation that map matches, players nearly match between rounds, and match lengths are within 5 minutes">Force-parse logs</abbr></label>
+                    </div>
                     <div class="form-group">
                         <input class="btn btn-primary" type="submit" value="Submit">
                     </div>
@@ -137,17 +141,22 @@ <h2>Hampalyzer &mdash; TFC Statistics</h2>
                 cache: false,
             });
 
-            let handler = function(data) {
-                if (data.success) {
+            let handler = function(response) {
+                if (response.success) {
                     $('#formResponse').attr('class', 'alert alert-primary')
-                        .html(`Your game has been parsed and is available at <a href="${ data.success.path }">${ data.success.path }</a>!`);
-                } else {
+                        .html(`Your game has been parsed and is available at <a href="${ response.success.path }">${ response.success.path }</a>!`);
+                } 
+                else if (response.failure) {
+                    $('#formResponse').attr('class', 'alert alert-danger')
+                        .html(`There was an error parsing the logs: <strong>${response.failure.error_reason}</strong>.<br />${response.failure.message}`);
+                } 
+                else {
                     $('#formResponse').attr('class', 'alert alert-danger')
-                        .html("There was an error parsing the logs.  More info: &laquo;" + (data.responseJSON || { error: "no additional info" }).error + "&raquo;");
+                        .html(`An unknown error occured: either the server did not respond or it talked funny to us.`);
                 }
             }
 
-            request.done(handler).fail(handler);
+            request.done(handler).fail((data) => handler(data.responseJSON));
         });
     </script>
 </body>

src/App.ts

@@ -13,6 +13,7 @@ import { FileCompression } from './fileCompression.js';
 import { default as fileParser, HampalyzerTemplates } from './fileParser.js';
 import { ParsedStats, Parser } from './parser.js';
 import TemplateUtils from './templateUtils.js';
+import { ParseResponse, ParsingError, ParsingOptions } from './constants.js';
 
 const envFilePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "../.env");
 dotenv.config({ path: envFilePath });
@@ -82,47 +83,50 @@ class App {
         });
 
         router.post('/parseGame', cors(), upload.array('logs[]', 2), async (req, res) => {
-            if (req?.files!['logs']?.length < 2) {
-                console.error("expected two files");
+            if (req.files && (req.files as Express.Multer.File[]).length < 2) {
+                res.status(400).json({ failure: { message: "expected two files in `logs[]`"}});
+                return;
             }
 
-            let parsedResult = await this.parseLogs([
+            const skipValidation = !!req.body.force;
+            const parserResponse = await this.parseLogs([
                 req.files![0].path,
-                req.files![1].path]);
+                req.files![1].path],
+                { skipValidation });
 
-            if (parsedResult == null) {
-                res.status(500).json({ error: "Failed to parse file (please pass logs to Hampster)" });
-            }
-            else {
+            if (parserResponse.success) {
                 // sanitize the outputPath by removing the webserverRoot path
                 // (e.g., remove /var/www/app.hampalyzer.com/html prefix)
-                let outputPath = parsedResult;
+                let outputPath = parserResponse.message;
                 if (outputPath.startsWith(this.webserverRoot)) {
                     outputPath = outputPath.slice(this.webserverRoot.length);
                 }
 
                 res.status(200).json({ success: { path: outputPath }});
             }
+            else {
+                const { error_reason, message } = parserResponse;
+                res.status(400).json({ failure: { error_reason, message } });
+            }
         });
 
         router.post('/parseLog', cors(), upload.single('logs[]'), async (req, res) => {
-            // res.status(500).json({ error: "Single log parsing is still a work in progress; try uploading two rounds of a game instead." });
-
-            let parsedResult = await this.parseLogs([req.file!.path]);
-
-            if (parsedResult == null) {
-                res.status(500).json({ error: "Failed to parse file (please pass logs to Hampster)" });
-            }
-            else {
+            const skipValidation = !!req.body.force;
+            const parserResponse = await this.parseLogs([req.file!.path], { skipValidation });
+            if (parserResponse.success) {
                 // sanitize the outputPath by removing the webserverRoot path
                 // (e.g., remove /var/www/app.hampalyzer.com/html prefix)
-                let outputPath = parsedResult;
+                let outputPath = parserResponse.message;
                 if (outputPath.startsWith(this.webserverRoot)) {
                     outputPath = outputPath.slice(this.webserverRoot.length);
                 }
 
                 res.status(200).json({ success: { path: outputPath }});
             }
+            else {
+                const { error_reason, message } = parserResponse;
+                res.status(400).json({ failure: { error_reason, message } });
+            }
         });
 
         router.get('/logs/:page_num', async (req, res) => {
@@ -150,8 +154,7 @@ class App {
     private async reparseLogs(): Promise<boolean> {
         let result;
         try {
-            result = await this.pool.query('SELECT log_file1, log_file2 FROM logs WHERE id > 42'); // before 42, wrong log filenames
-            // for (const game of result.rows) {
+            result = await this.pool.query('SELECT id, log_file1, log_file2 FROM logs WHERE id > 42'); // before 42, wrong log filenames
             for (let i = 0, len = result.rows.length; i < len; i++) {
                 const game = result.rows[i];
 
@@ -161,28 +164,52 @@ class App {
                     filenames.push(game.log_file2);
                 }
 
-                console.warn(`${i+1} / ${len} (${Math.round((i+1) / len * 1000) / 10}%) reparsing: ${filenames.join(" +  ")}`);
-
-                const parsedLog = await this.parseLogs(filenames, true /* reparse */);
-                if (!parsedLog) {
-                    console.error(`failed to parse logs ${filenames.join(" + ")}; aborting`);
-                    return false;
+                console.warn(`${i+1} / ${len} (${Math.round((i+1) / len * 1000) / 10}%) reparsing: ${filenames.join(" + ")}`);
+
+                const parsedLog = await this.parseLogs(filenames, { reparse: true });
+                if (!parsedLog.success) {
+                    // if it is a validation failure, mark it and move on to the next log.
+                    if (parsedLog.error_reason === 'MATCH_INVALID') {
+                        console.error(`LOG ${game.id} invalid: ${parsedLog.message}`);
+                        this.pool.query('UPDATE logs SET is_valid = FALSE WHERE id = $1', [game.id]);
+                        continue;
+                    }
+                    else {
+                        console.error(`failed to parse logs ${filenames.join(" + ")}; aborting`);
+                        return false;
+                    }
                 }
             }
         } catch (error: any) {
-            console.error("critical error: failed to connect to DB to reparse logs: " + error?.message);
+            console.error("critical error: failed to connect to DB to reparse logs: " + error?.message || error);
         }
 
         // at least some logs must have been reparsed
         return result && result.rows.length !== 0;
     }
 
-    private async parseLogs(filenames: string[], reparse?: boolean): Promise<string | undefined> {
+    private async parseLogs(filenames: string[], { reparse, skipValidation }: ParsingOptions): Promise<ParseResponse> {
         filenames = await FileCompression.ensureFilesCompressed(filenames, /*deleteOriginals=*/true);
         const parser = new Parser(...filenames)
 
-        return parser.parseRounds()
-            .then(allStats => fileParser(allStats, path.join(this.webserverRoot, this.outputRoot), this.templates, this.pool, reparse));
+        return parser.parseRounds(skipValidation)
+            .then(allStats => fileParser(allStats, path.join(this.webserverRoot, this.outputRoot), this.templates, this.pool, reparse))
+            .catch((error) => {
+                if (error instanceof ParsingError) {
+                    return <ParseResponse>{
+                        success: false,
+                        error_reason: error.name,
+                        message: error.message,
+                    };
+                }
+                else {
+                    return <ParseResponse>{
+                        success: false,
+                        error_reason: 'PARSING_FAILURE',
+                        message: error,
+                    }
+                }
+            });
     }
 }
 

src/classTracker.ts

@@ -2,16 +2,23 @@ import { EventHandlingPhase, EventSubscriber, HandlerRequest } from './eventSubs
 import { RoundState } from './roundState.js'
 import { Event } from './parser.js';
 import EventType from './eventType.js';
+import { ParsingError } from './constants.js';
 
 export class ClassTracker extends EventSubscriber {
     public phaseStart(phase: EventHandlingPhase, _roundState: RoundState): void {
         if (phase !== EventHandlingPhase.AfterGameTimeEpochEstablished)
-            throw "Unexpected phase";
+            throw new ParsingError({
+                name: 'LOGIC_FAILURE',
+                message: "Unexpected phase"
+            });
     }
 
     public phaseEnd(phase: EventHandlingPhase, _roundState: RoundState): void {
         if (phase !== EventHandlingPhase.AfterGameTimeEpochEstablished)
-            throw "Unexpected phase";
+            throw new ParsingError({
+                name: 'LOGIC_FAILURE',
+                message: "Unexpected phase"
+            });
     }
 
     public handleEvent(event: Event, _phase: EventHandlingPhase, _roundState: RoundState): HandlerRequest {
@@ -51,7 +58,10 @@ export class ClassTracker extends EventSubscriber {
                         event.playerToClass = currentClass;
                         return;
                     default:
-                        throw 'unknown playerDirection';
+                        throw new ParsingError({
+                            name: 'LOGIC_FAILURE',
+                            message: 'unknown playerDirection'
+                        });
                 }
             }
         }

src/constants.ts

@@ -2,6 +2,22 @@ import { TeamScore } from "./parserUtils.js";
 import { Event } from "./parser.js";
 import Player from "./player.js";
 
+export type ParseResponse = ParseResponseSuccess | ParseResponseFailure;
+
+interface ParseResponseSuccess {
+    success: true;
+    /** parsedlog slug */
+    message: string;
+};
+
+interface ParseResponseFailure {
+    success: false;
+    /** failure type */
+    error_reason: ParsingErrorName;
+    /** detail about what went wrong */
+    message: string;
+}
+
 export interface OutputStats {
     parse_name: string;  // 'user-friendly' URI slug
     log_name: string; // original log name as uploaded
@@ -324,4 +340,23 @@ export namespace TeamColor {
     export function toString(teamColor: TeamColor): string {
         return (TeamColor as any)[teamColor];
     }
-}
+}
+
+export interface ParsingOptions {
+    reparse?: boolean;
+    skipValidation?: boolean;
+}
+
+export type ParsingErrorName = 'MATCH_INVALID' | 'PARSING_FAILURE' | 'DATABASE_FAILURE' | 'LOGIC_FAILURE';
+export class ParsingError extends Error {
+    name: ParsingErrorName;
+    message: string;
+    cause: any; // unused
+
+    constructor({ name, message, cause}: { name: ParsingErrorName, message: string, cause?: any }) {
+        super();
+        this.name = name;
+        this.message = message;
+        this.cause = cause;
+    }
+}

src/eventSubscriberManager.ts

@@ -1,3 +1,4 @@
+import { ParsingError } from './constants.js';
 import { Event } from './parser.js';
 import { RoundState } from './roundState.js';
 
@@ -69,7 +70,11 @@ export class EventSubscriberManager {
                     }
                     catch (error: any) {
                         console.error(`[subscriber=${subscriber.constructor.name}, phase=${EventHandlingPhase[phase]}] failed (error=${error.message}) when handling line ${event.lineNumber}: ${event.rawLine}`);
-                        throw error;
+
+                        throw new ParsingError({
+                            name: 'LOGIC_FAILURE',
+                            message: error,
+                        });
                     }
                 });
                 if (shouldRemoveEvent) {