Native globus rewrite + some more refactoring (#125)

* refactored statistic

* refactoring + documentation

* finished refactoring on maintainers

* finished connector refactoring

* finished eslinting all remaining ts files

* documented server.ts + added markdown files

* refactored cli.ts

* fixed some typing issues

* revamped packages + redid express structure + more linting

* more type hinting + async removal on models

* linting on main src/ files

* changes to type assertions

* finished maintainers

* make types more meaningful

* finished with util rework + minor changes elsewhere

* finished linting + helper rework + better nullguarding + max line size

* fixed helper typo & slurmconnector doc

* add specialized manifest download functionality + lint action

* documentation initial commit + helper moving + abstract functions + initial lazy supervisor

* finished codebase overview doc

* finished codebase documentation doc

* added caching to folder uploaders + a bit more linting

* disable cached upload on data

* added cache refresing functionality

* fixed swagger error

* fixed cli

* readd typedoc

* more package updates

* added mysql

* disable uml

* move all documentation out of build

* fixed generate swagger bug

* reenable documentation

* fixed gitutil bug + updated packages

* recursive mkdirs

* more gitutil bugfixes

* silenced console.logs

* reenabled urlencoded

* fixed folder uploader bug

* more bugfixing + removing console logs

* fixed remotefsexists bug

* job upload now works

* update build script to copy example jsons

* fixed github actions

* added ruff for the python files for now

* add js globus sdk

* added typing to cachedupload

* switched to git sdk

* switch to fs promises

* automated branch resolution + out of date checking

* moved configs to example

* undo config.json move

* fixed action typo

* removed all refreshing code + added automatic logic to decide when to re-upload git repo

* added yaml parser

* add js globus sdk

* sync package lock

* rework folder typing to be more enumy

* leftover changes from folder rework

* added cache model

* force synchronize

* synchronize in the json

* actually force reinitialize

* turned strict on + updated packages + migrated to typeorm 0.3 + migrated to redis 5 + cleaned up redis classes

* reworked tsconfig

* remove typeorm-uml

* actually disable uml + add cache to entities

* actually disable uml instead of tsdoc

* fixed redis

* connect to redis client

* reworked cache + deprecated specialized manifest + added more git caching

* reworked refreshing in getexecutablemanifest

* reworked db to not be a class per typeorm guidelines

* leftover db port changes

* remove ormconfig

* initialize database in server.ts

* fixed some minor errors

* initialize datasource in init hello world

* reenable register

* move this.register

* fix db registering

* fix db registering

* add print statements

* debugging job2object

* updated dockerfile + removed print statements

* fixed typing on nullable model columns

* better undefined/null handling + typeorm function fixes

* added more is_testing clauses to console logs

* fixed registercache

* more fixes to hpcpath -> cachepath

* disable removing comments

* styling change + git manifest cache + new git routes

* rework server structure

* initialize cache

* change rimraf

* changed promises to awaits

* change unix time scaling

* fix linting in cli

* undo examples folder

* rename a file

* fixed merge bugs

* fixed linting

* change the eslint config

* revert back to commonjs

* change global variables

* disable typeorm redis caching

* update packages

* changed file structure + updated linting

* update file paths

* update file paths again

* fix paths and eslint

* more path fixes

* added axios-based globus ts impl

* fix globus imports

* fix entities

* more typeorm fixes

* fix typeorm again

* change a path

* add logging

* add submission id retrieval

cli.ts

@@ -1,6 +1,8 @@
 import { Command } from "commander";
-import dataSource from "./src/DB";
+
 import { Git } from "./src/models/Git";
+import dataSource from "./src/utils/DB";
+
 const pkg: {version: string} = require("../package.json"); // eslint-disable-line
 const cmd = new Command();
 

configs/config.ts

@@ -5,7 +5,7 @@ import {
  containerConfig,
  jupyterGlobusMapConfig,
  kernelConfig,
-} from "../src/types";
+} from "../src/utils/types";
 // eslint-disable-next-line import/order
 import rawConfig from "../config.json"; // base config
 import rawContainerConfig from "./container.json"; // docker container config

docker/docker-entrypoint.sh

@@ -29,4 +29,4 @@ EOT
 
 # run server
 redis-server /job_supervisor/docker/redis.conf --daemonize yes
-node /job_supervisor/production/server.js
+npm run start

docs/codebase.md

@@ -49,7 +49,7 @@ Dependencies:
 - `src/DB.ts`: provides a reference to mutate the underlying database *(not used)*
 - `src/errors.ts`: used for a custom `ConnectorError` on a failed SSH connection
 - `src/Helper.ts`: general helper functions (mainly type guards)
-- `src/lib/FileUtil.ts`: used for some helpful zipping/local file functions for the host server *(possibly wrong import, FileUtil does not exist)*
+- `src/shared/FileUtil.ts`: used for some helpful zipping/local file functions for the host server *(possibly wrong import, FileUtil does not exist)*
 - `src/BaseMaintainer.ts`: used to allow the relevant logs to propagate to the maintainer of the connector's job
 - `src/types.ts`: various custom types
 - `src/connectors/connectionPool.ts`: used to get the SSH configs to open SSH connections
@@ -74,7 +74,7 @@ Improvements
 
 ### Utility classes
 
-These classes are found in the `src/lib/` directory, and these generally serve as a static helper functions abstracting away more basic operations for other parts of the codebase. The functionalities for each one are described below:
+These classes are found in the `src/shared/` directory, and these generally serve as a static helper functions abstracting away more basic operations for other parts of the codebase. The functionalities for each one are described below:
 
 - `FolderUtil`: This class generally deals with files on the local server, providing functions that can zip and unzip folders/files along with functions that can detect if a file is zipped and delete folders/files. 
 
@@ -88,7 +88,7 @@ These classes are found in the `src/lib/` directory, and these generally serve a
 
  - `src/FolderUploader.ts`: to handle zipping/unzipping on local file uploads
  - `src/BaseConnector.ts`: for unzipping upon downloading something
- - `src/lib/GitUtil.ts`: unzipping files
+ - `src/shared/GitUtil.ts`: unzipping files
 
 - `GitUtil`: This class handles interactions with [github](https://github.com) repositories on the hosting server. It supports things like cloning/refreshing/pulling repositores from the GitHub remote link, and, more importantly, it supports the retrieval and processing of executable manifests stored in job GitHub repositores that describe how a job should be run. 
  - Update: new functions that just `wget` the manifests instead of fully repulling have been added as of commit `e2a4f0`. 
@@ -110,7 +110,7 @@ These classes are found in the `src/lib/` directory, and these generally serve a
  - `configs/config.ts`: needed to get redis credentials and globus ids
  - `src/DB.ts`: used to interface with the `globus_transfer_refresh_token` database to get the refresh token associated with an hpc's globus identity
  - `src/types.ts`: import custom functions needed for redis & configs/folders
- - `src/lib/Helper.ts`: for null guard type assertions
+ - `src/shared/Helper.ts`: for null guard type assertions
  - `src/PythonUtil.ts`: used to actually execute the python scripts that directly interface with globus via the Python SDK
  - Dependents:
  - `server.ts`: needed for a route that gives information regarding globus and to support downloading folders from the server
@@ -134,7 +134,7 @@ These classes are found in the `src/lib/` directory, and these generally serve a
  - `child_process`: used to run the python programs externally via forking
  - `configs/config.ts`: used for a testing flag controlling print statements
  - Dependents:
- - `src/lib/GlobusUtil.ts`: uses these functions to interface with globus via python scripts
+ - `src/shared/GlobusUtil.ts`: uses these functions to interface with globus via python scripts
  - `tools/globus-refresh-transfer-token.ts`: also uses these functions to interface with globus via python scripts
 - `XSEDEUtil`: This class provides functionality for accessing [XSEDE](https://www.xsede.org/) commands--in particular, it is able to log jobs to XSEDE, given that the HPC in question has the credentials to do so in the config. 
  - Dependencies: 
@@ -170,13 +170,13 @@ Dependencies:
 - `configs/config.ts`: used to access testing flags, hpc configs, and maintainer configs
 - `src/connectors/`: allows maintainers to actually communicate with HPCs
 - `src/DB.ts` & `src/models/*`: used to interface with `Job` repo for updates
-- `src/lib/Helper.ts`: general utility (null guarding)
+- `src/shared/Helper.ts`: general utility (null guarding)
 - `src/Supervisor.ts`: unused (supposed to be parent pointer)
 - `src/types.ts`: various custom types for configs/events/slurm
 - `src/FolderUploader.ts`: used to upload things to the HPC (in `CommunityContributionMaintainer`)
 - `src/GitUtil.ts`: used to get executable manifests for a given job (associated with a git repo; in `CommunityContributionMaintainer`)
-- `src/lib/JobUtil.ts`: used to have a result folder content manager (unclear whether this is separate or the same as the one in `server.ts`)
-- `src/lib/XSEDEUtil`: for job logging to XSEDE
+- `src/shared/JobUtil.ts`: used to have a result folder content manager (unclear whether this is separate or the same as the one in `server.ts`)
+- `src/shared/XSEDEUtil`: for job logging to XSEDE
 
 Dependents:
 
@@ -352,7 +352,7 @@ Dependencies:
 - `redis`: to implement the key-value store to keep track of credentials
 - `util`: for `redis` function storing with `promisify`
 - `configs/config.ts`: to get redis credentails
-- `src/lib/Helper.ts`: generating random ids for credentials (given the time)
+- `src/shared/Helper.ts`: generating random ids for credentials (given the time)
 - `src/types.ts`: miscellaneous assorted custom types used for typing
 
 Dependents:
@@ -388,7 +388,7 @@ Dependencies:
 - `src/connectors/ConnectionPool.ts`: used for keeping track of the number of jobs on each HPC & for ad-hoc additions of SSH connections for jobs with private credentials
 - `src/DB.ts` & `src/models/`: used to connect with the database to log errors with job initialization (and set the respective finishedAt time) and to register job events/logs upon job completion
 - `src/Emitter.ts`: used to register events/logs generated during the job maintain cycle
-- `src/lib/Helper.ts`: null/assertion checking
+- `src/shared/Helper.ts`: null/assertion checking
 - `src/maintainers/BaseMaintainer`: used to allow supervisors to create maintainers for queued jobs
 - `src/Queue.ts`: used to implement the job queue of a supervisor
 - `src/types.ts`: miscellaneous custom typings used

eslint.config.js

@@ -1,32 +1,30 @@
-/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-var-requires */
+/* eslint-disable */
 
-const globals = require("globals");
 const eslint = require("@eslint/js");
 const tseslint = require("typescript-eslint");
 const importPlugin = require("eslint-plugin-import");
-const { FlatCompat } = require("@eslint/eslintrc");
 const stylistic = require("@stylistic/eslint-plugin");
+const globals = require("globals");
 
-const compat = new FlatCompat({
- baseDirectory: __dirname,
-});
 
-module.exports = [
- ...compat.config({
- env: {
- node: true
- },
- parserOptions: {
- ecmaVersion: "latest",
- project: ["tsconfig(.*)?.json"]
- }
- }),
- {
- languageOptions: { globals: globals.browser },
- },
+module.exports = tseslint.config(
  eslint.configs.recommended,
+
  ...tseslint.configs.recommendedTypeChecked,
  ...tseslint.configs.stylisticTypeChecked,
+ {
+ languageOptions: {
+ parserOptions: {
+ project: true,
+ tsconfigRootDir: __dirname
+ },
+ globals: {
+ ...globals.node
+ }
+ },
+
+ },
+
  {
  files: ["**/*.ts", "**/*.tsx"],
  plugins: {
@@ -66,7 +64,7 @@ module.exports = [
  order: "asc",
  },
  groups: ["external", "builtin", "parent", ["sibling", "index"]],
- "newlines-between": "never",
+ "newlines-between": "always",
  pathGroups: [
  {
  group: "external",
@@ -103,4 +101,8 @@ module.exports = [
  ]
  },
  },
-];
+
+ {
+ ignores: ["production/*"]
+ }
+);