refactor: Storage of configs and data

.gitignore

@@ -1,12 +1,13 @@
+config.yaml
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 
 # C extensions
 *.so
-
 .idea/
+
 # Distribution / packaging
 .Python
 build/

README.md

@@ -11,7 +11,7 @@
 
 **Clone the repository:**
 ```bash
-git clone https://github.com/martvdm/TrainAPI.git
+$ git clone https://github.com/martvdm/TrainAPI.git
 ```
 
 **Install packages:**
@@ -21,47 +21,75 @@ pip install -r requirements.txt
 
 **Create a config file:**
 ```bash
-cp config.example.json config.json
+$ cp config.example.yaml config.yaml
 ```
 
-**Fill in the config.json file:**
-
-```json
-{
-  "token": "BOT_TOKEN",  <-- Replace with your discord-bot token
-  "language": "en, nl", <-- Your language
-  "api": {
-    ---> The NS-Primary key is needed to gain acces to the API
-    ---> make an account at "https://apiportal.ns.nl/signin?ReturnUrl=%2F"
-    "NS-PRIMARY": "Ocp-Apim-Subscription-Key" <-- Replace with your NS-Primary key
-  },
-  "app": {
-    "author": "Mart",
-    "invite": "https://discord.com/api/oauth2/authorize?client_id=CLIENT_ID&permissions=8&scope=bot%20applications.commands", <-- Modify the url with your client id
-    "testing": {
-      "server": "TEST-SERVER-ID", <-- Replace with your test server id (optional)
-      "channel": "TEST-CHANNEL-ID" <-- Replace with your test channel id (optional)
-    }
-  },
-  "database": { 
-    "host": "localhost", <-- Replace with your database IP
-    "port": 3036, <-- Replace with your database port
-    "database": "database", <-- Replace with your database name
-    "username": "root", <-- Replace with your database username
-    "password": ""  <-- Replace with your database password
-  }
-}
+**Fill in the config.yaml file:**
+> **Please follow the instructions:**
+>
+> make an account at https://apiportal.ns.nl/signin?ReturnUrl=%2F (You need an account to gain acces to the api)
+```yaml
+app:
+   name: trainAPI # Please fill in the name for your application
+   language: en # Choose between (en & nl)
+   version: 1.0.0 # PLEASE DONT CHANGE THIS
+   maintenance-mode: false # (NOT IMPLEMENTED YET)
+   storage:
+      use-mysql: false # If false the application uses a local file system (JSON)
+ # Customize your app with the following settings
+   customization:
+      theme-color: FFFFFF # This hexcolor-code will be used in embeds
+#----------------------------------------------------------------------------
+#  Discord bot settings
+#----------------------------------------------------------------------------
+   discord:
+        token:  # Your discord bot token
+        maintainers: # Discord id's of maintainers
+            - 287598871373283329
+        presence:
+            show-last-station: false # If set to true: presence text will change when a user requests information about station.
+            ## ^^^ Please note that this will affect the performance
+            default-message: "TrainAPI"
+            default-type: PLAYING #PLAYING, LISTENING, WATCHING, STREAMING
+
+#------------------------------------------------------------------------------
+# Database
+# ONLY SUPPORTS MYSQL
+#------------------------------------------------------------------------------
+# Please note that "use-mysql" must be set to true in the "app" section above, else this section will be ignored.
+#------------------------------------------------------------------------------
+database:
+  host: localhost
+  port: 3306
+  user: root
+  password: root
+  database: trainapi
+
+#------------------------------------------------------------------------------
+# All api credentials
+#------------------------------------------------------------------------------
+api:
+  refresh-interval: 3 # In minutes
+  # ^^^ 3 minutes IS RECOMMENDED, TO MUCH MAY AFFECT THE PERFORMANCE ALSO YOU CAN BE RATE LIMITED BY NS
+  ns-primary-key: SECRETKEY #PRIMARY KEY 
+  # ^^^ This key can be found in the APIPORTAL account dashboard
 ```
 
 **Run the bot:**
 ```bash
-python3 main.py
+$ python main.py
 ```
 
-**Invite the bot to your server**
+**Invite the bot to your server:**
 
-> replace the CLIENT_ID with the client id of your bot
-> 
-> https://discordapp.com/api/oauth2/authorize?client_id=CLIENT_ID&permissions=8&scope=bot%20applications.commands
+> When the application starts up it will send a log in the console. 
+> This log will contain the generated invite link.
 
+Example:
+```bash
+$ Config loaded
+$ Invite link for TrainAPI#5430:
+$ https://discordapp.com/api/oauth2/authorize?client_id=959101335008063558&permissions=544491302336&scope=applications.commands%20bot
+                ^^^ This link redirects to the invite page.
+```
 

__api__.py

@@ -1,12 +1,13 @@
 import json
 import http.client, urllib.request, urllib.parse, urllib.error, base64
+import yaml
 
-with open("./config.json") as jsonfile:
-    config = json.load(jsonfile)
+with open('config.yaml') as file:
+    config = yaml.full_load(file)
 
 def nsapi(url, params):
     headers = {
-        'Ocp-Apim-Subscription-Key': f"{config['api']['NS-PRIMARY']}",
+        'Ocp-Apim-Subscription-Key': f"{config['api']['ns-primary-key']}",
     }
     try:
         conn = http.client.HTTPSConnection('gateway.apiportal.ns.nl')

config.example.yaml

@@ -0,0 +1,50 @@
+#------------------------------------------------------------------------------
+# AUTHOR: Mart van der Molen
+# PLEASE READ THE README.md FILE FOR INSTRUCTIONS
+#
+#  GLOBAL SETTINGS:
+#------------------------------------------------------------------------------
+app:
+   name: trainAPI
+   language: en # Choose between (en & nl)
+   version: 1.0.0 # PLEASE DONT CHANGE THIS
+   maintenance-mode: false
+   storage:
+      use-mysql: false # If false the application uses a local file system (JSON)
+ # Customize your app with the following settings
+   customization:
+      theme-color: FFFFFF
+#----------------------------------------------------------------------------
+#  Discord bot settings
+#----------------------------------------------------------------------------
+   discord:
+        token:  # Your discord bot token
+        maintainers: # Discord id's of maintainers
+            - 287598871373283329
+        presence:
+            show-last-station: false # If set to true: presence text will change when a user requests information about station.
+            ## ^^^ Please note that this will affect the performance
+            default-message: "TrainAPI"
+            default-type: PLAYING #PLAYING, LISTENING, WATCHING, STREAMING
+
+#------------------------------------------------------------------------------
+# Database
+# ONLY SUPPORTS MYSQL
+#------------------------------------------------------------------------------
+# Please note that "use-mysql" must be set to true in the "app" section above, else this section will be ignored.
+#------------------------------------------------------------------------------
+database:
+  host: localhost
+  port: 3306
+  user: root
+  password: root
+  database: trainapi
+
+#------------------------------------------------------------------------------
+# All api credentials
+#------------------------------------------------------------------------------
+api:
+  refresh-interval: 3 #In minutes
+  # ^^^ 3 minutes IS RECOMMENDED, TO MUCH MAY AFFECT THE PERFORMANCE ALSO YOU CAN BE RATE LIMITED BY NS
+  ns-primary-key: SECRETKEY #PRIMARY KEY
+

database/__init__.py

@@ -1,13 +1,18 @@
 import mysql.connector
 import os
 
+def connect(config):
+    dbconfig = config['database']
+    conn = mysql.connector.connect(user=dbconfig['user'], password=dbconfig['password'], host=dbconfig['host'], port=dbconfig['port'], database=dbconfig['database'])
+    return conn
+
 def create_tables(config):
-    conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'], host=config['database']['host'], port=config['database']['port'], database=config['database']['database'])
+    conn = connect(config)
     import database.tables.trips as trips
     import database.tables.notifications as notifications
     import database.tables.disruptions as disruptions
     cursor = conn.cursor()
     disruptions.index(cursor)
     trips.index(cursor)
     notifications.index(cursor)
-    conn.close()
+    conn.close()

database/tables/disruptions.py

@@ -1,6 +1,6 @@
 import pandas as pd
 import mysql
-
+from database.__init__ import connect
 
 def index(cursor):
     sql = '''CREATE TABLE IF NOT EXISTS DISRUPTIONS(
@@ -14,7 +14,7 @@ def index(cursor):
 def check_action(disruption, config):
     action = 'none'
     situation = disruption['timespans'][0]['situation']['label']
-    conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'], host=config['database']['host'], port=config['database']['port'], database=config['database']['database'])
+    conn = connect(config)
     cursor = conn.cursor()
     search1 = f"SELECT * FROM DISRUPTIONS WHERE ID = '{disruption['id']}'"  # Check if the disruption is already in the database
     search2 = f"SELECT * FROM DISRUPTIONS WHERE ID = '{disruption['id']}' AND SITUATION = '{situation}'" # Check if the situation has changed

database/tables/notifications.py

@@ -1,5 +1,4 @@
-import mysql
-
+from database.__init__ import connect
 
 def index(cursor):
     sql = '''CREATE TABLE IF NOT EXISTS NOTIFICATIONS(
@@ -14,9 +13,7 @@ def index(cursor):
 def create(config, action, client_id, station):
     stationcode = station['stationCode']
     import pandas as pd
-    conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'],
-                                   host=config['database']['host'], port=config['database']['port'],
-                                   database=config['database']['database'])
+    conn = connect(config)
     cursor = conn.cursor()
     searchnotification = pd.read_sql(f"SELECT * FROM NOTIFICATIONS WHERE STATION = '{stationcode}' AND CLIENT_ID = '{client_id}'", conn)
     if action == 'subscribe':
@@ -41,9 +38,7 @@ def create(config, action, client_id, station):
 
 def find_users(config, stationcode):
     import pandas as pd
-    conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'],
-                                   host=config['database']['host'], port=config['database']['port'],
-                                   database=config['database']['database'])
+    conn = connect(config)
     sql = f"SELECT CLIENT_ID FROM NOTIFICATIONS WHERE STATION = '{stationcode}'"
     users = pd.read_sql(sql, conn)
     conn.close()

main.py

@@ -8,28 +8,29 @@
 from discord_slash.utils.manage_commands import create_option, create_choice
 import modules.commands as commandmodule
 import asyncio
+import yaml
 
 client = commands.Bot(command_prefix='!')
 slash = SlashCommand(client, sync_commands=True)
 
-with open("config.json") as jsonfile:
-    config = json.load(jsonfile)
+with open('config.yaml') as file:
+    config = yaml.full_load(file)
     print('\033[1;32mConfig loaded')
 
-
 @client.event
 async def on_ready():
-    print('\033[92mLoading data... \n \033[94mLoaded client: {0.user}'.format(client))
+    invitelink = f'https://discordapp.com/api/oauth2/authorize?client_id={client.user.id}&permissions=544491302336&scope=applications.commands%20bot'
+    print(f'Invite link for {client.user}:')
+    print(invitelink)
     import database.__init__ as db
     db.create_tables(config)  # Create tables if they don't exist
     import warnings
     warnings.filterwarnings("ignore", category=UserWarning)  # PandaSQL warning
-    print('\033[92mLoaded database')  # Print success message for table creation
-    await client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=f"ov-NL"))
+    await client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=f"{config['app']['discord']['presence']['default-message']}"))
     loops.start()  # Start loop for updating the database
 
 
-@tasks.loop(minutes=2)
+@tasks.loop(minutes=config['api']['refresh-interval'])
 async def loops():
     from modules.commands.notify import checknotifications
     await checknotifications(client, config)
@@ -68,4 +69,4 @@ async def notify(ctx, *, action, station):
     await index(ctx, action, station, config, client)
 
 
-client.run(config['token'])
+client.run(config['app']['discord']['token'])

modules/commands/station.py

@@ -14,11 +14,11 @@
 async def index(ctx, station, config, client):
     station = get_station(station)
     stationcode = station['stationCode']
-    embed = discord.Embed(title="Station: ", description=f"{station['name']}", color=0x000065)
+    embed = discord.Embed(title="Station: ", description=f"{station['name']}", color=config['app']['customization']['theme-color'])
     from __api__ import nsapi
     params = urllib.parse.urlencode({
         # Request parameters
-        'lang': f'{config["language"]}',
+        'lang': f'en',
         'station': f'{stationcode}',
         'uicCode': '',
         'dateTime': '',
@@ -60,7 +60,7 @@ async def departures(ctx, station, config, client):
     from __api__ import nsapi
     params = urllib.parse.urlencode({
         # Request parameters
-        'lang': f'{config["language"]}',
+        'lang': f'en',
         'station': f'{stationcode}',
         'uicCode': '',
         'dateTime': '',

request.py

@@ -1,10 +1,10 @@
 ## All request functions for the API's
 from __api__ import nsapi
 import http.client, urllib.request, urllib.parse, urllib.error, base64
-import json
+import yaml
 
-with open("config.json") as jsonfile:
-    config = json.load(jsonfile)
+with open('config.yaml') as file:
+    config = yaml.full_load(file)
 ## NS API
 
 
@@ -13,7 +13,7 @@ def get_station(station):
         'q':  f'{station}',
         'limit': 8,
         'details': 'false',
-        'lang': f'{ config["language"] }',
+        'lang': config['app']['language'],
     })
     url = f"/places-api/v2/places"
     get_results = nsapi(url, params)
@@ -25,7 +25,7 @@ def get_station(station):
         'station_code': f'{stationcode}',
         'limit': 1,
         'details': 'false',
-        'lang': f'{config["language"]}',
+        'lang': config['app']['language'],
     })
     station = nsapi(url, params)
     return station['payload'][0]['locations'][0]