From 34c6fda713003e78aa781292954885269a8d6eba Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:12:39 +0200 Subject: [PATCH 01/16] feat: config.yaml to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e1a075b..5647447 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +config.yaml # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -5,8 +6,8 @@ __pycache__/ # C extensions *.so - .idea/ + # Distribution / packaging .Python build/ From e5ce0ca2602ed9efc3a7a28740c8cc85b70eaf1f Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:13:52 +0200 Subject: [PATCH 02/16] feat: config json -> yaml --- config.example.json | 22 --------------------- config.example.yaml | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) delete mode 100644 config.example.json create mode 100644 config.example.yaml diff --git a/config.example.json b/config.example.json deleted file mode 100644 index 875afa6..0000000 --- a/config.example.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "token": "BOT_TOKEN", - "language": "en, nl", - "api": { - "NS-PRIMARY": "Ocp-Apim-Subscription-Key" - }, - "app": { - "author": "Mart", - "invite": "https://discord.com/api/oauth2/authorize?client_id=CLIENT_ID&permissions=8&scope=bot%20applications.commands", - "testing": { - "server": "TEST-SERVER-ID", - "channel": "TEST-CHANNEL-ID" - } - }, - "database": { - "host": "localhost", - "port": 3036, - "database": "database", - "username": "root", - "password": "" - } -} \ No newline at end of file diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..98be246 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,48 @@ +#------------------------------------------------------------------------------ +# AUTHOR: Mart van der Molen +# PLEASE READ THE README.md FILE FOR INSTRUCTIONS +# +# GLOBAL SETTINGS: +#------------------------------------------------------------------------------ +app: + name: trainAPI + version: 1.0.0 # PLEASE DONT CHANGE THIS + maintenance-mode: false + storage: + use-mysql: false # If false the application uses a local file system (JSON) + # Customize your app with the following settings + customization: + theme-color: FFFFFF + presence-text: TrainAPI +#---------------------------------------------------------------------------- +# Discord bot settings +#---------------------------------------------------------------------------- + discord: + token: # Your discord bot token + maintainers: # List of discord user ids + - 287598871373283329 + presence: + show-last-station: true + default-message: "TrainAPI" + default-type: PLAYING #PLAYING, LISTENING, WATCHING, STREAMING + +#------------------------------------------------------------------------------ +# Database +# ONLY SUPPORTS MYSQL +#------------------------------------------------------------------------------ +# Please note that "use-mysql" must be set to true in the "app" section above, else this section will be ignored. +#------------------------------------------------------------------------------ +database: + host: localhost + port: 3306 + user: root + password: root + database: trainapi + +#------------------------------------------------------------------------------ +# All api credentials +#------------------------------------------------------------------------------ +api: + refresh-interval: 60 # Refresh interval in seconds + ns-primary-key: SECRETKEY #PRIMARY KEY + From eaf1d50863c27f97c5eb0b04aeffb79852258686 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:14:19 +0200 Subject: [PATCH 03/16] refactor: requirements.txt add yaml --- requirements.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a54bcad..9a7c53a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ discord.py -asyncio +asyncio~=3.4.3 discord-py-slash-command -datetime -mysql +datetime~=4.4 +mysql~=0.0.3 mysql-connector-python==8.0.29 -pandas \ No newline at end of file +pandas~=1.4.2 +discord~=1.7.3 +PyYAML~=6.0 \ No newline at end of file From 5b22c4634b9531b8aedc19b57c92290fba314eeb Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:16:03 +0200 Subject: [PATCH 04/16] feat: database conn function --- database/__init__.py | 9 +++++++-- database/tables/disruptions.py | 4 ++-- database/tables/notifications.py | 11 +++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/database/__init__.py b/database/__init__.py index 964b803..aed56c3 100644 --- a/database/__init__.py +++ b/database/__init__.py @@ -1,8 +1,13 @@ import mysql.connector import os +def connect(config): + dbconfig = config['database'] + conn = mysql.connector.connect(user=dbconfig['user'], password=dbconfig['password'], host=dbconfig['host'], port=dbconfig['port'], database=dbconfig['database']) + return conn + def create_tables(config): - conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'], host=config['database']['host'], port=config['database']['port'], database=config['database']['database']) + conn = connect(config) import database.tables.trips as trips import database.tables.notifications as notifications import database.tables.disruptions as disruptions @@ -10,4 +15,4 @@ def create_tables(config): disruptions.index(cursor) trips.index(cursor) notifications.index(cursor) - conn.close() \ No newline at end of file + conn.close() diff --git a/database/tables/disruptions.py b/database/tables/disruptions.py index 37f6923..858410c 100644 --- a/database/tables/disruptions.py +++ b/database/tables/disruptions.py @@ -1,6 +1,6 @@ import pandas as pd import mysql - +from database.__init__ import connect def index(cursor): sql = '''CREATE TABLE IF NOT EXISTS DISRUPTIONS( @@ -14,7 +14,7 @@ def index(cursor): def check_action(disruption, config): action = 'none' situation = disruption['timespans'][0]['situation']['label'] - conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'], host=config['database']['host'], port=config['database']['port'], database=config['database']['database']) + conn = connect(config) cursor = conn.cursor() search1 = f"SELECT * FROM DISRUPTIONS WHERE ID = '{disruption['id']}'" # Check if the disruption is already in the database search2 = f"SELECT * FROM DISRUPTIONS WHERE ID = '{disruption['id']}' AND SITUATION = '{situation}'" # Check if the situation has changed diff --git a/database/tables/notifications.py b/database/tables/notifications.py index f106825..79401e4 100644 --- a/database/tables/notifications.py +++ b/database/tables/notifications.py @@ -1,5 +1,4 @@ -import mysql - +from database.__init__ import connect def index(cursor): sql = '''CREATE TABLE IF NOT EXISTS NOTIFICATIONS( @@ -14,9 +13,7 @@ def index(cursor): def create(config, action, client_id, station): stationcode = station['stationCode'] import pandas as pd - conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'], - host=config['database']['host'], port=config['database']['port'], - database=config['database']['database']) + conn = connect(config) cursor = conn.cursor() searchnotification = pd.read_sql(f"SELECT * FROM NOTIFICATIONS WHERE STATION = '{stationcode}' AND CLIENT_ID = '{client_id}'", conn) if action == 'subscribe': @@ -41,9 +38,7 @@ def create(config, action, client_id, station): def find_users(config, stationcode): import pandas as pd - conn = mysql.connector.connect(user=config['database']['username'], password=config['database']['password'], - host=config['database']['host'], port=config['database']['port'], - database=config['database']['database']) + conn = connect(config) sql = f"SELECT CLIENT_ID FROM NOTIFICATIONS WHERE STATION = '{stationcode}'" users = pd.read_sql(sql, conn) conn.close() From c6036bb2783ab7c510b8b14443212c90937c2fc9 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:16:40 +0200 Subject: [PATCH 05/16] refactor: __api__.py to new config --- __api__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/__api__.py b/__api__.py index fe1c00f..6f00ff4 100644 --- a/__api__.py +++ b/__api__.py @@ -1,12 +1,13 @@ import json import http.client, urllib.request, urllib.parse, urllib.error, base64 +import yaml -with open("./config.json") as jsonfile: - config = json.load(jsonfile) +with open('config.yaml') as file: + config = yaml.full_load(file) def nsapi(url, params): headers = { - 'Ocp-Apim-Subscription-Key': f"{config['api']['NS-PRIMARY']}", + 'Ocp-Apim-Subscription-Key': f"{config['api']['ns-primary-key']}", } try: conn = http.client.HTTPSConnection('gateway.apiportal.ns.nl') From ca25d44bfa38dc5dada9bee8e6c113ed148f45cb Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:17:11 +0200 Subject: [PATCH 06/16] refactor: main.py to new config --- main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 83ba6b0..fd9c280 100644 --- a/main.py +++ b/main.py @@ -8,15 +8,15 @@ from discord_slash.utils.manage_commands import create_option, create_choice import modules.commands as commandmodule import asyncio +import yaml client = commands.Bot(command_prefix='!') slash = SlashCommand(client, sync_commands=True) -with open("config.json") as jsonfile: - config = json.load(jsonfile) +with open('config.yaml') as file: + config = yaml.full_load(file) print('\033[1;32mConfig loaded') - @client.event async def on_ready(): print('\033[92mLoading data... \n \033[94mLoaded client: {0.user}'.format(client)) @@ -25,7 +25,7 @@ async def on_ready(): import warnings warnings.filterwarnings("ignore", category=UserWarning) # PandaSQL warning print('\033[92mLoaded database') # Print success message for table creation - await client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=f"ov-NL")) + await client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=f"{config['app']['discord']['presence']['default-message']}")) loops.start() # Start loop for updating the database @@ -68,4 +68,4 @@ async def notify(ctx, *, action, station): await index(ctx, action, station, config, client) -client.run(config['token']) +client.run(config['app']['discord']['token']) From aafdced51f7da806c5306a76e5b3d5fb7bd1ef1f Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:17:45 +0200 Subject: [PATCH 07/16] refactor: api request language default english --- modules/commands/station.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/commands/station.py b/modules/commands/station.py index 8e72849..0e677cc 100644 --- a/modules/commands/station.py +++ b/modules/commands/station.py @@ -14,11 +14,11 @@ async def index(ctx, station, config, client): station = get_station(station) stationcode = station['stationCode'] - embed = discord.Embed(title="Station: ", description=f"{station['name']}", color=0x000065) + embed = discord.Embed(title="Station: ", description=f"{station['name']}", color=config['app']['customization']['theme-color']) from __api__ import nsapi params = urllib.parse.urlencode({ # Request parameters - 'lang': f'{config["language"]}', + 'lang': f'en', 'station': f'{stationcode}', 'uicCode': '', 'dateTime': '', @@ -60,7 +60,7 @@ async def departures(ctx, station, config, client): from __api__ import nsapi params = urllib.parse.urlencode({ # Request parameters - 'lang': f'{config["language"]}', + 'lang': f'en', 'station': f'{stationcode}', 'uicCode': '', 'dateTime': '', From b4d0fcb0a6cfcdaea9c08e9d3b47f56713cfff0f Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:22:31 +0200 Subject: [PATCH 08/16] feat: add language to config --- config.example.yaml | 1 + request.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 98be246..e069fac 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -6,6 +6,7 @@ #------------------------------------------------------------------------------ app: name: trainAPI + language: en # Choose between (en & nl) version: 1.0.0 # PLEASE DONT CHANGE THIS maintenance-mode: false storage: diff --git a/request.py b/request.py index ec43477..1f6ba36 100644 --- a/request.py +++ b/request.py @@ -2,9 +2,10 @@ from __api__ import nsapi import http.client, urllib.request, urllib.parse, urllib.error, base64 import json +import yaml -with open("config.json") as jsonfile: - config = json.load(jsonfile) +with open('config.yaml') as file: + config = yaml.full_load(file) ## NS API From 2eb482ce9241049399ec96f1a3307f0c1dce9848 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:26:59 +0200 Subject: [PATCH 09/16] fix: config in request.py --- request.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/request.py b/request.py index 1f6ba36..d0ff929 100644 --- a/request.py +++ b/request.py @@ -1,7 +1,6 @@ ## All request functions for the API's from __api__ import nsapi import http.client, urllib.request, urllib.parse, urllib.error, base64 -import json import yaml with open('config.yaml') as file: @@ -14,7 +13,7 @@ def get_station(station): 'q': f'{station}', 'limit': 8, 'details': 'false', - 'lang': f'{ config["language"] }', + 'lang': config['app']['language'], }) url = f"/places-api/v2/places" get_results = nsapi(url, params) @@ -26,7 +25,7 @@ def get_station(station): 'station_code': f'{stationcode}', 'limit': 1, 'details': 'false', - 'lang': f'{config["language"]}', + 'lang': config['app']['language'], }) station = nsapi(url, params) return station['payload'][0]['locations'][0] From 9492685012309f4e024bf1b32ed254e3614d3e9b Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:49:15 +0200 Subject: [PATCH 10/16] refactor(docs): more clearly comments --- config.example.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index e069fac..c0f0f21 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,16 +14,16 @@ app: # Customize your app with the following settings customization: theme-color: FFFFFF - presence-text: TrainAPI #---------------------------------------------------------------------------- # Discord bot settings #---------------------------------------------------------------------------- discord: token: # Your discord bot token - maintainers: # List of discord user ids + maintainers: # Discord id's of maintainers - 287598871373283329 presence: - show-last-station: true + show-last-station: false # If set to true: presence text will change when a user requests information about station. + ## ^^^ Please note that this will affect the performance default-message: "TrainAPI" default-type: PLAYING #PLAYING, LISTENING, WATCHING, STREAMING @@ -44,6 +44,7 @@ database: # All api credentials #------------------------------------------------------------------------------ api: - refresh-interval: 60 # Refresh interval in seconds + refresh-interval: 3 #In minutes + # ^^^ 3 minutes IS RECOMMENDED, TO MUCH MAY AFFECT THE PERFORMANCE ALSO YOU CAN BE RATE LIMITED BY NS ns-primary-key: SECRETKEY #PRIMARY KEY From 7b504e7d23100ff17da5a22cdc15b774d72bcf5c Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 17:49:45 +0200 Subject: [PATCH 11/16] refactor(docs): update README.md for new config --- README.md | 84 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index b53c462..2997795 100644 --- a/README.md +++ b/README.md @@ -21,47 +21,69 @@ pip install -r requirements.txt **Create a config file:** ```bash -cp config.example.json config.json +cp config.example.yaml config.yaml ``` -**Fill in the config.json file:** - -```json -{ - "token": "BOT_TOKEN", <-- Replace with your discord-bot token - "language": "en, nl", <-- Your language - "api": { - ---> The NS-Primary key is needed to gain acces to the API - ---> make an account at "https://apiportal.ns.nl/signin?ReturnUrl=%2F" - "NS-PRIMARY": "Ocp-Apim-Subscription-Key" <-- Replace with your NS-Primary key - }, - "app": { - "author": "Mart", - "invite": "https://discord.com/api/oauth2/authorize?client_id=CLIENT_ID&permissions=8&scope=bot%20applications.commands", <-- Modify the url with your client id - "testing": { - "server": "TEST-SERVER-ID", <-- Replace with your test server id (optional) - "channel": "TEST-CHANNEL-ID" <-- Replace with your test channel id (optional) - } - }, - "database": { - "host": "localhost", <-- Replace with your database IP - "port": 3036, <-- Replace with your database port - "database": "database", <-- Replace with your database name - "username": "root", <-- Replace with your database username - "password": "" <-- Replace with your database password - } -} +**Fill in the config.yaml file:** +> **Please follow the instructions:** +> +> make an account at https://apiportal.ns.nl/signin?ReturnUrl=%2F (You need an account to gain acces to the api) +```yaml +app: + name: trainAPI # Please fill in the name for your application + language: en # Choose between (en & nl) + version: 1.0.0 # PLEASE DONT CHANGE THIS + maintenance-mode: false # (NOT IMPLEMENTED YET) + storage: + use-mysql: false # If false the application uses a local file system (JSON) + # Customize your app with the following settings + customization: + theme-color: FFFFFF # This hexcolor-code will be used in embeds +#---------------------------------------------------------------------------- +# Discord bot settings +#---------------------------------------------------------------------------- + discord: + token: # Your discord bot token + maintainers: # Discord id's of maintainers + - 287598871373283329 + presence: + show-last-station: false # If set to true: presence text will change when a user requests information about station. + ## ^^^ Please note that this will affect the performance + default-message: "TrainAPI" + default-type: PLAYING #PLAYING, LISTENING, WATCHING, STREAMING + +#------------------------------------------------------------------------------ +# Database +# ONLY SUPPORTS MYSQL +#------------------------------------------------------------------------------ +# Please note that "use-mysql" must be set to true in the "app" section above, else this section will be ignored. +#------------------------------------------------------------------------------ +database: + host: localhost + port: 3306 + user: root + password: root + database: trainapi + +#------------------------------------------------------------------------------ +# All api credentials +#------------------------------------------------------------------------------ +api: + refresh-interval: 3 # In minutes + # ^^^ 3 minutes IS RECOMMENDED, TO MUCH MAY AFFECT THE PERFORMANCE ALSO YOU CAN BE RATE LIMITED BY NS + ns-primary-key: SECRETKEY #PRIMARY KEY + # ^^^ This key can be found in the APIPORTAL account dashboard ``` **Run the bot:** ```bash -python3 main.py +python main.py ``` **Invite the bot to your server** -> replace the CLIENT_ID with the client id of your bot +> replace the {CLIENT_ID} with the client id of your bot > -> https://discordapp.com/api/oauth2/authorize?client_id=CLIENT_ID&permissions=8&scope=bot%20applications.commands +> https://discordapp.com/api/oauth2/authorize?client_id={CLIENT_ID}&permissions=544491302336&scope=applications.commands%20bot From bac5903e074083d8f6ab58a12a10b0b0829a26a1 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 18:02:13 +0200 Subject: [PATCH 12/16] feat: Generate invite link on start --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index fd9c280..83b24d4 100644 --- a/main.py +++ b/main.py @@ -19,7 +19,9 @@ @client.event async def on_ready(): - print('\033[92mLoading data... \n \033[94mLoaded client: {0.user}'.format(client)) + invitelink = f'https://discordapp.com/api/oauth2/authorize?client_id={client.user.id}&permissions=544491302336&scope=applications.commands%20bot' + print(f'Invite link for {client.user}:') + print(invitelink) import database.__init__ as db db.create_tables(config) # Create tables if they don't exist import warnings From 09e70ff8ae12c071b58c5532ab23c557045df049 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 18:02:44 +0200 Subject: [PATCH 13/16] refactor: removed message database loaded --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 83b24d4..eee410a 100644 --- a/main.py +++ b/main.py @@ -26,7 +26,6 @@ async def on_ready(): db.create_tables(config) # Create tables if they don't exist import warnings warnings.filterwarnings("ignore", category=UserWarning) # PandaSQL warning - print('\033[92mLoaded database') # Print success message for table creation await client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=f"{config['app']['discord']['presence']['default-message']}")) loops.start() # Start loop for updating the database From 2cc080a77e9df21ed8c505b0ec04db712f62e40c Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 18:03:12 +0200 Subject: [PATCH 14/16] refactor: refresh interval of loop configurable --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index eee410a..c655322 100644 --- a/main.py +++ b/main.py @@ -30,7 +30,7 @@ async def on_ready(): loops.start() # Start loop for updating the database -@tasks.loop(minutes=2) +@tasks.loop(minutes=config['api']['refresh-interval']) async def loops(): from modules.commands.notify import checknotifications await checknotifications(client, config) From 19a0920bd859045a68ba5475cbc612c9a28e4c79 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 18:41:22 +0200 Subject: [PATCH 15/16] refactor(docs): added link generator to README.md --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2997795..2f4dc6e 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,16 @@ api: python main.py ``` -**Invite the bot to your server** +**Invite the bot to your server:** -> replace the {CLIENT_ID} with the client id of your bot -> -> https://discordapp.com/api/oauth2/authorize?client_id={CLIENT_ID}&permissions=544491302336&scope=applications.commands%20bot +> When the application starts up it will send a log in the console. +> This log will contain the generated invite link. +Example: +```bash +$ Config loaded +$ Invite link for TrainAPI#5430: +$ https://discordapp.com/api/oauth2/authorize?client_id=959101335008063558&permissions=544491302336&scope=applications.commands%20bot + ^^^ This link redirects to the invite page. +``` From df5cd2c0c1f82935ffcce5be0bdd2f29f91d2396 Mon Sep 17 00:00:00 2001 From: Martvdm Date: Sat, 30 Apr 2022 20:01:31 +0200 Subject: [PATCH 16/16] refactor(docs): dollars to bash --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2f4dc6e..eacbdb7 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Clone the repository:** ```bash -git clone https://github.com/martvdm/TrainAPI.git +$ git clone https://github.com/martvdm/TrainAPI.git ``` **Install packages:** @@ -21,7 +21,7 @@ pip install -r requirements.txt **Create a config file:** ```bash -cp config.example.yaml config.yaml +$ cp config.example.yaml config.yaml ``` **Fill in the config.yaml file:** @@ -77,7 +77,7 @@ api: **Run the bot:** ```bash -python main.py +$ python main.py ``` **Invite the bot to your server:**