From c7288fb7c3c96a1c7080f3d8b85fbeb5db823142 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 23 Mar 2023 09:46:41 +0100 Subject: [PATCH 1/5] Move egio functions to etrago --- etrago/tools/constraints.py | 2 +- etrago/tools/db.py | 200 ++++++++++++++++++++++++++++++++++++ etrago/tools/network.py | 2 +- etrago/tools/utilities.py | 2 +- 4 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 etrago/tools/db.py diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index ab3650bca..b53ff3b10 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -23,7 +23,7 @@ """ import logging -from egoio.tools import db +from etrago.tools import db from pyomo.environ import Constraint from pypsa.descriptors import expand_series from pypsa.linopt import define_constraints, define_variables, get_var, linexpr diff --git a/etrago/tools/db.py b/etrago/tools/db.py new file mode 100644 index 000000000..d5de25ee8 --- /dev/null +++ b/etrago/tools/db.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# Copyright 2016-2018 Flensburg University of Applied Sciences, +# Europa-Universität Flensburg, +# Centre for Sustainable Energy Systems, +# DLR-Institute for Networked Energy Systems + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import configparser as cp +import keyring +import getpass +from sqlalchemy import create_engine +import oedialect + +def readcfg(filepath, section): + """ + Reads the configuration file. If section is not available, calls + create_oedb_config_file to add the new section to an existing config.ini. + + Parameters + ---------- + filepath : str + Absolute path of config file including the filename itself + section : str + Section in config file which contains connection details + Returns + ------- + cfg : configparser.ConfigParser + Used for configuration file parser language. + """ + + cfg = cp.ConfigParser() + cfg.read(filepath) + + if not cfg.has_section(section): + print('The section "{sec}" is not in the config file {file}.' + .format(sec=section, + file=filepath)) + cfg = create_oedb_config_file(filepath, section) + + return cfg + +def get_connection_details(section): + """ + Asks the user for the database connection details and returns them as a + ConfigParser-object. + + Parameters + ---------- + None + + Returns + ------- + cfg : configparser.ConfigParser + Used for configuration file parser language. + """ + print('Please enter your connection details:') + dialect = input('Enter input value for `dialect` (default: psycopg2): ') or 'psycopg2' + username = input('Enter value for `username`: ') + database = input('Enter value for `database`: ') + host = input('Enter value for `host`: ') + port = input('Enter value for `port` (default: 5432): ') or '5432' + + cfg = cp.ConfigParser() + cfg.add_section(section) + cfg.set(section, 'dialect', dialect) + cfg.set(section, 'username', username) + cfg.set(section, 'host', host) + cfg.set(section, 'port', port) + cfg.set(section, 'database', database) + pw = getpass.getpass(prompt="Enter your password/token to " \ + "store it in " + "keyring: ".format(database=section)) + keyring.set_password(section, cfg.get(section, "username"), pw) + + return cfg + +def create_oedb_config_file(filepath, section='oep'): + """ + + Parameters + ---------- + filepath : str + Absolute path of config file including the filename itself + section : str + Section in config file which contains connection details + + Returns + ------- + cfg : configparser.ConfigParser + Used for configuration file parser language. + """ + + cfg = get_connection_details(section) + + print('Do you want to store the connection details in the config file {file} ?' + .format(file=filepath)) + choice = '' + while choice not in ['y', 'n']: + choice = input('(y/n): ') + + if choice == 'y': + # create egoio dir if not existent + base_path = os.path.split(filepath)[0] + if not os.path.isdir(base_path): + os.mkdir(base_path) + print('The directory {path} was created.'.format(path=base_path)) + + with open(filepath, 'a') as configfile: + cfg.write(configfile) + pass + + + print('Template {0} with section "{1}" created.\nYou can manually edit' + ' the config file.' + .format(filepath, + section)) + else: + pass + + return cfg + +def connection(filepath=None, section='oep', readonly=False): + """ + Instantiate a database connection (for the use with SQLAlchemy). + + The keyword argument `filepath` specifies the location of the config file + that contains database connection information. If not given, the default + of `~/.etrago_database/config.ini` applies. + + Parameters + ---------- + filepath : str + Absolute path of config file including the filename itself + section : str + Section in config file containing database connection parameters. + Default: 'oep'. + readonly : bool + Set this option to True for creating a read-only and passwordless + engine for accessing the open energy platform. + Default: False. + + Returns + ------- + conn : sqlalchemy.engine + SQLalchemy engine object containing the connection details + """ + + if readonly: + conn = create_engine( + "postgresql+oedialect://openenergy-platform.org") + else: + # define default filepath if not provided + if filepath is None: + filepath = os.path.join(os.path.expanduser("~"), '.etrago_database', 'config.ini') + + # does the file exist? + if not os.path.isfile(filepath): + print('DB config file {file} not found. ' + 'This might be the first run of the tool. ' + .format(file=filepath)) + cfg = create_oedb_config_file(filepath, section=section) + else: + cfg = readcfg(filepath, section) + + try: + pw = cfg.get(section, "password") + except: + pw = keyring.get_password(section, + cfg.get(section, "username")) + if pw is None: + pw = getpass.getpass(prompt='No password found for database "{db}". ' + 'Enter your password to ' + 'store it in keyring: ' + .format(db=cfg.get(section, 'database'))) + keyring.set_password(section, cfg.get(section, "username"), pw) + + # establish connection and return it + conn = create_engine( + "postgresql+{dialect}://{user}:{password}@{host}:{port}/{db}".format( + dialect=cfg.get(section, 'dialect', fallback='psycopg2'), + user=cfg.get(section, 'username'), + password=pw, + host=cfg.get(section, 'host'), + port=cfg.get(section, 'port'), + db=cfg.get(section, 'database'))) + + return conn \ No newline at end of file diff --git a/etrago/tools/network.py b/etrago/tools/network.py index fa215352e..07c0b4666 100644 --- a/etrago/tools/network.py +++ b/etrago/tools/network.py @@ -24,7 +24,7 @@ import logging -from egoio.tools import db +from etrago.tools import db from pypsa.components import Network from sqlalchemy.orm import sessionmaker import pandas as pd diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index e1c266e13..bc2be0441 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -29,7 +29,7 @@ import math import os -from egoio.tools import db +from etrago.tools import db from pyomo.environ import Constraint, PositiveReals, Var from shapely.geometry import LineString, Point import geopandas as gpd From fc74e3ab2f01888f2803f971a04c82192fa57caf Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 23 Mar 2023 09:47:05 +0100 Subject: [PATCH 2/5] Remove egio from installation --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ea9b7ab6..dac4c1eb5 100755 --- a/setup.py +++ b/setup.py @@ -43,9 +43,9 @@ def read(*names, **kwargs): packages=find_packages(), include_package_data=True, install_requires=[ - "egoio == 0.4.7", "geoalchemy2 >= 0.3.0", "geopandas", + "keyring", "matplotlib >= 3.0.3", "oedialect", # PyPSA uses a deprecated import that errors with Pyomo 6.4.3. From 71c9bc27882de810c0e0864b9263aa4241f3f53d Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 23 Mar 2023 09:49:13 +0100 Subject: [PATCH 3/5] Apply black and isort --- etrago/tools/constraints.py | 3 +- etrago/tools/db.py | 152 +++++++++++++++++++++--------------- etrago/tools/network.py | 2 +- etrago/tools/utilities.py | 3 +- 4 files changed, 92 insertions(+), 68 deletions(-) diff --git a/etrago/tools/constraints.py b/etrago/tools/constraints.py index b53ff3b10..b77197e17 100755 --- a/etrago/tools/constraints.py +++ b/etrago/tools/constraints.py @@ -23,7 +23,6 @@ """ import logging -from etrago.tools import db from pyomo.environ import Constraint from pypsa.descriptors import expand_series from pypsa.linopt import define_constraints, define_variables, get_var, linexpr @@ -32,6 +31,8 @@ import pandas as pd import pyomo.environ as po +from etrago.tools import db + logger = logging.getLogger(__name__) __copyright__ = ( diff --git a/etrago/tools/db.py b/etrago/tools/db.py index d5de25ee8..3f9ca2a5a 100644 --- a/etrago/tools/db.py +++ b/etrago/tools/db.py @@ -17,18 +17,20 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import os import configparser as cp -import keyring import getpass +import os + from sqlalchemy import create_engine +import keyring import oedialect + def readcfg(filepath, section): - """ + """ Reads the configuration file. If section is not available, calls create_oedb_config_file to add the new section to an existing config.ini. - + Parameters ---------- filepath : str @@ -43,51 +45,60 @@ def readcfg(filepath, section): cfg = cp.ConfigParser() cfg.read(filepath) - + if not cfg.has_section(section): - print('The section "{sec}" is not in the config file {file}.' - .format(sec=section, - file=filepath)) - cfg = create_oedb_config_file(filepath, section) + print( + 'The section "{sec}" is not in the config file {file}.'.format( + sec=section, file=filepath + ) + ) + cfg = create_oedb_config_file(filepath, section) return cfg + def get_connection_details(section): """ Asks the user for the database connection details and returns them as a ConfigParser-object. - + Parameters ---------- None - + Returns ------- cfg : configparser.ConfigParser Used for configuration file parser language. """ - print('Please enter your connection details:') - dialect = input('Enter input value for `dialect` (default: psycopg2): ') or 'psycopg2' - username = input('Enter value for `username`: ') - database = input('Enter value for `database`: ') - host = input('Enter value for `host`: ') - port = input('Enter value for `port` (default: 5432): ') or '5432' + print("Please enter your connection details:") + dialect = ( + input("Enter input value for `dialect` (default: psycopg2): ") + or "psycopg2" + ) + username = input("Enter value for `username`: ") + database = input("Enter value for `database`: ") + host = input("Enter value for `host`: ") + port = input("Enter value for `port` (default: 5432): ") or "5432" cfg = cp.ConfigParser() cfg.add_section(section) - cfg.set(section, 'dialect', dialect) - cfg.set(section, 'username', username) - cfg.set(section, 'host', host) - cfg.set(section, 'port', port) - cfg.set(section, 'database', database) - pw = getpass.getpass(prompt="Enter your password/token to " \ - "store it in " - "keyring: ".format(database=section)) + cfg.set(section, "dialect", dialect) + cfg.set(section, "username", username) + cfg.set(section, "host", host) + cfg.set(section, "port", port) + cfg.set(section, "database", database) + pw = getpass.getpass( + prompt="Enter your password/token to " + "store it in " + "keyring: ".format(database=section) + ) keyring.set_password(section, cfg.get(section, "username"), pw) - + return cfg -def create_oedb_config_file(filepath, section='oep'): + +def create_oedb_config_file(filepath, section="oep"): """ Parameters @@ -96,43 +107,46 @@ def create_oedb_config_file(filepath, section='oep'): Absolute path of config file including the filename itself section : str Section in config file which contains connection details - + Returns ------- cfg : configparser.ConfigParser Used for configuration file parser language. """ - + cfg = get_connection_details(section) - print('Do you want to store the connection details in the config file {file} ?' - .format(file=filepath)) - choice = '' - while choice not in ['y', 'n']: - choice = input('(y/n): ') + print( + "Do you want to store the connection details in the config file {file} ?".format( + file=filepath + ) + ) + choice = "" + while choice not in ["y", "n"]: + choice = input("(y/n): ") - if choice == 'y': + if choice == "y": # create egoio dir if not existent base_path = os.path.split(filepath)[0] if not os.path.isdir(base_path): os.mkdir(base_path) - print('The directory {path} was created.'.format(path=base_path)) - - with open(filepath, 'a') as configfile: + print("The directory {path} was created.".format(path=base_path)) + + with open(filepath, "a") as configfile: cfg.write(configfile) pass - - - print('Template {0} with section "{1}" created.\nYou can manually edit' - ' the config file.' - .format(filepath, - section)) + + print( + 'Template {0} with section "{1}" created.\nYou can manually edit' + " the config file.".format(filepath, section) + ) else: pass - + return cfg -def connection(filepath=None, section='oep', readonly=False): + +def connection(filepath=None, section="oep", readonly=False): """ Instantiate a database connection (for the use with SQLAlchemy). @@ -151,7 +165,7 @@ def connection(filepath=None, section='oep', readonly=False): Set this option to True for creating a read-only and passwordless engine for accessing the open energy platform. Default: False. - + Returns ------- conn : sqlalchemy.engine @@ -159,18 +173,22 @@ def connection(filepath=None, section='oep', readonly=False): """ if readonly: - conn = create_engine( - "postgresql+oedialect://openenergy-platform.org") + conn = create_engine("postgresql+oedialect://openenergy-platform.org") else: # define default filepath if not provided if filepath is None: - filepath = os.path.join(os.path.expanduser("~"), '.etrago_database', 'config.ini') + filepath = os.path.join( + os.path.expanduser("~"), ".etrago_database", "config.ini" + ) # does the file exist? if not os.path.isfile(filepath): - print('DB config file {file} not found. ' - 'This might be the first run of the tool. ' - .format(file=filepath)) + print( + "DB config file {file} not found. " + "This might be the first run of the tool. ".format( + file=filepath + ) + ) cfg = create_oedb_config_file(filepath, section=section) else: cfg = readcfg(filepath, section) @@ -178,23 +196,27 @@ def connection(filepath=None, section='oep', readonly=False): try: pw = cfg.get(section, "password") except: - pw = keyring.get_password(section, - cfg.get(section, "username")) + pw = keyring.get_password(section, cfg.get(section, "username")) if pw is None: - pw = getpass.getpass(prompt='No password found for database "{db}". ' - 'Enter your password to ' - 'store it in keyring: ' - .format(db=cfg.get(section, 'database'))) + pw = getpass.getpass( + prompt='No password found for database "{db}". ' + "Enter your password to " + "store it in keyring: ".format( + db=cfg.get(section, "database") + ) + ) keyring.set_password(section, cfg.get(section, "username"), pw) # establish connection and return it conn = create_engine( "postgresql+{dialect}://{user}:{password}@{host}:{port}/{db}".format( - dialect=cfg.get(section, 'dialect', fallback='psycopg2'), - user=cfg.get(section, 'username'), + dialect=cfg.get(section, "dialect", fallback="psycopg2"), + user=cfg.get(section, "username"), password=pw, - host=cfg.get(section, 'host'), - port=cfg.get(section, 'port'), - db=cfg.get(section, 'database'))) + host=cfg.get(section, "host"), + port=cfg.get(section, "port"), + db=cfg.get(section, "database"), + ) + ) - return conn \ No newline at end of file + return conn diff --git a/etrago/tools/network.py b/etrago/tools/network.py index 07c0b4666..38251cf06 100644 --- a/etrago/tools/network.py +++ b/etrago/tools/network.py @@ -24,7 +24,6 @@ import logging -from etrago.tools import db from pypsa.components import Network from sqlalchemy.orm import sessionmaker import pandas as pd @@ -34,6 +33,7 @@ from etrago.cluster.electrical import ehv_clustering, run_spatial_clustering from etrago.cluster.gas import run_spatial_clustering_gas from etrago.cluster.snapshot import skip_snapshots, snapshot_clustering +from etrago.tools import db from etrago.tools.calc_results import calc_etrago_results from etrago.tools.execute import ( dispatch_disaggregation, diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index bc2be0441..e758d2af4 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -29,7 +29,6 @@ import math import os -from etrago.tools import db from pyomo.environ import Constraint, PositiveReals, Var from shapely.geometry import LineString, Point import geopandas as gpd @@ -38,6 +37,8 @@ import pypsa import sqlalchemy.exc +from etrago.tools import db + logger = logging.getLogger(__name__) From c081a174533fda2a10d925e938ecb1d48925cfba Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 25 Sep 2023 15:42:28 +0200 Subject: [PATCH 4/5] Import db functions from etrago --- etrago/tools/network.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etrago/tools/network.py b/etrago/tools/network.py index a98a45f1e..eeb8dd02b 100644 --- a/etrago/tools/network.py +++ b/etrago/tools/network.py @@ -30,14 +30,13 @@ import pandas as pd if "READTHEDOCS" not in os.environ: - from egoio.tools import db + from etrago.tools import db from etrago import __version__ from etrago.cluster.disaggregation import run_disaggregation from etrago.cluster.electrical import ehv_clustering, run_spatial_clustering from etrago.cluster.gas import run_spatial_clustering_gas from etrago.cluster.snapshot import skip_snapshots, snapshot_clustering -from etrago.tools import db from etrago.tools.calc_results import calc_etrago_results from etrago.tools.execute import ( dispatch_disaggregation, From 9cb4e5f3fb00b46060bcbfe0c316a2dd31226def Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Mon, 25 Sep 2023 15:44:12 +0200 Subject: [PATCH 5/5] Apply isort --- etrago/tools/utilities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etrago/tools/utilities.py b/etrago/tools/utilities.py index 9f99d4598..d44171bc1 100755 --- a/etrago/tools/utilities.py +++ b/etrago/tools/utilities.py @@ -36,10 +36,11 @@ import sqlalchemy.exc if "READTHEDOCS" not in os.environ: - from etrago.tools import db from shapely.geometry import Point import geopandas as gpd + from etrago.tools import db + logger = logging.getLogger(__name__)