From 38dea153bb7d8ab726c242d3b3c23df0b8d9e67e Mon Sep 17 00:00:00 2001 From: Johan Paduart Date: Wed, 14 Jul 2021 21:12:43 +0200 Subject: [PATCH 1/2] Enable nett generation for query_generation_per_plant - Pass on nett argument to query_generation_per_plant - Refactor and generalize _calc_nett_and_drop_reduntant_columns to cope with 3 column levels (unit/psr_type/aggregation) - Warning: this removes df.squeeze() at the end of the netting. df.squeeze() gives inconsistent results according to the number of columns in a dataframe (structure is different if by coincidence only one technology or plant unit is present in the data). --- entsoe/entsoe.py | 2 +- entsoe/parsers.py | 82 ++++++++++++++++++++++++++++------------------- 2 files changed, 50 insertions(+), 34 deletions(-) diff --git a/entsoe/entsoe.py b/entsoe/entsoe.py index 1c30739..1491f8a 100644 --- a/entsoe/entsoe.py +++ b/entsoe/entsoe.py @@ -1569,7 +1569,7 @@ def query_generation_per_plant( area = lookup_area(country_code) text = super(EntsoePandasClient, self).query_generation_per_plant( country_code=area, start=start, end=end, psr_type=psr_type) - df = parse_generation(text, per_plant=True) + df = parse_generation(text, per_plant=True, nett = nett) df.columns = df.columns.set_levels(df.columns.levels[0].str.encode('latin-1').str.decode('utf-8'), level=0) df = df.tz_convert(area.tz) # Truncation will fail if data is not sorted along the index in rare diff --git a/entsoe/parsers.py b/entsoe/parsers.py index 8b15e39..c56e9a1 100644 --- a/entsoe/parsers.py +++ b/entsoe/parsers.py @@ -10,6 +10,9 @@ GENERATION_ELEMENT = "inBiddingZone_Domain.mRID" CONSUMPTION_ELEMENT = "outBiddingZone_Domain.mRID" +CONSUMPTION = 'Actual Consumption' +GENERATION = 'Actual Aggregated' + def _extract_timeseries(xml_text): """ @@ -107,39 +110,52 @@ def parse_generation( return df +def _get_aggregation_level(df: pd.DataFrame) -> int: + """ + Returns the dataframe's column level corresponding to the aggregation. + Parameters + ---------- + df : pd.DataFrame + + Returns + ------- + int + """ + for ind, level in enumerate(df.columns.levels): + if set(level) <= set([CONSUMPTION, GENERATION]): + return ind + return -1 + + def _calc_nett_and_drop_redundant_columns( df: pd.DataFrame, nett: bool) -> pd.DataFrame: - def _calc_nett(_df): - try: - if set(['Actual Aggregated']).issubset(_df): - if set(['Actual Consumption']).issubset(_df): - _new = _df['Actual Aggregated'].fillna(0) - _df[ - 'Actual Consumption'].fillna(0) - else: - _new = _df['Actual Aggregated'].fillna(0) - else: - _new = -_df['Actual Consumption'].fillna(0) - - except KeyError: - print ('Netting production and consumption not possible. Column not found') - return _new - - if hasattr(df.columns, 'levels'): - if len(df.columns.levels[-1]) == 1: - # Drop the extra header, if it is redundant - df = df.droplevel(axis=1, level=-1) - elif nett: - frames = [] - for column in df.columns.levels[-2]: - new = _calc_nett(df[column]) - new.name = column - frames.append(new) - df = pd.concat(frames, axis=1) - else: - if nett: - df = _calc_nett(df) - elif len(df.columns) == 1: - df = df.squeeze() + """ + Calculates the net generation if needed. + Parameters + ---------- + df : pd.DataFrame + nett: bool + whether or not to net + + Returns + ------- + df : pd.DataFrame + """ + aggr_level = _get_aggregation_level(df) + if aggr_level >= 0 and nett: + gen = df.xs(GENERATION, 1, aggr_level).fillna(0) \ + if GENERATION in df.columns.levels[aggr_level] \ + else None + cons = df.xs(CONSUMPTION, 1, aggr_level).fillna(0) \ + if CONSUMPTION in df.columns.levels[aggr_level] \ + else 0 + + if gen is None: + # No generation is defined, only consumption (eg PSP in ES) + df = -cons + else: + # Net generation = generation - consumption + df = gen.add(-cons, fill_value = 0) return df @@ -422,9 +438,9 @@ def _parse_generation_timeseries(soup, per_plant: bool = False) -> pd.Series: # If OUT, this means Consumption is measured. # OUT means Consumption of a generation plant, eg. charging a pumped hydro plant if soup.find(CONSUMPTION_ELEMENT.lower()): - metric = 'Actual Consumption' + metric = CONSUMPTION else: - metric = 'Actual Aggregated' + metric = GENERATION name = [metric] From 48ec8c5725a91eda37f5fa167a1b9672a4221eb8 Mon Sep 17 00:00:00 2001 From: Johan Paduart Date: Wed, 14 Jul 2021 21:56:15 +0200 Subject: [PATCH 2/2] Add Georgia & Azerbaijan (fixes #104) --- entsoe/mappings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/entsoe/mappings.py b/entsoe/mappings.py index d39665e..3b2d112 100644 --- a/entsoe/mappings.py +++ b/entsoe/mappings.py @@ -48,6 +48,7 @@ def code(self): # List taken directly from the API Docs DE_50HZ = '10YDE-VE-------2', '50Hertz CA, DE(50HzT) BZA', 'Europe/Berlin', AL = '10YAL-KESH-----5', 'Albania, OST BZ / CA / MBA', 'Europe/Tirane', + AZ = '10Y1001A1001B05V', 'Azerbaijan', 'Asia/Baku', DE_AMPRION = '10YDE-RWENET---I', 'Amprion CA', 'Europe/Berlin', AT = '10YAT-APG------L', 'Austria, APG BZ / CA / MBA', 'Europe/Vienna', BY = '10Y1001A1001A51S', 'Belarus BZ / CA / MBA', 'Europe/Minsk', @@ -70,6 +71,7 @@ def code(self): MK = '10YMK-MEPSO----8', 'Former Yugoslav Republic of Macedonia, MEPSO BZ / CA / MBA', 'Europe/Skopje', FR = '10YFR-RTE------C', 'France, RTE BZ / CA / MBA', 'Europe/Paris', DE = '10Y1001A1001A83F', 'Germany', 'Europe/Berlin' + GE = '10Y1001A1001B012', 'Georgia', 'Asia/Tbilisi', GR = '10YGR-HTSO-----Y', 'Greece, IPTO BZ / CA/ MBA', 'Europe/Athens', HU = '10YHU-MAVIR----U', 'Hungary, MAVIR CA / BZ / MBA', 'Europe/Budapest', IS = 'IS', 'Iceland', 'Atlantic/Reykjavik',