diff --git a/.travis.yml b/.travis.yml index 8ac1b87..e5e4cca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,6 @@ env: - TOXENV=docs matrix: include: - - python: '2.7' - env: - - TOXENV=py27-cover,report,coveralls,codecov - - python: '2.7' - env: - - TOXENV=py27-nocov - python: '3.4' env: - TOXENV=py34-cover,report,coveralls,codecov @@ -44,18 +38,9 @@ matrix: sudo: required env: - TOXENV=py37-nocov - - python: 'pypy' - env: - - TOXENV=pypy-cover,report,coveralls,codecov - - python: 'pypy' - env: - - TOXENV=pypy-nocov - - python: 'pypy3' - env: - - TOXENV=pypy3-cover,report,coveralls,codecov - - python: 'pypy3' - env: - - TOXENV=pypy3-nocov + allow_failures: + - python: '3.7' + before_install: - python --version - uname -a @@ -88,11 +73,11 @@ install: fi set +x script: - - tox -v + - travis_wait 60 tox -v after_failure: - more .tox/log/* | cat - more .tox/*/log/* | cat notifications: email: on_success: never - on_failure: always + on_failure: never diff --git a/appveyor.yml b/appveyor.yml index 2837f54..c21aa62 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,29 +11,6 @@ environment: PYTHON_HOME: C:\Python27 PYTHON_VERSION: '2.7' PYTHON_ARCH: '32' - - TOXENV: 'py27-cover,report,codecov' - TOXPYTHON: C:\Python27\python.exe - PYTHON_HOME: C:\Python27 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '32' - - TOXENV: 'py27-cover,report,codecov' - TOXPYTHON: C:\Python27-x64\python.exe - WINDOWS_SDK_VERSION: v7.0 - PYTHON_HOME: C:\Python27-x64 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '64' - - - TOXENV: 'py27-nocov' - TOXPYTHON: C:\Python27\python.exe - PYTHON_HOME: C:\Python27 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '32' - - TOXENV: 'py27-nocov' - TOXPYTHON: C:\Python27-x64\python.exe - WINDOWS_SDK_VERSION: v7.0 - PYTHON_HOME: C:\Python27-x64 - PYTHON_VERSION: '2.7' - PYTHON_ARCH: '64' - TOXENV: 'py34-cover,report,codecov' TOXPYTHON: C:\Python34\python.exe @@ -125,6 +102,10 @@ environment: PYTHON_VERSION: '3.7' PYTHON_ARCH: '64' +matrix: + allow_failures: + - PYTHON_VERSION: '3.7' + init: - ps: echo $env:TOXENV - ps: ls C:\Python* diff --git a/docs/reference/fuchur.scripts.rst b/docs/reference/fuchur.scripts.rst new file mode 100644 index 0000000..2cf5af6 --- /dev/null +++ b/docs/reference/fuchur.scripts.rst @@ -0,0 +1,62 @@ +fuchur.scripts package +====================== + +Submodules +---------- + +fuchur.scripts.bus module +------------------------- + +.. automodule:: fuchur.scripts.bus + :members: + :undoc-members: + :show-inheritance: + +fuchur.scripts.capacity\_factors module +--------------------------------------- + +.. automodule:: fuchur.scripts.capacity_factors + :members: + :undoc-members: + :show-inheritance: + +fuchur.scripts.compute module +----------------------------- + +.. automodule:: fuchur.scripts.compute + :members: + :undoc-members: + :show-inheritance: + +fuchur.scripts.electricity module +--------------------------------- + +.. automodule:: fuchur.scripts.electricity + :members: + :undoc-members: + :show-inheritance: + +fuchur.scripts.grid module +-------------------------- + +.. automodule:: fuchur.scripts.grid + :members: + :undoc-members: + :show-inheritance: + +fuchur.scripts.heat module +-------------------------- + +.. automodule:: fuchur.scripts.heat + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: fuchur.scripts + :members: + :undoc-members: + :show-inheritance: diff --git a/setup.cfg b/setup.cfg index 1a5a21f..d49bed7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ universal = 1 [flake8] -max-line-length = 140 +max-line-length = 79 exclude = */migrations/* [tool:pytest] @@ -31,13 +31,15 @@ addopts = --tb=short [isort] -force_single_line = True -line_length = 120 +force_single_line = False +line_length = 79 known_first_party = fuchur default_section = THIRDPARTY forced_separate = test_fuchur not_skip = __init__.py skip = migrations +from_first = True +ignore_whitespace = True [matrix] # This is the configuration for the `./bootstrap.py` script. @@ -61,13 +63,10 @@ skip = migrations # - can use as many you want python_versions = - py27 py34 py35 py36 py37 - pypy - pypy3 dependencies = # 1.4: Django==1.4.16 !python_versions[py3*] diff --git a/setup.py b/setup.py index f02c5e1..2ffd1b7 100644 --- a/setup.py +++ b/setup.py @@ -1,73 +1,73 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function +from glob import glob +from os.path import basename, dirname, join, splitext import io import re -from glob import glob -from os.path import basename -from os.path import dirname -from os.path import join -from os.path import splitext -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup def read(*names, **kwargs): with io.open( join(dirname(__file__), *names), - encoding=kwargs.get('encoding', 'utf8') + encoding=kwargs.get("encoding", "utf8"), ) as fh: return fh.read() setup( - name='fuchur', - version='0.0.0', - license='BSD 3-Clause License', - description='An optimizaton model for Europe based on oemof.', - long_description='%s\n%s' % ( - re.compile('^.. start-badges.*^.. end-badges', re.M | re.S).sub('', read('README.rst')), - re.sub(':[a-z]+:`~?(.*?)`', r'``\1``', read('CHANGELOG.rst')) + name="fuchur", + version="0.0.0", + license="BSD 3-Clause License", + description="An optimizaton model for Europe based on oemof.", + long_description="%s\n%s" + % ( + re.compile("^.. start-badges.*^.. end-badges", re.M | re.S).sub( + "", read("README.rst") + ), + re.sub(":[a-z]+:`~?(.*?)`", r"``\1``", read("CHANGELOG.rst")), ), - author='Simon Hilpert', - author_email='simon.hilpert@uni-flensburg.de', - url='https://github.com/znes/fuchur', - packages=find_packages('src'), - package_dir={'': 'src'}, - py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + author="Simon Hilpert", + author_email="simon.hilpert@uni-flensburg.de", + url="https://github.com/znes/fuchur", + packages=find_packages("src"), + package_dir={"": "src"}, + py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, zip_safe=False, classifiers=[ - # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers - 'Development Status :: 1 - Planning', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: Unix', - 'Operating System :: POSIX', - 'Operating System :: Microsoft :: Windows', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', + # complete classifier list: + # + # http://pypi.python.org/pypi?%3Aaction=list_classifiers + # + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: Unix", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: Implementation :: CPython", # uncomment if you test on these interpreters: # 'Programming Language :: Python :: Implementation :: IronPython', # 'Programming Language :: Python :: Implementation :: Jython', # 'Programming Language :: Python :: Implementation :: Stackless', - 'Topic :: Utilities', + "Topic :: Utilities", ], keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], install_requires=[ - 'click', + "click", + "oemof.tabular==0.0.1.dev0", + "setuptools", # eg: 'aspectlib==1.1.1', 'six>=1.7', ], extras_require={ @@ -75,9 +75,14 @@ def read(*names, **kwargs): # 'rst': ['docutils>=0.11'], # ':python_version=="2.6"': ['argparse'], }, - entry_points={ - 'console_scripts': [ - 'fuchur = fuchur.cli:main', + dependency_links=( + [ + ( + "git+https://git@github.com/oemof/oemof-tabular.git" + "@features/postprocessing" + "#egg=oemof.tabular-0.0.1.dev0" + ) ] - }, + ), + entry_points={"console_scripts": ["fuchur = fuchur.cli:main"]}, ) diff --git a/src/fuchur/__init__.py b/src/fuchur/__init__.py index c57bfd5..4d8b5dc 100644 --- a/src/fuchur/__init__.py +++ b/src/fuchur/__init__.py @@ -1 +1,22 @@ -__version__ = '0.0.0' +import os + +import pkg_resources as pkg +import toml + +__version__ = "0.0.0" +__RAW_DATA_PATH__ = os.path.join(os.path.expanduser("~"), "fuchur-raw-data") + +if not os.path.exists(__RAW_DATA_PATH__): + os.makedirs(__RAW_DATA_PATH__) + +scenarios = { + scenario["name"]: scenario + for resource in pkg.resource_listdir("fuchur", "scenarios") + for scenario in [ + toml.loads( + pkg.resource_string( + "fuchur", os.path.join("scenarios", resource) + ).decode("UTF-8") + ) + ] +} diff --git a/src/fuchur/cli.py b/src/fuchur/cli.py index 3150575..c5c59de 100644 --- a/src/fuchur/cli.py +++ b/src/fuchur/cli.py @@ -3,8 +3,8 @@ Why does this file exist, and why not put this in __main__? - You might be tempted to import things from __main__ later, but that will cause - problems: the code will get executed twice: + You might be tempted to import things from __main__ later, but that will + cause problems: the code will get executed twice: - When you run `python -mfuchur` python will execute ``__main__.py`` as a script. That means there won't be any @@ -14,10 +14,199 @@ Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration """ +import copy +import os + +from oemof.tabular import datapackage import click +from fuchur.scripts import (bus, capacity_factors, electricity, grid, heat, + biomass) +import fuchur +import fuchur.scripts.compute + +import collections + +def update(d, u): + for k, v in u.items(): + if isinstance(v, collections.Mapping): + d[k] = update(d.get(k, {}), v) + else: + d[k] = v + return d + +# TODO: This definitely needs docstrings. +class Scenario(dict): + @classmethod + def from_path(cls, path): + fuchur.scenarios[path] = cls( + datapackage.building.read_build_config(path) + ) + if "name" in fuchur.scenarios[path]: + name = fuchur.scenarios[path]["name"] + fuchur.scenarios[name] = fuchur.scenarios[path] + return fuchur.scenarios[path] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if "parents" in self: + for parent in self["parents"]: + if parent in fuchur.scenarios: + scenario = copy.deepcopy(fuchur.scenarios[parent]) + else: + scenario = copy.deepcopy(type(self).from_path(parent)) + + # hackish, but necessary to get self (the child) right + # with all key/values of the parent and it's own key/value pairs + update(scenario, self) + update(self, scenario) + + + + +def _download_rawdata(): + datapackage.building.download_data( + "sftp://5.35.252.104/home/rutherford/fuchur-raw-data.zip", + username="rutherford", + directory=fuchur.__RAW_DATA_PATH__, + unzip_file="fuchur-raw-data/", + ) + +def _construct(config, ctx): + """ + + config: dict + Config dict for contructing the datapackage + """ + + datapackage.processing.clean( + path=ctx.obj["datapackage_dir"], directories=["data", "resources"] + ) + + datapackage.building.initialize( + config=config, directory=ctx.obj["datapackage_dir"] + ) + + bus.add(config["buses"], ctx.obj["datapackage_dir"]) + + biomass.add(config["buses"], ctx.obj["datapackage_dir"]) + + grid.ehighway_grid( + config["buses"]["electricity"], ctx.obj["datapackage_dir"]) + + electricity.ehighway_load( + config["buses"]["electricity"], + config["temporal"]["scenario_year"], + ctx.obj["datapackage_dir"], + "100% RES" + ) + + electricity.generation(config, ctx.obj["datapackage_dir"]) + + electricity.excess(ctx.obj["datapackage_dir"]) + + electricity.hydro_generation(config, ctx.obj["datapackage_dir"]) + + capacity_factors.pv( + config["buses"]["electricity"], + config["temporal"]["weather_year"], + config["temporal"]["scenario_year"], + ctx.obj["datapackage_dir"]) + + capacity_factors.wind( + config["buses"]["electricity"], + config["temporal"]["weather_year"], + config["temporal"]["scenario_year"], + ctx.obj["datapackage_dir"]) + + if ( + config["buses"]["heat"]["decentral"] + or config["buses"]["heat"]["central"] + ): + heat.load(config, ctx.obj["datapackage_dir"]) + + if config["buses"]["heat"]["decentral"]: + heat.decentral(config, ctx.obj["datapackage_dir"]) + + if config["buses"]["heat"]["central"]: + heat.central(config, ctx.obj["datapackage_dir"]) + + datapackage.building.infer_metadata( + package_name=config["name"], + foreign_keys={ + "bus": [ + "volatile", + "dispatchable", + "storage", + "heat_storage", + "load", + "ror", + "reservoir", + "phs", + "excess", + "shortage", + "boiler", + "commodity", + ], + "profile": ["load", "volatile", "heat_load", "ror", "reservoir"], + "from_to_bus": ["link", "conversion", "line"], + "chp": ["backpressure", "extraction"], + }, + path=ctx.obj["datapackage_dir"], + ) + + +@click.group(chain=True) +@click.option("--solver", default="gurobi", help="Choose solver") +@click.option( + "--datapackage-dir", + default=os.getcwd(), + help="Data to root directory of datapackage", +) +@click.option( + "--results-dir", + default=os.path.join(os.getcwd(), "results"), + help="Data directory for results", +) +@click.option( + "--temporal-resolution", + default=1, + help="Temporal resolution used for calculation.", +) +@click.option( + "--emission-limit", default=None, help="Limit for CO2 emission in tons" +) +@click.option( + "--safe", default=True, help="Protect results from being overwritten." +) +@click.pass_context +def cli(ctx, **kwargs): + ctx.obj = kwargs + + +# TODO: Document specifying built-in scenarios by `name`. +@cli.command() +@click.argument("config", type=str, default="config.json") +@click.pass_context +def construct(ctx, config): + if config in fuchur.scenarios: + config = fuchur.scenarios[config] + else: + config = Scenario.from_path(config) + _construct(config, ctx) + + + +@cli.command() +@click.pass_context +def compute(ctx): + fuchur.scripts.compute.compute(ctx) + + +@cli.command() +@click.pass_context +def download(ctx): + _download_rawdata(ctx) + -@click.command() -@click.argument('names', nargs=-1) -def main(names): - click.echo(repr(names)) +def main(): + cli(obj={}) diff --git a/src/fuchur/scenarios/el-2pv-cost.toml b/src/fuchur/scenarios/el-2pv-cost.toml new file mode 100644 index 0000000..bb9277b --- /dev/null +++ b/src/fuchur/scenarios/el-2pv-cost.toml @@ -0,0 +1,74 @@ +# Toml document to used for building model input +# datapackages + +title = "Fuchur build config for scenario 1" +description = "Electricity sector for DE and DK." +name = "el-2pv-cost" + +[owner] +name = "ZNES" + +[cost] +wacc = 0.07 + +[cost.factor] +pv = 2 +lithium_battery = 2 +acaes = 1 + +[temporal] +scenario_year = 2050 +weather_year = 2012 +demand_year = 2012 + +[potential] +renewables = "LIMES-EU" +biomass = "hotmaps" + +[buses] + +electricity = [ + "AT", + "BE", + "CH", + "CZ", + "DE", + "DK", + "FR", + "NO", + "PL", + "SE" +] + +biomass = [ + "AT", + "BE", + "CH", + "CZ", + "DE", + "DK", + "FR", + "NO", + "PL", + "SE" +] + +[buses.heat] + +decentral = [] + +central = [] + +[technologies] + +investment = [ + "wind_onshore", + "wind_offshore", + "pv", + "ocgt", + "ccgt", + "acaes", + "biomass", + "lithium_battery", + "coal" +] diff --git a/src/fuchur/scenarios/el-base.toml b/src/fuchur/scenarios/el-base.toml new file mode 100644 index 0000000..344b310 --- /dev/null +++ b/src/fuchur/scenarios/el-base.toml @@ -0,0 +1,74 @@ +# Toml document to used for building model input +# datapackages + +title = "Fuchur build config for scenario 1" +description = "Electricity sector for DE and DK." +name = "el-base" + +[owner] +name = "ZNES" + +[cost] +wacc = 0.07 + +[cost.factor] +pv = 1 +lithium_battery = 1 +acaes = 1 + +[temporal] +scenario_year = 2050 +weather_year = 2012 +demand_year = 2012 + +[potential] +renewables = "LIMES-EU" +biomass = "hotmaps" + +[buses] + +electricity = [ + "AT", + "BE", + "CH", + "CZ", + "DE", + "DK", + "FR", + "NO", + "PL", + "SE" +] + +biomass = [ + "AT", + "BE", + "CH", + "CZ", + "DE", + "DK", + "FR", + "NO", + "PL", + "SE" +] + +[buses.heat] + +decentral = [] + +central = [] + +[technologies] + +investment = [ + "wind_onshore", + "wind_offshore", + "pv", + "ocgt", + "ccgt", + "acaes", + "biomass", + "lithium_battery", + "coal" +] diff --git a/src/fuchur/scenarios/el-no-biomass.toml b/src/fuchur/scenarios/el-no-biomass.toml new file mode 100644 index 0000000..19fd033 --- /dev/null +++ b/src/fuchur/scenarios/el-no-biomass.toml @@ -0,0 +1,63 @@ +# Toml document to used for building model input +# datapackages + +title = "Fuchur build config for scenario 1" +description = "Electricity sector for DE and DK." +name = "el-no-biomass" + +[owner] +name = "ZNES" + +[cost] +wacc = 0.07 + +[cost.factor] +pv = 1 +lithium_battery = 1 +acaes = 1 + +[temporal] +scenario_year = 2050 +weather_year = 2012 +demand_year = 2012 + +[potential] +renewables = "LIMES-EU" +biomass = "hotmaps" + +[buses] + +electricity = [ + "AT", + "BE", + "CH", + "CZ", + "DE", + "DK", + "FR", + "NO", + "PL", + "SE" +] + +biomass = [ +] + +[buses.heat] + +decentral = [] + +central = [] + +[technologies] + +investment = [ + "wind_onshore", + "wind_offshore", + "pv", + "ocgt", + "ccgt", + "acaes", + "lithium_battery", + "coal" +] diff --git a/src/fuchur/scenarios/scenario2.toml b/src/fuchur/scenarios/scenario2.toml new file mode 100644 index 0000000..96197c4 --- /dev/null +++ b/src/fuchur/scenarios/scenario2.toml @@ -0,0 +1,107 @@ +# Toml document to used for building model input +# datapackages + +title = "Fuchur build config for scenario 1" +description = "Electricity sector for DE and DK." +name = "scenario2" + +[owner] +name = "ZNES" + +[raw-data] +path = "fuchur-raw-data" +download = true + +[cost] +wacc = 0.07 + +[cost.factor] +pv = 1 +lithium_battery = 1 +acaes = 1 + +[temporal] +scenario_year = 2050 +weather_year = 2012 +demand_year = 2012 + +[potential] +renewables = "LIMES-EU" +biomass = "hotmaps" + +[buses] + +electricity = [ + "AT", + "BE", + "BG", + "CH", + "CZ", + "DE", + "DK", + "ES", + "FI", + "FR", + "HR", + "HU", + "IE", + "IT", + "LT", + "LV", + "NL", + "NO", + "PL", + "PT", + "RO", + "SE", + "SI", + "SK", + "GB" +] + +biomass = [ + "AT", + "BE", + "BG", + "CH", + "CZ", + "DE", + "DK", + "ES", + "FI", + "FR", + "HR", + "HU", + "IE", + "IT", + "LT", + "LV", + "NL", + "NO", + "PL", + "PT", + "RO", + "SE", + "SI", + "SK", + "GB" +] + +[buses.heat] + +decentral = [] +central = [] + +[technologies] + +investment = [ + "wind_onshore", + "wind_offshore", + "pv", + "ocgt", + "ccgt", + "acaes", + "biomass", + "lithium_battery", + "coal" +] diff --git a/src/fuchur/scenarios/test.toml b/src/fuchur/scenarios/test.toml new file mode 100644 index 0000000..c178da8 --- /dev/null +++ b/src/fuchur/scenarios/test.toml @@ -0,0 +1,65 @@ +# Toml document to used for building model input +# datapackages + +title = "Fuchur build config for scenario 1" +description = "Electricity sector for DE and DK." +name = "test" + +[owner] +name = "ZNES" + +[raw-data] +path = "fuchur-raw-data" +download = true + +[cost] +wacc = 0.07 + +[cost.factor] +pv = 2 +lithium_battery = 2 +acaes = 1 + +[temporal] +scenario_year = 2050 +weather_year = 2012 +demand_year = 2012 + +[potential] +renewables = "LIMES-EU" +biomass = "hotmaps" + +[buses] + +electricity = [ + "DK", + "DE" +] + +biomass = [ + "DK", + "DE" +] + +[buses.heat] + +decentral = [ + "DE" +] +central = [ + "DE" +] + +[technologies] + +investment = [ + "wind_onshore", + "wind_offshore", + "pv", + "ocgt", + "ccgt", + "acaes", + "biomass", + "lithium_battery", + "coal" +] diff --git a/src/fuchur/scripts/__init__.py b/src/fuchur/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fuchur/scripts/biomass.py b/src/fuchur/scripts/biomass.py new file mode 100644 index 0000000..80f37d5 --- /dev/null +++ b/src/fuchur/scripts/biomass.py @@ -0,0 +1,69 @@ + + +import json +import os + +from datapackage import Package +import pandas as pd + +from oemof.tabular.datapackage import building + + +def add(buses, datapackage_dir): + """ + """ + commodities = {} + bus_elements = {} + + bio_potential = ( + Package( + "https://raw.githubusercontent.com/ZNES-datapackages/" + "technology-potential/master/datapackage.json" + ) + .get_resource("carrier") + .read(keyed=True) + ) + bio_potential = pd.DataFrame(bio_potential).set_index( + ["country", "carrier"] + ) + bio_potential.rename(index={"UK": "GB"}, inplace=True) + + bio_potential = bio_potential.loc[ + bio_potential["source"] == "hotmaps" + ].to_dict() + + if buses.get("biomass"): + for b in buses["biomass"]: + bus_name = '-'.join([b,"biomass", "bus"]) + commodity_name = '-'.join([b, "biomass", "commodity"]) + + commodities[commodity_name] = { + "type": "dispatchable", + "carrier": "biomass", + "bus": bus_name, + "capacity": float( + bio_potential["value"].get((b, "biomass"), 0) + ) + * 1e6, # TWh -> MWh + "output_parameters": json.dumps({"summed_max": 1}), + } + + bus_elements[bus_name] = { + "type": "bus", + "carrier": "biomass", + "geometry": None, + "balanced": True, + } + + if commodities: + building.write_elements( + "commodity.csv", + pd.DataFrame.from_dict(commodities, orient="index"), + os.path.join(datapackage_dir, "data/elements"), + ) + + building.write_elements( + "bus.csv", + pd.DataFrame.from_dict(bus_elements, orient="index"), + os.path.join(datapackage_dir, "data/elements"), + ) diff --git a/src/fuchur/scripts/bus.py b/src/fuchur/scripts/bus.py new file mode 100644 index 0000000..5b5f7c7 --- /dev/null +++ b/src/fuchur/scripts/bus.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" +This script constructs a pandas.Series `buses` with hub-names as index and +polygons of these buses as values. It uses the NUTS shapefile. + +""" + +import os + +from oemof.tabular.datapackage import building +from oemof.tabular.tools import geometry +import pandas as pd + +import fuchur + + +def electricity(buses, datapackage_dir, raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameters + ---------- + buses: dict + Dictionary with two keys: decentral and central and their values + beeing the names of the buses + datapackage_dir: str + Directory of datapackage where resources are stored + raw_data_path: str + Path to directory where raw data can be found + """ + + filepath = building.download_data( + 'http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/' + 'NUTS_2013_10M_SH.zip', + unzip_file='NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.shp', + directory=raw_data_path) + + building.download_data( + 'http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/' + 'NUTS_2013_10M_SH.zip', + unzip_file='NUTS_2013_10M_SH/data/NUTS_RG_10M_2013.dbf', + directory=raw_data_path) + + if not os.path.exists(filepath): + print("Shapefile data not found. Did you download raw data?") + # get nuts 1 regions for german neighbours + + nuts0 = pd.Series(geometry.nuts(filepath, nuts=0, tolerance=0.1)) + + nuts0.index = [i.replace("UK", "GB") for i in nuts0.index] + + el_buses = pd.Series(name="geometry") + el_buses.index.name = "name" + + for r in buses: + el_buses[r + "-electricity"] = nuts0[r] + building.write_geometries( + "bus.geojson", + el_buses, + os.path.join(datapackage_dir, "data", "geometries"), + ) + # Add electricity buses + bus_elements = {} + for b in el_buses.index: + bus_elements[b] = { + "type": "bus", + "carrier": "electricity", + "geometry": b, + "balanced": True, + } + + building.write_elements( + "bus.csv", + pd.DataFrame.from_dict(bus_elements, orient="index"), + os.path.join(datapackage_dir, "data", "elements"), + ) + + +def heat(buses, datapackage_dir): + """ + Parameters + ---------- + buses: dict + Dictionary with two keys: decentral and central and their values + beeing the names of the buses + datapackage_dir: str + Directory of datapackage where resources are stored + """ + bus_elements = {} + # Add heat buses per sub_bus and region (r) + for sub_bus, regions in buses.items(): + for region in regions: + bus_elements["-".join([region, "heat", sub_bus])] = { + "type": "bus", + "carrier": "heat", + "geometry": None, + "balanced": True, + } + + building.write_elements( + "bus.csv", + pd.DataFrame.from_dict(bus_elements, orient="index"), + os.path.join(datapackage_dir, "data", "elements"), + ) diff --git a/src/fuchur/scripts/capacity_factors.py b/src/fuchur/scripts/capacity_factors.py new file mode 100644 index 0000000..e6de66d --- /dev/null +++ b/src/fuchur/scripts/capacity_factors.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" +""" +import os + +from oemof.tabular.datapackage import building +import pandas as pd + +import fuchur + + +def pv(buses, weather_year, scenario_year, datapackage_dir, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameter + --------- + buses: array like + List with buses represented by iso country code + weather_year: integer or string + Year to select from raw data source + scenario_year: integer or string + Year to use for timeindex in tabular resource + datapackage_dir: string + Directory for tabular resource + raw_data_path: string + Path where raw data file `ninja_pv_europe_v1.1_merra2.csv` + is located + """ + filepath = building.download_data( + "https://www.renewables.ninja/static/downloads/ninja_europe_pv_v1.1.zip", + unzip_file="ninja_pv_europe_v1.1_merra2.csv", + directory=raw_data_path) + + year = str(weather_year) + + countries = buses + + raw_data = pd.read_csv(filepath, index_col=[0], parse_dates=True) + # for leap year... + raw_data = raw_data[ + ~((raw_data.index.month == 2) & (raw_data.index.day == 29)) + ] + + df = raw_data.loc[year] + + sequences_df = pd.DataFrame(index=df.index) + + for c in countries: + sequence_name = c + "-pv-profile" + sequences_df[sequence_name] = raw_data.loc[year][c].values + + sequences_df.index = building.timeindex( + year=str(scenario_year) + ) + building.write_sequences( + "volatile_profile.csv", + sequences_df, + directory=os.path.join(datapackage_dir, "data", "sequences"), + ) + + +def wind(buses, weather_year, scenario_year, datapackage_dir, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameter + --------- + buses: array like + List with buses represented by iso country code + weather_year: integer or string + Year to select from raw data source + scenario_year: integer or string + Year to use for timeindex in tabular resource + datapackage_dir: string + Directory for tabular resource + raw_data_path: string + Path where raw data file `ninja_wind_europe_v1.1_current_national.csv` + and `ninja_wind_europe_v1.1_current_national.csv` + is located + """ + near_term_path = building.download_data( + "https://www.renewables.ninja/static/downloads/ninja_europe_wind_v1.1.zip", + unzip_file= "ninja_wind_europe_v1.1_current_national.csv", + directory=raw_data_path) + + off_filepath = building.download_data( + "https://www.renewables.ninja/static/downloads/ninja_europe_wind_v1.1.zip", + unzip_file= "ninja_wind_europe_v1.1_future_nearterm_on-offshore.csv", + directory=raw_data_path) + + year = str(weather_year) + + near_term = pd.read_csv(near_term_path, index_col=[0], parse_dates=True) + # for lead year... + near_term = near_term[ + ~((near_term.index.month == 2) & (near_term.index.day == 29)) + ] + + offshore_data = pd.read_csv(off_filepath, index_col=[0], parse_dates=True) + offshore_data = offshore_data[ + ~((offshore_data.index.month == 2) & (offshore_data.index.day == 29)) + ] + + sequences_df = pd.DataFrame(index=near_term.loc[year].index) + + NorthSea = ["DE", "DK", "NO", "NL", "BE", "GB", "SE"] + + for c in buses: + # add offshore profile if country exists in offshore data columns + # and if its in NorthSea + if [ + col for col in offshore_data.columns if c + "_OFF" in col + ] and c in NorthSea: + sequences_df[c + "-offshore-profile"] = offshore_data[c + "_OFF"] + + sequence_name = c + "-onshore-profile" + sequences_df[sequence_name] = near_term.loc[year][c].values + + sequences_df.index = building.timeindex( + year=str(scenario_year) + ) + + building.write_sequences( + "volatile_profile.csv", + sequences_df, + directory=os.path.join(datapackage_dir, "data", "sequences"), + ) diff --git a/src/fuchur/scripts/compute.py b/src/fuchur/scripts/compute.py new file mode 100644 index 0000000..e56d47b --- /dev/null +++ b/src/fuchur/scripts/compute.py @@ -0,0 +1,111 @@ + +import json +import logging +import os + +from datapackage import Package +from oemof.solph import Bus, EnergySystem, Model, constraints +from oemof.tabular import facades +from oemof.tabular.datapackage import aggregation, processing +from oemof.tabular.tools import postprocessing as pp +import oemof.outputlib as outputlib + + +def compute(ctx): + """ + """ + p = Package( + os.path.join( + ctx.obj["datapackage_dir"], 'datapackage.json') + ) + + temporal_resolution = ctx.obj["temporal_resolution"] + emission_limit = ctx.obj["emission_limit"] + + # create results path + scenario_path = os.path.join(ctx.obj["results_dir"], p.descriptor["name"]) + if not os.path.exists(scenario_path): + os.makedirs(scenario_path) + + output_path = os.path.join(scenario_path, "output") + if not os.path.exists(output_path): + os.makedirs(output_path) + + # store used config file + # with open(os.path.join(scenario_path, "config.json"), "w") as outfile: + # json.dump(config, outfile, indent=4) + + # copy package either aggregated or the original one (only data!) + if temporal_resolution > 1: + logging.info("Aggregating for temporal aggregation ... ") + path = aggregation.temporal_skip( + os.path.join(ctx.obj["datapackage_dir"], "datapackage.json"), + temporal_resolution, + path=scenario_path, + name="input" + ) + else: + path = processing.copy_datapackage( + os.path.join(ctx.obj["datapackage_dir"], "datapackage.json"), + os.path.abspath(os.path.join(scenario_path, "input")), + subset="data", + ) + + es = EnergySystem.from_datapackage( + os.path.join(path, "datapackage.json"), + attributemap={}, + typemap=facades.TYPEMAP, + ) + + m = Model(es) + + if emission_limit is not None: + constraints.emission_limit(m, limit=emission_limit) + + m.receive_duals() + + m.solve(ctx.obj["solver"]) + + m.results = m.results() + + pp.write_results(m, output_path) + + modelstats = outputlib.processing.meta_results(m) + modelstats.pop("solver") + modelstats["problem"].pop("Sense") + # TODO: This is not model stats -> move somewhere else! + modelstats["temporal_resolution"] = temporal_resolution + modelstats["emission_limit"] = emission_limit + + with open(os.path.join(scenario_path, "modelstats.json"), "w") as outfile: + json.dump(modelstats, outfile, indent=4) + + supply_sum = ( + pp.supply_results( + results=m.results, + es=m.es, + bus=[b.label for b in es.nodes if isinstance(b, Bus)], + types=[ + "dispatchable", + "volatile", + "conversion", + "backpressure", + "extraction", + "reservoir", + ], + ) + .sum() + .reset_index() + ) + supply_sum["from"] = supply_sum.apply( + lambda x: "-".join(x["from"].label.split("-")[1::]), axis=1 + ) + supply_sum.drop("type", axis=1, inplace=True) + supply_sum = ( + supply_sum.set_index(["from", "to"]).unstack("from") + / 1e6 + * temporal_resolution + ) + supply_sum.columns = supply_sum.columns.droplevel(0) + summary = supply_sum # pd.concat([supply_sum, excess_share], axis=1) + summary.to_csv(os.path.join(scenario_path, 'summary.csv')) diff --git a/src/fuchur/scripts/electricity.py b/src/fuchur/scripts/electricity.py new file mode 100644 index 0000000..fbc1059 --- /dev/null +++ b/src/fuchur/scripts/electricity.py @@ -0,0 +1,598 @@ +# -*- coding: utf-8 -*- +""" +""" +import json +import os + +from datapackage import Package +from decimal import Decimal + +import pandas as pd +import numpy as np + +from oemof.tabular.datapackage import building +from oemof.tools.economics import annuity + +import fuchur + + + +def generic_investment(buses, investment_technologies, year, wacc, + potential_source, datapackage_dir, cost_factor={}, techmap = { + "ocgt": "dispatchable", + "ccgt": "dispatchable", + "st": "dispatchable", + "ce": "dispatchable", + "pv": "volatile", + "wind_onshore": "volatile", + "wind_offshore": "volatile", + "biomass": "conversion", + "lithium_battery": "storage", + "acaes": "storage"}): + """ + """ + + + technologies = pd.DataFrame( + Package( + "https://raw.githubusercontent.com/ZNES-datapackages/" + "technology-cost/master/datapackage.json" + ) + .get_resource("electricity") + .read(keyed=True) + ) + technologies = ( + technologies.groupby(["year", "tech", "carrier"]) + .apply(lambda x: dict(zip(x.parameter, x.value))) + .reset_index("carrier") + .apply(lambda x: dict({"carrier": x.carrier}, **x[0]), axis=1) + ) + technologies = technologies.loc[year].to_dict() + + potential = ( + Package( + "https://raw.githubusercontent.com/ZNES-datapackages/" + "technology-potential/master/datapackage.json" + ) + .get_resource("renewable") + .read(keyed=True) + ) + potential = pd.DataFrame(potential).set_index(["country", "tech"]) + potential = potential.loc[ + potential["source"] == potential_source + ].to_dict() + + for tech in technologies: + technologies[tech]["capacity_cost"] = technologies[tech][ + "capacity_cost" + ] * cost_factor.get(tech, 1) + if "storage_capacity_cost" in technologies[tech]: + technologies[tech]["storage_capacity_cost"] = technologies[tech][ + "storage_capacity_cost" + ] * cost_factor.get(tech, 1) + + # TODO: replace by datapackage + carrier = pd.read_csv( + os.path.join(fuchur.__RAW_DATA_PATH__, "carrier.csv"), index_col=[0, 1] + ).loc[("base", year)] + carrier.set_index("carrier", inplace=True) + + elements = {} + + for r in buses: + for tech, data in technologies.items(): + if tech in investment_technologies: + + element = data.copy() + elements[r + "-" + tech] = element + + if techmap.get(tech) == "dispatchable": + element.update( + { + "capacity_cost": annuity( + float(data["capacity_cost"]), + float(data["lifetime"]), + wacc, + ) + * 1000, # €/kW -> €/MW + "bus": r + "-electricity", + "type": "dispatchable", + "marginal_cost": ( + carrier.loc[data["carrier"]].cost + + carrier.loc[data["carrier"]].emission + * carrier.loc["co2"].cost + ) + / float(data["efficiency"]), + "tech": tech, + "capacity_potential": potential[ + "capacity_potential" + ].get((r, tech), "Infinity"), + "output_parameters": json.dumps( + { + "emission_factor": ( + carrier.loc[data["carrier"]].emission + / float(data["efficiency"]) + ) + } + ), + } + ) + + if techmap.get(tech) == "conversion": + element.update( + { + "capacity_cost": annuity( + float(data["capacity_cost"]), + float(data["lifetime"]), + wacc, + ) + * 1000, # €/kW -> €/M + "to_bus": r + "-electricity", + "from_bus": r + "-" + data["carrier"] + "-bus", + "type": "conversion", + "marginal_cost": ( + carrier.loc[data["carrier"]].cost + + carrier.loc[data["carrier"]].emission + * carrier.loc["co2"].cost + ) + / float(data["efficiency"]), + "tech": tech, + } + ) + + # ep = {'summed_max': float(bio_potential['value'].get( + # (r, tech), 0)) * 1e6}) # TWh to MWh + + if techmap.get(tech) == "volatile": + NorthSea = ["DE", "DK", "NO", "NL", "BE", "GB", "SE"] + + if "wind_off" in tech: + profile = r + "-wind-off-profile" + elif "wind_on" in tech: + profile = r + "-wind-on-profile" + elif "pv" in tech: + profile = r + "-pv-profile" + + e = { + "capacity_cost": annuity( + float(data["capacity_cost"]), + float(data["lifetime"]), + wacc, + ) + * 1000, + "capacity_potential": potential[ + "capacity_potential" + ].get((r, tech), 0), + "bus": r + "-electricity", + "tech": tech, + "type": "volatile", + "profile": profile, + } + # only add all technologies that are not offshore or + # if offshore in the NorthSea list + if "wind_off" in tech and r not in NorthSea: + pass + else: + element.update(e) + + elif techmap[tech] == "storage": + if tech == "acaes" and r != "DE": + capacity_potential = 0 + else: + capacity_potential = "Infinity" + element.update( + { + "capacity_cost": annuity( + float(data["capacity_cost"]) + + float(data["storage_capacity_cost"]) + / float(data["capacity_ratio"]), + float(data["lifetime"]), + wacc, + ) + * 1000, + "bus": r + "-electricity", + "tech": tech, + "type": "storage", + "efficiency": float(data["efficiency"]) + ** 0.5, # convert roundtrip to + # input / output efficiency + "marginal_cost": 0.0000001, + "loss": 0.01, + "capacity_potential": capacity_potential, + "capacity_ratio": data["capacity_ratio"], + } + ) + + df = pd.DataFrame.from_dict(elements, orient="index") + # drop storage capacity cost to avoid duplicate investment + df = df.drop("storage_capacity_cost", axis=1) + + df = df[(df[["capacity_potential"]] != 0).all(axis=1)] + + # write elements to CSV-files + for element_type in set(techmap.values()): + building.write_elements( + element_type + ".csv", + df.loc[df["type"] == element_type].dropna(how="all", axis=1), + directory=os.path.join(datapackage_dir, "data/elements"), + ) + + +def tyndp_generation(buses, vision, scenario_year, datapackage_dir, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + """ + filepath = building.download_data( + "https://www.entsoe.eu/Documents/TYNDP%20documents/TYNDP%202016/rgips/" + "TYNDP2016%20market%20modelling%20data.xlsx", + directory=raw_data_path) + df = pd.read_excel(filepath, sheet_name="NGC") + + efficiencies = { + 'biomass': 0.45, + 'coal': 0.45, + 'gas': 0.5, + 'uranium': 0.35, + 'oil': 0.35, + 'lignite': 0.4} + + max = { + 'biomass': 0.85, + 'coal': 0.85, + 'gas': 0.85, + 'uranium': 0.85, + 'oil': 0.85, + 'lignite': 0.85 + } + + visions = { + 'vision1': 41, + 'vision2': 80, + 'vision3': 119, + 'vision4': 158 + } + # 41:77 for 2030 vision 1 + # 80:116 for 2030 vision 2 or from ehighway scenario? + # .... + x = df.iloc[ + visions[vision]: visions[vision] + 36 + ] + x.columns = x.iloc[0,:] + x.drop(x.index[0], inplace=True) + x.rename(columns={ + x.columns[0]: 'country', + 'Hard coal': 'coal', + 'Nuclear': 'uranium'}, + inplace=True) + x.set_index('country', inplace=True) + x.dropna(axis=1, how='all', inplace=True) # drop unwanted cols + x['biomass'] = x['Biofuels'] + x['Others RES'] + x.drop(['Biofuels', 'Others RES'], axis=1, inplace=True) + x.columns = [i.lower().replace(' ', '-') for i in x.columns] + + carriers = pd.DataFrame( + Package('https://raw.githubusercontent.com/ZNES-datapackages/technology-cost/master/datapackage.json') + .get_resource('carrier').read(keyed=True)).set_index( + ['year', 'carrier', 'parameter']).sort_index() + + elements = {} + + for b in buses: + for carrier in x.columns: + element = {} + + if carrier in ['wind', 'solar']: + if "wind" in carrier: + profile = b + "-onshore-profile" + tech = 'onshore' + elif "solar" in carrier: + profile = b + "-pv-profile" + tech = 'pv' + + elements['-'.join([b, carrier, tech])] = element + e = { + "bus": b + "-electricity", + "tech": tech, + "carrier": carrier, + "capacity": x.at[b, carrier], + "type": "volatile", + "profile": profile, + } + + element.update(e) + + elif carrier in ['gas', 'coal', 'lignite', 'oil', 'uranium']: + if carrier == 'gas': + tech = 'gt' + else: + tech = 'st' + elements['-'.join([b, carrier, tech])] = element + marginal_cost = float( + carriers.at[(scenario_year, carrier, 'cost'), 'value'] + + carriers.at[(2014, carrier, 'emission-factor'), 'value'] + * carriers.at[(scenario_year, 'co2', 'cost'), 'value'] + ) / efficiencies[carrier] + + element.update({ + "carrier": carrier, + "capacity": x.at[b, carrier], + "bus": b + "-electricity", + "type": "dispatchable", + "marginal_cost": marginal_cost, + "output_parameters": json.dumps( + {"max": max[carrier]} + ), + "tech": tech, + } + ) + + elif carrier == 'others-non-res': + elements[b + "-" + carrier] = element + + element.update({ + "carrier": 'other', + "capacity": x.at[b, carrier], + "bus": b + "-electricity", + "type": "dispatchable", + "marginal_cost": 0, + "tech": 'other', + "output_parameters": json.dumps( + {"summed_max": 2000} + ) + } + ) + + elif carrier == "biomass": + elements["-".join([b, carrier, 'ce'])] = element + + element.update({ + "carrier": carrier, + "capacity": x.at[b, carrier], + "to_bus": b + "-electricity", + "efficiency": efficiencies[carrier], + "from_bus": b + "-biomass-bus", + "type": "conversion", + "carrier_cost": float( + carriers.at[(2030, carrier, 'cost'), 'value'] + ), + "tech": 'ce', + } + ) + + df = pd.DataFrame.from_dict(elements, orient="index") + df = df[df.capacity != 0] + + # write elements to CSV-files + for element_type in ['dispatchable', 'volatile', 'conversion']: + building.write_elements( + element_type + ".csv", + df.loc[df["type"] == element_type].dropna(how="all", axis=1), + directory=os.path.join(datapackage_dir, "data", "elements"), + ) + + + + +def nep_2019(year, datapackage_dir, scenario='B2030', bins=2, eaf=0.95, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + """ + + technologies = pd.DataFrame( + #Package('/home/planet/data/datapackages/technology-cost/datapackage.json') + Package('https://raw.githubusercontent.com/ZNES-datapackages/technology-cost/master/datapackage.json') + .get_resource('electricity').read(keyed=True)).set_index( + ['year', 'carrier', 'tech', 'parameter']) + + carriers = pd.DataFrame( + #Package('/home/planet/data/datapackages/technology-cost/datapackage.json') + Package('https://raw.githubusercontent.com/ZNES-datapackages/technology-cost/master/datapackage.json') + .get_resource('carrier').read(keyed=True)).set_index( + ['year', 'carrier', 'parameter', 'unit']).sort_index() + + sq = pd.read_csv(building.download_data( + "https://data.open-power-system-data.org/conventional_power_plants/" + "2018-12-20/conventional_power_plants_DE.csv", + directory=raw_data_path) + , encoding='utf-8') + + sq.set_index("id", inplace=True) + + nep = pd.read_excel(building.download_data( + "https://www.netzentwicklungsplan.de/sites/default/files/" + "paragraphs-files/Kraftwerksliste_%C3%9CNB_Entwurf_Szenariorahmen_2030_V2019.xlsx", + directory=raw_data_path) + , encoding='utf-8') + + pp = nep.loc[nep["Nettonennleistung " + scenario + " [MW]"] != 0]["BNetzA-ID"] + pp = list(set([i for i in pp.values if not pd.isnull(i)])) + df = sq.loc[pp] + + cond1 = df['country_code'] == 'DE' + cond2 = df['fuel'].isin(['Hydro']) + cond3 = (df['fuel'] == 'Other fuels') & (df['technology'] == 'Storage technologies') + + df = df.loc[cond1 & ~cond2 & ~cond3, :].copy() + + mapper = {('Biomass and biogas', 'Steam turbine'): ('biomass', 'st'), + ('Biomass and biogas', 'Combustion Engine'): ('biomass', 'ce'), + ('Hard coal', 'Steam turbine'): ('coal', 'st'), + ('Hard coal', 'Combined cycle'): ('coal', 'ccgt'), + ('Lignite', 'Steam turbine'): ('lignite', 'st'), + ('Natural gas', 'Gas turbine'): ('gas', 'ocgt'), + ('Natural gas', 'Steam turbine'): ('gas', 'st'), + ('Natural gas', 'Combined cycle'): ('gas', 'ccgt'), + ('Natural gas', 'Combustion Engine'): ('gas', 'st'), # other technology + ('Nuclear', 'Steam turbine'): ('uranium', 'st'), + ('Oil', 'Steam turbine'): ('oil', 'st'), + ('Oil', 'Gas turbine'): ('oil', 'st'), + ('Oil', 'Combined cycle'): ('oil', 'st'), + ('Other fuels', 'Steam turbine'): ('waste', 'chp'), + ('Other fuels', 'Combined cycle'): ('gas', 'ccgt'), + ('Other fuels', 'Gas turbine'): ('gas', 'ocgt'), + ('Waste', 'Steam turbine'): ('waste', 'chp'), + ('Waste', 'Combined cycle'): ('waste', 'chp'), + ('Other fossil fuels', 'Steam turbine'): ('coal', 'st'), + ('Other fossil fuels', 'Combustion Engine'): ('gas', 'st'), + ('Mixed fossil fuels', 'Steam turbine'): ('gas', 'st')} + + df['carrier'], df['tech'] = zip(*[mapper[tuple(i)] for i in df[['fuel', 'technology']].values]) + + etas = df.groupby(['carrier', 'tech']).mean()['efficiency_estimate'].to_dict() + index = df['efficiency_estimate'].isna() + df.loc[index, 'efficiency_estimate'] = \ + [etas[tuple(i)] for i in df.loc[index, ('carrier', 'tech')].values] + + index = df['carrier'].isin(['gas', 'coal', 'lignite']) + + df.loc[index, 'bins'] = df[index].groupby(['carrier', 'tech'])['capacity_net_bnetza']\ + .apply(lambda i: pd.qcut(i, bins, labels=False, duplicates='drop')) + + df['bins'].fillna(0, inplace=True) + + s = df.groupby(['country_code', 'carrier', 'tech', 'bins']).\ + agg({'capacity_net_bnetza': sum, 'efficiency_estimate': np.mean}) + + elements = {} + + co2 = carriers.at[(year, 'co2', 'cost', 'EUR/t'), 'value'] + + for (country, carrier, tech, bins), (capacity, eta) in s.iterrows(): + name = country + '-' + carrier + '-' + tech + '-' + str(bins) + + vom = technologies.at[(year, carrier, tech, 'vom'), 'value'] + ef = carriers.at[(2015, carrier, 'emission-factor', 't (CO2)/MWh'), 'value'] + fuel = carriers.at[(year, carrier, 'cost', 'EUR/MWh'), 'value'] + + marginal_cost = (fuel + vom + co2 * ef) / Decimal(eta) + + output_parameters = {"max": eaf} + + if carrier == "waste": + output_parameters.update({"summed_max": 2500}) + + element = { + 'bus': country + '-electricity', + 'tech': tech, + 'carrier': carrier, + 'capacity': capacity, + 'marginal_cost': float(marginal_cost), + 'output_parameters': json.dumps(output_parameters), + 'type': 'dispatchable'} + + elements[name] = element + + + building.write_elements( + 'dispatchable.csv', + pd.DataFrame.from_dict(elements, orient='index'), + directory=os.path.join(datapackage_dir, 'data', 'elements')) + + # add renewables + elements = {} + + b = 'DE' + for carrier, tech in [('wind', 'offshore'), ('wind', 'onshore'), + ('solar', 'pv'), ('biomass', 'ce')]: + element = {} + if carrier in ['wind', 'solar']: + if "onshore" == tech: + profile = b + "-onshore-profile" + capacity = 85500 + elif "offshore" == tech: + profile = b + "-offshore-profile" + capacity = 17000 + elif "pv" in tech: + profile = b + "-pv-profile" + capacity = 104500 + + elements["-".join([b, carrier, tech])] = element + e = { + "bus": b + "-electricity", + "tech": tech, + "carrier": carrier, + "capacity": capacity, + "type": "volatile", + "profile": profile, + } + + element.update(e) + + + elif carrier == "biomass": + elements["-".join([b, carrier, tech])] = element + + element.update({ + "carrier": carrier, + "capacity": 6000, + "to_bus": b + "-electricity", + "efficiency": 0.4, + "from_bus": b + "-biomass-bus", + "type": "conversion", + "carrier_cost": float( + carriers.at[(2030, carrier, 'cost'), 'value'] + ), + "tech": 'ce', + } + ) + + elements['DE-battery'] = { + "storage_capacity": 8 * 10000, # 8 h + "capacity": 10000, + "bus": "DE-electricity", + "tech": 'battery', + "carrier": 'electricity', + "type": "storage", + "efficiency": 0.9 + ** 0.5, # convert roundtrip to input / output efficiency + "marginal_cost": 0.0000001, + "loss": 0.01 + } + + + df = pd.DataFrame.from_dict(elements, orient="index") + + for element_type in ['volatile', 'conversion', 'storage']: + building.write_elements( + element_type + ".csv", + df.loc[df["type"] == element_type].dropna(how="all", axis=1), + directory=os.path.join(datapackage_dir, "data", "elements"), + ) + + +def excess(datapackage_dir): + """ + """ + path = os.path.join(datapackage_dir, "data", "elements") + buses = building.read_elements("bus.csv", directory=path) + + buses.index.name = "bus" + buses = buses.loc[buses['carrier'] == 'electricity'] + + elements = pd.DataFrame(buses.index) + elements["type"] = "excess" + elements["name"] = elements["bus"] + "-excess" + elements["marginal_cost"] = 0 + + elements.set_index("name", inplace=True) + + building.write_elements("excess.csv", elements, directory=path) + +def shortage(datapackage_dir): + """ + """ + path = os.path.join(datapackage_dir, "data", "elements") + buses = building.read_elements("bus.csv", directory=path) + + buses = buses.loc[buses['carrier'] == 'electricity'] + buses.index.name = "bus" + + elements = pd.DataFrame(buses.index) + elements["capacity"] = 10e10 + elements["type"] = "shortage" + elements["name"] = elements["bus"] + "-shortage" + elements["marginal_cost"] = 300 + + elements.set_index("name", inplace=True) + + building.write_elements("shortage.csv", elements, directory=path) diff --git a/src/fuchur/scripts/grid.py b/src/fuchur/scripts/grid.py new file mode 100644 index 0000000..fe4a9fd --- /dev/null +++ b/src/fuchur/scripts/grid.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +""" +""" +import os +import re + +from oemof.tabular.datapackage import building +import pandas as pd + +import fuchur + + +def _prepare_frame(df): + """ Prepare dataframe from ehighway excel sheet, see function: + ehighway_grid() + """ + df.dropna(how="all", axis=1, inplace=True) + df.drop(df.tail(1).index, inplace=True) + df.reset_index(inplace=True) + df["Links"] = df["Links"].apply(lambda row: row.upper()) + + df["Links"] = [i.replace("UK", "GB") for i in df["Links"]] # for ISO code + + # remove all links inside countries + df = df.loc[df["Links"].apply(_remove_links)] + + # strip down to letters only for grouping + df["Links"] = df["Links"].apply(lambda row: re.sub(r"[^a-zA-Z]+", "", row)) + + df = df.groupby(df["Links"]).sum() + + df.reset_index(inplace=True) + + df = pd.concat( + [ + pd.DataFrame( + df["Links"].apply(lambda row: [row[0:2], row[2:4]]).tolist(), + columns=["from", "to"], + ), + df, + ], + axis=1, + ) + + return df + + +# helper function for transshipment +def _remove_links(row): + """ Takes a row of the dataframe and returns True if the + link is within the country. + """ + r = row.split("-") + if r[0].split("_")[1].strip() == r[1].split("_")[1].strip(): + return False + else: + return True + + +def ehighway(buses, year, datapackage_dir, scenario = "100% RES", + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameter + --------- + buses: array like + List with buses represented by iso country code + year: integer + Scenario year to select. One of: 2030, 2050. If year is 2030, the + starting grid will be used, meaning the scenario argument will have no + impact + datapackage_dir: string + Directory for tabular resource + scenario: + Name of ehighway scenario to select. One of: + ["Large Scale RES", "100% RES", "Big & Market", "Fossil & Nuclear", + "Small & Local"], default: "100% RES" + raw_data_path: string + Path where raw data file `e-Highway_database_per_country-08022016.xlsx` + is located + """ + + filename = "e-Highway_database_per_country-08022016.xlsx" + + filepath = os.path.join(raw_data_path, filename) + + if os.path.exists(filepath): + # if file exist in archive use this file + df_2030 = pd.read_excel( + filepath, sheet_name="T93", index_col=[1], skiprows=[0, 1, 3] + ).fillna(0) + + df_2050 = pd.read_excel( + filepath, sheet_name="T94", index_col=[1], skiprows=[0, 1, 3] + ).fillna(0) + else: + raise FileNotFoundError( + "File for e-Highway capacities does not exist. Did you download?" + ) + + df_2050 = _prepare_frame(df_2050).set_index(["Links"]) + df_2030 = _prepare_frame(df_2030).set_index(["Links"]) + + elements = {} + for idx, row in df_2030.iterrows(): + if ( + row["from"] in buses + and row["to"] in buses + ): + + predecessor = row["from"] + "-electricity" + successor = row["to"] + "-electricity" + element_name = predecessor + "-" + successor + + if year == 2030: + capacity = row[scenario] + elif year == 2050: + capacity = (row[scenario] + + df_2050.to_dict()[scenario].get(idx, 0)) + + element = { + "type": "link", + "loss": 0.05, + "from_bus": predecessor, + "to_bus": successor, + "tech": "transshipment", + "capacity": capacity, + #"length": row["Length"], + } + + elements[element_name] = element + + building.write_elements( + "link.csv", + pd.DataFrame.from_dict(elements, orient="index"), + directory=os.path.join(datapackage_dir, "data/elements"), + ) + + +def tyndp(buses, datapackage_dir, raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameter + --------- + buses: array like + List with buses represented by iso country code + datapackage_dir: string + Directory for tabular resource + raw_data_path: string + Path where raw data file is located + """ + filepath = building.download_data( + "https://www.entsoe.eu/Documents/TYNDP%20documents/TYNDP2018/" + "Scenarios%20Data%20Sets/Input%20Data.xlsx", + directory=raw_data_path) + + df = pd.read_excel(filepath, sheet_name='NTC', index_col=[0], + skiprows=[1,2])[["CBA Capacities", "Unnamed: 3"]] + df.columns = ["=>", "<="] + df["links"] = df.index.astype(str) + df["links"] = df["links"].apply( + lambda row: (row.split('-')[0][0:2], row.split('-')[1][0:2])) + df = df.groupby(df["links"]).sum() + df.reset_index(inplace=True) + + df = pd.concat([ + pd.DataFrame(df["links"].apply(lambda row: [row[0], row[1]]).tolist(), + columns=['from', 'to']), + df[["=>", "<="]]], axis=1) + + elements = {} + for idx, row in df.iterrows(): + if ( + row["from"] in buses + and row["to"] in buses + ) and row["from"] != row["to"]: + + predecessor = row["from"] + "-electricity" + successor = row["to"] + "-electricity" + element_name = predecessor + "-" + successor + + element = { + "type": "link", + "loss": 0.05, + "from_bus": predecessor, + "to_bus": successor, + "tech": "transshipment", + "capacity": row["=>"] # still need to think how to + } + + elements[element_name] = element + + building.write_elements( + "link.csv", + pd.DataFrame.from_dict(elements, orient="index"), + directory=os.path.join(datapackage_dir, "data", "elements"), + ) diff --git a/src/fuchur/scripts/heat.py b/src/fuchur/scripts/heat.py new file mode 100644 index 0000000..e6b7723 --- /dev/null +++ b/src/fuchur/scripts/heat.py @@ -0,0 +1,475 @@ +# -*- coding: utf-8 -*- +""" +""" +import json +import os + +from datapackage import Package +from oemof.tabular.datapackage import building +from oemof.tools.economics import annuity +import pandas as pd + +import fuchur + + +def load(config, datapackage_dir): + """ + """ + # heat_load = pd.read_csv( + # os.path.join(fuchur.__RAW_DATA_PATH__, 'heat_load.csv')) + + elements = [] + for b in config["buses"]["heat"].get("central", []): + elements.append( + { + "name": b + "-central-heat-load", + "type": "load", + "bus": b + "-central-heat", + "amount": 200 * 1e6, + "profile": b + "-central-heat-load-profile", + "carrier": "heat", + } + ) + + # add decentral heat load + for b in config["buses"]["heat"].get("decentral", []): + elements.append( + { + "name": b + "-decentral-heat-load", + "type": "load", + "bus": b + "-decentral-heat", + "amount": 200 * 1e6, + "profile": b + "-decentral-heat-load-profile", + "carrier": "heat", + } + ) + + if elements: + building.write_elements( + "load.csv", + pd.DataFrame(elements).set_index("name"), + directory=os.path.join(datapackage_dir, "data/elements"), + ) + + central_heat_load_profile = pd.DataFrame( + data=pd.read_csv( + os.path.join( + fuchur.__RAW_DATA_PATH__, "central_heat_load_profiles.csv" + ), + sep=";", + )[config["buses"]["heat"].get("central", [])].values, + columns=[ + p + "-central-heat-load-profile" + for p in config["buses"]["heat"].get("central", []) + ], + index=pd.date_range( + str(config["temporal"]["scenario_year"]), + periods=8760, + freq="H", + ), + ) + + building.write_sequences( + "load_profile.csv", + central_heat_load_profile, + directory=os.path.join(datapackage_dir, "data/sequences"), + ) + + decentral_heat_load_profile = pd.DataFrame( + data=pd.read_csv( + os.path.join( + fuchur.__RAW_DATA_PATH__, "central_heat_load_profiles.csv" + ), + sep=";", + )[config["buses"]["heat"].get("decentral", [])].values, + columns=[ + p + "-decentral-heat-load-profile" + for p in config["buses"]["heat"].get("decentral", []) + ], + index=pd.date_range( + str(config["temporal"]["scenario_year"]), + periods=8760, + freq="H", + ), + ) + + building.write_sequences( + "load_profile.csv", + decentral_heat_load_profile, + directory=os.path.join(datapackage_dir, "data/sequences"), + ) + + +def decentral( + config, + datapackage_dir, + techmap={ + "backpressure": "backpressure", + "boiler_decentral": "dispatchable", + "heatpump_decentral": "conversion", + "hotwatertank_decentral": "storage", + }, +): + + wacc = config["cost"]["wacc"] + + technology_cost = Package( + "https://raw.githubusercontent.com/ZNES-datapackages/" + "technology-cost/master/datapackage.json" + ) + + technologies = pd.DataFrame( + technology_cost.get_resource("decentral_heat").read(keyed=True) + ) + + technologies = ( + technologies.groupby(["year", "tech", "carrier"]) + .apply(lambda x: dict(zip(x.parameter, x.value))) + .reset_index("carrier") + .apply(lambda x: dict({"carrier": x.carrier}, **x[0]), axis=1) + ) + technologies = technologies.loc[ + config["temporal"]["scenario_year"] + ].to_dict() + + carrier = pd.DataFrame( + technology_cost.get_resource("carrier").read(keyed=True) + ).set_index(["carrier", "parameter"]) + + # maybe we should prepare emission factors for scenario year... + emission = carrier[carrier.year == 2015] # 2015 as emission not change + + carrier = carrier[carrier.year == config["temporal"]["scenario_year"]] + + elements = dict() + + for b in config["buses"]["heat"].get("decentral", []): + for tech, entry in technologies.items(): + element_name = b + "-" + tech + heat_bus = b + "-decentral-heat" + + element = entry.copy() + + elements[element_name] = element + + if techmap.get(tech) == "backpressure": + element.update( + { + "type": techmap[tech], + "fuel_bus": "GL-" + entry["carrier"], + "carrier": entry["carrier"], + "fuel_cost": carrier.at[ + (entry["carrier"], "cost"), "value" + ], + "electricity_bus": b + "-electricity", + "heat_bus": heat_bus, + "thermal_efficiency": entry["thermal_efficiency"], + "input_parameters": json.dumps( + { + "emission_factor": float( + emission.at[ + (entry["carrier"], "emission-factor"), + "value", + ] + ) + } + ), + "electric_efficiency": entry["electrical_efficiency"], + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, # €/kW -> €/MW + } + ) + + elif techmap.get(tech) == "dispatchable": + element.update( + { + "type": techmap[tech], + "carrier": entry["carrier"], + "marginal_cost": ( + float( + carrier.at[(entry["carrier"], "cost"), "value"] + ) + / float(entry["efficiency"]) + ), + "bus": heat_bus, + "output_parameters": json.dumps( + { + "emission_factor": float( + emission.at[ + (entry["carrier"], "emission-factor"), + "value", + ] + ) + / float(entry["efficiency"]) + } + ), + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + } + ) + + elif techmap.get(tech) == "conversion": + element.update( + { + "type": techmap[tech], + "carrier": entry["carrier"], + "from_bus": b + "-electricity", + "to_bus": heat_bus, + "efficiency": entry["efficiency"], + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + } + ) + + elif techmap.get(tech) == "storage": + element.update( + { + "storage_capacity_cost": annuity( + float(entry["storage_capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + "bus": heat_bus, + "tech": tech, + "type": "storage", + "capacity_potential": "Infinity", + # rounttrip -> to in / out efficiency + "efficiency": float(entry["efficiency"]) ** 0.5, + "capacity_ratio": entry["capacity_ratio"], + } + ) + + elements = pd.DataFrame.from_dict(elements, orient="index") + + for type in set(techmap.values()): + building.write_elements( + type + ".csv", + elements.loc[elements["type"] == type].dropna(how="all", axis=1), + directory=os.path.join(datapackage_dir, "data", "elements"), + ) + + +def central( + config, + datapackage_dir, + techmap={ + "extraction": "extraction", + "boiler_central": "dispatchable", + "hotwatertank_central": "storage", + "heatpump_central": "conversion", + }, +): + """ + """ + + wacc = config["cost"]["wacc"] + + technology_cost = Package( + "https://raw.githubusercontent.com/ZNES-datapackages/" + "technology-cost/master/datapackage.json" + ) + + technologies = pd.DataFrame( + technology_cost.get_resource("central_heat").read(keyed=True) + ) + + technologies = ( + technologies.groupby(["year", "tech", "carrier"]) + .apply(lambda x: dict(zip(x.parameter, x.value))) + .reset_index("carrier") + .apply(lambda x: dict({"carrier": x.carrier}, **x[0]), axis=1) + ) + technologies = technologies.loc[ + config["temporal"]["scenario_year"] + ].to_dict() + + carrier = pd.DataFrame( + technology_cost.get_resource("carrier").read(keyed=True) + ).set_index(["carrier", "parameter"]) + + # maybe we should prepare emission factors for scenario year... + emission = carrier[carrier.year == 2015] # 2015 as emission not change + + carrier = carrier[carrier.year == config["temporal"]["scenario_year"]] + + elements = dict() + + for b in config["buses"]["heat"].get("central", []): + for tech, entry in technologies.items(): + element_name = b + "-" + tech + heat_bus = b + "-central-heat" + + element = entry.copy() + + elements[element_name] = element + + if techmap.get(tech) == "extraction": + element.update( + { + "type": techmap[tech], + "carrier": entry["carrier"], + "fuel_bus": "GL-" + entry["carrier"], + "carrier_cost": carrier.at[ + (entry["carrier"], "cost"), "value" + ], + "electricity_bus": "DE-electricity", + "heat_bus": heat_bus, + "thermal_efficiency": entry["thermal_efficiency"], + "input_parameters": json.dumps( + { + "emission_factor": float( + emission.at[ + (entry["carrier"], "emission-factor"), + "value", + ] + ) + } + ), + "electric_efficiency": entry["electrical_efficiency"], + "condensing_efficiency": entry[ + "condensing_efficiency" + ], + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + } + ) + + elif techmap.get(tech) == "backpressure": + element.update( + { + "type": techmap[tech], + "fuel_bus": "GL-" + entry["carrier"], + "carrier": entry["carrier"], + "fuel_cost": carrier.at[ + (entry["carrier"], "cost"), "value" + ], + "electricity_bus": b + "-electricity", + "heat_bus": heat_bus, + "thermal_efficiency": entry["thermal_efficiency"], + "input_parameters": json.dumps( + { + "emission_factor": float( + emission.at[ + (entry["carrier"], "emission-factor"), + "value", + ] + ) + } + ), + "electric_efficiency": entry["electrical_efficiency"], + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, # €/kW -> €/MW + } + ) + + elif techmap.get(tech) == "dispatchable": + element.update( + { + "type": techmap[tech], + "carrier": entry["carrier"], + "marginal_cost": ( + float( + carrier.at[(entry["carrier"], "cost"), "value"] + ) + / float(entry["efficiency"]) + ), + "bus": heat_bus, + "output_parameters": json.dumps( + { + "emission_factor": float( + emission.at[ + (entry["carrier"], "emission-factor"), + "value", + ] + ) + / float(entry["efficiency"]) + } + ), + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + } + ) + + elif techmap.get(tech) == "conversion": + element.update( + { + "type": techmap[tech], + "carrier": entry["carrier"], + "from_bus": b + "-electricity", + "to_bus": heat_bus, + "efficiency": entry["efficiency"], + "capacity_potential": "Infinity", + "tech": tech, + "capacity_cost": annuity( + float(entry["capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + } + ) + + elif techmap.get(tech) == "storage": + element.update( + { + "storage_capacity_cost": annuity( + float(entry["storage_capacity_cost"]), + float(entry["lifetime"]), + wacc, + ) + * 1000, + "bus": heat_bus, + "tech": tech, + "type": "storage", + "capacity_potential": "Infinity", + # rounttrip -> to in / out efficiency + "efficiency": float(entry["efficiency"]) ** 0.5, + "capacity_ratio": entry["capacity_ratio"], + } + ) + + elements = pd.DataFrame.from_dict(elements, orient="index") + + for type in set(techmap.values()): + building.write_elements( + type + ".csv", + elements.loc[elements["type"] == type].dropna(how="all", axis=1), + directory=os.path.join(datapackage_dir, "data", "elements"), + ) diff --git a/src/fuchur/scripts/hydro.py b/src/fuchur/scripts/hydro.py new file mode 100644 index 0000000..952de37 --- /dev/null +++ b/src/fuchur/scripts/hydro.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +""" +""" +import os + +from datapackage import Package + +from oemof.tabular.datapackage import building +from oemof.tools.economics import annuity +import pandas as pd + +import fuchur + + +def _get_hydro_inflow(inflow_dir=None): + """ Adapted from: + + https://github.com/FRESNA/vresutils/blob/master/vresutils/hydro.py + """ + + def read_inflow(country): + return pd.read_csv( + os.path.join(inflow_dir, "Hydro_Inflow_{}.csv".format(country)), + parse_dates={"date": [0, 1, 2]}, + ).set_index("date")["Inflow [GWh]"] + + europe = [ + "AT", + "BA", + "BE", + "BG", + "CH", + "CZ", + "DE", + "ES", + "FI", + "FR", + "HR", + "HU", + "IE", + "IT", + "KV", + "LT", + "LV", + "ME", + "MK", + "NL", + "NO", + "PL", + "PT", + "RO", + "RS", + "SE", + "SI", + "SK", + "UK", + ] + + hyd = pd.DataFrame({cname: read_inflow(cname) for cname in europe}) + + hyd.rename(columns={"UK": "GB"}, inplace=True) # for ISO country code + + hydro = hyd.resample("H").interpolate("cubic") + + # add last day of the dataset that is missing from resampling + last_day = pd.DataFrame( + index=pd.DatetimeIndex(start="20121231", freq="H", periods=24), + columns=hydro.columns, + ) + data = hyd.loc["2012-12-31"] + for c in last_day: + last_day.loc[:, c] = data[c] + + # need to drop last day because it comes in last day... + hydro = pd.concat([hydro.drop(hydro.tail(1).index), last_day]) + + # remove last day in Feb for leap years + hydro = hydro[~((hydro.index.month == 2) & (hydro.index.day == 29))] + + if True: # default norm + normalization_factor = hydro.index.size / float( + hyd.index.size + ) # normalize to new sampling frequency + # else: + # # conserve total inflow for each country separately + # normalization_factor = hydro.sum() / hyd.sum() + hydro /= normalization_factor + + return hydro + + +def generation(config, datapackage_dir, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + """ + countries, year = ( + config["buses"]["electricity"], + config["temporal"]["scenario_year"], + ) + + filepath = building.download_data( + 'https://zenodo.org/record/804244/files/hydropower.csv?download=1', + directory=raw_data_path) + + capacities = pd.read_csv( + filepath, + index_col=["ctrcode"], + ) + capacities.rename(index={"UK": "GB"}, inplace=True) # for iso code + + capacities.loc["CH"] = [8.8, 12, 1.9] # add CH elsewhere + + inflows = _get_hydro_inflow( + inflow_dir=os.path.join(raw_data_path, "Hydro_Inflow") + ) + + inflows = inflows.loc[ + inflows.index.year == config["temporal"]["weather_year"], : + ] + inflows["DK"], inflows["LU"] = 0, inflows["BE"] + + technologies = pd.DataFrame( + Package( + "https://raw.githubusercontent.com/ZNES-datapackages" + "/technology-cost/master/datapackage.json" + ) + .get_resource("electricity") + .read(keyed=True) + ) + technologies = ( + technologies.groupby(["year", "tech", "carrier"]) + .apply(lambda x: dict(zip(x.parameter, x.value))) + .reset_index("carrier") + .apply(lambda x: dict({"carrier": x.carrier}, **x[0]), axis=1) + ) + technologies = technologies.loc[year].to_dict() + + ror_shares = pd.read_csv( + os.path.join(raw_data_path, "ror_ENTSOe_Restore2050.csv"), + index_col="Country Code (ISO 3166-1)", + )["ror ENTSO-E\n+ Restore"] + + # ror + ror = pd.DataFrame(index=countries) + ror["type"], ror["tech"], ror["bus"], ror["capacity"], ror['carrier'] = ( + "volatile", + "ror", + ror.index.astype(str) + "-electricity", + ( + capacities.loc[ror.index, " installed hydro capacities [GW]"] + - capacities.loc[ + ror.index, " installed pumped hydro capacities [GW]" + ] + ) + * ror_shares[ror.index] + * 1000, + 'hydro' + ) + + ror = ror.assign(**technologies["ror"])[ror["capacity"] > 0].dropna() + ror["profile"] = ror["bus"] + "-" + ror["tech"] + "-profile" + + ror_sequences = (inflows[ror.index] * ror_shares[ror.index] * 1000) / ror[ + "capacity" + ] + ror_sequences.columns = ror_sequences.columns.map(ror["profile"]) + + # phs + phs = pd.DataFrame(index=countries) + phs["type"], phs["tech"], phs["bus"], phs["loss"], phs["capacity"], phs[ + "marginal_cost"], phs['carrier'] = ( + "storage", + "phs", + phs.index.astype(str) + "-electricity", + 0, + capacities.loc[phs.index, " installed pumped hydro capacities [GW]"] + * 1000, + 0.0000001, + "hydro" + ) + + phs["storage_capacity"] = phs["capacity"] * 6 # Brown et al. + # as efficieny in data is roundtrip use sqrt of roundtrip + phs["efficiency"] = float(technologies["phs"]["efficiency"]) ** 0.5 + phs = phs.assign(**technologies["phs"])[phs["capacity"] > 0].dropna() + + # other hydro / reservoir + rsv = pd.DataFrame(index=countries) + rsv["type"], rsv["tech"], rsv["bus"], rsv["loss"], rsv["capacity"], rsv[ + "storage_capacity"], rsv['carrier'] = ( + "reservoir", + "reservoir", + rsv.index.astype(str) + "-electricity", + 0, + ( + capacities.loc[ror.index, " installed hydro capacities [GW]"] + - capacities.loc[ + ror.index, " installed pumped hydro capacities [GW]" + ] + ) + * (1 - ror_shares[ror.index]) + * 1000, + capacities.loc[rsv.index, " reservoir capacity [TWh]"] * 1e6, + "hydro" + ) # to MWh + + rsv = rsv.assign(**technologies["rsv"])[rsv["capacity"] > 0].dropna() + rsv["profile"] = rsv["bus"] + "-" + rsv["tech"] + "-profile" + rsv[ + "efficiency" + ] = 1 # as inflow is already in MWelec -> no conversion needed + rsv_sequences = ( + inflows[rsv.index] * (1 - ror_shares[rsv.index]) * 1000 + ) # GWh -> MWh + rsv_sequences.columns = rsv_sequences.columns.map(rsv["profile"]) + + # write sequences to different files for better automatic foreignKey + # handling in meta data + building.write_sequences( + "reservoir_profile.csv", + rsv_sequences.set_index( + building.timeindex(year=str(config["temporal"]["scenario_year"])) + ), + directory=os.path.join(datapackage_dir, "data", "sequences"), + ) + + building.write_sequences( + "ror_profile.csv", + ror_sequences.set_index( + building.timeindex(year=str(config["temporal"]["scenario_year"])) + ), + directory=os.path.join(datapackage_dir, "data", "sequences"), + ) + + filenames = ["ror.csv", "phs.csv", "reservoir.csv"] + + for fn, df in zip(filenames, [ror, phs, rsv]): + df.index = df.index.astype(str) + "-hydro-" + df["tech"] + df["capacity_cost"] = df.apply( + lambda x: annuity( + float(x["capacity_cost"]) * 1000, + float(x["lifetime"]), + config["cost"]["wacc"], + ), + axis=1, + ) + building.write_elements( + fn, df, directory=os.path.join(datapackage_dir, "data", "elements") + ) diff --git a/src/fuchur/scripts/load.py b/src/fuchur/scripts/load.py new file mode 100644 index 0000000..ce39a63 --- /dev/null +++ b/src/fuchur/scripts/load.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +""" +""" +import os + +from oemof.tabular.datapackage import building + +import pandas as pd + +import fuchur + + +def tyndp(buses, scenario, datapackage_dir, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + """ + filepath = building.download_data( + "https://www.entsoe.eu/Documents/TYNDP%20documents/TYNDP2018/" + "Scenarios%20Data%20Sets/Input%20Data.xlsx", + directory=raw_data_path) + + df = pd.read_excel(filepath, sheet_name='Demand') + df['countries'] = [i[0:2] for i in df.index] # for aggregation by country + + elements = df.groupby('countries').sum()[scenario].to_frame() + elements.index.name = "bus" + elements = elements.loc[buses] + elements.reset_index(inplace=True) + elements["name"] = elements.apply( + lambda row: row.bus + "-electricity-load", axis=1 + ) + elements["profile"] = elements.apply( + lambda row: row.bus + "-electricity-load-profile", axis=1 + ) + elements["type"] = "load" + elements["carrier"] = "electricity" + elements.set_index("name", inplace=True) + elements.bus = [b + "-electricity" for b in elements.bus] + elements["amount"] = elements[scenario] * 1000 # MWh -> GWh + + building.write_elements( + "load.csv", + elements, + directory=os.path.join(datapackage_dir, "data", "elements"), + ) + +def ehighway(buses, year, datapackage_dir, scenario="100% RES", + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameter + --------- + buses: array like + List with buses represented by iso country code + year: integer + Scenario year to select. One of: 2040, 2050 + datapackage_dir: string + Directory for tabular resource + scenario: + Name of ehighway scenario to select. One of: + ["Large Scale RES", "100% RES", "Big & Market", "Fossil & Nuclear", + "Small & Local"], default: "100% RES" + raw_data_path: string + Path where raw data file `e-Highway_database_per_country-08022016.xlsx` + is located + """ + filename = "e-Highway_database_per_country-08022016.xlsx" + filepath = os.path.join(raw_data_path, filename) + + if year == 2050: + sheet = "T40" + elif year == 2040: + sheet = "T39" + else: + raise ValueError( + "Value of argument `year` must be integer 2040 or 2050!") + + if os.path.exists(filepath): + df = pd.read_excel(filepath, sheet_name=sheet, index_col=[0], + skiprows=[0,1]) + else: + raise FileNotFoundError( + "File for e-Highway loads does not exist. Did you download data?" + ) + + df.set_index("Scenario", inplace=True) # Scenario in same colum as ctrcode + df.drop(df.index[0:1], inplace=True) # remove row with units + df.dropna(how="all", axis=1, inplace=True) + + elements = df.loc[buses, scenario].to_frame() + elements = elements.rename(columns={scenario: "amount"}) + elements.index.name = "bus" + elements.reset_index(inplace=True) + elements["name"] = elements.apply( + lambda row: row.bus + "-electricity-load", axis=1 + ) + elements["profile"] = elements.apply( + lambda row: row.bus + "-electricity-load-profile", axis=1 + ) + elements["type"] = "load" + elements["carrier"] = "electricity" + elements.set_index("name", inplace=True) + elements.bus = [b + "-electricity" for b in elements.bus] + elements["amount"] = elements["amount"] * 1000 # to MWh + + path = os.path.join(datapackage_dir, "data", "elements") + building.write_elements("load.csv", elements, directory=path) + + +def opsd_profile(buses, demand_year, scenario_year, datapackage_dir, + raw_data_path=fuchur.__RAW_DATA_PATH__): + """ + Parameter + --------- + buses: array like + List with buses represented by iso country code + demand_year: integer or string + Demand year to select + scenario_year: integer or string + Year of scenario to use for timeindex to resource + datapackage_dir: string + Directory for tabular resource + raw_data_path: string + Path where raw data file + is located + """ + + filepath = building.download_data( + "https://data.open-power-system-data.org/time_series/2018-06-30/time_series_60min_singleindex.csv", + directory=raw_data_path + ) + + if os.path.exists(filepath): + raw_data = pd.read_csv(filepath, index_col=[0], parse_dates=True) + else: + raise FileNotFoundError( + "File for OPSD loads does not exist. Did you download data?" + ) + + suffix = "_load_old" + + countries = buses + + columns = [c + suffix for c in countries] + + timeseries = raw_data[str(demand_year)][columns] + + if timeseries.isnull().values.any(): + raise ValueError( + "Timeseries for load has NaN values. Select " + + "another demand year or use another data source." + ) + + load_total = timeseries.sum() + + load_profile = timeseries / load_total + + sequences_df = pd.DataFrame(index=load_profile.index) + + elements = building.read_elements( + "load.csv", directory=os.path.join(datapackage_dir, "data", "elements") + ) + + for c in countries: + # get sequence name from elements edge_parameters + # (include re-exp to also check for 'elec' or similar) + sequence_name = elements.at[ + elements.index[elements.index.str.contains(c)][0], "profile" + ] + + sequences_df[sequence_name] = load_profile[c + suffix].values + + if sequences_df.index.is_leap_year[0]: + sequences_df = sequences_df.loc[ + ~((sequences_df.index.month == 2) & (sequences_df.index.day == 29)) + ] + + sequences_df.index = building.timeindex( + year=str(scenario_year) + ) + + building.write_sequences( + "load_profile.csv", + sequences_df, + directory=os.path.join(datapackage_dir, "data/sequences"), + ) diff --git a/tests/test_fuchur.py b/tests/test_fuchur.py index 7eeaf9e..611f7cf 100644 --- a/tests/test_fuchur.py +++ b/tests/test_fuchur.py @@ -1,12 +1,82 @@ - from click.testing import CliRunner +import copy +import os -from fuchur.cli import main +from fuchur.cli import Scenario, cli +from fuchur.scripts import electricity +import fuchur +buses = [ + "AT", + "BE", + "BG", + "CH", + "CZ", + "DE", + "DK", + "ES", + "FI", + "FR", + "HR", + "HU", + "IE", + "IT", + "LT", + "LV", + "NL", + "NO", + "PL", + "PT", + "RO", + "SE", + "SI", + "SK", + "GB" +] def test_main(): runner = CliRunner() - result = runner.invoke(main, []) + result = runner.invoke(cli, []) + help = runner.invoke(cli, ["--help"]) - assert result.output == '()\n' + assert result.output == help.output assert result.exit_code == 0 + + +def test_builtin_scenario_availability(): + assert fuchur.scenarios.keys() == set( + ["el-2pv-cost", "el,-base", "el-no-biomass", "scenario2", "test"] + ) + + +def test_builtin_scenario_construction(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(cli, ["construct", "test"]) + assert result.exit_code == 0 + assert os.listdir(os.curdir) == [], ( + "\nIf you see this message, a test started failing that was" + "\nasserting the wrong fact anyway. The working directory should" + "\nnot be empty after running `fuchur construct test`.\n" + "\nNow that the working directory contains something, you can" + "\nstart correcting the test." + ) + + +def test_scenario_class(): + scenario = Scenario({"name": "child", "parents": ["test"]}) + expected = copy.deepcopy(fuchur.scenarios["test"]) + expected["name"] = "child" + expected["parents"] = ["test"] + assert scenario == expected + +def test_load(): + for s in ["Large Scale RES", "100% RES", "Big & Market", "Fossil & Nuclear", + "Small & Local"]: + for year in [2040, 2050]: + electricity.ehighway_load( + buses, year, os.path.join("/tmp", s, str(year)), scenario=s) + + electricity.opsd_load_profile( + buses, 2012, os.path.join("/tmp", "100% RES", str(2050))) +test_load() diff --git a/tox.ini b/tox.ini index 04e10d5..2b1558c 100644 --- a/tox.ini +++ b/tox.ini @@ -3,8 +3,6 @@ envlist = clean, check, docs, - py27-cover, - py27-nocov, py34-cover, py34-nocov, py35-cover, @@ -13,15 +11,11 @@ envlist = py36-nocov, py37-cover, py37-nocov, - pypy-cover, - pypy-nocov, - pypy3-cover, - pypy3-nocov, report [testenv] basepython = - {docs,spell}: {env:TOXPYTHON:python2.7} + {docs,spell}: {env:TOXPYTHON:python3} {bootstrap,clean,check,report,codecov,coveralls}: {env:TOXPYTHON:python3} setenv = PYTHONPATH={toxinidir}/tests @@ -33,6 +27,9 @@ deps = pytest-travis-fold commands = {posargs:pytest -vv --ignore=src} +install_command = + python -m pip install {opts} --process-dependency-links {packages} + [testenv:bootstrap] deps = @@ -105,20 +102,6 @@ commands = coverage erase skip_install = true deps = coverage -[testenv:py27-cover] -basepython = {env:TOXPYTHON:python2.7} -setenv = - {[testenv]setenv} -usedevelop = true -commands = - {posargs:pytest --cov --cov-report=term-missing -vv} -deps = - {[testenv]deps} - pytest-cov - -[testenv:py27-nocov] -basepython = {env:TOXPYTHON:python2.7} - [testenv:py34-cover] basepython = {env:TOXPYTHON:python3.4} setenv = @@ -171,37 +154,9 @@ commands = deps = {[testenv]deps} pytest-cov +ignore_errors = True [testenv:py37-nocov] basepython = {env:TOXPYTHON:python3.7} - -[testenv:pypy-cover] -basepython = {env:TOXPYTHON:pypy} -setenv = - {[testenv]setenv} -usedevelop = true -commands = - {posargs:pytest --cov --cov-report=term-missing -vv} -deps = - {[testenv]deps} - pytest-cov - -[testenv:pypy-nocov] -basepython = {env:TOXPYTHON:pypy} - -[testenv:pypy3-cover] -basepython = {env:TOXPYTHON:pypy3} -setenv = - {[testenv]setenv} -usedevelop = true -commands = - {posargs:pytest --cov --cov-report=term-missing -vv} -deps = - {[testenv]deps} - pytest-cov - -[testenv:pypy3-nocov] -basepython = {env:TOXPYTHON:pypy3} - - +ignore_errors = True