diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7c1cd70..e4c0486 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements-github.txt # Install dependencies from requirements.txt + python -m pip install -r requirements-gh-action.txt # Install dependencies - name: Run tests with pytest run: | diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml index cde8f89..cf6ed8d 100644 --- a/.github/workflows/typing.yml +++ b/.github/workflows/typing.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install mypy + python -m pip install -r requirements-mypy.txt # Install dependencies - name: Type check with mypy run: | diff --git a/README.md b/README.md index e5e597c..c92aa53 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,19 @@ # SuomiGeoData ## What is SuomiGeoData? -SuomiGeoData is a Python package designed to simplify the process of downloading and extracting geospatial data from Suomi, that is Finland. The features of the package include: +SuomiGeoData is a Python package whose concept originated on September 11, 2024. It is designed to simplify the process of downloading and extracting geospatial data from Suomi, that is Finland. The package offers the following features: -- **[Paituli](https://paituli.csc.fi/download.html)** - - Accessing the topographic database index map. + +* [Paituli](https://paituli.csc.fi/download.html) website + + - Provides access to vector format index maps for downloading DEM and the topographic database. + - Downloads DEM as raster files and the topographic database as shapefiles based on label names from the index maps. + + +## Roadmap + +* Enable downloading DEM for a specified area using a shapefile. +* Enable downloading the topographic database for a specified area using a shapefile. ## Easy Installation @@ -23,9 +32,16 @@ A brief example of how to start: >>> paituli = SuomiGeoData.Paituli() # get the topographic database index map ->>> im_tb = paituli.indexmap_tdb ->>> im_tb.shape -(3132, 3) +>>> paituli.indexmap_tdb.head() + + label path geometry + ------------------------------------------------------------------------------------------------------------- +0 K2344R mml/maastotietokanta/2022/shp/K2/K23/K2344R.sh... POLYGON ((104000 6606000, 104000 6618000, 1160... +1 K2334R mml/maastotietokanta/2022/shp/K2/K23/K2334R.sh... POLYGON ((104000 6582000, 104000 6594000, 1160... +2 K2343R mml/maastotietokanta/2022/shp/K2/K23/K2343R.sh... POLYGON ((104000 6594000, 104000 6606000, 1160... +3 K2443L mml/maastotietokanta/2022/shp/K2/K24/K2443L.sh... POLYGON ((92000 6642000, 92000 6654000, 104000... +4 K2443R mml/maastotietokanta/2022/shp/K2/K24/K2443R.sh... POLYGON ((104000 6642000, 104000 6654000, 1160... +... ``` ## Documentation diff --git a/SuomiGeoData/__init__.py b/SuomiGeoData/__init__.py index fae49e5..9c31413 100644 --- a/SuomiGeoData/__init__.py +++ b/SuomiGeoData/__init__.py @@ -6,4 +6,4 @@ ] -__version__ = '0.0.1' +__version__ = '0.1.0' diff --git a/SuomiGeoData/core.py b/SuomiGeoData/core.py index 84934cf..a4bdb45 100644 --- a/SuomiGeoData/core.py +++ b/SuomiGeoData/core.py @@ -33,3 +33,36 @@ def is_valid_write_shape_driver( output = False return output + + @property + def _url_prefix_paituli_dem_tdb( + self, + ) -> str: + + ''' + Returns the prefix url for downloading files + based on DEM and topographic database labels. + ''' + + output = 'https://www.nic.funet.fi/index/geodata/' + + return output + + @property + def default_http_headers( + self, + ) -> dict[str, str]: + + ''' + Returns the default http headers to be used for the web requests. + ''' + + output = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Host': 'www.nic.funet.fi', + 'Accept-Encoding': 'gzip, deflate, br, zstd', + 'Connection': 'keep-alive' + } + + return output diff --git a/SuomiGeoData/paituli.py b/SuomiGeoData/paituli.py index 73965c6..d7c4b84 100644 --- a/SuomiGeoData/paituli.py +++ b/SuomiGeoData/paituli.py @@ -1,5 +1,8 @@ import os +import io +import zipfile import geopandas +import requests import typing from .core import Core @@ -7,12 +10,16 @@ class Paituli: ''' - Executes downloading and extracting data from Paituli (https://paituli.csc.fi/download.html). + Executes downloading and extracting data from Paituli + (https://paituli.csc.fi/download.html). Attributes: ----------- + index_dem : GeoDataFrame + A GeoDataFrame containing the DEM index map. + index_td : GeoDataFrame - A GeoDataFrame containing the topographical database index map. + A GeoDataFrame containing the topographic database index map. ''' def __init__( @@ -20,8 +27,7 @@ def __init__( ) -> None: ''' - Initializes the class by loading the GeoDataFrame of index maps for - DEM (raster data) and topographical database (vector data). + Initializes the class by loading the GeoDataFrame of index maps. ''' # DEM index map @@ -45,8 +51,7 @@ def save_indexmap_dem( ) -> bool: ''' - Saves the GeoDataFrame of the DEM - index map to the specified file path. + Saves the GeoDataFrame of the DEM index map to the specified file path. Parameters ---------- @@ -185,3 +190,153 @@ def is_valid_label_tdb( ''' return label in self.tdb_labels + + def dem_download_by_labels( + self, + labels: list[str], + folder_path: str, + http_headers: typing.Optional[dict[str, str]] = None + ) -> bool: + + ''' + Downloads the DEM raster files for the given labels. + + Parameters + ---------- + labels : list of str + List of label names from the DEM index map. + + folder_path : str + Complete folder path to save the downloaded raster files. + + http_headers : dict, optional + HTTP headers to be used for the web request. If not provided, the default headers + :attr:`SuomiGeoData.core.Core.default_http_headers` will be used. + + Returns + ------- + bool + True if all the DEM raster files were successfully downloaded and + exist at the specified folder path, False otherwise. + ''' + + # check whether the input labels exist + for label in labels: + if self.is_valid_label_dem(label): + pass + else: + raise Exception( + f'The label "{label}" does not exist in the index map.' + ) + + # check the existence of the given folder path + if os.path.isdir(folder_path): + pass + else: + raise Exception( + f'The folder path "{folder_path}" is not a valid directory.' + ) + + # web request headers + if http_headers is None: + headers = Core().default_http_headers + else: + headers = http_headers + + # download topographic database + suffix_urls = self.indexmap_dem[self.indexmap_dem['label'].isin(labels)]['path'] + count = 1 + for label, url in zip(labels, suffix_urls): + label_url = Core()._url_prefix_paituli_dem_tdb + url + response = requests.get( + url=label_url, + headers=headers + ) + label_file = os.path.join( + folder_path, f'{label}.tif' + ) + with open(label_file, 'wb') as label_raster: + label_raster.write(response.content) + print( + f'Download of label {label} completed (count {count}/{len(labels)}).' + ) + count = count + 1 + + output = all(os.path.isfile(os.path.join(folder_path, f'{label}.tif')) for label in labels) + + return output + + def tdb_download_by_labels( + self, + labels: list[str], + folder_path: str, + http_headers: typing.Optional[dict[str, str]] = None + ) -> bool: + + ''' + Downloads the topographic database folders of shapefiles for the given labels. + + Parameters + ---------- + labels : list of str + List of label names from the topographic database index map. + + folder_path : str + Complete folder path to save the downloaded folder of shapefiles. + + http_headers : dict, optional + HTTP headers to be used for the web request. If not provided, the default headers + :attr:`SuomiGeoData.core.Core.default_http_headers` will be used. + + Returns + ------- + bool + True if all the topographic database folders were successfully downloaded and + exist at the specified folder path, False otherwise. + ''' + + # check whether the input labels exist + for label in labels: + if self.is_valid_label_tdb(label): + pass + else: + raise Exception( + f'The label "{label}" does not exist in the index map.' + ) + + # check the existence of the given folder path + if os.path.isdir(folder_path): + pass + else: + raise Exception( + f'The folder path "{folder_path}" is not a valid directory.' + ) + + # web request headers + if http_headers is None: + headers = Core().default_http_headers + else: + headers = http_headers + + # download topographic database + suffix_urls = self.indexmap_tdb[self.indexmap_tdb['label'].isin(labels)]['path'] + count = 1 + for label, url in zip(labels, suffix_urls): + label_url = Core()._url_prefix_paituli_dem_tdb + url + response = requests.get( + url=label_url, + headers=headers + ) + label_data = io.BytesIO(response.content) + with zipfile.ZipFile(label_data, 'r') as label_zip: + label_zip.extractall( + os.path.join(folder_path, label) + ) + print( + f'Download of label {label} completed (count {count}/{len(labels)}).' + ) + count = count + 1 + + output = all(os.path.isdir(os.path.join(folder_path, label)) for label in labels) + + return output diff --git a/docs/changelog.rst b/docs/changelog.rst index 32bd534..662b9ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,10 +3,37 @@ Release Notes ============= +Version 0.1.0 +------------- + +* **Release date:** 17-Sep-2024. + +* **Feature Additions:** + + * Access characteristics of the DEM index map. + * Download selected labels as raster files from the DEM index map. + * Download selected labels as shapefile folders from the topographic database index map. + +* **GitHub Actions Integration:** + + * Linting with `flake8` to enforce PEP8 code formatting. + * Type checking with `mypy` to verify annotations throughout the codebase. + * Code testing with `pytest` to ensure code reliability. + * Test Coverage with **Codecov** to monitor and report test coverage. + +* **Compatibity:** Verified with Python 3.10, 3.11, and 3.12. + +* **Documentation:** Added new badges to `README.md` to display statuses for linting, type-checking, testing, and coverage. + +* **Development status:** Upgraded to Pre-Alpha from Planning. + + Version 0.0.1 ------------- -* **Features:** Accessing the topographic database index map via the :class:`SuomiGeoData.Paituli` class. +* **Release date:** 14-Sep-2024. + +* **Features:** Functionality for accessing the characteristics of topographic database index map. * **Development status:** Planning. diff --git a/docs/index.rst b/docs/index.rst index 60561ac..88065cb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Welcome to SuomiGeoData's documentation! introduction installation + quickstart modules changelog diff --git a/docs/installation.rst b/docs/installation.rst index 28713e0..c489fd8 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,14 +2,27 @@ Installation ============ -Installation of the SuomiGeoData package is simple. +The installation of the package is straightforward. To prevent conflicts with other Python packages, it is recommended to create a separate Python environment. +Below are the steps for installing the package using different methods. -Install from pip ----------------- +Create a Python environment +--------------------------- + +Suppose your environment name is my_env, and you can create it by using the following steps through Anaconda distribution. .. code-block:: console + + conda create --name my_env + conda activate my_env + conda install pip + +Install from PyPI +----------------- + +.. code-block:: console + pip install SuomiGeoData @@ -20,3 +33,19 @@ Install from GitHub repository .. code-block:: console pip install git+https://github.com/debpal/SuomiGeoData.git + + +Install from source code in editable mode +----------------------------------------- + +For developers who want to modify the source code or contribute to the package, it is recommended to install in editable mode. +Navigate to your directory with the `my_env` Python environemnt activated, and run the following commands. +This allows you to make changes to the source code, with immediate reflection in the `my_env` environment without requiring reinstallation. + +.. code-block:: console + + pip install build + git clone https://github.com/debpal/SuomiGeoData.git + cd SuomiGeoData + python -m build + pip install -e . diff --git a/docs/introduction.rst b/docs/introduction.rst index 84ee2f6..d7da458 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -2,8 +2,13 @@ Introdution =========== -SuomiGeoData, a Python package born on September 11, 2024, is designed to simplify the process of -downloading, extracting, and analyzing geospatial data from Suomi, that is Finland. -The package has folloiwng features: -* Accessing the topographic database index map via the :class:`SuomiGeoData.Paituli` class. +SuomiGeoData is a Python package whose concept originated on September 11, 2024. It is designed to simplify the process of downloading and extracting geospatial data from Suomi, that is Finland. The package offers the following features: + + +* `Paituli website `_ + + - Provides access to the vector format index map for downloading DEM. + - Downloads selected labels as raster files from the DEM index map. + - Provides access to the vector format index map for downloading topographic database. + - Downloads selected labels as folders of shapefiles from the topographic database index map. \ No newline at end of file diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000..d2be86b --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,80 @@ +=========== +Quickstart +=========== + +This guide provides a quick overview to get started with :mod:`SuomiGeoData`. + + +Verify Installation +-------------------- +Ensure successful installation by running the following commands: + +.. code-block:: python + + import SuomiGeoData + paituli = SuomiGeoData.Paituli() + + +Index Map +---------- +Access and save the DEM index map: + +.. code-block:: python + + dem_gdf = paituli.indexmap_dem + dem_gdf.head() + # save the map with 'True' message a output + paituli.save_indexmap_dem( + file_path=r"C:\Users\Username\Desktop\Folder\indexmap_dem.shp" + ) + +Expected output: + +.. code-block:: text + + label path geometry + ----------------------------------------------------------------------------------------------------- + 0 K3244G mml/dem2m/2008_latest/K3/K32/K3244G.tif POLYGON ((206000 6654000, 206000 6660000, 2120... + 1 K3244H mml/dem2m/2008_latest/K3/K32/K3244H.tif POLYGON ((206000 6660000, 206000 6666000, 2120... + 2 K3222E mml/dem2m/2008_latest/K3/K32/K3222E.tif POLYGON ((128000 6654000, 128000 6660000, 1340... + 3 K3222A mml/dem2m/2008_latest/K3/K32/K3222A.tif POLYGON ((116000 6654000, 116000 6660000, 1220... + 4 K3222C mml/dem2m/2008_latest/K3/K32/K3222C.tif POLYGON ((122000 6654000, 122000 6660000, 1280... + + +Labels +-------- +Retrieve the list of label names for the topographic database: + +.. code-block:: python + + tdb_labels = paituli.tdb_labels + +Expected output: + +.. code-block:: text + + ['K2344R', + 'K2334R', + 'K2343R', + 'K2443L', + 'K2443R', + ...] + + +Download by labels +------------------ +Download raster and shapefiles using label names; +outputs 'True' if all data files are successfully downloaded. + +.. code-block:: python + + # download raster files + paituli.dem_download_by_labels( + labels=['K3244G', 'X4344A'], + folder_path=r"C:\Users\Username\Desktop\Folder" + ) + # download topographic database folder + paituli.tdb_download_by_labels( + labels=['K2344R', 'J3224R'], + folder_path=r"C:\Users\Username\Desktop\Folder" + ) \ No newline at end of file diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 55ec5ef..f2d0a36 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,5 @@ sphinx_rtd_theme geopandas>=1.0.1 +requests>=2.32.3 diff --git a/pyproject.toml b/pyproject.toml index 651fc31..d670b06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,19 +10,20 @@ authors = [ { name="Debasish Pal", email="bestdebasish@gmail.com" }, ] dependencies = [ - "geopandas>=1.0.1" + "geopandas>=1.0.1", + "requests>=2.32.3" ] readme = "README.md" requires-python = ">=3.10" classifiers = [ - "Development Status :: 1 - Planning", + "Development Status :: 2 - Pre-Alpha", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Intended Audience :: Education", - "Intended Audience :: Science/Research", + "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering :: GIS", "Topic :: Scientific/Engineering :: Hydrology" ] diff --git a/requirements-github.txt b/requirements-gh-action.txt similarity index 67% rename from requirements-github.txt rename to requirements-gh-action.txt index ad03a7b..308af20 100644 --- a/requirements-github.txt +++ b/requirements-gh-action.txt @@ -1,4 +1,4 @@ -geopandas>=1.0.1 pytest pytest-cov - +geopandas>=1.0.1 +requests>=2.32.3 diff --git a/requirements-mypy.txt b/requirements-mypy.txt new file mode 100644 index 0000000..e5738a6 --- /dev/null +++ b/requirements-mypy.txt @@ -0,0 +1,2 @@ +types-requests +mypy diff --git a/tests/test_paituli.py b/tests/test_paituli.py index fb80c9b..d6cfe26 100644 --- a/tests/test_paituli.py +++ b/tests/test_paituli.py @@ -57,8 +57,36 @@ def test_is_valid_label( assert class_instance.is_valid_label_tdb('invalid_label') is False -def test_check( +def test_download_by_labels( class_instance ): - assert 1 + 1 == 2 + # test for downloading DEM + with tempfile.TemporaryDirectory() as dem_dir: + # download test + class_instance.dem_download_by_labels(['X4344A'], dem_dir) is True + # download test with customized HTTP headers + class_instance.dem_download_by_labels(['X4344A'], dem_dir, http_headers={'Host': 'www.nic.funet.fi'}) is True + # error test for invalid label + with pytest.raises(Exception) as exc_info: + class_instance.dem_download_by_labels(['ABCDE'], dem_dir) + assert exc_info.value.args[0] == 'The label "ABCDE" does not exist in the index map.' + # errot test for invalid directory + with pytest.raises(Exception) as exc_info: + class_instance.dem_download_by_labels(['X4344A'], dem_dir) + assert exc_info.value.args[0] == f'The folder path "{dem_dir}" is not a valid directory.' + + # test for downloading topographical database + with tempfile.TemporaryDirectory() as tdb_dir: + # download test + class_instance.tdb_download_by_labels(['J3224R'], tdb_dir) is True + # download test with customized HTTP headers + class_instance.tdb_download_by_labels(['J3224R'], tdb_dir, http_headers={'Host': 'www.nic.funet.fi'}) is True + # error test for invalid label + with pytest.raises(Exception) as exc_info: + class_instance.tdb_download_by_labels(['ABCDE'], tdb_dir) + assert exc_info.value.args[0] == 'The label "ABCDE" does not exist in the index map.' + # errot test for invalid directory + with pytest.raises(Exception) as exc_info: + class_instance.tdb_download_by_labels(['J3224R'], tdb_dir) + assert exc_info.value.args[0] == f'The folder path "{tdb_dir}" is not a valid directory.'