diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b1c0f084..82106465 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM qgis/qgis:3.34.7 +FROM qgis/qgis:ltr # [Optional] Uncomment this section to install additional OS packages. RUN apt update -y && apt install -y pandoc zip diff --git a/.github/workflows/AequilibraeUpdates.yml b/.github/workflows/AequilibraeUpdates.yml index 55578919..62994bd2 100644 --- a/.github/workflows/AequilibraeUpdates.yml +++ b/.github/workflows/AequilibraeUpdates.yml @@ -10,7 +10,7 @@ jobs: strategy: max-parallel: 4 matrix: - container: ['qgis/qgis:latest', 'qgis/qgis:3.34.7'] + container: ['qgis/qgis:latest', 'qgis/qgis:ltr'] container: image: ${{ matrix.container }} steps: @@ -19,19 +19,18 @@ jobs: - name: Install dependencies run: | - python3 -m venv /tmp/venv_dev --system-site-package - . /tmp/venv_dev/bin/activate - python3 -m pip install setuptools - python3 -m pip install git+https://github.com/AequilibraE/aequilibrae.git@develop + python3 -m venv /tmp/.venv --system-site-packages + . /tmp/.venv/bin/activate + python3 -m pip install -U pip setuptools uv + python3 -m uv pip install "git+https://github.com/AequilibraE/aequilibrae.git@develop" touch ./qaequilibrae/packages/aequilibrae_version.txt - python3 -m pip install -r test/requirements-test.txt python3 ./ci/dependency_installation.py + python3 -m uv pip install -r test/requirements-test.txt --constraint ./ci/constraints.txt export PYTHONPATH=$(pwd)/qaequilibrae/packages:$PYTHONPATH echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - name: Run tests run: | - . /tmp/venv_dev/bin/activate export QT_QPA_PLATFORM=offscreen pwd - python3 -m pytest --cov-report term --cov-config=.coveragerc --cov=qaequilibrae test -v + python3 -m pytest --cov-report term --cov-config=.coveragerc --cov=qaequilibrae test diff --git a/.github/workflows/DevWorkflow.yml b/.github/workflows/DevWorkflow.yml index 63a45950..0ce39929 100644 --- a/.github/workflows/DevWorkflow.yml +++ b/.github/workflows/DevWorkflow.yml @@ -10,7 +10,7 @@ jobs: strategy: max-parallel: 4 matrix: - container: [ 'qgis/qgis:latest', 'qgis/qgis:3.34.7'] + container: [ 'qgis/qgis:latest', 'qgis/qgis:ltr'] container: image: ${{ matrix.container }} steps: @@ -18,17 +18,17 @@ jobs: - name: Install dependencies run: | - python3 -m venv /tmp/venv_test --system-site-package - . /tmp/venv_test/bin/activate - python3 -m pip install setuptools - python3 -m pip install -r test/requirements-test.txt + python3 -m venv /tmp/.venv --system-site-packages + . /tmp/.venv/bin/activate + python3 -m pip install -U pip setuptools uv python3 ./ci/dependency_installation.py + python3 -m uv pip install -r test/requirements-test.txt --constraint ./ci/constraints.txt export PYTHONPATH=$(pwd)/qaequilibrae/packages:$PYTHONPATH echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - name: Run tests run: | - . /tmp/venv_test/bin/activate + . /tmp/.venv/bin/activate export QT_QPA_PLATFORM=offscreen pwd python3 -m pytest --cov-config=.coveragerc --cov-report term-missing --cov=qaequilibrae test diff --git a/.github/workflows/WindowsRelease.yml b/.github/workflows/WindowsRelease.yml index d8435064..f7c4496e 100644 --- a/.github/workflows/WindowsRelease.yml +++ b/.github/workflows/WindowsRelease.yml @@ -12,7 +12,8 @@ jobs: runs-on: windows-latest strategy: matrix: - version: [qgis-ltr --version=3.34.6, qgis-ltr, qgis] + # we test the minimum LTR version required, the newest LTR, and the latest. + version: [qgis-ltr --version=3.34.10, qgis-ltr, qgis] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c7d1eadf..bd9aac1d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -20,8 +20,7 @@ jobs: python-version: 3.9 - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r qaequilibrae/requirements.txt + python -m pip install -U pip pip install -r docs/requirements-docs.txt - name: Build documentation diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e658168..88db72de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,8 +25,7 @@ jobs: run: | sudo apt-get update sudo apt install qttools5-dev-tools - python -m pip install --upgrade pip - python -m pip install qgis-plugin-ci==2.7.2 + python -m pip install -U pip qgis-plugin-ci - name: Compile translations if they exist run: | diff --git a/.github/workflows/translation.yml b/.github/workflows/translation.yml index 2b752b5c..640b2a90 100644 --- a/.github/workflows/translation.yml +++ b/.github/workflows/translation.yml @@ -26,8 +26,7 @@ jobs: run: | sudo apt-get update sudo apt install qttools5-dev-tools - python -m pip install --upgrade pip - python -m pip install qgis-plugin-ci==2.7.2 + python -m pip install -U pip qgis-plugin-ci - name: Create translatable files run: | diff --git a/ci/constraints.txt b/ci/constraints.txt new file mode 100644 index 00000000..ebd6b2b7 --- /dev/null +++ b/ci/constraints.txt @@ -0,0 +1 @@ +numpy<2.0 \ No newline at end of file diff --git a/docs/source/images/add_connectors_to_project.png b/docs/source/images/add_connectors_to_project.png index b6564547..dfaaef80 100644 Binary files a/docs/source/images/add_connectors_to_project.png and b/docs/source/images/add_connectors_to_project.png differ diff --git a/docs/source/images/gtfs_1.png b/docs/source/images/gtfs_1.png index 75069bd8..34de8526 100644 Binary files a/docs/source/images/gtfs_1.png and b/docs/source/images/gtfs_1.png differ diff --git a/docs/source/images/gtfs_4.png b/docs/source/images/gtfs_4.png index 430118a6..c67f04fe 100644 Binary files a/docs/source/images/gtfs_4.png and b/docs/source/images/gtfs_4.png differ diff --git a/docs/source/images/gtfs_5.png b/docs/source/images/gtfs_5.png index 7ab8c166..f74b72f5 100644 Binary files a/docs/source/images/gtfs_5.png and b/docs/source/images/gtfs_5.png differ diff --git a/docs/source/images/tripdistribution-ipf-1.png b/docs/source/images/tripdistribution-ipf-1.png index 661b4bc2..b5761d0f 100644 Binary files a/docs/source/images/tripdistribution-ipf-1.png and b/docs/source/images/tripdistribution-ipf-1.png differ diff --git a/docs/source/images/tripdistribution-ipf-2.png b/docs/source/images/tripdistribution-ipf-2.png index e85b0c85..ce7ce126 100644 Binary files a/docs/source/images/tripdistribution-ipf-2.png and b/docs/source/images/tripdistribution-ipf-2.png differ diff --git a/docs/source/images/tripdistribution-ipf-4.png b/docs/source/images/tripdistribution-ipf-4.png index 72fedee9..81f1b332 100644 Binary files a/docs/source/images/tripdistribution-ipf-4.png and b/docs/source/images/tripdistribution-ipf-4.png differ diff --git a/docs/source/menus_in_detail/build_model.rst b/docs/source/menus_in_detail/build_model.rst index 2f0d757c..4cd5f5e9 100644 --- a/docs/source/menus_in_detail/build_model.rst +++ b/docs/source/menus_in_detail/build_model.rst @@ -181,6 +181,11 @@ The *GUI* for this procedure is fairly straightforward, as shown below. :align: center :alt: Adding connectors +When creating centroids from zone centers, one can choose to limit the connector +to the zone or not. Plase notice if one choose to limit the connector creation to a +zone that has fewer nodes connected to links of the required types than the number of connectors will result +in fewer connectors being created than desired. + One would notice that nowhere in the *GUI* one can indicate which modes they want to see the network connected for or how to control how many connectors per mode will be created. Although it could be implemented, such a solution would diff --git a/docs/source/menus_in_detail/public_transport.rst b/docs/source/menus_in_detail/public_transport.rst index b0b03279..22be0903 100644 --- a/docs/source/menus_in_detail/public_transport.rst +++ b/docs/source/menus_in_detail/public_transport.rst @@ -19,7 +19,8 @@ Import GTFS To import a GTFS feed, click **Public transport > Import GTFS**. A new window with the importer will open. If it is the first time you are creating a GTFS feed for your project, it may take a little while -to create the public transport database in the project folder. In the GTFS importer window, you can click on +to create the public transport database in the project folder, and your QGIS screen might not be responsive +until the database is created in the project folder. In the GTFS importer window, you can click on *Add Feed* and point to the location in your machine where the GTFS data is. .. image:: ../images/gtfs_1.png @@ -41,7 +42,9 @@ and you will return to the GTFS importer screen. Notice that the feed information is now available at the *Feeds to import* table view. The first time you create a GTFS feed, the only option available is **Create new route system**, so you don't have to click on it. -Then, you can effectively import your GTFS feed to your project by clicking on **Execute Importer**. +If you want to map-match the existing transit routes, you can select **Allow map-match**. +Then, you can import your GTFS feed to your project by clicking on **Execute Importer**. + A window with a progress bar will open and once it is finished, you can check out the GTFS feed data you just imported in your project folder. diff --git a/docs/source/menus_in_detail/tripdistribution.rst b/docs/source/menus_in_detail/tripdistribution.rst index 6413a85f..f2e4f45a 100644 --- a/docs/source/menus_in_detail/tripdistribution.rst +++ b/docs/source/menus_in_detail/tripdistribution.rst @@ -19,20 +19,20 @@ presented in one of the subsections below. Iterative Proportional Fitting (IPF) ------------------------------------ -It is possible to balance the production/attraction vectors using IPF. There are two different -ways to load a vector's data: loading an \*.AED file or importing from an open layer. +It is possible to balance the production/attraction vectors using IPF. There are three different +ways to load a vector's data: loading a \*.CSV or \*.parquet file or importing from an open layer. We click on the Iterative Proportional Fitting option to open the menu. -Loading the vector from an \*.AED file is straightforward. Just select the *AequilibraE Data* option -in the menu, and click *Load*, pointing to the location of the vector in your machine. +Loading the vector from a \*.CSV or \*.parquet file is quite the same. Select your preferred option +in the menu, and click *Load*, pointing to the location of the vector file in your machine. .. image:: ../images/tripdistribution-ipf-1.png :width: 513 :align: center :alt: ipf_1 -Otherwise, case you are loading from an open layer, just click *Import from layer*, +Case you are loading from an open layer, just click *Import from layer*, point the available data layer, and the name of its index column. You can choose between *Use data* or *Save and use*. Case you choose to save, the vector will be saved in a temporary QGIS folder. @@ -48,7 +48,9 @@ After the vector is properly loaded, it will appear in the *Load datasets* tab. :align: center :alt: ipf_3 -You can now select the production/attraction (origin/destination) vectors. +You can now select the production/attraction (origin/destination) vectors. If your data comes +from a table/layer opened in QGIS, you'll notice that the *Index* collapsible list is deactivated because +the data index was selected when loading the data. .. image:: ../images/tripdistribution-ipf-4.png :width: 513 diff --git a/qaequilibrae/aequilibrae_version.txt b/qaequilibrae/aequilibrae_version.txt index b0e3367d..b7b451f6 100644 --- a/qaequilibrae/aequilibrae_version.txt +++ b/qaequilibrae/aequilibrae_version.txt @@ -1 +1 @@ -aequilibrae==1.0.1 \ No newline at end of file +aequilibrae==1.1.3 \ No newline at end of file diff --git a/qaequilibrae/download_extra_packages_class.py b/qaequilibrae/download_extra_packages_class.py index ef9e8119..c15ce683 100644 --- a/qaequilibrae/download_extra_packages_class.py +++ b/qaequilibrae/download_extra_packages_class.py @@ -2,6 +2,7 @@ import shutil import subprocess import sys +from importlib.util import find_spec from os.path import join, isdir from pathlib import Path @@ -39,6 +40,10 @@ def __init__(self): def install(self): reps = [] + command = f'"{self.find_python()}" -m pip install uv' + _ = self.execute(command) + print(command) + for file in self.dependency_files: flag = self.target_folder / file.name if flag.exists(): @@ -59,9 +64,10 @@ def install(self): def install_package(self, package): Path(self.target_folder).mkdir(parents=True, exist_ok=True) - install_command = f'-m pip install {package} -t "{self.target_folder}"' - if "ortools" in package.lower(): - install_command += " --no-deps" + spec = find_spec("uv") + is_uv = "" if spec is None else "uv" + + install_command = f'-m {is_uv} pip install {package} --target "{self.target_folder}"' command = f'"{self.find_python()}" {install_command}' print(command) @@ -96,6 +102,10 @@ def execute(self, command): return lines def find_python(self): + # Check if we're inside a virtual environment + if sys.prefix != sys.base_prefix: + return "python3" + sys_exe = Path(sys.executable) if sys.platform == "linux" or sys.platform == "linux2": # Unlike other platforms, linux uses the system python, lets see if we can guess it @@ -147,8 +157,8 @@ def clean_packages(self): if __name__ == "__main__": result = DownloadAll().install() - output = ("".join([str(x).upper() for x in result])) - + output = "".join([str(x).upper() for x in result]) + print(output) assert "ERROR" not in output diff --git a/qaequilibrae/message.py b/qaequilibrae/message.py index c299552d..883b7df7 100644 --- a/qaequilibrae/message.py +++ b/qaequilibrae/message.py @@ -29,7 +29,7 @@ def fourth_message(self): @property def messsage_five(self): a = self.tr("QAequilibraE requires Python 3.12.") - b = self.tr("Please install QGIS version 3.34.6+, 3.36.2+ or 3.38.0+ to make it work.") + b = self.tr("Please install QGIS version 3.34.10+ or 3.38.2+ to make it work.") return f"{a}\r\n{b}" def tr(self, text): diff --git a/qaequilibrae/metadata.txt b/qaequilibrae/metadata.txt index ac599072..1a13a4fd 100644 --- a/qaequilibrae/metadata.txt +++ b/qaequilibrae/metadata.txt @@ -1,10 +1,10 @@ [general] name=qaequilibrae -qgisMinimumVersion=3.34.6 +qgisMinimumVersion=3.34.10 qgisMaximumVersion=3.99 description=Transportation modeling toolbox for QGIS about=QAequilibraE is the GUI for AequilibraE, a transportation modeling package designed to be an open-source alternative to traditional commercial packages. It is a comprehensive set of tools for modeling and visualization, including incredibly fast equilibrium traffic assignment, synthetic gravity models, network editing, and GTFS importer. http://www.aequilibrae.com/. -version=1.0.3 +version=1.1.3 author=Pedro Camargo email=pedro@outerloop.io repository= https://github.com/AequilibraE/QAequilibraE diff --git a/qaequilibrae/modules/common_tools/__init__.py b/qaequilibrae/modules/common_tools/__init__.py index 553dc927..915b67b8 100644 --- a/qaequilibrae/modules/common_tools/__init__.py +++ b/qaequilibrae/modules/common_tools/__init__.py @@ -1,15 +1,14 @@ from .auxiliary_functions import * from .get_output_file_name import GetOutputFileName from .get_output_file_name import GetOutputFolderName -from .link_query_model import LinkQueryModel from .load_graph_layer_setting_dialog import LoadGraphLayerSettingDialog from .numpy_model import NumpyModel from .pandas_model import PandasModel from .database_model import DatabaseModel from .parameters_dialog import ParameterDialog from .report_dialog import ReportDialog -from .about_dialog import AboutDialog from .all_layers_from_toc import all_layers_from_toc from .log_dialog import LogDialog from .data_layer_from_dataframe import layer_from_dataframe from .table_field_lister import find_table_fields +from .geodataframe_from_data_layer import geodataframe_from_layer diff --git a/qaequilibrae/modules/common_tools/about_dialog.py b/qaequilibrae/modules/common_tools/about_dialog.py deleted file mode 100644 index 09b4a543..00000000 --- a/qaequilibrae/modules/common_tools/about_dialog.py +++ /dev/null @@ -1,90 +0,0 @@ -import logging -import os -from os.path import dirname, abspath -import requests -import yaml - -from qgis.PyQt import QtWidgets, uic -from .auxiliary_functions import standard_path - -try: - from aequilibrae.paths import release_name, release_version -except Exception as e: - logger = logging.getLogger("AequilibraEGUI") - logger.error(e.args) - release_name = "No Binaries available" - release_version = "No Binaries available" - -FORM_CLASS, _ = uic.loadUiType(os.path.join(os.path.dirname(__file__), "forms/ui_about.ui")) - - -class AboutDialog(QtWidgets.QDialog, FORM_CLASS): - def __init__(self, iface): - QtWidgets.QDialog.__init__(self) - self.iface = iface - self.setupUi(self) - self.path = standard_path() - - self.but_close.clicked.connect(self.exit_procedure) - - repository = "https://github.com/AequilibraE/QAequilibraE" - - d = dirname(dirname(dirname(abspath(__file__)))) - with open(os.path.join(d, "meta.yaml"), "r") as yml: - par = yaml.safe_load(yml) - - my_file = os.path.join(d, "metadata.txt") - b = "?" - with open(my_file, "r") as a: - for line in a.readlines(): - if line[:18] == "qgisMinimumVersion": - min_qgis = line[19:-1] if line[-1] == "\n" else line[19:] - continue - if line[:7] == "version": - b = line[8:-1] if line[-1] == "\n" else line[8:] - break - - developers = self.get_contributors(repository) - - self.all_items = [] - self.all_items.append([self.tr("AequilibraE Version name"), release_name]) - self.all_items.append([self.tr("AequilibraE Version number"), release_version]) - self.all_items.append([self.tr("GUI version"), b]) - self.all_items.append([self.tr("GUI Repository"), repository]) - self.all_items.append([self.tr("Minimum QGIS"), min_qgis]) - self.all_items.append([self.tr("Developers"), developers]) - self.all_items.append([self.tr("Sponsors"), par["sponsors"]]) - - self.assemble() - - def assemble(self): - titles = [] - row_count = 0 - - for r, t in self.all_items: - titles.append(r) - self.about_table.insertRow(row_count) - if isinstance(t, list): - lv = QtWidgets.QListWidget() - lv.addItems(t) - self.about_table.setCellWidget(row_count, 0, lv) - self.about_table.setRowHeight(row_count, len(t) * self.about_table.rowHeight(row_count)) - else: - self.about_table.setItem(row_count, 0, QtWidgets.QTableWidgetItem(self.tr(t))) - - row_count += 1 - self.about_table.setVerticalHeaderLabels(titles) - - def get_contributors(self, repo_url): - repo_name = repo_url.split("/")[-1] - owner_name = repo_url.split("/")[-2] - api_url = f"https://api.github.com/repos/{owner_name}/{repo_name}/contributors" - response = requests.get(api_url) - if response.status_code == 200: - contributors = [user["login"] for user in response.json()] - return [x for x in contributors if "[bot]" not in x] - else: - return None - - def exit_procedure(self): - self.close() diff --git a/qaequilibrae/modules/common_tools/forms/ui_about.ui b/qaequilibrae/modules/common_tools/forms/ui_about.ui deleted file mode 100644 index b86bfe28..00000000 --- a/qaequilibrae/modules/common_tools/forms/ui_about.ui +++ /dev/null @@ -1,52 +0,0 @@ - - - Dialog - - - - 0 - 0 - 510 - 600 - - - - AequilibraE - About - - - - - - QAbstractItemView::NoEditTriggers - - - false - - - false - - - false - - - true - - - - Info - - - - - - - - Close - - - - - - - - diff --git a/qaequilibrae/modules/common_tools/geodataframe_from_data_layer.py b/qaequilibrae/modules/common_tools/geodataframe_from_data_layer.py new file mode 100644 index 00000000..58d6d8b4 --- /dev/null +++ b/qaequilibrae/modules/common_tools/geodataframe_from_data_layer.py @@ -0,0 +1,24 @@ +import pandas as pd +import geopandas as gpd + +from qgis.core import QgsVectorLayer + + +def geodataframe_from_layer(layer: QgsVectorLayer) -> gpd.GeoDataFrame: + """Creates a gpd.GeoDataFrame from a data layer.""" + + data = [] + geoms = [] + + for feat in layer.getFeatures(): + data.append(feat.attributes()) + geoms.append(feat.geometry().asWkt()) + + columns = [field.name() for field in layer.fields()] + + df = pd.DataFrame(data, columns=columns) + + df = gpd.GeoDataFrame(df, geometry=gpd.GeoSeries.from_wkt(geoms), crs=layer.crs().authid()) + df["geoms"] = df["geometry"].to_wkb() + + return df diff --git a/qaequilibrae/modules/common_tools/link_query_model.py b/qaequilibrae/modules/common_tools/link_query_model.py deleted file mode 100644 index 7e0aa275..00000000 --- a/qaequilibrae/modules/common_tools/link_query_model.py +++ /dev/null @@ -1,28 +0,0 @@ -from PyQt5.QtCore import QAbstractTableModel -from qgis.PyQt.QtCore import Qt - - -# Largely adapted from http://stackoverflow.com/questions/28033633/using-large-record-set-with-qtableview-and-qabstracttablemodel-retrieve-visib -# Answer by Phil Cooper - - -class LinkQueryModel(QAbstractTableModel): - def __init__(self, narray, headerdata, parent=None, *args): - QAbstractTableModel.__init__(self, parent, *args) - self._array = narray - self.headerdata = headerdata - - def rowCount(self, parent): - return len(self._array) - - def columnCount(self, parent): - return len(self._array[0]) - - def data(self, index, role): - if index.isValid(): - if role == Qt.DisplayRole: - return str(self._array[index.row()][index.column()]) - - def headerData(self, col, orientation, role=Qt.DisplayRole): - if role == Qt.DisplayRole and orientation == Qt.Horizontal: - return self.headerdata[col] diff --git a/qaequilibrae/modules/distribution_procedures/apply_gravity_procedure.py b/qaequilibrae/modules/distribution_procedures/apply_gravity_procedure.py index 2461c557..c7c134eb 100644 --- a/qaequilibrae/modules/distribution_procedures/apply_gravity_procedure.py +++ b/qaequilibrae/modules/distribution_procedures/apply_gravity_procedure.py @@ -1,9 +1,11 @@ from aequilibrae.distribution import GravityApplication -from aequilibrae.utils.worker_thread import WorkerThread -from qgis.PyQt.QtCore import * +from aequilibrae.utils.interface.worker_thread import WorkerThread +from PyQt5.QtCore import pyqtSignal class ApplyGravityProcedure(WorkerThread): + signal = pyqtSignal(object) + def __init__(self, parentThread, **kwargs): WorkerThread.__init__(self, parentThread) self.gravity = GravityApplication(**kwargs) @@ -15,4 +17,4 @@ def doWork(self): self.gravity.apply() self.output = self.gravity.output self.report = self.gravity.report - self.jobFinished.emit("apply_gravity") + self.signal.emit(["finished"]) diff --git a/qaequilibrae/modules/distribution_procedures/calibrate_gravity_procedure.py b/qaequilibrae/modules/distribution_procedures/calibrate_gravity_procedure.py index 7a2b3601..fd8258da 100644 --- a/qaequilibrae/modules/distribution_procedures/calibrate_gravity_procedure.py +++ b/qaequilibrae/modules/distribution_procedures/calibrate_gravity_procedure.py @@ -1,9 +1,11 @@ from aequilibrae.distribution import GravityCalibration -from aequilibrae.utils.worker_thread import WorkerThread -from qgis.PyQt.QtCore import * +from aequilibrae.utils.interface.worker_thread import WorkerThread +from PyQt5.QtCore import pyqtSignal class CalibrateGravityProcedure(WorkerThread): + signal = pyqtSignal(object) + def __init__(self, parentThread, **kwargs): WorkerThread.__init__(self, parentThread) self.gravity = GravityCalibration(**kwargs) @@ -15,4 +17,4 @@ def doWork(self): self.gravity.calibrate() self.report = self.gravity.report self.model = self.gravity.model - self.jobFinished.emit("calibrate") + self.signal.emit(["finished"]) diff --git a/qaequilibrae/modules/distribution_procedures/distribution_models_dialog.py b/qaequilibrae/modules/distribution_procedures/distribution_models_dialog.py index 971898f7..63321167 100644 --- a/qaequilibrae/modules/distribution_procedures/distribution_models_dialog.py +++ b/qaequilibrae/modules/distribution_procedures/distribution_models_dialog.py @@ -1,33 +1,26 @@ -import importlib.util as iutil import os +import numpy as np +import pandas as pd from collections import OrderedDict from functools import partial -from os.path import join -import numpy as np +from aequilibrae.context import get_logger +from aequilibrae.matrix import AequilibraeMatrix from aequilibrae.distribution import SyntheticGravityModel from aequilibrae.distribution.synthetic_gravity_model import valid_functions -from aequilibrae.matrix import AequilibraeData, AequilibraeMatrix -from qgis.PyQt.QtWidgets import QAbstractItemView -from qaequilibrae.modules.matrix_procedures.matrix_lister import list_matrices -from qaequilibrae.modules.common_tools import PandasModel import qgis -from aequilibrae.context import get_logger from qgis.PyQt import QtWidgets, uic from qgis.PyQt.QtWidgets import QTableWidgetItem, QComboBox, QDoubleSpinBox, QAbstractItemView +from qaequilibrae.modules.common_tools.auxiliary_functions import standard_path +from qaequilibrae.modules.common_tools import PandasModel, ReportDialog, GetOutputFileName +from qaequilibrae.modules.distribution_procedures.ipf_procedure import IpfProcedure from qaequilibrae.modules.distribution_procedures.apply_gravity_procedure import ApplyGravityProcedure from qaequilibrae.modules.distribution_procedures.calibrate_gravity_procedure import CalibrateGravityProcedure -from qaequilibrae.modules.distribution_procedures.ipf_procedure import IpfProcedure -from qaequilibrae.modules.common_tools import GetOutputFileName -from qaequilibrae.modules.common_tools import ReportDialog -from qaequilibrae.modules.common_tools.auxiliary_functions import standard_path +from qaequilibrae.modules.matrix_procedures.matrix_lister import list_matrices from qaequilibrae.modules.matrix_procedures import LoadDatasetDialog, DisplayAequilibraEFormatsDialog FORM_CLASS, _ = uic.loadUiType(os.path.join(os.path.dirname(__file__), "forms/ui_distribution.ui")) -spec = iutil.find_spec("openmatrix") -has_omx = spec is not None - # TODO: Implement consideration of the "empty as zeros" for ALL distrbution models Should force inputs for trip distribution to be of FLOAT type @@ -64,11 +57,14 @@ def __init__(self, qgs_proj, mode=None): self.but_load_data.clicked.connect(self.load_datasets) self.but_load_model.clicked.connect(self.load_model) - self.cob_prod_data.currentIndexChanged.connect( - partial(self.change_vector_field, self.cob_prod_data, self.cob_prod_field, "data") + self.cob_data.currentIndexChanged.connect( + partial(self.change_vector_field, self.cob_data, self.cob_index, "data") + ) + self.cob_data.currentIndexChanged.connect( + partial(self.change_vector_field, self.cob_data, self.cob_prod_field, "data") ) - self.cob_atra_data.currentIndexChanged.connect( - partial(self.change_vector_field, self.cob_atra_data, self.cob_atra_field, "data") + self.cob_data.currentIndexChanged.connect( + partial(self.change_vector_field, self.cob_data, self.cob_atra_field, "data") ) self.cob_imped_mat.currentIndexChanged.connect( @@ -205,15 +201,23 @@ def load_datasets(self): dlg2 = LoadDatasetDialog(self.iface) dlg2.show() dlg2.exec_() - if isinstance(dlg2.dataset, AequilibraeData): - dataset_name = dlg2.dataset.file_path + if isinstance(dlg2.dataset, pd.DataFrame): + dataset_name = dlg2.output_name if dataset_name is not None: data_name = os.path.splitext(os.path.basename(dataset_name))[0] data_name = self.find_non_conflicting_name(data_name, self.datasets) self.datasets[data_name] = dlg2.dataset self.add_to_table(self.datasets, self.table_datasets) - self.load_comboboxes(self.datasets.keys(), self.cob_prod_data) - self.load_comboboxes(self.datasets.keys(), self.cob_atra_data) + self.load_comboboxes(self.datasets.keys(), self.cob_data) + # To use a QGIS layer as input, we deactivate part of the widgets + self._has_idx = True if dlg2.radio_layer.isChecked() else False + if self._has_idx: + self.cob_index.clear() + self.cob_index.setEnabled(False) + else: + qgis.utils.iface.messageBar().pushMessage( + "Warning: ", self.tr("You need to load a dataset to proceed"), level=1, duration=10 + ) def load_model(self): file_name = self.browse_outfile("mod") @@ -222,7 +226,7 @@ def load_model(self): self.update_model_parameters() except Exception as e: qgis.utils.iface.messageBar().pushMessage( - "Error", self.tr("Could not load model. {}").format(e.args), level=3, duration=10 + "Error: ", self.tr("Could not load model. {}").format(e.args), level=2, duration=10 ) def change_vector_field(self, cob_orig, cob_dest, dt): @@ -230,15 +234,15 @@ def change_vector_field(self, cob_orig, cob_dest, dt): d = str(cob_orig.currentText()) if len(d) > 0: if dt == "data": - for f in self.datasets[d].fields: - if np.issubdtype(self.datasets[d].data[f].dtype, np.integer) or np.issubdtype( - self.datasets[d].data[f].dtype, np.float64 + for f in self.datasets[d].columns: + if np.issubdtype(self.datasets[d][f].dtype, np.integer) or np.issubdtype( + self.datasets[d][f].dtype, np.float64 ): cob_dest.addItem(f) else: file_name = self.matrices.at[cob_orig.currentIndex(), "file_name"] mat = AequilibraeMatrix() - mat.load(join(self.project.matrices.fldr, file_name)) + mat.load(os.path.join(self.project.matrices.fldr, file_name)) cob_dest.addItems(mat.names) def load_comboboxes(self, list_to_load, data_cob): @@ -265,21 +269,17 @@ def add_to_table(self, dictio, table): for i, data_name in enumerate(dictio.keys()): table.setItem(i, 0, QTableWidgetItem(data_name)) - if isinstance(dictio[data_name], AequilibraeData): - table.setItem(i, 1, QTableWidgetItem(str(dictio[data_name].num_fields))) + if isinstance(dictio[data_name], pd.DataFrame): + table.setItem(i, 1, QTableWidgetItem(str(dictio[data_name].shape[1] - 1))) else: table.setItem(i, 1, QTableWidgetItem(str(dictio[data_name].cores))) def browse_outfile(self, file_type): file_types = { - "aed": ["AequilibraE dataset", ["Aequilibrae dataset(*.aed)"], ".aed"], "mod": ["Model file", ["Model file(*.mod)"], ".mod"], "aem": ["Matrix", ["Aequilibrae matrix(*.aem)"], ".aem"], } - if has_omx: - file_types["aem"] = ["Matrix", ["Open Matrix(*.omx)", "Aequilibrae matrix(*.aem)"], ".omx"] - ft = file_types[file_type] file_chosen, _ = GetOutputFileName(self, ft[0], ft[1], ft[2], self.path) return file_chosen @@ -290,19 +290,20 @@ def add_job_to_queue(self): if self.job != "ipf": imped_name = self.matrices.at[self.cob_imped_mat.currentIndex(), "file_name"] imped_matrix = AequilibraeMatrix() - imped_matrix.load(join(self.project.matrices.fldr, imped_name)) + imped_matrix.load(os.path.join(self.project.matrices.fldr, imped_name)) imped_matrix.computational_view([self.cob_imped_field.currentText()]) if self.job != "apply": seed_name = self.matrices.at[self.cob_seed_mat.currentIndex(), "file_name"] seed_matrix = AequilibraeMatrix() - seed_matrix.load(join(self.project.matrices.fldr, seed_name)) + seed_matrix.load(os.path.join(self.project.matrices.fldr, seed_name)) seed_matrix.computational_view([self.cob_seed_field.currentText()]) if self.job != "calibrate": - prod_vec = self.datasets[self.cob_prod_data.currentText()] + vec = self.datasets[self.cob_data.currentText()] + if not self._has_idx: + vec.set_index(self.cob_index.currentText(), inplace=True) prod_field = self.cob_prod_field.currentText() - atra_vec = self.datasets[self.cob_atra_data.currentText()] atra_field = self.cob_atra_field.currentText() if self.job == "ipf": @@ -310,9 +311,8 @@ def add_job_to_queue(self): if self.out_name is not None: args = { "matrix": seed_matrix, - "rows": prod_vec, + "vectors": vec, "row_field": prod_field, - "columns": atra_vec, "column_field": atra_field, "nan_as_zero": self.chb_empty_as_zero.isChecked(), } @@ -328,11 +328,10 @@ def add_job_to_queue(self): self.model.beta = float(self.table_model.cellWidget(i, 1).value()) args = { + "model": self.model, "impedance": imped_matrix, - "rows": prod_vec, + "vectors": vec, "row_field": prod_field, - "model": self.model, - "columns": atra_vec, "column_field": atra_field, "output": self.out_name, "nan_as_zero": self.chb_empty_as_zero.isChecked(), @@ -364,7 +363,7 @@ def add_job_to_queue(self): return self.add_job_to_list(worker_thread, self.out_name) else: - qgis.utils.iface.messageBar().pushMessage(self.tr("Procedure error: "), self.error, level=3) + qgis.utils.iface.messageBar().pushMessage(self.tr("Procedure error: "), self.error, level=3, duration=10) def add_job_to_list(self, job, out_name): self.job_queue[out_name] = job @@ -413,24 +412,25 @@ def check_data(self): return True def run_thread(self): - self.worker_thread.jobFinished.connect(self.job_finished_from_thread) - self.worker_thread.doWork() + self.worker_thread.signal.connect(self.signal_handler) + self.worker_thread.start() self.show() - def job_finished_from_thread(self, success): + def signal_handler(self, val): error = self.worker_thread.error if error is not None: - qgis.utils.iface.messageBar().pushMessage(self.tr("Procedure error: "), error.args[0], level=3) + qgis.utils.iface.messageBar().pushMessage(self.tr("Procedure error: "), error.args[0], level=2, duration=10) self.report.extend(self.worker_thread.report) - if success == "calibrate": - self.worker_thread.model.save(self.outfile) - - if success in ["apply_gravity", "finishedIPF"]: - self.worker_thread.output.export(self.outfile) + if val[0] == "finished": + if self.job == "calibrate": + self.worker_thread.model.save(self.outfile) + if self.job in ["apply", "ipf"]: + self.worker_thread.output.export(self.outfile) self.exit_procedure() def exit_procedure(self): + self.close() if self.report is not None: dlg2 = ReportDialog(self.iface, self.report) dlg2.show() diff --git a/qaequilibrae/modules/distribution_procedures/forms/ui_distribution.ui b/qaequilibrae/modules/distribution_procedures/forms/ui_distribution.ui index 6d7f24b8..7b8526a9 100644 --- a/qaequilibrae/modules/distribution_procedures/forms/ui_distribution.ui +++ b/qaequilibrae/modules/distribution_procedures/forms/ui_distribution.ui @@ -6,7 +6,7 @@ 0 0 - 511 + 509 334 @@ -115,18 +115,18 @@ - 10 - 170 + 190 + 160 291 30 - + - 9 - 111 + 20 + 170 57 17 @@ -138,38 +138,38 @@ - 10 - 76 + 190 + 120 291 30 - + - 9 + 190 40 291 30 - + - 9 - 134 + 190 + 80 291 30 - + - 12 - 10 + 20 + 130 64 17 @@ -178,6 +178,32 @@ Production + + + + 20 + 50 + 47 + 13 + + + + Vector + + + + + + 20 + 90 + 47 + 13 + + + + Index + + diff --git a/qaequilibrae/modules/distribution_procedures/forms/ui_gravity_parameters.ui b/qaequilibrae/modules/distribution_procedures/forms/ui_gravity_parameters.ui deleted file mode 100644 index ac1ef63d..00000000 --- a/qaequilibrae/modules/distribution_procedures/forms/ui_gravity_parameters.ui +++ /dev/null @@ -1,81 +0,0 @@ - - - gravity_parameters - - - - 0 - 0 - 308 - 85 - - - - AequilibraE - Gravity Parameters - - - - - 20 - 20 - 46 - 13 - - - - Alpha - - - - - - 20 - 54 - 46 - 13 - - - - Beta - - - - - - 60 - 15 - 121 - 25 - - - - - - - - - - 60 - 50 - 121 - 25 - - - - - - - 210 - 15 - 75 - 60 - - - - Done - - - - - - diff --git a/qaequilibrae/modules/distribution_procedures/ipf_procedure.py b/qaequilibrae/modules/distribution_procedures/ipf_procedure.py index 78ce6ecb..bd186fe2 100644 --- a/qaequilibrae/modules/distribution_procedures/ipf_procedure.py +++ b/qaequilibrae/modules/distribution_procedures/ipf_procedure.py @@ -1,9 +1,11 @@ from aequilibrae.distribution import Ipf - -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread +from PyQt5.QtCore import pyqtSignal class IpfProcedure(WorkerThread): + signal = pyqtSignal(object) + def __init__(self, parentThread, **kwargs): WorkerThread.__init__(self, parentThread) self.ipf = Ipf(**kwargs) @@ -15,4 +17,4 @@ def doWork(self): self.ipf.fit() self.report = self.ipf.report self.output = self.ipf.output - self.jobFinished.emit("finishedIPF") + self.signal.emit(["finished"]) diff --git a/qaequilibrae/modules/forms/ui_MatrixViewer.ui b/qaequilibrae/modules/forms/ui_MatrixViewer.ui deleted file mode 100644 index 799c5066..00000000 --- a/qaequilibrae/modules/forms/ui_MatrixViewer.ui +++ /dev/null @@ -1,269 +0,0 @@ - - - MatrixViewer - - - Qt::ApplicationModal - - - true - - - - 0 - 0 - 817 - 600 - - - - MainWindow - - - - - - 10 - 30 - 231 - 22 - - - - - - - 460 - 10 - 341 - 51 - - - - - 10 - - - - Marginals - - - - - 20 - 20 - 255 - 25 - - - - - - - None - - - true - - - - - - - Sum - - - true - - - - - - - Max - - - true - - - - - - - Min - - - true - - - - - - - - - - 10 - 10 - 81 - 16 - - - - - 10 - - - - Matrix cores - - - - - - 250 - 30 - 191 - 22 - - - - - - - 250 - 10 - 141 - 16 - - - - - 10 - - - - Matrix indices - - - - - - 680 - 551 - 131 - 23 - - - - - 10 - - - - Load new matrix - - - - - - 540 - 551 - 131 - 23 - - - - - 10 - - - - Close - - - - - - 240 - 534 - 281 - 41 - - - - Matrix type - - - - - 6 - 22 - 91 - 17 - - - - CSV Matrix - - - - - - 100 - 22 - 101 - 17 - - - - Numpy Array - - - - - - 209 - 22 - 61 - 17 - - - - OMX - - - true - - - - - - - 10 - 70 - 801 - 461 - - - - - - - - matrix_cores_selector - matrix_index_selector - marg_no_marg - marg_sum_marg - marg_max_marg - marg_min_marg - matrix_viewer - radioButton - radioButton_2 - radioButton_3 - but_close - but_load_new_matrix - - - - diff --git a/qaequilibrae/modules/forms/ui_SegmentingLines.ui b/qaequilibrae/modules/forms/ui_SegmentingLines.ui deleted file mode 100644 index c54e5d8c..00000000 --- a/qaequilibrae/modules/forms/ui_SegmentingLines.ui +++ /dev/null @@ -1,146 +0,0 @@ - - - Dialog - - - - 0 - 0 - 400 - 300 - - - - AequilibraE - Creating line segments - - - - - 30 - 240 - 341 - 32 - - - - Qt::Horizontal - - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - - - - - 130 - 50 - 151 - 17 - - - - Selected features only - - - - - - 130 - 20 - 251 - 24 - - - - - - - 20 - 23 - 71 - 16 - - - - Line Layer - - - - - - 14 - 90 - 101 - 20 - - - - Break features in - - - - - - 118 - 88 - 41 - 24 - - - - - - - 165 - 92 - 91 - 16 - - - - segments - - - - - comboBox - checkBox - lineEdit - buttonBox - - - - - buttonBox - accepted() - Dialog - accept() - - - 248 - 254 - - - 157 - 274 - - - - - buttonBox - rejected() - Dialog - reject() - - - 316 - 260 - - - 286 - 274 - - - - - diff --git a/qaequilibrae/modules/forms/ui_nodes_to_areas.ui b/qaequilibrae/modules/forms/ui_nodes_to_areas.ui deleted file mode 100644 index 4f6c635e..00000000 --- a/qaequilibrae/modules/forms/ui_nodes_to_areas.ui +++ /dev/null @@ -1,390 +0,0 @@ - - - nodes_to_area - - - Qt::ApplicationModal - - - - 0 - 0 - 400 - 478 - - - - - 9 - - - - AequilibraE - Node to Area data aggregation - - - true - - - - - 40 - 440 - 341 - 32 - - - - Qt::Horizontal - - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - - - - - 10 - 10 - 381 - 101 - - - - Data source - - - - - 160 - 60 - 211 - 24 - - - - - - - 160 - 30 - 211 - 24 - - - - - - - 10 - 64 - 161 - 16 - - - - Node field - - - - - - 10 - 34 - 161 - 16 - - - - Node Layer - - - - - - - 10 - 120 - 381 - 101 - - - - Data destination - - - - - 160 - 60 - 211 - 24 - - - - - - - 160 - 30 - 211 - 24 - - - - - - - 10 - 64 - 181 - 16 - - - - Area field - - - - - - 10 - 34 - 191 - 16 - - - - Area Layer - - - - - - - 10 - 230 - 381 - 111 - - - - Aggregation criteria - - - - true - - - - 160 - 80 - 211 - 24 - - - - 10 - - - - - - 160 - 50 - 211 - 24 - - - - 10 - - - - - - 10 - 84 - 151 - 16 - - - - Area field - - - - - - 10 - 54 - 181 - 16 - - - - Node field - - - - - - 10 - 23 - 231 - 17 - - - - Fields need to have matching values - - - false - - - - - - - 10 - 350 - 381 - 81 - - - - Operation - - - - - 10 - 20 - 51 - 17 - - - - Sum - - - true - - - - - - 70 - 20 - 51 - 17 - - - - Max - - - - - - 130 - 20 - 51 - 17 - - - - Min - - - - - - 190 - 20 - 81 - 17 - - - - Average - - - - - - 10 - 50 - 321 - 17 - - - - Divided value in case of multiple area overlapping - - - - - - - 10 - 444 - 118 - 23 - - - - 0 - - - - - nodelayer - nodefield - arealayer - areafield - needsmatching - nodematching - areamatching - sum - max - min - avg - divide_value - buttonBox - - - - - buttonBox - accepted() - nodes_to_area - accept() - - - 248 - 254 - - - 157 - 274 - - - - - buttonBox - rejected() - nodes_to_area - reject() - - - 316 - 260 - - - 286 - 274 - - - - - diff --git a/qaequilibrae/modules/gis/desire_lines_dialog.py b/qaequilibrae/modules/gis/desire_lines_dialog.py index 6aaa44c9..cadd01c9 100644 --- a/qaequilibrae/modules/gis/desire_lines_dialog.py +++ b/qaequilibrae/modules/gis/desire_lines_dialog.py @@ -114,7 +114,7 @@ def centers_item(self, item): return cell_widget def run_thread(self): - self.worker_thread.desire_lines.connect(self.signal_handler) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() self.exec_() @@ -128,19 +128,17 @@ def load_fields_to_combo_boxes(self): def signal_handler(self, val): # Signals that will come from traffic assignment - if val[0] == "zones finalized": - self.progressbar.setValue(val[1]) - elif val[0] == "text AoN": + if val[0] == "set_text": self.progress_label.setText(val[1]) # Signals that will come from desire lines procedure - elif val[0] == "job_size_dl": - self.progressbar.setRange(0, val[1]) - elif val[0] == "jobs_done_dl": - self.progressbar.setValue(val[1]) - elif val[0] == "text_dl": - self.progress_label.setText(val[1]) - elif val[0] == "finished_desire_lines_procedure": + elif val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progressbar.setValue(self.progressbar.value() + 1) + elif val[0] == "finished": self.job_finished_from_thread() def job_finished_from_thread(self): @@ -212,7 +210,7 @@ def run(self): self.tr("Inputs not loaded properly"), self.tr("You need the layer and at least one matrix_procedures core"), level=1, - duration=10 + duration=10, ) def throws_error(self, error_message): diff --git a/qaequilibrae/modules/gis/desire_lines_procedure.py b/qaequilibrae/modules/gis/desire_lines_procedure.py index bc149abf..b0290412 100644 --- a/qaequilibrae/modules/gis/desire_lines_procedure.py +++ b/qaequilibrae/modules/gis/desire_lines_procedure.py @@ -13,13 +13,13 @@ from scipy.spatial import Delaunay from PyQt5.QtCore import pyqtSignal from qgis.PyQt.QtCore import QVariant -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from qaequilibrae.modules.common_tools import get_vector_layer_by_name from aequilibrae.paths import allOrNothing class DesireLinesProcedure(WorkerThread): - desire_lines = pyqtSignal(object) + signal = pyqtSignal(object) def __init__( self, parentThread, layer: str, id_field: int, matrix: AequilibraeMatrix, matrix_hash: dict, dl_type: str @@ -47,13 +47,13 @@ def doWork(self): elif self.dl_type == "DelaunayLines": self.do_delaunay_lines() - self.desire_lines.emit(("finished_desire_lines_procedure", 0)) + self.signal.emit(["finished"]) def get_basic_data(self): layer = get_vector_layer_by_name(self.layer) idx = layer.dataProvider().fieldNameIndex(self.id_field) feature_count = layer.featureCount() - self.desire_lines.emit(("job_size_dl", feature_count)) + self.signal.emit(["start", feature_count, "Loading Layer Features"]) all_centroids = {} for P, feat in enumerate(layer.getFeatures()): geom = feat.geometry() @@ -61,8 +61,7 @@ def get_basic_data(self): point = list(geom.centroid().asPoint()) centroid_id = feat.attributes()[idx] all_centroids[centroid_id] = point - self.desire_lines.emit(("jobs_done_dl", P)) - self.desire_lines.emit(("text_dl", self.tr("Loading Layer Features: ") + str(P) + "/" + str(feature_count))) + self.signal.emit(["update", P, f"Loading Layer Features: {P} / {feature_count}"]) # Creating resulting layer EPSG_code = int(layer.crs().authid().split(":")[1]) desireline_layer = QgsVectorLayer("LineString?crs=epsg:" + str(EPSG_code), self.dl_type, "memory") @@ -85,26 +84,26 @@ def do_desire_lines(self): coord_index = np.zeros((max_zone, 2)) coord_index[coords[:, 0].astype(np.int64), 0] = coords[:, 1] coord_index[coords[:, 0].astype(np.int64), 1] = coords[:, 2] - self.desire_lines.emit(("text_dl", self.tr("Manipulating matrix indices"))) + self.signal.emit(["set_text", self.tr("Manipulating matrix indices")]) zones = self.matrix.index[:].shape[0] a = np.array(self.matrix.index[:], np.int64) ij, ji = np.meshgrid(a, a, sparse=False, indexing="ij") ij = ij.flatten() ji = ji.flatten() arrays = [ij, ji] - self.desire_lines.emit(("text_dl", self.tr("Collecting all matrices"))) - self.desire_lines.emit(("job_size_dl", len(self.matrix.view_names))) + self.signal.emit(["start", len(self.matrix.view_names), "Collecting all matrices"]) total_mat = np.zeros((zones, zones), np.float64) for i, mat in enumerate(self.matrix.view_names): m = self.matrix.get_matrix(mat) total_mat += m arrays.append(m.flatten()) - self.desire_lines.emit(("jobs_done_dl", i + 1)) + self.signal.emit(["update", i, f"Matrices collected: {i}"]) + # Eliminates the cells for which we don't have geography - self.desire_lines.emit(("text_dl", self.tr("Filtering zones with no geography available"))) + self.signal.emit(["set_text", self.tr("Filtering zones with no geography available")]) zones_with_no_geography = [x for x in self.matrix.index[:] if x not in all_centroids] if zones_with_no_geography: - self.desire_lines.emit(("job_size_dl", len(zones_with_no_geography))) + self.signal.emit(["start", len(zones_with_no_geography), "Zones with no geometry"]) for k, z in enumerate(zones_with_no_geography): i = self.matrix.matrix_hash[z] t = np.nansum(total_mat[i, :]) + np.nansum(total_mat[:, i]) @@ -112,8 +111,8 @@ def do_desire_lines(self): self.report.append(f"Zone {z} does not have a corresponding centroid/zone. Total flow {t}") total_mat[i, :] = 0 total_mat[:, i] = 0 - self.desire_lines.emit(("jobs_done_dl", k + 1)) - self.desire_lines.emit(("text_dl", self.tr("Filtering down to OD pairs with flows"))) + self.signal.emit(["update", k, f"Zones with no geometry: {k}"]) + self.signal.emit(["set_text", self.tr("Filtering down to OD pairs with flows")]) field_names = [x for x in self.matrix.view_names] nonzero = np.nonzero(total_mat.flatten()) arrays = np.vstack(arrays).transpose() @@ -134,7 +133,7 @@ def do_desire_lines(self): defaults1 = {x + "_AB": 0.0 for x in field_names} defaults = {x + "_BA": 0.0 for x in field_names} defaults = {**defaults, **defaults1} - self.desire_lines.emit(("text_dl", self.tr("Concatenating AB & BA flows"))) + self.signal.emit(["set_text", self.tr("Concatenating AB & BA flows")]) flows = rfn.join_by( ["from", "to"], flows_ab, flows_ba, jointype="outer", defaults=defaults, usemask=True, asrecarray=True ) @@ -143,8 +142,8 @@ def do_desire_lines(self): base_dl_fields.extend([QgsField(f, QVariant.Double)]) dlpr.addAttributes(base_dl_fields) desireline_layer.updateFields() - self.desire_lines.emit(("text_dl", self.tr("Creating Desire Lines"))) - self.desire_lines.emit(("job_size_dl", flows.shape[0])) + + self.signal.emit(["start", flows.shape[0], self.tr("Creating Desire Lines")]) all_features = [] for i, rec in enumerate(flows): a_node = rec[0] @@ -160,7 +159,7 @@ def do_desire_lines(self): attrs.extend([float(x) for x in list(rec)[2:]]) feature.setAttributes(attrs) all_features.append(feature) - self.desire_lines.emit(("jobs_done_dl", i)) + self.signal.emit(["update", i, f"Creating lines: {i} / {len(flows)}"]) if unnasigned > 0: self.report.append(f"Total non assigned flows (not counting intrazonals): {str(unnasigned)}") if flows.shape[0] > 1: @@ -181,41 +180,40 @@ def do_delaunay_lines(self): ) dlpr.addAttributes(base_dl_fields) desireline_layer.updateFields() - self.desire_lines.emit(("text_dl", self.tr("Building Delaunay dataset"))) + points = [] node_id_in_delaunay_results = {} i = 0 - self.desire_lines.emit(("job_size_dl", len(all_centroids))) + self.signal.emit(["start", len(all_centroids), self.tr("Building Delaunay dataset")]) for k, v in all_centroids.items(): - self.desire_lines.emit(("jobs_done_dl", i)) + self.signal.emit(["update", i, f"Building Delaunay dataset: {i}"]) points.append(v) node_id_in_delaunay_results[i] = k i += 1 - self.desire_lines.emit(("text_dl", self.tr("Computing Delaunay Triangles"))) + self.signal.emit(["set_text", self.tr("Computing Delaunay Triangles")]) tri = Delaunay(np.array(points)) # We process all the triangles to only get each edge once - self.desire_lines.emit(("text_dl", self.tr("Building Delaunay Network: Collecting Edges"))) edges = [] if self.python_version == 32: all_edges = tri.vertices else: all_edges = tri.simplices - self.desire_lines.emit(("job_size_dl", len(all_edges))) + self.signal.emit(["start", len(all_edges), self.tr("Building Delaunay Network: Collecting Edges")]) for j, triangle in enumerate(all_edges): - self.desire_lines.emit(("jobs_done_dl", j)) + self.signal.emit(["update", j, f"Collecting Edges: {j}"]) links = list(itertools.combinations(triangle, 2)) for i in links: edges.append([min(i[0], i[1]), max(i[0], i[1])]) - self.desire_lines.emit(("text_dl", self.tr("Building Delaunay Network: Getting unique edges"))) + self.signal.emit(["set_text", self.tr("Building Delaunay Network: Getting unique edges")]) edges = OrderedDict((str(x), x) for x in edges).values() + # Writing Delaunay layer - self.desire_lines.emit(("text_dl", self.tr("Building Delaunay Network: Assembling Layer"))) desireline_link_id = 1 data = [] dl_ids_on_links = {} - self.desire_lines.emit(("job_size_dl", len(edges))) + self.signal.emit(["start", len(edges), self.tr("Building Delaunay Network: Assembling Layer")]) for j, edge in enumerate(edges): - self.desire_lines.emit(("jobs_done_dl", j)) + self.signal.emit(["update", j, f"Assembling Layer: {j}"]) a_node = node_id_in_delaunay_results[edge[0]] a_point = all_centroids[a_node] a_point = QgsPointXY(a_point[0], a_point[1]) @@ -233,7 +231,7 @@ def do_delaunay_lines(self): data.append(line) dl_ids_on_links[desireline_link_id] = [a_node, b_node, 0, dist] desireline_link_id += 1 - self.desire_lines.emit(("text_dl", self.tr("Building graph"))) + self.signal.emit(["set_text", self.tr("Building graph")]) network = np.asarray(data) net = pd.DataFrame( @@ -265,19 +263,17 @@ def do_delaunay_lines(self): self.graph.set_blocked_centroid_flows(False) self.results = AssignmentResults() self.results.prepare(self.graph, self.matrix) - self.desire_lines.emit(("text_dl", self.tr("Assigning demand"))) - self.desire_lines.emit(("job_size_dl", self.matrix.index.shape[0])) - assigner = allOrNothing(self.matrix, self.graph, self.results) + self.signal.emit(["start", 1, self.tr("Assigning demand")]) + assigner = allOrNothing("aon", self.matrix, self.graph, self.results) assigner.execute() self.report = assigner.report - self.desire_lines.emit(("text_dl", self.tr("Collecting results"))) - self.desire_lines.emit(("text_dl", self.tr("Building resulting layer"))) + self.signal.emit(["update", 1, self.tr("Collecting results")]) features = [] max_edges = len(edges) - self.desire_lines.emit(("job_size_dl", max_edges)) + self.signal.emit(["start", max_edges, self.tr("Building resulting layer")]) link_loads = self.results.get_load_results() for i, link_id in enumerate(link_loads.index): - self.desire_lines.emit(("jobs_done_dl", i)) + self.signal.emit(["update", i, f"Building resulting layer: {i}"]) a_node, b_node, direct, dist = dl_ids_on_links[link_id] attr = [int(link_id), a_node, b_node, direct, dist] @@ -293,9 +289,9 @@ def do_delaunay_lines(self): for c in self.matrix.view_names: attr.extend( [ - float(link_loads.data[c + "_ab"][i]), - float(link_loads.data[c + "_ba"][i]), - float(link_loads.data[c + "_tot"][i]), + float(link_loads[c + "_ab"][link_id]), + float(link_loads[c + "_ba"][link_id]), + float(link_loads[c + "_tot"][link_id]), ] ) feature.setAttributes(attr) diff --git a/qaequilibrae/modules/gis/simple_tag_dialog.py b/qaequilibrae/modules/gis/simple_tag_dialog.py index d0c807ff..25319015 100644 --- a/qaequilibrae/modules/gis/simple_tag_dialog.py +++ b/qaequilibrae/modules/gis/simple_tag_dialog.py @@ -175,31 +175,12 @@ def matches_types(self): self.frommatchingtype = field.type() def run_thread(self): - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.ProgressText.connect(self.progress_text_from_thread) - self.worker_thread.finished_threaded_procedure.connect(self.finished_threaded_procedure) + self.worker_thread.signal.connect(self.signal_handler) self.OK.setEnabled(False) self.worker_thread.start() self.exec_() - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, value): - self.progressbar.setValue(value) - - def progress_text_from_thread(self, value): - self.lbl_operation.setText(value) - - def finished_threaded_procedure(self, procedure): - if self.worker_thread.error is not None: - qgis.utils.iface.messageBar().pushMessage( - self.tr("Input data not provided correctly"), self.worker_thread.error, level=3 - ) - self.close() - def run(self): error = False for wdgt in [self.fromlayer, self.fromfield, self.tolayer, self.tofield]: @@ -255,6 +236,26 @@ def string_order(self, order): elif order == 3: return self.tr("First found record is used.") + def signal_handler(self, val): + if val[0] == "start": + self.lbl_operation.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.lbl_operation.setText(val[2]) + self.progressbar.setValue(val[1]) + elif val[0] == "set_text": + self.lbl_operation.setText(val[1]) + self.progressbar.setValue(0) + elif val[0] == "finished": + self.lbl_operation.clear() + self.progressbar.reset() + if self.worker_thread.error is not None: + qgis.utils.iface.messageBar().pushMessage( + self.tr("Input data not provided correctly"), self.worker_thread.error, level=3 + ) + self.close() + def unload(self): pass diff --git a/qaequilibrae/modules/gis/simple_tag_procedure.py b/qaequilibrae/modules/gis/simple_tag_procedure.py index d71773a9..0cde7860 100644 --- a/qaequilibrae/modules/gis/simple_tag_procedure.py +++ b/qaequilibrae/modules/gis/simple_tag_procedure.py @@ -1,5 +1,5 @@ import numpy as np -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from qgis.PyQt.QtCore import pyqtSignal from qgis.core import QgsSpatialIndex, QgsCoordinateReferenceSystem, QgsCoordinateTransform, QgsProject @@ -8,10 +8,7 @@ class SimpleTAG(WorkerThread): - ProgressText = pyqtSignal(object) - ProgressValue = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parentThread, flayer, tlayer, ffield, tfield, fmatch, tmatch, operation, geo_types): WorkerThread.__init__(self, parentThread) @@ -52,17 +49,16 @@ def __init__(self, parentThread, flayer, tlayer, ffield, tfield, fmatch, tmatch, self.from_features = {} def doWork(self): - self.ProgressText.emit(self.tr("Initializing. Sit tight")) - self.ProgressValue.emit(0) + self.signal.emit(["set_text", self.tr("Initializing. Sit tight")]) + + to_layer_counts = self.to_layer.dataProvider().featureCount() + from_layer_counts = self.from_layer.dataProvider().featureCount() EPSG1 = QgsCoordinateReferenceSystem(f"EPSG:{int(self.from_layer.crs().authid().split(":")[1])}") EPSG2 = QgsCoordinateReferenceSystem(f"EPSG:{int(self.to_layer.crs().authid().split(":")[1])}") if EPSG1 != EPSG2: self.transform = QgsCoordinateTransform(EPSG1, EPSG2, QgsProject.instance()) - # PROGRESS BAR - self.ProgressMaxValue.emit(self.to_layer.dataProvider().featureCount()) - # FIELDS INDICES idx = self.from_layer.dataProvider().fieldNameIndex(self.ffield) fid = self.to_layer.dataProvider().fieldNameIndex(self.tfield) @@ -74,13 +70,12 @@ def doWork(self): self.to_match = {feature.id(): feature.attributes()[idq2] for (feature) in self.to_layer.getFeatures()} # We create an spatial self.index to hold all the features of the layer that will receive the data - self.ProgressText.emit(self.tr("Spatial index for target layer")) - self.ProgressValue.emit(0) + self.signal.emit(["start", to_layer_counts, self.tr("Spatial index for target layer")]) allfeatures = {} for i, feature in enumerate(self.to_layer.getFeatures()): + self.signal.emit(["update", i + 1, f"Target layer: {i}"]) self.index.addFeature(feature) allfeatures[feature.id()] = feature - self.ProgressValue.emit(i) self.all_attr = {} # Appending the line below would secure perfect results, but yields a VERY slow algorithm for when @@ -88,42 +83,34 @@ def doWork(self): # self.sequence_of_searches.append(self.from_count) # Dictionary with the FROM values - self.ProgressMaxValue.emit(self.from_layer.dataProvider().featureCount()) - self.ProgressText.emit(self.tr("Spatial index for source layer")) + self.signal.emit(["start", from_layer_counts, self.tr("Spatial index for source layer")]) self.from_val = {} for i, feature in enumerate(self.from_layer.getFeatures()): + if i % 1000 == 0: + self.signal.emit(["update", i + 1, f"Source layer: {i}"]) self.index_from.addFeature(feature) - self.ProgressValue.emit(i) self.from_val[feature.id()] = feature.attributes()[idx] self.from_features[feature.id()] = feature - # The spatial self.index for source layer - self.ProgressValue.emit(0) - + self.signal.emit(["start", to_layer_counts, self.tr("Performing spatial matching")]) self.from_count = len(self.from_val.keys()) # Number of features in the source layer - self.ProgressText.emit(self.tr("Performing spatial matching")) - self.ProgressValue.emit(0) - self.ProgressMaxValue.emit(self.to_layer.dataProvider().featureCount()) # Now we will have the code for all the possible configurations of input layer and output layer for i, feat in enumerate(self.to_layer.getFeatures()): - self.ProgressValue.emit(i) - + self.signal.emit(["update", i + 1, f"Features: {i}"]) self.chooses_match(feat) if feat.id() not in self.all_attr: self.all_attr[feat.id()] = None - self.ProgressValue.emit(0) - self.ProgressText.emit(self.tr("Writing data to target layer")) + self.signal.emit(["start", to_layer_counts, self.tr("Writing data to target layer")]) for i, feat in enumerate(self.to_layer.getFeatures()): - self.ProgressValue.emit(i) + self.signal.emit(["update", i + 1, f"Writing data to target layer: {i}"]) if self.all_attr[feat.id()] is not None: _ = self.to_layer.dataProvider().changeAttributeValues({feat.id(): {fid: self.all_attr[feat.id()]}}) self.to_layer.commitChanges() self.to_layer.updateFields() - self.ProgressValue.emit(self.to_layer.dataProvider().featureCount()) - self.finished_threaded_procedure.emit("procedure") + self.signal.emit(["finished"]) def chooses_match(self, feat): geom = feat.geometry() diff --git a/qaequilibrae/modules/matrix_procedures/display_aequilibrae_formats_dialog.py b/qaequilibrae/modules/matrix_procedures/display_aequilibrae_formats_dialog.py index d6c3246c..e1dcea73 100644 --- a/qaequilibrae/modules/matrix_procedures/display_aequilibrae_formats_dialog.py +++ b/qaequilibrae/modules/matrix_procedures/display_aequilibrae_formats_dialog.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd from math import ceil -from aequilibrae.matrix import AequilibraeMatrix, AequilibraeData +from aequilibrae.matrix import AequilibraeMatrix import qgis from qgis.PyQt import QtWidgets, uic, QtCore @@ -58,11 +58,8 @@ def __init__(self, qgis_project, file_path="", proj=False): def continue_with_data(self): self.setWindowTitle(self.tr("File path: {}").format(self.data_path)) - if self.data_type in ["AED", "AEM"]: - if self.data_type == "AED": - self.data_to_show = AequilibraeData() - elif self.data_type == "AEM": - self.data_to_show = AequilibraeMatrix() + if self.data_type == "AEM": + self.data_to_show = AequilibraeMatrix() if not self.from_proj: self.qgis_project.matrices[self.data_path] = self.data_to_show try: @@ -377,7 +374,7 @@ def add_matrix_parameters(self, idx, field): self.data_to_show.matrix[field] = self.data_to_show.matrix_view[:, :] def get_file_name(self): - formats = ["Aequilibrae matrix(*.aem)", "Aequilibrae dataset(*.aed)", "OpenMatrix(*.omx)"] + formats = ["Aequilibrae matrix(*.aem)", "OpenMatrix(*.omx)"] dflt = ".aem" data_path, data_type = GetOutputFileName( diff --git a/qaequilibrae/modules/matrix_procedures/forms/ui_vector_loader.ui b/qaequilibrae/modules/matrix_procedures/forms/ui_vector_loader.ui index 59337f41..c4bad3c5 100644 --- a/qaequilibrae/modules/matrix_procedures/forms/ui_vector_loader.ui +++ b/qaequilibrae/modules/matrix_procedures/forms/ui_vector_loader.ui @@ -38,7 +38,7 @@ 160 - 68 + 95 131 23 @@ -147,7 +147,7 @@ 250 - 36 + 53 190 24 @@ -160,7 +160,7 @@ 171 - 40 + 57 121 16 @@ -169,7 +169,7 @@ Index field - + 10 @@ -178,18 +178,21 @@ 20 + + true + - AequilibraE data + CSV true - + 10 - 40 + 70 161 17 @@ -205,7 +208,7 @@ 12 - 96 + 120 141 22 @@ -221,7 +224,7 @@ 304 - 68 + 95 136 23 @@ -271,9 +274,9 @@ 247 - 139 + 149 190 - 261 + 251 @@ -286,9 +289,9 @@ 13 - 139 + 149 190 - 261 + 251 @@ -339,7 +342,7 @@ 11 - 68 + 95 131 23 @@ -389,7 +392,7 @@ 250 - 10 + 17 190 24 @@ -402,7 +405,7 @@ 171 - 13 + 20 121 16 @@ -415,7 +418,7 @@ 283 - 122 + 130 131 16 @@ -424,6 +427,19 @@ Keep these fields only + + + + 10 + 40 + 82 + 17 + + + + Parquet + + diff --git a/qaequilibrae/modules/matrix_procedures/load_dataset_class.py b/qaequilibrae/modules/matrix_procedures/load_dataset_class.py index a717b84a..4579e594 100644 --- a/qaequilibrae/modules/matrix_procedures/load_dataset_class.py +++ b/qaequilibrae/modules/matrix_procedures/load_dataset_class.py @@ -1,18 +1,14 @@ -from qgis.PyQt.QtCore import QVariant import numpy as np -from uuid import uuid4 -from aequilibrae.utils.worker_thread import WorkerThread import struct -from aequilibrae.matrix import AequilibraeData +import pandas as pd +from aequilibrae.utils.interface.worker_thread import WorkerThread +from qgis.PyQt.QtCore import pyqtSignal, QVariant + from qaequilibrae.modules.common_tools.global_parameters import float_types, string_types, integer_types -from qgis.PyQt.QtCore import pyqtSignal class LoadDataset(WorkerThread): - ProgressText = pyqtSignal(object) - ProgressValue = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parent_thread, layer, index_field, fields, file_name): WorkerThread.__init__(self, parent_thread) @@ -21,12 +17,12 @@ def __init__(self, parent_thread, layer, index_field, fields, file_name): self.fields = fields self.error = None self.python_version = 8 * struct.calcsize("P") - self.output = AequilibraeData() + self.output = pd.DataFrame([]) self.output_name = file_name def doWork(self): feat_count = self.layer.featureCount() - self.ProgressMaxValue.emit(feat_count) + self.signal.emit(["start", feat_count, f"Total features: {feat_count}"]) # Create specification for the output file datafile_spec = {"entries": feat_count} @@ -55,28 +51,21 @@ def doWork(self): fields.append(str(field.name())) idxs.append(self.layer.dataProvider().fieldNameIndex(field.name())) - index_idx = self.layer.dataProvider().fieldNameIndex(self.index_field) datafile_spec["field_names"] = fields datafile_spec["data_types"] = types - - if self.output_name is None: - self.output_name = self.output.random_name() - else: - self.output_name += f"_{uuid4().hex[:10]}" datafile_spec["file_path"] = self.output_name if self.error is None: - self.output.create_empty(**datafile_spec) # Get all the data for p, feat in enumerate(self.layer.getFeatures()): for idx, field, empty in zip(idxs, fields, empties): if feat.attributes()[idx] == QVariant(): - self.output.data[field][p] = empty + self.output.loc[p, field] = empty else: - self.output.data[field][p] = feat.attributes()[idx] - self.output.index[p] = feat.attributes()[index_idx] - self.ProgressValue.emit(p) + self.output.loc[p, field] = feat.attributes()[idx] + self.signal.emit(["update", p, f"Feature count: {p}"]) + self.output.set_index(self.index_field, inplace=True) - self.ProgressValue.emit(feat_count) - self.finished_threaded_procedure.emit("Done") + self.signal.emit(["set_text", feat_count]) + self.signal.emit(["finished"]) diff --git a/qaequilibrae/modules/matrix_procedures/load_dataset_dialog.py b/qaequilibrae/modules/matrix_procedures/load_dataset_dialog.py index 001fa0ef..e7df078f 100644 --- a/qaequilibrae/modules/matrix_procedures/load_dataset_dialog.py +++ b/qaequilibrae/modules/matrix_procedures/load_dataset_dialog.py @@ -1,8 +1,8 @@ import os +import pandas as pd from functools import partial import qgis -from aequilibrae.matrix import AequilibraeData from qgis.PyQt import QtWidgets, uic, QtCore from qgis.PyQt.QtCore import Qt @@ -34,8 +34,9 @@ def __init__(self, iface, single_use=True): self.ignore_fields = [] self.single_use = single_use - self.radio_layer_matrix.clicked.connect(partial(self.size_it_accordingly, False)) - self.radio_aequilibrae.clicked.connect(partial(self.size_it_accordingly, False)) + self.radio_layer.clicked.connect(partial(self.size_it_accordingly, False)) + self.radio_csv.clicked.connect(partial(self.size_it_accordingly, False)) + self.radio_parquet.clicked.connect(partial(self.size_it_accordingly, False)) self.chb_all_fields.clicked.connect(self.set_tables_with_fields) self.but_adds_to_links.clicked.connect(self.append_to_list) @@ -43,7 +44,7 @@ def __init__(self, iface, single_use=True): self.cob_data_layer.currentIndexChanged.connect(self.load_fields_to_combo_boxes) self.but_removes_from_links.clicked.connect(self.removes_fields) # For adding skims - self.but_load.clicked.connect(self.load_from_aequilibrae_format) + self.but_load.clicked.connect(self.load_from_file) self.but_save_and_use.clicked.connect(self.load_the_vector) self.but_import_and_use.clicked.connect(self.load_just_to_use) @@ -54,8 +55,9 @@ def __init__(self, iface, single_use=True): self.cob_data_layer.addItem(layer.name()) if not self.single_use: - self.radio_layer_matrix.setChecked(True) - self.radio_aequilibrae.setEnabled(False) + self.radio_layer.setChecked(True) + self.radio_csv.setEnabled(False) + self.radio_parquet.setEnabled(False) self.but_import_and_use.setEnabled(False) self.but_load.setEnabled(False) self.but_save_and_use.setText(self.tr("Import")) @@ -86,11 +88,11 @@ def set_size(w, h): self.setMaximumSize(QtCore.QSize(w, h)) self.resize(w, h) - if self.radio_aequilibrae.isChecked(): - set_size(154, 100) + if self.radio_csv.isChecked() or self.radio_parquet.isChecked(): + set_size(154, 124) else: if final: - if self.radio_layer_matrix.isChecked(): + if self.radio_layer.isChecked(): if self.chb_all_fields.isChecked(): set_size(498, 120) self.progressbar.setMinimumHeight(100) @@ -140,9 +142,7 @@ def load_fields_to_combo_boxes(self): self.set_tables_with_fields() def run_thread(self): - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.finished_threaded_procedure.connect(self.finished_threaded_procedure) + self.worker_thread.signal.connect(self.signal_handler) self.chb_all_fields.setEnabled(False) self.but_load.setEnabled(False) @@ -150,50 +150,55 @@ def run_thread(self): self.worker_thread.start() self.exec_() - # VAL and VALUE have the following structure: (bar/text ID, value) - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, val): - self.progressbar.setValue(val) - - def finished_threaded_procedure(self): - self.but_load.setEnabled(True) - self.but_save_and_use.setEnabled(True) - self.chb_all_fields.setEnabled(True) - if self.worker_thread.error is not None: - qgis.utils.iface.messageBar().pushMessage( - self.tr("Error while loading vector:"), self.worker_thread.error, level=1 - ) - else: - self.dataset = self.worker_thread.output - self.exit_procedure() - - def load_from_aequilibrae_format(self): - out_name, _ = GetOutputFileName(self, "AequilibraE dataset", ["Aequilibrae dataset(*.aed)"], ".aed", self.path) + def signal_handler(self, val): + if val[0] == "start": + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progressbar.setValue(val[1]) + elif val[0] == "set_text": + self.progressbar.setValue(val[1]) + elif val[0] == "finished": + self.but_load.setEnabled(True) + self.but_save_and_use.setEnabled(True) + self.chb_all_fields.setEnabled(True) + if self.worker_thread.error is not None: + qgis.utils.iface.messageBar().pushMessage( + self.tr("Error while loading vector:"), self.worker_thread.error, level=1 + ) + else: + self.dataset = self.worker_thread.output + self.exit_procedure() + + def load_from_file(self): + if self.radio_csv.isChecked(): + out_name, _ = GetOutputFileName(self, "Load file", ["Comma-separated values (*.csv)"], ".csv", self.path) + elif self.radio_parquet.isChecked(): + out_name, _ = GetOutputFileName(self, "Load file", ["Parquet (*.parquet)"], ".parquet", self.path) self.load_with_file_name(out_name) def load_with_file_name(self, out_name): try: - self.dataset = AequilibraeData() - self.dataset.load(out_name) + if ".csv" in out_name: + self.dataset = pd.read_csv(out_name) + elif ".parquet" in out_name: + self.dataset = pd.read_parquet(out_name) + self.output_name = out_name except Exception as e: self.error = self.tr( - "Could not load file. It might be corrupted or not a valid AequilibraE file. {}".format(e.args) + "Could not load file. It might be corrupted or not a valid file format. {}".format(e.args) ) self.exit_procedure() def load_the_vector(self): self.set_output_name() - if self.radio_layer_matrix.isChecked() and self.error is None: + if self.radio_layer.isChecked() and self.error is None: self.output_name = self.layer.name() if self.cob_data_layer.currentIndex() < 0 or self.cob_index_field.currentIndex() < 0: self.error = self.tr("Invalid field chosen") index_field = self.cob_index_field.currentText() - if index_field in self.selected_fields: - self.selected_fields.remove(index_field) if len(self.selected_fields) > 0: self.worker_thread = LoadDataset( @@ -216,10 +221,9 @@ def set_output_name(self): if self.single_use: self.output_name = None else: + formats = ["Comma-separated values (*.csv)", "Parquet (*.parquet)"] self.error = None - self.output_name, _ = GetOutputFileName( - self, "AequilibraE dataset", ["Aequilibrae dataset(*.aed)"], ".aed", self.path - ) + self.output_name, _ = GetOutputFileName(self, "Save layer vector", formats, ".csv", self.path) if self.output_name is None: self.error = self.tr("No name provided for the output file") diff --git a/qaequilibrae/modules/matrix_procedures/load_matrix_class.py b/qaequilibrae/modules/matrix_procedures/load_matrix_class.py index 55bfd990..b4cd0429 100644 --- a/qaequilibrae/modules/matrix_procedures/load_matrix_class.py +++ b/qaequilibrae/modules/matrix_procedures/load_matrix_class.py @@ -4,17 +4,14 @@ import numpy as np from aequilibrae.matrix import AequilibraeMatrix -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from scipy.sparse import coo_matrix from qgis.PyQt.QtCore import pyqtSignal class LoadMatrix(WorkerThread): - ProgressValue = pyqtSignal(object) - ProgressText = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parentThread, **kwargs): WorkerThread.__init__(self, parentThread) @@ -31,9 +28,8 @@ def __init__(self, parentThread, **kwargs): def doWork(self): if self.matrix_type == "layer": - self.ProgressText.emit(self.tr("Loading from table")) feat_count = self.layer.featureCount() - self.ProgressMaxValue.emit(feat_count) + self.signal.emit(["start", feat_count, self.tr("Loading from table")]) # We read all the vectors and put in a list of lists matrix = [] @@ -45,11 +41,10 @@ def doWork(self): c = feat.attributes()[self.idx[2]] matrix.append([a, b, c]) if P % 1000 == 0: - self.ProgressValue.emit(int(P)) - self.ProgressText.emit(self.tr("Loading matrix: {}/{}").format(P, feat_count)) + txt = self.tr("Loading matrix: {}/{}").format(P, feat_count) + self.signal.emit(["update", int(P), txt]) - self.ProgressValue.emit(0) - self.ProgressText.emit(self.tr("Converting to a NumPy array")) + self.signal.emit(["set_text", self.tr("Converting to a NumPy array")]) matrix1 = np.array(matrix) # transform the list of lists in NumPy array del matrix @@ -68,7 +63,7 @@ def doWork(self): del matrix1 elif self.matrix_type == "numpy": - self.ProgressText.emit(self.tr("Loading from NumPy")) + self.signal.emit(["set_text", self.tr("Loading from NumPy")]) try: mat = np.load(self.numpy_file) if len(mat.shape[:]) == 2: @@ -91,5 +86,4 @@ def doWork(self): except Exception as e: self.report.append(f"Could not load array. {e.args}") - self.ProgressText.emit("") - self.finished_threaded_procedure.emit("LOADED-MATRIX") + self.signal.emit(["finished", "LOADED-MATRIX"]) diff --git a/qaequilibrae/modules/matrix_procedures/load_matrix_dialog.py b/qaequilibrae/modules/matrix_procedures/load_matrix_dialog.py index 9b950640..2be7a594 100644 --- a/qaequilibrae/modules/matrix_procedures/load_matrix_dialog.py +++ b/qaequilibrae/modules/matrix_procedures/load_matrix_dialog.py @@ -109,55 +109,55 @@ def load_fields_to_combo_boxes(self): self.field_cells.addItem(field.name()) def run_thread(self): - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.ProgressText.connect(self.progress_text_from_thread) - self.worker_thread.finished_threaded_procedure.connect(self.finished_threaded_procedure) + self.worker_thread.signal.connect(self.signal_handler) self.but_load.setEnabled(False) self.worker_thread.start() self.exec_() - # VAL and VALUE have the following structure: (bar/text ID, value) - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, val): - self.progressbar.setValue(val) - - def progress_text_from_thread(self, val): - self.progress_label.setText(val) - - def finished_threaded_procedure(self, param): - self.but_load.setEnabled(True) - if self.worker_thread.report: - dlg2 = ReportDialog(self.iface, self.worker_thread.report) - dlg2.show() - dlg2.exec_() - else: - if param == "LOADED-MATRIX": - self.compressed.setVisible(True) - self.progress_label.setVisible(False) - - if self.__current_name in self.matrices.keys(): - i = 1 - while self.__current_name + "_" + str(i) in self.matrices.keys(): - i += 1 - self.__current_name = self.__current_name + "_" + str(i) - - self.matrices[self.__current_name] = self.worker_thread.matrix - self.matrix_count += 1 - self.update_matrix_list() - - if not self.multiple: - self.update_matrix_hashes() - - elif param == "REBLOCKED MATRICES": - self.matrix = self.worker_thread.matrix - if self.compressed.isChecked(): - pass - # compression not implemented yet - self.exit_procedure() + def signal_handler(self, val): + if val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progress_label.setText(val[2]) + self.progressbar.setValue(val[1]) + elif val[0] == "set_text": + self.progress_label.setText(val[1]) + self.progressbar.setValue(0) + elif val[0] == "finished": + self.progress_label.clear() + self.progressbar.reset() + self.but_load.setEnabled(True) + if self.worker_thread.report: + dlg2 = ReportDialog(self.iface, self.worker_thread.report) + dlg2.show() + dlg2.exec_() + else: + if val[1] == "LOADED-MATRIX": + self.compressed.setVisible(True) + self.progress_label.setVisible(False) + + if self.__current_name in self.matrices.keys(): + i = 1 + while self.__current_name + "_" + str(i) in self.matrices.keys(): + i += 1 + self.__current_name = self.__current_name + "_" + str(i) + + self.matrices[self.__current_name] = self.worker_thread.matrix + self.matrix_count += 1 + self.update_matrix_list() + + if not self.multiple: + self.update_matrix_hashes() + + elif val[1] == "REBLOCKED MATRICES": + self.matrix = self.worker_thread.matrix + if self.compressed.isChecked(): + pass + # compression not implemented yet + self.exit_procedure() def __create_appropriate_name(self, nm: str) -> str: nm = nm.replace(" ", "_") diff --git a/qaequilibrae/modules/matrix_procedures/mat_reblock.py b/qaequilibrae/modules/matrix_procedures/mat_reblock.py index e5f16fcc..0e00e056 100644 --- a/qaequilibrae/modules/matrix_procedures/mat_reblock.py +++ b/qaequilibrae/modules/matrix_procedures/mat_reblock.py @@ -1,16 +1,13 @@ import numpy as np from aequilibrae.matrix import AequilibraeMatrix -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from scipy.sparse import coo_matrix from qgis.PyQt.QtCore import pyqtSignal class MatrixReblocking(WorkerThread): - ProgressValue = pyqtSignal(object) - ProgressText = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parentThread, **kwargs): WorkerThread.__init__(self, parentThread) @@ -27,9 +24,7 @@ def __init__(self, parentThread, **kwargs): def doWork(self): if self.sparse: # Builds the hash - self.ProgressMaxValue.emit(self.num_matrices) - self.ProgressValue.emit(0) - self.ProgressText.emit(self.tr("Building correspondence")) + self.signal.emit(["start", self.num_matrices, self.tr("Building correspondence")]) indices = None p = 0 @@ -44,7 +39,7 @@ def doWork(self): all_indices = np.hstack((indices, froms, tos)) indices = np.unique(all_indices) p += 1 - self.ProgressValue.emit(p) + self.signal.emit(["update", p, f"counter: {p}"]) compact_shape = int(indices.shape[0]) else: compact_shape = 0 @@ -62,8 +57,7 @@ def doWork(self): self.matrix.index[:] = indices[:] k = 0 - self.ProgressMaxValue.emit(self.num_matrices) - self.ProgressText.emit(self.tr("Reblocking matrices")) + self.signal.emit(["start", self.num_matrices, self.tr("Reblocking matrices")]) new_mat = None for mat_name, mat in self.matrices.items(): @@ -73,10 +67,10 @@ def doWork(self): new_mat["from"][mat["from"] == j] = v new_mat["to"][mat["to"] == j] = v k += 1 - self.ProgressValue.emit(k) + self.signal.emit(["update", k, f"sparse {k}"]) else: k += 1 - self.ProgressValue.emit(1) + self.signal.emit(["update", 1, f"non-sparse{k}"]) # In order to differentiate the zeros from the NaNs in the future matrix if new_mat is None: @@ -94,5 +88,5 @@ def doWork(self): del mat del new_mat - self.ProgressText.emit(self.tr("Matrix Reblocking finalized")) - self.finished_threaded_procedure.emit("REBLOCKED MATRICES") + self.signal.emit(["set_text", self.tr("Matrix Reblocking finalized")]) + self.signal.emit(["finished", "REBLOCKED MATRICES"]) diff --git a/qaequilibrae/modules/network/Network_preparation_procedure.py b/qaequilibrae/modules/network/Network_preparation_procedure.py index 33a65c23..da06754a 100644 --- a/qaequilibrae/modules/network/Network_preparation_procedure.py +++ b/qaequilibrae/modules/network/Network_preparation_procedure.py @@ -3,15 +3,12 @@ from qgis._core import QgsField, QgsFeatureRequest, QgsPointXY, QgsVectorLayer, QgsGeometry, QgsFeature, QgsSpatialIndex from qgis.PyQt.QtCore import QVariant -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from qaequilibrae.modules.common_tools import get_vector_layer_by_name class NetworkPreparationProcedure(WorkerThread): - ProgressValue = pyqtSignal(object) - ProgressText = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__( self, @@ -41,14 +38,12 @@ def doWork(self): layer = get_vector_layer_by_name(line_layer) feat_count = layer.featureCount() - self.ProgressMaxValue.emit(3) - self.ProgressValue.emit(0) - self.ProgressText.emit(self.tr("Duplicating line layer")) + self.signal.emit(["set_text", self.tr("Duplicating line layer")]) # We create the new line layer and load it in memory self.epsg_code = int(layer.crs().authid().split(":")[1]) new_line_layer = self.duplicate_layer(layer, "Linestring", self.new_line_layer) - self.ProgressValue.emit(1) + self.signal.emit(["set_text", self.tr("Line layer duplicated")]) # Add the A_Node and B_node fields to the layer field_names = [x.name().upper() for x in new_line_layer.dataProvider().fields().toList()] @@ -57,7 +52,7 @@ def doWork(self): if f not in field_names: _ = new_line_layer.dataProvider().addAttributes([QgsField(f, QVariant.Int)]) new_line_layer.updateFields() - self.ProgressValue.emit(2) + self.signal.emit(["set_text", self.tr("Adding fields to line layer")]) # If we have node IDs, we iterate over the ID field to make sure they are unique if node_ids: @@ -65,10 +60,9 @@ def doWork(self): else: self.with_lines_only(feat_count, new_line_layer) - self.ProgressText.emit("DONE") + self.signal.emit(["finished"]) def with_lines_only(self, feat_count, new_line_layer): - self.ProgressMaxValue.emit(feat_count) # Create node layer new_node_layer = QgsVectorLayer( f"Point?crs=epsg:{self.epsg_code}&field=ID:integer", self.new_node_layer, "memory" @@ -82,11 +76,13 @@ def with_lines_only(self, feat_count, new_line_layer): ] all_nodes = np.zeros(feat_count * 2, dtype=DTYPE) line = 0 + txt = self.tr("Links read: {}/{}") + self.signal.emit(["start", feat_count, txt.format(0, feat_count)]) + # Let's read all links and the coordinates for their extremities for p, feat in enumerate(new_line_layer.getFeatures()): if p % 500 == 0: - self.ProgressValue.emit(int(p)) - self.ProgressText.emit(self.tr("Links read: {}/{}").format(p, feat_count)) + self.signal.emit(["update", p, txt.format(p, feat_count)]) link = list(feat.geometry().asPolyline()) if link: @@ -105,15 +101,17 @@ def with_lines_only(self, feat_count, new_line_layer): all_nodes[line][2] = link_id all_nodes[line][3] = 1 line += 1 + + self.signal.emit(["update", feat_count, txt.format(feat_count, feat_count)]) + # Now we sort the nodes and assign IDs to them all_nodes = np.sort(all_nodes, order=["LAT", "LONG"]) lat0 = -100000.0 longit0 = -100000.0 incremental_ids = self.node_start - 1 p = 0 - self.ProgressMaxValue.emit(feat_count * 2) - self.ProgressText.emit(self.tr("Computing node IDs: {}/{}").format(0, feat_count * 2)) - self.ProgressMaxValue.emit(feat_count * 2) + txt = self.tr("Computing node IDs: {}/{}") + self.signal.emit(["start", int(feat_count * 2), txt.format(0, feat_count * 2)]) for i in all_nodes: p += 1 lat, longit, link_id, position, node_id = i @@ -126,14 +124,15 @@ def with_lines_only(self, feat_count, new_line_layer): i[4] = incremental_ids if p % 2000 == 0: - self.ProgressValue.emit(int(p)) - self.ProgressText.emit(self.tr("Computing node IDs: {}/{}").format(p, feat_count * 2)) - self.ProgressValue.emit(int(feat_count * 2)) - self.ProgressText.emit(self.tr("Computing node IDs: {}/{}").format(feat_count * 2, feat_count * 2)) + self.signal.emit(["update", p, txt.format(p, int(feat_count * 2))]) + + self.signal.emit(["update", int(feat_count * 2), txt.format(int(feat_count * 2), int(feat_count * 2))]) + # And we write the node layer as well node_id0 = -1 p = 0 - self.ProgressMaxValue.emit(incremental_ids) + txt = self.tr("Writing new node layer: {}/{}") + self.signal.emit(["start", incremental_ids, txt.format(0, incremental_ids)]) cfeatures = [] for i in all_nodes: lat, longit, link_id, position, node_id = i @@ -147,16 +146,15 @@ def with_lines_only(self, feat_count, new_line_layer): node_id0 = node_id if p % 500 == 0: - self.ProgressValue.emit(int(p)) - self.ProgressText.emit(self.tr("Writing new node layer: {}/{}").format(p, incremental_ids)) + self.signal.emit(["update", incremental_ids, txt.format(p, incremental_ids)]) _ = new_node_layer.dataProvider().addFeatures(cfeatures) del cfeatures new_node_layer.commitChanges() - self.ProgressValue.emit(int(incremental_ids)) - self.ProgressText.emit(self.tr("Writing new node layer: {}/{}").format(incremental_ids, incremental_ids)) + self.signal.emit(["update", incremental_ids, txt.format(incremental_ids, incremental_ids)]) + # Now we write all the node _IDs back to the line layer - self.ProgressText.emit(self.tr("Writing node IDs to links: {}/{}").format(0, feat_count * 2)) - self.ProgressMaxValue.emit(feat_count * 2) + txt = self.tr("Writing node IDs to links: {}/{}") + self.signal.emit(["start", int(feat_count * 2), txt.format(0, int(feat_count * 2))]) fid1 = new_line_layer.dataProvider().fieldNameIndex("A_NODE") fid2 = new_line_layer.dataProvider().fieldNameIndex("B_NODE") for p, i in enumerate(all_nodes): @@ -168,10 +166,10 @@ def with_lines_only(self, feat_count, new_line_layer): new_line_layer.dataProvider().changeAttributeValues({int(link_id): {fid2: int(node_id)}}) if p % 50 == 0: - self.ProgressValue.emit(int(p)) - self.ProgressText.emit(self.tr("Writing node IDs to links: {}/{}").format(p, feat_count * 2)) - self.ProgressValue.emit(int(p)) - self.ProgressText.emit(self.tr("Writing node IDs to links: {}/{}").format(feat_count * 2, feat_count * 2)) + self.signal.emit(["update", int(p), txt.format(p, int(feat_count * 2))]) + + self.signal.emit(["update", int(feat_count * 2), txt.format(int(feat_count * 2), int(feat_count * 2))]) + new_line_layer.commitChanges() self.new_line_layer = new_line_layer self.new_node_layer = new_node_layer @@ -181,11 +179,10 @@ def with_node_ids(self, feat_count, new_line_layer, node_ids, node_layer): nodes = get_vector_layer_by_name(node_layer) index = QgsSpatialIndex() idx = nodes.dataProvider().fieldNameIndex(node_ids) - self.ProgressMaxValue.emit(nodes.featureCount()) - self.ProgressValue.emit(0) + self.signal.emit(["start", nodes.featureCount(), "Checking node layer"]) for P, feat in enumerate(nodes.getFeatures()): - self.ProgressText.emit(self.tr("Checking node layer: {}/{}").format(str(P), str(nodes.featureCount()))) - self.ProgressValue.emit(P) + txt = self.tr("Checking node layer: {}/{}").format(str(P), str(nodes.featureCount())) + self.signal.emit(["update", P, txt]) index.addFeature(feat) i_d = feat.attributes()[idx] if i_d in ids: @@ -197,12 +194,12 @@ def with_node_ids(self, feat_count, new_line_layer, node_ids, node_layer): break ids.append(i_d) if self.error is None: - self.ProgressMaxValue.emit(new_line_layer.featureCount()) + self.signal.emit(["start", new_line_layer.featureCount(), "Processing links"]) P = 0 for feat in new_line_layer.getFeatures(): P += 1 - self.ProgressValue.emit(int(P)) - self.ProgressText.emit(self.tr("Processing links: {}/{}").format(str(P), str(feat_count))) + txt = self.tr("Processing links: {}/{}").format(str(P), str(feat_count)) + self.signal.emit(["update", int(P), txt]) # We search for matches for all AB nodes ab_nodes = [("A_NODE", 0), ("B_NODE", -1)] diff --git a/qaequilibrae/modules/network/adds_connectors_dialog.py b/qaequilibrae/modules/network/adds_connectors_dialog.py index c5216773..2d417e28 100644 --- a/qaequilibrae/modules/network/adds_connectors_dialog.py +++ b/qaequilibrae/modules/network/adds_connectors_dialog.py @@ -43,6 +43,8 @@ def __init__(self, qgis_project): self.layer_box.layerChanged.connect(self.set_fields) self.layer_box.setFilters(QgsMapLayerProxyModel.PointLayer) + self.chb_zone.setVisible(False) + self.but_process.clicked.connect(self.run) def centroid_source(self): @@ -50,6 +52,9 @@ def centroid_source(self): self.field_box.setEnabled(self.rdo_layer.isChecked()) self.field_box.setVisible(not self.rdo_zone.isChecked()) self.lbl_radius.setVisible(not self.rdo_zone.isChecked()) + self.sb_radius.setVisible(not self.rdo_zone.isChecked()) + self.layer_box.setVisible(not self.rdo_zone.isChecked()) + self.chb_zone.setVisible(self.rdo_zone.isChecked()) def set_fields(self): self.field_box.setLayer(self.layer_box.currentLayer()) @@ -74,36 +79,34 @@ def run(self): "source": source, } - if source != "zone": + if source == "zone": + parameters["limit_to_zone"] = self.chb_zone.isChecked() + else: parameters["radius"] = self.sb_radius.value() if source == "layer": parameters["layer"] = self.layer_box.currentLayer() parameters["field"] = self.field_box.currentField() + self.worker_thread = AddsConnectorsProcedure(qgis.utils.iface.mainWindow(), **parameters) self.run_thread() def run_thread(self): - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressText.connect(self.progress_text_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.jobFinished.connect(self.job_finished_from_thread) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() - self.show() - - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, value): - self.progressbar.setValue(value) - - def progress_text_from_thread(self, value): - self.progress_label.setText(value) - - def job_finished_from_thread(self, success): - self.but_process.setEnabled(True) - self.project.network.links.refresh() - self.project.network.nodes.refresh() - self.exit_procedure() + self.exec_() + + def signal_handler(self, val): + if val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progressbar.setValue(val[1]) + elif val[0] == "finished": + self.but_process.setEnabled(True) + self.project.network.links.refresh() + self.project.network.nodes.refresh() + self.exit_procedure() def exit_procedure(self): self.close() diff --git a/qaequilibrae/modules/network/adds_connectors_procedure.py b/qaequilibrae/modules/network/adds_connectors_procedure.py index bd1a7835..3ee3df47 100644 --- a/qaequilibrae/modules/network/adds_connectors_procedure.py +++ b/qaequilibrae/modules/network/adds_connectors_procedure.py @@ -1,20 +1,25 @@ import shapely.wkb from shapely.geometry import Point -from aequilibrae.project.database_connection import database_connection -from aequilibrae.utils.db_utils import commit_and_close -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from PyQt5.QtCore import pyqtSignal class AddsConnectorsProcedure(WorkerThread): - ProgressValue = pyqtSignal(object) - ProgressText = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__( - self, parentThread, qgis_project, link_types, modes, num_connectors, source, radius=None, layer=None, field=None + self, + parentThread, + qgis_project, + link_types, + modes, + num_connectors, + source, + limit_to_zone=True, + radius=None, + layer=None, + field=None, ): WorkerThread.__init__(self, parentThread) self.qgis_project = qgis_project @@ -26,6 +31,7 @@ def __init__( self.source = source self.layer = layer self.field = field + self.limit_to_zone = limit_to_zone def doWork(self): if self.source == "zone": @@ -35,55 +41,52 @@ def doWork(self): else: self.do_from_layer() - self.ProgressText.emit("DONE") + self.project.network.nodes.refresh() + self.project.network.links.refresh() + self.signal.emit(["finished"]) def do_from_zones(self): zoning = self.project.zoning - with commit_and_close(database_connection("network")) as conn: - tot_zones = [x[0] for x in conn.execute("select count(*) from zones")][0] - self.ProgressMaxValue.emit(tot_zones) - - zones = [x[0] for x in conn.execute("select zone_id from zones")] - - for counter, zone_id in enumerate(zones): - zone = zoning.get(zone_id) + self.signal.emit(["start", len(zoning.all_zones()), "Adding connectors from zones"]) + for idx, (zone_id, zone) in enumerate(zoning.all_zones().items()): zone.add_centroid(None) for mode_id in self.modes: - zone.connect_mode(mode_id=mode_id, link_types=self.link_types, connectors=self.num_connectors) - self.ProgressValue.emit(counter + 1) + zone.connect_mode( + mode_id=mode_id, + link_types=self.link_types, + connectors=self.num_connectors, + limit_to_zone=self.limit_to_zone, + ) + self.signal.emit(["update", idx + 1, f"Connector from zone: {zone_id}"]) def do_from_network(self): nodes = self.project.network.nodes nodes.refresh() - self.ProgressMaxValue.emit(self.project.network.count_centroids()) - with commit_and_close(database_connection("network")) as conn: - centroids = [x[0] for x in conn.execute("select node_id from nodes where is_centroid=1")] + centroids = nodes.data[nodes.data["is_centroid"] == 1].node_id.tolist() + + self.signal.emit(["start", self.project.network.count_centroids(), "Adding connectors from nodes"]) for counter, zone_id in enumerate(centroids): node = nodes.get(zone_id) geo = self.polygon_from_radius(node.geometry) for mode_id in self.modes: node.connect_mode(area=geo, mode_id=mode_id, link_types=self.link_types, connectors=self.num_connectors) - self.ProgressValue.emit(counter + 1) + self.signal.emit(["update", counter + 1, f"Connector from node: {zone_id}"]) def do_from_layer(self): - fields = self.layer.fields() - idx = fields.indexOf(self.field) - features = list(self.layer.getFeatures()) - self.ProgressMaxValue.emit(len(features)) - nodes = self.project.network.nodes nodes.refresh() - for counter, feat in enumerate(features): - zone_id = feat.attributes()[idx] - node = nodes.new_centroid(zone_id) + + self.signal.emit(["start", self.layer.featureCount(), "Adding connectors from layer"]) + for counter, feat in enumerate(self.layer.getFeatures()): + node = nodes.new_centroid(feat.id()) node.geometry = shapely.wkb.loads(feat.geometry().asWkb().data()) node.save() geo = self.polygon_from_radius(node.geometry) for mode_id in self.modes: node.connect_mode(area=geo, mode_id=mode_id, link_types=self.link_types, connectors=self.num_connectors) - self.ProgressValue.emit(counter + 1) + self.signal.emit(["update", counter + 1, f"Connector from layer feature: {feat.id()}"]) def polygon_from_radius(self, point: Point): # We approximate with the radius of the Earth at the equator diff --git a/qaequilibrae/modules/network/forms/ui_add_connectors.ui b/qaequilibrae/modules/network/forms/ui_add_connectors.ui index 6b53cf42..a1243e48 100644 --- a/qaequilibrae/modules/network/forms/ui_add_connectors.ui +++ b/qaequilibrae/modules/network/forms/ui_add_connectors.ui @@ -16,7 +16,7 @@ - AequilibraE - Add zoning layer to project + AequilibraE - Add centroid connectors @@ -51,30 +51,6 @@ - - - - From network - - - true - - - - - - - Zone centers - - - - - - - Layer - - - @@ -91,6 +67,13 @@ + + + + Layer + + + @@ -104,6 +87,23 @@ + + + + Zone centers + + + + + + + From network + + + true + + + @@ -113,35 +113,53 @@ Configurations - + Allowed link types - - - - Modes to connect + + + + Qt::Vertical - + + QSizePolicy::Fixed + + + + 20 + 20 + + + - - - - true + + + + + 0 + 0 + + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + 1 - - - true + + + Modes to connect - + @@ -159,36 +177,31 @@ - - - - Qt::Vertical + + + + true - - QSizePolicy::Fixed + + + + + + Limit connector creation to zone - - - 20 - 20 - + + true - + - - - - - 0 - 0 - - - - 1 + + + + true - + Qt::LeftToRight @@ -197,7 +210,7 @@ Connectors per centroid - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter diff --git a/qaequilibrae/modules/network/network_preparation_dialog.py b/qaequilibrae/modules/network/network_preparation_dialog.py index 5538c1e8..3ac95cad 100644 --- a/qaequilibrae/modules/network/network_preparation_dialog.py +++ b/qaequilibrae/modules/network/network_preparation_dialog.py @@ -47,21 +47,22 @@ def __init__(self, iface): self.set_columns_nodes() def run_thread(self): - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressText.connect(self.progress_text_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.jobFinished.connect(self.job_finished_from_thread) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() self.show() - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, value): - self.progressbar.setValue(value) - - def progress_text_from_thread(self, value): - self.progress_label.setText(value) + def signal_handler(self, val): + if val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progressbar.setValue(val[1]) + elif val[0] == "set_text": + self.progress_label.setText(val[1]) + self.progressbar.reset() + elif val[0] == "finished": + self.job_finished_from_thread() def set_columns_nodes(self): self.cbb_node_fields.clear() @@ -87,7 +88,7 @@ def uses_nodes(self): # self.cbb_node_layer.hideEvent() self.np_node_start.setEnabled(True) - def job_finished_from_thread(self, success): + def job_finished_from_thread(self): if self.worker_thread.error is not None: qgis.utils.iface.messageBar().pushMessage(self.tr("Node layer error: "), self.worker_thread.error, level=3) else: diff --git a/qaequilibrae/modules/paths_procedures/forms/advanced_graph_details.ui b/qaequilibrae/modules/paths_procedures/forms/advanced_graph_details.ui deleted file mode 100644 index 91287728..00000000 --- a/qaequilibrae/modules/paths_procedures/forms/advanced_graph_details.ui +++ /dev/null @@ -1,181 +0,0 @@ - - - advanced_graph_details - - - - 0 - 0 - 329 - 246 - - - - Graph Centroids setting - - - - - 210 - 210 - 111 - 31 - - - - - 10 - - - - Cancel - - - - - - 9 - 10 - 111 - 17 - - - - - 10 - - - - Set centroids - - - - - false - - - - 10 - 40 - 311 - 161 - - - - QFrame::StyledPanel - - - QFrame::Raised - - - - - 14 - 120 - 251 - 17 - - - - - 10 - - - - Block path through connectors - - - - - - 67 - 11 - 241 - 30 - - - - - - - 14 - 18 - 51 - 16 - - - - - 10 - - - - Layer - - - - - - 14 - 57 - 51 - 16 - - - - - 10 - - - - ID field - - - - - - 67 - 90 - 131 - 20 - - - - - 8 - 50 - false - false - - - - Selected nodes only - - - - - - 67 - 49 - 241 - 30 - - - - - - - - QgsMapLayerComboBox - QComboBox -
qgsmaplayercombobox.h
- - - - chb_path_through_centroids - chb_set_centroids - but_ok - - - - diff --git a/qaequilibrae/modules/paths_procedures/forms/ui_traffic_assignment.ui b/qaequilibrae/modules/paths_procedures/forms/ui_traffic_assignment.ui index cfc704a3..ad2d5eff 100644 --- a/qaequilibrae/modules/paths_procedures/forms/ui_traffic_assignment.ui +++ b/qaequilibrae/modules/paths_procedures/forms/ui_traffic_assignment.ui @@ -667,9 +667,6 @@ 9 - - AlignCenter - @@ -680,9 +677,6 @@ 9 - - AlignCenter - @@ -867,8 +861,49 @@ Assignment + + + + + 600 + 0 + + + + false + + + false + + + Outputs + + + + + + false + + + Save complete path file + + + + + + + + + + Result_name + + + + + + - + 9 @@ -887,7 +922,7 @@ - + true @@ -936,75 +971,6 @@ - - - - CANCEL - - - - - - - true - - - - - - - - 0 - 170 - 0 - - - - - - - - - 240 - 240 - 240 - - - - - - - - - 51 - 153 - 255 - - - - - - - - 0 - - - true - - - - - - - - 9 - - - - Status Message 0 - - - @@ -1086,6 +1052,13 @@ + + + + CANCEL + + + @@ -1161,47 +1134,6 @@ - - - - - 600 - 0 - - - - false - - - false - - - Outputs - - - - - - false - - - Save complete path file - - - - - - - - - - Result_name - - - - - - diff --git a/qaequilibrae/modules/paths_procedures/impedance_matrix_dialog.py b/qaequilibrae/modules/paths_procedures/impedance_matrix_dialog.py index acdeec54..c19f7547 100644 --- a/qaequilibrae/modules/paths_procedures/impedance_matrix_dialog.py +++ b/qaequilibrae/modules/paths_procedures/impedance_matrix_dialog.py @@ -126,17 +126,21 @@ def check_name_exists(self): def run_thread(self): self.do_dist_matrix.setVisible(False) - self.progressbar.setRange(0, self.graph.num_zones) - self.worker_thread.skimming.connect(self.signal_handler) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() self.exec_() def signal_handler(self, val): - if val[0] == "zones finalized": + if val[0] == "start": + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progress_label.setText(val[2]) self.progressbar.setValue(val[1]) - elif val[0] == "text skimming": + elif val[0] == "set_text": self.progress_label.setText(val[1]) - elif val[0] == "finished_threaded_procedure": + elif val[0] == "finished": + self.progressbar.reset() + self.progress_label.clear() self.finished_threaded_procedure() def finished_threaded_procedure(self): diff --git a/qaequilibrae/modules/paths_procedures/point_tool.py b/qaequilibrae/modules/paths_procedures/point_tool.py index 6af235c4..ca5e35ff 100644 --- a/qaequilibrae/modules/paths_procedures/point_tool.py +++ b/qaequilibrae/modules/paths_procedures/point_tool.py @@ -5,7 +5,7 @@ class PointTool(QgsMapTool): - clicked = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, canvas): QgsMapTool.__init__(self, canvas) @@ -29,7 +29,7 @@ def canvasReleaseEvent(self, event): y = event.pos().y() self.point = self.canvas.getCoordinateTransform().toMapCoordinates(x, y) - self.clicked.emit(1) + self.signal.emit(1) def activate(self): pass diff --git a/qaequilibrae/modules/paths_procedures/show_shortest_path_dialog.py b/qaequilibrae/modules/paths_procedures/show_shortest_path_dialog.py index e91081cb..01b885a0 100644 --- a/qaequilibrae/modules/paths_procedures/show_shortest_path_dialog.py +++ b/qaequilibrae/modules/paths_procedures/show_shortest_path_dialog.py @@ -101,18 +101,18 @@ def clear_memory_layer(self): self.link_features = None def search_for_point_from(self): - self.clickTool.clicked.connect(self.fill_path_from) + self.clickTool.signal.connect(self.fill_path_from) self.iface.mapCanvas().setMapTool(self.clickTool) self.from_but.setEnabled(False) def search_for_point_to(self): self.iface.mapCanvas().setMapTool(self.clickTool) - self.clickTool.clicked.connect(self.fill_path_to) + self.clickTool.signal.connect(self.fill_path_to) self.to_but.setEnabled(False) def search_for_point_to_after_from(self): self.iface.mapCanvas().setMapTool(self.clickTool) - self.clickTool.clicked.connect(self.fill_path_to) + self.clickTool.signal.connect(self.fill_path_to) def fill_path_to(self): self.to_node = self.find_point() diff --git a/qaequilibrae/modules/paths_procedures/traffic_assignment_dialog.py b/qaequilibrae/modules/paths_procedures/traffic_assignment_dialog.py index 905e9889..30f3218b 100644 --- a/qaequilibrae/modules/paths_procedures/traffic_assignment_dialog.py +++ b/qaequilibrae/modules/paths_procedures/traffic_assignment_dialog.py @@ -11,6 +11,7 @@ from aequilibrae.project.database_connection import database_connection from aequilibrae.utils.db_utils import read_and_close from tempfile import gettempdir +import time import qgis from qgis.PyQt import QtWidgets, uic @@ -62,7 +63,7 @@ def __init__(self, qgis_project): self.cancel_all.clicked.connect(self.exit_procedure) # Signals for the algorithm tab - for q in [self.progressbar0, self.progressbar1, self.progress_label0, self.progress_label1]: + for q in [self.progressbar, self.progress_label]: q.setVisible(False) for algo in self.assignment.all_algorithms: @@ -228,7 +229,7 @@ def _create_traffic_class(self): class_name = self.ln_class_name.text() if class_name in self.traffic_classes: - qgis.utils.iface.messageBar().pushMessage(self.tr("Class name already used"), "", level=2) + qgis.utils.iface.messageBar().pushMessage(self.tr("Class name already used"), "", level=2, duration=10) self.but_add_skim.setEnabled(True) @@ -321,14 +322,11 @@ def __remove_class(self): self.__edit_skimming_modes() def run_thread(self): - self.worker_thread.assignment.connect(self.signal_handler) - if self.cb_choose_algorithm.currentText() != "all-or-nothing": - self.worker_thread.equilibration.connect(self.equilibration_signal_handler) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() self.exec_() def job_finished_from_thread(self): - # self.report = self.worker_thread.report self.produce_all_outputs() self.exit_procedure() @@ -337,16 +335,10 @@ def run(self): if not self.check_data(): qgis.utils.iface.messageBar().pushMessage(self.tr("Input error"), self.error, level=3, duration=10) - algorithm = self.cb_choose_algorithm.currentText() self.miter = int(self.max_iter.text()) - if algorithm != "all-or-nothing": - for q in [self.progressbar1, self.progress_label1]: - q.setVisible(True) - self.progressbar1.setRange(0, self.miter) - - for q in [self.progressbar0, self.progress_label0]: + for q in [self.progressbar, self.progress_label]: q.setVisible(True) - self.progressbar0.setRange(0, self.project.network.count_centroids()) + self.progressbar.setRange(0, self.project.network.count_centroids()) self.assignment.set_classes(list(self.traffic_classes.values())) self.assignment.set_vdf(self.cob_vdf.currentText()) @@ -383,24 +375,15 @@ def check_data(self): return tries_setup def signal_handler(self, val): - if val[0] == "zones finalized": - self.progressbar0.setValue(val[1]) - elif val[0] == "text AoN": - self.progress_label0.setText(val[1]) - elif val[0] == "finished_threaded_procedure": - self.progressbar0.setValue(0) - if self.cb_choose_algorithm.currentText() == "all-or-nothing": - self.job_finished_from_thread() - - def equilibration_signal_handler(self, val): - if val[0] == "iterations": - self.progressbar1.setValue(val[1]) - self.iter = val[1] - elif val[0] == "rgap": - self.rgap = val[1] - elif val[0] == "finished_threaded_procedure": + if val[0] == "start": + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + self.progress_label.setText(val[2]) + elif val[0] == "update": + self.progressbar.setValue(val[1]) + self.progress_label.setText(val[2]) + elif val[0] == "finished": self.job_finished_from_thread() - self.progress_label1.setText(f"{self.iter}/{self.miter} - Rel. Gap {self.rgap:.2E}") # Save link flows to disk def produce_all_outputs(self): diff --git a/qaequilibrae/modules/processing_provider/Add_connectors.py b/qaequilibrae/modules/processing_provider/Add_connectors.py index c4fc6089..58edac65 100644 --- a/qaequilibrae/modules/processing_provider/Add_connectors.py +++ b/qaequilibrae/modules/processing_provider/Add_connectors.py @@ -36,21 +36,20 @@ def initAlgorithm(self, config=None): ) def processAlgorithm(self, parameters, context, model_feedback): - feedback = QgsProcessingMultiStepFeedback(2, model_feedback) - # Checks if we have access to aequilibrae library if iutil.find_spec("aequilibrae") is None: sys.exit(self.tr("AequilibraE module not found")) from aequilibrae import Project + feedback = QgsProcessingMultiStepFeedback(2, model_feedback) + feedback.pushInfo(self.tr("Opening project")) project = Project() project.open(parameters["project_path"]) nodes = project.network.nodes - centroids = nodes.data - centroids = centroids.loc[centroids["is_centroid"] == 1] + centroids = nodes.data[nodes.data["is_centroid"] == 1] feedback.pushInfo(" ") feedback.setCurrentStep(1) @@ -62,7 +61,7 @@ def processAlgorithm(self, parameters, context, model_feedback): for _, node in centroids.iterrows(): cnt = nodes.get(node.node_id) - cnt.connect_mode(cnt.geometry.buffer(0.01), mode_id=mode, connectors=num_connectors) + cnt.connect_mode(mode_id=mode, connectors=num_connectors) feedback.pushInfo(" ") feedback.setCurrentStep(2) diff --git a/qaequilibrae/modules/processing_provider/export_matrix.py b/qaequilibrae/modules/processing_provider/export_matrix.py index cda57d8f..45d86d75 100644 --- a/qaequilibrae/modules/processing_provider/export_matrix.py +++ b/qaequilibrae/modules/processing_provider/export_matrix.py @@ -39,28 +39,30 @@ def initAlgorithm(self, config=None): ) def processAlgorithm(self, parameters, context, model_feedback): - - src_path = parameters["src"] - file_format = [".csv", ".omx", ".aem"] - format = file_format[parameters["output_format"]] - dst_path = join(parameters["dst"], f"{Path(src_path).stem}.{format}") - # Checks if we have access to aequilibrae library if iutil.find_spec("aequilibrae") is None: sys.exit(self.tr("AequilibraE module not found")) from aequilibrae.matrix import AequilibraeMatrix - if src_path[-3:] == "omx": - tmpmat = AequilibraeMatrix() - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".aem").name - tmpmat.create_from_omx(tmp, src_path) - tmpmat.export(tmp) - src_path = tmp - tmpmat.close() + file_format = ["csv", "omx", "aem"] + format = file_format[parameters["output_format"]] + file_name, ext = parameters["src"].split("/")[-1].split(".") + dst_path = join(parameters["dst"], f"{file_name}.{format}") + + kwargs = {"file_path": dst_path, "memory_only": False} mat = AequilibraeMatrix() - mat.load(src_path) - mat.export(dst_path) + + if ext == "omx": + if format == "omx": + mat.create_from_omx(omx_path=parameters["src"], **kwargs) + elif format in ["csv", "aem"]: + mat.create_from_omx(parameters["src"]) + mat.export(dst_path) + elif ext == "aem": + mat.load(parameters["src"]) + mat.export(dst_path) + mat.close() return {"Output": dst_path} diff --git a/qaequilibrae/modules/project_procedures/add_zones_procedure.py b/qaequilibrae/modules/project_procedures/add_zones_procedure.py index fa898b15..2b5b3f0f 100644 --- a/qaequilibrae/modules/project_procedures/add_zones_procedure.py +++ b/qaequilibrae/modules/project_procedures/add_zones_procedure.py @@ -1,13 +1,11 @@ import shapely.wkb -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread from PyQt5.QtCore import pyqtSignal class AddZonesProcedure(WorkerThread): - ProgressValue = pyqtSignal(object) - ProgressText = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parentThread, project, area_layer, select_only, add_centroids, field_correspondence): WorkerThread.__init__(self, parentThread) @@ -19,7 +17,8 @@ def __init__(self, parentThread, project, area_layer, select_only, add_centroids def doWork(self): features = list(self.lyr.selectedFeatures()) if self.select_only else list(self.lyr.getFeatures()) - self.emit_messages(message=self.tr("Importing zones"), value=0, max_val=len(features)) + + self.signal.emit(["start", len(features), self.tr("Importing zones")]) idx_id = self.field_corresp["zone_id"] zoning = self.project.zoning @@ -33,13 +32,5 @@ def doWork(self): zone.save() if self.add_centroids: zone.add_centroid(None) - self.emit_messages(value=i + 1) - self.jobFinished.emit("DONE") - - def emit_messages(self, message="", value=-1, max_val=-1): - if len(message) > 0: - self.ProgressText.emit(message) - if value >= 0: - self.ProgressValue.emit(value) - if max_val >= 0: - self.ProgressMaxValue.emit(max_val) + self.signal.emit(["update", i + 1, f"Num. zones: {i}"]) + self.signal.emit(["finished"]) diff --git a/qaequilibrae/modules/project_procedures/adds_zones_dialog.py b/qaequilibrae/modules/project_procedures/adds_zones_dialog.py index d499055c..f2422b56 100644 --- a/qaequilibrae/modules/project_procedures/adds_zones_dialog.py +++ b/qaequilibrae/modules/project_procedures/adds_zones_dialog.py @@ -60,10 +60,7 @@ def run(self): field_correspondence, ) - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressText.connect(self.progress_text_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.jobFinished.connect(self.job_finished_from_thread) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() self.show() @@ -107,17 +104,15 @@ def centers_item(self, item): cell_widget.setLayout(lay_out) return cell_widget - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, value): - self.progressbar.setValue(value) - - def progress_text_from_thread(self, value): - self.progress_label.setText(value) - - def job_finished_from_thread(self, success): - self.exit_procedure() + def signal_handler(self, val): + if val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progressbar.setValue(val[1]) + elif val[0] == "finished": + self.exit_procedure() def exit_procedure(self): self.close() diff --git a/qaequilibrae/modules/project_procedures/creates_transponet_dialog.py b/qaequilibrae/modules/project_procedures/creates_transponet_dialog.py index e53df595..41b9aad7 100644 --- a/qaequilibrae/modules/project_procedures/creates_transponet_dialog.py +++ b/qaequilibrae/modules/project_procedures/creates_transponet_dialog.py @@ -325,25 +325,20 @@ def exit_procedure(self): self.close() def run_thread(self): - self.worker_thread.ProgressValue.connect(self.progress_value_from_thread) - self.worker_thread.ProgressText.connect(self.progress_text_from_thread) - self.worker_thread.ProgressMaxValue.connect(self.progress_range_from_thread) - self.worker_thread.jobFinished.connect(self.job_finished_from_thread) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() - self.show() - - def progress_range_from_thread(self, val): - self.progressbar.setRange(0, val) - - def progress_value_from_thread(self, value): - self.progressbar.setValue(value) - - def progress_text_from_thread(self, value): - self.progress_label.setText(value) - - def job_finished_from_thread(self, success): - if self.worker_thread.report: - dlg2 = ReportDialog(self.iface, self.worker_thread.report) - dlg2.show() - dlg2.exec_() - self.exit_procedure() + self.exec_() + + def signal_handler(self, val): + if val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": + self.progressbar.setValue(val[1]) + elif val[0] == "finished": + if self.worker_thread.report: + dlg2 = ReportDialog(self.iface, self.worker_thread.report) + dlg2.show() + dlg2.exec_() + self.exit_procedure() diff --git a/qaequilibrae/modules/project_procedures/creates_transponet_procedure.py b/qaequilibrae/modules/project_procedures/creates_transponet_procedure.py index 50ad8536..d4c1883b 100644 --- a/qaequilibrae/modules/project_procedures/creates_transponet_procedure.py +++ b/qaequilibrae/modules/project_procedures/creates_transponet_procedure.py @@ -1,19 +1,19 @@ +import numpy as np +from string import ascii_letters from PyQt5.QtCore import pyqtSignal + +from aequilibrae import Project from aequilibrae.context import get_logger -from aequilibrae.project import Project -from aequilibrae.project.database_connection import database_connection from aequilibrae.utils.db_utils import commit_and_close -from aequilibrae.utils.worker_thread import WorkerThread -from string import ascii_letters +from aequilibrae.utils.interface.worker_thread import WorkerThread +from aequilibrae.project.database_connection import database_connection +from qaequilibrae.modules.common_tools.geodataframe_from_data_layer import geodataframe_from_layer logger = get_logger() class CreatesTranspoNetProcedure(WorkerThread): - ProgressValue = pyqtSignal(object) - ProgressText = pyqtSignal(object) - ProgressMaxValue = pyqtSignal(object) - finished_threaded_procedure = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parentThread, proj_folder, node_layer, node_fields, link_layer, link_fields): WorkerThread.__init__(self, parentThread) @@ -27,131 +27,95 @@ def __init__(self, parentThread, proj_folder, node_layer, node_fields, link_laye self.project: Project def doWork(self): - self.emit_messages(message=self.tr("Initializing project"), value=0, max_val=1) + self.signal.emit(["start", 5, self.tr("Initializing project")]) self.project = Project() self.project.new(self.proj_folder) + self.signal.emit(["update", 1, "Project created"]) # Add the required extra fields to the link layer - self.emit_messages(message=self.tr("Adding extra network data fields to database"), value=0, max_val=1) + self.signal.emit(["update", 2, self.tr("Adding extra fields to links layer")]) self.additional_fields_to_layers("links", self.link_layer, self.link_fields) + self.signal.emit(["update", 3, self.tr("Adding extra fields to nodes layer")]) self.additional_fields_to_layers("nodes", self.node_layer, self.node_fields) + self.signal.emit(["update", 4, self.tr("Building links layer")]) self.transfer_layer_features("links", self.link_layer, self.link_fields) + + self.signal.emit(["update", 5, self.tr("Renumbering nodes layer")]) self.renumber_nodes() - self.emit_messages(message=self.tr("Creating layer triggers"), value=0, max_val=1) - self.emit_messages(message=self.tr("Spatial indices"), value=0, max_val=1) - self.ProgressText.emit("DONE") + self.signal.emit(["finished"]) # Adds the non-standard fields to a layer def additional_fields_to_layers(self, table, layer, layer_fields): - with commit_and_close(database_connection("network")) as conn: + fields = layer.dataProvider().fields() + + data = self.project.network.links if table == "links" else self.project.network.nodes + + existing_fields = data.data.columns.tolist() - fields = layer.dataProvider().fields() - string_fields = [] - - field_names = conn.execute(f"PRAGMA table_info({table});").fetchall() - existing_fields = [f[1].lower() for f in field_names] - - for f in set(layer_fields.keys()): - if f.lower() in existing_fields: - continue - field = fields[layer_fields[f]] - field_length = field.length() - if not field.isNumeric(): - field_type = "char" - string_fields.append(f) - else: - field_type = "INTEGER" if "Int" in field.typeName() else "REAL" - try: - sql = "alter table " + table + " add column " + f + " " + field_type + "(" + str(field_length) + ")" - conn.execute(sql) - self.project.conn.commit() - except Exception as e: - logger.error(sql) - logger.error(e.args) - self.report.append(f"field {str(f)} could not be added") - - return string_fields + for f in set(layer_fields.keys()): + if f.lower() in existing_fields: + continue + field = fields[layer_fields[f]] + if not field.isNumeric(): + field_type = "TEXT" + else: + field_type = "INTEGER" if "integer" in field.typeName() else "REAL" + data.fields.add(f, "Field automatically added during project creation from layers", field_type) + + data.refresh_fields() def renumber_nodes(self): - max_val = self.node_layer.maximumValue(self.node_fields["node_id"]) + nodes = self.project.network.nodes.data + nodes = nodes[["node_id", "geometry"]] + nodes.columns = ["nid", "geometry"] + + gdf = geodataframe_from_layer(self.node_layer) + # We ensure that `is_centroid` is always integer + cnt = gdf.columns[self.node_fields["is_centroid"]] + gdf[cnt] = gdf[cnt].astype(np.int64) + + columns = [gdf.columns.tolist()[idx] for idx in self.node_fields.values() if idx >= 0] + columns.extend(["geometry"]) + gdf = gdf[columns] + + gdf = gdf.sjoin(nodes) + gdf.drop(columns={"geometry", "index_right"}, inplace=True) + + flds = list(self.node_fields.keys()) + setting = [f"{fld}=?" for fld in flds if fld != "node_id"] + sql_values = f'UPDATE nodes SET {",".join(setting)} WHERE node_id=?;' + + sql_id = "UPDATE nodes SET node_id=? WHERE node_id=?;" + with commit_and_close(database_connection("network")) as conn: - num_nodes = conn.execute("select max(node_id) from nodes").fetchone()[0] - max_val += num_nodes - logger.info(max_val) - - conn.execute("BEGIN;") - conn.execute("Update Nodes set node_id=node_id+?", [max_val]) - conn.execute("COMMIT;") - - self.emit_messages(message=self.tr("Transferring nodes"), value=0, max_val=self.node_layer.featureCount()) - - find_sql = """SELECT node_id - FROM nodes - WHERE geometry = GeomFromWKB(?, ?) AND - ROWID IN ( - SELECT ROWID FROM SpatialIndex WHERE f_table_name = 'nodes' AND - search_frame = GeomFromWKB(?, ?))""" - - flds = list(self.node_fields.keys()) - setting = [f"{fld}=?" for fld in flds] - update_sql = f'Update nodes set {",".join(setting)} where node_id=?' - - crs = int(self.node_layer.crs().authid().split(":")[1]) - for j, f in enumerate(self.node_layer.getFeatures()): - self.emit_messages(value=j + 1) - attrs = [ - self.convert_data(f.attributes()[val]) if val >= 0 else None for val in self.node_fields.values() - ] - wkb = f.geometry().asWkb().data() - node_id = conn.execute(find_sql, [wkb, crs, wkb, crs]).fetchall() - if not node_id: - continue - attrs.append(node_id) - logger.info([update_sql, attrs]) - conn.commit() + conn.executemany(sql_values, gdf.iloc[:, 1:].to_records(index=False)) + conn.executemany(sql_id, gdf.iloc[:, :1].join(gdf.iloc[:, -1:]).to_records(index=False)) def transfer_layer_features(self, table, layer, layer_fields): - self.emit_messages(message=self.tr("Transferring {}").format(table), value=0, max_val=layer.featureCount()) + # We ensure that `link_id`, `a_node`, `b_node`, and `direction` fields are always integers + gdf = geodataframe_from_layer(layer).infer_objects() - field_titles = ",".join(layer_fields.keys()) - all_modes = set() - all_link_types = set() - data_to_add = [] - sql = f"""INSERT INTO {table} ({field_titles} , geometry) - VALUES ({','.join(['?'] * len(layer_fields.keys()))},GeomFromWKB(?, ?))""" + all_modes = set("".join(gdf.iloc[:, layer_fields["modes"]].unique())) + all_link_types = gdf.iloc[:, layer_fields["link_type"]].unique() + self.__add_linktypes_and_modes(all_link_types, all_modes) crs = int(layer.crs().authid().split(":")[1]) + fields = [k for k, v in layer_fields.items() if v >= 0] + field_titles = ",".join(fields) + sql = f"""INSERT INTO {table} ({field_titles},geometry) + VALUES ({','.join(['?'] * len(fields))},GeomFromWKB(?, {crs}))""" - for j, f in enumerate(layer.getFeatures()): - self.emit_messages(value=j + 1) - attrs = [self.convert_data(f.attributes()[val]) if val >= 0 else None for val in layer_fields.values()] - attrs.extend([f.geometry().asWkb().data(), crs]) - data_to_add.append(attrs) + columns = [gdf.columns.tolist()[idx] for idx in layer_fields.values() if idx >= 0] + columns.extend(["geoms"]) - if table == "links": - all_modes.update(list(f.attributes()[layer_fields["modes"]])) - all_link_types.update([f.attributes()[layer_fields["link_type"]]]) + gdf = gdf[columns] with commit_and_close(database_connection("network")) as conn: - if table == "links": - self.__add_linktypes_and_modes(all_link_types, all_modes, conn) - - for data in data_to_add: - try: - conn.execute(sql, data) - except Exception as e: - logger.info(f"Failed inserting record {data[0]} for {table}") - logger.info(e.args) - logger.info([sql, data]) - if data[0]: - msg = f"feature with id {data[0]} could not be added to layer {table}" - else: - msg = f"feature with no node id present. It could not be added to layer {table}" - self.report.append(msg) - - def __add_linktypes_and_modes(self, all_link_types, all_modes, conn): + conn.executemany(sql, gdf.to_records(index=False)) + + def __add_linktypes_and_modes(self, all_link_types, all_modes): # We check if all modes exist modes = self.project.network.modes current_modes = list(modes.all_modes().keys()) @@ -163,7 +127,7 @@ def __add_linktypes_and_modes(self, all_link_types, all_modes, conn): modes.add(new_mode) new_mode.save() logger.info(f"{new_mode.description} --> ({md})") - conn.commit() + # We check if all link types exist link_types = self.project.network.link_types current_lt = [lt.link_type for lt in link_types.all_types().values()] @@ -174,18 +138,6 @@ def __add_linktypes_and_modes(self, all_link_types, all_modes, conn): new_link_type = link_types.new(letters[0]) letters = letters[1:] new_link_type.link_type = lt - new_link_type.description = "Link_type automatically added during project creation from layers" + new_link_type.description = "Link type automatically added during project creation from layers" new_link_type.save() logger.info(new_link_type.description + f" --> ({new_link_type.link_type})") - conn.commit() - - def convert_data(self, value): - return None if type(value) is None else value - - def emit_messages(self, message="", value=-1, max_val=-1): - if len(message) > 0: - self.ProgressText.emit(message) - if value >= 0: - self.ProgressValue.emit(value) - if max_val >= 0: - self.ProgressMaxValue.emit(max_val) diff --git a/qaequilibrae/modules/project_procedures/project_from_osm_dialog.py b/qaequilibrae/modules/project_procedures/project_from_osm_dialog.py index ee0e4adb..a2c2a4b3 100644 --- a/qaequilibrae/modules/project_procedures/project_from_osm_dialog.py +++ b/qaequilibrae/modules/project_procedures/project_from_osm_dialog.py @@ -119,7 +119,7 @@ def run(self): self.qgis_project.project = Project() self.qgis_project.project.new(self.output_path.text()) - self.qgis_project.project.network.netsignal.connect(self.signal_handler) + self.qgis_project.project.network.signal.connect(self.signal_handler) self.qgis_project.project.network.create_from_osm(box(*bbox)) @@ -136,13 +136,16 @@ def leave(self): dlg2.exec_() def signal_handler(self, val): - if val[0] == "Value": + if val[0] == "start": + self.progress_label.setText(val[2]) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + elif val[0] == "update": self.progressbar.setValue(val[1]) - elif val[0] == "maxValue": - self.progressbar.setRange(0, val[1]) - elif val[0] == "text": + elif val[0] == "set_text": self.progress_label.setText(val[1]) - elif val[0] == "finished_threaded_procedure": + self.progressbar.reset() + elif val[0] == "finished": # lines = self.qgis_project.project.network.count_links() # nodes = self.qgis_project.project.network.count_nodes() # self.report.append(reporter(f"{lines:,} links generated")) diff --git a/qaequilibrae/modules/public_transport_procedures/forms/gtfs_importer.ui b/qaequilibrae/modules/public_transport_procedures/forms/gtfs_importer.ui index 3de4a3e3..848f5c07 100644 --- a/qaequilibrae/modules/public_transport_procedures/forms/gtfs_importer.ui +++ b/qaequilibrae/modules/public_transport_procedures/forms/gtfs_importer.ui @@ -7,7 +7,7 @@ 0 0 446 - 505 + 433 @@ -30,22 +30,31 @@ + + false + - - + + 0 0 + + + 0 + 30 + + - Resetting Transit Tables + Execute Importer - - + + 0 @@ -54,17 +63,17 @@ - 0 - 30 + 130 + 0 - Execute Importer + Add to Existing Routes - - + + 0 @@ -76,17 +85,24 @@ - - + + 0 0 - - Qt::Horizontal - + + + Feeds to import + + + + 10 + + + @@ -108,52 +124,42 @@ - - + + 0 0 - - - Feeds to import - - - - 10 - - - + + Qt::Horizontal + - - + + 0 0 - - Qt::Horizontal + + Resetting Transit Tables - - + + 0 0 - - Rebuild active transport networks - - - true + + Qt::Horizontal @@ -179,22 +185,13 @@ - - - - - 0 - 0 - - - - - 130 - 0 - - + + - Add to Existing Routes + Allow map-match + + + false @@ -212,31 +209,9 @@ - - - - - - 0 - 0 - - - - - 0 - 30 - - - - Running - - - true - - - - - + + + 0 @@ -254,8 +229,8 @@ - - + + 0 @@ -276,25 +251,6 @@ - - - - - 0 - 0 - - - - - 0 - 30 - - - - 0 - - - @@ -303,7 +259,6 @@ but_add list_feeds - chb_active_net_rebuild rdo_clear but_execute diff --git a/qaequilibrae/modules/public_transport_procedures/gtfs_importer.py b/qaequilibrae/modules/public_transport_procedures/gtfs_importer.py index 3b5b95e6..1fc973a2 100644 --- a/qaequilibrae/modules/public_transport_procedures/gtfs_importer.py +++ b/qaequilibrae/modules/public_transport_procedures/gtfs_importer.py @@ -40,6 +40,20 @@ def __init__(self, qgis_project): self.setFixedHeight(380) self.items = [self.config_box, self.progress_box] + self.__transit_tables = [ + "agencies", + "fare_attributes", + "fare_rules", + "fare_zones", + "pattern_mapping", + "route_links", + "routes", + "stop_connectors", + "stops", + "trips", + "trips_schedule", + ] + def add_gtfs_feed(self): self._p = Transit(self.qgis_project.project) self.dlg2 = GTFSFeed(self.qgis_project, self._p) @@ -69,26 +83,14 @@ def execute_importer(self): if self.rdo_clear.isChecked() and self.is_pt_database: with commit_and_close(database_connection("transit")) as conn: - for table in [ - "agencies", - "fare_attributes", - "fare_rules", - "fare_zones", - "pattern_mapping", - "route_links", - "routes", - "stop_connectors", - "stops", - "trips", - "trips_schedule", - ]: + for table in self.__transit_tables: conn.execute(f"DELETE FROM {table};") for _, feed in enumerate(self.feeds): feed.signal.connect(self.signal_handler) - - feed.set_allow_map_match(True) - feed.doWork() + if self.check_allow_map_match.isChecked(): + feed.set_allow_map_match() + feed.execute_import() self.qgis_project.projectManager.removeTab(0) update_project_layers(self.qgis_project) @@ -96,17 +98,17 @@ def execute_importer(self): self.close() def signal_handler(self, val): - if len(val) == 1: - return - - bar = self.progressBar if val[1] == "master" else self.progressBar2 - lbl = self.lbl_progress if val[1] == "master" else self.lbl_progress2 if val[0] == "start": - lbl.setText(val[3]) - bar.setRange(0, val[2]) - bar.setValue(0) + self.progressbar.setValue(0) + self.progressbar.setMaximum(val[1]) + self.progress_label.setText(val[2]) elif val[0] == "update": - bar.setValue(val[2]) - if val[1] != "master" and bar.maximum() == val[2]: - self.progressBar.setValue(self.progressBar.value() + 1) + self.progressbar.setValue(val[1]) + self.progress_label.setText(val[2]) + elif val[0] == "set_text": + self.progressbar.reset() + self.progress_label.setText(val[1]) + elif val[0] == "finished": + self.progressbar.reset() + self.progress_label.clear() diff --git a/qaequilibrae/modules/public_transport_procedures/transit_supply_metrics.py b/qaequilibrae/modules/public_transport_procedures/transit_supply_metrics.py index 73fa6fc5..09b6decc 100644 --- a/qaequilibrae/modules/public_transport_procedures/transit_supply_metrics.py +++ b/qaequilibrae/modules/public_transport_procedures/transit_supply_metrics.py @@ -23,7 +23,9 @@ def __init__(self, path: None): patt_sql = """Select pattern_id, route_id, seated_capacity s_capacity, total_capacity t_capacity from routes""" - stop_sql = f"""Select stop_id, stop, name stop_name, agency_id, route_type, transit_zone zone_id from stops""" + stop_sql = ( + f"""Select stop_id, stop, name stop_name, agency_id, route_type, transit_fare_zone zone_id from stops""" + ) stop_pat_sql = """Select pattern_id, cast(from_stop as text) stop_id from route_links UNION ALL diff --git a/qaequilibrae/modules/routing_procedures/tsp_dialog.py b/qaequilibrae/modules/routing_procedures/tsp_dialog.py index 7dde1d91..52ddb083 100644 --- a/qaequilibrae/modules/routing_procedures/tsp_dialog.py +++ b/qaequilibrae/modules/routing_procedures/tsp_dialog.py @@ -104,11 +104,11 @@ def run(self): self.run_thread() def run_thread(self): - self.worker_thread.finished.connect(self.finished) + self.worker_thread.signal.connect(self.signal_handler) self.worker_thread.start() self.exec_() - def finished(self): + def signal_handler(self): ns = self.worker_thread.node_sequence if len(ns) < 2: return diff --git a/qaequilibrae/modules/routing_procedures/tsp_procedure.py b/qaequilibrae/modules/routing_procedures/tsp_procedure.py index b97bc836..5a4afe26 100644 --- a/qaequilibrae/modules/routing_procedures/tsp_procedure.py +++ b/qaequilibrae/modules/routing_procedures/tsp_procedure.py @@ -1,10 +1,10 @@ from PyQt5.QtCore import pyqtSignal from aequilibrae.paths import NetworkSkimming -from aequilibrae.utils.worker_thread import WorkerThread +from aequilibrae.utils.interface.worker_thread import WorkerThread class TSPProcedure(WorkerThread): - finished = pyqtSignal(object) + signal = pyqtSignal(object) def __init__(self, parentThread, graph, depot, vehicles): WorkerThread.__init__(self, parentThread) @@ -74,4 +74,4 @@ def distance_callback(from_index, to_index): self.node_sequence.append(p) plan_output += f" {p}\n" self.report.append(plan_output) - self.finished.emit("TSP") + self.signal.emit(["finished"]) diff --git a/qaequilibrae/requirements.txt b/qaequilibrae/requirements.txt index c6436a52..12c0dfb4 100644 --- a/qaequilibrae/requirements.txt +++ b/qaequilibrae/requirements.txt @@ -1,5 +1 @@ -pyarrow==15.0.2 -ortools -absl-py -immutabledict -protobuf \ No newline at end of file +ortools \ No newline at end of file diff --git a/test/data/NetworkPreparation/link.csv b/test/data/NetworkPreparation/link.csv index b965bf10..f27339d7 100644 --- a/test/data/NetworkPreparation/link.csv +++ b/test/data/NetworkPreparation/link.csv @@ -1,5 +1,5 @@ -link_id,a_node,b_node,direction,distance,modes,link_type,geometry -1,1,2,1,1,c,test,"LINESTRING (0 0, 1 0)" -2,2,3,1,1,c,test,"LINESTRING (1 0, 1 1)" -3,3,4,1,1,c,test,"LINESTRING (1 1, 0 1)" -4,4,1,1,1,c,test,"LINESTRING (0 1, 0 0)" +a_node,b_node,direction,distance,modes,link_type,name,geometry,link_id +11,12,1,1.0,a,test,first link,"LINESTRING (0 0, 1 0)",1 +12,13,1,1.0,r,test,second link,"LINESTRING (1 0, 1 1)",2 +13,14,0,1.0,x,test,third link,"LINESTRING (1 1, 0 1)",3 +14,11,-1,1.0,xrc,test,fourth link,"LINESTRING (0 1, 0 0)",4 diff --git a/test/data/NetworkPreparation/node.csv b/test/data/NetworkPreparation/node.csv index cb5d9b67..9acaa085 100644 --- a/test/data/NetworkPreparation/node.csv +++ b/test/data/NetworkPreparation/node.csv @@ -1,5 +1,5 @@ node_id,is_centroid,x,y -1,0,0,0 -2,0,1,0 -3,0,1,1 -4,0,0,1 +11,0,0,0 +12,1,1,0 +13,1,1,1 +14,0,0,1 diff --git a/test/data/SiouxFalls_project/synthetic_future_vector.aed b/test/data/SiouxFalls_project/synthetic_future_vector.aed deleted file mode 100644 index f641db43..00000000 Binary files a/test/data/SiouxFalls_project/synthetic_future_vector.aed and /dev/null differ diff --git a/test/data/SiouxFalls_project/synthetic_future_vector.parquet b/test/data/SiouxFalls_project/synthetic_future_vector.parquet new file mode 100644 index 00000000..f2ccbd24 Binary files /dev/null and b/test/data/SiouxFalls_project/synthetic_future_vector.parquet differ diff --git a/test/data/coquimbo_project/public_transport.sqlite b/test/data/coquimbo_project/public_transport.sqlite index 1f849bf4..c6ee1a9c 100644 Binary files a/test/data/coquimbo_project/public_transport.sqlite and b/test/data/coquimbo_project/public_transport.sqlite differ diff --git a/test/test_adds_centroid_connector.py b/test/test_add_centroid_connector.py similarity index 91% rename from test/test_adds_centroid_connector.py rename to test/test_add_centroid_connector.py index d8fc17c3..7c3752bd 100644 --- a/test/test_adds_centroid_connector.py +++ b/test/test_add_centroid_connector.py @@ -1,5 +1,4 @@ import pytest -from time import sleep from shapely.geometry import Point from aequilibrae.utils.db_utils import read_and_close from aequilibrae.project.database_connection import database_connection @@ -10,9 +9,11 @@ from qaequilibrae.modules.network.adds_connectors_dialog import AddConnectorsDialog -def test_add_connectors_from_zones(pt_no_feed): +@pytest.mark.parametrize("in_zone", [True, False]) +def test_add_connectors_from_zones(pt_no_feed, in_zone): dialog = AddConnectorsDialog(pt_no_feed) dialog.rdo_zone.setChecked(True) + dialog.chb_zone.setChecked(in_zone) dialog.lst_modes.setCurrentRow(1) dialog.lst_link_types.setCurrentRow(11) @@ -23,11 +24,6 @@ def test_add_connectors_from_zones(pt_no_feed): dialog.run() - sleep(2) - - dialog.project.network.links.refresh() - dialog.project.network.nodes.refresh() - with read_and_close(database_connection("network")) as conn: node_count = conn.execute("select count(node_id) from nodes where is_centroid=1").fetchone()[0] link_count = conn.execute("select count(name) from links where name like 'centroid connector%'").fetchone()[0] @@ -62,11 +58,6 @@ def test_add_connectors_from_network(pt_no_feed, node_id, radius, point): assert dialog.sb_radius.value() == radius - sleep(2) - - dialog.project.network.links.refresh() - dialog.project.network.nodes.refresh() - with read_and_close(database_connection("network")) as conn: node_count = conn.execute("select count(node_id) from nodes where is_centroid=1").fetchone()[0] link_count = conn.execute("select count(name) from links where name like 'centroid connector%'").fetchone()[0] @@ -113,16 +104,13 @@ def test_add_connectors_from_layer(pt_no_feed): dialog = AddConnectorsDialog(pt_no_feed) dialog.rdo_layer.setChecked(True) + dialog.set_fields() + dialog.lst_modes.setCurrentRow(1) dialog.lst_link_types.setCurrentRow(11) dialog.run() - sleep(2) - - dialog.project.network.links.refresh() - dialog.project.network.nodes.refresh() - with read_and_close(database_connection("network")) as conn: node_count = conn.execute("select count(node_id) from nodes where is_centroid=1").fetchone()[0] link_count = conn.execute("select count(name) from links where name like 'centroid connector%'").fetchone()[0] diff --git a/test/test_create_transponet.py b/test/test_create_transponet.py new file mode 100644 index 00000000..4bfb1573 --- /dev/null +++ b/test/test_create_transponet.py @@ -0,0 +1,149 @@ +from uuid import uuid4 +from os.path import join +from shutil import copytree + +from aequilibrae import Project +from qgis.PyQt import QtWidgets +from qgis.core import QgsProject, QgsVectorLayer +from qaequilibrae.modules.project_procedures.creates_transponet_dialog import CreatesTranspoNetDialog +from qaequilibrae.modules.project_procedures.creates_transponet_procedure import CreatesTranspoNetProcedure + + +def load_test_layer(path): + for file_name in ["link", "node"]: + csv_path = f"/{path}/{file_name}.csv" + + if file_name == "link": + uri = "file://{}?delimiter=,&crs=epsg:4326&wktField={}".format(csv_path, "geometry") + else: + uri = "file://{}?delimiter=,&crs=epsg:4326&xField={}&yField={}".format(csv_path, "x", "y") + + layer = QgsVectorLayer(uri, file_name, "delimitedtext") + + if not layer.isValid(): + print(f"{file_name} layer failed to load!") + else: + QgsProject.instance().addMapLayer(layer) + + +def test_dialog(ae, tmp_path): + path = join(tmp_path, uuid4().hex) + copytree("test/data/NetworkPreparation", path) + + load_test_layer(path) + + dialog = CreatesTranspoNetDialog(ae) + dialog.project_destination.setText(path) + + links_columns = [ + "link_id", + "a_node", + "b_node", + "direction", + "distance", + "modes", + "link_type", + "link_id", + "a_node", + "b_node", + "direction", + "distance", + "modes", + "link_type", + ] + nodes_columns = ["node_id", "is_centroid", "node_id", "is_centroid"] + + child = dialog.findChildren(QtWidgets.QComboBox) + links_chd = [] + nodes_chd = [] + for chd in child: + if chd.count() == 8: + links_chd.append(chd) + elif chd.count() == 4: + nodes_chd.append(chd) + + for idx, chd in enumerate(links_chd): + i = chd.findText(links_columns[idx]) + chd.setCurrentIndex(i) + + for idx, chd in enumerate(nodes_chd): + i = chd.findText(nodes_columns[idx]) + chd.setCurrentIndex(i) + + dialog.create_net() + + QgsProject.instance().removeAllMapLayers() + + # Test assertions + project = Project() + project.open(dialog.worker_thread.proj_folder) + + project_links = project.network.links.data + assert project_links.shape[0] == 4 + + project_nodes = project.network.nodes.data + assert project_nodes.shape[0] == 4 + assert project_nodes[project_nodes["is_centroid"] == 1].shape[0] == 2 + + link_types = project.network.link_types + assert "a" in link_types.all_types().keys() + + modes = project.network.modes + for mode in ["a", "r", "x"]: + assert mode in modes.all_modes().keys() + + +def test_procedure(ae, tmp_path): + path = join(tmp_path, uuid4().hex) + copytree("test/data/NetworkPreparation", path) + + load_test_layer(path) + + nodes = QgsProject.instance().mapLayersByName("node")[0] + links = QgsProject.instance().mapLayersByName("link")[0] + + links_fields = { + "link_id": 7, + "a_node": 0, + "b_node": 1, + "direction": 2, + "distance": 3, + "modes": 4, + "link_type": 5, + "name": -1, + "cycleway": -1, + "cycleway_right": -1, + "cycleway_left": -1, + "busway": -1, + "busway_right": -1, + "busway_left": -1, + "lanes_ab": -1, + "lanes_ba": -1, + "capacity_ab": -1, + "capacity_ba": -1, + "speed_ab": -1, + "speed_ba": -1, + } + nodes_fields = {"node_id": 0, "is_centroid": 1} + + proj_folder = join(path, "project") + parent = CreatesTranspoNetDialog(ae) + dialog = CreatesTranspoNetProcedure(parent, proj_folder, nodes, nodes_fields, links, links_fields) + dialog.doWork() + + project = Project() + project.open(proj_folder) + + project_links = project.network.links.data + assert project_links.shape[0] == 4 + + project_nodes = project.network.nodes.data + assert project_nodes.shape[0] == 4 + assert project_nodes[project_nodes["is_centroid"] == 1].shape[0] == 2 + + link_types = project.network.link_types + assert "a" in link_types.all_types().keys() + + modes = project.network.modes + for mode in ["a", "r", "x"]: + assert mode in modes.all_modes().keys() diff --git a/test/test_display_aequilibrae_formats.py b/test/test_display_aequilibrae_formats.py index 1ca6946d..6146fdea 100644 --- a/test/test_display_aequilibrae_formats.py +++ b/test/test_display_aequilibrae_formats.py @@ -19,7 +19,7 @@ def test_display_data_no_path(ae, mocker): @pytest.mark.parametrize("has_project", [True, False]) -@pytest.mark.parametrize("path", ("matrices/demand.aem", "matrices/SiouxFalls.omx", "synthetic_future_vector.aed")) +@pytest.mark.parametrize("path", ("matrices/demand.aem", "matrices/SiouxFalls.omx")) def test_display_data_with_path(tmpdir, ae_with_project, mocker, has_project, path): file_path = f"test/data/SiouxFalls_project/{path}" name, extension = path.split(".") @@ -35,15 +35,9 @@ def test_display_data_with_path(tmpdir, ae_with_project, mocker, has_project, pa dialog.export() dialog.exit_procedure() - if extension in ["aem", "omx"]: - assert np.sum(dialog.data_to_show.matrix["matrix"]) == 360600 - assert "matrix" in dialog.list_cores - assert "taz" in dialog.list_indices - elif extension == "aed": - assert dialog.list_cores == ["origins", "destinations"] - assert sum(dialog.data_to_show.data["origins"]) == 436740 - assert sum(dialog.data_to_show.data["destinations"]) == 436740 - + assert np.sum(dialog.data_to_show.matrix["matrix"]) == 360600 + assert "matrix" in dialog.list_cores + assert "taz" in dialog.list_indices assert dialog.error is None assert dialog.data_type == extension.upper() assert os.path.isfile(f"{tmpdir}/{name}.csv") diff --git a/test/test_distribution_procedures.py b/test/test_distribution_procedures.py index 5aa2b63d..828eccff 100644 --- a/test/test_distribution_procedures.py +++ b/test/test_distribution_procedures.py @@ -1,14 +1,16 @@ from os.path import isfile, splitext, basename import numpy as np +import pandas as pd import openmatrix as omx import pytest from qgis.core import QgsProject -from aequilibrae.matrix import AequilibraeData, AequilibraeMatrix +from aequilibrae.matrix import AequilibraeMatrix +from qaequilibrae.modules.matrix_procedures.load_dataset_dialog import LoadDatasetDialog from qaequilibrae.modules.distribution_procedures.distribution_models_dialog import DistributionModelsDialog -@pytest.mark.parametrize("method", ("dataset", "open_layer")) +@pytest.mark.parametrize("method", ["csv", "parquet", "open layer"]) def test_ipf(ae_with_project, folder_path, mocker, method, load_synthetic_future_vector): file_path = f"{folder_path}/demand_ipf_D.aem" @@ -19,45 +21,37 @@ def test_ipf(ae_with_project, folder_path, mocker, method, load_synthetic_future dialog = DistributionModelsDialog(ae_with_project, mode="ipf") - if method == "dataset": - dataset_path = "test/data/SiouxFalls_project/synthetic_future_vector.aed" - dataset = AequilibraeData() - dataset.load(dataset_path) + if method == "csv": + dataset_path = "test/data/SiouxFalls_project/synthetic_future_vector.csv" + dataset = pd.read_csv(dataset_path) + elif method == "parquet": + dataset_path = "test/data/SiouxFalls_project/synthetic_future_vector.parquet" + dataset = pd.read_parquet(dataset_path) + elif method == "open layer": + layer = QgsProject.instance().mapLayersByName("synthetic_future_vector")[0] + dialog.iface.setActiveLayer(layer) + if method in ["csv", "parquet"]: data_name = splitext(basename(dataset_path))[0] - dialog.datasets[data_name] = dataset + + dialog.cob_index.setCurrentText("index") + dialog._has_idx = False else: - layer = QgsProject.instance().mapLayersByName("synthetic_future_vector")[0] - dialog.iface.setActiveLayer(layer) - idx = [] - origin = [] - destination = [] - for feat in layer.getFeatures(): - f = feat.attributes() - idx.append(f[0]) - origin.append(f[1]) - destination.append(f[2]) - args = { - "entries": 24, - "field_names": ["origins", "destinations"], - "data_types": [np.float64, np.float64], - "file_path": f"{folder_path}/synthetic_future_vector_CSV.aed", - } - - dataset = AequilibraeData() - dataset.create_empty(**args) - - dataset.origins[:] = origin[:] - dataset.destinations[:] = destination[:] - dataset.index[:] = idx[:] - - dialog.datasets["synthetic_future_vector_CSV"] = dataset + dataset = LoadDatasetDialog(dialog.iface) + dataset.radio_layer.setChecked(True) + dataset.size_it_accordingly(True) + dataset.cob_index_field.setCurrentText("index") + dataset.layer = layer + dataset.load_the_vector() + + dialog.datasets["synthetic_future_vector"] = dataset.dataset + dialog._has_idx = True + dialog.cob_index.clear() + dialog.cob_index.setEnabled(False) dialog.outfile = file_path - - dialog.load_comboboxes(dialog.datasets.keys(), dialog.cob_prod_data) - dialog.load_comboboxes(dialog.datasets.keys(), dialog.cob_atra_data) + dialog.load_comboboxes(dialog.datasets.keys(), dialog.cob_data) temp = list(dialog.matrices["name"]) demand_idx = temp.index("demand.aem") @@ -134,36 +128,34 @@ def test_calibrate_gravity(run_assignment, method, folder_path, mocker): assert "function: POWER" in file_text -@pytest.mark.parametrize(("method", "ext"), [("negative", "X"), ("power", "Y"), ("gamma", "Z")]) -def test_apply_gravity(ae_with_project, method, ext, folder_path, mocker): +@pytest.mark.parametrize("method", ["negative", "power", "gamma"]) +def test_apply_gravity(ae_with_project, method, folder_path, mocker): - file_path = f"{folder_path}/matrices/ADJ-TrafficAssignment_DP_{ext}.omx" + file_path = f"{folder_path}/matrices/ADJ-TrafficAssignment_DP.omx" mocker.patch( "qaequilibrae.modules.distribution_procedures.distribution_models_dialog.DistributionModelsDialog.browse_outfile", return_value=file_path, ) - dataset_name = "test/data/SiouxFalls_project/synthetic_future_vector.aed" - - dataset = AequilibraeData() - dataset.load(dataset_name) + dataset_path = "test/data/SiouxFalls_project/synthetic_future_vector.csv" + dataset = pd.read_csv(dataset_path) - data_name = splitext(basename(dataset_name))[0] + data_name = splitext(basename(dataset_path))[0] dialog = DistributionModelsDialog(ae_with_project, mode="apply") + dialog._has_idx = False dialog.datasets[data_name] = dataset - dialog.load_comboboxes(dialog.datasets.keys(), dialog.cob_prod_data) - dialog.load_comboboxes(dialog.datasets.keys(), dialog.cob_atra_data) + dialog.load_comboboxes(dialog.datasets.keys(), dialog.cob_data) temp = list(dialog.matrices["name"]) imped_idx = temp.index(f"trafficassignment_dp_x_car_omx") dialog.cob_imped_mat.setCurrentIndex(imped_idx) dialog.cob_imped_field.setCurrentText("free_flow_time_final") - dialog.cob_prod_data.setCurrentText("synthetic_future_vector") + dialog.cob_data.setCurrentText("synthetic_future_vector") + dialog.cob_index.setCurrentText("index") dialog.cob_prod_field.setCurrentText("origins") - dialog.cob_atra_data.setCurrentText("synthetic_future_vector") dialog.cob_atra_field.setCurrentText("destinations") if method == "negative": diff --git a/test/test_gis_desire_lines.py b/test/test_gis_desire_lines.py index 1ca0e9b8..4a3685ab 100644 --- a/test/test_gis_desire_lines.py +++ b/test/test_gis_desire_lines.py @@ -32,7 +32,7 @@ def test_click_create_with_layers(ae_with_project, qtbot, timeoutDetector, load_ assert len(exceptions) == 0 # default is delaunay assert dialog.progress_label.text() == "Building resulting layer" # Last displayed line - assert dialog.progressbar.value() == 61 # This number should match something in the SiouxFalls data + assert dialog.progressbar.value() == 62 # This number should match the number of edges in the SiouxFalls data # test that something cool happened on the map? @@ -47,7 +47,7 @@ def test_click_create_with_layers_desired_selected(ae_with_project, qtbot, timeo qtbot.mouseClick(dialog.create_dl, Qt.LeftButton) assert len(exceptions) == 0 assert dialog.progress_label.text() == "Creating Desire Lines" # Last displayed line - assert dialog.progressbar.value() == 275 # This number should match something in the SiouxFalls data + assert dialog.progressbar.value() == 276 # This number should match the number of edges in the SiouxFalls data # test that something cool happened on the map? diff --git a/test/test_impedance_matrix.py b/test/test_impedance_matrix.py new file mode 100644 index 00000000..12a75b9f --- /dev/null +++ b/test/test_impedance_matrix.py @@ -0,0 +1,14 @@ +from qaequilibrae.modules.paths_procedures.impedance_matrix_dialog import ImpedanceMatrixDialog + + +def test_import_impedance_matrix(ae_with_project): + dialog = ImpedanceMatrixDialog(ae_with_project) + + dialog.block_paths.setChecked(False) + dialog.line_matrix.setText("imped_matrix_car") + dialog.available_skims_table.selectRow(4) # add free_flow_time + dialog.append_to_list() + dialog.available_skims_table.selectRow(7) # add distance + dialog.append_to_list() + + dialog.run_skimming() diff --git a/test/test_load_dataset.py b/test/test_load_dataset.py index caaed924..3aa2e66e 100644 --- a/test/test_load_dataset.py +++ b/test/test_load_dataset.py @@ -1,49 +1,37 @@ import pytest -import numpy as np -import qgis +from qgis.core import QgsProject from qaequilibrae.modules.matrix_procedures.load_dataset_dialog import LoadDatasetDialog -from qaequilibrae.modules.matrix_procedures.load_dataset_class import LoadDataset -@pytest.mark.parametrize("method", ["aequilibrae data", "open layer"]) -def test_load_dialog(ae_with_project, method, folder_path, load_synthetic_future_vector): +@pytest.mark.parametrize("method", ["csv", "parquet", "open layer"]) +def test_load_dialog(ae_with_project, method, folder_path, load_synthetic_future_vector, timeoutDetector): dialog = LoadDatasetDialog(ae_with_project) dialog.path = folder_path - if method == "aequilibrae data": - dialog.radio_aequilibrae.setChecked(True) + if method in ["csv", "parquet"]: dialog.load_fields_to_combo_boxes() dialog.cob_data_layer.setCurrentText("synthetic_future_vector") - out_name = f"{folder_path}/synthetic_future_vector.aed" + out_name = f"{folder_path}/synthetic_future_vector.{method}" dialog.load_with_file_name(out_name) - assert dialog.selected_fields == ["index", "origins", "destinations"] assert dialog.worker_thread is None - arr = dialog.dataset.data.tolist() - else: - dialog.radio_layer_matrix.setChecked(True) - dialog.load_fields_to_combo_boxes() + elif method == "open layer": + layer = QgsProject.instance().mapLayersByName("synthetic_future_vector")[0] + + dialog.layer = layer + dialog.radio_layer.setChecked(True) + + dialog.size_it_accordingly(True) + dialog.cob_index_field.setCurrentText("index") + dialog.output_name = f"{folder_path}/synthetic_future_vector.csv" dialog.cob_data_layer.setCurrentText("synthetic_future_vector") dialog.single_use = True - dialog.output_name = f"{folder_path}/synthetic_future_vector_TEST.aed" - dialog.set_output_name() - dialog.selected_fields.remove("index") - dialog.worker_thread = LoadDataset( - qgis.utils.iface.mainWindow(), - layer=dialog.layer, - index_field="index", - fields=dialog.selected_fields, - file_name=dialog.output_name, - ) - dialog.worker_thread.doWork() - - assert dialog.selected_fields == ["origins", "destinations"] - assert dialog.dataset is None - arr = dialog.worker_thread.output.data.tolist() - - assert len(arr) == 24 - assert np.sum(arr, axis=0)[1] == np.sum(arr, axis=0)[2] > 0 + dialog.load_the_vector() + + assert dialog.selected_fields == ["index", "origins", "destinations"] + assert dialog.dataset.shape[0] == 24 + assert (dialog.dataset.sum(axis=0)["origins"] == dialog.dataset.sum(axis=0)["destinations"]) > 0 diff --git a/test/test_load_matrix.py b/test/test_load_matrix.py index 8ba6023a..aa89f6fe 100644 --- a/test/test_load_matrix.py +++ b/test/test_load_matrix.py @@ -1,5 +1,5 @@ -from PyQt5.QtCore import QTimer, QVariant import numpy as np +from PyQt5.QtCore import QTimer, QVariant from qgis.core import QgsProject, QgsVectorLayer, QgsField, QgsFeature from qaequilibrae.modules.matrix_procedures.load_matrix_dialog import LoadMatrixDialog @@ -61,9 +61,9 @@ def test_save_matrix(ae_with_project, folder_path): dialog.field_to.setCurrentText("D") dialog.field_cells.setCurrentText("Ton") dialog.has_errors() + dialog.worker_thread.signal.connect(dialog.signal_handler) dialog.worker_thread.doWork() dialog.worker_thread.report = None - dialog.finished_threaded_procedure("LOADED-MATRIX") dialog.build_worker_thread() dialog.worker_thread.doWork() diff --git a/test/test_load_project_data.py b/test/test_load_project_data.py index b36e1d30..e6fc4632 100644 --- a/test/test_load_project_data.py +++ b/test/test_load_project_data.py @@ -18,6 +18,7 @@ def test_no_project(ae, mocker, qtbot): dialog.close() +# TODO: Re-write the tests - they're really time consuming @pytest.mark.parametrize("button_clicked", [True, False]) def test_project(run_assignment, mocker, qtbot, button_clicked): proj = run_assignment diff --git a/test/test_network_preparation.py b/test/test_network_preparation.py index 87535cd1..1000ccdb 100644 --- a/test/test_network_preparation.py +++ b/test/test_network_preparation.py @@ -58,8 +58,8 @@ def test_prepare_network(tmp_path, ae, is_node): node_start=int(dialog.np_node_start.text()), ) + dialog.worker_thread.signal.connect(dialog.signal_handler) dialog.worker_thread.doWork() - dialog.job_finished_from_thread("DONE") all_layers = [layer.name() for layer in QgsProject.instance().mapLayers().values()] assert "net_links" in all_layers diff --git a/test/test_qaequilibrae_menu_with_project.py b/test/test_qaequilibrae_menu_with_project.py index c2c81645..df9341a2 100644 --- a/test/test_qaequilibrae_menu_with_project.py +++ b/test/test_qaequilibrae_menu_with_project.py @@ -1,4 +1,3 @@ -import pytest from PyQt5.QtCore import QTimer from PyQt5.QtWidgets import QApplication diff --git a/test/test_traffic_assignment.py b/test/test_traffic_assignment.py index 13937ece..a241d3ea 100644 --- a/test/test_traffic_assignment.py +++ b/test/test_traffic_assignment.py @@ -50,11 +50,16 @@ def test_single_class(ae_with_project, qtbot): dialog.cob_capacity.setCurrentText("capacity") dialog.cob_ffttime.setCurrentText("free_flow_time") dialog.cb_choose_algorithm.setCurrentText("bfw") - dialog.max_iter.setText("500") + dialog.max_iter.setText("25") dialog.rel_gap.setText("0.001") dialog.run() + with pytest.raises(ValueError): + dialog.produce_all_outputs() + + dialog.close() + pth = Path(dialog.project.project_base_path) results = pth / "results_database.sqlite" assert isfile(results) @@ -92,7 +97,7 @@ def test_single_class(ae_with_project, qtbot): in file_text ) assert "INFO ; Traffic Assignment specification" in file_text - assert "{{'VDF parameters': {{'alpha': 0.15, 'beta': 4.0}}, 'VDF function': 'bpr', 'Number of cores': {}, 'Capacity field': 'capacity', 'Time field': 'free_flow_time', 'Algorithm': 'bfw', 'Maximum iterations': 30, 'Target RGAP': 0.001}}".format( + assert "{{'VDF parameters': {{'alpha': 0.15, 'beta': 4.0}}, 'VDF function': 'bpr', 'Number of cores': {}, 'Capacity field': 'capacity', 'Time field': 'free_flow_time', 'Algorithm': 'bfw', 'Maximum iterations': 25, 'Target RGAP': 0.001}}".format( num_cores ) @@ -167,10 +172,13 @@ def test_multiclass(ae_with_project, qtbot): with pytest.raises(ValueError): dialog.produce_all_outputs() + dialog.close() + # Assert we have a non-null result and that results are actually stored in the file pth = Path(dialog.project.project_base_path) results = pth / "results_database.sqlite" assert isfile(results) + con = sqlite3.connect(results) assert con.execute(f"SELECT ROUND(SUM(PCE_tot), 4) FROM {test_name}").fetchone()[0] > 0 assert con.execute(f"SELECT ROUND(SUM(car_tot), 4) FROM {test_name}").fetchone()[0] > 0 @@ -210,3 +218,49 @@ def test_multiclass(ae_with_project, qtbot): assert "{{'VDF parameters': {{'alpha': 0.15, 'beta': 4.0}}, 'VDF function': 'bpr', 'Number of cores': {}, 'Capacity field': 'capacity', 'Time field': 'free_flow_time', 'Algorithm': 'bfw', 'Maximum iterations': 20, 'Target RGAP': 0.001}}".format( num_cores ) + + +def test_all_or_nothing(ae_with_project, qtbot): + dialog = TrafficAssignmentDialog(ae_with_project) + + test_name = f"TestTrafficAssignment_AON_{uuid4().hex[:6]}" + dialog.output_scenario_name.setText(test_name) + dialog.cob_matrices.setCurrentText("demand.aem") + + dialog.tbl_core_list.selectRow(0) + dialog.cob_mode_for_class.setCurrentIndex(0) + dialog.ln_class_name.setText("car") + dialog.pce_setter.setValue(1.0) + dialog.chb_check_centroids.setChecked(False) + qtbot.mouseClick(dialog.but_add_class, Qt.LeftButton) + + # Skimming + dialog.cob_skims_available.setCurrentText("free_flow_time") + qtbot.mouseClick(dialog.but_add_skim, Qt.LeftButton) + dialog.cob_skims_available.setCurrentText("distance") + qtbot.mouseClick(dialog.but_add_skim, Qt.LeftButton) + + dialog.tbl_vdf_parameters.cellWidget(0, 1).setText("0.15") + dialog.tbl_vdf_parameters.cellWidget(1, 1).setText("4.0") + dialog.cob_vdf.setCurrentText("BPR") + dialog.cob_capacity.setCurrentText("capacity") + dialog.cob_ffttime.setCurrentText("free_flow_time") + dialog.cb_choose_algorithm.setCurrentText("all-or-nothing") + + dialog.run() + + with pytest.raises(ValueError): + dialog.produce_all_outputs() + + dialog.close() + + pth = Path(dialog.project.project_base_path) + results = pth / "results_database.sqlite" + assert isfile(results) + + # Assert we have a non-null result and that results are actually stored in the file + con = sqlite3.connect(results) + assert con.execute(f"SELECT ROUND(SUM(matrix_tot), 4) FROM {test_name}").fetchone()[0] == 885_300.0 + + skims = pth / f"matrices/{test_name}_car.omx" + assert isfile(skims)