From 3079c8a03d11319d1cffb8c03ea211a7783e67c3 Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Mon, 18 Nov 2024 16:39:08 +0100 Subject: [PATCH 01/10] Import let module --- .../components/disclaimer-dialog/disclaimer-dialog.component.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/client-shared/src/lib/components/disclaimer-dialog/disclaimer-dialog.component.ts b/libs/client-shared/src/lib/components/disclaimer-dialog/disclaimer-dialog.component.ts index 4ddf9d7a..5c93e1bb 100644 --- a/libs/client-shared/src/lib/components/disclaimer-dialog/disclaimer-dialog.component.ts +++ b/libs/client-shared/src/lib/components/disclaimer-dialog/disclaimer-dialog.component.ts @@ -3,6 +3,7 @@ import { Component, inject } from '@angular/core'; import { MatDialogActions, MatDialogContent, MatDialogRef } from '@angular/material/dialog'; import { MatDivider } from '@angular/material/divider'; import { TranslateModule, TranslateService } from '@ngx-translate/core'; +import { LetModule } from '@rx-angular/template/let'; import { map } from 'rxjs'; import { ButtonComponent } from '../../components/button'; import { LanguageSelectorComponent } from '../../components/language-selector'; @@ -23,6 +24,7 @@ const LEGAL_BASE_URL = 'https://www.swissgeol.ch/datenschutz'; ButtonComponent, AsyncPipe, TranslateModule, + LetModule, ], }) export class DisclaimerDialogComponent { From 791a3572dae4cbfd99cc8f8ed77a4b3cfd73c9ee Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Tue, 19 Nov 2024 09:18:57 +0100 Subject: [PATCH 02/10] Reset favoritesOnly and fix display of heatmap --- .../src/lib/state/asset-search/asset-search.effects.ts | 5 +++-- .../src/lib/state/asset-search/asset-search.selector.ts | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts b/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts index 0a4be95e..f7a37778 100644 --- a/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts +++ b/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts @@ -79,7 +79,8 @@ export class AssetSearchEffects { return !deepEqual(params.query, storeQuery) || params.assetId != storeDetail?.assetId; }), map(([params, storeQuery, storeDetail, isMostRecentPage]) => { - const hasNoQueryParams = Object.values(params.query).every((v) => v == null || v == false); + const { favoritesOnly, ...query } = params.query; + const hasNoQueryParams = Object.values(query).every((v) => v == null); const hasQueryOrAssetIdInStore = !Object.values(storeQuery).every((v) => v == null || v == false) || storeDetail; // We only use the values from the store if all of the below are true: @@ -87,7 +88,7 @@ export class AssetSearchEffects { // - There are query or assetId values in the store // - The current page is the most recent page if (hasNoQueryParams && hasQueryOrAssetIdInStore && isMostRecentPage) { - return actions.runCombinedSearch({ query: storeQuery, assetId: storeDetail?.assetId }); + return actions.runCombinedSearch({ query: { ...storeQuery, favoritesOnly }, assetId: storeDetail?.assetId }); } return actions.runCombinedSearch({ query: params.query, assetId: params.assetId }); }) diff --git a/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts b/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts index 62e13b36..03a02c80 100644 --- a/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts +++ b/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts @@ -62,7 +62,8 @@ export const selectStudies = createSelector(assetSearchFeature, (state) => state export const selectHasDefaultFilters = createSelector( assetSearchFeature, - ({ query, currentAsset }) => currentAsset == null && Object.values(query).every((value) => value === undefined) + ({ query, currentAsset }) => + currentAsset == null && Object.values(query).every((value) => value === undefined || value == false) ); export const selectCurrentAssetDetailVM = createSelector( From de4bf440ad0b7d8b740671ebfb00b2e13d99089f Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Tue, 19 Nov 2024 12:41:08 +0100 Subject: [PATCH 03/10] Remove deletions of assetKindItems and assetFormatItem --- .../src/features/asset-edit/asset-edit.repo.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/apps/server-asset-sg/src/features/asset-edit/asset-edit.repo.ts b/apps/server-asset-sg/src/features/asset-edit/asset-edit.repo.ts index bf14a32d..a9d875b5 100644 --- a/apps/server-asset-sg/src/features/asset-edit/asset-edit.repo.ts +++ b/apps/server-asset-sg/src/features/asset-edit/asset-edit.repo.ts @@ -284,18 +284,6 @@ export class AssetEditRepo implements Repo Date: Tue, 19 Nov 2024 13:38:49 +0100 Subject: [PATCH 04/10] Only allow one concurrent pipeline run for different publish pipelines --- .github/workflows/publish-edge.yml | 3 +++ .github/workflows/publish-rc.yml | 3 +++ .github/workflows/release.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/publish-edge.yml b/.github/workflows/publish-edge.yml index 5b4c48ae..d51713e0 100644 --- a/.github/workflows/publish-edge.yml +++ b/.github/workflows/publish-edge.yml @@ -19,6 +19,9 @@ on: description: "Tag the commit and published image with `edge`." default: true +concurrency: + group: "publish-edge" + permissions: write-all env: diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 610b02b7..a8cab828 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -15,6 +15,9 @@ on: required: false default: "edge" +concurrency: + group: "publish-rc" + permissions: write-all env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 64456ce1..34331106 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,9 @@ on: Assign the `release-candidate` tag to this release. default: true +concurrency: + group: "release" + permissions: write-all env: From 164d1a80cb394905b02b53c461dbcd1523a9abf8 Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Tue, 19 Nov 2024 15:23:52 +0100 Subject: [PATCH 05/10] Reset pre-release number on first minor release --- .github/scripts/find-version.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/find-version.js b/.github/scripts/find-version.js index f549771c..fc2186bd 100644 --- a/.github/scripts/find-version.js +++ b/.github/scripts/find-version.js @@ -10,6 +10,7 @@ const findNextVersion = (tags, branch) => { if (version.preRelease == null || version.patch !== 0) { version.minor += 1; version.patch = 0; + version.preRelease = null; } } else { // It's a patch. From bdebde7ee769b87a4c5733ab2f80a047698b7a01 Mon Sep 17 00:00:00 2001 From: Jannic Veith Date: Tue, 19 Nov 2024 16:25:04 +0100 Subject: [PATCH 06/10] Add script to import geometries --- scripts/2024-10-01_import-new-geometries/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/2024-10-01_import-new-geometries/main.py b/scripts/2024-10-01_import-new-geometries/main.py index b5f3d904..159114c4 100644 --- a/scripts/2024-10-01_import-new-geometries/main.py +++ b/scripts/2024-10-01_import-new-geometries/main.py @@ -5,6 +5,8 @@ from geopandas import GeoDataFrame import os +from shapely import Polygon + # The following parameters need to be adjusted to the correct values: filedir = '' filename = 'geometries.gpkg' @@ -30,6 +32,11 @@ def write_insert_statements(f: TextIO, gdf: GeoDataFrame, geom_type: str): f.write( f"INSERT INTO study_{geom_type} (asset_id, geom_quality_item_code, geom) VALUES ((SELECT asset_id FROM asset WHERE sgs_id = {row['IDSGS_neu']}), 'revised', ST_GeomFromText('{row['geometry'].wkt}', 2056));\n") +def remove_interior_polygons(gdf: GeoDataFrame): + '''Remove interior polygons from a GeoDataFrame of polygons.''' + gdf['geometry'] = gdf['geometry'].apply(lambda x: Polygon(x.exterior.coords)) + return gdf + def main(): path = os.path.join(filedir, filename) gdf_points = read_gdf(file=path, layer='backup_20240829__points', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict) @@ -37,6 +44,7 @@ def main(): gdf_polygons = read_gdf(file=path, layer='backup_20240829__polygons', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict) gdf_lines = gdf_lines.explode() # explode multi-linestrings to linestrings gdf_polygons = gdf_polygons.explode() # explode multi-polygons to polygons + gdf_polygons = remove_interior_polygons(gdf_polygons) # remove holes from polygons gdf_points = gdf_points[~gdf_points['geometry'].is_empty] # remove empty geometries sgs_ids = np.unique(np.concatenate( From 1ea82c9e7b198a00be7417476adfd7e3f3b0b3ab Mon Sep 17 00:00:00 2001 From: Jannic Veith Date: Tue, 19 Nov 2024 16:30:19 +0100 Subject: [PATCH 07/10] Add script to simplify geometries --- .editorconfig | 3 + .../2024-10-01_import-new-geometries/main.py | 56 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.editorconfig b/.editorconfig index 819d984b..f3c831fc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,3 +13,6 @@ max_line_length = 120 [*.md] max_line_length = off trim_trailing_whitespace = false + +[*.py] +indent_size = 4 diff --git a/scripts/2024-10-01_import-new-geometries/main.py b/scripts/2024-10-01_import-new-geometries/main.py index 159114c4..2f3b038d 100644 --- a/scripts/2024-10-01_import-new-geometries/main.py +++ b/scripts/2024-10-01_import-new-geometries/main.py @@ -7,11 +7,14 @@ from shapely import Polygon +verbose = False + # The following parameters need to be adjusted to the correct values: filedir = '' filename = 'geometries.gpkg' output_filename = 'import_geometries.sql' output_sgsids_filename = 'sgsids.txt' +output_sgsids_simplified_filename = 'sgsids_simplified.txt' # The following sgsids are not in the database, they were manually translated to the probably correct sgsid: sgsid_translation_dict = {} @@ -20,7 +23,7 @@ allowed_bemerkung = ['erraten', 'erraten_B', 'neu', 'neu (?)', 'unsich kein B'] def read_gdf(file: str, layer: str, allowed_bemerkung: list, translation_dict: dict): - '''Read a geopandas dataframe from a file and layer, filter out entries with Bemerkung not in allowed_bemerkung and translate the sgsid according to translationDict.''' + """Read a geopandas dataframe from a file and layer, filter out entries with Bemerkung not in allowed_bemerkung and translate the sgsid according to translationDict.""" gdf = gpd.read_file(file, layer=layer) print(gdf['Bemerkung'].value_counts()) gdf = gdf[(gdf['Bemerkung'].isin(allowed_bemerkung))] @@ -37,13 +40,61 @@ def remove_interior_polygons(gdf: GeoDataFrame): gdf['geometry'] = gdf['geometry'].apply(lambda x: Polygon(x.exterior.coords)) return gdf +simplified_sgsids = [] +simplified_rows = [0] + +def simplify_geometry(gdf: GeoDataFrame): + """ Simplify the geometries of large Polygons. + Use different tolerances depending on the number of points in the geometry. + 1000 - 10000 points: 100 + 10000 - 50000 points: 1000 + > 50000 points: 2000 + """ + def simplify_by(row, tolerance): + num_points = count_points(row['geometry']) + simplified = row['geometry'].simplify(tolerance, preserve_topology=True) + simplified_sgsids.append(row['IDSGS_neu']) + simplified_rows[0] += num_points - count_points(simplified) + if verbose: + print( + f" ||| SGSID Simplified {num_points} - {len(simplified.exterior.coords)} = {num_points - len(simplified.exterior.coords)}") + return simplified + + def count_points(geom): + if geom.geom_type == 'Polygon': + return len(geom.exterior.coords) + elif geom.geom_type == 'MultiPolygon': + return sum([count_points(p) for p in geom.geoms]) + else: + return 0 + + def simplify(row): + if row['geometry'].geom_type in ('Polygon', 'MultiPolygon'): + num_points = count_points(row['geometry']) + else: + return row['geometry'] + + if num_points < 1000: + return row['geometry'] + elif num_points < 10000: + return simplify_by(row, 100) + elif num_points < 50000: + return simplify_by(row, 1000) + else: + return simplify_by(row, 2000) + + gdf['geometry'] = gdf.apply(simplify, axis=1) + print(f"Simplified {len(simplified_sgsids)} geometries, removed {simplified_rows[0]} points.") + return gdf + def main(): path = os.path.join(filedir, filename) gdf_points = read_gdf(file=path, layer='backup_20240829__points', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict) gdf_lines = read_gdf(file=path, layer='backup_20240829__lines', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict) gdf_polygons = read_gdf(file=path, layer='backup_20240829__polygons', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict) gdf_lines = gdf_lines.explode() # explode multi-linestrings to linestrings - gdf_polygons = gdf_polygons.explode() # explode multi-polygons to polygons + gdf_polygons = simplify_geometry(gdf_polygons) + gdf_polygons = gdf_polygons.explode() # explode multi-polygons to polygons gdf_polygons = remove_interior_polygons(gdf_polygons) # remove holes from polygons gdf_points = gdf_points[~gdf_points['geometry'].is_empty] # remove empty geometries @@ -53,6 +104,7 @@ def main(): gdf_polygons['IDSGS_neu'].unique()])) np.savetxt(os.path.join(filedir, output_sgsids_filename), sgs_ids.astype(int), fmt='%d', delimiter=',') + np.savetxt(os.path.join(filedir, output_sgsids_simplified_filename), np.unique(simplified_sgsids).astype(int), fmt='%d', delimiter=',') # for each sgsid write a sql delete statement to delete all entries with this sgsid. Commit the commands every 1000 rows. with open(os.path.join(filedir, output_filename), 'w') as f: From 85b8249a718b3dee7bfe846b04aeb163600b0d3a Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Tue, 19 Nov 2024 17:30:31 +0100 Subject: [PATCH 08/10] Update README.md for db dump creation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 681dc435..a555641b 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Be aware that you need to manually insert the `{DB_*}` values beforehand. ```bash cd development -docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table workgroups_on_users --exclude-table _prisma_migrations -n public > /dump.sql' +docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table workgroups_on_users --exclude-table _prisma_migrations --exclude-table asset_test --exclude-table asset_user_bak --exclude-table favorite -n public > /dump.sql' ``` > The export will output warnings related to circular foreign-key constraints. From ade30e058b4e945412d3061afd33ab65d9dd67b2 Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Mon, 25 Nov 2024 08:02:40 +0100 Subject: [PATCH 09/10] Fix client build warnings --- apps/client-asset-sg/project.json | 2 +- package-lock.json | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/client-asset-sg/project.json b/apps/client-asset-sg/project.json index acc6be8d..8de86db8 100644 --- a/apps/client-asset-sg/project.json +++ b/apps/client-asset-sg/project.json @@ -19,7 +19,7 @@ "assets": ["apps/client-asset-sg/src/favicon.ico", "apps/client-asset-sg/src/assets"], "styles": ["apps/client-asset-sg/src/styles.scss"], "scripts": [], - "allowedCommonJsDependencies": ["tsafe", "xml-utils", "pbf", "rbush", "earcut", "@prisma/client"] + "allowedCommonJsDependencies": ["tsafe", "xml-utils", "pbf", "rbush", "earcut", "@prisma/client", "validator"] }, "configurations": { "production": { diff --git a/package-lock.json b/package-lock.json index 6a7bb821..36e41398 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13828,9 +13828,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001621", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001621.tgz", - "integrity": "sha512-+NLXZiviFFKX0fk8Piwv3PfLPGtRqJeq2TiNoUff/qB5KJgwecJTvCXDpmlyP/eCI/GUEmp/h/y5j0yckiiZrA==", + "version": "1.0.30001684", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001684.tgz", + "integrity": "sha512-G1LRwLIQjBQoyq0ZJGqGIJUXzJ8irpbjHLpVRXDvBEScFJ9b17sgK6vlx0GAJFE21okD7zXl08rRRUfq6HdoEQ==", "funding": [ { "type": "opencollective", @@ -13844,7 +13844,8 @@ "type": "github", "url": "https://github.com/sponsors/ai" } - ] + ], + "license": "CC-BY-4.0" }, "node_modules/caseless": { "version": "0.12.0", From 0495380ae3de4b92695faa36967ac51ecda2ea10 Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Mon, 25 Nov 2024 17:41:24 +0100 Subject: [PATCH 10/10] Fix studies being invisible when asset is selected --- .../src/lib/components/map/map.component.ts | 11 ++++++----- .../lib/state/asset-search/asset-search.effects.ts | 8 ++++---- .../lib/state/asset-search/asset-search.selector.ts | 10 +++++++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/libs/asset-viewer/src/lib/components/map/map.component.ts b/libs/asset-viewer/src/lib/components/map/map.component.ts index 2a01bf50..45a45295 100644 --- a/libs/asset-viewer/src/lib/components/map/map.component.ts +++ b/libs/asset-viewer/src/lib/components/map/map.component.ts @@ -21,7 +21,8 @@ import { selectAssetSearchPolygon, selectAssetSearchResultData, selectCurrentAssetDetail, - selectHasDefaultFilters, + selectHasNoActiveFilters, + selectIsSearchQueryEmpty, selectStudies, } from '../../state/asset-search/asset-search.selector'; import { AppStateWithMapControl } from '../../state/map-control/map-control.reducer'; @@ -129,8 +130,8 @@ export class MapComponent implements AfterViewInit, OnChanges, OnDestroy { this.controller.addControl(this.controls.draw); this.subscription.add( - this.store.select(selectHasDefaultFilters).subscribe((hasDefaultFilters) => { - if (hasDefaultFilters) { + this.store.select(selectIsSearchQueryEmpty).subscribe((isSearchQueryEmpty) => { + if (isSearchQueryEmpty) { this.controls.zoom.resetZoom(); } }) @@ -155,8 +156,8 @@ export class MapComponent implements AfterViewInit, OnChanges, OnDestroy { private initializeStoreBindings() { this.subscription.add( - this.store.select(selectHasDefaultFilters).subscribe((showStudies) => { - this.controller.setShowHeatmap(showStudies); + this.store.select(selectHasNoActiveFilters).subscribe((hasNoActiveFilters) => { + this.controller.setShowHeatmap(hasNoActiveFilters); }) ); this.subscription.add( diff --git a/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts b/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts index f7a37778..218d92c1 100644 --- a/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts +++ b/libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts @@ -18,7 +18,7 @@ import { selectAssetSearchIsInitialized, selectAssetSearchQuery, selectCurrentAssetDetail, - selectHasDefaultFilters, + selectIsSearchQueryEmpty, selectSearchLoadingState, selectStudies, } from './asset-search.selector'; @@ -221,9 +221,9 @@ export class AssetSearchEffects { this.actions$.pipe( ofType(actions.updateResults), map(({ results }) => results.page.total !== 0), - withLatestFrom(this.store.select(selectHasDefaultFilters)), - map(([hasResults, hasNoFilters]) => - !hasResults || hasNoFilters ? actions.closeResults() : actions.openResults() + withLatestFrom(this.store.select(selectIsSearchQueryEmpty)), + map(([hasResults, isSearchQueryEmpty]) => + !hasResults || isSearchQueryEmpty ? actions.closeResults() : actions.openResults() ) ) ); diff --git a/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts b/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts index 03a02c80..197042a6 100644 --- a/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts +++ b/libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts @@ -60,12 +60,16 @@ export const selectAssetSearchTotalResults = createSelector(assetSearchFeature, export const selectCurrentAssetDetail = createSelector(assetSearchFeature, (state) => state.currentAsset); export const selectStudies = createSelector(assetSearchFeature, (state) => state.studies); -export const selectHasDefaultFilters = createSelector( +export const selectHasNoActiveFilters = createSelector(assetSearchFeature, ({ query }) => hasNoActiveFilters(query)); + +export const selectIsSearchQueryEmpty = createSelector( assetSearchFeature, - ({ query, currentAsset }) => - currentAsset == null && Object.values(query).every((value) => value === undefined || value == false) + ({ query, currentAsset }) => currentAsset == null && hasNoActiveFilters(query) ); +const hasNoActiveFilters = (query: AssetSearchQuery): boolean => + Object.values(query).every((value) => value === undefined || value == false); + export const selectCurrentAssetDetailVM = createSelector( fromAppShared.selectRDReferenceData, selectCurrentAssetDetail,