diff --git a/CHANGES.md b/CHANGES.md index 637ed0db81..99d7598e3c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,11 @@ - #3485: New feature for registry records manager: able to search record by keywords & added records list view - Display organizational unit ID info in the user general info panel +- Improved Registry event processing performance - #3100: Conenctor SDK: Allow runtime filter code to be supply to connectors via Helm config - Fixed an issue with docker build utiliy +- #3408: auto reset search dataset result cache when login status changes +- Allow users to open datasets & distributions in "Raw Record Editor" when the user has edit permission ## v2.3.1 diff --git a/magda-migrator-registry-db/sql/V2_8__optimise_event_aspect_query.sql b/magda-migrator-registry-db/sql/V2_8__optimise_event_aspect_query.sql new file mode 100644 index 0000000000..18d42abc08 --- /dev/null +++ b/magda-migrator-registry-db/sql/V2_8__optimise_event_aspect_query.sql @@ -0,0 +1,6 @@ +-- this will make planner to use gin index for distributions rather than seq scan +-- event web hook processing will be much faster +CREATE INDEX recordsaspects_data_distributions_gin_with_aspect_cond_idx + ON recordaspects USING gin + ((data -> 'distributions'::text)) + WHERE aspectid = 'dataset-distributions'; \ No newline at end of file diff --git a/magda-registry-api/src/main/resources/application.conf b/magda-registry-api/src/main/resources/application.conf index 921ff5fb0e..691fe7a257 100644 --- a/magda-registry-api/src/main/resources/application.conf +++ b/magda-registry-api/src/main/resources/application.conf @@ -34,7 +34,7 @@ akka { # `Timeout-Access` header to the request, which enables programmatic # customization of the timeout period and timeout response for each # request individually. - request-timeout = 30s + request-timeout = 60s # The time after which an idle connection will be automatically closed. # Set to `infinite` to completely disable idle connection timeouts. @@ -78,7 +78,7 @@ db-query { # the default timeout setting used for most common queries that serve rest APIs. # we put it in place via DBSession.queryTimeout(seconds: Int) - default-timeout = "90s" + default-timeout = "180s" # Similar to `default-timeout`. But this setting allow us to use different (often longer) timeout settings for long queries. # e.g. Trim operations. diff --git a/magda-registry-api/src/main/scala/au/csiro/data61/magda/registry/RecordPersistence.scala b/magda-registry-api/src/main/scala/au/csiro/data61/magda/registry/RecordPersistence.scala index f71fa9bb74..a8828e1927 100644 --- a/magda-registry-api/src/main/scala/au/csiro/data61/magda/registry/RecordPersistence.scala +++ b/magda-registry-api/src/main/scala/au/csiro/data61/magda/registry/RecordPersistence.scala @@ -1772,8 +1772,25 @@ class DefaultRecordPersistence(config: Config) sqls"order by sequence DESC" else sqls"order by sequence" + /** + * Why we need to "fence" the query? + * select * from ( + * select xxx,xx from xxx where xxx + * order by xx + * ) as tmp_tab order by xx offset xxx limit xxx + * + * When both order by & (limit or offset) involved might make planner + * give up using full text index and choose to use sequence idx for filtering & sorting + * + * The "fence" make it always use full text index and sort by key sequence + * sort by key sequence could be very expense but in real world they are much less expensive than full text search. + * Our records table's frequent update nature might also contribute to it (planer won't get accurate cost estimate) + * vacuum analyze won't help either + * + */ val result = - sql"""select Records.sequence as sequence, + sql"""select * from ( + select Records.sequence as sequence, Records.recordId as recordId, Records.name as recordName, (select array_agg(aspectId) from RecordAspects ${SQLSyntax @@ -1787,6 +1804,7 @@ class DefaultRecordPersistence(config: Config) Records.tenantId as tenantId from Records ${SQLSyntax.where(SQLUtils.toAndConditionOpt(whereClauseParts: _*))} + ${orderBy}) as tmp_results ${orderBy} offset ${start.getOrElse(0)} limit ${limit + 1}""" diff --git a/magda-typescript-common/src/test/JsonConnector.spec.ts b/magda-typescript-common/src/test/JsonConnector.spec.ts index 2ca8ddf0ea..0db9a243ee 100644 --- a/magda-typescript-common/src/test/JsonConnector.spec.ts +++ b/magda-typescript-common/src/test/JsonConnector.spec.ts @@ -309,6 +309,79 @@ describe("JsonConnector", () => { }); }); + it("Will cralw only one dataset (`blah2-dataset-2`) and one distribution with `customJsFilterCode`", () => { + const { scope, connector } = setupCrawlTest({ + customJsFilterCode: `if(type === 'Dataset' && jsonData.blah != 'blah2-dataset-2') { + return false; + } else { + return true; + }` + }); + const receivedRecords: Record[] = []; + + scope + .persist() + .put(new RegExp("/records")) + .reply(200, (uri: string, requestBody: any) => { + receivedRecords.push(requestBody); + }); + scope.delete(/.*/).reply(202); + + return connector.run().then((result) => { + scope.done(); + expect(receivedRecords.length).to.equal(3); + const datasets = receivedRecords.filter( + (record) => !!record.aspects["dataset-distributions"] + ); + expect(datasets.length).to.equal(1); + expect(datasets[0].name).to.equal("blah2-dataset-2"); + const dists = receivedRecords.filter( + (record) => !record.aspects["dataset-distributions"] + ); + expect(dists.length).to.equal(2); + expect(dists.map((dist) => dist.name)).to.include.members([ + "blah-dist-1", + "blah-dist-2" + ]); + }); + }); + + it("Will cralw only one dataset (`blah2-dataset-1`) with `customJsFilterCode`", () => { + const { scope, connector } = setupCrawlTest({ + customJsFilterCode: `if(type === 'Dataset' && jsonData.blah != 'blah-dataset-1') { + return false; + } else if(type === 'Distribution' && jsonData.blah != 'blah-dist-2') { + return false; + }else { + return true; + }` + }); + const receivedRecords: Record[] = []; + + scope + .persist() + .put(new RegExp("/records")) + .reply(200, (uri: string, requestBody: any) => { + receivedRecords.push(requestBody); + }); + scope.delete(/.*/).reply(202); + + return connector.run().then((result) => { + scope.done(); + expect(receivedRecords.length).to.equal(2); + const datasets = receivedRecords.filter( + (record) => !!record.aspects["dataset-distributions"] + ); + expect(datasets.length).to.equal(1); + expect(datasets[0].name).to.equal("blah-dataset-1"); + const dists = receivedRecords.filter( + (record) => !record.aspects["dataset-distributions"] + ); + expect(dists.length).to.equal(1); + expect(dists[0].name).to.equal("blah-dist-2"); + }); + }); + function setupCrawlTest( config: FakeConnectorSourceConfig = {}, transformerConfig: JsonTransformerOptions = {} as JsonTransformerOptions @@ -362,22 +435,23 @@ class FakeJsonTransformer extends JsonTransformer { } getNameFromJsonOrganization(jsonOrganization: any): string { - return "name"; + return jsonOrganization?.name ? jsonOrganization.name : "name"; } getNameFromJsonDataset(jsonDataset: any): string { - return "name"; + return jsonDataset?.name ? jsonDataset.name : "name"; } getNameFromJsonDistribution( jsonDistribution: any, jsonDataset: any ): string { - return "name"; + return jsonDistribution?.name ? jsonDistribution.name : "name"; } } type FakeConnectorSourceConfig = { extras?: JsonConnectorConfigExtraMetaData; presetRecordAspects?: JsonConnectorConfigPresetAspect[]; + customJsFilterCode?: string; }; class FakeConnectorSource implements ConnectorSource { @@ -386,6 +460,7 @@ class FakeConnectorSource implements ConnectorSource { readonly hasFirstClassOrganizations: boolean = false; presetRecordAspects: JsonConnectorConfigPresetAspect[] = null; extras: JsonConnectorConfigExtraMetaData = null; + customJsFilterCode: string = ""; constructor(config: FakeConnectorSourceConfig = {}) { if (config.extras) { @@ -394,6 +469,9 @@ class FakeConnectorSource implements ConnectorSource { if (config.presetRecordAspects) { this.presetRecordAspects = config.presetRecordAspects; } + if (config.customJsFilterCode) { + this.customJsFilterCode = config.customJsFilterCode; + } } getJsonDataset(id: string): Promise { @@ -422,24 +500,32 @@ class FakeConnectorSource implements ConnectorSource { getJsonDatasets(): AsyncPage { return AsyncPage.single([ { - blah: "blah" + blah: "blah-dataset-1", + name: "blah-dataset-1" }, { - blah: "blah" + blah: "blah2-dataset-2", + name: "blah2-dataset-2" } ]); } getJsonFirstClassOrganizations(): AsyncPage { return AsyncPage.single([ { - blah: "blah" + blah: "blah-org-1", + name: "blah-org-1" } ]); } getJsonDistributions(): AsyncPage { return AsyncPage.single([ { - blah: "blah" + blah: "blah-dist-1", + name: "blah-dist-1" + }, + { + blah: "blah-dist-2", + name: "blah-dist-2" } ]); } diff --git a/magda-web-client/src/AppContainer.js b/magda-web-client/src/AppContainer.js index db0bd8401a..2de19f8c50 100644 --- a/magda-web-client/src/AppContainer.js +++ b/magda-web-client/src/AppContainer.js @@ -13,13 +13,10 @@ import Notification from "./Components/Common/Notification"; import { hideTopNotification } from "./actions/topNotificationAction"; import Routes from "./Routes"; - import { Medium } from "./Components/Common/Responsive"; - import { Route, Switch } from "react-router-dom"; - import { getPluginFooter } from "./externalPluginComponents"; - +import inPopUpMode from "./helpers/inPopUpMode"; import "./AppContainer.scss"; const ExternalFooterComponent = getPluginFooter(); @@ -31,6 +28,7 @@ class AppContainer extends React.Component { } render() { + const isPopUp = inPopUpMode(); return (
@@ -54,71 +52,73 @@ class AppContainer extends React.Component {
- - {/** turn off top margin for home page only */} - - ExternalFooterComponent ? ( - - ) : ( -