feat: Add subject filter

netwerk-digitaal-erfgoed · Dec 5, 2023 · a33914c · a33914c
1 parent 647e2db
commit a33914c
Show file tree

Hide file tree

Showing 16 changed files with 1,962 additions and 2,524 deletions.
diff --git a/jest.config.js b/jest.config.js
@@ -10,9 +10,9 @@ export default {
  coverageReporters: ['json-summary', 'text'],
  coverageThreshold: {
  global: {
- lines: 20.8,
- statements: 20.8,
- branches: 11.9,
+ lines: 21.91,
+ statements: 21.91,
+ branches: 12.76,
  functions: 21.05,
  },
  },

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -37,14 +37,16 @@
  "n3": "^1.17.2",
  "pino": "^8.16.2",
  "rdf-data-factory": "^1.1.2",
- "rdf-ext": "^2.4.0",
+ "rdf-dereference": "^2.2.0",
+ "rdf-ext": "^2.5.0",
  "rdf-js": "^4.0.2"
  },
  "devDependencies": {
  "@rdfjs/types": "^1.1.0",
  "@types/jest": "^28.1.8",
  "@types/n3": "^1.16.4",
  "@types/node": "^18.18.12",
+ "@types/rdf-ext": "^2.2.5",
  "gts": "^5.2.0",
  "jest": "^29.7.0",
  "jest-coverage-thresholds-bumper": "^1.1.0",

diff --git a/queries/analysis/class-partition.rq b/queries/analysis/class-partition.rq
@@ -8,6 +8,7 @@ CONSTRUCT {
  ] .
 } WHERE {
  SELECT (COUNT(?type) AS ?entities) ?type {
+ #subjectFilter#
  ?s a ?type .
  }
  GROUP BY ?type

diff --git a/queries/analysis/class-properties.rq b/queries/analysis/class-properties.rq
@@ -11,6 +11,7 @@ CONSTRUCT {
  ] .
 } WHERE {
  SELECT ?p ?t (COUNT(DISTINCT ?s) AS ?subjects) {
+ #subjectFilter#
  ?s a ?t ;
  ?p ?o .
  }

diff --git a/queries/analysis/entity-properties.rq b/queries/analysis/entity-properties.rq
@@ -8,6 +8,7 @@ CONSTRUCT {
  ] .
 } WHERE {
  SELECT (COUNT(?p) AS ?entities) ?p {
+ #subjectFilter#
  ?s ?p ?o .
  }
  GROUP BY ?p

diff --git a/queries/analysis/object-literals.rq b/queries/analysis/object-literals.rq
@@ -6,6 +6,7 @@ CONSTRUCT {
  nde:distinctObjectsLiteral ?total .
 } WHERE {
  SELECT (COUNT(?o) as ?total) {
+ #subjectFilter#
  ?s ?p ?o .
  FILTER(ISLITERAL(?o))
  }

diff --git a/queries/analysis/object-uri-space.rq b/queries/analysis/object-uri-space.rq
@@ -7,6 +7,7 @@ CONSTRUCT {
  void:triples ?count .
 } WHERE {
  SELECT DISTINCT ?prefix (COUNT(?prefix) AS ?count) {
+ #subjectFilter#
  ?s ?p ?o .
  FILTER(ISIRI(?o))
  BIND(REPLACE(STR(?o), "([^/]+$)", "") AS ?prefix)

diff --git a/queries/analysis/object-uris.rq b/queries/analysis/object-uris.rq
@@ -6,6 +6,7 @@ CONSTRUCT {
  nde:distinctObjectsURI ?total .
 } WHERE {
  SELECT (COUNT(?o) as ?total) {
+ #subjectFilter#
  ?s ?p ?o .
  FILTER(ISIRI(?o))
  }

diff --git a/queries/analysis/properties.rq b/queries/analysis/properties.rq
@@ -5,6 +5,7 @@ CONSTRUCT {
  void:properties ?count .
 } WHERE {
  SELECT (COUNT(DISTINCT ?p) as ?count) {
+ #subjectFilter#
  ?s ?p ?o
  }
 }
diff --git a/queries/analysis/subjects.rq b/queries/analysis/subjects.rq
@@ -5,6 +5,7 @@ CONSTRUCT {
  void:distinctSubjects ?count .
 } WHERE {
  SELECT (COUNT(DISTINCT ?s) as ?count) {
+ #subjectFilter#
  ?s ?p ?o .
  FILTER(!ISBLANK(?s))
  }

diff --git a/queries/analysis/triples.rq b/queries/analysis/triples.rq
@@ -5,6 +5,7 @@ CONSTRUCT {
  void:triples ?count .
 } WHERE {
  SELECT (COUNT(*) as ?count) {
+ #subjectFilter#
  ?s ?p ?o
  }
 }
diff --git a/queries/selection/supplemental.ttl b/queries/selection/supplemental.ttl
@@ -0,0 +1,14 @@
+@prefix nde: <https://data.netwerkdigitaalerfgoed.nl/def/> .
+
+<http://data.bibliotheken.nl/id/dataset/albac> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/albac>" .
+<http://data.bibliotheken.nl/id/dataset/brinkman> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/brinkman>" .
+<http://data.bibliotheken.nl/id/dataset/corps> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/corps>" .
+<http://data.bibliotheken.nl/id/dataset/dbnla> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/dbnla>" .
+<http://data.bibliotheken.nl/id/dataset/dbnlt> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/dbnlt>" .
+<http://data.bibliotheken.nl/id/dataset/gtt> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/gtt>" .
+<http://data.bibliotheken.nl/id/dataset/kbcode> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/kbcode>" .
+<http://data.bibliotheken.nl/id/dataset/nbt> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/nbt>" .
+<http://data.bibliotheken.nl/id/dataset/persons> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/persone>" .
+<http://data.bibliotheken.nl/id/dataset/rise-alba> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/rise-alba>" .
+<http://data.bibliotheken.nl/id/dataset/rise-centsprenten> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/rise-centsprenten>" .
+<http://data.bibliotheken.nl/id/dataset/stcn> nde:subjectFilter "?s schema:mainEntityOfPage/schema:isPartOf <http://data.bibliotheken.nl/id/dataset/stcn>" .
diff --git a/src/analyzer.ts b/src/analyzer.ts
@@ -71,18 +71,21 @@ export class SparqlQueryAnalyzer implements Analyzer {
  type?: string
  ): Promise<AsyncIterator<Quad> & ResultStream<Quad>> {
  try {
- return await new QueryEngine().queryQuads(this.query, {
- initialBindings: this.bindingsFactory.fromRecord({
- dataset: this.dataFactory.namedNode(dataset.iri),
- }) as unknown as Bindings,
- sources: [
- {
- type: 'sparql',
- value: endpoint,
- },
- ],
- httpTimeout: 300_000, // Some SPARQL queries really take this long.
- });
+ return await new QueryEngine().queryQuads(
+ this.query.replace('#subjectFilter#', dataset.subjectFilter ?? ''),
+ {
+ initialBindings: this.bindingsFactory.fromRecord({
+ dataset: this.dataFactory.namedNode(dataset.iri),
+ }) as unknown as Bindings,
+ sources: [
+ {
+ type: 'sparql',
+ value: endpoint,
+ },
+ ],
+ httpTimeout: 300_000, // Some SPARQL queries really take this long.
+ }
+ );
  } catch (e) {
  if (type !== undefined) {
  // Retry without explicit SPARQL type, which is needed for endpoints that offer a SPARQL Service Description.

diff --git a/src/dataset.ts b/src/dataset.ts
@@ -1,7 +1,11 @@
 export class Dataset {
  constructor(
  public readonly iri: string,
- public distributions: Distribution[]
+ public distributions: Distribution[],
+
+ // On the level of the dataset instead of the distribution because distribution may not have a URI, so cannot be
+ // referenced from supplemental.ttl.
+ public subjectFilter?: string
  ) {}
 
  public getSparqlDistribution(): Distribution | null {

diff --git a/src/selector.ts b/src/selector.ts
@@ -1,6 +1,10 @@
 import {Dataset, Distribution} from './dataset.js';
 import {QueryEngine} from '@comunica/query-sparql';
-import {Quad} from 'n3';
+import {DataFactory, Quad} from 'n3';
+import {resolve} from 'node:path';
+import rdfDereferencer from 'rdf-dereference';
+import namedNode = DataFactory.namedNode;
+import factory from 'rdf-ext';
 
 export interface Selector {
  select(): Promise<Set<Dataset>>;
@@ -15,6 +19,14 @@ export class SparqlQuerySelector implements Selector {
  private readonly queryEngine: QueryEngine
  ) {}
  async select(): Promise<Set<Dataset>> {
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+ // @ts-ignore
+ const {data} = await rdfDereferencer.default.dereference(
+ resolve('queries/selection/supplemental.ttl'),
+ {localFiles: true}
+ );
+ const supplementalStore = await factory.dataset().import(data);
+
  const quadStream = await this.queryEngine.queryQuads(this.config.query, {
  sources: [
  {
@@ -34,7 +46,19 @@ export class SparqlQuerySelector implements Selector {
  quad.predicate.value &&
  'http://www.w3.org/ns/dcat#Dataset' === quad.object.value
  ) {
- dataset = new Dataset(quad.subject.value, []);
+ const subjectFilter = [
+ ...supplementalStore.match(
+ quad.subject,
+ namedNode(
+ 'https://data.netwerkdigitaalerfgoed.nl/def/subjectFilter'
+ )
+ ),
+ ][0]?.object.value;
+ dataset = new Dataset(
+ quad.subject.value,
+ [],
+ subjectFilter ? subjectFilter + '.' : undefined
+ );
  datasets.add(dataset);
  }