Merge pull request #863 from europeana/feat/MET-6213-Debias-Consolida…

…tion

MET-6213 Debias Consolidation

projects/sandbox/cypress/e2e/debias.cy.ts

@@ -0,0 +1,36 @@
+context('Sandbox', () => {
+  describe('Debias', () => {
+    const force = { force: true };
+    const selDebiasLink = 'li .debias-link';
+    const txtNoDetections = 'No Biases Found';
+    const pollInterval = 2000;
+
+    it('should not allow debias checks for failed datasets', () => {
+      cy.visit('/dataset/909');
+      cy.wait(1000);
+      cy.get(selDebiasLink).should('not.exist');
+    });
+
+    it('should show an empty report', () => {
+      cy.visit('/dataset/28');
+      cy.wait(pollInterval);
+      cy.get(selDebiasLink)
+        .last()
+        .click(force);
+      cy.wait(1);
+      cy.get(selDebiasLink)
+        .last()
+        .click(force);
+      cy.contains(txtNoDetections).should('exist');
+    });
+
+    it('should show a report', () => {
+      cy.visit('/dataset/3');
+      cy.wait(pollInterval);
+      cy.get(selDebiasLink)
+        .last()
+        .click(force);
+      cy.contains(txtNoDetections).should('not.exist');
+    });
+  });
+});

projects/sandbox/src/app/_data/static-data.ts

@@ -42,3 +42,190 @@ export const problemPatternData = {
     problemPatternTitle: 'Extremely long values'
   }
 };
+
+export const isoLanguageNames: { [key: string]: string } = {
+  aa: 'Afar',
+  ab: 'Abkhazian',
+  ae: 'Avestan',
+  af: 'Afrikaans',
+  ak: 'Akan',
+  am: 'Amharic',
+  an: 'Aragonese',
+  ar: 'Arabic',
+  as: 'Assamese',
+  av: 'Avaric',
+  ay: 'Aymara',
+  az: 'Azerbaijani',
+  ba: 'Bashkir',
+  be: 'Belarusian',
+  bg: 'Bulgarian',
+  bh: 'Bihari languages',
+  bi: 'Bislama',
+  bm: 'Bambara',
+  bn: 'Bengali',
+  bo: 'Tibetan',
+  br: 'Breton',
+  bs: 'Bosnian',
+  ca: 'Catalan; Valencian',
+  ce: 'Chechen',
+  ch: 'Chamorro',
+  co: 'Corsican',
+  cr: 'Cree',
+  cs: 'Czech',
+  cu: 'Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic',
+  cv: 'Chuvash',
+  cy: 'Welsh',
+  da: 'Danish',
+  de: 'German',
+  dv: 'Divehi; Dhivehi; Maldivian',
+  dz: 'Dzongkha',
+  ee: 'Ewe',
+  el: 'Greek, Modern (1453-)',
+  en: 'English',
+  eo: 'Esperanto',
+  es: 'Spanish; Castilian',
+  et: 'Estonian',
+  eu: 'Basque',
+  fa: 'Persian',
+  ff: 'Fulah',
+  fi: 'Finnish',
+  fj: 'Fijian',
+  fo: 'Faroese',
+  fr: 'French',
+  fy: 'Western Frisian',
+  ga: 'Irish',
+  gd: 'Gaelic; Scomttish Gaelic',
+  gl: 'Galician',
+  gn: 'Guarani',
+  gu: 'Gujarati',
+  gv: 'Manx',
+  ha: 'Hausa',
+  he: 'Hebrew',
+  hi: 'Hindi',
+  ho: 'Hiri Motu',
+  hr: 'Croatian',
+  ht: 'Haitian; Haitian Creole',
+  hu: 'Hungarian',
+  hy: 'Armenian',
+  hz: 'Herero',
+  ia: 'Interlingua (International Auxiliary Language Association)',
+  id: 'Indonesian',
+  ie: 'Interlingue; Occidental',
+  ig: 'Igbo',
+  ii: 'Sichuan Yi; Nuosu',
+  ik: 'Inupiaq',
+  io: 'Ido',
+  is: 'Icelandic',
+  it: 'Italian',
+  iu: 'Inuktitut',
+  ja: 'Japanese',
+  jv: 'Javanese',
+  ka: 'Georgian',
+  kg: 'Kongo',
+  ki: 'Kikuyu; Gikuyu',
+  kj: 'Kuanyama; Kwanyama',
+  kk: 'Kazakh',
+  kl: 'Kalaallisut; Greenlandic',
+  km: 'Central Khmer',
+  kn: 'Kannada',
+  ko: 'Korean',
+  kr: 'Kanuri',
+  ks: 'Kashmiri',
+  ku: 'Kurdish',
+  kv: 'Komi',
+  kw: 'Cornish',
+  ky: 'Kirghiz; Kyrgyz',
+  la: 'Latin',
+  lb: 'Luxembourgish; Letzeburgesch',
+  lg: 'Ganda',
+  li: 'Limburgan; Limburger; Limburgish',
+  ln: 'Lingala',
+  lo: 'Lao',
+  lt: 'Lithuanian',
+  lu: 'Luba-Katanga',
+  lv: 'Latvian',
+  mg: 'Malagasy',
+  mh: 'Marshallese',
+  mi: 'Maori',
+  mk: 'Macedonian',
+  ml: 'Malayalam',
+  mn: 'Mongolian',
+  mr: 'Marathi',
+  ms: 'Malay',
+  mt: 'Maltese',
+  my: 'Burmese',
+  na: 'Nauru',
+  nb: 'Bokmål, Norwegian; Norwegian Bokmål',
+  nd: 'Ndebele, North; North Ndebele',
+  ne: 'Nepali',
+  ng: 'Ndonga',
+  nl: 'Dutch; Flemish',
+  nn: 'Norwegian Nynorsk; Nynorsk, Norwegian',
+  no: 'Norwegian',
+  nr: 'Ndebele, South; South Ndebele',
+  nv: 'Navajo; Navaho',
+  ny: 'Chichewa; Chewa; Nyanja',
+  oc: 'Occitan (post 1500)',
+  oj: 'Ojibwa',
+  om: 'Oromo',
+  or: 'Oriya',
+  os: 'Ossetian; Ossetic',
+  pa: 'Panjabi; Punjabi',
+  pi: 'Pali',
+  pl: 'Polish',
+  ps: 'Pushto; Pashto',
+  pt: 'Portuguese',
+  qu: 'Quechua',
+  rm: 'Romansh',
+  rn: 'Rundi',
+  ro: 'Romanian; Moldavian; Moldovan',
+  ru: 'Russian',
+  rw: 'Kinyarwanda',
+  sa: 'Sanskrit',
+  sc: 'Sardinian',
+  sd: 'Sindhi',
+  se: 'Northern Sami',
+  sg: 'Sango',
+  si: 'Sinhala; Sinhalese',
+  sk: 'Slovak',
+  sl: 'Slovenian',
+  sm: 'Samoan',
+  sn: 'Shona',
+  so: 'Somali',
+  sq: 'Albanian',
+  sr: 'Serbian',
+  ss: 'Swati',
+  st: 'Sotho, Southern',
+  su: 'Sundanese',
+  sv: 'Swedish',
+  sw: 'Swahili',
+  ta: 'Tamil',
+  te: 'Telugu',
+  tg: 'Tajik',
+  th: 'Thai',
+  ti: 'Tigrinya',
+  tk: 'Turkmen',
+  tl: 'Tagalog',
+  tn: 'Tswana',
+  to: 'Tonga (Tonga Islands)',
+  tr: 'Turkish',
+  ts: 'Tsonga',
+  tt: 'Tatar',
+  tw: 'Twi',
+  ty: 'Tahitian',
+  ug: 'Uighur; Uyghur',
+  uk: 'Ukrainian',
+  ur: 'Urdu',
+  uz: 'Uzbek',
+  ve: 'Venda',
+  vi: 'Vietnamese',
+  vo: 'Volapük',
+  wa: 'Walloon',
+  wo: 'Wolof',
+  xh: 'Xhosa',
+  yi: 'Yiddish',
+  yo: 'Yoruba',
+  za: 'Zhuang; Chuang',
+  zh: 'Chinese',
+  zu: 'Zulu'
+};

projects/sandbox/src/app/_mocked/mocked-sandbox.service.ts

@@ -189,6 +189,13 @@ export class MockSandboxService {
     } as unknown) as DebiasReport);
   }
 
+  runDebiasReport(datasetId: string): Observable<boolean> {
+    if (this.errorMode) {
+      return this.getError('mock runDebiasReport throws error');
+    }
+    return of(parseInt(datasetId) % 2 === 0);
+  }
+
   /**
    * getLanguages
    * gets the language options

projects/sandbox/src/app/_services/sandbox.service.spec.ts

@@ -19,6 +19,8 @@ import {
 import {
   DatasetInfo,
   DatasetStatus,
+  DebiasInfo,
+  DebiasReport,
   FieldOption,
   ProblemPattern,
   ProblemPatternsDataset,
@@ -281,4 +283,31 @@ describe('sandbox service', () => {
       .send(mockRecordReport);
     sub.unsubscribe();
   }));
+
+  it('should get the debias info', () => {
+    const datasetId = '123';
+    const sub = service.getDebiasInfo(datasetId).subscribe((di: DebiasInfo) => {
+      expect(di).toBeTruthy();
+    });
+    mockHttp.expect('GET', `/dataset/${datasetId}/debias/info`).send(datasetId);
+    sub.unsubscribe();
+  });
+
+  it('should get the debias report', () => {
+    const datasetId = '123';
+    const sub = service.getDebiasReport(datasetId).subscribe((dr: DebiasReport) => {
+      expect(dr).toBeTruthy();
+    });
+    mockHttp.expect('GET', `/dataset/${datasetId}/debias/report`).send(datasetId);
+    sub.unsubscribe();
+  });
+
+  it('should run the debias report', () => {
+    const datasetId = '123';
+    const sub = service.runDebiasReport(datasetId).subscribe((tf: boolean) => {
+      expect(tf).toBeTruthy();
+    });
+    mockHttp.expect('POST', `/dataset/${datasetId}/debias`).send(datasetId);
+    sub.unsubscribe();
+  });
 });

projects/sandbox/src/app/_services/sandbox.service.ts

@@ -241,15 +241,15 @@ export class SandboxService {
     );
   }
 
-  runDebiasReport(datasetId: number): Observable<boolean> {
+  runDebiasReport(datasetId: string): Observable<boolean> {
     return this.http.post<boolean>(`${apiSettings.apiHost}/dataset/${datasetId}/debias`, {});
   }
 
-  getDebiasReport(datasetId: number): Observable<DebiasReport> {
+  getDebiasReport(datasetId: string): Observable<DebiasReport> {
     return this.http.get<DebiasReport>(`${apiSettings.apiHost}/dataset/${datasetId}/debias/report`);
   }
 
-  getDebiasInfo(datasetId: number): Observable<DebiasInfo> {
+  getDebiasInfo(datasetId: string): Observable<DebiasInfo> {
     return this.http.get<DebiasInfo>(`${apiSettings.apiHost}/dataset/${datasetId}/debias/info`);
   }
 }

projects/sandbox/src/app/_translate/format-dc-field.pipe.spec.ts

@@ -0,0 +1,16 @@
+import { FormatDcFieldPipe } from '.';
+import { DebiasSourceField } from '../_models';
+
+describe('FormatDcFieldPipe', () => {
+  it('should transform', () => {
+    const pipe = new FormatDcFieldPipe();
+    expect(pipe.transform('hello')).toEqual('hello');
+    expect(pipe.transform(DebiasSourceField.DC_TITLE)).toEqual('dc:title');
+    expect(pipe.transform(DebiasSourceField.DC_DESCRIPTION)).toEqual('dc:description');
+    expect(pipe.transform(DebiasSourceField.DC_TYPE_LITERAL)).toEqual('dc:type literal');
+    expect(pipe.transform(DebiasSourceField.DC_TYPE_REFERENCE)).toEqual('dc:type reference');
+    expect(pipe.transform(DebiasSourceField.DC_SUBJECT_LITERAL)).toEqual('dc:subject literal');
+    expect(pipe.transform(DebiasSourceField.DC_SUBJECT_REFERENCE)).toEqual('dc:subject reference');
+    expect(pipe.transform(DebiasSourceField.DCTERMS_ALTERNATIVE)).toEqual('dc:terms alternative');
+  });
+});

projects/sandbox/src/app/_translate/format-language.pipe.spec.ts

@@ -0,0 +1,10 @@
+import { FormatLanguagePipe } from '.';
+
+describe('FormatLanguagePipe', () => {
+  it('should transform', () => {
+    const pipe = new FormatLanguagePipe();
+    expect(pipe.transform('en')).toEqual('English');
+    expect(pipe.transform('de')).toEqual('German');
+    expect(pipe.transform('xxx')).toEqual('xxx');
+  });
+});

projects/sandbox/src/app/_translate/format-language.pipe.ts

@@ -0,0 +1,13 @@
+import { Pipe, PipeTransform } from '@angular/core';
+import { isoLanguageNames } from '../_data';
+
+@Pipe({
+  name: 'formatLanguage',
+  standalone: true
+})
+export class FormatLanguagePipe implements PipeTransform {
+  transform(value: string): string {
+    const translated = isoLanguageNames[value];
+    return translated || value;
+  }
+}

projects/sandbox/src/app/_translate/highlight-matches-and-link.pipe.spec.ts

@@ -0,0 +1,38 @@
+import { HighlightMatchesAndLinkPipe } from './';
+import { DebiasTag } from '../_models';
+
+describe('highlight matches and link pipe', () => {
+  let pipe: HighlightMatchesAndLinkPipe;
+
+  const getTagOpen = (href: string): string => {
+    return `<a href="${href}" class="term-highlight external-link-debias" target="_blank">`;
+  };
+
+  beforeEach(() => {
+    pipe = new HighlightMatchesAndLinkPipe();
+  });
+
+  it('should not alter the string if no args are supplied', () => {
+    expect(pipe.transform('hello')).toBe('hello');
+  });
+
+  it('should not alter the string if empty args are supplied', () => {
+    expect(pipe.transform('hello', [])).toBe('hello');
+    expect(pipe.transform('hello', [[]])).toBe('hello');
+  });
+
+  it('should highlight', () => {
+    const href = 'http://www.the-link.com';
+
+    const dts: Array<DebiasTag> = [
+      {
+        start: 1,
+        end: 2,
+        length: 1,
+        uri: href
+      }
+    ];
+
+    expect(pipe.transform('hello', [dts])).toBe('h' + getTagOpen(href) + 'e</a>llo');
+  });
+});

projects/sandbox/src/app/_translate/highlight-matches-and-link.pipe.ts

@@ -12,7 +12,7 @@ import { DebiasTag } from '../_models';
 export class HighlightMatchesAndLinkPipe implements PipeTransform {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   transform(value: string, args?: Array<any>): string {
-    if (args && args.length > 0 && args[0].length > 0) {
+    if (args && args.length > 0) {
       const tags = args[0] as Array<DebiasTag>;
       if (tags.length === 0) {
         return value;

projects/sandbox/src/app/_translate/index.ts

@@ -1,5 +1,6 @@
 export * from './format-dc-field.pipe';
 export * from './format-harvest-url.pipe';
+export * from './format-language.pipe';
 export * from './format-license.pipe';
 export * from './format-tier-dimension.pipe';
 export * from './highlight-match.pipe';