diff --git a/.env b/.env index a897d5532..f6057c019 100644 --- a/.env +++ b/.env @@ -69,7 +69,7 @@ REPOSITORY=ghcr.io/jhu-sheridan-libraries/idc-isle-dc TAG=upstream-20200824-f8d1e8e-23-g9fe79fc # Docker image and tag for snapshot image -SNAPSHOT_TAG=upstream-20201007-739693ae-213-g3669e99.1616768917 +SNAPSHOT_TAG=upstream-20201007-739693ae-226-gf12f7708.1617383406 # IdP, SP entity URIs and base URLs SP_BASEURL=https://islandora-idc.traefik.me diff --git a/.gitignore b/.gitignore index 2c7bfe6c7..bf8564ca4 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,5 @@ certs *node_modules end-to-end/reports tests/10-migration-backend-tests/testcafe/screenshots +tests/11-file-deletion-tests/testcafe/screenshots +tests/12-migration-derivative-tests/testcafe/screenshots diff --git a/codebase/composer.json b/codebase/composer.json index 388e9c998..d7775e8e1 100644 --- a/codebase/composer.json +++ b/codebase/composer.json @@ -38,6 +38,14 @@ { "type": "vcs", "url": "git@github.com:jhu-idc/idc_ui_module.git" + }, + { + "type": "vcs", + "url": "https://github.com/jhu-idc/idc_migration" + }, + { + "type": "vcs", + "url": "https://github.com/jhu-idc/migrate_file" } ], "require": { @@ -58,7 +66,6 @@ "drupal/epp": "^1.0", "drupal/facets": "^1.3", "drupal/libraries": "3.x-dev@dev", - "drupal/migrate_file": "^1.1", "drupal/migrate_source_ui": "^1.0@RC", "drupal/pager_serializer": "^1.0", "drupal/pdf": "1.x-dev", @@ -76,6 +83,7 @@ "jhu-idc/idc_ui_module": "dev-main", "jhu-idc/islandora_defaults": "dev-8.x-1.x", "jhu_idc/idc_migration": "^1.0", + "jhu-idc/migrate_file": "^1.1", "vlucas/phpdotenv": "^4.0", "webflo/drupal-finder": "^1.0.0", "zaporylie/composer-drupal-optimizations": "^1.0" diff --git a/codebase/composer.lock b/codebase/composer.lock index e16092503..e031103c3 100644 --- a/codebase/composer.lock +++ b/codebase/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ddec2ca9e1b9f3e133e1e8245344f175", + "content-hash": "50ead380aac04c0153faf55b1c1d46cc", "packages": [ { "name": "alchemy/zippy", @@ -292,6 +292,16 @@ "zend", "zikula" ], + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://tidelift.com/funding/github/packagist/composer/composer", + "type": "tidelift" + } + ], "time": "2020-04-07T06:57:05+00:00" }, { @@ -3806,6 +3816,14 @@ { "name": "borisson_", "homepage": "https://www.drupal.org/user/2393360" + }, + { + "name": "drunken monkey", + "homepage": "https://www.drupal.org/user/205582" + }, + { + "name": "mkalkbrenner", + "homepage": "https://www.drupal.org/user/124705" } ], "description": "The Facet module allows site builders to easily create and manage faceted search interfaces.", @@ -4177,10 +4195,22 @@ "GPL-2.0-or-later" ], "authors": [ + { + "name": "benjifisher", + "homepage": "https://www.drupal.org/user/683300" + }, + { + "name": "bradjones1", + "homepage": "https://www.drupal.org/user/405824" + }, { "name": "jungle", "homepage": "https://www.drupal.org/user/2919723" }, + { + "name": "lhridley", + "homepage": "https://www.drupal.org/user/1223730" + }, { "name": "twistor", "homepage": "https://www.drupal.org/user/473738" @@ -4501,54 +4531,6 @@ }, "time": "2019-06-25T07:35:46+00:00" }, - { - "name": "drupal/migrate_file", - "version": "1.1.0", - "source": { - "type": "git", - "url": "https://git.drupalcode.org/project/migrate_file.git", - "reference": "8.x-1.1" - }, - "dist": { - "type": "zip", - "url": "https://ftp.drupal.org/files/projects/migrate_file-8.x-1.1.zip", - "reference": "8.x-1.1", - "shasum": "4a24edc577541b5aa67682a4f89c8dfd7c0788d5" - }, - "require": { - "drupal/core": "^8" - }, - "type": "drupal-module", - "extra": { - "drupal": { - "version": "8.x-1.1", - "datestamp": "1539064380", - "security-coverage": { - "status": "covered", - "message": "Covered by Drupal's security advisory policy" - } - } - }, - "notification-url": "https://packages.drupal.org/8/downloads", - "license": [ - "GPL-2.0+" - ], - "authors": [ - { - "name": "drclaw", - "homepage": "https://www.drupal.org/user/823702" - } - ], - "description": "Additional support for migrating files including downloading remote files and using remote uris (without download)", - "homepage": "https://www.drupal.org/project/migrate_file", - "keywords": [ - "Drupal" - ], - "support": { - "source": "http://cgit.drupalcode.org/migrate_file", - "issues": "https://www.drupal.org/project/issues/migrate_file" - } - }, { "name": "drupal/migrate_plus", "version": "5.1.0", @@ -6834,6 +6816,35 @@ }, "time": "2021-03-17T14:28:23+00:00" }, + { + "name": "jhu-idc/migrate_file", + "version": "1.1.1", + "source": { + "type": "git", + "url": "https://github.com/jhu-idc/migrate_file.git", + "reference": "126a1e180b9e014b7b28008efd10d0c8328b3fff" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jhu-idc/migrate_file/zipball/126a1e180b9e014b7b28008efd10d0c8328b3fff", + "reference": "126a1e180b9e014b7b28008efd10d0c8328b3fff", + "shasum": "" + }, + "type": "drupal-module", + "license": [ + "GPL-2.0+" + ], + "description": "Additional support for migrating files including downloading remote files and using remote uris (without download)", + "homepage": "https://www.drupal.org/project/migrate_file", + "keywords": [ + "Drupal" + ], + "support": { + "issues": "https://www.drupal.org/project/issues/migrate_file", + "source": "http://cgit.drupalcode.org/migrate_file" + }, + "time": "2021-03-22T13:37:50+00:00" + }, { "name": "jhu-idc/reference_value_pair", "version": "dev-main", @@ -6865,16 +6876,16 @@ }, { "name": "jhu_idc/idc_migration", - "version": "v1.0.1", + "version": "v1.0.2", "source": { "type": "git", - "url": "https://github.com/emetsger/idc_migration.git", - "reference": "f7059e32d1da7a50c09d47422eed7bc089059c7b" + "url": "https://github.com/jhu-idc/idc_migration.git", + "reference": "e878fe4e629b0aaeee8d89548ef4f8757ab7e422" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/emetsger/idc_migration/zipball/f7059e32d1da7a50c09d47422eed7bc089059c7b", - "reference": "f7059e32d1da7a50c09d47422eed7bc089059c7b", + "url": "https://api.github.com/repos/jhu-idc/idc_migration/zipball/e878fe4e629b0aaeee8d89548ef4f8757ab7e422", + "reference": "e878fe4e629b0aaeee8d89548ef4f8757ab7e422", "shasum": "" }, "require": { @@ -6903,10 +6914,9 @@ ], "description": "Custom migration plugins for IDC", "support": { - "source": "https://github.com/emetsger/idc_migration/tree/master", - "issues": "https://github.com/emetsger/idc_migration/issues" + "source": "https://github.com/jhu-idc/idc_migration/tree/v1.0.2" }, - "time": "2020-12-10T16:08:15+00:00" + "time": "2021-03-19T15:39:24+00:00" }, { "name": "laminas/laminas-diactoros", @@ -7096,6 +7106,12 @@ "feed", "laminas" ], + "funding": [ + { + "url": "https://funding.communitybridge.org/projects/laminas-project", + "type": "community_bridge" + } + ], "time": "2021-01-04T19:20:24+00:00" }, { @@ -7188,6 +7204,12 @@ "laminas", "zf" ], + "funding": [ + { + "url": "https://funding.communitybridge.org/projects/laminas-project", + "type": "community_bridge" + } + ], "time": "2020-09-14T14:23:00+00:00" }, { @@ -7325,6 +7347,12 @@ "transform", "write" ], + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], "time": "2020-09-05T08:40:12+00:00" }, { @@ -7497,6 +7525,16 @@ } ], "description": "Mime-type detection for Flysystem", + "funding": [ + { + "url": "https://github.com/frankdejonge", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/league/flysystem", + "type": "tidelift" + } + ], "time": "2020-10-18T11:50:25+00:00" }, { @@ -8018,6 +8056,16 @@ "archive", "tar" ], + "funding": [ + { + "url": "https://github.com/mrook", + "type": "github" + }, + { + "url": "https://www.patreon.com/michielrook", + "type": "patreon" + } + ], "time": "2021-01-18T19:32:54+00:00" }, { @@ -8298,6 +8346,12 @@ } ], "description": "PHPMailer is a full-featured email creation and transfer class for PHP", + "funding": [ + { + "url": "https://github.com/synchro", + "type": "github" + } + ], "time": "2020-07-14T18:50:27+00:00" }, { @@ -11005,6 +11059,20 @@ ], "description": "Symfony Config Component", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-07-23T09:37:51+00:00" }, { @@ -11203,6 +11271,20 @@ ], "description": "Symfony CssSelector Component", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-03-16T08:31:04+00:00" }, { @@ -11427,6 +11509,20 @@ ], "description": "Symfony DomCrawler Component", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-07-16T09:41:49+00:00" }, { @@ -11536,6 +11632,20 @@ ], "description": "Symfony Filesystem Component", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-05-30T17:48:24+00:00" }, { @@ -11585,6 +11695,20 @@ ], "description": "Symfony Finder Component", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-02-14T07:34:21+00:00" }, { @@ -11773,6 +11897,20 @@ "symfony", "words" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-10-24T12:01:57+00:00" }, { @@ -11835,6 +11973,20 @@ "polyfill", "portable" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2021-01-07T16:49:33+00:00" }, { @@ -11898,6 +12050,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2021-01-07T16:49:33+00:00" }, { @@ -11962,6 +12128,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-10-23T14:02:19+00:00" }, { @@ -12032,6 +12212,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2021-01-07T16:49:33+00:00" }, { @@ -12099,6 +12293,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2021-01-07T17:09:11+00:00" }, { @@ -12162,6 +12370,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2021-01-07T16:49:33+00:00" }, { @@ -12323,6 +12545,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2021-01-07T16:49:33+00:00" }, { @@ -12389,6 +12625,20 @@ "portable", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-10-23T14:02:19+00:00" }, { @@ -12495,6 +12745,20 @@ "property path", "reflection" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-10-31T22:44:29+00:00" }, { @@ -12849,6 +13113,20 @@ "utf-8", "utf8" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-10-24T12:08:07+00:00" }, { @@ -13072,6 +13350,20 @@ "debug", "dump" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-06-24T13:34:53+00:00" }, { @@ -13442,6 +13734,16 @@ "env", "environment" ], + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/vlucas/phpdotenv", + "type": "tidelift" + } + ], "time": "2020-07-14T19:22:52+00:00" }, { @@ -13956,6 +14258,16 @@ "ssl", "tls" ], + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://tidelift.com/funding/github/packagist/composer/composer", + "type": "tidelift" + } + ], "time": "2020-04-08T08:27:21+00:00" }, { @@ -14036,6 +14348,20 @@ "dependency", "package" ], + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://github.com/composer", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/composer/composer", + "type": "tidelift" + } + ], "time": "2020-07-16T10:57:00+00:00" }, { @@ -14096,6 +14422,20 @@ "spdx", "validator" ], + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://github.com/composer", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/composer/composer", + "type": "tidelift" + } + ], "time": "2020-07-15T15:35:07+00:00" }, { @@ -14140,6 +14480,20 @@ "Xdebug", "performance" ], + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://github.com/composer", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/composer/composer", + "type": "tidelift" + } + ], "time": "2020-06-04T11:16:35+00:00" }, { @@ -14196,6 +14550,20 @@ "constructor", "instantiate" ], + "funding": [ + { + "url": "https://www.doctrine-project.org/sponsorship.html", + "type": "custom" + }, + { + "url": "https://www.patreon.com/phpdoctrine", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/doctrine%2Finstantiator", + "type": "tidelift" + } + ], "time": "2020-05-29T17:27:14+00:00" }, { @@ -14669,6 +15037,12 @@ "object", "object graph" ], + "funding": [ + { + "url": "https://tidelift.com/funding/github/packagist/myclabs/deep-copy", + "type": "tidelift" + } + ], "time": "2020-06-29T13:22:24+00:00" }, { @@ -15932,6 +16306,16 @@ "parser", "validator" ], + "funding": [ + { + "url": "https://github.com/Seldaek", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/seld/jsonlint", + "type": "tidelift" + } + ], "time": "2020-04-30T19:05:18+00:00" }, { @@ -16084,6 +16468,20 @@ ], "description": "Symfony BrowserKit Component", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-04-27T06:55:12+00:00" }, { @@ -16146,6 +16544,20 @@ "redlock", "semaphore" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-07-23T12:42:41+00:00" }, { @@ -16211,6 +16623,20 @@ ], "description": "Symfony PHPUnit Bridge", "homepage": "https://symfony.com", + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-07-22T22:00:00+00:00" }, { @@ -16267,6 +16693,20 @@ "polyfill", "shim" ], + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], "time": "2020-07-14T12:35:20+00:00" }, { diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml new file mode 100644 index 000000000..0a9878292 --- /dev/null +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_audio.yml @@ -0,0 +1,92 @@ +uuid: 4f2f31ea-1678-484a-9b0e-54e91f3bc089 +langcode: en +status: true +dependencies: { } +id: idc_ingest_media_audio +class: null +field_plugin_method: null +cck_plugin_method: null +migration_tags: null +migration_group: idc_ingest +label: '(3g) Ingest New Audio Media' +source: + plugin: csv + ids: + - local_id + path: 'Will be populated by the Migrate Source UI' + constants: + STATUS: true + DISPLAY: true + ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" +process: + _url_filename: + - plugin: callback + callable: parse_url + source: url + - plugin: extract + index: + - path + - plugin: callback + callable: basename + _url_filepath: + plugin: concat + source: + - constants/TMP_FS + - '@_url_filename' + _download_filepath: + plugin: file_copy + source: + - url + - '@_url_filepath' + file_exists: 1 + _file_sha: + plugin: callback + callable: sha1_file + source: '@_download_filepath' + _destination_filepath: + plugin: pairtree + source: '@_file_sha' + _destination_drupalpath: + plugin: concat + source: + - constants/PUBLIC_FS + - '@_destination_filepath' + field_original_name: original_name + field_media_audio_file/target_id: + plugin: file_import + source: '@_download_filepath' + move: true + reuse: false + rename: false + id_only: true + destination: '@_destination_drupalpath' + mimetype: mime_type + filename: name + field_mime_type: mime_type + name: name + field_media_of: + plugin: migration_lookup + migration: idc_ingest_new_items + no_stub: true + source: media_of + field_media_use: + - + plugin: skip_on_empty + method: process + source: media_use + - + plugin: explode + delimiter: '|' + - + plugin: entity_lookup + entity_type: taxonomy_term + value_key: name + bundle_key: vid + bundle: islandora_media_use + status: constants/STATUS +destination: + plugin: 'entity:media' + default_bundle: audio +migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml index 7664b6e92..9fd1191bb 100644 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_document.yml @@ -18,30 +18,68 @@ source: STATUS: true DISPLAY: true ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" process: + _url_filename: + - plugin: callback + callable: parse_url + source: url + - plugin: extract + index: + - path + - plugin: callback + callable: basename + _url_filepath: + plugin: concat + source: + - constants/TMP_FS + - '@_url_filename' + _download_filepath: + plugin: file_copy + source: + - url + - '@_url_filepath' + file_exists: 1 + _file_sha: + plugin: callback + callable: sha1_file + source: '@_download_filepath' + _destination_filepath: + plugin: pairtree + source: '@_file_sha' + _destination_drupalpath: + plugin: concat + source: + - constants/PUBLIC_FS + - '@_destination_filepath' + field_original_name: original_name field_media_document/target_id: plugin: file_import - source: document_url + source: '@_download_filepath' + move: true reuse: false - rename: true + rename: false id_only: true - field_media_document/description: name - field_media_document/display: constants/DISPLAY - field_mime_type: - plugin: explode - source: document_mime_type - delimiter: ; - name: name + destination: '@_destination_drupalpath' + mimetype: mime_type + filename: name + field_mime_type: mime_type + name/value: name field_media_of: plugin: migration_lookup migration: idc_ingest_new_items - source: nodeid_local no_stub: true + source: media_of field_media_use: + - + plugin: skip_on_empty + method: process + source: media_use - plugin: explode - source: use - delimiter: ; + delimiter: '|' + strict: false - plugin: entity_lookup entity_type: taxonomy_term @@ -49,15 +87,6 @@ process: bundle_key: vid bundle: islandora_media_use status: constants/STATUS - uid: - - - plugin: entity_lookup - entity_type: user - value_key: name - source: user - - - plugin: default_value - default_value: 1 destination: plugin: 'entity:media' default_bundle: document diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml new file mode 100644 index 000000000..76bdfe3d6 --- /dev/null +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_extracted_text.yml @@ -0,0 +1,93 @@ +uuid: 5861c879-474a-4f00-adc8-c61e368bb35a +langcode: en +status: true +dependencies: { } +id: idc_ingest_media_extracted_text +class: null +field_plugin_method: null +cck_plugin_method: null +migration_tags: null +migration_group: idc_ingest +label: '(3d) Ingest New Extracted Text Media' +source: + plugin: csv + ids: + - local_id + path: 'Will be populated by the Migrate Source UI' + constants: + STATUS: true + DISPLAY: true + ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" + FORMAT: "basic_html" +process: + _url_filename: + - plugin: callback + callable: parse_url + source: url + - plugin: extract + index: + - path + - plugin: callback + callable: basename + _url_filepath: + plugin: concat + source: + - constants/TMP_FS + - '@_url_filename' + _download_filepath: + plugin: file_copy + source: + - url + - '@_url_filepath' + file_exists: 1 + _file_sha: + plugin: callback + callable: sha1_file + source: '@_download_filepath' + _destination_filepath: + plugin: pairtree + source: '@_file_sha' + _destination_drupalpath: + plugin: concat + source: + - constants/PUBLIC_FS + - '@_destination_filepath' + field_media_file/target_id: + plugin: file_import + source: '@_download_filepath' + move: true + reuse: false + rename: false + id_only: true + destination: '@_destination_drupalpath' + mimetype: mime_type + filename: name + field_mime_type: mime_type + name/value: name + field_media_of: + plugin: migration_lookup + migration: idc_ingest_new_items + no_stub: true + source: media_of + field_media_use: + - + plugin: skip_on_empty + method: process + source: media_use + - + plugin: explode + delimiter: '|' + strict: false + - + plugin: entity_lookup + entity_type: taxonomy_term + value_key: name + bundle_key: vid + bundle: islandora_media_use + status: constants/STATUS +destination: + plugin: 'entity:media' + default_bundle: extracted_text +migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml new file mode 100644 index 000000000..febf50403 --- /dev/null +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_file.yml @@ -0,0 +1,92 @@ +uuid: 8d9a085b-24a7-42db-b0cb-4e92b35f0abf +langcode: en +status: true +dependencies: { } +id: idc_ingest_media_file +class: null +field_plugin_method: null +cck_plugin_method: null +migration_tags: null +migration_group: idc_ingest +label: '(3b) Ingest New Generic File Media' +source: + plugin: csv + ids: + - local_id + path: 'Will be populated by the Migrate Source UI' + constants: + STATUS: true + DISPLAY: true + ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" +process: + _url_filename: + - plugin: callback + callable: parse_url + source: url + - plugin: extract + index: + - path + - plugin: callback + callable: basename + _url_filepath: + plugin: concat + source: + - constants/TMP_FS + - '@_url_filename' + _download_filepath: + plugin: file_copy + source: + - url + - '@_url_filepath' + file_exists: 1 + _file_sha: + plugin: callback + callable: sha1_file + source: '@_download_filepath' + _destination_filepath: + plugin: pairtree + source: '@_file_sha' + _destination_drupalpath: + plugin: concat + source: + - constants/PUBLIC_FS + - '@_destination_filepath' + field_original_name: original_name + field_media_file/target_id: + plugin: file_import + source: '@_download_filepath' + move: true + reuse: false + rename: false + id_only: true + destination: '@_destination_drupalpath' + mimetype: mime_type + filename: name + field_mime_type: mime_type + name: name + field_media_of: + plugin: migration_lookup + migration: idc_ingest_new_items + no_stub: true + source: media_of + field_media_use: + - + plugin: skip_on_empty + method: process + source: media_use + - + plugin: explode + delimiter: '|' + - + plugin: entity_lookup + entity_type: taxonomy_term + value_key: name + bundle_key: vid + bundle: islandora_media_use + status: constants/STATUS +destination: + plugin: 'entity:media' + default_bundle: file +migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_files.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_files.yml deleted file mode 100644 index e2c781603..000000000 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_files.yml +++ /dev/null @@ -1,28 +0,0 @@ -uuid: 8d9a085b-24a7-42db-b0cb-4e92b35f0abf -langcode: en -status: true -dependencies: { } -id: idc_ingest_media_files -class: null -field_plugin_method: null -cck_plugin_method: null -migration_tags: null -migration_group: idc_ingest -label: '(3b) Ingest New Generic Files Media' -source: - plugin: csv - ids: - - local_id - path: 'Will be populated by the Migrate Source UI' - constants: - STATUS: true - ADMIN: 1 -process: - uri: file_uri - filename: file_name - filemime: file_type - status: constants/STATUS - uid: user -destination: - plugin: 'entity:file' -migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml new file mode 100644 index 000000000..6b2af9afd --- /dev/null +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_image.yml @@ -0,0 +1,93 @@ +uuid: 52f4d32a-e761-4f27-8d80-9cc5863a9486 +langcode: en +status: true +dependencies: { } +id: idc_ingest_media_image +class: null +field_plugin_method: null +cck_plugin_method: null +migration_tags: null +migration_group: idc_ingest +label: '(3a) Ingest New Image Media' +source: + plugin: csv + ids: + - local_id + path: 'Will be populated by the Migrate Source UI' + constants: + STATUS: true + DISPLAY: true + ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" +process: + _url_filename: + - plugin: callback + callable: parse_url + source: url + - plugin: extract + index: + - path + - plugin: callback + callable: basename + _url_filepath: + plugin: concat + source: + - constants/TMP_FS + - '@_url_filename' + _download_filepath: + plugin: file_copy + source: + - url + - '@_url_filepath' + file_exists: 1 + _file_sha: + plugin: callback + callable: sha1_file + source: '@_download_filepath' + _destination_filepath: + plugin: pairtree + source: '@_file_sha' + _destination_drupalpath: + plugin: concat + source: + - constants/PUBLIC_FS + - '@_destination_filepath' + field_original_name: original_name + field_media_image/target_id: + plugin: file_import + source: '@_download_filepath' + move: true + reuse: false + rename: false + id_only: true + destination: '@_destination_drupalpath' + mimetype: mime_type + filename: name + field_media_image/alt: alt_text + field_mime_type: mime_type + name: name + field_media_of: + plugin: migration_lookup + migration: idc_ingest_new_items + no_stub: true + source: media_of + field_media_use: + - + plugin: skip_on_empty + method: process + source: media_use + - + plugin: explode + delimiter: '|' + - + plugin: entity_lookup + entity_type: taxonomy_term + value_key: name + bundle_key: vid + bundle: islandora_media_use + status: constants/STATUS +destination: + plugin: 'entity:media' + default_bundle: image +migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_images.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_images.yml deleted file mode 100644 index 39b60b67d..000000000 --- a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_images.yml +++ /dev/null @@ -1,58 +0,0 @@ -uuid: 52f4d32a-e761-4f27-8d80-9cc5863a9486 -langcode: en -status: true -dependencies: { } -id: idc_ingest_media_images -class: null -field_plugin_method: null -cck_plugin_method: null -migration_tags: null -migration_group: idc_ingest -label: '(3a) Ingest New Image Media' -source: - plugin: csv - ids: - - local_id - path: 'Will be populated by the Migrate Source UI' - constants: - STATUS: true - ADMIN: 1 -process: - field_media_image/target_id: - plugin: file_import - source: image_url - reuse: false - rename: true - id_only: true - field_mime_type: image_mime_type - name: name - field_media_of: - plugin: migration_lookup - migration: idc_ingest_new_items - source: nodeid_local - no_stub: true - field_media_use: - - - plugin: explode - source: use - delimiter: ; - - - plugin: entity_lookup - entity_type: taxonomy_term - value_key: name - bundle_key: vid - bundle: islandora_media_use - status: constants/STATUS - uid: - - - plugin: entity_lookup - entity_type: user - value_key: name - source: user - - - plugin: default_value - default_value: 1 -destination: - plugin: 'entity:media' - default_bundle: image -migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_remote_video.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_remote_video.yml new file mode 100644 index 000000000..3f924c9d2 --- /dev/null +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_remote_video.yml @@ -0,0 +1,35 @@ +uuid: 3ed66557-28cb-46c2-96a7-92f9f6806887 +langcode: en +status: true +dependencies: { } +id: idc_ingest_media_remote_video +class: null +field_plugin_method: null +cck_plugin_method: null +migration_tags: null +migration_group: idc_ingest +label: '(3f) Ingest Remote Video Media' +source: + plugin: csv + ids: + - local_id + path: 'Will be populated by the Migrate Source UI' + constants: + STATUS: true + DISPLAY: true + ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" +process: + name: name + field_media_oembed_video: url + field_media_of: + plugin: migration_lookup + migration: idc_ingest_new_items + no_stub: true + source: media_of + status: constants/STATUS +destination: + plugin: 'entity:media' + default_bundle: remote_video +migration_dependencies: null diff --git a/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml new file mode 100644 index 000000000..499b60e53 --- /dev/null +++ b/codebase/config/sync/migrate_plus.migration.idc_ingest_media_video.yml @@ -0,0 +1,92 @@ +uuid: 90046574-3b49-421a-a94b-05138c841f3e +langcode: en +status: true +dependencies: { } +id: idc_ingest_media_video +class: null +field_plugin_method: null +cck_plugin_method: null +migration_tags: null +migration_group: idc_ingest +label: '(3e) Ingest New Video Media' +source: + plugin: csv + ids: + - local_id + path: 'Will be populated by the Migrate Source UI' + constants: + STATUS: true + DISPLAY: true + ADMIN: 1 + PUBLIC_FS: "public://" + TMP_FS: "/tmp/" +process: + _url_filename: + - plugin: callback + callable: parse_url + source: url + - plugin: extract + index: + - path + - plugin: callback + callable: basename + _url_filepath: + plugin: concat + source: + - constants/TMP_FS + - '@_url_filename' + _download_filepath: + plugin: file_copy + source: + - url + - '@_url_filepath' + file_exists: 1 + _file_sha: + plugin: callback + callable: sha1_file + source: '@_download_filepath' + _destination_filepath: + plugin: pairtree + source: '@_file_sha' + _destination_drupalpath: + plugin: concat + source: + - constants/PUBLIC_FS + - '@_destination_filepath' + field_original_name: original_name + field_media_video_file/target_id: + plugin: file_import + source: '@_download_filepath' + move: true + reuse: false + rename: false + id_only: true + destination: '@_destination_drupalpath' + mimetype: mime_type + filename: name + field_mime_type: mime_type + name: name + field_media_of: + plugin: migration_lookup + migration: idc_ingest_new_items + no_stub: true + source: media_of + field_media_use: + - + plugin: skip_on_empty + method: process + source: media_use + - + plugin: explode + delimiter: '|' + - + plugin: entity_lookup + entity_type: taxonomy_term + value_key: name + bundle_key: vid + bundle: islandora_media_use + status: constants/STATUS +destination: + plugin: 'entity:media' + default_bundle: video +migration_dependencies: null diff --git a/tests/10-migration-backend-tests.sh b/tests/10-migration-backend-tests.sh index 4933680d9..c1932d571 100755 --- a/tests/10-migration-backend-tests.sh +++ b/tests/10-migration-backend-tests.sh @@ -1,10 +1,21 @@ -#!/bin/sh +#!/bin/bash set -e BASE_TEST_FOLDER=10-migration-backend-tests DRUPAL_CONTAINER_NAME=$(docker ps | awk '{print $NF}'|grep drupal) CURRENT_DIR=$(pwd) +# The Docker registry used to obtain the migration assets image +assets_repo=${MIGRATION_ASSETS_REPO:-ghcr.io/jhu-sheridan-libraries/idc-isle-dc} +# The name of the Docker image for migration assets +assets_image=${MIGRATION_ASSETS_IMAGE:-migration-assets} +# The migration assets image tag +assets_image_tag=${MIGRATION_ASSETS_IMAGE_TAG:-088c482.1617637226} +# The *external* port the migration assets HTTP server listens on +ext_assets_port=${MIGRATION_ASSETS_PORT:-8081} +# The name used by the migration assets container +assets_container=${MIGRATION_ASSETS_CONTAINER:-migration-assets} + # Locate test directory if [ ! -d ${BASE_TEST_FOLDER} ] ; then @@ -22,17 +33,22 @@ fi TESTCAFE_TESTS_FOLDER=$(pwd)/testcafe +# Start the backend that serves the media files to be migrated +# Listens internally on port 80 (addressed as http:///assets/) +docker run --name ${assets_container} --network gateway --rm -d ${assets_repo}/${assets_image}:${assets_image_tag} +trap "docker stop ${assets_container}" EXIT + # Execute migrations using testcafe docker run --network gateway -v "${TESTCAFE_TESTS_FOLDER}":/tests testcafe/testcafe --screenshots path=/tests/screenshots,takeOnFails=true chromium /tests/**/*.js # Verify migrations using go - # Build docker image (TODO: should it be defined in docker-compose.yml to avoid any env issues?) -docker build -t local:migration-backend-tests ./verification +docker build -t local/migration-backend-tests ./verification # Execute tests in docker image, on the same docker network (gateway, idc_default?) as Drupal # TODO: expose logs when failing tests? -docker run --network gateway --rm local:migration-backend-tests +# N.B. trailing slash on the BASE_ASSETS_URL is important. uses the internal URL. +docker run --network gateway --rm -e BASE_ASSETS_URL=http://${assets_container}/assets/ local/migration-backend-tests # Back to parent directory cd ${CURRENT_DIR} diff --git a/tests/10-migration-backend-tests/README.md b/tests/10-migration-backend-tests/README.md index 11f5637f0..15f716266 100644 --- a/tests/10-migration-backend-tests/README.md +++ b/tests/10-migration-backend-tests/README.md @@ -46,13 +46,19 @@ Migration tests may be invoked in isolation by running the `10-migration-backend The migration tests do participate in the general IDC test framework, and should run automatically when `make test` is invoked. +The tests will attempt to start an HTTP server on port 8081. If that port is already in use on your machine, test startup will fail. Use a different port by setting `MIGRATION_ASSETS_PORT` to an open port when running tests: + + MIGRATION_ASSETS_PORT=9090 ./10-migration-backend-tests.sh + +Migration assets can be reached at `http://localhost:${MIGRATION_ACCESS_PORT}/assets/` (note the trailing slash). + ## How the tests work - an overview The `10-migration-backend-tests.sh` script is the "controller" of the tests. It is responsible for executing the various test frameworks and controls the shell exit code. Each test framework executes in a Docker container, so there are no dependencies or configuration required to perform the tests, except for a working Docker. -The controller script invokes testcafe first, which will perform migrations that result in resources being created in Drupal. Next, the controller will invoke Go tests which verify the resources were created correctly (e.g. that the data in the migration CSV files are present in the Drupal resources). +The controller script invokes testcafe first, which will perform migrations that result in resources being created in Drupal. Next, the controller will start an HTTP server which provides access to binary files used for the media tests. Finally, the Go tests are invoked which verify the resources were created correctly (e.g. that the data in the migration CSV files are present in the Drupal resources). -Because one test framework creates resources and another test framework verifies the resources, there is unfortunate coupling that may not be readily apparent. If the CSV files in testcafe are modified, the verification code must almost certainly be updated to account for the changes in the Drupal resources. +Because one test framework (testcafe) creates resources and another test framework (go) verifies the resources, there is unfortunate coupling that may not be readily apparent. If the CSV files in testcafe are modified, the verification code must almost certainly be updated to account for the changes in the Drupal resources. ### Testcafe performs the migration diff --git a/tests/10-migration-backend-tests/testcafe/migrate_tests.spec.js b/tests/10-migration-backend-tests/testcafe/migrate_tests.spec.js index 1d0f06475..5811a9870 100644 --- a/tests/10-migration-backend-tests/testcafe/migrate_tests.spec.js +++ b/tests/10-migration-backend-tests/testcafe/migrate_tests.spec.js @@ -18,7 +18,13 @@ const migrate_geolocation_taxonomy = 'idc_ingest_taxonomy_geolocation'; const migrate_language_taxonomy = 'idc_ingest_taxonomy_language'; const migrate_new_items = 'idc_ingest_new_items'; const migrate_new_collection = 'idc_ingest_new_collection'; -const migrate_media_images = 'idc_ingest_media_images'; +const migrate_media_image = 'idc_ingest_media_image'; +const migrate_media_document = 'idc_ingest_media_document'; +const migrate_media_extracted_text = 'idc_ingest_media_extracted_text'; +const migrate_media_file = 'idc_ingest_media_file'; +const migrate_media_video = 'idc_ingest_media_video'; +const migrate_media_remote_video = 'idc_ingest_media_remote_video'; +const migrate_media_audio = 'idc_ingest_media_audio'; const migrate_resource_types = 'idc_ingest_taxonomy_resourcetypes'; const migrate_subject_taxonomy = 'idc_ingest_taxonomy_subject'; const migrate_corporatebody_taxonomy = 'idc_ingest_taxonomy_corporatebody'; @@ -310,16 +316,97 @@ test('Perform Repository Object Migration', async t => { }); -test('Perform Image Media Migration', async t => { +test('Perform Media Migrations', async t => { - await t - .click(selectMigration) - .click(migrationOptions.withAttribute('value', migrate_media_images)); + // Migrate the Collection and Repository Object the Media will be attached to - await t - .setFilesToUpload('#edit-source-file', [ - './migrations/image-media.csv' - ]) - .click('#edit-import'); + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_new_collection)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-collection.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_new_items)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-islandora_object.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_image)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-image.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_document)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-document.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_extracted_text)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-extracted_text.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_file)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-file.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_video)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-video.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_remote_video)); + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-remote_video.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_audio)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/media-audio.csv' + ]) + .click('#edit-import'); }); diff --git a/tests/10-migration-backend-tests/testcafe/migrations/image-media.csv b/tests/10-migration-backend-tests/testcafe/migrations/image-media.csv deleted file mode 100644 index b6b536ab0..000000000 --- a/tests/10-migration-backend-tests/testcafe/migrations/image-media.csv +++ /dev/null @@ -1,2 +0,0 @@ -local_id,name,nodeid_local,use,image_url,image_title,image_alt_text,image_mime_type -media_01,moonrise.jpeg,repo_obj_49,Original File;Preservation Master File,https://drive.google.com/uc?export=download&id=1A1bWYtNnkKwff8B7XsjJpmzU66cLfb74,,,image/jpeg diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-audio.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-audio.csv new file mode 100644 index 000000000..344b42db5 --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-audio.csv @@ -0,0 +1,2 @@ +local_id,name,original_name,mime_type,media_of,media_use,url +media_audio_00001,Moo Cow,moo.mp3,audio/mpeg,media_io_01,Original File|Preservation Master File,http://migration-assets/assets/audio/moo.mp3 diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-collection.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-collection.csv new file mode 100644 index 000000000..f49923cbc --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-collection.csv @@ -0,0 +1,2 @@ +local_id,title,title_language,alternative_title,member_of,contact_email,contact_name,collection_number,description,finding_aid +media-collection-01,Media Collection,eng,,,,,,Collection for Media Migrations;eng, diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-document.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-document.csv new file mode 100644 index 000000000..5a1f97cf7 --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-document.csv @@ -0,0 +1,3 @@ +local_id,name,original_name,mime_type,media_of,media_use,url +media_doc_00001,Fuji Acros Datasheet,Fuji_acros.pdf,application/pdf,media_io_01,Original File|Preservation Master File,http://migration-assets/assets/document/Fuji_acros.pdf +media_doc_00002,Fuji Acros Datasheet,Fuji_acros.pdf,application/pdf,media_io_02,Original File|Preservation Master File,http://migration-assets/assets/document/Fuji_acros.pdf \ No newline at end of file diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-extracted_text.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-extracted_text.csv new file mode 100644 index 000000000..1b7d4061c --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-extracted_text.csv @@ -0,0 +1,2 @@ +local_id,name,mime_type,media_of,media_use,url +media_ext_00001,Hello World,text/plain,media_io_01,Original File|Preservation Master File,http://migration-assets/assets/extracted_text/hello_world.txt diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-file.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-file.csv new file mode 100644 index 000000000..37d0e47c5 --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-file.csv @@ -0,0 +1,2 @@ +local_id,name,original_name,mime_type,media_of,media_use,url +media_file_00001,Ilford Temperature Compensation Chart,ilford_temperature-compensation-chart.pdf,application/pdf,media_io_01,Original File|Preservation Master File,http://migration-assets/assets/file/ilford_temperature-compensation-chart.pdf diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-image.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-image.csv new file mode 100644 index 000000000..f19c88353 --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-image.csv @@ -0,0 +1,2 @@ +local_id,name,original_name,mime_type,media_of,media_use,url,alt_text +media_img_00001,Looking For Fossils,TRP_7767.jpg,image/jpeg,media_io_01,Original File|Preservation Master File,http://migration-assets/assets/image/TRP_7767.jpg,Image alt text \ No newline at end of file diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-islandora_object.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-islandora_object.csv new file mode 100644 index 000000000..e8d403d76 --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-islandora_object.csv @@ -0,0 +1,3 @@ +local_id,title,abstract,access_rights,access_terms,alt_title,collection_number,contributor,copyright,copyright_holder,creator,date_available,date_copyrighted,date_created,date_published,description,digital_identifier,digital_publisher,display_hints,dspace_identifier,dspace_itemid,extent,finding_aid,genre,geoportal_link,issn,is_part_of,item_barcode,jhir_uri,language,library_catalog_link,member_of,model,oclc_number,publisher,publisher_country,resource_type,spatial_coverage,subject,table_of_contents,title_language,years +media_io_01,Media Migration Repository Item One,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,media-collection-01,,,,,,,,,, +media_io_02,Media Migration Repository Item Two,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,media-collection-01,,,,,,,,,, \ No newline at end of file diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-remote_video.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-remote_video.csv new file mode 100644 index 000000000..6d65a9f94 --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-remote_video.csv @@ -0,0 +1,2 @@ +local_id,name,media_of,url +media_rvideo_00001,A Tour of Go,media_io_01,https://www.youtube.com/watch?v=ytEkHepK08c diff --git a/tests/10-migration-backend-tests/testcafe/migrations/media-video.csv b/tests/10-migration-backend-tests/testcafe/migrations/media-video.csv new file mode 100644 index 000000000..5eb21947c --- /dev/null +++ b/tests/10-migration-backend-tests/testcafe/migrations/media-video.csv @@ -0,0 +1,2 @@ +local_id,name,original_name,mime_type,media_of,media_use,url +media_video_00001,Chair Pop Video,chair-pop-gif.mp4,video/mp4,media_io_01,Original File|Preservation Master File,http://migration-assets/assets/video/chair-pop-gif.mp4 diff --git a/tests/10-migration-backend-tests/verification/expected/deletion-file.json b/tests/10-migration-backend-tests/verification/expected/deletion-file.json new file mode 100644 index 000000000..d5d06c5ad --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/deletion-file.json @@ -0,0 +1,17 @@ +{ + "type": "media", + "bundle": "file", + "name": "FP4 Datasheet", + "original_name": "ilford_fp4.pdf", + "size": 413378, + "mime_type": "application/pdf", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "File Deletion Repository Item One", + "uri": { + "url": "/sites/default/files/04/7f/86/c0c26cf42ee9c6eb17910599d3802d2f98", + "value": "public://04/7f/86/c0c26cf42ee9c6eb17910599d3802d2f98" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-audio.json b/tests/10-migration-backend-tests/verification/expected/media-audio.json new file mode 100644 index 000000000..0487b3239 --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-audio.json @@ -0,0 +1,17 @@ +{ + "type": "media", + "bundle": "audio", + "name": "Moo Cow", + "original_name": "moo.mp3", + "size": 82539, + "mime_type": "audio/mpeg", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "Media Migration Repository Item One", + "uri": { + "url": "/sites/default/files/76/6d/51/a112896d48d3457fb3c5fa5fbdf661713d", + "value": "public://76/6d/51/a112896d48d3457fb3c5fa5fbdf661713d" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-document.json b/tests/10-migration-backend-tests/verification/expected/media-document.json new file mode 100644 index 000000000..749c46739 --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-document.json @@ -0,0 +1,17 @@ +{ + "type": "media", + "bundle": "document", + "name": "Fuji Acros Datasheet", + "original_name": "Fuji_acros.pdf", + "size": 98884, + "mime_type": "application/pdf", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "Media Migration Repository Item One", + "uri": { + "url": "/sites/default/files/c9/a0/60/c39365820edc5d1a51f221d49e96a8a730", + "value": "public://c9/a0/60/c39365820edc5d1a51f221d49e96a8a730" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-extracted_text.json b/tests/10-migration-backend-tests/verification/expected/media-extracted_text.json new file mode 100644 index 000000000..19b9f4aa2 --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-extracted_text.json @@ -0,0 +1,22 @@ +{ + "type": "media", + "bundle": "extracted_text", + "name": "Hello World", + "original_name": "hello_world.txt", + "size": 29, + "mime_type": "text/plain", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "Media Migration Repository Item One", + "uri": { + "url": "/sites/default/files/53/06/43/aaed48b24d41bbe075579df620204d06d1", + "value": "public://53/06/43/aaed48b24d41bbe075579df620204d06d1" + }, + "extracted_text": { + "value": "Hello, extracted text world!
\n", + "format": "basic_html", + "processed": "Hello, extracted text world!
" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-file.json b/tests/10-migration-backend-tests/verification/expected/media-file.json new file mode 100644 index 000000000..f5611eda0 --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-file.json @@ -0,0 +1,17 @@ +{ + "type": "media", + "bundle": "file", + "name": "Ilford Temperature Compensation Chart", + "original_name": "ilford_temperature-compensation-chart.pdf", + "size": 13507, + "mime_type": "application/pdf", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "Media Migration Repository Item One", + "uri": { + "url": "/sites/default/files/cd/95/2b/54f0e4b3fea3082a3d3297c39f8052c9f7", + "value": "public://cd/95/2b/54f0e4b3fea3082a3d3297c39f8052c9f7" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-image.json b/tests/10-migration-backend-tests/verification/expected/media-image.json new file mode 100644 index 000000000..3a34771cb --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-image.json @@ -0,0 +1,20 @@ +{ + "type": "media", + "bundle": "image", + "name": "Looking For Fossils", + "original_name": "TRP_7767.jpg", + "alt_text": "Image alt text", + "size": 7584545, + "height": 2239, + "width": 3378, + "mime_type": "image/jpeg", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "Media Migration Repository Item One", + "uri": { + "url": "/sites/default/files/2021-03/trp_7767.jpg", + "value": "public://2021-03/trp_7767.jpg" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-remote_video.json b/tests/10-migration-backend-tests/verification/expected/media-remote_video.json new file mode 100644 index 000000000..dca8690cd --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-remote_video.json @@ -0,0 +1,7 @@ +{ + "type": "media", + "bundle": "remote_video", + "name": "A Tour of Go", + "media_of": "Media Migration Repository Item One", + "embed_url": "https://www.youtube.com/watch?v=ytEkHepK08c" +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected/media-video.json b/tests/10-migration-backend-tests/verification/expected/media-video.json new file mode 100644 index 000000000..1614da409 --- /dev/null +++ b/tests/10-migration-backend-tests/verification/expected/media-video.json @@ -0,0 +1,17 @@ +{ + "type": "media", + "bundle": "video", + "name": "Chair Pop Video", + "original_name": "chair-pop-gif.mp4", + "size": 214707, + "mime_type": "video/mp4", + "use": [ + "Preservation Master File", + "Original File" + ], + "media_of": "Media Migration Repository Item One", + "uri": { + "url": "/sites/default/files/34/e9/34/86f9a355930ac2c14f067082e6d584ea24", + "value": "public://34/e9/34/86f9a355930ac2c14f067082e6d584ea24" + } +} \ No newline at end of file diff --git a/tests/10-migration-backend-tests/verification/expected_json_types_test.go b/tests/10-migration-backend-tests/verification/expected_json_types_test.go index 3878bedd1..96c80a2ec 100644 --- a/tests/10-migration-backend-tests/verification/expected_json_types_test.go +++ b/tests/10-migration-backend-tests/verification/expected_json_types_test.go @@ -307,3 +307,42 @@ type LanguageString struct { Value string LangCode string `json:"language"` } + +type ExpectedMediaGeneric struct { + Type string + Bundle string + Name string + OriginalName string `json:"original_name"` + Size int + MimeType string `json:"mime_type"` + MediaUse []string `json:"use"` + MediaOf string `json:"media_of"` + Uri struct { + Url string + Value string + } +} + +type ExpectedMediaImage struct { + ExpectedMediaGeneric + AltText string `json:"alt_text"` + Height int + Width int +} + +type ExpectedMediaExtractedText struct { + ExpectedMediaGeneric + ExtractedText struct { + Value string + Format string + Processed string + } `json:"extracted_text"` +} + +type ExpectedMediaRemoteVideo struct { + Type string + Bundle string + Name string + EmbedUrl string `json:"embed_url"` + MediaOf string `json:"media_of"` +} diff --git a/tests/10-migration-backend-tests/verification/go.mod b/tests/10-migration-backend-tests/verification/go.mod index 653ae856c..c51de50a8 100644 --- a/tests/10-migration-backend-tests/verification/go.mod +++ b/tests/10-migration-backend-tests/verification/go.mod @@ -2,4 +2,7 @@ module 10-migration-backend-tests go 1.15 -require github.com/stretchr/testify v1.6.1 +require ( + github.com/logrusorgru/aurora/v3 v3.0.0 + github.com/stretchr/testify v1.6.1 +) diff --git a/tests/10-migration-backend-tests/verification/go.sum b/tests/10-migration-backend-tests/verification/go.sum index 1f1e7af97..05910779d 100644 --- a/tests/10-migration-backend-tests/verification/go.sum +++ b/tests/10-migration-backend-tests/verification/go.sum @@ -1,8 +1,9 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= +github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/tests/10-migration-backend-tests/verification/jsonapi_types_test.go b/tests/10-migration-backend-tests/verification/jsonapi_types_test.go index d9d8cdb2c..399f24fe6 100644 --- a/tests/10-migration-backend-tests/verification/jsonapi_types_test.go +++ b/tests/10-migration-backend-tests/verification/jsonapi_types_test.go @@ -41,13 +41,24 @@ func (json *JsonApiUrl) String() string { return u.String() } -func (jar *JsonApiUrl) get(v interface{}) { +// Get the JSON API content from the URL and unmarshal the response into the supplied interface (which must be a +// pointer). This method asserts that there is a single object in the `data` element of the JSON response. +func (jar *JsonApiUrl) getSingle(v interface{}) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res, body := getResource(jar.t.(*testing.T), jar.String()) defer func() { _ = res.Close }() unmarshalSingleResponse(jar.t.(*testing.T), body, res, &JsonApiResponse{}).to(v) } +// Get the JSON API content from the URL and unmarshal the response into the supplied interface (which must be a +// pointer). +func (jar *JsonApiUrl) get(v interface{}) { + // retrieve json of the migrated entity from the jsonapi and unmarshal the single response + res, body := getResource(jar.t.(*testing.T), jar.String()) + defer func() { _ = res.Close }() + unmarshalResponse(jar.t.(*testing.T), body, res, &JsonApiResponse{}, nil).to(v) +} + // Encapsulates a generic JSON API response type JsonApiResponse struct { Data []map[string]interface{} @@ -104,7 +115,7 @@ func (jad *JsonApiData) resolve(t *testing.T, v interface{}) { value: jad.Id, } - u.get(v) + u.getSingle(v) } // Represents the results of a JSONAPI query for a single Person from the Person Taxonomy @@ -651,3 +662,184 @@ func (rd RelData) metaInt(field string) (int, error) { return -1, fmt.Errorf("%w: %s", ErrMissing, field) } + +// https://islandora-idc.traefik.me/jsonapi/media/image?filter[id]=090690a5-4db5-4d72-a94e-3b26a90b516b +type JsonApiImageMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + JsonApiMediaAttributes + JsonApiImageMediaAttributes + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + File struct { + Data RelData + } `json:"field_media_image"` + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiMediaAttributes struct { + FileSize int `json:"field_file_size"` + MimeType string `json:"field_mime_type"` + OriginalName string `json:"field_original_name"` + Name string +} + +type JsonApiMediaRelationships struct { + MediaUse struct { + Data []JsonApiData + } `json:"field_media_use"` + MediaOf struct { + Data JsonApiData + } `json:"field_media_of"` +} + +type JsonApiImageMediaAttributes struct { + Height int `json:"field_height"` + Width int `json:"field_width"` +} + +type JsonApiDocumentMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + JsonApiMediaAttributes + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + File struct { + Data RelData + } `json:"field_media_document"` + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiAudioMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + JsonApiMediaAttributes + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + File struct { + Data RelData + } `json:"field_media_audio_file"` + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiExtractedTextMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + JsonApiMediaAttributes + JsonApiExtractedTextMediaAttributes + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + File struct { + Data RelData + } `json:"field_media_file"` + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiExtractedTextMediaAttributes struct { + EditedText struct { + Value string + Format string + Processed string + } `json:"field_edited_text"` +} + +type JsonApiGenericFileMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + JsonApiMediaAttributes + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + File struct { + Data RelData + } `json:"field_media_file"` + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiRemoteVideoMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + Name string + EmbedUrl string `json:"field_media_oembed_video"` + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiVideoMedia struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + JsonApiMediaAttributes + } `json:"attributes"` + JsonApiRelationships struct { + JsonApiMediaRelationships + File struct { + Data RelData + } `json:"field_media_video_file"` + } `json:"relationships"` + } `json:"data"` +} + +type JsonApiFile struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + Filename string + Uri struct { + Url string + Value string + } + MimeType string `json:"filemime"` + FileSize int + CreatedDate string `json:"created"` + ChangedDate string `json:"changed"` + } `json:"attributes"` + } `json:"data"` +} + +type JsonApiMediaUse struct { + JsonApiData []struct { + Type DrupalType + Id string + JsonApiAttributes struct { + Name string + Description struct { + Value string + Format string + Processed string + } + ExternalUri struct { + Uri string + Title string + } `json:"field_external_uri"` + } `json:"attributes"` + JsonApiRelationships struct { + } `json:"relationships"` + } `json:"data"` +} diff --git a/tests/10-migration-backend-tests/verification/verify_migrations_test.go b/tests/10-migration-backend-tests/verification/verify_migrations_test.go index 1839fd38d..3c37d399d 100644 --- a/tests/10-migration-backend-tests/verification/verify_migrations_test.go +++ b/tests/10-migration-backend-tests/verification/verify_migrations_test.go @@ -1,15 +1,19 @@ package main import ( + "crypto/sha1" "encoding/json" "errors" "fmt" + . "github.com/logrusorgru/aurora/v3" "github.com/stretchr/testify/assert" + "io" "io/ioutil" "log" "net/http" "os" "path/filepath" + "strings" "testing" ) @@ -23,8 +27,35 @@ const ( // The base URL of the test instance of IDC. // TODO: consult env DrupalBaseurl = "https://islandora-idc.traefik.me" + + // Env var name for the base URL to media assets + AssetsBaseUrl = "BASE_ASSETS_URL" ) +func TestMain(m *testing.M) { + var ( + res *http.Response + err error + ) + + assetsUrl := os.Getenv(AssetsBaseUrl) + if assetsUrl != "" { + if res, err = http.Get(assetsUrl); err != nil { + log.Println(Sprintf(Red("Assets container (%s) is not up, media tests will fail: %s"), assetsUrl, BrightRed(err.Error()))) + } else { + defer res.Body.Close() + if res.StatusCode != 200 { + log.Println(Sprintf(Red("Unexpected status code %d from %s, media tests will fail."), + res.StatusCode, assetsUrl)) + } + } + } else { + log.Println(Sprintf(Red("%s env var is not defined, media tests will fail."), AssetsBaseUrl)) + } + + os.Exit(m.Run()) +} + // Verifies that the Person migrated by testcafe persons-01.csv and persons-02.csv // match the expected fields and values present in taxonomy-person-01.json func Test_VerifyTaxonomyTermPerson_Person1(t *testing.T) { @@ -57,7 +88,7 @@ func verifyTaxonomyTermPerson(t *testing.T, fileName string, restOfName string) // retrieve json of the migrated entity from the jsonapi and unmarshal the single response personRes := &JsonApiPerson{} - u.get(personRes) + u.getSingle(personRes) // for each field in expected json, // see if the expected field matches the actual field from retrieved json @@ -89,7 +120,7 @@ func verifyTaxonomyTermPerson(t *testing.T, fileName string, restOfName string) // retrieve json of the resolved entity from the jsonapi personRes = &JsonApiPerson{} - u.get(personRes) + u.getSingle(personRes) relSchemaKnows := personRes.JsonApiData[0] // sanity @@ -119,7 +150,7 @@ func Test_VerifyTaxonomyTermAccessRights(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response accessRightsRes := &JsonApiAccessRights{} - u.get(accessRightsRes) + u.getSingle(accessRightsRes) actual := accessRightsRes.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -155,7 +186,7 @@ func Test_VerifyTaxonomyCopyrightAndUse(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response copyrightRes := &JsonApiCopyrightAndUse{} - u.get(copyrightRes) + u.getSingle(copyrightRes) actual := copyrightRes.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -191,7 +222,7 @@ func Test_VerifyTaxonomyTermFamily(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response familyres := &JsonApiFamily{} - u.get(familyres) + u.getSingle(familyres) sourceId := familyres.JsonApiData[0].Id assert.NotEmpty(t, sourceId) @@ -223,7 +254,7 @@ func Test_VerifyTaxonomyTermFamily(t *testing.T) { // retrieve json of the resolved entity from the jsonapi familyres = &JsonApiFamily{} - u.get(familyres) + u.getSingle(familyres) relSchemaKnowsAbout := familyres.JsonApiData[0] // sanity @@ -256,7 +287,7 @@ func Test_VerifyTaxonomyTermGenre(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response genreRes := &JsonApiGenre{} - u.get(genreRes) + u.getSingle(genreRes) actual := genreRes.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -292,7 +323,7 @@ func Test_VerifyTaxonomyTermGeolocation(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiGeolocation{} - u.get(res) + u.getSingle(res) actual := res.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -337,7 +368,7 @@ func Test_VerifyTaxonomyTermResourceType(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiResourceType{} - u.get(res) + u.getSingle(res) actual := res.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -373,7 +404,7 @@ func Test_VerifyTaxonomySubject(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiSubject{} - u.get(res) + u.getSingle(res) actual := res.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -409,7 +440,7 @@ func Test_VerifyTaxonomyTermLanguage(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiLanguage{} - u.get(res) + u.getSingle(res) actual := res.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -446,7 +477,7 @@ func Test_VerifyTaxonomyTermCorporateBody(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiCorporateBody{} - u.get(res) + u.getSingle(res) actual := res.JsonApiData[0] assert.Equal(t, expectedJson.Type, actual.Type.entity()) @@ -487,7 +518,7 @@ func Test_VerifyTaxonomyTermCorporateBody(t *testing.T) { value: relData[0].Id, } target := &JsonApiCorporateBody{} - u.get(target) + u.getSingle(target) assert.Equal(t, expectedJson.Relationship[0].Name, target.JsonApiData[0].JsonApiAttributes.Name) // "Parent Organization" -> 'schema:subOrganization' -> "My Corporate Body" @@ -514,7 +545,7 @@ func Test_VerifyCollection(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiCollection{} - u.get(res) + u.getSingle(res) sourceId := res.JsonApiData[0].Id assert.NotEmpty(t, sourceId) @@ -573,7 +604,7 @@ func Test_VerifyCollection(t *testing.T) { value: memberOfData.Id, } memberCol := JsonApiCollection{} - u.get(&memberCol) + u.getSingle(&memberCol) assert.Equal(t, expectedJson.MemberOf[i], memberCol.JsonApiData[0].JsonApiAttributes.Title) @@ -599,7 +630,7 @@ func Test_VerifyRepositoryItem(t *testing.T) { // retrieve json of the migrated entity from the jsonapi and unmarshal the single response res := &JsonApiIslandoraObj{} - u.get(res) + u.getSingle(res) actual := res.JsonApiData[0] sourceId := actual.Id assert.NotEmpty(t, sourceId) @@ -841,8 +872,426 @@ func Test_VerifyRepositoryItem(t *testing.T) { } } -func Test_VerifyMediaAndFile(t *testing.T) { +// Two media with identical file content will have different File entities, but each File entity will reference the +// the same file URI. The file URI should be based on the checksum of the bytestream's content. Allowing different +// File entities allows the same bytestream to have different file metadata (i.e. be known by one name in one Media, +// and known by a different name in another Media). +func Test_VerifyDuplicateMediaAndFile(t *testing.T) { + // There are two Media with this name that were migrated by testcafe; they use the same file, so the File entity + // linked by these Media should be byte-for-byte identical. The File entities will be different, but their URIs + // will reference the same content. + name := "Fuji Acros Datasheet" + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: "media", + drupalBundle: "document", + filter: "name", + value: name, + } + + res := JsonApiDocumentMedia{} + u.get(&res) + + // Sanity check the response contains what we expect + assert.NotEmpty(t, res) + assert.Equal(t, 2, len(res.JsonApiData)) + for i := range res.JsonApiData { + assert.Equal(t, name, res.JsonApiData[i].JsonApiAttributes.Name) + } + + var ( + fileEntityId string + fileEntityUri string + resolvedFiles []JsonApiFile + ) + + // The two media should have different File entities + for i := range res.JsonApiData { + if fileEntityId == "" { + fileEntityId = res.JsonApiData[i].JsonApiRelationships.File.Data.Id + } else { + assert.NotEqual(t, fileEntityId, res.JsonApiData[i].JsonApiRelationships.File.Data.Id) + } + + // (while we're ranging over the response data, resolve the file entities) + file := JsonApiFile{} + res.JsonApiData[i].JsonApiRelationships.File.Data.resolve(t, &file) + resolvedFiles = append(resolvedFiles, file) + } + + // sanity + assert.Equal(t, 2, len(resolvedFiles)) + + // The two media should have the same URI (because the bytestreams are identical) + for i := range resolvedFiles { + if fileEntityUri == "" { + fileEntityUri = resolvedFiles[i].JsonApiData[0].JsonApiAttributes.Uri.Value + } else { + assert.Equal(t, fileEntityUri, resolvedFiles[i].JsonApiData[0].JsonApiAttributes.Uri.Value) + } + } + + // download one of the files and confirm the URI is based on the checksum of the content + var ( + fileRes *http.Response + err error + ) + // TODO obtain from env + baseUri := "https://islandora-idc.traefik.me/" + fileUrl := fmt.Sprintf("%s%s", baseUri, resolvedFiles[0].JsonApiData[0].JsonApiAttributes.Uri.Url) + // TODO: set truncate to false in migration def + // public://c9/a0/60/c39365820edc5d1a51f221d49e96a8a730 -> c9a060c39365820edc5d1a51f221d49e96a8a730 + expectedChecksum := strings.ReplaceAll(strings.ReplaceAll(resolvedFiles[0].JsonApiData[0].JsonApiAttributes.Uri.Value, "/", ""), "public:", "") + fileRes, err = http.Get(fileUrl) + assert.Nil(t, err) + defer fileRes.Body.Close() + assert.Equal(t, 200, fileRes.StatusCode) + hash := sha1.New() + io.Copy(hash, fileRes.Body) + actualChecksum := hash.Sum(nil) + assert.Equal(t, expectedChecksum, fmt.Sprintf("%x", actualChecksum)) +} + +func Test_VerifyMediaDocument(t *testing.T) { + expectedJson := &ExpectedMediaGeneric{} + unmarshalJson(t, "media-document.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, "media", expectedJson.Type) + assert.Equal(t, "document", expectedJson.Bundle) + + // There are two media with name that were migrated by testcafe + name := "Fuji Acros Datasheet" + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: "media", + drupalBundle: "document", + filter: "name", + value: name, + } + + res := JsonApiDocumentMedia{} + u.get(&res) + + // use the first media + document := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Size, document.JsonApiAttributes.FileSize) + assert.Equal(t, expectedJson.MimeType, document.JsonApiAttributes.MimeType) + assert.Equal(t, expectedJson.OriginalName, document.JsonApiAttributes.OriginalName) + assert.Equal(t, expectedJson.Name, document.JsonApiAttributes.Name) + + // Resolve relationships and verify + + assert.Equal(t, 2, len(expectedJson.MediaUse)) + assert.Equal(t, len(expectedJson.MediaUse), len(document.JsonApiRelationships.MediaUse.Data)) + for i := range document.JsonApiRelationships.MediaUse.Data { + use := JsonApiMediaUse{} + document.JsonApiRelationships.MediaUse.Data[i].resolve(t, &use) + assert.Equal(t, expectedJson.MediaUse[i], use.JsonApiData[0].JsonApiAttributes.Name) + } + + mediaOf := JsonApiIslandoraObj{} + document.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) +} + +func Test_VerifyMediaImage(t *testing.T) { + expectedJson := &ExpectedMediaImage{} + unmarshalJson(t, "media-image.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, "media", expectedJson.Type) + assert.Equal(t, "image", expectedJson.Bundle) + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: "media", + drupalBundle: "image", + filter: "name", + value: "Looking For Fossils", + } + + res := JsonApiImageMedia{} + u.getSingle(&res) + + // use the first media + image := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Size, image.JsonApiAttributes.FileSize) + assert.Equal(t, expectedJson.MimeType, image.JsonApiAttributes.MimeType) + assert.Equal(t, expectedJson.OriginalName, image.JsonApiAttributes.OriginalName) + assert.Equal(t, expectedJson.Name, image.JsonApiAttributes.Name) + assert.Equal(t, expectedJson.Height, image.JsonApiAttributes.Height) + assert.Equal(t, expectedJson.Width, image.JsonApiAttributes.Width) + + // Resolve relationships and verify + + assert.Equal(t, expectedJson.AltText, image.JsonApiRelationships.File.Data.Meta["alt"]) + + assert.Equal(t, 2, len(expectedJson.MediaUse)) + assert.Equal(t, len(expectedJson.MediaUse), len(image.JsonApiRelationships.MediaUse.Data)) + for i := range image.JsonApiRelationships.MediaUse.Data { + use := JsonApiMediaUse{} + image.JsonApiRelationships.MediaUse.Data[i].resolve(t, &use) + assert.Equal(t, expectedJson.MediaUse[i], use.JsonApiData[0].JsonApiAttributes.Name) + } + + mediaOf := JsonApiIslandoraObj{} + image.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) +} + +func Test_VerifyMediaExtractedText(t *testing.T) { + expectedJson := &ExpectedMediaExtractedText{} + expectedType := "media" + expectedBundle := "extracted_text" + unmarshalJson(t, "media-extracted_text.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, expectedType, expectedJson.Type) + assert.Equal(t, expectedBundle, expectedJson.Bundle) + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: expectedType, + drupalBundle: expectedBundle, + filter: "name", + value: expectedJson.Name, + } + + res := JsonApiExtractedTextMedia{} + u.getSingle(&res) + ext := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Name, ext.JsonApiAttributes.Name) + assert.Equal(t, expectedJson.MimeType, ext.JsonApiAttributes.MimeType) + assert.EqualValues(t, expectedJson.ExtractedText, ext.JsonApiAttributes.EditedText) + + // Resolve relationships and verify + + assert.Equal(t, 2, len(expectedJson.MediaUse)) + assert.Equal(t, len(expectedJson.MediaUse), len(ext.JsonApiRelationships.MediaUse.Data)) + for i := range ext.JsonApiRelationships.MediaUse.Data { + use := JsonApiMediaUse{} + ext.JsonApiRelationships.MediaUse.Data[i].resolve(t, &use) + assert.Equal(t, expectedJson.MediaUse[i], use.JsonApiData[0].JsonApiAttributes.Name) + } + + mediaOf := JsonApiIslandoraObj{} + ext.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) + file := JsonApiFile{} + ext.JsonApiRelationships.File.Data.resolve(t, &file) + assert.EqualValues(t, expectedJson.Uri, file.JsonApiData[0].JsonApiAttributes.Uri) + assert.Equal(t, expectedJson.Size, file.JsonApiData[0].JsonApiAttributes.FileSize) + assert.Equal(t, expectedJson.MimeType, file.JsonApiData[0].JsonApiAttributes.MimeType) + assert.Equal(t, expectedJson.Name, file.JsonApiData[0].JsonApiAttributes.Filename) +} + +func Test_VerifyMediaFile(t *testing.T) { + expectedJson := &ExpectedMediaGeneric{} + expectedType := "media" + expectedBundle := "file" + unmarshalJson(t, "media-file.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, expectedType, expectedJson.Type) + assert.Equal(t, expectedBundle, expectedJson.Bundle) + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: expectedType, + drupalBundle: expectedBundle, + filter: "name", + value: expectedJson.Name, + } + + res := JsonApiGenericFileMedia{} + u.getSingle(&res) + genericFile := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Name, genericFile.JsonApiAttributes.Name) + assert.Equal(t, expectedJson.MimeType, genericFile.JsonApiAttributes.MimeType) + assert.EqualValues(t, expectedJson.OriginalName, genericFile.JsonApiAttributes.OriginalName) + assert.Equal(t, expectedJson.Size, genericFile.JsonApiAttributes.FileSize) + + // Resolve relationships and verify + + assert.Equal(t, 2, len(expectedJson.MediaUse)) + assert.Equal(t, len(expectedJson.MediaUse), len(genericFile.JsonApiRelationships.MediaUse.Data)) + for i := range genericFile.JsonApiRelationships.MediaUse.Data { + use := JsonApiMediaUse{} + genericFile.JsonApiRelationships.MediaUse.Data[i].resolve(t, &use) + assert.Equal(t, expectedJson.MediaUse[i], use.JsonApiData[0].JsonApiAttributes.Name) + } + + mediaOf := JsonApiIslandoraObj{} + genericFile.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) + + file := JsonApiFile{} + genericFile.JsonApiRelationships.File.Data.resolve(t, &file) + assert.EqualValues(t, expectedJson.Uri, file.JsonApiData[0].JsonApiAttributes.Uri) + assert.Equal(t, expectedJson.Size, file.JsonApiData[0].JsonApiAttributes.FileSize) + assert.Equal(t, expectedJson.MimeType, file.JsonApiData[0].JsonApiAttributes.MimeType) + assert.Equal(t, expectedJson.Name, file.JsonApiData[0].JsonApiAttributes.Filename) +} + +func Test_VerifyMediaAudio(t *testing.T) { + expectedJson := &ExpectedMediaGeneric{} + expectedType := "media" + expectedBundle := "audio" + unmarshalJson(t, "media-audio.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, expectedType, expectedJson.Type) + assert.Equal(t, expectedBundle, expectedJson.Bundle) + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: expectedType, + drupalBundle: expectedBundle, + filter: "name", + value: expectedJson.Name, + } + + res := JsonApiAudioMedia{} + u.getSingle(&res) + audio := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Name, audio.JsonApiAttributes.Name) + assert.Equal(t, expectedJson.MimeType, audio.JsonApiAttributes.MimeType) + assert.EqualValues(t, expectedJson.OriginalName, audio.JsonApiAttributes.OriginalName) + assert.Equal(t, expectedJson.Size, audio.JsonApiAttributes.FileSize) + + // Resolve relationships and verify + + assert.Equal(t, 2, len(expectedJson.MediaUse)) + assert.Equal(t, len(expectedJson.MediaUse), len(audio.JsonApiRelationships.MediaUse.Data)) + for i := range audio.JsonApiRelationships.MediaUse.Data { + use := JsonApiMediaUse{} + audio.JsonApiRelationships.MediaUse.Data[i].resolve(t, &use) + assert.Equal(t, expectedJson.MediaUse[i], use.JsonApiData[0].JsonApiAttributes.Name) + } + + mediaOf := JsonApiIslandoraObj{} + audio.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) + + file := JsonApiFile{} + audio.JsonApiRelationships.File.Data.resolve(t, &file) + assert.EqualValues(t, expectedJson.Uri, file.JsonApiData[0].JsonApiAttributes.Uri) + assert.Equal(t, expectedJson.Size, file.JsonApiData[0].JsonApiAttributes.FileSize) + assert.Equal(t, expectedJson.MimeType, file.JsonApiData[0].JsonApiAttributes.MimeType) + assert.Equal(t, expectedJson.Name, file.JsonApiData[0].JsonApiAttributes.Filename) +} + +func Test_VerifyMediaVideo(t *testing.T) { + expectedJson := &ExpectedMediaGeneric{} + expectedType := "media" + expectedBundle := "video" + unmarshalJson(t, "media-video.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, expectedType, expectedJson.Type) + assert.Equal(t, expectedBundle, expectedJson.Bundle) + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: expectedType, + drupalBundle: expectedBundle, + filter: "name", + value: expectedJson.Name, + } + + res := JsonApiVideoMedia{} + u.getSingle(&res) + video := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Name, video.JsonApiAttributes.Name) + assert.Equal(t, expectedJson.MimeType, video.JsonApiAttributes.MimeType) + assert.EqualValues(t, expectedJson.OriginalName, video.JsonApiAttributes.OriginalName) + assert.Equal(t, expectedJson.Size, video.JsonApiAttributes.FileSize) + + // Resolve relationships and verify + + assert.Equal(t, 2, len(expectedJson.MediaUse)) + assert.Equal(t, len(expectedJson.MediaUse), len(video.JsonApiRelationships.MediaUse.Data)) + for i := range video.JsonApiRelationships.MediaUse.Data { + use := JsonApiMediaUse{} + video.JsonApiRelationships.MediaUse.Data[i].resolve(t, &use) + assert.Equal(t, expectedJson.MediaUse[i], use.JsonApiData[0].JsonApiAttributes.Name) + } + + mediaOf := JsonApiIslandoraObj{} + video.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) + + file := JsonApiFile{} + video.JsonApiRelationships.File.Data.resolve(t, &file) + assert.EqualValues(t, expectedJson.Uri, file.JsonApiData[0].JsonApiAttributes.Uri) + assert.Equal(t, expectedJson.Size, file.JsonApiData[0].JsonApiAttributes.FileSize) + assert.Equal(t, expectedJson.MimeType, file.JsonApiData[0].JsonApiAttributes.MimeType) + assert.Equal(t, expectedJson.Name, file.JsonApiData[0].JsonApiAttributes.Filename) +} + +func Test_VerifyMediaRemoteVideo(t *testing.T) { + expectedJson := &ExpectedMediaRemoteVideo{} + expectedType := "media" + expectedBundle := "remote_video" + unmarshalJson(t, "media-remote_video.json", &expectedJson) + + // sanity check the expected json + assert.Equal(t, expectedType, expectedJson.Type) + assert.Equal(t, expectedBundle, expectedJson.Bundle) + + u := &JsonApiUrl{ + t: t, + baseUrl: DrupalBaseurl, + drupalEntity: expectedType, + drupalBundle: expectedBundle, + filter: "name", + value: expectedJson.Name, + } + + res := JsonApiRemoteVideoMedia{} + u.getSingle(&res) + video := res.JsonApiData[0] + + // Verify attributes + + assert.Equal(t, expectedJson.Name, video.JsonApiAttributes.Name) + assert.Equal(t, expectedJson.EmbedUrl, video.JsonApiAttributes.EmbedUrl) + + // Resolve relationships and verify + + // TODO: media_of not supported for remote_video? + //mediaOf := JsonApiIslandoraObj{} + //video.JsonApiRelationships.MediaOf.Data.resolve(t, &mediaOf) + //assert.Equal(t, expectedJson.MediaOf, mediaOf.JsonApiData[0].JsonApiAttributes.Title) } // Searches the file system for the named file. The `name` should not contain any path components or separators. @@ -906,9 +1355,19 @@ func unmarshalJson(t *testing.T, filename string, value interface{}) { // Unmarshal a JSONAPI response body and assert that exactly one data element is present func unmarshalSingleResponse(t *testing.T, body []byte, res *http.Response, value *JsonApiResponse) *JsonApiResponse { + unmarshalResponse(t, body, res, value, func(value *JsonApiResponse) { + assert.Equal(t, 1, len(value.Data), "Exactly one JSONAPI data element is expected in the response, but found %d element(s)", len(value.Data)) + }) + return value +} + +// Unmarshal a JSONAPI response body and perform supplied assertions on the response +func unmarshalResponse(t *testing.T, body []byte, res *http.Response, value *JsonApiResponse, responseAssertions func(res *JsonApiResponse)) *JsonApiResponse { err := json.Unmarshal(body, value) assert.Nil(t, err, "Error unmarshaling JSONAPI response body: %s", err) - assert.Equal(t, 1, len(value.Data), "Exactly one JSONAPI data element is expected in the response, but found %d element(s)", len(value.Data)) + if responseAssertions != nil { + responseAssertions(value) + } return value } diff --git a/tests/11-file-deletion-tests.sh b/tests/11-file-deletion-tests.sh new file mode 100755 index 000000000..c5152f9a1 --- /dev/null +++ b/tests/11-file-deletion-tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +BASE_TEST_FOLDER=11-file-deletion-tests +DRUPAL_CONTAINER_NAME=$(docker ps | awk '{print $NF}'|grep drupal) +CURRENT_DIR=$(pwd) + +# The Docker registry used to obtain the migration assets image +assets_repo=${MIGRATION_ASSETS_REPO:-ghcr.io/jhu-sheridan-libraries/idc-isle-dc} +# The name of the Docker image for migration assets +assets_image=${MIGRATION_ASSETS_IMAGE:-migration-assets} +# The migration assets image tag +assets_image_tag=${MIGRATION_ASSETS_IMAGE_TAG:-088c482.1617637226} +# The *external* port the migration assets HTTP server listens on +ext_assets_port=${MIGRATION_ASSETS_PORT:-8081} +# The name used by the migration assets container +assets_container=${MIGRATION_ASSETS_CONTAINER:-migration-assets} + +# Locate test directory +if [ ! -d ${BASE_TEST_FOLDER} ] ; +then + BASE_TEST_FOLDER=tests/${BASE_TEST_FOLDER} + if [ ! -d ${BASE_TEST_FOLDER} ] ; + then + echo "Missing expected test directory ${BASE_TEST_FOLDER}" + exit 1 + else + cd ${BASE_TEST_FOLDER} + fi +else + cd ${BASE_TEST_FOLDER} +fi + +TESTCAFE_TESTS_FOLDER=$(pwd)/testcafe + +# Start the backend that serves the media files to be migrated +# Listens internally on port 80 (addressed as http:///assets/) +docker run --name ${assets_container} --network gateway --rm -d ${assets_repo}/${assets_image}:${assets_image_tag} +trap "docker stop ${assets_container}" EXIT + +# Execute migrations using testcafe +docker run --network gateway -v "${TESTCAFE_TESTS_FOLDER}":/tests testcafe/testcafe --screenshots path=/tests/screenshots,takeOnFails=true chromium /tests/**/*.js + +# Back to parent directory +cd ${CURRENT_DIR} diff --git a/tests/11-file-deletion-tests/testcafe/file_deletion_tests.spec.js b/tests/11-file-deletion-tests/testcafe/file_deletion_tests.spec.js new file mode 100644 index 000000000..c9a52b45b --- /dev/null +++ b/tests/11-file-deletion-tests/testcafe/file_deletion_tests.spec.js @@ -0,0 +1,131 @@ +import https from 'https'; +import {Selector} from 'testcafe'; +import {adminUser} from './roles.js'; + + +fixture`File Deletion Tests` + .page`https://islandora-idc.traefik.me/migrate_source_ui` + .beforeEach(async t => { + await t + .useRole(adminUser); + }); + +const migrate_new_items = 'idc_ingest_new_items'; +const migrate_new_collection = 'idc_ingest_new_collection'; +const migrate_media_file = 'idc_ingest_media_file'; + +const selectMigration = Selector('#edit-migrations'); +const migrationOptions = selectMigration.find('option'); + + +test('Migrate Files to be Deleted', async t => { + // Migrate the Collection and Repository Object the Media will be attached to + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_new_collection)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/filedeletion-collection.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_new_items)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/filedeletion-islandora_object.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_file)); + + // Migrate the File to be deleted + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/filedeletion-file.csv' + ]) + .click('#edit-import'); + + let fileListing = 'https://islandora-idc.traefik.me/admin/content/files/'; + let mediaListing = 'https://islandora-idc.traefik.me/admin/content/media'; + + + // Navigate to Files listing, verify File is present, and used in one place + await t.navigateTo(fileListing); + const fileToDelete = Selector('div.view-content').find('a').withText('FP4 Datasheet'); + await t.expect(fileToDelete.count).eql(1); + const tr = fileToDelete.parent('tr'); + const referenceCell = tr.child('td').nth(-1); + await t.expect(referenceCell.innerText).eql('1 place'); + + // Navigate to Media listing, verify Media is present containing the File to be deleted + await t.navigateTo(mediaListing); + const mediaToDelete = Selector('div.view-content').find('a').withText('FP4 Datasheet'); + await t.expect(mediaToDelete.count).eql(1); + + // Delete the Media and the File + await t.click(mediaToDelete.parent('tr').find('input')); + await t.click(Selector('div.view-content').find('input').withAttribute('name', 'op')); + await t.click(Selector('input').withAttribute('name', 'op').withAttribute('value', 'Delete')); + + // After deleting the media, there are two messages that show on the flash: + // 'Deleted 2 items.' (A green informational message) + // '1 item has not been deleted because you do not have the necessary permissions.' (A yellow warning message) + + await t.expect(Selector('div').withAttribute('aria-label', 'Status message').innerText) + .contains('Deleted 2 items.') + await t.expect(Selector('div').withAttribute('aria-label', 'Warning message').innerText) + .contains('1 item has not been deleted because you do not have the necessary permissions.') + + // The two items that would normally be deleted are the Media and the File, but the warning message indicates the + // that something (the File, in this case) *was not* deleted. + + // Verify the Media is gone + await t.expect(mediaToDelete.count).eql(0); + + // Navigate to the File listing, verify File is present with zero references (although we wanted the file to be + // deleted, apparently it can't be due to some permissions issue that is not understood). + await t.navigateTo(fileListing); + await t.expect(fileToDelete.count).eql(1); + await t.expect(referenceCell.innerText).eql('0 places'); + + // It isn't clear as to why the File entity survives. The warning message claims it is a permissions issue, but + // we are logged in as the admin. Logging into the container and examining the filesystem permissions shows that + // the nginx user ought to be able to delete the file (and in fact, you can test this is true by execing into the + // container as the nginx user and rm'ing a file). + + // Perform HEAD on File's URL and verify it is a 200 (i.e. the bytes are still there) + + let url = await fileToDelete.getAttribute('href'); + var statusCode = -1; + + const executeReq = () => { + url = url.replace('http:', 'https:') + console.log(url); + + const options = { + method: 'HEAD' + }; + + return new Promise(resolve => { + const req = https.request(url, options, (res) => { + statusCode = res.statusCode; + resolve(); + res.on('data', () => { + // do nothing + }) + }); + req.end(); + }) + }; + + await executeReq(); + await t.expect(200).eql(statusCode); +}); diff --git a/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-collection.csv b/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-collection.csv new file mode 100644 index 000000000..a65743bb1 --- /dev/null +++ b/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-collection.csv @@ -0,0 +1,2 @@ +local_id,title,title_language,alternative_title,member_of,contact_email,contact_name,collection_number,description,finding_aid +deletion-collection-01,File Deletion Collection,eng,,,,,,Collection for File Deletion tests;eng, diff --git a/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-file.csv b/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-file.csv new file mode 100644 index 000000000..38f49b007 --- /dev/null +++ b/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-file.csv @@ -0,0 +1,2 @@ +local_id,name,original_name,mime_type,media_of,media_use,url +deletion_file_00001,FP4 Datasheet,ilford_fp4.pdf,application/pdf,filedelete_io_01,Original File|Preservation Master File,http://migration-assets/assets/file/ilford_fp4.pdf diff --git a/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-islandora_object.csv b/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-islandora_object.csv new file mode 100644 index 000000000..5012af494 --- /dev/null +++ b/tests/11-file-deletion-tests/testcafe/migrations/filedeletion-islandora_object.csv @@ -0,0 +1,2 @@ +local_id,title,abstract,access_rights,access_terms,alt_title,collection_number,contributor,copyright,copyright_holder,creator,date_available,date_copyrighted,date_created,date_published,description,digital_identifier,digital_publisher,display_hints,dspace_identifier,dspace_itemid,extent,finding_aid,genre,geoportal_link,issn,is_part_of,item_barcode,jhir_uri,language,library_catalog_link,member_of,model,oclc_number,publisher,publisher_country,resource_type,spatial_coverage,subject,table_of_contents,title_language,years +filedelete_io_01,File Deletion Repository Item One,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,deletion-collection-01,,,,,,,,,, \ No newline at end of file diff --git a/tests/11-file-deletion-tests/testcafe/roles.js b/tests/11-file-deletion-tests/testcafe/roles.js new file mode 100644 index 000000000..7c46ea18a --- /dev/null +++ b/tests/11-file-deletion-tests/testcafe/roles.js @@ -0,0 +1,11 @@ +import { Role } from 'testcafe'; + +/** + * Drupal administrator via local login + */ +export const adminUser = Role('https://islandora-idc.traefik.me/user/login', async t => { + await t + .typeText('#edit-name', 'admin') + .typeText('#edit-pass', 'password') + .click('#edit-submit'); +}); diff --git a/tests/12-migration-derivative-tests.sh b/tests/12-migration-derivative-tests.sh new file mode 100755 index 000000000..20fe165f3 --- /dev/null +++ b/tests/12-migration-derivative-tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +BASE_TEST_FOLDER=12-migration-derivative-tests +DRUPAL_CONTAINER_NAME=$(docker ps | awk '{print $NF}'|grep drupal) +CURRENT_DIR=$(pwd) + +# The Docker registry used to obtain the migration assets image +assets_repo=${MIGRATION_ASSETS_REPO:-ghcr.io/jhu-sheridan-libraries/idc-isle-dc} +# The name of the Docker image for migration assets +assets_image=${MIGRATION_ASSETS_IMAGE:-migration-assets} +# The migration assets image tag +assets_image_tag=${MIGRATION_ASSETS_IMAGE_TAG:-088c482.1617637226} +# The *external* port the migration assets HTTP server listens on +ext_assets_port=${MIGRATION_ASSETS_PORT:-8081} +# The name used by the migration assets container +assets_container=${MIGRATION_ASSETS_CONTAINER:-migration-assets} + +# Locate test directory +if [ ! -d ${BASE_TEST_FOLDER} ] ; +then + BASE_TEST_FOLDER=tests/${BASE_TEST_FOLDER} + if [ ! -d ${BASE_TEST_FOLDER} ] ; + then + echo "Missing expected test directory ${BASE_TEST_FOLDER}" + exit 1 + else + cd ${BASE_TEST_FOLDER} + fi +else + cd ${BASE_TEST_FOLDER} +fi + +TESTCAFE_TESTS_FOLDER=$(pwd)/testcafe + +# Start the backend that serves the media files to be migrated +# Listens internally on port 80 (addressed as http:///assets/) +docker run --name ${assets_container} --network gateway --rm -d ${assets_repo}/${assets_image}:${assets_image_tag} +trap "docker stop ${assets_container}" EXIT + +# Execute migrations using testcafe +docker run --network gateway -v "${TESTCAFE_TESTS_FOLDER}":/tests testcafe/testcafe --screenshots path=/tests/screenshots,takeOnFails=true chromium /tests/**/*.js + +# Back to parent directory +cd ${CURRENT_DIR} diff --git a/tests/12-migration-derivative-tests/testcafe/migrate_derivative.spec.js b/tests/12-migration-derivative-tests/testcafe/migrate_derivative.spec.js new file mode 100644 index 000000000..e5c7e6a40 --- /dev/null +++ b/tests/12-migration-derivative-tests/testcafe/migrate_derivative.spec.js @@ -0,0 +1,96 @@ +import {Selector} from 'testcafe'; +import {adminUser} from './roles.js'; + + +fixture`Migration Derivative Tests` + .page`https://islandora-idc.traefik.me/migrate_source_ui` + .beforeEach(async t => { + await t + .useRole(adminUser); + }); + +const migrate_new_items = 'idc_ingest_new_items'; +const migrate_new_collection = 'idc_ingest_new_collection'; +const migrate_media_file = 'idc_ingest_media_file'; + +const selectMigration = Selector('#edit-migrations'); +const migrationOptions = selectMigration.find('option'); + +const contentList = "https://islandora-idc.traefik.me/admin/content"; + +test('Migrate Images for Derivative Generation', async t => { + + // migrate the test objects into Drupal + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_new_collection)); + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/derivative-collection.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_new_items)) + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/derivative-islandora_object.csv' + ]) + .click('#edit-import'); + + await t + .click(selectMigration) + .click(migrationOptions.withAttribute('value', migrate_media_file)) + + await t + .setFilesToUpload('#edit-source-file', [ + './migrations/derivative-file.csv' + ]) + .click('#edit-import'); + + // verify the presence of the islandora object + const io_name = "Derivative Repository Item One" + await t.navigateTo(contentList) + const io = Selector('div.view-content').find('a').withText(io_name) + await t.expect(io.count).eql(1); + + // list its media + + await t.click(io) + await t.click(Selector('#rid-content').find('a').withText('Media')) + + // assert the presence of the original media + const media_name = "Map Image"; + const media = Selector('div.view-content').find('a').withText(media_name); + await t.expect(media.count).eql(1); + + // assert expected attributes of the original media + await t.expect(media.parent('tr').child('td').nth(2).innerText).eql('File') + await t.expect(media.parent('tr').child('td').nth(3).innerText).eql('image/tiff') + await t.expect(media.parent('tr').child('td').nth(4).innerText).contains('Preservation Master File') + await t.expect(media.parent('tr').child('td').nth(4).innerText).contains('Original File') + + // assert the presence of a derivative thumbnail and service image + // (increase timeout in case derivatives haven't been created yet?) + const service_derivative = Selector('div.view-content').find('a').withText('Service File.jpg'); + const thumb_derivative = Selector('div.view-content').find('a').withText('Thumbnail Image.jpg'); + + console.log("Checking for derivatives ...") + const service_count = await service_derivative.count + const thumb_count = await thumb_derivative.count + + // if a derivative isn't present yet, it may be because it hasn't been generated yet. + // in that case, wait 10 seconds and refresh the page, and see if it appears. + if (service_count < 1 || thumb_count < 1) { + console.log("Derivatives haven't appeared. Sleeping for 10 seconds, then trying again ...") + // sleep 10 seconds, refresh the page + await t.wait(10000); + await t.eval(() => location.reload(true)); + } + + await t.expect(service_derivative.count).eql(1); + await t.expect(thumb_derivative.count).eql(1); +}); \ No newline at end of file diff --git a/tests/12-migration-derivative-tests/testcafe/migrations/derivative-collection.csv b/tests/12-migration-derivative-tests/testcafe/migrations/derivative-collection.csv new file mode 100644 index 000000000..1efa2ff51 --- /dev/null +++ b/tests/12-migration-derivative-tests/testcafe/migrations/derivative-collection.csv @@ -0,0 +1,2 @@ +local_id,title,title_language,alternative_title,member_of,contact_email,contact_name,collection_number,description,finding_aid +derivative-collection-01,Derivative Collection,eng,,,,,,Collection for Derivative tests;eng, \ No newline at end of file diff --git a/tests/12-migration-derivative-tests/testcafe/migrations/derivative-file.csv b/tests/12-migration-derivative-tests/testcafe/migrations/derivative-file.csv new file mode 100644 index 000000000..0d2dd06bc --- /dev/null +++ b/tests/12-migration-derivative-tests/testcafe/migrations/derivative-file.csv @@ -0,0 +1,2 @@ +local_id,name,original_name,mime_type,media_of,media_use,url +derivative_media_file_00001,Map Image,map-image.tif,image/tiff,derivative_io_01,Original File|Preservation Master File,http://migration-assets/assets/image/map-image.tif diff --git a/tests/12-migration-derivative-tests/testcafe/migrations/derivative-islandora_object.csv b/tests/12-migration-derivative-tests/testcafe/migrations/derivative-islandora_object.csv new file mode 100644 index 000000000..c66d193e9 --- /dev/null +++ b/tests/12-migration-derivative-tests/testcafe/migrations/derivative-islandora_object.csv @@ -0,0 +1,2 @@ +local_id,title,abstract,access_rights,access_terms,alt_title,collection_number,contributor,copyright,copyright_holder,creator,date_available,date_copyrighted,date_created,date_published,description,digital_identifier,digital_publisher,display_hints,dspace_identifier,dspace_itemid,extent,finding_aid,genre,geoportal_link,issn,is_part_of,item_barcode,jhir_uri,language,library_catalog_link,member_of,model,oclc_number,publisher,publisher_country,resource_type,spatial_coverage,subject,table_of_contents,title_language,years +derivative_io_01,Derivative Repository Item One,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,derivative-collection-01,Image,,,,,,,,, \ No newline at end of file diff --git a/tests/12-migration-derivative-tests/testcafe/roles.js b/tests/12-migration-derivative-tests/testcafe/roles.js new file mode 100644 index 000000000..7c46ea18a --- /dev/null +++ b/tests/12-migration-derivative-tests/testcafe/roles.js @@ -0,0 +1,11 @@ +import { Role } from 'testcafe'; + +/** + * Drupal administrator via local login + */ +export const adminUser = Role('https://islandora-idc.traefik.me/user/login', async t => { + await t + .typeText('#edit-name', 'admin') + .typeText('#edit-pass', 'password') + .click('#edit-submit'); +});