diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..1d5edf8 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,75 @@ +name: Build and test + +on: + workflow_call: + inputs: + oarepo: + description: OARepo version (11, 12, ...) + required: true + default: 11 + type: string + +env: + OAREPO_VERSION: ${{ inputs.oarepo }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Configure sysctl limits + run: | + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + + - name: Runs Opensearch + uses: ankane/setup-opensearch@v1 + with: + plugins: analysis-icu + + - name: Start Redis + uses: supercharge/redis-github-action@1.7.0 + with: + redis-version: ${{ matrix.redis-version }} + + - name: Run tests + run: | + ./run-tests.sh + + - name: Build package to publish + run: | + .venv-builder/bin/python setup.py sdist bdist_wheel + + - name: Freeze packages + run: | + .venv-builder/bin/pip freeze > requirements.txt + .venv-tests/bin/pip freeze >>requirements.txt + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: requirements.txt + path: requirements.txt diff --git a/.github/workflows/manual.yaml b/.github/workflows/manual.yaml new file mode 100644 index 0000000..e05df2c --- /dev/null +++ b/.github/workflows/manual.yaml @@ -0,0 +1,15 @@ +name: Dispatch + +on: + workflow_dispatch: + inputs: + oarepo: + description: OARepo version (11, 12, ...) + required: true + default: 11 + +jobs: + build: + uses: ./.github/workflows/build.yaml + with: + oarepo: ${{ github.event.inputs.oarepo }} diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 0000000..5431749 --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,41 @@ +name: Build, test and publish + +on: push + +permissions: + id-token: write + contents: read + +jobs: + build11: + uses: ./.github/workflows/build.yaml + with: + oarepo: 11 + + build12: + uses: ./.github/workflows/build.yaml + with: + oarepo: 12 + + publish: + runs-on: ubuntu-latest + needs: build11 + steps: + - name: Use built artifacts + uses: actions/download-artifact@v3 + with: + name: dist + path: dist + + - name: List files + run: | + ls -la + ls -la dist + + - name: Publish package + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip_existing: true + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0287eb8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,93 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# Idea software family +.idea/ + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +venv/ +.venv/ +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Vim swapfiles +.*.sw? + +tests/test.db + +.venv* +.direnv + +docs/migration/data + +.env +.envrc +/.python-version +/poetry.lock +example/data +.DS_Store + +test-model + +# Testing +sample/ + +tests/test-sample-app +tests/test-sample-site + +example_document/ +dist/ + +.model_venv/ +.vscode + +thesis diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1e43fbc --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (C) 2021 CESNET. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..2f9e419 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include oarepo_doi * +prune tests +prune thesis \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e1a9dad --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# OARepo DOI \ No newline at end of file diff --git a/format.sh b/format.sh new file mode 100644 index 0000000..e09be13 --- /dev/null +++ b/format.sh @@ -0,0 +1,3 @@ +black oarepo_requests tests --target-version py310 +autoflake --in-place --remove-all-unused-imports --recursive oarepo_requests tests +isort oarepo_requests tests --profile black diff --git a/oarepo_doi/__init__.py b/oarepo_doi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oarepo_doi/actions/__init__.py b/oarepo_doi/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oarepo_doi/actions/doi.py b/oarepo_doi/actions/doi.py new file mode 100644 index 0000000..12623ac --- /dev/null +++ b/oarepo_doi/actions/doi.py @@ -0,0 +1,12 @@ +from invenio_requests.customizations import SubmitAction +from invenio_requests.resolvers.registry import ResolverRegistry +from invenio_records_resources.proxies import current_service_registry + +from oarepo_doi.api import create_doi + + +class DoiDraftAction(SubmitAction): + def execute(self, identity, uow): + topic = self.request.topic.resolve() + # create_doi(topic, event=None) + super().execute(identity, uow) diff --git a/oarepo_doi/api.py b/oarepo_doi/api.py new file mode 100644 index 0000000..d939856 --- /dev/null +++ b/oarepo_doi/api.py @@ -0,0 +1,59 @@ +import requests +import json +from invenio_base.utils import obj_or_import_string +from flask import current_app + + +def create_doi(service, record, data, event = None ): + """ if event = None, doi will be created as a draft.""" + + mapping = obj_or_import_string(service.mapping[record.schema])() + errors = mapping.metadata_check(data) + if len(errors) > 0 and event: + return #todo: dois can not be published with missing mandatory values + + request_metadata = mapping.create_datacite_payload(data) + + if event: + request_metadata["data"]["attributes"]["event"] = event + + request_metadata["data"]["attributes"]["prefix"] = service.prefix + + request = requests.post(url=service.url, json=request_metadata, headers={'Content-type': 'application/vnd.api+json'}, + auth=(service.username, service.password) + ) + + if request.status_code != 201: + raise requests.ConnectionError("Expected status code 201, but got {}".format(request.status_code)) + + content = request.content.decode('utf-8') + json_content = json.loads(content) + doi_value = json_content['data']['id'] + mapping.add_doi(record, data, doi_value) + + +def edit_doi(service, record, event = None): + """ edit existing draft """ + + mapping = obj_or_import_string(service.mapping[record.schema])() + errors = mapping.metadata_check(record) + if len(errors) > 0 and event: + return #todo: dois can not be published with missing mandatory values + doi_value = mapping.get_doi(record) + if doi_value: + if not service.url.endswith('/'): + url = service.url + '/' + else: + url = service.url + url = url + doi_value.replace("/", "%2F") + + request_metadata = mapping.create_datacite_payload(record) + if event: + request_metadata["data"]["attributes"]["event"] = event + + request = requests.put(url=url, json=request_metadata, headers={'Content-type': 'application/vnd.api+json'}, + auth=(service.username, service.password)) + + if request.status_code != 200: + raise requests.ConnectionError("Expected status code 200, but got {}".format(request.status_code)) + diff --git a/oarepo_doi/ext.py b/oarepo_doi/ext.py new file mode 100644 index 0000000..9d9a263 --- /dev/null +++ b/oarepo_doi/ext.py @@ -0,0 +1,20 @@ + +class OARepoDOI(object): + """OARepo DOI extension.""" + + def __init__(self, app=None): + """Extension initialization.""" + if app: + self.init_app(app) + + def init_app(self, app): + """Flask application initialization.""" + self.init_config(app) + app.extensions["oarepo-doi"] = self + + def init_config(self, app): + """Initialize configuration.""" + if "DATACITE_URL" not in app.config: + app.config["DATACITE_URL"] = 'https://api.datacite.org/dois' + if "DATACITE_MODE" not in app.config: + app.config["DATACITE_MODE"] = "ON_EVENT" diff --git a/oarepo_doi/services/__init__.py b/oarepo_doi/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oarepo_doi/services/components/__init__.py b/oarepo_doi/services/components/__init__.py new file mode 100644 index 0000000..7acbdb0 --- /dev/null +++ b/oarepo_doi/services/components/__init__.py @@ -0,0 +1,38 @@ +import json +import requests +from invenio_records_resources.services.records.components import ServiceComponent +from flask import current_app +from invenio_base.utils import obj_or_import_string + +from oarepo_doi.api import create_doi, edit_doi + + +class DoiComponent(ServiceComponent): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.mode = current_app.config.get("DATACITE_MODE") + self.username = current_app.config.get("DATACITE_USERNAME") + self.password = current_app.config.get("DATACITE_PASSWORD") + self.url = current_app.config.get("DATACITE_URL") + self.prefix = current_app.config.get("DATACITE_PREFIX") + self.mapping = current_app.config.get("DATACITE_MAPPING") + + def create(self, identity, data=None, record=None, **kwargs): + if self.mode == "AUTOMATIC_DRAFT": + create_doi(self, record,data, None) + + def update_draft(self, identity, data=None, record=None, **kwargs): + if self.mode == "AUTOMATIC_DRAFT": + edit_doi(self, record) + + def update(self, identity, data=None, record=None, **kwargs): + if self.mode == "AUTOMATIC_DRAFT" or self.mode == "AUTOMATIC": + edit_doi(self, record) + + def publish(self, identity, data=None, record=None, **kwargs): + if self.mode == "AUTOMATIC": + create_doi(self, record, data, "publish") + if self.mode == "AUTOMATIC_DRAFT": + edit_doi(self, record, "publish") diff --git a/oarepo_doi/types/__init__.py b/oarepo_doi/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oarepo_doi/types/doi.py b/oarepo_doi/types/doi.py new file mode 100644 index 0000000..8d91018 --- /dev/null +++ b/oarepo_doi/types/doi.py @@ -0,0 +1,13 @@ +from invenio_requests.customizations import RequestType + +from oarepo_doi.actions.doi import DoiDraftAction + + +class DoiRequestType(RequestType): + available_actions = { + **RequestType.available_actions, + "create_draft": DoiDraftAction, + "publish_draft": DoiPublishAction, + } + + receiver_can_be_none = True \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8289f07 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build_system] +requires = ["setuptools", "wheel", "babel>2.8"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 0000000..3ccc274 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +OAREPO_VERSION=${OAREPO_VERSION:-11} + +BUILDER_VENV=".venv-builder" +if test -d $BUILDER_VENV ; then + rm -rf $BUILDER_VENV +fi + +python3 -m venv $BUILDER_VENV +. $BUILDER_VENV/bin/activate +pip install "oarepo[tests]==${OAREPO_VERSION}.*" +pip install -U setuptools pip wheel + +pip install -e ".[tests]" + +VENV_TESTS=".venv-tests" + +if test -d ./thesis; then + rm -rf ./thesis +fi +if test -d $VENV_TESTS ; then + rm -rf $VENV_TESTS +fi +oarepo-compile-model ./tests/thesis.yaml --output-directory ./thesis -vvv + +python3 -m venv $VENV_TESTS +source $VENV_TESTS/bin/activate + +pip install -U setuptools pip wheel +pip install "oarepo[tests]==${OAREPO_VERSION}.*" +pip install "./thesis[tests]" +pytest ./thesis/tests -vvv + +pytest tests -vvv diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..a05d221 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,34 @@ +[metadata] +name = oarepo-doi +version = 1.0.0 +description = +authors = Alzbeta Pokorna +readme = README.md +long_description = file:README.md +long_description_content_type = text/markdown + + +[options] +python = >=3.10 +install_requires = + invenio-requests + +packages = find: + +[tool.setuptools.packages.find] +include = ['oarepo_requests.*'] + +[options.package_data] +* = *.json, *.rst, *.md, *.json5, *.jinja2 + + + +[options.extras_require] +tests = + oarepo-model-builder-tests + +[options.entry_points] +invenio_base.apps = + oarepo_doi = oarepo_doi.ext:OARepoDOI +invenio_base.api_apps = + oarepodoi = oarepo_doi.ext:OARepoDOI diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c3164ab --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,212 @@ +import os +import yaml +import pytest +from flask_principal import Identity, Need, UserNeed +from flask_security.utils import hash_password, login_user +from invenio_access.models import ActionRoles +from invenio_access.permissions import superuser_access +from invenio_accounts.models import Role +from invenio_accounts.testutils import login_user_via_session +from invenio_app.factory import create_api +from invenio_requests.customizations import CommentEventType, LogEventType +from invenio_requests.proxies import current_request_type_registry, current_requests +from invenio_requests.records.api import Request, RequestEventFormat +from thesis.proxies import current_service +from thesis.records.api import ThesisRecord +# from thesis.proxies import current_service +# from thesis.records.api import ThesisRecord + + +@pytest.fixture(scope="function") +def sample_metadata_list(): + data_path = f"thesis/data/sample_data.yaml" + docs = list(yaml.load_all(open(data_path), Loader=yaml.SafeLoader)) + return docs + + +@pytest.fixture(scope="module") +def create_app(instance_path, entry_points): + """Application factory fixture.""" + return create_api + + +@pytest.fixture(scope="module") +def app_config(app_config): + app_config["REQUESTS_REGISTERED_EVENT_TYPES"] = [LogEventType(), CommentEventType()] + app_config["SEARCH_HOSTS"] = [ + { + "host": os.environ.get("OPENSEARCH_HOST", "localhost"), + "port": os.environ.get("OPENSEARCH_PORT", "9200"), + } + ] + app_config["JSONSCHEMAS_HOST"] = "localhost" + app_config[ + "RECORDS_REFRESOLVER_CLS" + ] = "invenio_records.resolver.InvenioRefResolver" + app_config[ + "RECORDS_REFRESOLVER_STORE" + ] = "invenio_jsonschemas.proxies.current_refresolver_store" + app_config["CACHE_TYPE"] = "SimpleCache" + app_config["DATACITE_PREFIX"] = "123456" + + return app_config + + +@pytest.fixture(scope="module") +def identity_simple(): + """Simple identity fixture.""" + i = Identity(1) + i.provides.add(UserNeed(1)) + i.provides.add(Need(method="system_role", value="any_user")) + i.provides.add(Need(method="system_role", value="authenticated_user")) + return i + + +@pytest.fixture(scope="module") +def identity_simple_2(): + """Simple identity fixture.""" + i = Identity(2) + i.provides.add(UserNeed(2)) + i.provides.add(Need(method="system_role", value="any_user")) + i.provides.add(Need(method="system_role", value="authenticated_user")) + return i + + +@pytest.fixture(scope="module") +def requests_service(app): + """Request Factory fixture.""" + + return current_requests.requests_service + + +@pytest.fixture(scope="module") +def request_events_service(app): + """Request Factory fixture.""" + service = current_requests.request_events_service + return service + + +@pytest.fixture() +def create_request(requests_service): + """Request Factory fixture.""" + + def _create_request(identity, input_data, receiver, request_type, **kwargs): + """Create a request.""" + # Need to use the service to get the id + item = requests_service.create( + identity, input_data, request_type=request_type, receiver=receiver, **kwargs + ) + return item._request + + return _create_request + + +@pytest.fixture() +def submit_request(create_request, requests_service, **kwargs): + """Opened Request Factory fixture.""" + + def _submit_request(identity, data, **kwargs): + """Create and submit a request.""" + request = create_request(identity, input_data=data, **kwargs) + id_ = request.id + return requests_service.execute_action(identity, id_, "submit", data)._request + + return _submit_request + + +@pytest.fixture(scope="module") +def users(app): + """Create example users.""" + # This is a convenient way to get a handle on db that, as opposed to the + # fixture, won't cause a DB rollback after the test is run in order + # to help with test performance (creating users is a module -if not higher- + # concern) + from invenio_db import db + + with db.session.begin_nested(): + datastore = app.extensions["security"].datastore + + su_role = Role(name="superuser-access") + db.session.add(su_role) + + su_action_role = ActionRoles.create(action=superuser_access, role=su_role) + db.session.add(su_action_role) + + user1 = datastore.create_user( + email="user1@example.org", password=hash_password("password"), active=True + ) + user2 = datastore.create_user( + email="user2@example.org", password=hash_password("password"), active=True + ) + admin = datastore.create_user( + email="admin@example.org", password=hash_password("password"), active=True + ) + admin.roles.append(su_role) + + db.session.commit() + return [user1, user2, admin] + + +@pytest.fixture() +def client_with_login(client, users): + """Log in a user to the client.""" + user = users[0] + login_user(user) + login_user_via_session(client, email=user.email) + return client + + +@pytest.fixture(scope="function") +def request_record_input_data(): + """Input data to a Request record.""" + ret = { + "title": "Doc1 approval", + "payload": { + "content": "Can you approve my document doc1 please?", + "format": RequestEventFormat.HTML.value, + }, + } + return ret + + +@pytest.fixture(scope="module") +def record_service(): + return current_service + + +@pytest.fixture(scope="function") +def example_topic_draft(record_service, identity_simple): + draft = record_service.create(identity_simple, {}) + return draft._obj + + +@pytest.fixture(scope="function") +def example_topic(record_service, identity_simple): + draft = record_service.create(identity_simple, {}) + record = record_service.publish(identity_simple, draft.id) + id_ = record.id + record = ThesisRecord.pid.resolve(id_) + return record + + +@pytest.fixture(scope="module") +def identity_creator(identity_simple): # for readability + return identity_simple + + +@pytest.fixture(scope="module") +def identity_receiver(identity_simple_2): # for readability + return identity_simple_2 + + +@pytest.fixture(scope="function") +def request_with_receiver_user( + requests_service, example_topic, identity_creator, users +): + receiver = users[1] + type_ = current_request_type_registry.lookup("generic_request", quiet=True) + request_item = requests_service.create( + identity_creator, {}, type_, receiver=receiver, topic=example_topic + ) + request = Request.get_record(request_item.id) + return request_item diff --git a/tests/test_requests.py b/tests/test_requests.py new file mode 100644 index 0000000..0399c0c --- /dev/null +++ b/tests/test_requests.py @@ -0,0 +1,33 @@ +from invenio_access.permissions import system_identity +from invenio_base.utils import obj_or_import_string + + + +def test_datacite_config(app): + assert app.config["DATACITE_URL"] == 'https://api.datacite.org/dois' + + assert "DATACITE_PREFIX" in app.config + + + + +# def test_request(app, client_with_login): +# with client_with_login.get(f"/thesis/") as c: +# assert c.status_code == 200 +# +# +# def test_create(app, db, record_service, sample_metadata_list, search_clear): +# created_records = [] +# for sample_metadata_point in sample_metadata_list: +# created_records.append( +# record_service.create(system_identity, sample_metadata_point) +# ) +# for sample_metadata_point, created_record in zip( +# sample_metadata_list, created_records +# ): +# created_record_reread = record_service.read( +# system_identity, created_record["id"] +# ) +# assert ( +# created_record_reread.data["metadata"] == sample_metadata_point["metadata"] +# ) diff --git a/tests/thesis.yaml b/tests/thesis.yaml new file mode 100644 index 0000000..449d29e --- /dev/null +++ b/tests/thesis.yaml @@ -0,0 +1,12 @@ +record: + use: + - invenio + module: + qualified: thesis + permissions: + presets: [ 'everyone' ] + properties: + metadata: + properties: + title: + type: keyword \ No newline at end of file