[explorer/rest]: init setup rest endpoint for explorer

Add a new repo for Explorer/REST

- Added ci script
- Added Facade to process db for block and blocks.
- Added DB connection and connect to nem db
- Added Flask to handle endpoint response for block and blocks.

.github/buildConfiguration.yaml

@@ -2,6 +2,9 @@ builds:
   - name: Explorer Nodewatch
     path: explorer/nodewatch
 
+  - name: Explorer Rest
+    path: explorer/rest
+
   - name: Faucet Authenticator
     path: faucet/authenticator
 

.github/codecov.yaml

@@ -12,6 +12,12 @@ coverage:
         flags:
           - explorer-nodewatch
 
+      explorer-rest:
+        target: auto
+        threshold: 1%
+        flags:
+          - explorer-rest
+
       faucet-authenticator:
         target: auto
         threshold: 1%

.github/dependabot.yaml

@@ -21,6 +21,17 @@ updates:
     commit-message:
       prefix: '[dependency]'
 
+  - package-ecosystem: pip
+    directory: /explorer/rest
+    schedule:
+      interval: weekly
+      day: sunday
+    target-branch: dev
+    labels: [Explorer]
+    versioning-strategy: increase-if-necessary
+    commit-message:
+      prefix: '[dependency]'
+
   - package-ecosystem: npm
     directory: /faucet/authenticator
     schedule:

README.md

@@ -6,6 +6,7 @@ It includes our optin manager.
 | component | lint | build | test | coverage | package |
 |-----------|------|-------|------|----------| ------- |
 | [@explorer/nodewatch](explorer/nodewatch) | [![lint][explorer-nodewatch-lint]][explorer-nodewatch-job] | | [![test][explorer-nodewatch-test]][explorer-nodewatch-job] | [![][explorer-nodewatch-cov]][explorer-nodewatch-cov-link]
+| [@explorer/rest](explorer/rest) | [![lint][explorer-rest-lint]][explorer-rest-job] | | [![test][explorer-rest-test]][explorer-rest-job] | [![][explorer-rest-cov]][explorer-rest-cov-link]
 | [@faucet/authenticator](faucet/authenticator) | [![lint][faucet-authenticator-lint]][faucet-authenticator-job] | | [![test][faucet-authenticator-test]][faucet-authenticator-job]| [![][faucet-authenticator-cov]][faucet-authenticator-cov-link] |
 | [@faucet/backend](faucet/backend) | [![lint][faucet-backend-lint]][faucet-backend-job] | | [![test][faucet-backend-test]][faucet-backend-job]| [![][faucet-backend-cov]][faucet-backend-cov-link] |
 | [@faucet/frontend](faucet/frontend) | [![lint][faucet-frontend-lint]][faucet-frontend-job] | [![build][faucet-frontend-build]][faucet-frontend-job] | [![test][faucet-frontend-test]][faucet-frontend-job]| [![][faucet-frontend-cov]][faucet-frontend-cov-link] |
@@ -28,6 +29,12 @@ Detailed version can be seen on [codecov.io][product-cov-link].
 [explorer-nodewatch-cov]: https://codecov.io/gh/symbol/product/branch/dev/graph/badge.svg?token=SSYYBMK0M7&flag=explorer-nodewatch
 [explorer-nodewatch-cov-link]: https://codecov.io/gh/symbol/product/tree/dev/explorer/nodewatch
 
+[explorer-rest-job]: https://jenkins.symboldev.com/blue/organizations/jenkins/Symbol%2Fgenerated%2Fproduct%2Fexplorer-rest/activity?branch=dev
+[explorer-rest-lint]: https://jenkins.symboldev.com/buildStatus/icon?job=Symbol%2Fgenerated%2Fproduct%2Fexplorer-rest%2Fdev%2F&config=explorer-rest-lint
+[explorer-rest-test]: https://jenkins.symboldev.com/buildStatus/icon?job=Symbol%2Fgenerated%2Fproduct%2Fexplorer-rest%2Fdev%2F&config=explorer-rest-test
+[explorer-rest-cov]: https://codecov.io/gh/symbol/product/branch/dev/graph/badge.svg?token=SSYYBMK0M7&flag=explorer-rest
+[explorer-rest-cov-link]: https://codecov.io/gh/symbol/product/tree/dev/explorer/rest
+
 [faucet-authenticator-job]: https://jenkins.symboldev.com/blue/organizations/jenkins/Symbol%2Fgenerated%2Fproduct%2Ffaucet-authenticator/activity?branch=dev
 [faucet-authenticator-lint]: https://jenkins.symboldev.com/buildStatus/icon?job=Symbol%2Fgenerated%2Fproduct%2Ffaucet-authenticator%2Fdev%2F&config=faucet-authenticator-lint
 [faucet-authenticator-test]: https://jenkins.symboldev.com/buildStatus/icon?job=Symbol%2Fgenerated%2Fproduct%2Ffaucet-authenticator%2Fdev%2F&config=faucet-authenticator-test

explorer/rest/.coveragerc

@@ -0,0 +1,2 @@
+[run]
+source = rest

explorer/rest/Jenkinsfile

@@ -0,0 +1,11 @@
+defaultCiPipeline {
+	operatingSystem = ['ubuntu']
+	instanceSize = 'medium'
+
+	ciBuildDockerfile = 'postgres.Dockerfile'
+
+	packageId = 'explorer-rest'
+
+	codeCoverageTool = 'coverage'
+	minimumCodeCoverage = 95
+}

explorer/rest/dev_requirements.txt

@@ -0,0 +1,5 @@
+pytest==7.2.2
+pytest-aiohttp==1.0.4
+coverage>=6.3
+testing.postgresql==1.3.0
+psycopg2-binary==2.9.6

explorer/rest/requirements.txt

@@ -0,0 +1,6 @@
+aiohttp==3.8.4
+Flask==2.2.3
+psycopg2-binary==2.9.6
+symbol-sdk-python==3.0.11
+zenlog==1.1
+Path==16.6.0

explorer/rest/rest/__init__.py

@@ -0,0 +1,93 @@
+import configparser
+from collections import namedtuple
+from pathlib import Path
+
+from flask import Flask, abort, jsonify, request
+from zenlog import log
+
+from rest.facade.NemRestFacade import NemRestFacade
+
+DatabaseConfig = namedtuple('DatabaseConfig', ['database', 'user', 'password', 'host', 'port'])
+
+
+def create_app():
+	app = Flask(__name__)
+
+	setup_error_handlers(app)
+
+	nem_api_facade = setup_nem_facade(app)
+	setup_nem_routes(app, nem_api_facade)
+
+	return app
+
+
+def setup_nem_facade(app):
+	app.config.from_envvar('EXPLORER_REST_SETTINGS')
+	config = configparser.ConfigParser()
+	db_path = Path(app.config.get('DATABASE_CONFIG_FILEPATH'))
+
+	log.info(f'loading database config from {db_path}')
+
+	config.read(db_path)
+
+	nem_db_config = config['nem_db']
+	db_params = DatabaseConfig(
+		nem_db_config['database'],
+		nem_db_config['user'],
+		nem_db_config['password'],
+		nem_db_config['host'],
+		nem_db_config['port']
+	)
+
+	return NemRestFacade(db_params)
+
+
+def setup_nem_routes(app, nem_api_facade):
+	@app.route('/api/nem/block/<height>')
+	def api_get_nem_block_by_height(height):
+		try:
+			height = int(height)
+			if height < 1:
+				raise ValueError()
+
+		except ValueError:
+			abort(400)
+
+		result = nem_api_facade.get_block(height)
+		if not result:
+			abort(404)
+
+		return jsonify(result)
+
+	@app.route('/api/nem/blocks')
+	def api_get_nem_blocks():
+		try:
+			limit = int(request.args.get('limit', 10))
+			offset = int(request.args.get('offset', 0))
+			min_height = int(request.args.get('min_height', 1))
+
+			if limit < 0 or offset < 0 or min_height < 1:
+				raise ValueError()
+
+		except ValueError:
+			abort(400)
+
+		return jsonify(nem_api_facade.get_blocks(limit=limit, offset=offset, min_height=min_height))
+
+
+def setup_error_handlers(app):
+	@app.errorhandler(404)
+	def not_found(_):
+		response = {
+			'status': 404,
+			'message': 'Resource not found'
+		}
+		return jsonify(response), 404
+
+	@app.errorhandler(400)
+	def bad_request(_):
+		response = {
+			'status': 400,
+			'message': 'Bad request'
+		}
+		return jsonify(response), 400

explorer/rest/rest/db/DatabaseConnection.py

@@ -0,0 +1,54 @@
+from psycopg2 import pool
+
+
+class DatabaseConnectionPool:
+	"""Database connection pool class."""
+
+	def __init__(self, db_config, min_connections=1, max_connections=10):
+		"""Initialize the database connection pool with given configurations."""
+
+		self.db_config = db_config
+		self.min_connections = min_connections
+		self.max_connections = max_connections
+		self._pool = self._create_pool()
+
+	def _create_pool(self):
+		return pool.SimpleConnectionPool(
+			self.min_connections,
+			self.max_connections,
+			database=self.db_config.database,
+			user=self.db_config.user,
+			password=self.db_config.password,
+			host=self.db_config.host,
+			port=self.db_config.port
+		)
+
+	def connection(self):
+		"""Acquires a managed database connection instance from the pool."""
+
+		return PooledConnection(self._pool)
+
+
+class PooledConnection:
+	"""
+	Represents a managed database connection from the connection pool.
+	Intended for use within a context manager (`with` statement).
+	"""
+
+	def __init__(self, connection_pool):
+		"""Initialize with a reference to a connection pool."""
+
+		self._pool = connection_pool
+		self.connection = None
+
+	def __enter__(self):
+		"""Acquire a database connection from the pool upon entering the context of a `with` statement."""
+
+		self.connection = self._pool.getconn()
+		return self.connection
+
+	def __exit__(self, exc_type, exc_value, traceback):
+		"""Ensure the connection is returned to the pool upon exiting the context of a `with` statement."""
+
+		if self.connection:
+			self._pool.putconn(self.connection)

explorer/rest/rest/db/NemDatabase.py

@@ -0,0 +1,55 @@
+from binascii import hexlify
+
+from rest.model.Block import BlockView
+
+from .DatabaseConnection import DatabaseConnectionPool
+
+
+def _format_bytes(buffer):
+	return hexlify(buffer).decode('utf8').upper()
+
+
+class NemDatabase(DatabaseConnectionPool):
+	"""Database containing Nem blockchain data."""
+
+	@staticmethod
+	def _create_block_view(result):
+		return BlockView(
+			height=result[0],
+			timestamp=str(result[1]),
+			total_fees=result[2],
+			total_transactions=result[3],
+			difficulty=result[4],
+			block_hash=_format_bytes(result[5]),
+			signer=_format_bytes(result[6]),
+			signature=_format_bytes(result[7])
+		)
+
+	def get_block(self, height):
+		"""Gets block by height in database."""
+
+		with self.connection() as connection:
+			cursor = connection.cursor()
+			cursor.execute('''
+				SELECT *
+				FROM blocks
+				WHERE height = %s
+			''', (height,))
+			result = cursor.fetchone()
+
+			return self._create_block_view(result) if result else None
+
+	def get_blocks(self, limit, offset, min_height):
+		"""Gets blocks pagination in database."""
+
+		with self.connection() as connection:
+			cursor = connection.cursor()
+			cursor.execute('''
+				SELECT *
+				FROM blocks
+				WHERE height >= %s
+				LIMIT %s OFFSET %s
+			''', (min_height, limit, offset,))
+			results = cursor.fetchall()
+
+			return [self._create_block_view(result) for result in results]

explorer/rest/rest/facade/NemRestFacade.py

@@ -0,0 +1,24 @@
+from rest.db.NemDatabase import NemDatabase
+
+
+class NemRestFacade:
+	"""Nem Rest Facade."""
+
+	def __init__(self, db_config):
+		"""Creates a facade object."""
+
+		self.nem_db = NemDatabase(db_config)
+
+	def get_block(self, height):
+		"""Gets block by height."""
+
+		block = self.nem_db.get_block(height)
+
+		return block.to_dict() if block else None
+
+	def get_blocks(self, limit, offset, min_height):
+		"""Gets blocks pagination."""
+
+		blocks = self.nem_db.get_blocks(limit, offset, min_height)
+
+		return [block.to_dict() for block in blocks]

explorer/rest/rest/model/Block.py

@@ -0,0 +1,40 @@
+class BlockView:
+	def __init__(self, height, timestamp, total_fees, total_transactions, difficulty, block_hash, signer, signature):
+		"""Create Block view."""
+
+		# pylint: disable=too-many-arguments
+
+		self.height = height
+		self.timestamp = timestamp
+		self.total_fees = total_fees
+		self.total_transactions = total_transactions
+		self.difficulty = difficulty
+		self.block_hash = block_hash
+		self.signer = signer
+		self.signature = signature
+
+	def __eq__(self, other):
+		return isinstance(other, BlockView) and all([
+			self.height == other.height,
+			self.timestamp == other.timestamp,
+			self.total_fees == other.total_fees,
+			self.total_transactions == other.total_transactions,
+			self.difficulty == other.difficulty,
+			self.block_hash == other.block_hash,
+			self.signer == other.signer,
+			self.signature == other.signature
+		])
+
+	def to_dict(self):
+		"""Formats the block info as a dictionary."""
+
+		return {
+			'height': self.height,
+			'timestamp': str(self.timestamp),
+			'totalFees': self.total_fees,
+			'totalTransactions': self.total_transactions,
+			'difficulty': self.difficulty,
+			'hash': str(self.block_hash),
+			'signer': str(self.signer),
+			'signature': str(self.signature)
+		}

explorer/rest/scripts/ci/lint.sh

@@ -0,0 +1 @@
+../../../../linters/python/lint.sh

explorer/rest/scripts/ci/setup_lint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+python3 -m pip install -r "$(git rev-parse --show-toplevel)/linters/python/lint_requirements.txt"
+python3 -m pip install -r requirements.txt
+python3 -m pip install -r dev_requirements.txt

explorer/rest/scripts/ci/setup_test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+export POSTGRES_DB="explorer_rest"
+export POSTGRES_USER="postgres"
+export POSTGRES_PASSWORD="testPassword"
+export POSTGRES_HOST_AUTH_METHOD="password"
+export PGDATA=/var/lib/postgresql/data/pgdata
+
+initdb -U "${POSTGRES_USER}" -D "${PGDATA}"
+pg_ctl start --wait
+pg_ctl status
+
+psql -U "${POSTGRES_USER}" --command "CREATE USER ubuntu WITH SUPERUSER PASSWORD 'ubuntu';"

explorer/rest/scripts/ci/test.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+
+TEST_RUNNER=$([ "$1" = "code-coverage" ] && echo "coverage run --append" || echo "python3")
+${TEST_RUNNER} -m pytest --asyncio-mode=auto -v