diff --git a/.ci/aptPackagesToInstall.txt b/.ci/aptPackagesToInstall.txt new file mode 100644 index 0000000..dbb7a09 --- /dev/null +++ b/.ci/aptPackagesToInstall.txt @@ -0,0 +1 @@ +python3-ujson diff --git a/.ci/pythonPackagesToInstallFromGit.txt b/.ci/pythonPackagesToInstallFromGit.txt new file mode 100644 index 0000000..cdd0eae --- /dev/null +++ b/.ci/pythonPackagesToInstallFromGit.txt @@ -0,0 +1 @@ +https://github.com/KOLANICH/transformerz.py.git diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c9162b9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/.templateMarker b/.github/.templateMarker new file mode 100644 index 0000000..5e3a3e0 --- /dev/null +++ b/.github/.templateMarker @@ -0,0 +1 @@ +KOLANICH/python_project_boilerplate.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..89ff339 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + allow: + - dependency-type: "all" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..7fe33b3 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,15 @@ +name: CI +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: typical python workflow + uses: KOLANICH-GHActions/typical-python-workflow@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..577d55b --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__ +*.py[co] +/*.egg-info +*.srctrlbm +*.srctrldb +build +dist +.eggs +monkeytype.sqlite3 +/tests/testSavedDataRootDir +/.ipynb_checkpoints diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..39b3634 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,53 @@ +#image: pypy:latest +image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest + +variables: + DOCKER_DRIVER: overlay2 + SAST_ANALYZER_IMAGE_TAG: latest + SAST_DISABLE_DIND: "true" + SAST_CONFIDENCE_LEVEL: 5 + CODECLIMATE_VERSION: latest + +include: + - template: SAST.gitlab-ci.yml + - template: Code-Quality.gitlab-ci.yml + - template: License-Management.gitlab-ci.yml + +build: + tags: + - shared + - linux + stage: build + variables: + GIT_DEPTH: "1" + PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages + + before_script: + - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables` + - apt-get update + - apt-get -y install python3-lz4 python3-msgpack python3-brotli python3-cbor python3-ujson + - pip3 install --upgrade https://gitlab.com/KOLANICH/py-lmdb/-/jobs/artifacts/gitlab/raw/wheels/lmdb-0.CI_cpython_latest-py3-none-any.whl?job=build + - python3 ./fix_python_modules_paths.py + - mkdir -p ./tests/databasesFiles + - mount -t ramfs ramfs ./tests/databasesFiles + + script: + - python3 setup.py bdist_wheel + - mv ./dist/*.whl ./dist/Cache-0.CI-py3-none-any.whl + - pip3 install --upgrade -e ./[msgpack,lz4,brotli,zopflipy,cbor,zstd,lmdb] + - coverage run --source=Cache -m pytest --junitxml=./rspec.xml ./tests/test.py + - coverage report -m + - coverage xml + + coverage: /^TOTAL(?:\s+\d+){4}\s+(\d+%).+/ + + cache: + paths: + - $PYTHONUSERBASE + + artifacts: + paths: + - dist + reports: + junit: ./rspec.xml + cobertura: ./coverage.xml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..81ade8d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +os: + - linux + - osx +dist: trusty +language: python +python: + - 3.4 + - 3.5 + - 3.6 + - 3.7 + - nightly + - pypy3 + - pypy3-nightly +before_install: + - pip3 install --upgrade setuptools setuptools_scm coveralls +install: + - python setup.py install +script: + - coverage run --source=Cache python tests/test.py test +after_success: + - coveralls diff --git a/Code_Of_Conduct.md b/Code_Of_Conduct.md new file mode 100644 index 0000000..bcaa2bf --- /dev/null +++ b/Code_Of_Conduct.md @@ -0,0 +1 @@ +No codes of conduct! \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..20f0fa8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include UNLICENSE +include *.md +include tests +include .editorconfig diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..19504bc --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,56 @@ +urm.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/) +====== +~~[wheel (GitLab)](https://gitlab.com/KOLANICH/urm.py/-/jobs/artifacts/master/raw/dist/urm-0.CI-py3-none-any.whl?job=build)~~ +~~[wheel (GHA via `nightly.link`)](https://nightly.link/KOLANICH-libs/urm.py/workflows/CI/master/urm-0.CI-py3-none-any.whl)~~ +~~![GitLab Build Status](https://gitlab.com/KOLANICH/urm.py/badges/master/pipeline.svg)~~ +~~![GitLab Coverage](https://gitlab.com/KOLANICH/urm.py/badges/master/coverage.svg)~~ +~~[![GitHub Actions](https://github.com/KOLANICH-libs/urm.py/workflows/CI/badge.svg)](https://github.com/KOLANICH-libs/urm.py/actions/)~~ +[![Libraries.io Status](https://img.shields.io/librariesio/github/KOLANICH-libs/urm.py.svg)](https://libraries.io/github/KOLANICH-libs/urm.py) +[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://codeberg.org/KOLANICH-tools/antiflash.py) + +Unrelational mapper. + +Sometimes you need to store some data somewhen and lazily retrieve as class fields and we don't want to write lot of boilerplate code and maintain this piece of shit. Object-relational mappers solve this problem by mapping classes to database tables, objects - to rows, object hierarchy to public and foreign keys. + +But sometimes we don't need to store the data in relational databases. We need to store data in entities like files, archives, remote servers using REST/GraphQL API, etc. So we generalize a bit. + +This stuff utilizes my other library `transformerz` of composable 2-way transformations. + +Tutorial +-------- + +It is [strongly](/issue/1) recommended to read the tutorial before using the lib. [`tutorial.ipynb`](./tutorial.ipynb)[![NBViewer](https://nbviewer.org/static/ico/ipynb_icon_16x16.png)](https://nbviewer.org/urls/codeberg.org/KOLANICH-libs/urm.py/blob/master/tutorial.ipynb) + +TL;DR. Data model +----------------- + +We tie stored entities not to objects, but properties in classes. Entities are stored in underlying key-value storage. + +* When we first time access a property in the class, it is loaded into a cache and then returned. +* subsequent accesses return the stuff from the cache. +* we can `save` the stuff from the cache to the storage explicitly. +* to create a storage from scratch, we put the data into a cache and then `save` it. + +Then we heavily abstrage the stuff. + +A `key` is a `tuple` of strings and numbers. It is an unique identifier of a piece of data. It is decoupled from the actual storage implementation. We address data by keys. + +To define the way we are going to store data we need to answer to the following "orthogonal" questions: + +* WHERE are we going to store it? `Saver` object is an answer. `Mapper.saver` answers the question. +* WHAT is the mapping between our internal `key`s and `key`s in the storage of this piece of data? A `key` is an answer. `Mapper.key` answers this question. + +For cold storage we need to answer an additional question: + +* HOW are we going to serialize the data? `transformerz.Transformer` object is an answer. `ColdMapper.serializer` answers this question. + +So `Mapper` object answers the questions related to storage of values. + +To create a bidirectional mapping between class properties, we need to answer the following questions: + +* What pattern of access to the data should be optimized for? `FieldStrategy` subclasses are the answers. + * `ColdStrategy` optimizes for frequent rather cheap accesses to volatile data. + * Requires to know how we STORE the data. `ColdMapper` object is an answer. `FieldStrategy.cold` answers this question. + * `CachedStrategy` optimizes for frequent rather expensive accesses to not very volatile data. + * *Additionally* requires to know how we CACHE the data? `HotMapper` object is an answer. `FieldStrategy.hot` answers this question. + * How do we create our internal `key`s? `Field` subclasses contain the answers. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..efb9808 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..11085e5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.2.0", "wheel", "setuptools_scm[toml]>=3.4.3"] +build-backend = "setuptools.build_meta" + +[project] +name = "urm" +description = "Unrelational mapper. Saves and loads data not only from key-value storages." +readme = "ReadMe.md" +authors = [{name = "KOLANICH"}] +license = {text = "Unlicense"} +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: Public Domain", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = ["urm"] +urls = {Homepage = "https://codeberg.org/KOLANICH-libs/urm.py"} +requires-python = ">=3.4" +dependencies = [ + "transformerz", # @ git+https://codeberg.org/KOLANICH-libs/transformerz.py.git +] +dynamic = ["version"] + +[tool.setuptools] +packages = ["urm"] +zip-safe = true +include-package-data = false + +[tool.setuptools_scm] diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..0e5ca5c --- /dev/null +++ b/tests/test.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +import typing +import sys +import unittest +from collections import OrderedDict + +dict = OrderedDict + +from pathlib import Path + +try: + import ujson as json +except ImportError: + import json + +from transformerz.core import TransformerBase +from transformerz.serialization.json import jsonFancySerializer +from transformerz.text import utf8Transformer + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from urm.core import Dynamic +from urm.fields import CachedStrategy, ColdStrategy, Field, Field0D, FieldND +from urm.mappers import ColdMapper, HotMapper +from urm.mappers.key import PrefixKeyMapper, fieldNameKeyMapper +from urm.mappers.serializer import JustReturnSerializerMapper +from urm.ProtoBundle import ProtoBundle +from urm.storers.cold import FileSaver +from urm.storers.hot import CollectionCacher, PrefixCacher + +ourTransformer = utf8Transformer + jsonFancySerializer + +savedDataRootDir = Path(Path(__file__).parent / "testSavedDataRootDir") # a directory where a file will reside +savedDataRootDir.mkdir(parents=True, exist_ok=True) + +ourSaver = FileSaver(savedDataRootDir, "json") # json is the extension of the files used. `ourSaver` will be populated into `saver` property in future + + +def constantParamsSerializerMapper(parent: ProtoBundle) -> TransformerBase: # it will be populated into `serializer` property in future + return ourTransformer + + +class Tests(unittest.TestCase): + @staticmethod + def createAClassWithStrategyFromGroundUp(strategy) -> ProtoBundle: + class B(ProtoBundle): + __slots__ = ("_scalarField",) + scalarField = Field0D(None) + scalarField.strategy = strategy + + return B + + def getBrandNewOurStorer(self) -> ColdMapper: + return ColdMapper(fieldNameKeyMapper, ourSaver, constantParamsSerializerMapper) + + def getBrandNewOurCacher(self) -> HotMapper: + return HotMapper(fieldNameKeyMapper, PrefixCacher()) + + def verifyBundleWithJSONFileSaver(self, testClass: typing.Type, isCached: bool) -> None: + A = testClass + a = A() + + oldValue = {"a": ["b", "c"]} + a.scalarField = oldValue + with self.subTest("Simple round-trip"): + self.assertEqual(a.scalarField, oldValue) + + if isCached: + a.save() + + with self.subTest("Initial persistence"): + fileName = savedDataRootDir / (A.scalarField.strategy.name + ".json") + self.assertEqual(json.loads(fileName.read_text()), oldValue) + + newValue = 100500 + fileName.write_text(json.dumps(newValue)) + if isCached: + with self.subTest("Read from cache"): + self.assertEqual(a.scalarField, oldValue) # we read from cache! + a.save() + + with self.subTest("Noninvalidation by lack of write"): + self.assertEqual(json.loads(fileName.read_text()), newValue) # because we don't write back from the cache, if it is not invalidated by a write! + + # returning to the state with the old value everywhere ... + a.scalarField = oldValue + a.save() + + with self.subTest("Non-write if not requested"): + a.scalarField = newValue + self.assertEqual(json.loads(fileName.read_text()), oldValue) # because not yet written back from the cache! + + with self.subTest("Update"): + a.save() + self.assertEqual(json.loads(fileName.read_text()), newValue) # because only now it is written back from the cache! + + else: + self.assertEqual(a.scalarField, newValue) # the new value immediately available as changed in the cold storage + + def test_ColdStrategy_FileSaver_Field0D_FromGroundUp(self) -> None: + ourStorer = self.getBrandNewOurStorer() + coldStrategy = ColdStrategy(ourStorer) + + self.verifyBundleWithJSONFileSaver(self.createAClassWithStrategyFromGroundUp(coldStrategy), False) + + def test_CachedStrategy_FileSaver_Field0D_FromGroundUp(self): + ourStorer = self.getBrandNewOurStorer() + ourCacher = self.getBrandNewOurCacher() + cachedStrategy = CachedStrategy(ourStorer, ourCacher) + self.verifyBundleWithJSONFileSaver(self.createAClassWithStrategyFromGroundUp(cachedStrategy), True) + + def test_ColdStrategy_FileSaver_Field0D(self) -> None: + ourStorer = self.getBrandNewOurStorer() + + class A: + scalarField = Field0D(ourStorer) # uncached + + self.verifyBundleWithJSONFileSaver(A, False) + + def test_CachedStrategy_FileSaver_Field0D(self): + ourStorer = self.getBrandNewOurStorer() + ourCacher = self.getBrandNewOurCacher() + + class B(ProtoBundle): + __slots__ = ("_scalarField",) + scalarField = Field0D(ourStorer, ourCacher) # cached + + self.verifyBundleWithJSONFileSaver(B, True) + + def test_CachedStrategy_FileSaver_Field1D(self) -> None: + vectorKeyMapper = PrefixKeyMapper() + ourVectorStorer = ColdMapper(vectorKeyMapper, ourSaver, constantParamsSerializerMapper) + ourVectorCacher = HotMapper(vectorKeyMapper, CollectionCacher(dict)) # our hot storage is a dict, but we can plug there any collection + + class C(ProtoBundle): + vectorField = FieldND(ourVectorStorer, ourVectorCacher) # cached + + c = C() + c.vectorField["aaaa"] = 10 + c.vectorField["bbbb"] = {25: 36} + c.vectorField["cccc"] = {"25": 36} + c.save() + self.assertEqual((savedDataRootDir / "aaaa.json").read_text(), str(c.vectorField["aaaa"])) + self.assertNotEqual(json.loads((savedDataRootDir / "bbbb.json").read_text()), c.vectorField["bbbb"]) # because it is JSON! + self.assertEqual(json.loads((savedDataRootDir / "cccc.json").read_text()), c.vectorField["cccc"]) + + def test_ColdStrategy_FileSaver_Field0D_Dynamic(self) -> None: + controlledPathKeyMapper = PrefixKeyMapper(Dynamic("name")) + ourNameControlledStorer = ColdMapper(controlledPathKeyMapper, ourSaver, constantParamsSerializerMapper) + ourCacher = self.getBrandNewOurCacher() + + class Pocket(ProtoBundle): + __slots__ = ("name", "_shit") + shit = Field0D(ourNameControlledStorer, ourCacher) + + def __init__(self, name: str): + self.name = name + + ptchkPocket = Pocket("ptchk") + ptchkPocket.shit = 2 + ptchkPocket.save() + (savedDataRootDir / "ptchk.json").write_text(str(json.loads((savedDataRootDir / "ptchk.json").read_text()) - 1)) + ptchkPocket.shit = None # invalidates cache + self.assertEqual(ptchkPocket.shit, 1) + ptchkPocket.shit -= 1 + self.assertEqual(json.loads((savedDataRootDir / "ptchk.json").read_text()), 1) + ptchkPocket.save() + self.assertEqual(json.loads((savedDataRootDir / "ptchk.json").read_text()), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tutorial.ipynb b/tutorial.ipynb new file mode 100644 index 0000000..c8eb06c --- /dev/null +++ b/tutorial.ipynb @@ -0,0 +1,391 @@ +{ + "cells": [{ + "cell_type": "markdown", + "metadata": {}, + "source": ["# Lesson 1: Transformers\n", "\n", "Transformers are just a composable way to serialize the stuff. `unprocess` serializes, `process` parses."] + }, { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": ["testJsonDict = {\"abolish\": [\"patent\", \"copyright\"]}"] + }, { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["{\n", "\t\"abolish\": [\n", "\t\t\"patent\",\n", "\t\t\"copyright\"\n", "\t]\n", "}\n"] + } + ], + "source": ["from transformerz.serialization.json import jsonFancySerializer\n", "\n", "print(jsonFancySerializer.unprocess(testJsonDict))"] + }, { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["[None]\n"] + } + ], + "source": ["print(jsonFancySerializer.process(\"[null]\"))"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": [" They can be composed using `+` operation. "] + }, { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["'{\\n\\t\"abolish\": [\\n\\t\\t\"patent\",\\n\\t\\t\"copyright\"\\n\\t]\\n}'\n"] + } + ], + "source": ["from transformerz.serialization.pon import ponSerializer # JSON <-> JavaScript === PON <-> Python\n", "\n", "print((ponSerializer + jsonFancySerializer).unprocess(testJsonDict)) # Returns a \"PON\" `str` in which a JSON string is serialized"] + }, { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["\"{'abolish': ['patent', 'copyright']}\"\n"] + } + ], + "source": ["print((jsonFancySerializer + ponSerializer).unprocess(testJsonDict)) # Returns a JSON `str` in which \"PON\" string is serialized"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["But you cannot save strings into files, you need to save bytes into files ..."] + }, { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["b'{\\n\\t\"abolish\": [\\n\\t\\t\"patent\",\\n\\t\\t\"copyright\"\\n\\t]\\n}'\n"] + } + ], + "source": ["from transformerz.text import utf8Transformer\n", "\n", "ourTransformer = utf8Transformer + jsonFancySerializer\n", "print(ourTransformer.unprocess(testJsonDict)) # Returns raw bytes of a \"PON\" string is serialized"] + }, { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": ["del testJsonDict"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["The data can be compressed. For compression we use the stuff available in my fork of `kaitai.compress` library (I hope it would be merged somewhen). `BinaryProcessor` is an adapter allowing to use the stuff from that lib."] + }, { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["b'x\\x9c\\xed\\xc6\\xa1\\r\\x00 \\x0c\\x000\\xcd\\xce\\x98\\xdeG\\x04E\\x82A\\xef\\x7f\\x14_\\xb4\\xaa3F\\x9e\\xde7KDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD~b=s\\x83\\xe3\\x84'\n"] + } + ], + "source": ["from transformerz.compression import BinaryProcessor\n", "from transformerz.kaitai.compress import Zlib\n", "\n", "zlibProcessor = BinaryProcessor(\"zlib\", Zlib()) # You must name your processor!\n", "print((zlibProcessor + ourTransformer).unprocess([\"fuck\"] * 8000)) # Returns ZLib-compressed UTF-8 encoded JSON string"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["# Lesson 2: Concepts of cold and hot storage and objects representing them"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["There exist a pair of concepts of storage in computer science. Cold storage and hot storage.\n", "\n", "* Cold storage is permanent and costly to access. It is used for long-term storage of data and distribution on physical medium. The examples are HDD, magnetic tape, flash memory, a piece of paper with handwritten data, a DVD, a holograph, anything mentioned within a safe, or even [pieces of glass with dots burned in thew with a laser](https://www.microsoft.com/en-us/research/project/project-silica/) [stored in a abandoned mine in permafrost](https://archiveprogram.github.com/).\n", "* Hot storage may be not permanent, but it must be efficient to access. It is usally RAM.\n", "\n", " We use the term `store` for cold storage and the term `cache` for hot storage. In our case hot storage is usually RAM and requires no explicit serialization (the data are stored the way defined by runtime and compiler), and cold storage is usually HDD/SSD requiring data to be serialized before written and unserialized after being read."] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["## Defining cold storage\n", "To define the way we are going to STORE data we need to answer to the following \"orthogonal\" questions:\n", "* WHERE are we going to store it? `Saver` object is an answer. `Mapper.saver` answers the question."] + }, { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": ["# Here is an example of a saver saving the data into a file. The first argument is a path to the file (without extension!). Second argument is the extension.\n", "from pathlib import Path\n", "from urm.storers.cold import FileSaver\n", "\n", "savedDataRootDir = Path(\"./tests/testSavedDataRootDir\") # a directory where a file will reside\n", "ourSaver = FileSaver(savedDataRootDir, \"json\") # json is the extension of the files used. `ourSaver` will be populated into `saver` property in future"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["* WHAT is the mapping between our internal `key`s and `key`s in the storage of this piece of data? A `key` is an answer. `Mapper.key` answers this question."] + }, { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": ["from urm.mappers.key import fieldNameKeyMapper\n", "\n", "keyMapper = fieldNameKeyMapper # We don't need a key mapper currently, since we deal with scalars in this example"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["For cold storage we need to answer an additional question:\n", "* HOW are we going to serialize the data? `transformerz.Transformer` object is an answer. `ColdMapper.serializer` answers this question. It is a function, that returns a transformer we will use to serialize the data. It allows you to change the transformer depending on some conditions, some of which can be encoded in other serialized data."] + }, { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": ["from transformerz.core import TransformerBase\n", "from urm.ProtoBundle import ProtoBundle\n", "\n", "def constantParamsSerializerMapper(parent: ProtoBundle) -> TransformerBase: # it will be populated into `serializer` property in future\n", " return ourTransformer"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["So `ColdMapper` object answers the questions related to storage of values in cold storage. Let's construct it!"] + }, { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": ["from urm.mappers import ColdMapper\n", "\n", "ourStorer = ColdMapper(keyMapper, ourSaver, constantParamsSerializerMapper)"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["## Defining hot storage\n", "To work with data we need a hot storage."] + }, { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": ["from urm.mappers.key import PrefixKeyMapper, fieldNameKeyMapper\n", "from urm.mappers import HotMapper\n", "from urm.storers.hot import PrefixCacher\n", "\n", "ourCacher = HotMapper(fieldNameKeyMapper, PrefixCacher())"] + }, { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["# Lesson 3: Fields, strategies and bundles\n", "\n", "A `key` is a `tuple` of strings and numbers. It is an unique identifier of a piece of data. It is decoupled from the actual storage implementation. We address data by keys."] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["A field strategy is an object of `FieldStrategy` class that describes our pattern of loading/storing data. There are 2:\n", "* \"cold\" one (`ColdStrategy` class): always load data from medium on getting the field value and always store data to medium when the field is assigned with a value.\n", "* \"cached\" one (`CachedStrategy` class): on accesses only alter the data in the hot storage (aka `cache`). Load data to cache from cold storage the first time it is read. Store the data to cold storage when explicitly asked.\n", "\n", "As you see, the most basic strategy is the cold one. The strategy using only hot storage makes completely no sense by itself, to use it you don't need all this framework.\n", "\n", "So, let's get familiar with the cold strategy first.\n", "\n", "## Defining a cold strategy"] + }, { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": ["from urm.fields import ColdStrategy\n", "\n", "coldStrategy = ColdStrategy(ourStorer) # Don't do like this in real code, a strategy object must never be reused! We have a better way to set it."] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["## Defining our class with a property backed by cold storage\n", "\n", "Now we define a class, which properties are backed by cold storage."] + }, { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": ["from urm.fields import Field, Field0D, FieldND\n", "from urm.mappers.serializer import JustReturnSerializerMapper\n", "\n", "class A:\n", " __slots__ = ()\n", " scalarField = Field0D(None)\n", " scalarField.strategy = coldStrategy # Don't define the field like this, we have a better option. This way is about how strategies work"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["... and test it ..."] + }, { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "scrolled": false + }, + "outputs": [{ + "data": { + "text/plain": ["True"] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["import json\n", "\n", "a = A()\n", "dataToSave = {\"a\": [\"b\", \"c\"]}\n", "a.scalarField = dataToSave\n", "json.loads((savedDataRootDir / (A.scalarField.strategy.name + \".json\")).read_text()) == dataToSave # the data read from the file by another way must match the value we have saved!"] + }, { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["100500"] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["(savedDataRootDir / (A.scalarField.strategy.name + \".json\")).write_text(\"100500\") # we replace the value in the file ...\n", "a.scalarField # ... and the returned value changes"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["## Defining a cached strategy\n", "\n", "A cached strategy requires both cold and hot mappers."] + }, { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": ["from urm.fields import CachedStrategy\n", "\n", "cachedStrategy = CachedStrategy(ourStorer, ourCacher)"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["## Defining our class with a property backed by cached storage\n", "\n", "Now we define a class, which properties are backed by cached storage.\n", "* Such classes must inherit from `ProtoBundle`!\n", "* Hot storage is placed into an attr of the class prefixed with an underscore, so they have to be added into slots."] + }, { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": ["class B(ProtoBundle):\n", " __slots__ = (\"_scalarField\",)\n", " scalarField = Field0D(None)\n", " scalarField.strategy = cachedStrategy # Don't define the field like this in production, we have a better option. This way is only to explain how strategies work"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["... and test it ..."] + }, { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["True"] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["b = B() # since we use the same storer, the data is loaded from the same storage\n", "b.scalarField == a.scalarField"] + }, { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["True"] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["dataToSave2 = {\"d\": [\"e\", \"f\"]}\n", "b.scalarField = dataToSave2\n", "b.scalarField == dataToSave2"] + }, { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["False"] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["# but the data in cold storage is not automatically updated ...\n", "b.scalarField == dataToSave"] + }, { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["False"] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["# We can save the data ....\n", "b.save()\n", "a.scalarField == dataToSave"] + }, { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["True"] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["a.scalarField == dataToSave2"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["The way above is useful for just understanding how the stuff works. In real code you gonna create the storage-backed fields using the following syntax:"] + }, { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [{ + "data": { + "text/plain": ["True"] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": ["class A:\n", " scalarField = Field0D(ourStorer) # uncached\n", "\n", "class B(ProtoBundle):\n", " __slots__ = (\"_scalarField\",)\n", " scalarField = Field0D(ourStorer, ourCacher) # cached\n", "\n", "\n", "a = A()\n", "b = B()\n", "b.scalarField == a.scalarField"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["# Lesson 4: Bird-eye picture"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["To create a bidirectional mapping between class properties, we need to answer the following questions:\n", "* How does data flows between hot and cold storages? `FieldStrategy` subclasses are the answers.\n", "* How do we STORE the data? `ColdMapper` object is an answer. `FieldStrategy.cold\" answers this question.\n", "* How do we CACHE the data? `HotMapper` object is an answer. `FieldStrategy.hot\" answers this question.\n", "* How do we create our internal `key`s? `Field` subclasses contain the answers."] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["# Lesson 5: File-backed collections\n", "\n", "For fields containing collections of objects mapped to unrelational entities you need key mappers, mapping keys. For scalars keys always were empty. For collections the keys are provided by the user when indexing."] + }, { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["True\n", "False\n", "True\n"] + } + ], + "source": ["from urm.storers.hot import CollectionCacher\n", "\n", "vectorKeyMapper = PrefixKeyMapper()\n", "ourVectorStorer = ColdMapper(vectorKeyMapper, ourSaver, constantParamsSerializerMapper)\n", "ourVectorCacher = HotMapper(vectorKeyMapper, CollectionCacher(dict)) # our hot storage is a dict, but we can plug there any collection\n", "\n", "class C(ProtoBundle):\n", " vectorField = FieldND(ourVectorStorer, ourVectorCacher) # cached\n", "\n", "\n", "c = C()\n", "c.vectorField[\"aaaa\"] = 10\n", "c.vectorField[\"bbbb\"] = {25: 36}\n", "c.vectorField[\"cccc\"] = {\"25\": 36}\n", "c.save()\n", "print((savedDataRootDir / \"aaaa.json\").read_text() == str(c.vectorField[\"aaaa\"]))\n", "print(json.loads((savedDataRootDir / \"bbbb.json\").read_text()) == c.vectorField[\"bbbb\"]) # False, because it is JSON!\n", "print(json.loads((savedDataRootDir / \"cccc.json\").read_text()) == c.vectorField[\"cccc\"])"] + }, { + "cell_type": "markdown", + "metadata": {}, + "source": ["# Lesson 6: Controlling paths with dynamic attributes and cache invalidation\n", "\n", "To resolve paths dynamically we have a wrapper object `Dynamic`. It is a path in object hierarchy.\n", "To invalidate cache, set the value to None"] + }, { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [{ + "name": "stdout", + "output_type": "stream", + "text": ["Wn: hv y brgt??\n", "ptchk: Y nw hv 1\n", "1\n", "0\n"] + } + ], + "source": ["from urm.core import Dynamic\n", "from urm.fields import Field0D, FieldND\n", "from urm.mappers.serializer import JustReturnSerializerMapper\n", "\n", "controlledPathKeyMapper = PrefixKeyMapper(Dynamic(\"name\"))\n", "ourNameControlledStorer = ColdMapper(controlledPathKeyMapper, ourSaver, constantParamsSerializerMapper)\n", "\n", "class Pocket(ProtoBundle):\n", " __slots__ = (\"name\", \"_shit\")\n", " shit = Field0D(ourNameControlledStorer, ourCacher)\n", " def __init__(self, name: str):\n", " self.name = name\n", "\n", "\n", "ptchkPocket = Pocket(\"ptchk\")\n", "ptchkPocket.shit = 2\n", "ptchkPocket.save()\n", "print(\"Wn: hv y brgt??\")\n", "(savedDataRootDir / \"ptchk.json\").write_text(str(json.loads((savedDataRootDir / \"ptchk.json\").read_text()) - 1))\n", "ptchkPocket.shit = None # invalidates cache\n", "print(\"ptchk: Y nw hv\", ptchkPocket.shit)\n", "ptchkPocket.shit -= 1\n", "print(json.loads((savedDataRootDir / \"ptchk.json\").read_text()))\n", "ptchkPocket.save()\n", "print(json.loads((savedDataRootDir / \"ptchk.json\").read_text()))"] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/urm.srctrlprj b/urm.srctrlprj new file mode 100644 index 0000000..036373d --- /dev/null +++ b/urm.srctrlprj @@ -0,0 +1,17 @@ + + + + + Python Source Group + + .py + + + ./urm + + enabled + Python Source Group + + + 8 + diff --git a/urm/ProtoBundle.py b/urm/ProtoBundle.py new file mode 100644 index 0000000..2836e0a --- /dev/null +++ b/urm/ProtoBundle.py @@ -0,0 +1,41 @@ +import typing +from abc import ABC, ABCMeta + +from .fields import Field, FieldND, CachedStrategy + + +class _ProtoBundle(ABC): + __slots__ = () + __SAVED_ATTRS__ = frozenset() + + +class ProtoBundleMeta(ABCMeta): + __slots__ = () + + def __new__(cls: typing.Type["ProtoBundleMeta"], className: str, parents: typing.Tuple[type, ...], attrs: typing.Dict[str, typing.Any], *args, **kwargs) -> typing.Type["_ProtoBundle"]: + attrs = type(attrs)(attrs) + savedAttrs = [] + attsToSetParent = [] + for k, v in attrs.items(): + if isinstance(v, Field): + if isinstance(v.strategy, CachedStrategy): + savedAttrs.append(k) + if savedAttrs: + attrs["__SAVED_ATTRS__"] = parents[0].__SAVED_ATTRS__ | frozenset(savedAttrs) + res = super().__new__(cls, className, parents, attrs, *args, **kwargs) + return res + + +class ProtoBundle(_ProtoBundle, metaclass=ProtoBundleMeta): + __slots__ = () + + def _saveProp(self, propName: str) -> None: + prop = getattr(self.__class__, propName) + prop.strategy.save(self) + + def save(self, propName: typing.Optional[str] = None) -> None: + if propName is None: + for a in self.__class__.__SAVED_ATTRS__: + self._saveProp(a) + else: + self._saveProp(propName) diff --git a/urm/__init__.py b/urm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/urm/core.py b/urm/core.py new file mode 100644 index 0000000..e65818c --- /dev/null +++ b/urm/core.py @@ -0,0 +1,19 @@ +import typing + +from transformerz.core import TransformerBase + +KeyT = typing.Tuple[str, ...] +KeyMapperCallableT = typing.Callable[["_ProtoBundle", "Field", KeyT], KeyT] +SerializerMapperCallableT = typing.Callable[["_ProtoBundle"], TransformerBase] + + +class Dynamic: + __slots__ = ("path",) + + def __init__(self, path: typing.Union[KeyT, str]) -> None: + if isinstance(path, str): + path = (path,) + self.path = path + + def __repr__(self): + return self.__class__.__name__ + "(" + repr(self.path) + ")" diff --git a/urm/fields.py b/urm/fields.py new file mode 100644 index 0000000..2d67917 --- /dev/null +++ b/urm/fields.py @@ -0,0 +1,139 @@ +import typing +from abc import ABC, abstractmethod + +from .core import KeyT +from .mappers import ColdMapper, HotMapper + + +class FieldStrategy(ABC): + """Defines a storage-backed field in a class.""" + + __slots__ = ("name", "cold") + + def __init__(self, cold: ColdMapper, name: typing.Optional[str] = None) -> None: + self.name = name + self.cold = cold + + @abstractmethod + def get(self, parent: "_ProtoBundle", key: KeyT) -> typing.Any: + raise NotImplementedError + + @abstractmethod + def set(self, parent: "_ProtoBundle", key: KeyT, newV: typing.Any) -> None: + raise NotImplementedError + + +class ColdStrategy(FieldStrategy): + """Defines a storage-backed uncached immediately-written field in a class.""" + + __slots__ = () + + def get(self, parent: "_ProtoBundle", key: KeyT) -> typing.Any: + return self.cold.load(parent, self, key) + + def set(self, parent: "_ProtoBundle", key: KeyT, newV: typing.Any) -> None: + self.cold.save(parent, self, key, newV) + + +class CachedStrategy(FieldStrategy): + """Defines a storage-backed cached field in a class.""" + + __slots__ = ("hot", "modified") + + def __init__(self, cold: ColdMapper, hot: HotMapper, name: typing.Optional[str] = None) -> None: + super().__init__(cold, name) + self.hot = hot + self.modified = set() + + def get(self, parent: "_ProtoBundle", key: KeyT) -> typing.Any: + res = self.hot.load(parent, self, key) + if res is None: + res = self.cold.load(parent, self, key) + self.hot.save(parent, self, key, res) + return res + + def set(self, parent: "_ProtoBundle", key: KeyT, newV: typing.Any) -> None: + self.hot.save(parent, self, key, newV) + self.modified |= {key} + + def _saveItem(self, parent: "_ProtoBundle", key: KeyT) -> None: + res = self.hot.load(parent, self, key) + self.cold.save(parent, self, key, res) + + def save(self, parent: "_ProtoBundle", key: KeyT = None) -> None: + if key is not None: + self._saveItem(parent, key) + self.modified -= {key} + else: + self._saveAll(parent) + + def _saveAll(self, parent: "_ProtoBundle") -> None: + for key in self.modified: + self._saveItem(parent, key) + self.modified = type(self.modified)() + + +class _Field: + __slots__ = ("strategy",) + + def __init__(self, strategy) -> None: + self.strategy = strategy + + def __set_name__(self, owner: typing.Type["_ProtoBundle"], name: str) -> None: + if self.strategy.name is None: + self.strategy.name = name + + def save(self, parent: "_ProtoBundle"): + return self.strategy.save(parent) + + +class Field(_Field): + __slots__ = () + + def __init__(self, cold: ColdMapper, hot: HotMapper = None, name: typing.Optional[str] = None) -> None: + if hot is None: + strategy = ColdStrategy(cold, name) + else: + strategy = CachedStrategy(cold, hot, name) + super().__init__(strategy) + + +class Field0D(Field): + __slots__ = () + + def __get__(self, inst: "_ProtoBundle", cls: typing.Optional[typing.Type["_ProtoBundle"]] = None) -> typing.Any: + if inst is not None: + return self.strategy.get(inst, ()) + else: + return self + + def __set__(self, inst: "_ProtoBundle", newV: typing.Any) -> None: + if inst is not None: + self.strategy.set(inst, (), newV) + + +class FieldNDAccessor: + __slots__ = ("strategy", "parent") + + def __init__(self, strategy, parent): + self.strategy = strategy + self.parent = parent + + def __getitem__(self, key: KeyT) -> typing.Any: + return self.strategy.get(self.parent, key) + + def __setitem__(self, key: KeyT, newV: typing.Any) -> None: + self.strategy.set(self.parent, key, newV) + + +class FieldND(Field): + __slots__ = () + + def __get__(self, inst: "_ProtoBundle", cls=None) -> typing.Any: + if inst is not None: + return FieldNDAccessor(self.strategy, inst) + else: + return self + + def __set__(self, inst: "_ProtoBundle", newV: typing.Any) -> None: + raise NotImplementedError diff --git a/urm/mappers/__init__.py b/urm/mappers/__init__.py new file mode 100644 index 0000000..2427c74 --- /dev/null +++ b/urm/mappers/__init__.py @@ -0,0 +1,54 @@ +import typing +from abc import ABC, abstractmethod + +from ..core import KeyMapperCallableT, SerializerMapperCallableT, KeyT +from ..storers import Storer, Saver, Cacher + + +class Mapper(ABC): + """Groups mapping required for storage of the data.""" + + __slots__ = ("key", "storer") + + def __init__(self, keyMapper: KeyMapperCallableT, storer: Storer) -> None: + self.key = keyMapper + self.storer = storer + + def load(self, parent: "_ProtoBundle", field: "Field", key: KeyT) -> typing.Any: + path = self.key(parent, field, key) + rawRes = self.storer.get(parent, path) + return rawRes + + def save(self, parent: "_ProtoBundle", field: "Field", key: KeyT, value: typing.Any) -> None: + path = self.key(parent, field, key) + self.storer.set(parent, path, value) + + +class HotMapper(Mapper): + """Groups mapping required for caching of the data.""" + + __slots__ = () + + def __init__(self, keyMapper: KeyMapperCallableT, storer: Cacher) -> None: + super().__init__(keyMapper, storer) + + +class ColdMapper(Mapper): + """Groups mapping required for COLD storage of the data.""" + + __slots__ = ("serializer",) + + def __init__(self, keyMapper: KeyMapperCallableT, storer: Saver, serializerMapper: SerializerMapperCallableT) -> None: + super().__init__(keyMapper, storer) + self.key = keyMapper + self.serializer = serializerMapper + + def load(self, parent: "_ProtoBundle", field: "Field", key: KeyT) -> typing.Any: + rawRes = super().load(parent, field, key) + ser = self.serializer(parent) + return ser.process(rawRes) + + def save(self, parent: "_ProtoBundle", field: "Field", key: KeyT, value: typing.Any) -> None: + ser = self.serializer(parent) + rawRes = ser.unprocess(value) + super().save(parent, field, key, rawRes) diff --git a/urm/mappers/key.py b/urm/mappers/key.py new file mode 100644 index 0000000..39f3af8 --- /dev/null +++ b/urm/mappers/key.py @@ -0,0 +1,28 @@ +import typing + +from ..core import KeyT +from ..utils import resolveDynamics, toKey + + +class PrefixKeyMapper: + __slots__ = ("prefix",) + + def __init__(self, *prefix: typing.Tuple[str]) -> None: + self.prefix = tuple(prefix) + + def __call__(self, parent: "_ProtoBundle", field: "Field", key: KeyT) -> KeyT: + return toKey(resolveDynamics(parent, self.prefix)) + toKey(key) + + +class PostfixKeyMapper: + __slots__ = ("postfix",) + + def __init__(self, *postfix): + self.postfix = postfix + + def __call__(self, parent: "_ProtoBundle", field: "Field", key: KeyT) -> KeyT: + return toKey(key) + toKey(resolveDynamics(parent, self.postfix)) + + +def fieldNameKeyMapper(parent: "_ProtoBundle", field: "Field", key: KeyT) -> KeyT: # pylint:disable=unused-argument + return toKey(field.name) + toKey(key) diff --git a/urm/mappers/serializer.py b/urm/mappers/serializer.py new file mode 100644 index 0000000..6831075 --- /dev/null +++ b/urm/mappers/serializer.py @@ -0,0 +1,16 @@ +import typing + +from transformerz.core import TransformerBase + +from ..core import Dynamic +from ..utils import resolveDynamics + + +class JustReturnSerializerMapper: + __slots__ = ("res",) + + def __init__(self, res: typing.Union[TransformerBase, Dynamic]) -> None: + self.res = res + + def __call__(self, parent: "_ProtoBundle") -> TransformerBase: + return resolveDynamics(parent, self.res) diff --git a/urm/storers/__init__.py b/urm/storers/__init__.py new file mode 100644 index 0000000..9ad9574 --- /dev/null +++ b/urm/storers/__init__.py @@ -0,0 +1,24 @@ +import typing +from abc import ABC, abstractmethod + +from ..core import KeyT + + +class Storer(ABC): + __slots__ = () + + @abstractmethod + def get(self, parent: "_ProtoBundle", key: KeyT): + raise NotImplementedError + + @abstractmethod + def set(self, parent: "_ProtoBundle", key: KeyT, value: typing.Any): + raise NotImplementedError + + +class Cacher(Storer): # pylint:disable=abstract-method + __slots__ = () + + +class Saver(Storer): # pylint:disable=abstract-method + __slots__ = () diff --git a/urm/storers/cold.py b/urm/storers/cold.py new file mode 100644 index 0000000..c3cf79b --- /dev/null +++ b/urm/storers/cold.py @@ -0,0 +1,30 @@ +import typing +from pathlib import Path + +from . import Saver +from ..core import Dynamic, KeyT +from ..ProtoBundle import _ProtoBundle +from ..utils import resolveDynamics + + +class FileSaver(Saver): + __slots__ = ("root", "ext") + + def __init__(self, root: typing.Union[Dynamic, Path], ext: str = None) -> None: + self.root = root + self.ext = ext + + def getFSPath(self, parent: _ProtoBundle, key: KeyT) -> Path: # pylint:disable=unused-argument + root = resolveDynamics(parent, self.root) + ext = resolveDynamics(parent, self.ext) + if ext is not None: + return root._make_child(key[:-1])._make_child((".".join((key[-1], ext)),)) + return root._make_child(key) + + def get(self, parent: _ProtoBundle, key: KeyT): + return self.getFSPath(parent, key).read_bytes() + + def set(self, parent: _ProtoBundle, key: KeyT, value: typing.Any) -> int: + p = self.getFSPath(parent, key) + p.parent.mkdir(parents=True, exist_ok=True) + return p.write_bytes(value) diff --git a/urm/storers/hot.py b/urm/storers/hot.py new file mode 100644 index 0000000..2ea565f --- /dev/null +++ b/urm/storers/hot.py @@ -0,0 +1,52 @@ +import typing + +from . import Cacher +from ..core import KeyT +from ..ProtoBundle import _ProtoBundle +from ..utils import adaptKeyToContainers + + +class ValueCacher(Cacher): + __slots__ = ("value",) + + def __init__(self) -> None: + self.value = None + + def get(self, parent: _ProtoBundle, key: KeyT): + assert not key, "Key must be always empty for this cacher, but got " + repr(key) + return self.value + + def set(self, parent: _ProtoBundle, key: KeyT, value: typing.Any): + assert not key, "Key must be always empty for this cacher, but got " + repr(key) + self.value = value + + +class CollectionCacher(ValueCacher): + __slots__ = ("ctor",) + + def __init__(self, ctor) -> None: + self.ctor = ctor + super().__init__() + + def get(self, parent: _ProtoBundle, key: KeyT): + return self.value[adaptKeyToContainers(key)] + + def set(self, parent: _ProtoBundle, key: KeyT, value: typing.Any): + if self.value is None: + self.value = self.ctor() + self.value[adaptKeyToContainers(key)] = value + + +class PrefixCacher(Cacher): + __slots__ = ("prefix",) + + def __init__(self, prefix: str = "_") -> None: + self.prefix = prefix + + def get(self, parent: _ProtoBundle, key: KeyT) -> typing.Any: + assert len(key) == 1, "This cacher may be used only with keys of len(key)==1, but got " + repr(key) + return getattr(parent, self.prefix + key[0], None) + + def set(self, parent: _ProtoBundle, key: KeyT, value: typing.Any) -> None: + assert len(key) == 1, "This cacher may be used only with keys of len(key)==1, but got " + repr(key) + setattr(parent, self.prefix + key[0], value) diff --git a/urm/utils.py b/urm/utils.py new file mode 100644 index 0000000..5c9948d --- /dev/null +++ b/urm/utils.py @@ -0,0 +1,38 @@ +import typing + +from .core import Dynamic, KeyT + + +def getPathAttr(parent: typing.Any, path: KeyT) -> typing.Any: + v = parent + for comp in path: + v = getattr(v, comp) + return v + + +def setPathAttr(parent: typing.Any, path: KeyT, v: typing.Any): + o = parent + for comp in path[:-1]: + o = getattr(o, comp) + setattr(o, path[-1], v) + + +def adaptKeyToContainers(key: KeyT) -> typing.Union[str, KeyT]: + if len(key) == 1: + return key[0] + return key + + +def toKey(key: typing.Union[str, KeyT]) -> KeyT: + if not isinstance(key, tuple): + return (key,) + + return key + + +def resolveDynamics(parent: typing.Any, key: KeyT) -> typing.Any: + if isinstance(key, Dynamic): + return getPathAttr(parent, key.path) + if isinstance(key, tuple): + return tuple(resolveDynamics(parent, el) for el in key) + return key