From 410ba5d8a978459d2effc02ce7b18747223f8718 Mon Sep 17 00:00:00 2001 From: Stehlampe2020 Date: Sat, 5 Aug 2023 13:57:24 +0200 Subject: [PATCH] Add methods to l2db.py and bump version number of SPEC.md --- .idea/L2DB.iml | 6 ++-- .idea/misc.xml | 2 +- SPEC.md | 68 ++++++++++++++++++------------------ l2db.py | 95 ++++++++++++++++++++++++++++++++++++++++---------- setup.py | 1 + 5 files changed, 116 insertions(+), 56 deletions(-) diff --git a/.idea/L2DB.iml b/.idea/L2DB.iml index fa7a615..1966f12 100644 --- a/.idea/L2DB.iml +++ b/.idea/L2DB.iml @@ -1,8 +1,10 @@ - - + + + + diff --git a/.idea/misc.xml b/.idea/misc.xml index dc9ea49..6b977f3 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/SPEC.md b/SPEC.md index cd823be..5e47d4e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,7 +1,7 @@ # L2DB file format specification *If you want to make an alternative implementation of this format, use this document as a reference to ensure compatibility.* -- Version 1.0.3 -- Copyright (c) by Lampe2020 +- Version 1.1.0 +- Copyright (c) by Christian Lampe - If strings in this spec contain a variable name enclosed in double curly braces this means that that part of the string shouldn't be taken literally but instead replaced with the appropriate content, if not specified otherwise. - "spec", "the spec" or "this spec" in the following document refer to this specification unless otherwise specified. @@ -65,7 +65,7 @@ The length of this string is specified in the "Index length" bytes in the [heade Each entry consists of 8 bytes for the index number(s) followed by three non-`null` bytes for the value type and a variable amount of non-`null` bytes for the name which is terminated by one `null`-byte. If the type is unknown it will be interpreted as raw and a warning should be emitted stating `Unknown format -{{format}}! Interpreting as raw`. +{{format}}! Interpreting as 'raw'`. The order of these entries does not need to be maintained but can be. If the flag `X64_INDEXES` is not set the index numbers will be two `uint32`s which refer to the starting and end offset of the value's data. @@ -78,9 +78,7 @@ Be aware that in this case the values can still switch order but then the offset The data section is a pure concatenation of all values in the whole database. ## Value types -*If implicit type conversions are done, emit a warning `Could not assign '{{new_type}}' to a key of type -'{{old_type}}'. Implicitly converted the key to '{{new_type}}'`, with `new_type` being the new value's type Identifier -(see table below) and `old_type` being the previous type Identifier (see table below) of the key. +*If implicit type conversions are done, emit a warning `Implicitly converted '{{old_type}}' to '{{new_type}}'`. If a type conversion fails or isn't possible, raise a `L2DBTypeError` exception with the message `Could not assign value of type '{{val_type}}' to key of type '{{key_type}}'`, with `val_type` being the value's type Identifier (see table below) and `key_type` being the key's type Identifier (see table below), optionally extend the message with @@ -90,7 +88,8 @@ table below) and `key_type` being the key's type Identifier (see table below), o |:---------------------:|:----------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Whole number | `int` | Any positive or negative 64-bit whole number. (aka.`long`[^2]) If a positive number too large for a normal `long` is tried to assign, implicitly convert the key to a `uin` if that allows for storing the value, otherwise fail. | | Positive whole number | `uin` | Any positive 64-bit whole number. (aka.`unsigned long`[^2]) If a negative number is tried to assign, implicitly convert the key to a `int` if that allows for storing the value, otherwise fail. | -| Number | `flt` | Any positive or negative 64-bit number. (aka.`double`[^2]) | +| Floating point number | `flt` | Any positive or negative 64-bit number. (aka.`double`[^2])
*Note that this will sooner or later be removed in favor of `fpn`* | +| Number | `fpn` | Any positive or negative 64-bit number, stored in a custom format.
*Note: `fpn` should currently automatically get converted to `flt` and strict implementations should emit a warning with the message `'fpn' is not implemented yet as there is no standard for it`* | | Boolean | `bol` | True or False. Is stored in a single byte which is set to either 0x01 (True) or 0x00 (False). If a `int`, `uin` or `flt` 0 or 1 or raw `null`-byte (`\0`) or one-byte (`\1`) is tried to assign, implicitly convert the value to `True` for 1 and `False` for 0. If `null` is tried to assign, implicitly convert the key to `nul`.
*Note: in strict implementations the `DIRTY` bit should be set if this byte is anything other than 0x00 or 0x01!* | | String | `str` | Any UTF-8 encoded string. | | Raw | `raw` | Any sequence of bytes. | @@ -117,23 +116,25 @@ method.* ### `open()` -| Argument name | Default value | Optional? | Possible values | -|:---------------:|:--------------:|:---------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `source` | | No | - File path, as String
- `r` or `rw` binary file handle
- `bytes` to act on as if they were the file content
- `dict` with zero or more valid key-value pairs, invalid pairs are tried to convert or are otherwise discarded with a warning stating `Could not load key '{{keyname}}', discarding it`. | -| `mode` | `'rw'` | Yes | String with any combination of letters described in the [modes](#modes) table. | -| `runtime_flags` | empty `tuple` | Yes | A list of strings that specify each one runtime flag name to be enabled. All runtime flags are by default disabled. | -| Return value | | | The database object that has been opened by calling this method | +| Argument name | Default value | Optional? | Possible values | +|:---------------:|:--------------:|:---------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `source` | | No | File path, as String
- `rb`, `r+b` or `w+b` file handle
- `bytes` to act on as if they were the file content
- `dict` with zero or more valid key-value pairs, invalid pairs are tried to convert or are otherwise discarded with a warning stating `Could not load key '{{keyname}}', discarding it`. | +| `mode` | `'rw'` | Yes | String with any combination of letters described in the [modes](#modes) table. | +| `runtime_flags` | empty `tuple` | Yes | A list of strings that specify each one runtime flag name to be enabled. All runtime flags are by default disabled. | +| Return value | | | The database object that has been opened by calling this method | This method populates the `L2DB` with the new content and (if there were more than zero keys in the `L2DB` before) emits a warning stating `Old content of L2DB has been discarded in favor of new content`. This method is also called by the object constructor to populate the database. +If the `source` is a file handle, `mode` is ignored and taken from the file handle's `mode` attribute. If that +is invalid for L2DB the mode is set by the `mode` argument. ### `read()` | Argument name | Default value | Optional? | Possible values | |:-------------:|:-------------:|:---------:|:-----------------------------------------------------------------------------------------------------| | `key` | | No | Any string that occurs as a key name in the currently-opened DB | -| `type` | `None` | Yes | Any three-letter [type Identifier](#value-types) that the value should be converted to after reading | +| `vtype` | `None` | Yes | Any three-letter [type Identifier](#value-types) that the value should be converted to after reading | | Return value | `None` | | The value of the read key | This method returns the value of the requested key if possible, if no type is given the data is returned as the stored @@ -150,7 +151,8 @@ random). | Argument name | Default value | Optional? | Possible values | |:-------------:|:---------------------:|:---------:|:-------------------------------------------------------------------------------------------------------------------| | `key` | | No | Any string that doesn't contain a `null`-byte | -| `type` | `None` | Yes | Any three-letter [type Identifier](#value-types) that the stored value should be converted to before writing | +| `value` | | No | Any value storable in an L2DB format | +| `vtype` | `None` | Yes | Any three-letter [type Identifier](#value-types) that the stored value should be converted to before writing | | Return value | `{'key':'','val':''}` | | A `dict` with the keys `key` and `val` which contains the given key and value as if they had been read from the DB | This method stores any value given to it into the database with the key provided to it. @@ -163,8 +165,7 @@ If a specific type is given the values is converted to | `key` | | No | Any string that matches an existing key stored in the DB | | Return value | `{'key':'','val':''}` | | A `dict` with the keys `key` and `val` which contains the given key and value as they were stored in the DB | -Removes the given key along with its value from the DB. In file mode this function renames the key to `---deleted---`, -removes its value and sets its type to `nul`, no matter if it already exists. +Removes the given key along with its value from the DB. If the key doesn't already exist, it raises a `L2DBKeyError` exception with the message `{{key}} could not be found`, with `key` in single quotes and any contained single quotes escaped with a backslash. @@ -173,15 +174,14 @@ found`, with `key` in single quotes and any contained single quotes escaped with | Argument name | Default value | Optional? | Possible values | |:-------------:|:-------------:|:---------:|:----------------------------------------------------------------------------------------------| -| `keyname` | | No | Any string matching a key's name or (if `fromval` is set) an empty string | -| `type` | | No | Any three-letter string matching one of the type Identifiers (see [type table](#value-types)) | +| `key` | | No | Any string matching a key's name or (if `fromval` is set) an empty string | +| `vtype` | | No | Any three-letter string matching one of the type Identifiers (see [type table](#value-types)) | | `fromval` | `None` | Yes | Any value representable as one of the [L2DB-compatible types](#value-types) | | Return value | | | The converted value | Converts the key along with its value to the target type, if that fails a `L2DBTypeError` exception should be raised -with the message `Could not convert {{keyname}} to type '{{type}}'`, with `keyname` in single-quotes except if the name -contains single-quotes, then double-quotes. -If `fromval` is set `keyname` is ignored and the value to convert is taken from `fromval` instead of the DB. +with the `key` name and `vtype` and if `fromval` is set `key` should be `None`. +If `fromval` is set the given `key` name is ignored and the value to convert is taken from `fromval` instead of the DB. If a `flt` is converted to any whole number type it simply loses its decimals (not rounded but cut off) and if any whole number type is converted to `flt` it gets 0 as the only decimal place. Examples: `1.999 -> 1`, `-3.7 -> 3` and `1 -> 1.0` @@ -201,18 +201,18 @@ random, the implementer decides which one). ### `flush()` -| Argument name | Default value | Optional? | Possible values | -|:-------------:|:-------------:|:---------:|:---------------------------------------------------| -| `filename` | `None` | Yes | any string or binary file handle with write access | -| `move` | `False` | Yes | any boolean | -| Return value | | | | +| Argument name | Default value | Optional? | Possible values | +|:-------------:|:-------------:|:---------:|:----------------------------------------------------------------------------------| +| `file` | `None` | Yes | any string which is a valid file path or file handle in `wb`, `r+b` or `w+b` mode | +| `move` | `False` | Yes | any boolean | +| Return value | | | | This method flushes the buffered changes to the given file or (if none given) to the file the database has been read from. If the database is in [file mode](#modes) this will just clone the database file to the new location, see the [file mode's description](#modes). *Note: If no file is given and none has been used to initialize the database this method shall raise a -`FileNotFoundError` with the message `No file specified!`!* +`FileNotFoundError` with the message `No file specified`!* ### `cleanup()` | Argument name | Default value | Optional? | Possible values | @@ -232,12 +232,12 @@ After the check the `DIRTY` flag is reset and (if the runtime-flag `verbose` is ## Error classes -| Error name | Default message | Explanation | -|:---------------------:|:-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `L2DBError` | empty string | Base error type, base class for all other L2DB errors to inherit from | -| `L2DBVersionMismatch` | `The database follows the spec version {{db_ver}} but the implementation follows the spec version {{imp_ver}}. Conversion failed.` | `db_ver` is the `major.minor` version of the spec that the database file follows and `imp_ver` is the `major.minor` version of the spec that the implementation follows. | -| `L2DBTypeError` | `Could not convert '{{keyname}}' to type '{{type}}'` | `keyname` is the name of the key tried to convert (if it contains single quotes they should be escaped with backslashes) and `type` is the target type. | -| `L2DBKeyError` | `Key '{{key}}' could not be found` | `key` is the name of the key that could not be found (if it contains single quotes they should be escaped with backslashes). | +| Error name | Default message | Explanation | +|:---------------------:|:-------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `L2DBError` | empty string | Base error type, base class for all other L2DB errors to inherit from | +| `L2DBVersionMismatch` | `database follows spec version {{db_ver}} but implementation follows spec version {{imp_ver}}. Conversion failed.` | `db_ver` is the `major.minor` version of the spec that the database file follows and `imp_ver` is the `major.minor` version of the spec that the implementation follows. | +| `L2DBTypeError` | `Could not convert key '{{keyname}}' to type '{{type}}'` or `Could not convert value to type '{{type}}'` | `keyname` is the name of the key tried to convert (if it contains single quotes they should be escaped with backslashes) and `type` is the target type. | +| `L2DBKeyError` | `Key '{{key}}' could not be found` | `key` is the name of the key that could not be found (if it contains single quotes they should be escaped with backslashes). | diff --git a/l2db.py b/l2db.py index 7cc8cb9..9b053dc 100644 --- a/l2db.py +++ b/l2db.py @@ -1,20 +1,23 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" -The L2DB database format, version 1. (c) Lampe2020 -L2DB supports the following data types: - * keys: string (UTF-8-encoded text) - * Values: integer (32-bit), long (64-bit), float (32-bit), double (64-bit), string (UTF-8-encoded text), raw (binary data) - -#################################################################################### -# NOTE that this code doesn't fully comply with the standard defined in SPEC.md! # -#################################################################################### -→ This is the case because I changed some things when actually coming up with a spec instead of just randomly creating spaghetti code. +spec_version = '1.1.0' +implementation_version = '0.1.0-pre-alpha+python3-above-.7' + +__doc__ = f""" +L2DB {spec_version} - implementation {implementation_version} +Both version numbers follow the SemVer 2.0.0 standard (http://semver.org/spec/v2.0.0.html) + +Simple binary database format made by Christian Lampe + +Spec: see SPEC.md +This module is the Python3-based example implementation of the database format, feel free to make a better +implementation. +This implementation is a strict implementation, so it follows even the rules for strict implementations. """ import collections.abc as collections -import struct +import struct, warnings, semver ##################################################################### # Helper functions - must be moved into `L2DB.__helpers()` later on # @@ -36,25 +39,79 @@ def getbit(seq, pos): ############## class L2DBError(Exception): + """L2DB base exception""" def __init__(self, message=''): + self.message = message super().__init__(self.message) class L2DBVersionMismatch(L2DBError): - def __init__(self, db_ver='0.0.0', imp_ver='0.0.0'): - super().__init__( - f'The database follows the spec version {db_ver} but the implementation follows the spec version {imp_ver}.\ - Conversion failed.' - ) + """Raised when conversion between spec versions fails""" + def __init__(self, db_ver='0.0.0-please+replace', imp_ver=implementation_version): + self.message = f'database follows spec version {db_ver} but implementation follows spec version {imp_ver}. Conversion failed.' + super().__init__(self.message) class L2DBTypeError(L2DBError): - def __init__(self, keyname='', ktype='inv'): # Renamed `type` to `ktype` because `type` is a Python3-builtin + """Raised when conversion between value types fails""" + def __init__(self, key='', vtype='inv'): # Renamed `type` to `vtype` because `type` is a Python3-builtin toreplace = ("'", "\\'") - super().__init__(f"Could not convert '{keyname.replace(*toreplace)}' to type '{ktype.replace(*toreplace)}'") + self.message = f"Could not convert key '{key.replace(*toreplace)}' to type '{vtype.replace(*toreplace)}'" if ( + key!=None) else f"Could not convert value to type '{vtype.replace(*toreplace)}'" + super().__init__(self.message) class L2DBKeyError(L2DBError): + """Raised when an unaccessible key is accessed""" def __init__(self, key=''): toreplace = ("'", "\\'") - super().__init__(f"Key '{key.replace(*toreplace)}' could not be found") + self.message = f"Key '{key.replace(*toreplace)}' could not be found" + super().__init__(self.message) + +######## +# L2DB # +######## + +class L2DB: + __doc__ = f'L2DB {spec_version} in implementation {implementation_version}' + spec = spec_version + implementation = implementation_version + def __init__(self, source, mode, runtime_flags): + self.__db = {} + self.source, self.mode, self.runtime_flags = source, mode, runtime_flags + self.open(source, mode, runtime_flags) + + def open(self, source, mode='rw', runtime_flags=()): + """Populates the L2DB with new content and sets its source location if applicable. + Accepts modes 'r', 'w', 'f' and any combination of those three.""" + + def read(self, key, vtype=None): + """Returns the value of the key, optionally converts it to `vtype`. + Raises an L2DBKeyError if the key is not found. + Raises an L2DBTypeError if the value cannot be converted to the specified `vtype`.""" + + def write(self, key, value, vtype=None): + """Writes `value` to `key`, optionally converts it to `vtype`. + Raises an L2DBKeyError if the key name is invalid. + Raises an L2DBTypeError if the value cannot be converted to the specified `vtype`.""" + + def delete(self, key): + """Removes a key from the L2DB.""" + + def convert(self, key, vtype, fromval): + """Converts the key or value to type `vtype`.""" + + def dump(self): + """Dumps all key-value pairs as a `dict`""" + + def flush(self, file=None, move=False): + """Flushes the buffered changes to the file the database has been initialized with. + If a file is specified this flushes the changes to that file instead and changes the database source to the new + file if `move` is True. + Raises a FileNotFoundError with the message 'No file specified' if the database has no source file and none is + specified.""" + + def cleanup(self, only_flag=False, dont_rescue=False): + """Tries to repair the database and unsets the `DIRTY` flag. + Skips all repairs if `only_flag` is True. + Discards corrupted key-value pairs instead of rescuing them if `dont_rescue` is True.""" ######## # Test # diff --git a/setup.py b/setup.py index c44e050..b4fde22 100644 --- a/setup.py +++ b/setup.py @@ -4,4 +4,5 @@ name='L2DB', version='0.1.0-pre-alpha', scripts=['l2db.py'], + install_requires=['semver~=3.0.0'] )