diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 219c9a0..554fb86 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -28,31 +28,9 @@ jobs: run: | sudo apt update > /dev/null sudo apt-get -q update - sudo apt-get -q -y install tmux vim libncurses5 - wget -q https://github.com/FirebirdSQL/firebird/releases/download/R2_5_8/FirebirdCS-2.5.8.27089-0.amd64.tar.gz - tar xzf FirebirdCS-2.5.8.27089-0.amd64.tar.gz - pushd FirebirdCS-2.5.8.27089-0.amd64 - tar xzf buildroot.tar.gz - mv opt/firebird /opt - popd - echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/firebird/lib" >> "$GITHUB_ENV" + sudo apt-get -q -y install tmux vim libncurses5 fbclient2 python -m pip -q install -U pip virtualenv flake8 pytest if [ -f requirements.txt ]; then pip -q install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Unzip sample db - run: | - ./fb_export -h - - name: Export test json data - run: | - ./fb_export -d test/employee.fdb -e -o test/json -b - - name: Run tests with pytest - run: | - pytest - name: Setup lhotari ssh session on disaster - if: ${{ failure() }} + if: ${{ failure() || success() }} uses: lhotari/action-upterm@v1 diff --git a/.gitignore b/.gitignore index 68bc17f..3c4498e 100644 --- a/.gitignore +++ b/.gitignore @@ -121,7 +121,7 @@ celerybeat.pid # Environments .env -.venv +.venv* env/ venv/ ENV/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 76bf46f..e7afb64 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,8 +11,8 @@ "program": "src/fb_export.py", //"${file}", "console": "integratedTerminal", "justMyCode": true, - "args": [ "-d", "test/employee.fdb", // "-e", // "-c", // "-l", //"-m 10", // "-u", "SYSDBA", "-p", "masterkey", - "-o" , "export", // "-F", "json" + "args": [ "test/employee.fdb", // "-e", // "-c", // "-l", //"-m 10", // "-u", "SYSDBA", "-p", "masterkey", + // "-o" , "OutDir", // "-F", "json" ] , },], //other launch setting diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..424bc57 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Change Log + +## [0.2.0] - 2024-05-16 +### Added: +Make it an importable unit. Add CHANGELOG.md. Remove shell script fb_export and replace with +python script. + +## [0.1.0] - 2024-05-10 +### Initial upload + +Working with Firebird 2.5 and also via Github Actions runner on ubuntu-latest. + + diff --git a/fb_export b/fb_export deleted file mode 100755 index 3154642..0000000 --- a/fb_export +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python src/fb_export.py $* diff --git a/pyproject.toml b/pyproject.toml index 448aa7c..67da7bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,12 @@ requires = ["setuptools",] build-backend = "setuptools.build_meta" [project] -name = "firebird-export" +name = "fb_export" authors = [ {name = "Inus Scheepers", email = "In.uS.sc@gmail.com"} ] description = "Export Firebird database to CSV, JSON or other pandas format from dataframe" -version = "0.1.0" +dynamic = ["version"] readme = "README.md" requires-python = ">=3.9" dependencies = [ @@ -49,15 +49,11 @@ classifiers = [ "Topic :: File Formats", ] -[project.scripts] -firebird-export = "fb_export" - [project.urls] Homepage = "https://github.com/inus/firebird-export" -#Documentation = "https://readthedocs.org" -#Repository = "https://github.com/me/spam.git" -#Issues = "https://github.com/me/spam/issues" -#Changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md" +Repository = "https://github.com/inus/firebird-export.git" +Issues = "https://github.com/inus/firebird-export/issues" +Changelog = "https://github.com/inus/firebird-export/CHANGELOG.md" [tool.pytest.ini_options] testpaths = "test" @@ -70,3 +66,11 @@ log_date_format = "%Y-%m-%d %H:%M:%S" #filterwarnings = "ignore" #norecursedirs = docs build +[project.scripts] +fb_export = "fb_export.fb_export:main" + +[tool.setuptools] +package-dir = {"fb_export" = "src"} + +[tool.setuptools.dynamic] +version = {attr = "fb_export.about.VERSION"} diff --git a/src/about.py b/src/about.py index 3f83e09..4361f98 100644 --- a/src/about.py +++ b/src/about.py @@ -1,3 +1,3 @@ # about.py -VERSION = "0.0.1" -DATE = '2024-05-10' +VERSION = "0.2.0" +DATE = '2024-05-16' diff --git a/src/args.py b/src/args.py index 7b125e5..2617c65 100644 --- a/src/args.py +++ b/src/args.py @@ -1,34 +1,38 @@ # Firebird export arguments from about import VERSION - import argparse -def get_args(): +def get_args(*fbe_args): + parser = argparse.ArgumentParser(description="Firebird export") + + parser.add_argument( 'path_to_db', type=str, + help="Firebird server alias or database name \ + (eg. employee) or path to database file, eg ./employee.fdb") + parser.add_argument('--version', action='version', version='%(prog)s version ' + VERSION) parser.add_argument("-e", "--export", action='store_true', default=False, help="Export data") - parser.add_argument('-b', '--brief', action='store_true', default=False,) + parser.add_argument('-b', '--brief', action='store_true', default=False,help="Shorter info") parser.add_argument('-l', '--limit', action='store_true', default=False, - help="Limit tables and fields to those in customselect.py") - parser.add_argument('-m', '--maxrows', help="Max number of rows returned in export") - - #fixme - numsamples to be conditional on sampledata + help="Limit tables and fields to those in limit_sql.py") + parser.add_argument('-m', '--maxrows', help="Max number of rows to export") parser.add_argument('-s', '--sampledata', action='store_true', default=False, - help="Also show sample record") + help="Also show sample data records") parser.add_argument('-n', '--numsamples', default=3, help="number of sample rows") - parser.add_argument('-c', '--combine', action='store_true', default=False, - help="Combine output into a single file") + parser.add_argument('-j', '--join', action='store_true', default=False, + help="Join output files") format = [ 'csv', 'json',] # 'excel', 'sql', 'hdf', 'pickle', 'html' ....]? parser.add_argument("-F", "--format", choices=format, default='csv', help="Export output format, default .CSV") parser.add_argument("-o", "--outdir", type=str, dest='outdir', default='Export', help="Output directory") parser.add_argument("-u", "--user", type=str, default='SYSDBA', help="Firebird DB username") parser.add_argument("-p", "--password", type=str, default='masterkey', help="Firebird DB password") - parser.add_argument("-d", "--database", dest='path_to_db', type=str, required=True, - default="tests/employee.fdb", - help="Firebird server alias or database name (eg. employee) or full path if file, eg ./employee.fdb") - args = parser.parse_args() + if len(fbe_args) == 0: # Command line + args = parser.parse_args() + else: # From module + args = parser.parse_args( fbe_args[0]) + return args diff --git a/src/fb_export.py b/src/fb_export.py index caf3a79..ca7abc3 100755 --- a/src/fb_export.py +++ b/src/fb_export.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Dump firebase data +# Export Firebird database contents to CSV import fdb from shutil import rmtree @@ -7,7 +7,13 @@ import sys import os from pathlib import Path + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + + from args import get_args +#import .args from dftools import fix_fields from utils import mkdirs @@ -20,14 +26,13 @@ unzip_testdb() -def main(): - args = get_args() - +def main(*fbe_arg): + args = get_args(fbe_arg) if os.getenv('GITHUB_ACTIONS'): con = fdb.connect(args.path_to_db, user=args.user, password=args.password, fb_library_name='/opt/firebird/lib/libfbembed.so') else: - con = fdb.connect(args.path_to_db, user=args.user, password=args.password) + con = fdb.connect(args.path_to_db) print("Connected to ", con.database_name, ' via ', con.firebird_version, file=sys.stderr) @@ -122,7 +127,7 @@ def main(): except: print("Error converting DF to json: " + table, file=sys.stderr) - if args.combine: + if args.join: filename = dbf_name.rstrip('.fdb') + '.' + args.format fmode = 'a' else: