Update CI/CD and use dataclasses instead of namedtuples

.github/workflows/build.yaml

@@ -0,0 +1,47 @@
+name: Build
+on:
+  push:
+    branches: [ main, dev ]
+  pull_request:
+    branches: [ main, dev ]
+jobs:
+  build:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+    steps:
+      - name: Start Kafka docker
+        run: docker run -d -p 9092:9092 -e ADV_HOST=127.0.0.1 lensesio/fast-data-dev
+      - uses: actions/checkout@v2
+      - name: Fetch complete history for all tags and branches
+        run: git fetch --prune --unshallow
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup pip
+        run: python -m pip install --upgrade pip setuptools wheel
+      - name: Install package
+        run: pip install .[dev]
+      - name: Run black
+        run: black --check .
+      - name: Run flake8
+        run: flake8 .
+      - name: Run isort
+        run: isort --check --profile=black .
+      - name: Run mypy
+        run: mypy .
+      - name: Run pytest
+        run: py.test --cov=./ --cov-report=xml
+      - name: Run Sphinx doctest
+        run: python -m sphinx -b doctest docs docs/_build
+      - name: Run Sphinx HTML
+        run: python -m sphinx -b html -W docs docs/_build
+      - name: Upload coverge to Codecov
+        uses: codecov/codecov-action@v1
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.8'
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/codeql-anaylsis.yaml

@@ -0,0 +1,22 @@
+name: CodeQL
+on:
+  push:
+    branches: [ main, dev ]
+  pull_request:
+    branches: [ main, dev ]
+  schedule:
+    - cron: '21 2 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: 'python'
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/pypi-upload.yaml

@@ -0,0 +1,31 @@
+name: Upload to PyPI
+on:
+  release:
+    types: [created]
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Fetch complete history for all tags and branches
+      run: git fetch --prune --unshallow
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine setuptools-scm[toml]
+    - name: Build distribution
+      run: python setup.py sdist bdist_wheel
+    - name: Publish to PyPI Test
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_TEST_TOKEN }}
+      run: twine upload --repository testpypi dist/*
+    - name: Publish to PyPI
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+      run: twine upload --repository pypi dist/*

.gitignore

@@ -111,3 +111,6 @@ venv.bak/
 
 # KQ specific
 output.log
+
+# setuptools-scm
+kq/version.py

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    hooks:
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.7.0
+    hooks:
+      - id: isort
+        args: [ --profile, black ]
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.790
+    hooks:
+      - id: mypy
+        args: [ --ignore-missing-imports ]
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
+    hooks:
+      - id: flake8

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2016 Joohwan Oh
+Copyright (c) 2016,2017,2018,2019,2020,2021 Joohwan Oh
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST.in

@@ -1,2 +1,2 @@
-include README.rst LICENSE
+include README.md LICENSE
 prune tests

README.md

@@ -0,0 +1,121 @@
+## KQ: Kafka Job Queue for Python
+
+![Build](https://github.com/joowani/kq/workflows/Build/badge.svg)
+![CodeQL](https://github.com/joowani/kq/workflows/CodeQL/badge.svg)
+[![codecov](https://codecov.io/gh/joowani/kq/branch/master/graph/badge.svg?token=d6ooyuUCl6)](https://codecov.io/gh/joowani/kq)
+
+**KQ (Kafka Queue)** is a lightweight Python library which lets you queue and
+execute jobs asynchronously using [Apache Kafka](https://kafka.apache.org/). It is 
+built on top of [kafka-python](https://github.com/dpkp/kafka-python).
+
+### Announcements
+
+* Python 3.5 will not be supported from kq v3.0.0.
+* See [releases](https://github.com/joowani/kq/releases) for latest updates.
+
+### Requirements
+
+* [Apache Kafka](https://kafka.apache.org) 0.9+
+* Python 3.6+
+
+### Installation
+
+Install using [pip](https://pip.pypa.io):
+
+```shell
+pip install kq
+```
+
+### Usage
+
+Start your Kafka instance. 
+Example using [Kafka Docker](https://github.com/lensesio/fast-data-dev):
+
+```shell
+docker run -p 9092:9092 -e ADV_HOST=127.0.0.1 lensesio/fast-data-dev
+```
+
+Define your KQ ``worker.py`` module:
+
+```python
+import logging
+
+from kafka import KafkaConsumer
+from kq import Worker
+
+# Set up logging.
+formatter = logging.Formatter('[%(levelname)s] %(message)s')
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(formatter)
+logger = logging.getLogger('kq.worker')
+logger.setLevel(logging.DEBUG)
+logger.addHandler(stream_handler)
+
+# Set up a Kafka consumer.
+consumer = KafkaConsumer(
+    bootstrap_servers='127.0.0.1:9092',
+    group_id='group',
+    auto_offset_reset='latest'
+)
+
+# Set up a worker.
+worker = Worker(topic='topic', consumer=consumer)
+worker.start()
+```
+
+Start your worker:
+
+```shell
+python my_worker.py
+[INFO] Starting Worker(hosts=127.0.0.1:9092 topic=topic, group=group) ...
+```
+
+Enqueue a function call:
+
+```python
+import requests
+
+from kafka import KafkaProducer
+from kq import Queue
+
+# Set up a Kafka producer.
+producer = KafkaProducer(bootstrap_servers='127.0.0.1:9092')
+
+# Set up a queue.
+queue = Queue(topic='topic', producer=producer)
+
+# Enqueue a function call.
+job = queue.enqueue(requests.get, 'https://google.com')
+
+# You can also specify the job timeout, Kafka message key and partition.
+job = queue.using(timeout=5, key=b'foo', partition=0).enqueue(requests.get, 'https://google.com')
+```
+
+The worker executes the job in the background:
+
+```shell
+python my_worker.py
+[INFO] Starting Worker(hosts=127.0.0.1:9092, topic=topic, group=group) ...
+[INFO] Processing Message(topic=topic, partition=0, offset=0) ...
+[INFO] Executing job c7bf2359: requests.api.get('https://www.google.com')
+[INFO] Job c7bf2359 returned: <Response [200]>
+```
+
+See [documentation](https://kq.readthedocs.io) for more information.
+
+### Contributing
+
+Set up dev environment:
+
+```shell
+cd ~/your/kq/clone          # Activate venv if you have one
+pip install -e .[dev]       # Install dev dependencies (black, mypy, pre-commit etc.)
+pre-commit install          # Install git pre-commit hooks
+py.test                     # Run unit tests
+```
+
+Run unit tests:
+```shell
+docker run -p 9092:9092 -e ADV_HOST=127.0.0.1 lensesio/fast-data-dev  # Start Kafka docker
+py.test
+```