ci: refactor project configuration and workflows

- Migrate from setup.py to pyproject.toml
- Update GitHub Actions workflows for PyPI publishing
- Restructure requirements files
- Consolidate package data configuration in pyproject.toml
- Remove MANIFEST.in in favor of pyproject.toml configuration

Details:
- Centralize all dependencies in pyproject.toml
- Set up proper optional dependencies for dev, test, and docs
- Update CI/CD pipeline for automated testing and PyPI deployment
- Reorganize requirements into separate files for different environments
- Improve package data handling

.github/workflows/cleanup.yml

@@ -0,0 +1,28 @@
+# .gitHub/workflows/cleanup.yml
+name: Cleanup Deployments
+
+on:
+  workflow_dispatch:  # 允許手動觸發
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      deployments: write
+      actions: write
+      contents: write
+
+    steps:
+      - name: Delete github-pages deployments
+        uses: strumwolf/delete-deployment-environment@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          environment: github-pages
+          onlyRemoveDeployments: true
+
+      - name: Delete pypi deployments
+        uses: strumwolf/delete-deployment-environment@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          environment: pypi
+          onlyRemoveDeployments: true

.github/workflows/publish.yml

@@ -0,0 +1,54 @@
+# .gitHub/workflows/publish.yml
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'  # 當推送版本標籤時觸發，如 v0.1.0
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel build twine
+
+      - name: Extract version from tag
+        id: get_version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
+
+      - name: Verify version matches
+        run: |
+          VERSION=$(python setup.py --version)
+          if [ "$VERSION" != "${{ env.VERSION }}" ]; then
+            echo "Version mismatch: Tag version (${{ env.VERSION }}) doesn't match package version ($VERSION)"
+            exit 1
+          fi
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to Test PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
+        run: |
+          twine upload --repository testpypi dist/*
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          twine upload dist/*

.github/workflows/pytest.yml

@@ -23,7 +23,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install pytest
-          pip install -r requirements.txt
+          pip install -r requirements/requirements.txt
           pip install -e .
 
       - name: Run tests

AeroViz/rawDataReader/script/Aurora.py

@@ -38,7 +38,7 @@ def _QC(self, _df):
         _df = _df.loc[(_df['B'] > _df['G']) & (_df['G'] > _df['R'])]
 
         # use IQR_QC
-        _df = self.time_aware_IQR_QC(_df)
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
         # make sure all columns have values, otherwise set to nan
         return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/NEPH.py

@@ -74,7 +74,7 @@ def _QC(self, _df):
         # _df = _df.loc[(_df['B'] > _df['G']) & (_df['G'] > _df['R'])]
 
         # use IQR_QC
-        _df = self.time_aware_IQR_QC(_df)
+        _df = self.time_aware_IQR_QC(_df, time_window='1h')
 
         # make sure all columns have values, otherwise set to nan
         return _df.dropna(how='any').reindex(_index)

AeroViz/rawDataReader/script/OCEC.py

@@ -87,6 +87,6 @@ def _QC(self, _df):
             _df.loc[_df[col] <= threshold, col] = np.nan
 
         # use IQR_QC
-        _df = self.time_aware_IQR_QC(_df, time_window='1h')
+        _df = self.time_aware_IQR_QC(_df)
 
         return _df.dropna(subset=['Thermal_OC', 'Optical_OC']).reindex(_index)

AeroViz/rawDataReader/script/SMPS.py

@@ -56,6 +56,10 @@ def _raw_reader(self, file):
             _df_smps.columns = _df_smps.columns.astype(float)
             _df_smps = _df_smps.loc[_df_smps.index.dropna().copy()]
 
+            if _df_smps.columns[0] != 11.8:
+                print(f'file_name: {file.name}')
+                return None
+
             return _df_smps.apply(to_numeric, errors='coerce')
 
     # QC data

AeroViz/rawDataReader/script/TEOM.py

@@ -40,7 +40,7 @@ def _QC(self, _df):
 
         # QC data in 1 hr
         # use time_aware_IQR_QC
-        _df = self.time_aware_IQR_QC(_df, time_window='1h')
+        _df = self.time_aware_IQR_QC(_df, time_window='6h')
 
         # remove data where size < 50% in 1-hr
         points_per_hour = Timedelta('1h') / Timedelta(self.meta['freq'])

README.md

@@ -59,7 +59,7 @@ RawDataReader supports a wide range of aerosol instruments, including NEPH, SMPS
 various file types and time resolutions, making data processing efficient and standardized.
 
 For a detailed list of supported instruments, file types, and data columns, please refer to
-our [RawDataReader Usage Guide](docs/user_guide/RawDataReader) in the `docs` folder.
+our [RawDataReader Usage Guide](docs/guide/RawDataReader) in the `docs` folder.
 
 ### Key Features:
 
@@ -105,10 +105,10 @@ For detailed documentation, please refer to the `docs` folder, which includes:
 
 <div align="center">
 
-| Documentation                              | Description                |
-|--------------------------------------------|----------------------------|
-| [User Guide](docs/user_guide)              | Basic usage instructions   |
-| [Changelog](docs/changelog.md)             | List of changes            |
+| Documentation                  | Description              |
+|--------------------------------|--------------------------|
+| [User Guide](docs/guide)       | Basic usage instructions |
+| [Changelog](docs/changelog.md) | List of changes          |
 
 </div>
 

docs/index.md

@@ -0,0 +1,133 @@
+## <div align="center">AeroViz for Aerosol Science Visualization</div>
+
+<p align="center">
+
+  <img alt="Static Badge" src="https://img.shields.io/badge/python-3.12-blue?logo=python">
+  <img alt="Static Badge" src="https://img.shields.io/badge/License-MIT-yellow">
+  <img alt="Static Badge" src="https://img.shields.io/badge/github-updating-red?logo=github">
+  <img alt="Static Badge" src="https://img.shields.io/badge/testing-green?logo=Pytest&logoColor=blue">
+
+</p>
+
+<div align="center">
+
+<a href="https://github.com/Alex870521"><img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-social-github.png?raw=true" width="3%" alt="Alex870521 GitHub"></a>
+<img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-transparent.png?raw=true" width="3%">
+<a href="https://www.linkedin.com/in/Alex870521/"><img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-social-linkedin.png?raw=true" width="3%" alt="Alex870521 LinkedIn"></a>
+<img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-transparent.png?raw=true" width="3%">
+<a href="https://medium.com/@alex870521"><img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-social-medium.png?raw=true" width="3%" alt="Alex870521 Medium"></a>
+
+</div>
+
+## <div align="center">Key Features</div>
+
+* Data Reading: Supports reading multiple aerosol data formats.
+* Data Visualization: Offers various charts and graphs, including time series plots, distribution plots, and correlation
+  matrices.
+* Data Processing: Includes multiple data processing tools, such as linear regression and Mie theory calculations.
+
+## <div align="center">Installation</div>
+
+```bash
+pip install AeroViz
+```
+
+For Windows users: Run `install_windows.bat`
+
+For Linux and Mac users: Run `install_unix.bat`
+
+## <div align="center">Quick Start</div>
+
+```python
+from datetime import datetime
+from pathlib import Path
+from AeroViz import RawDataReader, DataProcess, plot
+
+# Read data from a supported instrument
+data = RawDataReader('NEPH', Path('/path/to/data'), start=datetime(2024, 2, 1), end=datetime(2024, 4, 30))
+
+# Create a visualization
+plot.timeseries(data, y='scattering_coefficient')
+```
+
+For more detailed usage instructions, please refer to our [User Guide]().
+
+## <div align="center"> RawDataReader
+
+RawDataReader supports a wide range of aerosol instruments, including NEPH, SMPS, AE33, and many more. It handles
+various file types and time resolutions, making data processing efficient and standardized.
+
+For a detailed list of supported instruments, file types, and data columns, please refer to
+our [RawDataReader Usage Guide](docs/guide/RawDataReader) in the `docs` folder.
+
+### Key Features:
+
+- Supports multiple aerosol instruments
+- Applies customizable quality control measures
+- Offers flexible data filtering and resampling options
+- Enables easy data export to CSV format
+
+### Supported Instruments
+
+The AeroViz project currently supports data from the following instruments:
+
+- SMPS (Scanning Mobility Particle Sizer)
+- APS (Aerodynamic Particle Sizer)
+- GRIMM (GRIMM Aerosol Technik)
+- TEOM (Continuous Ambient Particulate Monitor)
+- NEPH (Nephelometer)
+- Aurora (Nephelometer)
+- AE33 (Aethalometer Model 33)
+- AE43 (Aethalometer Model 43)
+- BC1054 (Black Carbon Monitor 1054)
+- MA350 (MicroAeth MA350)
+- OCEC (Organic Carbon Elemental Carbon Analyzer)
+- IGAC (In-situ Gas and Aerosol Compositions monitor)
+- XRF (X-ray Fluorescence Spectrometer)
+- VOC (Volatile Organic Compounds Monitor)
+
+> **Note:** We are continuously working to support more instruments. Please check back for updates or contribute to our
+> project on GitHub.
+
+## <div align="center">DataProcess</div>
+
+The AeroViz project currently supports the following processing methods:
+
+- **Chemistry**:
+- **Optical**
+- **SizeDistr**
+- **VOC**
+
+## <div align="center">Documentation</div>
+
+For detailed documentation, please refer to the `docs` folder, which includes:
+
+<div align="center">
+
+| Documentation                  | Description              |
+|--------------------------------|--------------------------|
+| [User Guide](docs/guide)       | Basic usage instructions |
+| [Changelog](docs/changelog.md) | List of changes          |
+
+</div>
+
+## <div align="center">Related Source</div>
+
+* #### [PyMieScatt](https://github.com/bsumlin/PyMieScatt.git)
+* #### [py-smps](https://github.com/quant-aq/py-smps.git)
+* #### [ContainerHandle](https://github.com/yrr-Su/ContainerHandle.git)
+
+## <div align="center">Contact</div>
+
+For bug reports and feature requests please visit [GitHub Issues](https://github.com/Alex870521/DataPlot/issues).
+
+<div align="center">
+
+<a href="https://github.com/Alex870521"><img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-social-github.png?raw=true" width="3%" alt="Alex870521 GitHub"></a>
+<img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-transparent.png?raw=true" width="3%">
+<a href="https://www.linkedin.com/in/Alex870521/"><img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-social-linkedin.png?raw=true" width="3%" alt="Alex870521 LinkedIn"></a>
+<img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-transparent.png?raw=true" width="3%">
+<a href="https://medium.com/@alex870521"><img src="https://github.com/Alex870521/AeroViz/blob/main/assets/media/logo-social-medium.png?raw=true" width="3%" alt="Alex870521 Medium"></a>
+
+
+</div>

pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "AeroViz"
+version = "0.1.9.5"
+description = "Aerosol science"
+authors = [{ name = "alex", email = "[email protected]" }]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.12"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "pandas==2.2.2",
+    "numpy==1.26.4",
+    "matplotlib==3.8.4",
+    "scipy==1.14.0",
+    "seaborn==0.13.2",
+    "scikit-learn==1.5.1",
+    "windrose==1.9.2",
+    "cartopy==0.24.1",
+    "tabulate==0.9.0",
+    "rich~=13.7.1",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.1.0",
+    "pytest-mock>=3.10.0",
+]
+dev = [
+    "black>=23.0",
+    "isort>=5.12.0",
+    "flake8>=6.0.0",
+    "mypy>=1.5.0",
+    "build",
+    "twine",
+]
+docs = [
+    "mkdocs>=1.4.0",
+    "mkdocs-material>=8.0",
+    "mkdocstrings[python]>=0.18.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Alex870521/AeroViz"
+Repository = "https://github.com/Alex870521/AeroViz"
+Issues = "https://github.com/Alex870521/AeroViz/issues"
+
+# 只保留一個 setuptools 配置部分
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["AeroViz*"]
+exclude = ["tests*"]
+namespaces = false
+
+[tool.setuptools.package-data]
+AeroViz = ["*", "**/*"]

requirements/requirements-dev.txt

@@ -0,0 +1 @@
+-e .[dev,test]

requirements/requirements-docs.txt

@@ -0,0 +1 @@
+-e .[docs]