From 4e8f44d572684bc822c5edae61745cd74023c864 Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 19:25:50 +0200 Subject: [PATCH 1/8] BLD: - first test of a GitHub CI for testing --- .github/workflows/python-package.yml | 36 ++++++++++++++++++++++++++++ pyproject.toml | 2 +- requirements/git_ci.txt | 9 +++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-package.yml create mode 100644 requirements/git_ci.txt diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..54e4d20 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,36 @@ +name: Build and Test + +on: + push: + branches: + - develop # Change from main to develop + pull_request: + branches: + - develop # Change from main to develop + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9, 3.10, 3.11, 3.12] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip .["git_ci"] + + - name: Build Cython code + run: | + python setup.py build_ext --inplace + + - name: Run tests + run: | + pytest -n=auto -x diff --git a/pyproject.toml b/pyproject.toml index 3ecf2bb..9a137c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ package-data = {"*" = ["AUTHORS.txt", "VERSION.txt"]} version = {file = "robust_hermite_ft/VERSION.txt"} readme = {file = ["README.rst"]} dependencies = {file = "requirements/base.txt"} -optional-dependencies = {fast = {file = "requirements/fast.txt"}, dev = {file = "requirements/dev.txt"}, examples = {file = "requirements/examples.txt"}} +optional-dependencies = {fast = {file = "requirements/fast.txt"}, dev = {file = "requirements/dev.txt"}, examples = {file = "requirements/examples.txt"}, git_ci = {file = "requirements/git_ci.txt"}} [tool.isort] profile = "black" diff --git a/requirements/git_ci.txt b/requirements/git_ci.txt new file mode 100644 index 0000000..92bc99a --- /dev/null +++ b/requirements/git_ci.txt @@ -0,0 +1,9 @@ +black +coverage +cython>=3.0.10 +cython-lint>=0.16.0 +numba>=0.55.0 +pytest +pytest-cov +pytest-xdist +ruff \ No newline at end of file From 9f84d93153fc0c08456b3cd74058088a3f0e96c0 Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 19:28:54 +0200 Subject: [PATCH 2/8] FIX: - fixed wrong pip install and python versions in CI ? --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 54e4d20..79679c3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9, 3.10, 3.11, 3.12] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 @@ -25,7 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip .["git_ci"] + pip install .["git_ci"] - name: Build Cython code run: | From 5e201f0c60246d7f5537f234f3052d507545dd8f Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 19:33:59 +0200 Subject: [PATCH 3/8] FIX: - fixed loading of test files that do not contain test but dependencies that cannot be loaded ? --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..fddcd1d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +norecursedirs = tests/reference_files \ No newline at end of file From 408d8184c172aea95c249fe74a24774cca795cbe Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 19:41:03 +0200 Subject: [PATCH 4/8] BUG: - escaped imports of unaccessible dependencies for test-file generation in GitHub CI tests by making the `if __name__ == "__main__"` include these imports and function definitions --- .../generate_hermfunc_references.py | 271 +++++++++--------- 1 file changed, 141 insertions(+), 130 deletions(-) diff --git a/tests/reference_files/generate_hermfunc_references.py b/tests/reference_files/generate_hermfunc_references.py index f82f8b6..c75ec5d 100644 --- a/tests/reference_files/generate_hermfunc_references.py +++ b/tests/reference_files/generate_hermfunc_references.py @@ -14,18 +14,9 @@ import json import os from dataclasses import asdict, dataclass, field -from functools import partial -from multiprocessing import Pool -from time import perf_counter -from typing import Dict, Tuple +from typing import Dict, List, Tuple import numpy as np -from sympy import Symbol as sp_Symbol -from sympy import exp as sp_exp -from sympy import pi as sp_pi -from sympy import sqrt as sp_sqrt -from sympy import symbols as sp_symbols -from tqdm import tqdm # === Constants === @@ -61,7 +52,7 @@ class HermiteFunctionsParameters: n: int alpha: float - ns_for_single_function: list[int] = field(default_factory=list) + ns_for_single_function: List[int] = field(default_factory=list) @dataclass @@ -86,132 +77,152 @@ class ReferenceHermiteFunctionsMetadata: x_values: np.ndarray -# === Functions === +# === Main code === +if __name__ == "__main__": -def _eval_sym_hermite_worker( - row_index: int, - x: np.ndarray, - x_sym: sp_Symbol, - n: int, - alpha: float, - expressions: np.ndarray, - num_digits: int, -) -> Tuple[int, np.ndarray]: - """ - Worker function to evaluate the Hermite functions at the given points ``x``. - - """ - - # the Hermite functions are evaluated at the given points - hermite_function_values = np.empty(shape=n + 1, dtype=np.float64) - - # the Hermite functions are evaluated using the recurrence relation - for iter_j in range(0, n + 1): - # the expression for the Hermite function is evaluated - hermite_expression = expressions[iter_j] - hermite_function_values[iter_j] = hermite_expression.subs( - x_sym, x[row_index] / alpha - ).evalf(n=num_digits) - - return row_index, hermite_function_values - - -def _eval_sym_dilated_hermite_function_basis( - x: np.ndarray, - n: int, - alpha: float, - num_digits: int = 16, -) -> np.ndarray: - """ - Evaluates the first ``n + 1`` dilated Hermite functions at the given points ``x``. - They are defined as - - .. image:: docs/hermite_functions/equations/DilatedHermiteFunctions.png - - Parameters - ---------- - x : :class:`np.ndarray` of shape (m,) - The points at which the Hermite functions are evaluated. - n : :class:`int` - The order of the Hermite functions. - alpha : :class:`float` - The scaling factor of the independent variable ``x``. - num_digits : :class:`int`, default=16 - The number of digits used in the symbolic evaluation of the Hermite functions. - For orders ``n >= 50`` and high ``x / alpha``-values, the symbolic evaluation - might be inaccurate. In this case, going to quadruple precision - (``n_digits~=32``) or higher might be necessary. - - Returns - ------- - hermite_function_basis : :class:`np.ndarray` of shape (m, n + 1) - The values of the first ``n + 1`` dilated Hermite functions evaluated at the - points ``x``. - - """ - - # the Hermite functions are evaluated using their recurrence relation given by - # h_{n+1}(x) = sqrt(2 / (n + 1)) * x * h_{n}(x) - sqrt(n / (n + 1)) * h_{n-1}(x) - # with the initial conditions h_{-1}(x) = 0 and - # h_{0}(x) = pi**(-1/4) * exp(-x**2 / 2) - x_sym = sp_symbols("x") - hermite_expressions = np.empty(shape=(n + 1), dtype=object) - - # the first two Hermite function expressions are defined with the involved Gaussian - # function not multiplied in yet to avoid the build-up of large expressions - h_i_minus_1 = 0 - h_i = sp_exp(-(x_sym**2) / 2) / sp_sqrt(sp_sqrt(sp_pi)) # type: ignore - hermite_expressions[0] = h_i - - # the Hermite functions are evaluated using the recurrence relation - for iter_j in tqdm(range(0, n), desc="Generating Hermite expressions", leave=False): - h_i_plus_1 = ( - sp_sqrt(2 / (iter_j + 1)) * x_sym * h_i - - sp_sqrt(iter_j / (iter_j + 1)) * h_i_minus_1 # type: ignore - ) - h_i_minus_1, h_i = h_i, h_i_plus_1 - hermite_expressions[iter_j + 1] = h_i - - # the Hermite functions are evaluated at the given points - hermite_function_basis = np.empty(shape=(x.size, n + 1), dtype=np.float64) - - # the evaluation is done in parallel to speed up the process but a progress bar is - # used to keep track of the progress - with Pool() as pool: - worker = partial( - _eval_sym_hermite_worker, - x=x, - x_sym=x_sym, - n=n, - alpha=alpha, - expressions=hermite_expressions, - num_digits=num_digits, - ) - results = list( - tqdm( - pool.imap(worker, range(0, x.size)), - total=x.size, - desc="Evaluating Hermite functions", - leave=False, + # === Imports === + + from functools import partial + from multiprocessing import Pool + from time import perf_counter + + from sympy import Symbol as sp_Symbol + from sympy import exp as sp_exp + from sympy import pi as sp_pi + from sympy import sqrt as sp_sqrt + from sympy import symbols as sp_symbols + from tqdm import tqdm + + # === Functions === + + def _eval_sym_hermite_worker( + row_index: int, + x: np.ndarray, + x_sym: sp_Symbol, + n: int, + alpha: float, + expressions: np.ndarray, + num_digits: int, + ) -> Tuple[int, np.ndarray]: + """ + Worker function to evaluate the Hermite functions at the given points ``x``. + + """ + + # the Hermite functions are evaluated at the given points + hermite_function_values = np.empty(shape=n + 1, dtype=np.float64) + + # the Hermite functions are evaluated using the recurrence relation + for iter_j in range(0, n + 1): + # the expression for the Hermite function is evaluated + hermite_expression = expressions[iter_j] + hermite_function_values[iter_j] = hermite_expression.subs( + x_sym, x[row_index] / alpha + ).evalf(n=num_digits) + + return row_index, hermite_function_values + + def _eval_sym_dilated_hermite_function_basis( + x: np.ndarray, + n: int, + alpha: float, + num_digits: int = 16, + ) -> np.ndarray: + """ + Evaluates the first ``n + 1`` dilated Hermite functions at the given points + ``x``. + They are defined as + + .. image:: docs/hermite_functions/equations/DilatedHermiteFunctions.png + + Parameters + ---------- + x : :class:`np.ndarray` of shape (m,) + The points at which the Hermite functions are evaluated. + n : :class:`int` + The order of the Hermite functions. + alpha : :class:`float` + The scaling factor of the independent variable ``x``. + num_digits : :class:`int`, default=16 + The number of digits used in the symbolic evaluation of the Hermite + functions. + For orders ``n >= 50`` and high ``x / alpha``-values, the symbolic + evaluation might be inaccurate. In this case, going to quadruple precision + (``n_digits~=32``) or higher might be necessary. + + Returns + ------- + hermite_function_basis : :class:`np.ndarray` of shape (m, n + 1) + The values of the first ``n + 1`` dilated Hermite functions evaluated at the + points ``x``. + + """ + + # the Hermite functions are evaluated using their recurrence relation given by + # h_{n+1}(x) = sqrt(2 / (n + 1)) * x * h_{n}(x) - sqrt(n / (n + 1)) * h_{n-1}(x) + # with the initial conditions h_{-1}(x) = 0 and + # h_{0}(x) = pi**(-1/4) * exp(-x**2 / 2) + x_sym = sp_symbols("x") + hermite_expressions = np.empty(shape=(n + 1), dtype=object) + + # the first two Hermite function expressions are defined with the involved + # Gaussian function not multiplied in yet to avoid the build-up of large + # expressions + h_i_minus_1 = 0 + h_i = sp_exp(-(x_sym**2) / 2) / sp_sqrt(sp_sqrt(sp_pi)) # type: ignore + hermite_expressions[0] = h_i + + # the Hermite functions are evaluated using the recurrence relation + for iter_j in tqdm( + range(0, n), + desc="Generating Hermite expressions", + leave=False, + ): + h_i_plus_1 = ( + sp_sqrt(2 / (iter_j + 1)) * x_sym * h_i + - sp_sqrt(iter_j / (iter_j + 1)) * h_i_minus_1 # type: ignore + ) + h_i_minus_1, h_i = h_i, h_i_plus_1 + hermite_expressions[iter_j + 1] = h_i + + # the Hermite functions are evaluated at the given points + hermite_function_basis = np.empty(shape=(x.size, n + 1), dtype=np.float64) + + # the evaluation is done in parallel to speed up the process but a progress bar + # is used to keep track of the progress + with Pool() as pool: + worker = partial( + _eval_sym_hermite_worker, + x=x, + x_sym=x_sym, + n=n, + alpha=alpha, + expressions=hermite_expressions, + num_digits=num_digits, + ) + results = list( + tqdm( + pool.imap(worker, range(0, x.size)), + total=x.size, + desc="Evaluating Hermite functions", + leave=False, + ) ) - ) - - # the results are stored in the matrix - for row_idx, row_values in results: - hermite_function_basis[row_idx, ::] = row_values - - return hermite_function_basis / np.sqrt(alpha) + # the results are stored in the matrix + for row_idx, row_values in results: + hermite_function_basis[row_idx, ::] = row_values -# === Main === + return hermite_function_basis / np.sqrt(alpha) -# this part generates NumPy binary files for the first 250 dilated Hermite functions -# with different scaling factors evaluated at high precision for a series of 501 points -# in the range [-45, 45] -# NOTE: it is important that the number of points is odd to have a point at exactly 0 + # === Test file generation === -if __name__ == "__main__": + # this part generates NumPy binary files for the first 250 dilated Hermite functions + # with different scaling factors evaluated at high precision for a series of 501 + # points in the range [-45, 45] + # NOTE: it is important that the number of points is odd to have a point at + # exactly 0 # --- Setup --- From d8f158dfc571e4e12d8134af8aadbbbaa4c67d6c Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 19:47:28 +0200 Subject: [PATCH 5/8] TST: - test whether Cython build can be removed from GitHub actions for testing ? --- .github/workflows/python-package.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 79679c3..049bcb1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -27,10 +27,6 @@ jobs: python -m pip install --upgrade pip setuptools wheel pip install .["git_ci"] - - name: Build Cython code - run: | - python setup.py build_ext --inplace - - name: Run tests run: | pytest -n=auto -x From 21329285fedb32f144f83d8adae7c8182eedd75d Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 19:49:11 +0200 Subject: [PATCH 6/8] TST: - reverted removal of Cython for testing --- .github/workflows/python-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 049bcb1..79679c3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -27,6 +27,10 @@ jobs: python -m pip install --upgrade pip setuptools wheel pip install .["git_ci"] + - name: Build Cython code + run: | + python setup.py build_ext --inplace + - name: Run tests run: | pytest -n=auto -x From 08849e47c3c8f54ff2a38af62dfa5dc09466ace5 Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 20:26:13 +0200 Subject: [PATCH 7/8] PKG: - moved the full package back into an `src`-folder BLD: - updated `pyproject.toml` and `setup.py` to account for the movement to the `src`-folder - removed dedicated Cython build from CI test pipeline --- .github/workflows/python-package.yml | 6 +----- pyproject.toml | 2 +- setup.py | 8 +++++--- {robust_hermite_ft => src/robust_hermite_ft}/AUTHORS.txt | 0 {robust_hermite_ft => src/robust_hermite_ft}/VERSION.txt | 0 {robust_hermite_ft => src/robust_hermite_ft}/__init__.py | 0 .../robust_hermite_ft}/hermite_functions/.gitignore | 0 .../robust_hermite_ft}/hermite_functions/__init__.py | 0 .../robust_hermite_ft}/hermite_functions/_c_hermite.pyx | 0 .../robust_hermite_ft}/hermite_functions/_interface.py | 0 .../robust_hermite_ft}/hermite_functions/_numba_funcs.py | 0 .../robust_hermite_ft}/hermite_functions/_numpy_funcs.py | 0 12 files changed, 7 insertions(+), 9 deletions(-) rename {robust_hermite_ft => src/robust_hermite_ft}/AUTHORS.txt (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/VERSION.txt (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/__init__.py (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/hermite_functions/.gitignore (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/hermite_functions/__init__.py (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/hermite_functions/_c_hermite.pyx (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/hermite_functions/_interface.py (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/hermite_functions/_numba_funcs.py (100%) rename {robust_hermite_ft => src/robust_hermite_ft}/hermite_functions/_numpy_funcs.py (100%) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 79679c3..ebdf4e6 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -22,15 +22,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies and Build package run: | python -m pip install --upgrade pip setuptools wheel pip install .["git_ci"] - - name: Build Cython code - run: | - python setup.py build_ext --inplace - - name: Run tests run: | pytest -n=auto -x diff --git a/pyproject.toml b/pyproject.toml index 9a137c2..a0600cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ include-package-data = true package-data = {"*" = ["AUTHORS.txt", "VERSION.txt"]} [tool.setuptools.dynamic] -version = {file = "robust_hermite_ft/VERSION.txt"} +version = {file = "src/robust_hermite_ft/VERSION.txt"} readme = {file = ["README.rst"]} dependencies = {file = "requirements/base.txt"} optional-dependencies = {fast = {file = "requirements/fast.txt"}, dev = {file = "requirements/dev.txt"}, examples = {file = "requirements/examples.txt"}, git_ci = {file = "requirements/git_ci.txt"}} diff --git a/setup.py b/setup.py index 1887cdc..6d2a11f 100644 --- a/setup.py +++ b/setup.py @@ -10,12 +10,12 @@ import Cython.Compiler.Options import numpy as np from Cython.Build import cythonize -from setuptools import Extension, setup +from setuptools import Extension, find_packages, setup # === Constants === SOURCES = [ - "robust_hermite_ft/hermite_functions/_c_hermite.pyx", + "src/robust_hermite_ft/hermite_functions/_c_hermite.pyx", ] # === Setup === @@ -41,8 +41,10 @@ ] setup( + package_dir={"": "src"}, + packages=find_packages("src"), ext_modules=cythonize(CY_MODULES, nthreads=1, annotate=True), package_data={"robust_hermite_ft": ["*.pxd"]}, # include pxd files - include_package_data=False, # ignore other files + include_package_data=True, zip_safe=False, ) diff --git a/robust_hermite_ft/AUTHORS.txt b/src/robust_hermite_ft/AUTHORS.txt similarity index 100% rename from robust_hermite_ft/AUTHORS.txt rename to src/robust_hermite_ft/AUTHORS.txt diff --git a/robust_hermite_ft/VERSION.txt b/src/robust_hermite_ft/VERSION.txt similarity index 100% rename from robust_hermite_ft/VERSION.txt rename to src/robust_hermite_ft/VERSION.txt diff --git a/robust_hermite_ft/__init__.py b/src/robust_hermite_ft/__init__.py similarity index 100% rename from robust_hermite_ft/__init__.py rename to src/robust_hermite_ft/__init__.py diff --git a/robust_hermite_ft/hermite_functions/.gitignore b/src/robust_hermite_ft/hermite_functions/.gitignore similarity index 100% rename from robust_hermite_ft/hermite_functions/.gitignore rename to src/robust_hermite_ft/hermite_functions/.gitignore diff --git a/robust_hermite_ft/hermite_functions/__init__.py b/src/robust_hermite_ft/hermite_functions/__init__.py similarity index 100% rename from robust_hermite_ft/hermite_functions/__init__.py rename to src/robust_hermite_ft/hermite_functions/__init__.py diff --git a/robust_hermite_ft/hermite_functions/_c_hermite.pyx b/src/robust_hermite_ft/hermite_functions/_c_hermite.pyx similarity index 100% rename from robust_hermite_ft/hermite_functions/_c_hermite.pyx rename to src/robust_hermite_ft/hermite_functions/_c_hermite.pyx diff --git a/robust_hermite_ft/hermite_functions/_interface.py b/src/robust_hermite_ft/hermite_functions/_interface.py similarity index 100% rename from robust_hermite_ft/hermite_functions/_interface.py rename to src/robust_hermite_ft/hermite_functions/_interface.py diff --git a/robust_hermite_ft/hermite_functions/_numba_funcs.py b/src/robust_hermite_ft/hermite_functions/_numba_funcs.py similarity index 100% rename from robust_hermite_ft/hermite_functions/_numba_funcs.py rename to src/robust_hermite_ft/hermite_functions/_numba_funcs.py diff --git a/robust_hermite_ft/hermite_functions/_numpy_funcs.py b/src/robust_hermite_ft/hermite_functions/_numpy_funcs.py similarity index 100% rename from robust_hermite_ft/hermite_functions/_numpy_funcs.py rename to src/robust_hermite_ft/hermite_functions/_numpy_funcs.py From ff9d28599a288b255fb54c24c596a6f3297cdbd0 Mon Sep 17 00:00:00 2001 From: MothNik Date: Sat, 20 Jul 2024 20:29:28 +0200 Subject: [PATCH 8/8] BLD: - changed CI pipeline target branches --- .github/workflows/python-package.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ebdf4e6..9c1a64b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -3,10 +3,11 @@ name: Build and Test on: push: branches: - - develop # Change from main to develop + - main pull_request: branches: - - develop # Change from main to develop + - main + - develop jobs: build: