diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 7a1d178..7d5c7de 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -13,9 +13,9 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install dependencies diff --git a/.github/workflows/python-test-push.yml b/.github/workflows/python-test-push.yml index fddd100..eb4e863 100644 --- a/.github/workflows/python-test-push.yml +++ b/.github/workflows/python-test-push.yml @@ -8,26 +8,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9, 3.11, 3.12] # 3 + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install package with test dependencies run: | python -m pip install --upgrade pip - pip install pytest flake8 pytest-flake8-v2 mypy pytest-mypy pytest-cov \ - types-setuptools - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -e .[test] - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --exit-zero --show-source --statistics # --max-complexity=10 + run: flake8 src/pyrcn tests + - name: Type-check with mypy + run: mypy src/pyrcn - name: Test with pytest - run: | - pytest + run: pytest --cov=src/pyrcn --cov-branch --cov-report=xml --cov-report=term-missing - name: Test installed PyRCN run: | pip install . diff --git a/.readthedocs.yml b/.readthedocs.yml index cea00f3..3f38006 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,9 +7,9 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.9" + python: "3.12" # You can also specify other tool versions: # nodejs: "16" # rust: "1.55" diff --git a/README.md b/README.md index fbd322c..2e3d7d6 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ If you plan to contribute to ``PyRCN``, you can also install the package from so Clone the Git repository: ``` -git clone https://github.com/TUD-STKS/PyRCN.git +git clone https://github.com/PlasmaControl/PyRCN.git ``` Install the package using ``setup.py``: @@ -124,19 +124,19 @@ reg.fit(X=X[:8000], y=y[:8000]) y_pred = reg.predict(X[8000:]) # output is the prediction for each input example ``` -An extensive introduction to getting started with PyRCN is included in the [examples](https://github.com/TUD-STKS/PyRCN/blob/main/examples) directory. -The notebook [digits](https://github.com/TUD-STKS/PyRCN/blob/main/examples/digits.ipynb) or its corresponding [Python script](https://github.com/TUD-STKS/PyRCN/blob/main/examples/digits.py) show how to set up an ESN for a small hand-written digit recognition experiment. +An extensive introduction to getting started with PyRCN is included in the [examples](https://github.com/PlasmaControl/PyRCN/blob/main/examples) directory. +The notebook [digits](https://github.com/PlasmaControl/PyRCN/blob/main/examples/digits.ipynb) or its corresponding [Python script](https://github.com/PlasmaControl/PyRCN/blob/main/examples/digits.py) show how to set up an ESN for a small hand-written digit recognition experiment. Launch the digits notebook on Binder: -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TUD-STKS/PyRCN/main?filepath=examples%2Fdigits.ipynb) +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/PlasmaControl/PyRCN/main?filepath=examples%2Fdigits.ipynb) -The notebook [PyRCN_Intro](https://github.com/TUD-STKS/PyRCN/blob/main/examples/PyRCN_Intro.ipynb) or its corresponding [Python script](https://github.com/TUD-STKS/PyRCN/blob/main/examples/PyRCN_Intro.py) show how to construct different RCNs with building blocks. +The notebook [PyRCN_Intro](https://github.com/PlasmaControl/PyRCN/blob/main/examples/PyRCN_Intro.ipynb) or its corresponding [Python script](https://github.com/PlasmaControl/PyRCN/blob/main/examples/PyRCN_Intro.py) show how to construct different RCNs with building blocks. -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TUD-STKS/PyRCN/main?filepath=examples%2FPyRCN_Intro.ipynb) +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/PlasmaControl/PyRCN/main?filepath=examples%2FPyRCN_Intro.ipynb) -The notebook [Impulse responses](https://github.com/TUD-STKS/PyRCN/blob/main/examples/esn_impulse_responses.ipynb) is an interactive tool to demonstrate the impact of different hyper-parameters on the impulse responses of an ESN. +The notebook [Impulse responses](https://github.com/PlasmaControl/PyRCN/blob/main/examples/esn_impulse_responses.ipynb) is an interactive tool to demonstrate the impact of different hyper-parameters on the impulse responses of an ESN. -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TUD-STKS/PyRCN/main?filepath=examples%2Fesn_impulse_responses.ipynb) +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/PlasmaControl/PyRCN/main?filepath=examples%2Fesn_impulse_responses.ipynb) Fore more advanced examples, please have a look at our [Automatic Music Transcription Repository](https://github.com/TUD-STKS/Automatic-Music-Transcription), in which we provide an entire feature extraction, training and test pipeline for multipitch tracking and for note onset detection using PyRCN. This is currently transferred to this repository. diff --git a/docs/source/development.rst b/docs/source/development.rst index 9c8a67e..28cbd1b 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -29,5 +29,5 @@ Whenever you find something not explained well, please inform Peter Steiner `peter.steiner@pyrcn.net `_. -.. _GitHub: https://github.com/TUD-STKS/PyRCN -.. _issue tracker on GitHub: https://github.com/TUD-STKS/PyRCN/issues +.. _GitHub: https://github.com/PlasmaControl/PyRCN +.. _issue tracker on GitHub: https://github.com/PlasmaControl/PyRCN/issues diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 5301f88..d39d15b 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -14,7 +14,7 @@ command in a PowerShell or CommandLine in Windows, or in a shell in Linux/MacOS: python --version As any package, **PyRCN** has several dependencies as listed in the `requirements.txt - `_. To avoid any + `_. To avoid any unexpected interaction with the basic system as installed on your computer, we highly recommend using a virtual environment @@ -55,7 +55,7 @@ Installation from source We only recommend you the installation of PyRCN from source if you would like to contribute to PyRCN. Therefore, please find the source code of **PyRCN** on `GitHub -`_. +`_. You can download the latest stable version from the ``main`` branch. To work with older or unstable versions of **PyRCN**,. you can checkout the ``dev`` branch or any other @@ -68,4 +68,4 @@ The installation then work similar as before using `pip` in your command line: pip install -e /path/to/pyrcn -.. _issue tracker on GitHub: https://github.com/TUD-STKS/PyRCN/issues +.. _issue tracker on GitHub: https://github.com/PlasmaControl/PyRCN/issues diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index b7a5fa1..84afd14 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -14,7 +14,7 @@ similar, and PyRCN aims to unify the development of ESNs and ELMs. Many examples can be found in the PyRCN repository. Some useful examples can also be -found in the `PyRCN repository `_ +found in the `PyRCN repository `_ with many Jupyter notebooks. PyRCN is inspired by `ReservoirPy `_, another RC toolbox diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index da541db..ca68bc7 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -6,35 +6,35 @@ package. You can view them online or download Python scripts: -https://github.com/TUD-STKS/PyRCN/tree/main/examples +https://github.com/PlasmaControl/PyRCN/tree/main/examples The notebook -`PyRCN_Intro `_ +`PyRCN_Intro `_ or its corresponding -`Python script `_ +`Python script `_ show how to construct different RCNs with building blocks. .. image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/TUD-STKS/PyRCN/main?filepath=examples%2FPyRCN_Intro.ipynb + :target: https://mybinder.org/v2/gh/PlasmaControl/PyRCN/main?filepath=examples%2FPyRCN_Intro.ipynb The notebook `Impulse responses -`_ +`_ is an interactive tool to demonstrate the impact of different hyper-parameters on the impulse responses of an ESN. .. image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/TUD-STKS/PyRCN/main?filepath=examples%2Fesn_impulse_responses.ipynb + :target: https://mybinder.org/v2/gh/PlasmaControl/PyRCN/main?filepath=examples%2Fesn_impulse_responses.ipynb The Jupyter notebook -`digits `_ +`digits `_ or its corresponding -`Python script `_ +`Python script `_ show how to set up an ESN for a small hand-written digit recognition experiment. .. image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/TUD-STKS/PyRCN/main?filepath=examples%2Fdigits.ipynb + :target: https://mybinder.org/v2/gh/PlasmaControl/PyRCN/main?filepath=examples%2Fdigits.ipynb Fore more advanced examples, please have a look at our `Automatic Music Transcription Repository diff --git a/examples/PyRCN_Intro.ipynb b/examples/PyRCN_Intro.ipynb index ff56a72..e32f07d 100644 --- a/examples/PyRCN_Intro.ipynb +++ b/examples/PyRCN_Intro.ipynb @@ -722,7 +722,7 @@ "\n", "This complex use-case requires a serious hyper-parameter tuning. To keep the code example simple, we did not include the optimization in this paper and refer the interested readers to the Jupyter Notebook [^1] that was developed to produce these results.\n", "\n", - "[^1]: https://github.com/TUD-STKS/PyRCN/blob/main/examples/digits.ipynb" + "[^1]: https://github.com/PlasmaControl/PyRCN/blob/main/examples/digits.ipynb" ] }, { diff --git a/mypy.ini b/mypy.ini index 23ccd42..12fedc2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,6 @@ [mypy] # global options +# third-party scientific packages ship no type stubs / py.typed marker +ignore_missing_imports = True # add this to find packages without an __init__.py file namespace_packages = True strict_optional = True diff --git a/pyproject.toml b/pyproject.toml index 1066e63..3c584cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,39 +6,63 @@ build-backend = "setuptools.build_meta" [project] name = "PyRCN" -version = "0.0.18" +dynamic = ["version"] +description = "A scikit-learn-compatible framework for Reservoir Computing in Python" +readme = "README.md" +requires-python = ">=3.10" authors = [ { name="Peter Steiner", email="peter.steiner@pyrcn.net" }, ] -readme = "README.md" -requires-python = ">=3.8" classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Science/Research", "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", ] dependencies = [ - "torch", - "torchvision", - "torchaudio", - "scikit-learn", - "pandas", + "scikit-learn>=1.6", + "numpy>=1.18.1", + "scipy>=1.4.0", + "joblib>=0.13.2", + "pandas>=1.0.0", +] + +[project.optional-dependencies] +test = [ + "pytest>=6.5.0", + "pytest-cov", + "pytest-mypy", + "mypy", + "flake8", +] +examples = [ + "matplotlib", + "seaborn", + "ipywidgets", + "ipympl", + "tqdm", ] +[project.urls] +Homepage = "https://pyrcn.net/" +Documentation = "https://pyrcn.readthedocs.io/" +Source = "https://github.com/PlasmaControl/PyRCN/" +Issues = "https://github.com/PlasmaControl/PyRCN/issues" + +[tool.setuptools.dynamic] +version = { attr = "pyrcn._version.__version__" } + +[tool.setuptools.packages.find] +where = ["src"] + [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra -q" testpaths = [ "tests", ] - -[tool.setuptools.packages.find] -where = ["src"] - -[project.urls] -Homepage = "https://pyrcn.net/" -Documentation = "https://pyrcn.readthedocs.io/" -Source = "https://github.com/TUD-STKS/PyRCN/" -Issues = "https://github.com/TUD-STKS/PyRCN/issues" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 386f6d8..0000000 --- a/pytest.ini +++ /dev/null @@ -1,20 +0,0 @@ -# pytest.ini -[pytest] -minversion = 6.5.0 -addopts = - -ra -q -v - --doctest-modules - --junitxml=junit/test-results.xml - --cov=src/pyrcn - --cov-branch - --cov-report=xml - --cov-report=html - --cov-report=term-missing - # --pep257 - # --flake8 - --mypy - -testpaths = - src/pyrcn - docs - tests diff --git a/requirements.txt b/requirements.txt index 305f2e5..4ce35d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ setuptools>=46.4.0 -scikit-learn>=1.4 +scikit-learn>=1.6 numpy>=1.18.1 scipy>=1.4.0 joblib>=0.13.2 pandas>=1.0.0 -typing-extensions -requests[matplotlib] -requests[seaborn] -requests[ipywidgets] -requests[ipympl] -requests[tqdm] +matplotlib +seaborn +ipywidgets +ipympl +tqdm diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 198986e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[metadata] -name = PyRCN -version = 0.0.18 -author = Peter Steiner -author_email = peter.steiner@pyrcn.net -description = A scikit-learn-compatible framework for Reservoir Computing in Python -long_description = file: README.md -long_description_content_type = text/markdown -url = https://github.com/TUD-STKS/PyRCN -project_urls = - Documentation = https://pyrcn.readthedocs.io/ - Funding = https://pyrcn.net/ - Source = https://github.com/TUD-STKS/PyRCN/ - Bug Tracker = https://github.com/TUD-STKS/PyRCN/issues -classifiers = - Programming Language :: Python :: 3 - Development Status :: 2 - Pre-Alpha - License :: OSI Approved :: BSD License - Operating System :: OS Independent - Intended Audience :: Science/Research - -[options] -package_dir = - = src -packages = find: -python_requires = >=3.9 - -[options.packages.find] -where = src - -[tool:pytest] -testpaths = tests diff --git a/src/pyrcn/__init__.py b/src/pyrcn/__init__.py index 22779e7..9cb1a6c 100644 --- a/src/pyrcn/__init__.py +++ b/src/pyrcn/__init__.py @@ -2,12 +2,12 @@ # Authors: Peter Steiner , # License: BSD 3 clause -from ._version import __version__ +from __future__ import annotations from . import (base, echo_state_network, extreme_learning_machine, - linear_model, model_selection, nn, postprocessing, - preprocessing, projection, util) - + linear_model, model_selection, postprocessing, preprocessing, + projection, util) +from ._version import __version__ __all__ = ('__version__', 'base', @@ -15,7 +15,6 @@ 'extreme_learning_machine', 'linear_model', 'model_selection', - 'nn', 'postprocessing', 'preprocessing', 'projection', diff --git a/src/pyrcn/_version.py b/src/pyrcn/_version.py index 5e9a3c7..34284b2 100644 --- a/src/pyrcn/_version.py +++ b/src/pyrcn/_version.py @@ -1,2 +1,4 @@ """Version indicator for PyRCN.""" -__version__ = "0.0.17post1" +from __future__ import annotations + +__version__ = "0.0.18" diff --git a/src/pyrcn/base/__init__.py b/src/pyrcn/base/__init__.py index ffed84b..c9a69bf 100644 --- a/src/pyrcn/base/__init__.py +++ b/src/pyrcn/base/__init__.py @@ -10,16 +10,17 @@ of Reservoir Computing Networks’, under review. """ +from __future__ import annotations + # Authors: Peter Steiner , # License: BSD 3 clause - from ._activations import (ACTIVATIONS, ACTIVATIONS_INVERSE, ACTIVATIONS_INVERSE_BOUNDS) -from ._base import ( - _uniform_random_input_weights, _uniform_random_weights, - _normal_random_weights, _make_sparse, _antisymmetric_weights, - _unitary_spectral_radius, _uniform_random_bias, - _normal_random_recurrent_weights, _uniform_random_recurrent_weights) +from ._base import (_antisymmetric_weights, _make_sparse, + _normal_random_recurrent_weights, _normal_random_weights, + _uniform_random_bias, _uniform_random_input_weights, + _uniform_random_recurrent_weights, _uniform_random_weights, + _unitary_spectral_radius) __all__ = ( 'ACTIVATIONS', 'ACTIVATIONS_INVERSE', 'ACTIVATIONS_INVERSE_BOUNDS', diff --git a/src/pyrcn/base/_activations.py b/src/pyrcn/base/_activations.py index ce07766..e45f134 100644 --- a/src/pyrcn/base/_activations.py +++ b/src/pyrcn/base/_activations.py @@ -1,11 +1,64 @@ -"""The :mod:`activations` contains various activation functions for PyRCN.""" +"""The :mod:`activations` contains various activation functions for PyRCN.""" # Authors: Peter Steiner # License: BSD 3 clause +from __future__ import annotations + +from collections.abc import Callable + import numpy as np -from sklearn.neural_network._base import ACTIVATIONS -from typing import Dict, Callable +from scipy.special import expit as logistic_sigmoid + + +def inplace_identity(X: np.ndarray) -> None: + """ + Compute the identity function inplace. + + This is a no-op, kept for API consistency with the other activations. + + Parameters + ---------- + X : ndarray + The input data. + """ + # Nothing to do + + +def inplace_tanh(X: np.ndarray) -> None: + """ + Compute the hyperbolic tangent function inplace. + + Parameters + ---------- + X : ndarray + The input data. + """ + np.tanh(X, out=X) + + +def inplace_logistic(X: np.ndarray) -> None: + """ + Compute the logistic sigmoid function inplace. + + Parameters + ---------- + X : ndarray + The input data. + """ + logistic_sigmoid(X, out=X) + + +def inplace_relu(X: np.ndarray) -> None: + """ + Compute the rectified linear unit function inplace. + + Parameters + ---------- + X : ndarray + The input data. + """ + np.maximum(X, 0, out=X) def inplace_softplus(X: np.ndarray) -> None: @@ -74,7 +127,7 @@ def inplace_identity_inverse(X: np.ndarray) -> None: X : ndarray The input data. """ - ACTIVATIONS['identity'](X) + inplace_identity(X) def inplace_logistic_inverse(X: np.ndarray) -> None: @@ -102,7 +155,7 @@ def inplace_relu_inverse(X: np.ndarray) -> None: X : ndarray The input data. """ - ACTIVATIONS['relu'](X) + inplace_relu(X) def inplace_bounded_relu_inverse(X: np.ndarray) -> None: @@ -119,14 +172,20 @@ def inplace_bounded_relu_inverse(X: np.ndarray) -> None: X : ndarray The input data. """ - ACTIVATIONS['bounded_relu'](X) + inplace_bounded_relu(X) -ACTIVATIONS.update({'bounded_relu': inplace_bounded_relu, - 'softmax': inplace_softmax, - 'softplus': inplace_softplus}) +ACTIVATIONS: dict[str, Callable] = { + 'identity': inplace_identity, + 'tanh': inplace_tanh, + 'logistic': inplace_logistic, + 'relu': inplace_relu, + 'bounded_relu': inplace_bounded_relu, + 'softmax': inplace_softmax, + 'softplus': inplace_softplus, +} -ACTIVATIONS_INVERSE: Dict[str, Callable] = { +ACTIVATIONS_INVERSE: dict[str, Callable] = { 'tanh': inplace_tanh_inverse, 'identity': inplace_identity_inverse, 'logistic': inplace_logistic_inverse, @@ -134,7 +193,7 @@ def inplace_bounded_relu_inverse(X: np.ndarray) -> None: 'bounded_relu': inplace_bounded_relu_inverse } -ACTIVATIONS_INVERSE_BOUNDS: Dict[str, tuple] = { +ACTIVATIONS_INVERSE_BOUNDS: dict[str, tuple] = { 'tanh': (-.99, .99), 'identity': (-np.inf, np.inf), 'logistic': (0.01, .99), diff --git a/src/pyrcn/base/_base.py b/src/pyrcn/base/_base.py index cc30388..f279728 100644 --- a/src/pyrcn/base/_base.py +++ b/src/pyrcn/base/_base.py @@ -3,22 +3,17 @@ # Authors: Peter Steiner # License: BSD 3 clause -import sys +from __future__ import annotations import numpy as np import scipy -from scipy.sparse.linalg import eigs as eigens from scipy.sparse.linalg import ArpackNoConvergence - -if sys.version_info >= (3, 8): - from typing import Union -else: - from typing import Union +from scipy.sparse.linalg import eigs as eigens def _antisymmetric_weights( - weights: Union[np.ndarray, scipy.sparse.csr_matrix]) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + weights: np.ndarray | scipy.sparse.csr_matrix) \ + -> np.ndarray | scipy.sparse.csr_matrix: """ Transform a given weight matrix to get antisymmetric, e.g., compute weights - weights.T @@ -39,9 +34,9 @@ def _antisymmetric_weights( def _unitary_spectral_radius( - weights: Union[np.ndarray, scipy.sparse.csr_matrix], + weights: np.ndarray | scipy.sparse.csr_matrix, random_state: np.random.RandomState) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + -> np.ndarray | scipy.sparse.csr_matrix: """ Normalize a given weight matrix to the unitary spectral radius, e.g., the maximum absolute eigenvalue. @@ -94,7 +89,7 @@ def _make_sparse(k_in: int, dense_weights: np.ndarray, for neuron in range(n_outputs): all_indices = np.arange(n_inputs) - keep_indices = np.random.choice(n_inputs, k_in, replace=False) + keep_indices = random_state.choice(n_inputs, k_in, replace=False) zero_indices = np.setdiff1d(all_indices, keep_indices) dense_weights[zero_indices, neuron] = 0 @@ -104,7 +99,7 @@ def _make_sparse(k_in: int, dense_weights: np.ndarray, def _normal_random_weights( n_inputs: int, n_outputs: int, k_in: int, random_state: np.random.RandomState) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + -> np.ndarray | scipy.sparse.csr_matrix: """ Sparse or dense normal random weights. @@ -136,7 +131,7 @@ def _normal_random_weights( def _uniform_random_weights( n_inputs: int, n_outputs: int, k_in: int, random_state: np.random.RandomState) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + -> np.ndarray | scipy.sparse.csr_matrix: """ Sparse or dense uniform random weights in range [-1, 1]. @@ -168,7 +163,7 @@ def _uniform_random_weights( def _uniform_random_input_weights( n_features_in: int, hidden_layer_size: int, fan_in: int, random_state: np.random.RandomState) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + -> np.ndarray | scipy.sparse.csr_matrix: """ Return uniform random input weights in range [-1, 1]. @@ -216,7 +211,7 @@ def _uniform_random_bias( def _uniform_random_recurrent_weights( hidden_layer_size: int, fan_in: int, random_state: np.random.RandomState) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + -> np.ndarray | scipy.sparse.csr_matrix: """ Return uniformly distributed random reservoir weights. @@ -241,7 +236,7 @@ def _uniform_random_recurrent_weights( def _normal_random_recurrent_weights( hidden_layer_size: int, fan_in: int, random_state: np.random.RandomState) \ - -> Union[np.ndarray, scipy.sparse.csr_matrix]: + -> np.ndarray | scipy.sparse.csr_matrix: """ Return normally distributed random reservoir weights. diff --git a/src/pyrcn/base/blocks/__init__.py b/src/pyrcn/base/blocks/__init__.py index b0cf1e5..8c71053 100644 --- a/src/pyrcn/base/blocks/__init__.py +++ b/src/pyrcn/base/blocks/__init__.py @@ -3,12 +3,12 @@ # Authors: Peter Steiner # License: BSD 3 clause -from ._input_to_node import ( - InputToNode, PredefinedWeightsInputToNode, BatchIntrinsicPlasticity) -from ._node_to_node import ( - NodeToNode, EulerNodeToNode, PredefinedWeightsNodeToNode, - HebbianNodeToNode) +from __future__ import annotations +from ._input_to_node import (BatchIntrinsicPlasticity, InputToNode, + PredefinedWeightsInputToNode) +from ._node_to_node import (EulerNodeToNode, HebbianNodeToNode, NodeToNode, + PredefinedWeightsNodeToNode) __all__ = ( 'InputToNode', 'PredefinedWeightsInputToNode', 'BatchIntrinsicPlasticity', diff --git a/src/pyrcn/base/blocks/_input_to_node.py b/src/pyrcn/base/blocks/_input_to_node.py index b01b5ae..e358111 100644 --- a/src/pyrcn/base/blocks/_input_to_node.py +++ b/src/pyrcn/base/blocks/_input_to_node.py @@ -6,24 +6,23 @@ from __future__ import annotations import sys -from scipy.sparse import csr_matrix -from scipy.sparse import issparse +from typing import Literal + import numpy as np -from sklearn.utils.validation import _deprecate_positional_args +from scipy.sparse import csr_matrix, issparse from sklearn.base import BaseEstimator, TransformerMixin -from sklearn.utils import check_random_state, deprecated -from sklearn.utils.extmath import safe_sparse_dot from sklearn.exceptions import NotFittedError from sklearn.preprocessing import StandardScaler +from sklearn.utils import check_random_state, deprecated +from sklearn.utils.extmath import safe_sparse_dot +from sklearn.utils.validation import validate_data from ...base import (ACTIVATIONS, ACTIVATIONS_INVERSE, ACTIVATIONS_INVERSE_BOUNDS, _uniform_random_bias, _uniform_random_input_weights) -from typing import Union, Literal, Optional - -class InputToNode(BaseEstimator, TransformerMixin): +class InputToNode(TransformerMixin, BaseEstimator): """ InputToNode class for reservoir computing modules. @@ -67,7 +66,6 @@ class InputToNode(BaseEstimator, TransformerMixin): A set of predefined bias weights. """ - @_deprecate_positional_args def __init__(self, *, hidden_layer_size: int = 500, sparsity: float = 1., @@ -75,10 +73,10 @@ def __init__(self, *, 'relu', 'bounded_relu'] = 'tanh', input_scaling: float = 1., input_shift: float = 0., bias_scaling: float = 1., bias_shift: float = 0., - k_in: Union[int, None] = None, - random_state: Union[int, np.random.RandomState, None] = 42, - predefined_input_weights: Optional[np.ndarray] = None, - predefined_bias_weights: Optional[np.ndarray] = None) -> None: + k_in: int | None = None, + random_state: int | np.random.RandomState | None = 42, + predefined_input_weights: np.ndarray | None = None, + predefined_bias_weights: np.ndarray | None = None) -> None: """Construct the InputToNode.""" self.hidden_layer_size = hidden_layer_size self.sparsity = sparsity @@ -112,8 +110,7 @@ def fit(self, X: np.ndarray, y: None = None) -> InputToNode: self : returns a fitted InputToNode. """ self._validate_hyperparameters() - self._validate_data(X, y) - self._check_n_features(X, reset=True) + validate_data(self, X) if self.k_in is not None: self.sparsity = float(self.k_in) / float(X.shape[1]) fan_in = int(np.rint(self.hidden_layer_size * self.sparsity)) @@ -162,7 +159,7 @@ def transform(self, X: np.ndarray, y: None = None) -> np.ndarray: @staticmethod def _node_inputs(X: np.ndarray, - input_weights: Union[np.ndarray, csr_matrix], + input_weights: np.ndarray | csr_matrix, input_scaling: float, input_shift: float, bias: np.ndarray, bias_scaling: float, bias_shift: float)\ -> np.ndarray: @@ -202,25 +199,25 @@ def _validate_hyperparameters(self) -> None: self._random_state = check_random_state(self.random_state) if self.hidden_layer_size <= 0: - raise ValueError("hidden_layer_size must be > 0, got {0}." + raise ValueError("hidden_layer_size must be > 0, got {}." .format(self.hidden_layer_size)) if self.sparsity <= 0. or self.sparsity > 1.: - raise ValueError("sparsity must be between 0. and 1., got {0}." + raise ValueError("sparsity must be between 0. and 1., got {}." .format(self.sparsity)) if self.input_activation not in ACTIVATIONS: - raise ValueError("The activation_function '{0}' is not supported." - "Supported activations are {1}." + raise ValueError("The activation_function '{}' is not supported." + "Supported activations are {}." .format(self.input_activation, ACTIVATIONS)) if self.input_scaling <= 0.: - raise ValueError("input_scaling must be > 0, got {0}." + raise ValueError("input_scaling must be > 0, got {}." .format(self.input_scaling)) if self.bias_scaling < 0: - raise ValueError("bias must be > 0, got {0}." + raise ValueError("bias must be > 0, got {}." .format(self.bias_scaling)) if self.k_in is not None and (self.k_in <= 0 or self.k_in >= self.hidden_layer_size): raise ValueError("k_in must be > 0 and < self.hidden_layer_size" - " {0}, got {1}." + " {}, got {}." .format(self.hidden_layer_size, self.k_in)) def __sizeof__(self) -> int: @@ -241,7 +238,7 @@ def __sizeof__(self) -> int: self._input_weights.nbytes + sys.getsizeof(self._random_state) @property - def input_weights(self) -> Union[np.ndarray, csr_matrix]: + def input_weights(self) -> np.ndarray | csr_matrix: """ Return the input weights. @@ -297,20 +294,19 @@ class PredefinedWeightsInputToNode(InputToNode): random_state : Union[int, np.random.RandomState, None], default = 42 """ - @_deprecate_positional_args def __init__(self, predefined_input_weights: np.ndarray, *, input_activation: Literal['tanh', 'identity', 'logistic', 'relu', 'bounded_relu'] = 'tanh', input_scaling: float = 1., input_shift: float = 0., - predefined_bias_weights: Optional[np.ndarray] = None, + predefined_bias_weights: np.ndarray | None = None, bias_scaling: float = 0., bias_shift: float = 0., - random_state: Union[int, np.random.RandomState, None] = 42)\ + random_state: int | np.random.RandomState | None = 42)\ -> None: """Construct the PredefinedWeightsInputToNode.""" if predefined_input_weights.ndim != 2: raise ValueError('predefined_input_weights has not the expected' - 'ndim 2, given {0}.' + 'ndim 2, given {}.' .format(predefined_input_weights.shape)) super().__init__(hidden_layer_size=predefined_input_weights.shape[1], input_activation=input_activation, @@ -338,35 +334,6 @@ def fit(self, X: np.ndarray, y: None = None) -> InputToNode: return self -""" -class NonlinearVectorAutoregression(InputToNode): - # - # Non-linear vector autoregression (NVAR) class - - # - - @_deprecate_positional_args - def __init__(self, *, - delay: int = 2, order: int = 2, stride: int = 1) -> None: - super().__init__() - self.delay = delay - self.order = order - self.stride = stride - self._linear_dimension = 0 - self._non_linear_dimension = 0 - - def fit(self, X: np.ndarray, y: None = None) -> InputToNode: - self._validate_hyperparameters() - self._validate_data(X, y) - n_samples, n_features = X.shape - self._linear_dimension = self.delay * n_features - self._non_linear_dimension = comb( - self._linear_dimension + self.order - 1, self.order) - self.hidden_layer_size = self._linear_dimension + \ - self._non_linear_dimension -""" - - class BatchIntrinsicPlasticity(InputToNode): """ BatchIntrinsicPlasticity class for reservoir computing modules. @@ -399,7 +366,6 @@ class BatchIntrinsicPlasticity(InputToNode): random_state : Union[int, np.random.RandomState, None], default = 42 """ - @_deprecate_positional_args def __init__(self, *, distribution: Literal['exponential', 'uniform', 'normal'] = 'normal', @@ -407,7 +373,7 @@ def __init__(self, *, input_activation: Literal['tanh', 'identity', 'logistic', 'relu', 'bounded_relu'] = 'tanh', hidden_layer_size: int = 500, sparsity: float = 1., - random_state: Union[int, np.random.RandomState, None] = 42): + random_state: int | np.random.RandomState | None = 42): """Construct the BatchIntrinsicPlasticity InputToNode.""" super().__init__(input_activation=input_activation, hidden_layer_size=hidden_layer_size, @@ -415,8 +381,8 @@ def __init__(self, *, self.distribution = distribution self.algorithm = algorithm self._scaler = StandardScaler() - self._m = 1 - self._c = 0 + self._m = 1. + self._c = 0. IN_DISTRIBUTION_PARAMS = {'exponential': (-.5, -.5), 'uniform': (.7, .0), @@ -528,7 +494,7 @@ def _fit_neumann(self, X: np.ndarray, y: None = None) -> InputToNode: t.sort() ACTIVATIONS_INVERSE[self.input_activation](t) else: - raise ValueError('Not a valid activation inverse, got {0}' + raise ValueError('Not a valid activation inverse, got {}' .format(self.distribution)) v = safe_sparse_dot(np.linalg.pinv(phi), t) @@ -551,7 +517,7 @@ def _fit_dresden(self, X: np.ndarray, y: None = None) -> InputToNode: """ if self.input_activation != 'tanh': raise ValueError('This algorithm is working with tanh-activation' - 'only, got {0}'.format(self.input_activation)) + 'only, got {}'.format(self.input_activation)) super().fit(X, y=y) s = BatchIntrinsicPlasticity._node_inputs( @@ -568,8 +534,8 @@ def _validate_hyperparameters(self) -> None: super()._validate_hyperparameters() if self.algorithm not in {'neumann', 'dresden'}: - raise ValueError('The selected algorithm is unknown, got {0}' + raise ValueError('The selected algorithm is unknown, got {}' .format(self.algorithm)) if self.distribution not in {'exponential', 'uniform', 'normal'}: - raise ValueError('The selected distribution is unknown, got {0}' + raise ValueError('The selected distribution is unknown, got {}' .format(self.distribution)) diff --git a/src/pyrcn/base/blocks/_node_to_node.py b/src/pyrcn/base/blocks/_node_to_node.py index f1fc96d..70c3938 100644 --- a/src/pyrcn/base/blocks/_node_to_node.py +++ b/src/pyrcn/base/blocks/_node_to_node.py @@ -6,24 +6,21 @@ from __future__ import annotations import sys - -from scipy.sparse.csr import csr_matrix -from scipy.sparse import issparse +from typing import Literal import numpy as np -from sklearn.utils.validation import _deprecate_positional_args +from scipy.sparse import csr_matrix, issparse from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.exceptions import NotFittedError from sklearn.utils import check_random_state, deprecated from sklearn.utils.extmath import safe_sparse_dot -from sklearn.exceptions import NotFittedError +from sklearn.utils.validation import validate_data from ...base import (ACTIVATIONS, _normal_random_recurrent_weights, _uniform_random_recurrent_weights) -from typing import Union, Literal, Optional - -class NodeToNode(BaseEstimator, TransformerMixin): +class NodeToNode(TransformerMixin, BaseEstimator): """ NodeToNode class for reservoir computing modules. @@ -63,7 +60,6 @@ class NodeToNode(BaseEstimator, TransformerMixin): A set of predefined recurrent weights. """ - @_deprecate_positional_args def __init__(self, *, hidden_layer_size: int = 500, sparsity: float = 1., reservoir_activation: Literal['tanh', 'identity', @@ -71,9 +67,9 @@ def __init__(self, *, 'bounded_relu'] = 'tanh', spectral_radius: float = 1., leakage: float = 1., bidirectional: bool = False, - k_rec: Union[int, np.integer, None] = None, - random_state: Union[int, np.random.RandomState, None] = 42, - predefined_recurrent_weights: Optional[np.ndarray] = None + k_rec: int | np.integer | None = None, + random_state: int | np.random.RandomState | None = 42, + predefined_recurrent_weights: np.ndarray | None = None ) -> None: """Construct the NodeToNode.""" self.hidden_layer_size = hidden_layer_size @@ -105,8 +101,7 @@ def fit(self, X: np.ndarray, y: None = None) -> NodeToNode: self : returns a trained NodeToNode. """ self._validate_hyperparameters() - self._validate_data(X, y) - self._check_n_features(X, reset=True) + validate_data(self, X) if self.k_rec is not None: self.sparsity = float(self.k_rec) / float(X.shape[1]) @@ -182,27 +177,28 @@ def _validate_hyperparameters(self) -> None: self._random_state = check_random_state(self.random_state) if self.hidden_layer_size <= 0: - raise ValueError("hidden_layer_size must be > 0, got {0}%s." - .format(self.hidden_layer_size)) + raise ValueError( + "hidden_layer_size must be > 0, " + f"got {self.hidden_layer_size}") if self.sparsity <= 0. or self.sparsity > 1.: - raise ValueError("sparsity must be between 0. and 1., got {0}." + raise ValueError("sparsity must be between 0. and 1., got {}." .format(self.sparsity)) if self.reservoir_activation not in ACTIVATIONS: - raise ValueError("The activation_function {0} is not supported. " - "Supported activations are {1}." + raise ValueError("The activation_function {} is not supported. " + "Supported activations are {}." .format(self.reservoir_activation, ACTIVATIONS)) if self.spectral_radius < 0.: - raise ValueError("spectral_radius must be >= 0, got {0}." + raise ValueError("spectral_radius must be >= 0, got {}." .format(self.spectral_radius)) if self.leakage <= 0. or self.leakage > 1.: - raise ValueError("leakage must be between 0. and 1., got {0}." + raise ValueError("leakage must be between 0. and 1., got {}." .format(self.leakage)) if self.bidirectional not in [False, True]: raise ValueError("bidirectional must be either False or True," - "got {0}.".format(self.bidirectional)) + "got {}.".format(self.bidirectional)) if self.k_rec is not None and ( self.k_rec <= 0 or self.k_rec >= self.hidden_layer_size): - raise ValueError("k_rec must be > 0, got {0}.".format(self.k_rec)) + raise ValueError(f"k_rec must be > 0, got {self.k_rec}.") def __sizeof__(self) -> int: """ @@ -222,13 +218,13 @@ def __sizeof__(self) -> int: sys.getsizeof(self.random_state) @property - def recurrent_weights(self) -> Union[np.ndarray, csr_matrix]: + def recurrent_weights(self) -> np.ndarray | csr_matrix: """ Return the recurrent weights. Returns ------- - recurrent_weights : Union[np.ndarray, scipy.sparse.csr.csr_matrix] + recurrent_weights : Union[np.ndarray, scipy.sparse.csr_matrix] of size (hidden_layer_size, hidden_layer_size) """ return self._recurrent_weights @@ -271,7 +267,6 @@ class EulerNodeToNode(NodeToNode): random_state : Union[int, np.random.RandomState, None], default = 42 """ - @_deprecate_positional_args def __init__(self, *, hidden_layer_size: int = 500, sparsity: float = 1., @@ -281,9 +276,9 @@ def __init__(self, *, recurrent_scaling: float = 1., gamma: float = 0.001, epsilon: float = 0.01, - k_rec: Union[int, np.integer, None] = None, - random_state: Union[int, np.random.RandomState, - None] = 42) -> None: + k_rec: int | np.integer | None = None, + random_state: (int | np.random.RandomState | + None) = 42) -> None: """Construct the EulerNodeToNode.""" super().__init__(hidden_layer_size=hidden_layer_size, sparsity=sparsity, @@ -308,8 +303,7 @@ def fit(self, X: np.ndarray, y: None = None) -> NodeToNode: self : returns a trained EulerNodeToNode. """ self._validate_hyperparameters() - self._validate_data(X, y) - self._check_n_features(X, reset=True) + validate_data(self, X) if self.k_rec is not None: self.sparsity = float(self.k_rec) / float(X.shape[1]) @@ -398,7 +392,6 @@ class PredefinedWeightsNodeToNode(NodeToNode): Whether to work bidirectional. """ - @_deprecate_positional_args def __init__(self, predefined_recurrent_weights: np.ndarray, *, reservoir_activation: Literal['tanh', 'identity', @@ -410,7 +403,7 @@ def __init__(self, """Construct the PredefinedWeightsNodeToNode.""" if predefined_recurrent_weights.ndim != 2: raise ValueError('predefined_recurrent_weights has not the ' - 'expected ndim {0}, given 2.' + 'expected ndim {}, given 2.' .format(predefined_recurrent_weights.shape)) super().__init__( hidden_layer_size=predefined_recurrent_weights.shape[0], @@ -483,7 +476,6 @@ class HebbianNodeToNode(NodeToNode): Method used to fit the recurrent weights. """ - @_deprecate_positional_args def __init__(self, *, hidden_layer_size: int = 500, sparsity: float = 1., @@ -493,10 +485,10 @@ def __init__(self, *, spectral_radius: float = 1., leakage: float = 1., bidirectional: bool = False, - k_rec: Union[int, np.integer, None] = None, - random_state: Union[int, np.random.RandomState, None] = 42, + k_rec: int | np.integer | None = None, + random_state: int | np.random.RandomState | None = 42, learning_rate: float = 0.01, - epochs: Union[int, np.integer] = 100, + epochs: int | np.integer = 100, training_method: Literal['hebbian', 'anti_hebbian', 'oja', 'anti_oja'] = 'hebbian'): """Construct the HebbianNodeToNode.""" diff --git a/src/pyrcn/datasets/__init__.py b/src/pyrcn/datasets/__init__.py index 0b6955a..94f5238 100644 --- a/src/pyrcn/datasets/__init__.py +++ b/src/pyrcn/datasets/__init__.py @@ -3,7 +3,8 @@ # Authors: Peter Steiner # License: BSD 3 clause -from ._base import mackey_glass, lorenz, load_digits +from __future__ import annotations +from ._base import load_digits, lorenz, mackey_glass __all__ = 'mackey_glass', 'lorenz', 'load_digits' diff --git a/src/pyrcn/datasets/_base.py b/src/pyrcn/datasets/_base.py index 08bba4f..a37b7e9 100644 --- a/src/pyrcn/datasets/_base.py +++ b/src/pyrcn/datasets/_base.py @@ -3,11 +3,14 @@ # Authors: Peter Steiner # License: BSD 3 clause -from typing import Union, Tuple, Callable, Any, List, Dict +from __future__ import annotations + +import collections +from collections.abc import Callable +from typing import Any + import numpy as np from scipy.integrate import solve_ivp -import collections -from sklearn.utils.validation import _deprecate_positional_args from sklearn.datasets import load_digits as sklearn_load_digits from sklearn.utils import Bunch @@ -51,12 +54,11 @@ def _runge_kutta(equation: Callable, x_t: float, h: float = 1., return x_t + h*(k_1 + 2*k_2 + 2*k_3 + k_4) / 6 -@_deprecate_positional_args def mackey_glass(n_timesteps: int, n_future: int = 1, tau: int = 17, beta: float = 0.2, gamma: float = 0.1, n: int = 10, x_0: float = 1.2, h: float = 1.0, - random_state: Union[int, np.random.RandomState, None] = 42) \ - -> Tuple[np.ndarray, np.ndarray]: + random_state: int | np.random.RandomState | None = 42) \ + -> tuple[np.ndarray, np.ndarray]: r""" Mackey-Glass time-series. @@ -137,11 +139,10 @@ def mackey_glass(n_timesteps: int, n_future: int = 1, tau: int = 17, return x[:-n_future], x[n_future:] -@_deprecate_positional_args def lorenz(n_timesteps: int, n_future: int = 1, sigma: float = 10., rho: float = 28., beta: float = 8./3., - x_0: Union[List, np.ndarray] = [1.0, 1.0, 1.0], h: float = 0.03, - **kwargs: Dict) -> Tuple[np.ndarray, np.ndarray]: + x_0: list | np.ndarray | None = None, h: float = 0.03, + **kwargs: dict) -> tuple[np.ndarray, np.ndarray]: r""" Lorenz time-series. @@ -191,11 +192,14 @@ def lorenz(n_timesteps: int, n_future: int = 1, sigma: float = 10., Conference on Artificial Neural Networks (pp. 494-505). Springer, Cham. """ + if x_0 is None: + x_0 = [1.0, 1.0, 1.0] + timesteps = np.arange(0., (n_timesteps + n_future) * h, h) def lorenz_differential_equation(t: int, - state: Tuple[float, float, float]) \ - -> Tuple[float, float, float]: + state: tuple[float, float, float]) \ + -> tuple[float, float, float]: x, y, z = state dx_dt = sigma * (y - x) dy_dt = x * (rho - z) @@ -209,10 +213,9 @@ def lorenz_differential_equation(t: int, lorenz_solution.y.T[:-n_future, :], lorenz_solution.y.T[n_future:, :]) -@_deprecate_positional_args -def load_digits(*, n_class: Union[int, np.integer] = 10, +def load_digits(*, n_class: int | np.integer = 10, return_X_y: bool = False, as_frame: bool = False, - as_sequence: bool = False) -> Union[Bunch, tuple]: + as_sequence: bool = False) -> Bunch | tuple: """ Load and return the digits dataset (classification). diff --git a/src/pyrcn/echo_state_network/__init__.py b/src/pyrcn/echo_state_network/__init__.py index dae096f..9020afa 100644 --- a/src/pyrcn/echo_state_network/__init__.py +++ b/src/pyrcn/echo_state_network/__init__.py @@ -16,9 +16,10 @@ State Networks’, Jan. 2012, doi: 10.1007/978-3-642-35289-8_36. """ +from __future__ import annotations + # Authors: Peter Steiner # License: BSD 3 clause - from ._esn import ESNClassifier, ESNRegressor __all__ = ('ESNClassifier', 'ESNRegressor', ) diff --git a/src/pyrcn/echo_state_network/_esn.py b/src/pyrcn/echo_state_network/_esn.py index ab28b56..c4d7915 100644 --- a/src/pyrcn/echo_state_network/_esn.py +++ b/src/pyrcn/echo_state_network/_esn.py @@ -4,26 +4,25 @@ # License: BSD 3 clause from __future__ import annotations + import sys +from typing import Any, Literal + +from joblib import Parallel, delayed import numpy as np -from sklearn.base import (BaseEstimator, ClassifierMixin, RegressorMixin, - MultiOutputMixin, is_regressor, clone) -from sklearn.linear_model._base import LinearModel +from sklearn.base import (BaseEstimator, ClassifierMixin, MultiOutputMixin, + RegressorMixin, clone, is_regressor) +from sklearn.exceptions import NotFittedError +from sklearn.preprocessing import LabelBinarizer +from sklearn.utils.validation import validate_data from ..base.blocks import InputToNode, NodeToNode -from ..util import concatenate_sequences from ..linear_model import IncrementalRegression from ..projection import MatrixToValueProjection -from sklearn.utils.validation import _deprecate_positional_args -from sklearn.preprocessing import LabelBinarizer -from sklearn.exceptions import NotFittedError - -from joblib import Parallel, delayed - -from typing import Union, Dict, Any, Optional, Literal +from ..util import concatenate_sequences -class ESNRegressor(BaseEstimator, MultiOutputMixin, RegressorMixin): +class ESNRegressor(RegressorMixin, MultiOutputMixin, BaseEstimator): """ Echo State Network regressor. @@ -41,7 +40,8 @@ class ESNRegressor(BaseEstimator, MultiOutputMixin, RegressorMixin): ```input_to_node```. If ```None```, a ```pyrcn.base.blocks.NodeToNode``` object is instantiated. - regressor : Union[IncrementalRegression, LinearModel, None], default=None + regressor : Union[IncrementalRegression, RegressorMixin, None], + default=None Regressor object such as derived from ``BaseEstimator``. This regressor will automatically be cloned each time prior to fitting. If ```None```, a ```pyrcn.linear_model.IncrementalRegression``` @@ -61,13 +61,12 @@ class ESNRegressor(BaseEstimator, MultiOutputMixin, RegressorMixin): default=None """ - @_deprecate_positional_args def __init__(self, *, - input_to_node: Optional[InputToNode] = None, - node_to_node: Optional[NodeToNode] = None, - regressor: Union[IncrementalRegression, - LinearModel, None] = None, - requires_sequence: Union[Literal["auto"], bool] = "auto", + input_to_node: InputToNode | None = None, + node_to_node: NodeToNode | None = None, + regressor: (IncrementalRegression | + RegressorMixin | None) = None, + requires_sequence: Literal["auto"] | bool = "auto", decision_strategy: Literal["winner_takes_all", "median", "last_value"] = "winner_takes_all", verbose: bool = True, @@ -147,7 +146,7 @@ def __radd__(self, other: ESNRegressor) -> ESNRegressor: else: return self.__add__(other) - def get_params(self, deep: bool = True) -> Dict: + def get_params(self, deep: bool = True) -> dict: """Get all parameters of the ESNRegressor.""" if deep: return {**self.input_to_node.get_params(), @@ -200,7 +199,7 @@ def _check_if_sequence(self, X: np.ndarray, y: np.ndarray) -> None: """ if X.ndim > 2 or y.ndim > 2: raise ValueError("Could not determine a valid structure," - "because X has {0} and y has {1} dimensions." + "because X has {} and y has {} dimensions." "Only 1 or 2 dimensions allowed." .format(X.ndim, y.ndim)) self.requires_sequence = X.ndim == 1 @@ -226,7 +225,7 @@ def _check_if_sequence_to_value(self, self._sequence_to_value = not np.any(len_X == len_y) def partial_fit(self, X: np.ndarray, y: np.ndarray, - transformer_weights: Union[None, np.ndarray] = None, + transformer_weights: None | np.ndarray = None, postpone_inverse: bool = False) -> ESNRegressor: """ Fit the regressor partially. @@ -248,16 +247,15 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, self : Returns a trained ```ESNRegressor``` model. """ self._validate_hyperparameters() - self._validate_data(X=X, y=y, multi_output=True) + validate_data(self, X=X, y=y, multi_output=True) # input_to_node try: hidden_layer_state = self._input_to_node.transform(X) except NotFittedError as e: if self.verbose: - print('input_to_node has not been fitted yet: {0}'.format(e)) + print(f'input_to_node has not been fitted yet: {e}') hidden_layer_state = self._input_to_node.fit_transform(X) - pass # node_to_node try: @@ -265,15 +263,15 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, hidden_layer_state) except NotFittedError as e: if self.verbose: - print('node_to_node has not been fitted yet: {0}'.format(e)) + print(f'node_to_node has not been fitted yet: {e}') hidden_layer_state = self._node_to_node.fit_transform( hidden_layer_state) - pass # regression if not hasattr(self._regressor, 'partial_fit') and postpone_inverse: - raise BaseException('Regressor has no attribute partial_fit, got' - '{0}'.format(self._regressor)) + raise TypeError( + "Regressor has no attribute partial_fit, " + f"got {self._regressor}") elif not hasattr(self._regressor, 'partial_fit') \ and not postpone_inverse: self._regressor.fit(hidden_layer_state, y) @@ -283,8 +281,8 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, return self def fit(self, X: np.ndarray, y: np.ndarray, - n_jobs: Union[int, np.integer, None] = None, - transformer_weights: Optional[np.ndarray] = None) -> ESNRegressor: + n_jobs: int | np.integer | None = None, + transformer_weights: np.ndarray | None = None) -> ESNRegressor: """ Fit the regressor. @@ -313,7 +311,7 @@ def fit(self, X: np.ndarray, y: np.ndarray, self._input_to_node.fit(X) self._node_to_node.fit(self._input_to_node.transform(X)) else: - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) self._input_to_node.fit(X) self._node_to_node.fit(self._input_to_node.transform(X)) # self._regressor = self._regressor.__class__() @@ -324,8 +322,8 @@ def fit(self, X: np.ndarray, y: np.ndarray, def _sequence_fit(self, X: np.ndarray, y: np.ndarray, sequence_ranges: np.ndarray, - n_jobs: Union[int, np.integer, - None] = None) -> ESNRegressor: + n_jobs: (int | np.integer | + None) = None) -> ESNRegressor: """ Call partial_fit for each sequence. Runs parallel if more than one job. @@ -406,7 +404,7 @@ def _validate_hyperparameters(self) -> None: and hasattr(self.input_to_node, "fit_transform") and hasattr(self.input_to_node, "transform")): raise TypeError("All input_to_node should be transformers and" - "implement fit and transform '{0}' (type {1}) " + "implement fit and transform '{}' (type {}) " "doesn't".format(self.input_to_node, type(self.input_to_node))) @@ -414,18 +412,18 @@ def _validate_hyperparameters(self) -> None: and hasattr(self.node_to_node, "fit_transform") and hasattr(self.node_to_node, "transform")): raise TypeError("All node_to_node should be transformers and" - "implement fit and transform '{0}' (type {1}) " + "implement fit and transform '{}' (type {}) " "doesn't".format(self.node_to_node, type(self.node_to_node))) if (self._requires_sequence != "auto" and not isinstance(self._requires_sequence, bool)): - raise ValueError('Invalid value for requires_sequence, got {0}' + raise ValueError('Invalid value for requires_sequence, got {}' .format(self._requires_sequence)) if not is_regressor(self._regressor): raise TypeError("The last step should be a regressor and " - "implement fit and predict '{0}' (type {1})" + "implement fit and predict '{}' (type {})" "doesn't".format(self._regressor, type(self._regressor))) @@ -442,25 +440,25 @@ def __sizeof__(self) -> int: sys.getsizeof(self._node_to_node) + sys.getsizeof(self._regressor) @property - def regressor(self) -> Union[LinearModel, IncrementalRegression]: + def regressor(self) -> RegressorMixin | IncrementalRegression: """ Return the regressor. Returns ------- - regressor : LinearModel + regressor : RegressorMixin """ return self._regressor @regressor.setter - def regressor(self, regressor: Union[LinearModel, - IncrementalRegression]) -> None: + def regressor(self, regressor: (RegressorMixin | + IncrementalRegression)) -> None: """ Set the regressor. Parameters ---------- - regressor : LinearModel + regressor : RegressorMixin """ self._regressor = regressor @@ -587,7 +585,7 @@ def decision_strategy(self, decision_strategy: Literal["winner_takes_all", self._decision_strategy = decision_strategy @property - def requires_sequence(self) -> Union[Literal["auto"], bool]: + def requires_sequence(self) -> Literal["auto"] | bool: """ Return the requires_sequence parameter. @@ -599,7 +597,7 @@ def requires_sequence(self) -> Union[Literal["auto"], bool]: @requires_sequence.setter def requires_sequence(self, - requires_sequence: Union[Literal["auto"], bool])\ + requires_sequence: Literal["auto"] | bool)\ -> None: """ Set the requires_sequence parameter. @@ -612,7 +610,7 @@ def requires_sequence(self, self._requires_sequence = requires_sequence -class ESNClassifier(ESNRegressor, ClassifierMixin): +class ESNClassifier(ClassifierMixin, ESNRegressor): """ Echo State Network classifier. @@ -630,8 +628,9 @@ class ESNClassifier(ESNRegressor, ClassifierMixin): ```input_to_node```. If ```None```, a ```pyrcn.base.blocks.NodeToNode()``` object is instantiated. - regressor : Union[IncrementalRegression, LinearModel, None], default=None - Regressor object such as derived from ``LinearModel``. This + regressor : Union[IncrementalRegression, RegressorMixin, None], + default=None + Regressor object such as derived from ``RegressorMixin``. This regressor will automatically be cloned each time prior to fitting. If ```None```, a ```pyrcn.linear_model.IncrementalRegression()``` object is instantiated. @@ -649,13 +648,12 @@ class ESNClassifier(ESNRegressor, ClassifierMixin): keyword arguments passed to the subestimators if this is desired. """ - @_deprecate_positional_args def __init__(self, *, - input_to_node: Optional[InputToNode] = None, - node_to_node: Optional[NodeToNode] = None, - regressor: Union[IncrementalRegression, - LinearModel, None] = None, - requires_sequence: Union[Literal["auto"], bool] = "auto", + input_to_node: InputToNode | None = None, + node_to_node: NodeToNode | None = None, + regressor: (IncrementalRegression | + RegressorMixin | None) = None, + requires_sequence: Literal["auto"] | bool = "auto", decision_strategy: Literal["winner_takes_all", "median", "last_value"] = "winner_takes_all", verbose: bool = False, @@ -670,9 +668,9 @@ def __init__(self, *, self._sequence_to_value = False def partial_fit(self, X: np.ndarray, y: np.ndarray, - transformer_weights: Optional[np.ndarray] = None, + transformer_weights: np.ndarray | None = None, postpone_inverse: bool = False, - classes: Optional[np.ndarray] = None) -> ESNClassifier: + classes: np.ndarray | None = None) -> ESNClassifier: """ Fit the regressor partially. @@ -699,7 +697,7 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, ------- self : returns a trained ESNClassifier model """ - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) self._encoder.fit(classes) super().partial_fit(X, self._encoder.transform(y), transformer_weights=None, @@ -707,9 +705,9 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, return self def fit(self, X: np.ndarray, y: np.ndarray, - n_jobs: Union[int, np.integer, None] = None, - transformer_weights: Union[None, - np.ndarray] = None) -> ESNClassifier: + n_jobs: int | np.integer | None = None, + transformer_weights: (None | + np.ndarray) = None) -> ESNClassifier: """ Fit the classifier. @@ -739,7 +737,7 @@ def fit(self, X: np.ndarray, y: np.ndarray, self._input_to_node.fit(X) self._node_to_node.fit(self._input_to_node.transform(X)) else: - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) self._input_to_node.fit(X) self._node_to_node.fit(self._input_to_node.transform(X)) self._encoder = LabelBinarizer().fit(y) diff --git a/src/pyrcn/extreme_learning_machine/__init__.py b/src/pyrcn/extreme_learning_machine/__init__.py index 5146ca5..da564b1 100644 --- a/src/pyrcn/extreme_learning_machine/__init__.py +++ b/src/pyrcn/extreme_learning_machine/__init__.py @@ -13,10 +13,11 @@ applications’, p. 489-501, 2006, doi: 10.1016/j.neucom.2005.12.126. """ +from __future__ import annotations + # Authors: Peter Steiner , # Michael Schindler # License: BSD 3 clause - from ._elm import ELMClassifier, ELMRegressor __all__ = ('ELMClassifier', 'ELMRegressor') diff --git a/src/pyrcn/extreme_learning_machine/_elm.py b/src/pyrcn/extreme_learning_machine/_elm.py index b0de47a..8c345db 100644 --- a/src/pyrcn/extreme_learning_machine/_elm.py +++ b/src/pyrcn/extreme_learning_machine/_elm.py @@ -1,28 +1,28 @@ """The :mod:`extreme_learning_machine` contains the ELMRegressor and ELMClassifier.""" +from __future__ import annotations + # Authors: Peter Steiner # License: BSD 3 clause - from __future__ import annotations + import sys -from typing import Union, Any, Optional +from typing import Any +from joblib import Parallel, delayed import numpy as np -from sklearn.base import (BaseEstimator, ClassifierMixin, RegressorMixin, - MultiOutputMixin, is_regressor, clone) -from sklearn.linear_model._base import LinearModel +from sklearn.base import (BaseEstimator, ClassifierMixin, MultiOutputMixin, + RegressorMixin, clone, is_regressor) +from sklearn.exceptions import NotFittedError +from sklearn.preprocessing import LabelBinarizer +from sklearn.utils.validation import validate_data from ..base.blocks import InputToNode from ..linear_model import IncrementalRegression -from sklearn.utils.validation import _deprecate_positional_args -from sklearn.preprocessing import LabelBinarizer -from sklearn.exceptions import NotFittedError - -from joblib import Parallel, delayed -class ELMRegressor(BaseEstimator, MultiOutputMixin, RegressorMixin): +class ELMRegressor(RegressorMixin, MultiOutputMixin, BaseEstimator): """ Extreme Learning Machine regressor. @@ -35,8 +35,9 @@ class ELMRegressor(BaseEstimator, MultiOutputMixin, RegressorMixin): Any ```InputToNode``` object that transforms the inputs. If ```None```, a ```pyrcn.base.blocks.InputToNode``` object is instantiated. - regressor : Union[IncrementalRegression, LinearModel, None], default=None - Regressor object such as derived from ``LinearModel``. This + regressor : Union[IncrementalRegression, RegressorMixin, None], + default=None + Regressor object such as derived from ``RegressorMixin``. This regressor will automatically be cloned each time prior to fitting. If ```None```, a ```pyrcn.linear_model.IncrementalRegression``` object is instantiated. @@ -50,12 +51,11 @@ class ELMRegressor(BaseEstimator, MultiOutputMixin, RegressorMixin): default=None """ - @_deprecate_positional_args def __init__(self, *, - input_to_node: Optional[InputToNode] = None, - regressor: Union[IncrementalRegression, - LinearModel, None] = None, - chunk_size: Optional[int] = None, + input_to_node: InputToNode | None = None, + regressor: (IncrementalRegression | + RegressorMixin | None) = None, + chunk_size: int | None = None, verbose: bool = False, **kwargs: Any) -> None: """Construct the ELMRegressor.""" @@ -149,7 +149,7 @@ def set_params(self, **parameters: dict) -> ELMRegressor: return self def partial_fit(self, X: np.ndarray, y: np.ndarray, - transformer_weights: Union[np.ndarray, None] = None, + transformer_weights: np.ndarray | None = None, postpone_inverse: bool = False) -> ELMRegressor: """ Fit the regressor partially. @@ -172,19 +172,19 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, self : Returns a trained ```ELMRegressor``` model. """ if not hasattr(self._regressor, 'partial_fit'): - raise BaseException('regressor has no attribute partial_fit, got' - '{0}'.format(self._regressor)) + raise TypeError( + "regressor has no attribute partial_fit, " + f"got {self._regressor}") self._validate_hyperparameters() - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) # input_to_node try: hidden_layer_state = self._input_to_node.transform(X) except NotFittedError as e: if self.verbose: - print('input_to_node has not been fitted yet: {0}'.format(e)) + print(f'input_to_node has not been fitted yet: {e}') hidden_layer_state = self._input_to_node.fit_transform(X) - pass # regression if self._regressor: @@ -193,9 +193,9 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, return self def fit(self, X: np.ndarray, y: np.ndarray, - n_jobs: Union[int, np.integer, None] = None, - transformer_weights: Union[np.ndarray, - None] = None) -> ELMRegressor: + n_jobs: int | np.integer | None = None, + transformer_weights: (np.ndarray | + None) = None) -> ELMRegressor: """ Fit the regressor. @@ -216,7 +216,7 @@ def fit(self, X: np.ndarray, y: np.ndarray, self : Returns a trained ELMRegressor model. """ self._validate_hyperparameters() - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) self._input_to_node.fit(X) @@ -253,7 +253,7 @@ def fit(self, X: np.ndarray, y: np.ndarray, transformer_weights=transformer_weights, postpone_inverse=False) else: - raise ValueError('chunk_size invalid {0}'.format(self._chunk_size)) + raise ValueError(f'chunk_size invalid {self._chunk_size}') return self def predict(self, X: np.ndarray) -> np.ndarray: @@ -279,19 +279,19 @@ def _validate_hyperparameters(self) -> None: and hasattr(self.input_to_node, "fit_transform") and hasattr(self.input_to_node, "transform")): raise TypeError("All input_to_node should be transformers and" - "implement fit and transform '{0}' (type {1})" + "implement fit and transform '{}' (type {})" "doesn't".format(self.input_to_node, type(self.input_to_node))) if (self._chunk_size is not None and (not isinstance(self._chunk_size, int) or self._chunk_size < 0)): - raise ValueError('Invalid value for chunk_size, got {0}' + raise ValueError('Invalid value for chunk_size, got {}' .format(self._chunk_size)) if not is_regressor(self._regressor): raise TypeError("The last step should be a regressor and" - "implement fit and predict '{0}' (type {1}) " + "implement fit and predict '{}' (type {}) " "doesn't".format(self._regressor, type(self._regressor))) @@ -308,19 +308,19 @@ def __sizeof__(self) -> int: sys.getsizeof(self._regressor) @property - def regressor(self) -> Union[LinearModel, IncrementalRegression]: + def regressor(self) -> RegressorMixin | IncrementalRegression: """ Return the regressor. Returns ------- - regressor : LinearModel + regressor : RegressorMixin """ return self._regressor @regressor.setter - def regressor(self, regressor: Union[LinearModel, - IncrementalRegression]) -> None: + def regressor(self, regressor: (RegressorMixin | + IncrementalRegression)) -> None: """ Set the regressor. @@ -373,7 +373,7 @@ def hidden_layer_state(self, X: np.ndarray) -> np.ndarray: return hidden_layer_state @property - def chunk_size(self) -> Union[None, int, np.integer]: + def chunk_size(self) -> None | int | np.integer: """ Return the chunk_size, in which X will be chopped. @@ -384,7 +384,7 @@ def chunk_size(self) -> Union[None, int, np.integer]: return self._chunk_size @chunk_size.setter - def chunk_size(self, chunk_size: Union[int, None]) -> None: + def chunk_size(self, chunk_size: int | None) -> None: """ Set the chunk_size, in which X will be chopped. @@ -395,7 +395,7 @@ def chunk_size(self, chunk_size: Union[int, None]) -> None: self._chunk_size = chunk_size -class ELMClassifier(ELMRegressor, ClassifierMixin): +class ELMClassifier(ClassifierMixin, ELMRegressor): """ Extreme Learning Machine classifier. @@ -408,8 +408,9 @@ class ELMClassifier(ELMRegressor, ClassifierMixin): Any ```InputToNode``` object that transforms the inputs. If ```None```, a ```pyrcn.base.blocks.InputToNode``` object is instantiated. - regressor : Union[IncrementalRegression, LinearModel, None], default=None - Regressor object such as derived from ``LinearModel``. This + regressor : Union[IncrementalRegression, RegressorMixin, None], + default=None + Regressor object such as derived from ``RegressorMixin``. This regressor will automatically be cloned each time prior to fitting. If ```None```, a ```pyrcn.linear_model.IncrementalRegression``` object is instantiated. @@ -423,12 +424,11 @@ class ELMClassifier(ELMRegressor, ClassifierMixin): default=None """ - @_deprecate_positional_args def __init__(self, *, - input_to_node: Optional[InputToNode] = None, - regressor: Union[IncrementalRegression, - LinearModel, None] = None, - chunk_size: Optional[int] = None, verbose: bool = False, + input_to_node: InputToNode | None = None, + regressor: (IncrementalRegression | + RegressorMixin | None) = None, + chunk_size: int | None = None, verbose: bool = False, **kwargs: Any) -> None: """Construct the ELMClassifier.""" super().__init__(input_to_node=input_to_node, regressor=regressor, @@ -436,9 +436,9 @@ def __init__(self, *, self._encoder = LabelBinarizer() def partial_fit(self, X: np.ndarray, y: np.ndarray, - transformer_weights: Optional[np.ndarray] = None, + transformer_weights: np.ndarray | None = None, postpone_inverse: bool = False, - classes: Optional[np.ndarray] = None) -> ELMClassifier: + classes: np.ndarray | None = None) -> ELMClassifier: """ Fit the classifier partially. @@ -466,7 +466,7 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, ------- self : returns a trained ELMClassifier model """ - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) self._encoder.fit(classes) @@ -476,8 +476,8 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, return self def fit(self, X: np.ndarray, y: np.ndarray, - n_jobs: Union[int, np.integer, None] = None, - transformer_weights: Optional[np.ndarray] = None) -> ELMClassifier: + n_jobs: int | np.integer | None = None, + transformer_weights: np.ndarray | None = None) -> ELMClassifier: """ Fit the classifier. @@ -497,7 +497,7 @@ def fit(self, X: np.ndarray, y: np.ndarray, ------- self : Returns a trained ELMClassifier model. """ - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) self._encoder = LabelBinarizer().fit(y) super().fit(X, self._encoder.transform(y), n_jobs=n_jobs, transformer_weights=None) diff --git a/src/pyrcn/linear_model/__init__.py b/src/pyrcn/linear_model/__init__.py index 478b0cd..d705ab3 100644 --- a/src/pyrcn/linear_model/__init__.py +++ b/src/pyrcn/linear_model/__init__.py @@ -3,6 +3,8 @@ # Authors: Peter Steiner # License: BSD 3 clause +from __future__ import annotations + from ._incremental_regression import IncrementalRegression __all__ = ('IncrementalRegression',) diff --git a/src/pyrcn/linear_model/_incremental_regression.py b/src/pyrcn/linear_model/_incremental_regression.py index edd3a1f..fe3420d 100644 --- a/src/pyrcn/linear_model/_incremental_regression.py +++ b/src/pyrcn/linear_model/_incremental_regression.py @@ -6,17 +6,17 @@ from __future__ import annotations import sys -from typing import Union, cast +from typing import cast import numpy as np from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import _deprecate_positional_args -from sklearn.utils.extmath import safe_sparse_dot -from sklearn.preprocessing import StandardScaler from sklearn.exceptions import NotFittedError +from sklearn.preprocessing import StandardScaler +from sklearn.utils.extmath import safe_sparse_dot +from sklearn.utils.validation import validate_data -class IncrementalRegression(BaseEstimator, RegressorMixin): +class IncrementalRegression(RegressorMixin, BaseEstimator): """ Linear regression. @@ -50,7 +50,6 @@ class IncrementalRegression(BaseEstimator, RegressorMixin): ``fit_intercept = False``. """ - @_deprecate_positional_args def __init__(self, *, alpha: float = 1e-5, fit_intercept: bool = True, normalize: bool = False): @@ -91,7 +90,7 @@ def partial_fit(self, X: np.ndarray, y: np.ndarray, self : returns a partially fitted IncrementalRegression model """ if validate: - self._validate_data(X, y, multi_output=True) + validate_data(self, X, y, multi_output=True) X_preprocessed = self._preprocessing( X, partial_normalize=partial_normalize) @@ -217,7 +216,7 @@ def __sizeof__(self) -> int: self._output_weights.nbytes + sys.getsizeof(self.scaler) @property - def coef_(self) -> Union[np.ndarray, None]: + def coef_(self) -> np.ndarray | None: """ Return the output weights without intercept. diff --git a/src/pyrcn/metrics/__init__.py b/src/pyrcn/metrics/__init__.py index 277b1e0..834e622 100644 --- a/src/pyrcn/metrics/__init__.py +++ b/src/pyrcn/metrics/__init__.py @@ -5,25 +5,25 @@ results. """ +from __future__ import annotations + # Author: Peter Steiner # License: BSD 3 clause - -from ._classification import (accuracy_score, balanced_accuracy_score, - classification_report, cohen_kappa_score, - confusion_matrix, f1_score, fbeta_score, - hamming_loss, hinge_loss, jaccard_score, - log_loss, matthews_corrcoef, - precision_recall_fscore_support, precision_score, - recall_score, zero_one_loss, brier_score_loss, - multilabel_confusion_matrix) from ..metrics._regression import (explained_variance_score, max_error, - mean_absolute_error, mean_squared_error, - mean_squared_log_error, - median_absolute_error, - mean_absolute_percentage_error, r2_score, + mean_absolute_error, + mean_absolute_percentage_error, + mean_gamma_deviance, mean_poisson_deviance, + mean_squared_error, mean_squared_log_error, mean_tweedie_deviance, - mean_poisson_deviance, mean_gamma_deviance) - + median_absolute_error, r2_score) +from ._classification import (accuracy_score, balanced_accuracy_score, + brier_score_loss, classification_report, + cohen_kappa_score, confusion_matrix, f1_score, + fbeta_score, hamming_loss, hinge_loss, + jaccard_score, log_loss, matthews_corrcoef, + multilabel_confusion_matrix, + precision_recall_fscore_support, precision_score, + recall_score, zero_one_loss) __all__ = ('accuracy_score', 'balanced_accuracy_score', diff --git a/src/pyrcn/metrics/_classification.py b/src/pyrcn/metrics/_classification.py index dd3c664..9f78222 100644 --- a/src/pyrcn/metrics/_classification.py +++ b/src/pyrcn/metrics/_classification.py @@ -6,34 +6,27 @@ the lower the better. """ +from __future__ import annotations + # Authors: Peter Steiner # License: BSD 3 clause -import sys +from typing import Literal import numpy as np - -from sklearn.utils.validation import (check_consistent_length, - _deprecate_positional_args) -import sklearn.metrics as sklearn_metrics from scipy.sparse import csr_matrix -from sklearn.metrics._classification import\ - _check_targets as sklearn_check_targets - -if sys.version_info >= (3, 8): - from typing import Tuple, Union, Optional, Dict, Literal -else: - from typing_extensions import Literal - from typing import Tuple, Union, Optional, Dict +import sklearn.metrics as sklearn_metrics +from sklearn.utils.multiclass import type_of_target +from sklearn.utils.validation import check_consistent_length def _check_targets(y_true: np.ndarray, y_pred: np.ndarray, - sample_weight: Optional[np.ndarray] = None) \ - -> Tuple[Literal["multilabel-indicator", + sample_weight: np.ndarray | None = None) \ + -> tuple[Literal["multilabel-indicator", "multiclass", "binary"], - Union[np.ndarray, csr_matrix], - Union[np.ndarray, csr_matrix], - Union[np.ndarray, csr_matrix, None]]: + np.ndarray | csr_matrix, + np.ndarray | csr_matrix, + np.ndarray | csr_matrix | None]: """ Check that y_true and y_pred belong to the same classification task. @@ -67,14 +60,13 @@ def _check_targets(y_true: np.ndarray, y_pred: np.ndarray, [check_consistent_length(y_t, y_p) for y_t, y_p in zip(y_true, y_pred)] y_true = np.concatenate(y_true) y_pred = np.concatenate(y_pred) - y_type, y_true, y_pred = sklearn_check_targets(y_true, y_pred) + y_type = type_of_target(y_true) return y_type, y_true, y_pred, sample_weight -@_deprecate_positional_args def accuracy_score(y_true: np.ndarray, y_pred: np.ndarray, *, normalize: bool = True, - sample_weight: Optional[np.ndarray] = None) -> float: + sample_weight: np.ndarray | None = None) -> float: """ Accuracy classification score. @@ -124,11 +116,10 @@ def accuracy_score(y_true: np.ndarray, y_pred: np.ndarray, *, sample_weight=sample_weight) -@_deprecate_positional_args def confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray, *, - labels: Optional[np.ndarray] = None, - sample_weight: Optional[np.ndarray] = None, - normalize: Optional[Literal["true", "predicted"]] = None)\ + labels: np.ndarray | None = None, + sample_weight: np.ndarray | None = None, + normalize: Literal["true", "predicted"] | None = None)\ -> np.ndarray: """ Compute confusion matrix to evaluate the accuracy of a classification. @@ -189,10 +180,9 @@ def confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray, *, sample_weight=sample_weight, normalize=normalize) -@_deprecate_positional_args def multilabel_confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - labels: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, + labels: np.ndarray | None = None, samplewise: bool = False) -> np.ndarray: """ Compute a confusion matrix for each class or sample. @@ -258,11 +248,10 @@ def multilabel_confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray, *, labels=labels, samplewise=samplewise) -@_deprecate_positional_args def cohen_kappa_score(y1: np.ndarray, y2: np.ndarray, *, - labels: Optional[np.ndarray] = None, - weights: Optional[Literal["linear", "quadratic"]] = None, - sample_weight: Optional[np.ndarray] = None) -> float: + labels: np.ndarray | None = None, + weights: Literal["linear", "quadratic"] | None = None, + sample_weight: np.ndarray | None = None) -> float: r""" Cohen' kappa: a statistic that measures inter-annotator agreement. @@ -322,15 +311,14 @@ class labels [2]_. sample_weight=sample_weight) -@_deprecate_positional_args def jaccard_score(y_true: np.ndarray, y_pred: np.ndarray, *, - labels: Optional[np.ndarray] = None, - pos_label: Union[str, int] = 1, - average: Optional[Literal['micro', 'macro', 'samples', - 'weighted', 'binary']] = 'binary', - sample_weight: Optional[np.ndarray] = None, + labels: np.ndarray | None = None, + pos_label: str | int = 1, + average: None | (Literal['micro', 'macro', 'samples', + 'weighted', 'binary']) = 'binary', + sample_weight: np.ndarray | None = None, zero_division: Literal["warn", 0, 1] = "warn")\ - -> Union[float, np.ndarray]: + -> float | np.ndarray: """ Jaccard similarity coefficient score. @@ -419,9 +407,8 @@ def jaccard_score(y_true: np.ndarray, y_pred: np.ndarray, *, zero_division=zero_division) -@_deprecate_positional_args def matthews_corrcoef(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None,) -> float: + sample_weight: np.ndarray | None = None,) -> float: """ Compute the Matthews correlation coefficient (MCC). @@ -479,10 +466,9 @@ def matthews_corrcoef(y_true: np.ndarray, y_pred: np.ndarray, *, sample_weight=sample_weight) -@_deprecate_positional_args def zero_one_loss(y_true: np.ndarray, y_pred: np.ndarray, *, normalize: bool = True, - sample_weight: Optional[np.ndarray] = None) -> float: + sample_weight: np.ndarray | None = None) -> float: """ Zero-one classification loss. @@ -529,15 +515,14 @@ def zero_one_loss(y_true: np.ndarray, y_pred: np.ndarray, *, sample_weight=sample_weight) -@_deprecate_positional_args def f1_score(y_true: np.ndarray, y_pred: np.ndarray, *, - labels: Optional[np.ndarray] = None, - pos_label: Union[str, int] = 1, - average: Optional[Literal['micro', 'macro', 'samples', 'weighted', - 'binary']] = 'binary', - sample_weight: Optional[np.ndarray] = None, + labels: np.ndarray | None = None, + pos_label: str | int = 1, + average: None | (Literal['micro', 'macro', 'samples', 'weighted', + 'binary']) = 'binary', + sample_weight: np.ndarray | None = None, zero_division: Literal["warn", 0, 1] = "warn")\ - -> Union[float, np.ndarray]: + -> float | np.ndarray: """ Compute the F1 score, also known as balanced F-score or F-measure. @@ -632,15 +617,14 @@ def f1_score(y_true: np.ndarray, y_pred: np.ndarray, *, zero_division=zero_division) -@_deprecate_positional_args def fbeta_score(y_true: np.ndarray, y_pred: np.ndarray, beta: float, *, - labels: Optional[np.ndarray] = None, - pos_label: Union[str, int] = 1, - average: Optional[Literal['micro', 'macro', 'samples', - 'weighted', 'binary']] = 'binary', - sample_weight: Optional[np.ndarray] = None, + labels: np.ndarray | None = None, + pos_label: str | int = 1, + average: None | (Literal['micro', 'macro', 'samples', + 'weighted', 'binary']) = 'binary', + sample_weight: np.ndarray | None = None, zero_division: Literal["warn", 0, 1] = "warn")\ - -> Union[float, np.ndarray]: + -> float | np.ndarray: """ Compute the F-beta score. @@ -735,21 +719,20 @@ def fbeta_score(y_true: np.ndarray, y_pred: np.ndarray, beta: float, *, return f -@_deprecate_positional_args def precision_recall_fscore_support(y_true: np.ndarray, y_pred: np.ndarray, *, beta: float = 1.0, - labels: Optional[np.ndarray] = None, - pos_label: Union[str, int] = 1, - average: Optional[Literal[ + labels: np.ndarray | None = None, + pos_label: str | int = 1, + average: None | (Literal[ 'micro', 'macro', 'samples', - 'weighted', 'binary']] = 'binary', - warn_for: Tuple = ('precision', 'recall', + 'weighted', 'binary']) = 'binary', + warn_for: tuple = ('precision', 'recall', 'f-score'), - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, zero_division: Literal["warn", 0, 1] = "warn")\ - -> Tuple[Union[float, np.ndarray], Union[float, np.ndarray], - Union[float, np.ndarray], Optional[np.ndarray]]: + -> tuple[float | np.ndarray, float | np.ndarray, + float | np.ndarray, np.ndarray | None]: """ Compute precision, recall, F-measure and support for each class. @@ -868,16 +851,15 @@ def precision_recall_fscore_support(y_true: np.ndarray, y_pred: np.ndarray, *, sample_weight=sample_weight, zero_division=zero_division) -@_deprecate_positional_args def precision_score(y_true: np.ndarray, y_pred: np.ndarray, *, - labels: Optional[np.ndarray] = None, - pos_label: Union[str, int] = 1, - average: Optional[Literal[ - 'micro', 'macro', 'samples', 'weighted', 'binary']] + labels: np.ndarray | None = None, + pos_label: str | int = 1, + average: None | (Literal[ + 'micro', 'macro', 'samples', 'weighted', 'binary']) = 'binary', - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, zero_division: Literal["warn", 0, 1] = "warn")\ - -> Union[float, np.ndarray]: + -> float | np.ndarray: """ Compute the precision. @@ -962,15 +944,14 @@ def precision_score(y_true: np.ndarray, y_pred: np.ndarray, *, return p -@_deprecate_positional_args def recall_score(y_true: np.ndarray, y_pred: np.ndarray, *, - labels: Optional[np.ndarray] = None, - pos_label: Union[str, int] = 1, - average: Optional[Literal['micro', 'macro', 'samples', - 'weighted', 'binary']] = 'binary', - sample_weight: Optional[np.ndarray] = None, + labels: np.ndarray | None = None, + pos_label: str | int = 1, + average: None | (Literal['micro', 'macro', 'samples', + 'weighted', 'binary']) = 'binary', + sample_weight: np.ndarray | None = None, zero_division: Literal["warn", 0, 1] = "warn")\ - -> Union[float, np.ndarray]: + -> float | np.ndarray: """ Compute the recall. @@ -1054,9 +1035,8 @@ def recall_score(y_true: np.ndarray, y_pred: np.ndarray, *, return r -@_deprecate_positional_args def balanced_accuracy_score(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, adjusted: bool = False) -> float: """ Compute the balanced accuracy. @@ -1118,14 +1098,13 @@ def balanced_accuracy_score(y_true: np.ndarray, y_pred: np.ndarray, *, adjusted=adjusted) -@_deprecate_positional_args def classification_report(y_true: np.ndarray, y_pred: np.ndarray, *, - labels: Optional[np.ndarray] = None, - target_names: Optional[np.ndarray] = None, - sample_weight: Optional[np.ndarray] = None, + labels: np.ndarray | None = None, + target_names: np.ndarray | None = None, + sample_weight: np.ndarray | None = None, digits: int = 2, output_dict: bool = False, zero_division: Literal["warn", 0, 1] = "warn") \ - -> Union[str, Dict]: + -> str | dict: """ Build a text report showing the main classification metrics. @@ -1197,9 +1176,8 @@ def classification_report(y_true: np.ndarray, y_pred: np.ndarray, *, zero_division=zero_division) -@_deprecate_positional_args def hamming_loss(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None) -> float: + sample_weight: np.ndarray | None = None) -> float: """ Compute the average Hamming loss. @@ -1259,11 +1237,10 @@ def hamming_loss(y_true: np.ndarray, y_pred: np.ndarray, *, y_true, y_pred, sample_weight=sample_weight) -@_deprecate_positional_args def log_loss(y_true: np.ndarray, y_pred: np.ndarray, *, eps: float = 1e-15, normalize: bool = True, - sample_weight: Optional[np.ndarray] = None, - labels: Optional[np.ndarray] = None) -> float: + sample_weight: np.ndarray | None = None, + labels: np.ndarray | None = None) -> float: r""" Log loss, aka logistic loss or cross-entropy loss. @@ -1329,10 +1306,9 @@ def log_loss(y_true: np.ndarray, y_pred: np.ndarray, *, eps: float = 1e-15, sample_weight=sample_weight, labels=labels) -@_deprecate_positional_args def hinge_loss(y_true: np.ndarray, pred_decision: np.ndarray, *, - labels: Optional[np.ndarray] = None, - sample_weight: Optional[np.ndarray] = None) -> float: + labels: np.ndarray | None = None, + sample_weight: np.ndarray | None = None) -> float: """ Average hinge loss (non-regularized). @@ -1388,10 +1364,9 @@ def hinge_loss(y_true: np.ndarray, pred_decision: np.ndarray, *, sample_weight=sample_weight) -@_deprecate_positional_args def brier_score_loss(y_true: np.ndarray, y_prob: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - pos_label: Optional[int] = None) -> float: + sample_weight: np.ndarray | None = None, + pos_label: int | None = None) -> float: """ Compute the Brier score loss. diff --git a/src/pyrcn/metrics/_regression.py b/src/pyrcn/metrics/_regression.py index b35a586..a4c4cff 100644 --- a/src/pyrcn/metrics/_regression.py +++ b/src/pyrcn/metrics/_regression.py @@ -6,37 +6,30 @@ the lower the better. """ +from __future__ import annotations + # Authors: Peter Steiner # License: BSD 3 clause -import sys +from typing import Any, Literal import numpy as np - -from sklearn.utils.validation import (check_consistent_length, - _deprecate_positional_args) import sklearn.metrics as sklearn_metrics -from sklearn.metrics._regression\ - import _check_reg_targets as sklearn_check_reg_targets - -if sys.version_info >= (3, 8): - from typing import Any, Tuple, Union, Optional, Literal -else: - from typing_extensions import Literal - from typing import Any, Tuple, Union, Optional +from sklearn.utils.multiclass import type_of_target +from sklearn.utils.validation import check_consistent_length def _check_reg_targets(y_true: np.ndarray, y_pred: np.ndarray, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = None, + "variance_weighted"] | None) = None, dtype: str = "numeric")\ - -> Tuple[Any, np.ndarray, np.ndarray, Optional[np.ndarray], - Union[np.ndarray, Literal["raw_values", - "uniform_average", - "variance_weighted"], - None]]: + -> tuple[ + Any, np.ndarray, np.ndarray, np.ndarray | None, + np.ndarray + | Literal["raw_values", "uniform_average", "variance_weighted"] + | None]: """ Check that y_true and y_pred belong to the same regression task. @@ -78,17 +71,15 @@ def _check_reg_targets(y_true: np.ndarray, y_pred: np.ndarray, [check_consistent_length(y_t, y_p) for y_t, y_p in zip(y_true, y_pred)] y_true = np.concatenate(y_true) y_pred = np.concatenate(y_pred) - y_type, y_true, y_pred, multioutput = sklearn_check_reg_targets( - y_true, y_pred, multioutput, dtype) + y_type = type_of_target(y_true) return y_type, y_true, y_pred, sample_weight, multioutput -@_deprecate_positional_args def mean_absolute_error(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = "uniform_average")\ + "variance_weighted"] | None) = "uniform_average")\ -> float: """Mean absolute error regression loss. @@ -132,10 +123,10 @@ def mean_absolute_error(y_true: np.ndarray, y_pred: np.ndarray, *, def mean_absolute_percentage_error(y_true: np.ndarray, y_pred: np.ndarray, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = + "variance_weighted"] | None) = "uniform_average")\ -> float: """ @@ -186,12 +177,11 @@ def mean_absolute_percentage_error(y_true: np.ndarray, y_pred: np.ndarray, multioutput=multioutput) -@_deprecate_positional_args def mean_squared_error(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = "uniform_average", + "variance_weighted"] | None) = "uniform_average", squared: bool = True) -> float: """Mean squared error regression loss. @@ -228,17 +218,20 @@ def mean_squared_error(y_true: np.ndarray, y_pred: np.ndarray, *, check_consistent_length(y_true, y_pred, sample_weight) else: check_consistent_length(y_true, y_pred) - return sklearn_metrics.mean_squared_error( + if squared: + return sklearn_metrics.mean_squared_error( + y_true=y_true, y_pred=y_pred, sample_weight=sample_weight, + multioutput=multioutput) + return sklearn_metrics.root_mean_squared_error( y_true=y_true, y_pred=y_pred, sample_weight=sample_weight, - multioutput=multioutput, squared=squared) + multioutput=multioutput) -@_deprecate_positional_args def mean_squared_log_error(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = + "variance_weighted"] | None) = "uniform_average") -> float: """Mean squared logarithmic error regression loss. @@ -279,12 +272,11 @@ def mean_squared_log_error(y_true: np.ndarray, y_pred: np.ndarray, *, multioutput=multioutput) -@_deprecate_positional_args def median_absolute_error(y_true: np.ndarray, y_pred: np.ndarray, *, - multioutput: Union[np.ndarray, Literal[ + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = "uniform_average", - sample_weight: Optional[np.ndarray] = None) -> float: + "variance_weighted"] | None) = "uniform_average", + sample_weight: np.ndarray | None = None) -> float: """Median absolute error regression loss. Median absolute error output is non-negative floating point. The best value @@ -327,12 +319,11 @@ def median_absolute_error(y_true: np.ndarray, y_pred: np.ndarray, *, sample_weight=sample_weight) -@_deprecate_positional_args def explained_variance_score(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ "raw_values", "uniform_average", - "variance_weighted"], None] = + "variance_weighted"] | None) = "uniform_average") -> float: """Explained variance regression score function. @@ -379,12 +370,11 @@ def explained_variance_score(y_true: np.ndarray, y_pred: np.ndarray, *, multioutput=multioutput) -@_deprecate_positional_args def r2_score(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, - multioutput: Union[np.ndarray, Literal[ - "raw_values", "uniform_average", "variance_weighted"], - None] = "uniform_average") -> float: + sample_weight: np.ndarray | None = None, + multioutput: (np.ndarray | Literal[ + "raw_values", "uniform_average", "variance_weighted"] | + None) = "uniform_average") -> float: """ R^2 (coefficient of determination) regression score function. @@ -475,9 +465,8 @@ def max_error(y_true: np.ndarray, y_pred: np.ndarray) -> float: return sklearn_metrics.max_error(y_true=y_true, y_pred=y_pred) -@_deprecate_positional_args def mean_tweedie_deviance(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, power: float = 0) -> float: """Mean Tweedie deviance regression loss. @@ -523,9 +512,8 @@ def mean_tweedie_deviance(y_true: np.ndarray, y_pred: np.ndarray, *, y_true=y_true, y_pred=y_pred, sample_weight=sample_weight, power=power) -@_deprecate_positional_args def mean_poisson_deviance(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None) -> float: + sample_weight: np.ndarray | None = None) -> float: """Mean Poisson deviance regression loss. Poisson deviance is equivalent to the Tweedie deviance with @@ -550,9 +538,8 @@ def mean_poisson_deviance(y_true: np.ndarray, y_pred: np.ndarray, *, y_true, y_pred, sample_weight=sample_weight, power=1) -@_deprecate_positional_args def mean_gamma_deviance(y_true: np.ndarray, y_pred: np.ndarray, *, - sample_weight: Optional[np.ndarray] = None) -> float: + sample_weight: np.ndarray | None = None) -> float: """ Mean Gamma deviance regression loss. diff --git a/src/pyrcn/model_selection/__init__.py b/src/pyrcn/model_selection/__init__.py index 7f3e327..97c0bfd 100644 --- a/src/pyrcn/model_selection/__init__.py +++ b/src/pyrcn/model_selection/__init__.py @@ -4,6 +4,8 @@ # Simon Stone # License: BSD 3 clause +from __future__ import annotations + from ._search import SequentialSearchCV, SHGOSearchCV __all__ = ('SequentialSearchCV', 'SHGOSearchCV') diff --git a/src/pyrcn/model_selection/_search.py b/src/pyrcn/model_selection/_search.py index ca5124b..f1fccf2 100644 --- a/src/pyrcn/model_selection/_search.py +++ b/src/pyrcn/model_selection/_search.py @@ -6,17 +6,16 @@ from __future__ import annotations -from sklearn.base import BaseEstimator, is_classifier, clone -from sklearn.model_selection._search import BaseSearchCV -from sklearn.utils.validation import indexable, _check_method_params -from sklearn.model_selection._split import check_cv +from collections.abc import Callable, Iterable +import time +from typing import Any import numpy as np -import time from scipy import optimize -from collections.abc import Iterable - -from typing import Union, Optional, Callable, Dict, Any, List, Tuple +from sklearn.base import BaseEstimator, clone, is_classifier +from sklearn.model_selection._search import BaseSearchCV +from sklearn.model_selection._split import check_cv +from sklearn.utils.validation import _check_method_params, indexable class SequentialSearchCV(BaseSearchCV): @@ -112,15 +111,15 @@ class SequentialSearchCV(BaseSearchCV): def __init__(self, estimator: BaseEstimator, searches: list, - scoring: Union[str, Callable, list, tuple, dict, None] = None, - n_jobs: Union[int, np.integer, None] = None, + scoring: str | Callable | list | tuple | dict | None = None, + n_jobs: int | np.integer | None = None, refit: bool = True, - cv: Union[int, np.integer, Iterable, None] = None, - verbose: Union[int, np.integer] = 0, - pre_dispatch: Union[int, np.integer, str] = '2*n_jobs', - error_score: Union[int, float] = np.nan) -> None: + cv: int | np.integer | Iterable | None = None, + verbose: int | np.integer = 0, + pre_dispatch: int | np.integer | str = '2*n_jobs', + error_score: int | float = np.nan) -> None: """Construct the SequentialSearchCV.""" - self.estimator: Optional[BaseEstimator] = None + self.estimator: BaseEstimator | None = None super().__init__( estimator, scoring=scoring, n_jobs=n_jobs, refit=refit, cv=cv, verbose=verbose, pre_dispatch=pre_dispatch, @@ -137,8 +136,8 @@ def _run_search(self, evaluate_candidates: Callable) -> None: """ evaluate_candidates(self.searches) - def fit(self, X: np.ndarray, y: Optional[np.ndarray], *, - groups: Optional[np.ndarray] = None, **fit_params: Any)\ + def fit(self, X: np.ndarray, y: np.ndarray | None, *, + groups: np.ndarray | None = None, **fit_params: Any)\ -> SequentialSearchCV: """ Run fit with all sets of parameters. @@ -158,15 +157,15 @@ def fit(self, X: np.ndarray, y: Optional[np.ndarray], *, Parameters passed to the ```fit``` method of the estimator. """ def evaluate_candidates(searches: list) -> None: - self.all_cv_results_: Dict[str, dict] = {} - self.all_best_estimator_: Dict[str, BaseEstimator] = {} - self.all_best_score_: Dict[str, Any] = {} - self.all_best_params_: Dict[str, dict] = {} - self.all_best_index_: Dict[str, int] = {} - self.all_scorer_: Dict[str, Any] = {} - self.all_n_splits_: Dict[str, int] = {} - self.all_refit_time_: Dict[str, float] = {} - self.all_multimetric_: Dict[str, bool] = {} + self.all_cv_results_: dict[str, dict] = {} + self.all_best_estimator_: dict[str, BaseEstimator] = {} + self.all_best_score_: dict[str, Any] = {} + self.all_best_params_: dict[str, dict] = {} + self.all_best_index_: dict[str, int] = {} + self.all_scorer_: dict[str, Any] = {} + self.all_n_splits_: dict[str, int] = {} + self.all_refit_time_: dict[str, float] = {} + self.all_multimetric_: dict[str, bool] = {} for name, search, params, *kwargs in searches: if len(kwargs) == 1 and 'refit' in kwargs[0].keys(): result = search( @@ -293,7 +292,7 @@ def best_params_(self) -> dict: return {} @property - def best_index_(self) -> Union[int, np.integer]: + def best_index_(self) -> int | np.integer: """ The index (of the cv_results_ arrays) which corresponds to the best candidate. @@ -314,7 +313,7 @@ def best_index_(self) -> Union[int, np.integer]: return 0 @property - def scorer_(self) -> Dict: + def scorer_(self) -> dict: """ Scorer function used on the held out data. @@ -330,7 +329,7 @@ def scorer_(self) -> Dict: return self.all_scorer_[self.searches[-1][0]] @property - def n_splits_(self) -> Union[int, np.integer]: + def n_splits_(self) -> int | np.integer: """ The number of cross-validation splits (folds/iterations). @@ -467,10 +466,10 @@ class SHGOSearchCV(BaseSearchCV): GridSearchCV : Does exhaustive search over a grid of parameters. """ - def __init__(self, estimator: BaseEstimator, func: Callable, params: Dict, - *, args: Tuple = (), - constraints: Union[Dict, List, None] = None, - refit: bool = True, cv: Optional[int] = None, + def __init__(self, estimator: BaseEstimator, func: Callable, params: dict, + *, args: tuple = (), + constraints: dict | list | None = None, + refit: bool = True, cv: int | None = None, return_train_score: bool = False) -> None: super().__init__(estimator=estimator, refit=refit, cv=cv, return_train_score=return_train_score) @@ -479,8 +478,8 @@ def __init__(self, estimator: BaseEstimator, func: Callable, params: Dict, self.args = args self.constraints = constraints - def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, *, - groups: Optional[np.ndarray] = None, + def fit(self, X: np.ndarray, y: np.ndarray | None = None, *, + groups: np.ndarray | None = None, **fit_params: dict) -> SHGOSearchCV: """ Run the optimization based on the parameters defined before. diff --git a/src/pyrcn/nn/__init__.py b/src/pyrcn/nn/__init__.py deleted file mode 100644 index 959ef30..0000000 --- a/src/pyrcn/nn/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""These are the basic building blocks for RCNs in PyTorch.""" -from ._forward_layers import ELM, LeakyELM -from ._recurrent_layers import ( - ESNCell, IdentityESNCell, IdentityEuSNCell, ESN, DelayLineReservoirCell, - DelayLineReservoirESN, DelayLineReservoirWithFeedbackCell, - DelayLineReservoirWithFeedbackESN, SimpleCycleReservoirCell, - SimpleCycleReservoirESN) - - -__all__ = [ - "ELM", "LeakyELM", "ESNCell", "ESN", "IdentityESNCell", "IdentityEuSNCell", - "DelayLineReservoirCell", "DelayLineReservoirESN", - "DelayLineReservoirWithFeedbackCell", "DelayLineReservoirWithFeedbackESN", - "SimpleCycleReservoirCell", "SimpleCycleReservoirESN"] diff --git a/src/pyrcn/nn/_activations.py b/src/pyrcn/nn/_activations.py deleted file mode 100644 index 88f30db..0000000 --- a/src/pyrcn/nn/_activations.py +++ /dev/null @@ -1,26 +0,0 @@ -"""The :mod:`activations` contains various activation functions for PyRCN.""" - -# Authors: Peter Steiner -# License: BSD 3 clause - -from torch import nn - - -ACTIVATIONS = { - "elu": nn.ELU, - "hard_shrink": nn.Hardshrink, - "hard_sigmoid": nn.Hardsigmoid, - "hard_tanh": nn.Hardtanh, - "hard_swish": nn.Hardswish, - "identity": nn.Identity, - "leaky_relu": nn.LeakyReLU, - "log_sigmoid": nn.LogSigmoid, - "relu": nn.ReLU, - "relu_6": nn.ReLU6, - "selu": nn.SELU, - "sigmoid": nn.Sigmoid, - "silu": nn.SiLU, - "mish": nn.Mish, - "tanh": nn.Tanh, - "tanh_shrink": nn.Tanhshrink, -} diff --git a/src/pyrcn/nn/_forward_layers.py b/src/pyrcn/nn/_forward_layers.py deleted file mode 100644 index a195f9c..0000000 --- a/src/pyrcn/nn/_forward_layers.py +++ /dev/null @@ -1,345 +0,0 @@ -"""Forward RCN layers.""" -from typing import Optional, Union, Tuple, List, Callable, Literal -import torch -import torch.nn as nn -import numpy as np - -from ..util import value_to_tuple, batched - - -class Linear(nn.Linear): - r""" - Applies a linear transformation to the incoming data: :math:`y = xA^T + b` - - This module supports :ref:`TensorFloat32`. - - On certain ROCm devices, when using float16 inputs this module will use - :ref:`different precision` for backward. - - Parameters - ---------- - in_features : int - Size of each input sample. - out_features : int - Size of each output sample. - bias : bool, default = True - If set to ``False``, the layer will not use an additive bias. - - Notes - ----- - - Shape: - - Input: :math:`(*, H_{in})` where :math:`*` means any number of - dimensions including none and :math:`H_{in} = \text{in\_features}`. - - Output: :math:`(*, H_{out})` where all but the last dimension are the - same shape as the input and :math:`H_{out} = \text{out\_features}`. - - Attributes # noqa - ---------- - weight : torch.Tensor, - shape = :math:`(\text{out\_features}, \text{in\_features})`. - The hidden weights of the module. - bias : torch.Tensor, shape = :math:`(\text{out\_features})`. - The learnable bias of the module. If :attr:`bias` is ``True``, the - values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` - where :math:`k = \frac{1}{\text{in\_features}}`. - """ - - def __init__(self, in_features: int, out_features: int, bias: bool = True, - input_scaling: float = 1., bias_scaling: float = 0., - bias_shift: float = 0., input_sparsity: float = 0.9, - device: Optional[str] = None, dtype: Optional = None): - self.input_scaling = input_scaling - self.bias_scaling = bias_scaling - self.bias_shift = bias_shift - self.input_sparsity = input_sparsity - - super().__init__(in_features=in_features, out_features=out_features, - bias=bias, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for name, weight in self.named_parameters(): - weight.requires_grad_(False) - if "weight" in name: - nn.init.sparse_(weight, sparsity=self.input_sparsity, - std=self.input_scaling) - elif "bias" in name: - nn.init.uniform_(weight, -self.bias_scaling + self.bias_shift, - self.bias_scaling + self.bias_shift) - - def set_external_weights(self, weights: torch.Tensor) -> None: - """ - Set externally initialized weights for the layer. - - Parameters - ---------- - weights : torch.Tensor, shape = (out_features, in_features). - The externally initialized weight tensor. - """ - if weights.shape != (self.out_features, self.in_features): - raise ValueError( - f"Shape of weights ({weights.shape}) does not match the " - f"expected shape ({(self.out_features, self.in_features)}).") - - with torch.no_grad(): - self.weight.copy_(weights) - self.weight.requires_grad_(False) - - def set_external_bias(self, bias: torch.Tensor) -> None: - """ - Set externally initialized bias for the layer. - - Parameters - ---------- - bias : torch.Tensor, shape = (out_features, ). - The externally initialized bias tensor. - """ - if bias.shape != (self.out_features, ): - raise ValueError( - f"Shape of bias ({bias.shape}) does not match the expected " - f"shape ({(self.out_features, )}).") - - with torch.no_grad(): - self.bias.copy_(bias) - self.bias.requires_grad_(False) - - -class ELM(nn.Sequential): - r""" - Applies a multi-layer Extreme Learning Machine with :math:`\tanh` or an - arbitrary non-linearity to an input. - - For each element in the input, each layer computes the following function: - - .. math:: - h = \tanh(x W_{ih}^T + b_{ih}) - - where :math:`h` is the hidden state, and :math:`x` is the input. - If :attr:`nonlinearity` is not :math:'\tanh', then the specific - non-linearity is used instead of :math:`\tanh`. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_sizes : Tuple[int, ...] - The number of features in each hidden layer `h`. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : Union[float, Tuple[float, ...]], default = 1. - Scales the input weights `w_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_scaling : Union[float, Tuple[float, ...]], default = 0. - Scales the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_shift : Union[float, Tuple[float, ...]], default = 0. - Shifts the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - input_sparsity : Union[float, Tuple[float, ...]], default = 0.9 - Ratio between zero and non-zero values in the input weights `w_ih`. If - it is a Tuple, it needs to have `num_layer` entries for each layer. If - it is a number, the value in each layer is the same. - activation_layer : Optional[Callable[..., nn.Module]], default = nn.ReLU - The non-linearity to use. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Leaky integration is not supported. However, leaky integrator deep RCN - models can be built using the ``LeakyELM``. - - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - """ - - def __init__( - self, input_size: int, hidden_sizes: Tuple[int, ...], - bias: bool = True, - input_scaling: Union[float, Tuple[float, ...]] = 1., - bias_scaling: Union[float, Tuple[float, ...]] = 0., - bias_shift: Union[float, Tuple[float, ...]] = 0., - input_sparsity: Union[float, Tuple[float, ...]] = 0.9, - activation_layer: Optional[Callable[..., nn.Module]] = nn.ReLU, - device: Optional[str] = None, dtype: Optional = None): - self.num_layers = len(hidden_sizes) - self.input_scaling = value_to_tuple(input_scaling, self.num_layers) - self.bias_scaling = value_to_tuple(bias_scaling, self.num_layers) - self.bias_shift = value_to_tuple(bias_shift, self.num_layers) - self.input_sparsity = value_to_tuple(input_sparsity, self.num_layers) - layers = [] - in_features = input_size - for layer_id, out_features in enumerate(hidden_sizes): - layers.append(Linear(in_features, out_features, bias=bias, - input_scaling=self.input_scaling[layer_id], - bias_scaling=self.bias_scaling[layer_id], - bias_shift=self.bias_shift[layer_id], - input_sparsity=self.input_sparsity[layer_id], - device=device, dtype=dtype)) - layers.append(activation_layer()) - in_features = out_features - super().__init__(*layers) - - def set_external_weights(self, external_weights: List[torch.Tensor]) \ - -> None: - """ - Set externally initialized weights for each layer. - - Parameters - ---------- - external_weights : List[torch.Tensor]. - The externally initialized weight tensors. - """ - k = 0 - for name, layer in self.named_modules(): - if name != "": - if int(name) % 2 == 0: - layer.set_external_weights(external_weights[k]) - k += 1 - - def set_external_bias(self, external_bias: List[torch.Tensor]) -> None: - """ - Set externally initialized bias for each layer. - - Parameters - ---------- - external_bias : List[torch.Tensor]. - The externally initialized bias tensors. - """ - k = 0 - for name, layer in self.named_modules(): - if name != "": - if int(name) % 2 == 0: - layer.set_external_bias(external_bias[k]) - k += 1 - - -class LeakyELM(ELM): - r""" - Applies a multi-layer Extreme Learning Machine with :math:`\tanh` or an - arbitrary non-linearity to an input. - - For each element in the input, each layer computes the following function: - - .. math:: - h = \tanh(x W_{ih}^T + b_{ih}) - - where :math:`h` is the hidden state, and :math:`x` is the input. - If :attr:`nonlinearity` is not :math:'\tanh', then the specific - non-linearity is used instead of :math:`\tanh`. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_sizes : Tuple[int, ...] - The number of features in each hidden layer `h`. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : Union[float, Tuple[float, ...]], default = 1. - Scales the input weights `w_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_scaling : Union[float, Tuple[float, ...]], default = 0. - Scales the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_shift : Union[float, Tuple[float, ...]], default = 0. - Shifts the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - input_sparsity : Union[float, Tuple[float, ...]], default = 0.9 - Ratio between zero and non-zero values in the input weights `w_ih`. If - it is a Tuple, it needs to have `num_layer` entries for each layer. If - it is a number, the value in each layer is the same. - activation_layer : Optional[Callable[..., nn.Module]], default = nn.ReLU - The non-linearity to use. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Leaky integration is not supported. However, leaky integrator deep RCN - models can be built using the ``RCNCell``. - - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - """ - - def __init__( - self, input_size: int, hidden_sizes: Tuple[int, ...], - bias: bool = True, - input_scaling: Union[float, Tuple[float, ...]] = 1., - bias_scaling: Union[float, Tuple[float, ...]] = 0., - bias_shift: Union[float, Tuple[float, ...]] = 0., - input_sparsity: Union[float, Tuple[float, ...]] = 0.9, - leakage: Union[float, Tuple[float, ...]] = 1.0, - activation_layer: Optional[Callable[..., nn.Module]] = nn.ReLU, - device: Optional[str] = None, dtype: Optional = None): - self.leakage = value_to_tuple(leakage, self.num_layers) - super().__init__( - input_size=input_size, hidden_sizes=hidden_sizes, bias=bias, - input_scaling=input_scaling, bias_scaling=bias_scaling, - bias_shift=bias_shift, input_sparsity=input_sparsity, - activation_layer=activation_layer, device=device, dtype=dtype) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - r""" - Forward function of the neural network. Pass the input features through - the hidden layers and return the hidden layer states of the final - layer. - - Parameters - ---------- - input : Tensor, shape = :math:`(*, H_{in})`. - Tensor containing the input features. The input must also be a - packed variable length sequence. See - :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - Returns - ------- - output : Tensor, shape = :math:`(*, H_{out})`. - Tensor containing the output features `(h_t)` from the last layer - of the ELM. - """ - if input.dim() not in (1, 2): - raise ValueError(f"LeakyELM: Expected input to be 1D or 2D, " - f"got {input.shape}D instead") - is_batched = input.dim() == 2 - if not is_batched: - input = input.unsqueeze(0) - - batch_size = input.shape[0] - layer_id = 0 - layer_output = torch.zeros_like(input) - for weights, activation in batched(self.modules(), n=2): - out_features = weights.out_features - layer_output = torch.zeros((batch_size + 1, out_features)) - for k in range(batch_size): - layer_output[k+1] = \ - (1 - self.leakage[layer_id]) * layer_output[k] + \ - self.leakage[layer_id] * activation(weights(input[k])) - input = layer_output[1:] - layer_id += 1 - return layer_output[1:] diff --git a/src/pyrcn/nn/_recurrent_layers.py b/src/pyrcn/nn/_recurrent_layers.py deleted file mode 100644 index 85fc9fc..0000000 --- a/src/pyrcn/nn/_recurrent_layers.py +++ /dev/null @@ -1,1273 +0,0 @@ -"""Recurrent RCN layers.""" -from typing import Optional, Tuple, Union -import torch -import torch.nn as nn - -from . import init -from ..util import value_to_tuple - - -class ESN(nn.RNN): - r""" - Applies a multi-layer Echo State Network with :math:`\tanh` or - :math:`\text{ReLU}` non-linearity to an input sequence. - - For each element in the input sequence, each layer computes the following - function: - - .. math:: - h_t = \tanh(x_t W_{ih}^T + b_{ih} + h_{t-1}W_{hh}^T) - - where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the - input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the - previous layer at time `t-1` or the initial hidden state at time `0`. - If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used - instead of :math:`\tanh`. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - num_layers : int, default = 1 - Number of recurrent layers. E.g., setting ``num_layers=2`` would mean - stacking two RCNs together to form a `deep RCN`, with the second RCN - taking in outputs of the first RCN and computing the final results. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : Union[float, Tuple[float, ...]], default = 1. - Scales the input weights `w_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - spectral_radius : Union[float, Tuple[float, ...]], default = 1. - Scales the recurrent weights `w_hh`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_scaling : Union[float, Tuple[float, ...]], default = 0. - Scales the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_shift : Union[float, Tuple[float, ...]], default = 0. - Shifts the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - input_sparsity : Union[float, Tuple[float, ...]], default = 0.9 - Ratio between zero and non-zero values in the input weights `w_ih`. If - it is a Tuple, it needs to have `num_layer` entries for each layer. If - it is a number, the value in each layer is the same. - recurrent_sparsity : Union[float, Tuple[float, ...]], default = 0.9 - Ratio between zero and non-zero values in the recurrent weights `w_hh`. - If it is a Tuple, it needs to have `num_layer` entries for each layer. - If it is a number, the value in each layer is the same. - batch_first : bool, default = False - If ``True``, then the input and output tensors are provided as - `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that - this does not apply to hidden or cell states. See the Inputs/Outputs - sections below for details. - bidirectional: bool, default = False - If ``True``, becomes a bidirectional RCN. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Leaky integration is not supported. However, leaky integrator deep RCN - models can be built using the ``RCNCell``. - - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih_l[k] : torch.Tensor, shape = `(hidden_size, input_size)` or - `(hidden_size, num_directions * hidden_size)`. - The input-hidden weights of the k-th layer of shape - `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is - `(hidden_size, num_directions * hidden_size)`. - weight_hh_l[k] : torch.Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of the k-th layer of shape - `(hidden_size, hidden_size)`. - bias_ih_l[k] : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of the k-th layer of shape - `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, num_layers: int = 1, - nonlinearity: str = 'tanh', bias: bool = True, - input_scaling: Union[float, Tuple[float, ...]] = 1., - spectral_radius: Union[float, Tuple[float, ...]] = 1., - bias_scaling: Union[float, Tuple[float, ...]] = 0., - bias_shift: Union[float, Tuple[float, ...]] = 0., - input_sparsity: Union[float, Tuple[float, ...]] = 0.9, - recurrent_sparsity: Union[float, Tuple[float, ...]] = 0.9, - batch_first: bool = False, bidirectional: bool = False, - device: Optional[str] = None, dtype: Optional = None): - self.input_scaling = value_to_tuple(input_scaling, num_layers) - self.spectral_radius = value_to_tuple(spectral_radius, num_layers) - self.bias_scaling = value_to_tuple(bias_scaling, num_layers) - self.bias_shift = value_to_tuple(bias_shift, num_layers) - self.input_sparsity = value_to_tuple(input_sparsity, num_layers) - self.recurrent_sparsity = value_to_tuple(recurrent_sparsity, - num_layers) - - super().__init__( - input_size=input_size, hidden_size=hidden_size, - num_layers=num_layers, nonlinearity=nonlinearity, bias=bias, - batch_first=batch_first, dropout=0.0, bidirectional=bidirectional, - device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for name, weight in self.named_parameters(): - _, _, layer = name.split("_") - layer_idx = int(layer[1:]) - weight.requires_grad_(False) - if "weight_ih" in name: - nn.init.sparse_( - weight, sparsity=self.input_sparsity[layer_idx], - std=self.input_scaling[layer_idx]) - elif "weight_hh" in name: - nn.init.sparse_( - weight, sparsity=self.recurrent_sparsity[layer_idx], - std=1.) - init.spectral_norm_(weight) - weight *= self.spectral_radius[layer_idx] - elif "bias_ih" in name: - nn.init.uniform_( - weight, -(self.bias_scaling + self.bias_shift)[layer_idx], - (self.bias_scaling + self.bias_shift)[layer_idx]) - elif "bias_hh" in name: - nn.init.zeros_(weight) - else: - print(name) - - def forward(self, input: torch.Tensor, hx: Optional[torch.Tensor] = None) \ - -> Tuple[torch.Tensor, torch.Tensor]: - r""" - Forward function of the neural network. Pass the input features through - the hidden layers and return the hidden layer states of the final - layer. - - Parameters - ---------- - input : Tensor, shape = :math:`(L, H_{in})` or :math:`(L, N, H_{in})` - or :math:`(N, L, H_{in})`. - Tensor containing the input features. The shape for unbatched - input is :math:`(L, H_{in})`. For batched input, the shape is - :math:`(L, N, H_{in})` when ``batch_first=False``, or - :math:`(N, L, H_{in})` when ``batch_first=True``. The input can - also be a packed variable length sequence. See - :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - hx : Optional[Tensor], shape :math:`(D * \text{num\_layers}, H_{out})` - or :math:`(D * \text{num\_layers}, N, H_{out}) - Tensor containing the initial hidden state. The shape for unbatched - input is :math:`(D * \text{num\_layers}, H_{out})`. For batched - input, the shape is :math:`(D * \text{num\_layers}, N, H_{out})`. - Defaults to zero if not provided. - - Returns - ------- - output : Tensor, shape = :math:`(L, D * H_{out})` or - :math:`(L, N, D * H_{out})` or :math:`(N, L, D * H_{out})`. - Tensor containing the output features `(h_t)` from the last layer - of the RNN, for each `t`. The shape for unbatched input is - :math:`(L, D * H_{out})`. For batched input, the shape is - :math:`(L, N, D * H_{out})` when ``batch_first=False``, or - :math:`(N, L, D * H_{out})` when ``batch_first=True``. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the - input, the output will also be a packed sequence. - h_n : Tensor, shape = :math:`(D * \text{num\_layers}` or - :math:`(D * \text{num\_layers}, N, H_{out})`. - Tensor containing the final hidden state for each element in the - batch. The shape for unbatched input is - :math:`(D * \text{num\_layers}, H_{out})`. For batched input, - the shape is :math:`(D * \text{num\_layers}, N, H_{out})`. - """ - return super().forward(input, hx) - - -class DelayLineReservoirESN(ESN): - r""" - Applies a multi-layer Echo State Network with :math:`\tanh` or - :math:`\text{ReLU}` non-linearity to an input sequence. - - For each element in the input sequence, each layer computes the following - function: - - .. math:: - h_t = \tanh(x_t W_{ih}^T + b_{ih} + h_{t-1}W_{hh}^T) - - where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the - input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the - previous layer at time `t-1` or the initial hidden state at time `0`. - If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used - instead of :math:`\tanh`. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - num_layers : int, default = 1 - Number of recurrent layers. E.g., setting ``num_layers=2`` would mean - stacking two RCNs together to form a `deep RCN`, with the second RCN - taking in outputs of the first RCN and computing the final results. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : Union[float, Tuple[float, ...]], default = 1. - Scales the input weights `w_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - forward_weight : Union[float, Tuple[float, ...]], default = 1. - Scales the forward weights in `w_hh`. If it is a Tuple, it needs to - have `num_layer` entries for each layer. If it is a number, the value - in each layer is the same. - bias_scaling : Union[float, Tuple[float, ...]], default = 0. - Scales the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_shift : Union[float, Tuple[float, ...]], default = 0. - Shifts the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - batch_first : bool, default = False - If ``True``, then the input and output tensors are provided as - `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that - this does not apply to hidden or cell states. See the Inputs/Outputs - sections below for details. - bidirectional: bool, default = False - If ``True``, becomes a bidirectional RCN. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Leaky integration is not supported. However, leaky integrator deep RCN - models can be built using the ``RCNCell``. - - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih_l[k] : torch.Tensor, shape = `(hidden_size, input_size)` or - `(hidden_size, num_directions * hidden_size)`. - The input-hidden weights of the k-th layer of shape - `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is - `(hidden_size, num_directions * hidden_size)`. - weight_hh_l[k] : torch.Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of the k-th layer of shape - `(hidden_size, hidden_size)`. - bias_ih_l[k] : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of the k-th layer of shape - `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, num_layers: int = 1, - nonlinearity: str = 'tanh', bias: bool = True, - input_scaling: Union[float, Tuple[float, ...]] = 1., - forward_weight: Union[float, Tuple[float, ...]] = 1., - bias_scaling: Union[float, Tuple[float, ...]] = 0., - bias_shift: Union[float, Tuple[float, ...]] = 0., - batch_first: bool = False, bidirectional: bool = False, - device: Optional[str] = None, dtype: Optional = None): - self.forward_weight = value_to_tuple(forward_weight, num_layers) - - super().__init__( - input_size=input_size, hidden_size=hidden_size, - num_layers=num_layers, nonlinearity=nonlinearity, bias=bias, - input_scaling=input_scaling, spectral_radius=0., - bias_scaling=bias_scaling, bias_shift=bias_shift, - input_sparsity=0., recurrent_sparsity=0., batch_first=batch_first, - bidirectional=bidirectional, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for name, weight in self.named_parameters(): - _, _, layer = name.split("_") - layer_idx = int(layer[1:]) - weight.requires_grad_(False) - if "weight_ih" in name: - init.bernoulli_( - weight, p=0.5, std=self.input_scaling[layer_idx]) - elif "weight_hh" in name: - init.dlr_weights_( - weight, forward_weight=self.forward_weight[layer_idx]) - elif "bias_ih" in name: - init.bernoulli_( - weight, -(self.bias_scaling + self.bias_shift)[layer_idx], - (self.bias_scaling + self.bias_shift)[layer_idx]) - elif "bias_hh" in name: - nn.init.zeros_(weight) - else: - print(name) - - -class DelayLineReservoirWithFeedbackESN(ESN): - r""" - Applies a multi-layer Echo State Network with :math:`\tanh` or - :math:`\text{ReLU}` non-linearity to an input sequence. - - For each element in the input sequence, each layer computes the following - function: - - .. math:: - h_t = \tanh(x_t W_{ih}^T + b_{ih} + h_{t-1}W_{hh}^T) - - where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the - input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the - previous layer at time `t-1` or the initial hidden state at time `0`. - If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used - instead of :math:`\tanh`. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - num_layers : int, default = 1 - Number of recurrent layers. E.g., setting ``num_layers=2`` would mean - stacking two RCNs together to form a `deep RCN`, with the second RCN - taking in outputs of the first RCN and computing the final results. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : Union[float, Tuple[float, ...]], default = 1. - Scales the input weights `w_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - forward_weight : Union[float, Tuple[float, ...]], default = 1. - Scales the forward weights in `w_hh`. If it is a Tuple, it needs to - have `num_layer` entries for each layer. If it is a number, the value - in each layer is the same. - feedback_weight : Union[float, Tuple[float, ...]], default = 1. - Scales the feedback weights in `w_hh`. If it is a Tuple, it needs to - have `num_layer` entries for each layer. If it is a number, the value - in each layer is the same. - bias_scaling : Union[float, Tuple[float, ...]], default = 0. - Scales the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_shift : Union[float, Tuple[float, ...]], default = 0. - Shifts the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - batch_first : bool, default = False - If ``True``, then the input and output tensors are provided as - `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that - this does not apply to hidden or cell states. See the Inputs/Outputs - sections below for details. - bidirectional: bool, default = False - If ``True``, becomes a bidirectional RCN. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Leaky integration is not supported. However, leaky integrator deep RCN - models can be built using the ``RCNCell``. - - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih_l[k] : torch.Tensor, shape = `(hidden_size, input_size)` or - `(hidden_size, num_directions * hidden_size)`. - The input-hidden weights of the k-th layer of shape - `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is - `(hidden_size, num_directions * hidden_size)`. - weight_hh_l[k] : torch.Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of the k-th layer of shape - `(hidden_size, hidden_size)`. - bias_ih_l[k] : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of the k-th layer of shape - `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, num_layers: int = 1, - nonlinearity: str = 'tanh', bias: bool = True, - input_scaling: Union[float, Tuple[float, ...]] = 1., - forward_weight: Union[float, Tuple[float, ...]] = 1., - feedback_weight: Union[float, Tuple[float, ...]] = 1., - bias_scaling: Union[float, Tuple[float, ...]] = 0., - bias_shift: Union[float, Tuple[float, ...]] = 0., - batch_first: bool = False, bidirectional: bool = False, - device: Optional[str] = None, dtype: Optional = None): - self.forward_weight = value_to_tuple(forward_weight, num_layers) - self.feedback_weight = value_to_tuple(feedback_weight, num_layers) - - super().__init__( - input_size=input_size, hidden_size=hidden_size, - num_layers=num_layers, nonlinearity=nonlinearity, bias=bias, - input_scaling=input_scaling, spectral_radius=0., - bias_scaling=bias_scaling, bias_shift=bias_shift, - input_sparsity=0., recurrent_sparsity=0., batch_first=batch_first, - bidirectional=bidirectional, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for name, weight in self.named_parameters(): - _, _, layer = name.split("_") - layer_idx = int(layer[1:]) - weight.requires_grad_(False) - if "weight_ih" in name: - init.bernoulli_( - weight, p=0.5, std=self.input_scaling[layer_idx]) - elif "weight_hh" in name: - init.dlrb_weights_( - weight, forward_weight=self.forward_weight[layer_idx], - feedback_weight=self.feedback_weight[layer_idx]) - elif "bias_ih" in name: - init.bernoulli_( - weight, -(self.bias_scaling + self.bias_shift)[layer_idx], - (self.bias_scaling + self.bias_shift)[layer_idx]) - elif "bias_hh" in name: - nn.init.zeros_(weight) - else: - print(name) - - -class SimpleCycleReservoirESN(ESN): - r""" - Applies a multi-layer Echo State Network with :math:`\tanh` or - :math:`\text{ReLU}` non-linearity to an input sequence. - - For each element in the input sequence, each layer computes the following - function: - - .. math:: - h_t = \tanh(x_t W_{ih}^T + b_{ih} + h_{t-1}W_{hh}^T) - - where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the - input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the - previous layer at time `t-1` or the initial hidden state at time `0`. - If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used - instead of :math:`\tanh`. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - num_layers : int, default = 1 - Number of recurrent layers. E.g., setting ``num_layers=2`` would mean - stacking two RCNs together to form a `deep RCN`, with the second RCN - taking in outputs of the first RCN and computing the final results. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : Union[float, Tuple[float, ...]], default = 1. - Scales the input weights `w_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - forward_weight : Union[float, Tuple[float, ...]], default = 1. - Scales the forward weights in `w_hh`. If it is a Tuple, it needs to - have `num_layer` entries for each layer. If it is a number, the value - in each layer is the same. - bias_scaling : Union[float, Tuple[float, ...]], default = 0. - Scales the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - bias_shift : Union[float, Tuple[float, ...]], default = 0. - Shifts the bias weights `b_ih`. If it is a Tuple, it needs to have - `num_layer` entries for each layer. If it is a number, the value in - each layer is the same. - batch_first : bool, default = False - If ``True``, then the input and output tensors are provided as - `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that - this does not apply to hidden or cell states. See the Inputs/Outputs - sections below for details. - bidirectional: bool, default = False - If ``True``, becomes a bidirectional RCN. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Leaky integration is not supported. However, leaky integrator deep RCN - models can be built using the ``RCNCell``. - - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih_l[k] : torch.Tensor, shape = `(hidden_size, input_size)` or - `(hidden_size, num_directions * hidden_size)`. - The input-hidden weights of the k-th layer of shape - `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is - `(hidden_size, num_directions * hidden_size)`. - weight_hh_l[k] : torch.Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of the k-th layer of shape - `(hidden_size, hidden_size)`. - bias_ih_l[k] : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of the k-th layer of shape - `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, num_layers: int = 1, - nonlinearity: str = 'tanh', bias: bool = True, - input_scaling: Union[float, Tuple[float, ...]] = 1., - forward_weight: Union[float, Tuple[float, ...]] = 1., - bias_scaling: Union[float, Tuple[float, ...]] = 0., - bias_shift: Union[float, Tuple[float, ...]] = 0., - batch_first: bool = False, bidirectional: bool = False, - device: Optional[str] = None, dtype: Optional = None): - self.forward_weight = value_to_tuple(forward_weight, num_layers) - - super().__init__( - input_size=input_size, hidden_size=hidden_size, - num_layers=num_layers, nonlinearity=nonlinearity, bias=bias, - input_scaling=input_scaling, spectral_radius=0., - bias_scaling=bias_scaling, bias_shift=bias_shift, - input_sparsity=0., recurrent_sparsity=0., batch_first=batch_first, - bidirectional=bidirectional, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for name, weight in self.named_parameters(): - _, _, layer = name.split("_") - layer_idx = int(layer[1:]) - weight.requires_grad_(False) - if "weight_ih" in name: - init.bernoulli_( - weight, p=0.5, std=self.input_scaling[layer_idx]) - elif "weight_hh" in name: - init.scr_weights_( - weight, forward_weight=self.forward_weight[layer_idx]) - elif "bias_ih" in name: - init.bernoulli_( - weight, -(self.bias_scaling + self.bias_shift)[layer_idx], - (self.bias_scaling + self.bias_shift)[layer_idx]) - elif "bias_hh" in name: - nn.init.zeros_(weight) - else: - print(name) - - -class ESNCell(nn.RNNCell): - r""" - An Echo State Network cell with :math:`\tanh` or :math:`\text{ReLU}` - non-linearity. - - .. math:: - h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h) - - If :attr:`nonlinearity` is ``'relu'``, then ReLU is used instead of tanh. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : float, default = 1. - Scales the input weights `w_ih`. - spectral_radius : float, default = 1. - Scales the recurrent weights `w_hh`. - bias_scaling : float, default = 0. - Scales the bias weights `b_ih`. - bias_shift : float, default = 0. - Shifts the bias weights `b_ih`. - leakage : float, default = 1. - Parameter to determine the degree of leaky integration. - input_sparsity : float, default = 0.9 - Ratio between zero and non-zero values in the input weights `w_ih`. - recurrent_sparsity : float, default = 0.9 - Ratio between zero and non-zero values in the recurrent weights `w_hh`. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih : torch.Tensor, shape = `(hidden_size, input_size)`. - The input-hidden weights of shape `(hidden_size, input_size)`. - weight_hh : torch.Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of shape `(hidden_size, hidden_size)`. - bias_ih : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of shape `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, bias: bool = True, - nonlinearity: str = "tanh", input_scaling: float = 1., - spectral_radius: float = 1., bias_scaling: float = 0., - bias_shift: float = 0., leakage: float = 1., - input_sparsity: float = 0.9, recurrent_sparsity: float = 0.9, - device: Optional[str] = None, dtype: Optional = None): - self.input_scaling = input_scaling - self.spectral_radius = spectral_radius - self.bias_scaling = bias_scaling - self.bias_shift = bias_shift - self.leakage = leakage - self.input_sparsity = input_sparsity - self.recurrent_sparsity = recurrent_sparsity - self.input_sparsity = input_sparsity - self.recurrent_sparsity = recurrent_sparsity - super().__init__( - input_size=input_size, hidden_size=hidden_size, bias=bias, - nonlinearity=nonlinearity, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for weight in self.parameters(): - weight.requires_grad_(False) - nn.init.sparse_(self.weight_ih, sparsity=self.input_sparsity, - std=self.input_scaling) - nn.init.sparse_(self.weight_hh, sparsity=self.recurrent_sparsity, - std=1.) - init.spectral_norm_(self.weight_hh) - self.weight_hh *= self.spectral_radius - if self.bias: - nn.init.uniform_(self.bias_ih, - -self.bias_scaling + self.bias_shift, - self.bias_scaling + self.bias_shift) - nn.init.zeros_(self.bias_hh) - - def set_external_input_weights(self, weights: torch.Tensor) -> None: - """ - Set externally initialized input weights for the layer. - - Parameters - ---------- - weights : torch.Tensor, shape = (out_features, in_features). - The externally initialized weight tensor. - """ - if weights.shape != (self.hidden_size, self.input_size): - raise ValueError( - f"Shape of weights ({weights.shape}) does not match the " - f"expected shape ({(self.hidden_size, self.input_size)}).") - - with torch.no_grad(): - self.weight_ih.copy_(weights) - self.weight_ih.requires_grad_(False) - - def set_external_recurrent_weights(self, weights: torch.Tensor) -> None: - """ - Set externally initialized recurrent weights for the layer. - - Parameters - ---------- - weights : torch.Tensor, shape = (out_features, in_features). - The externally initialized weight tensor. - """ - if weights.shape != (self.hidden_size, self.hidden_size): - raise ValueError( - f"Shape of weights ({weights.shape}) does not match the " - f"expected shape ({(self.hidden_size, self.hidden_size)}).") - - with torch.no_grad(): - self.weight_hh.copy_(weights) - self.weight_hh.requires_grad_(False) - - def set_external_bias(self, bias: torch.Tensor) -> None: - """ - Set externally initialized bias for the layer. - - Parameters - ---------- - bias : torch.Tensor, shape = (out_features, ). - The externally initialized bias tensor. - """ - if bias.shape != (self.out_features, ): - raise ValueError( - f"Shape of bias ({bias.shape}) does not match the expected " - f"shape ({(self.out_features, )}).") - - with torch.no_grad(): - self.bias_ih.copy_(bias) - self.bias_ih.requires_grad_(False) - - def forward(self, input: torch.Tensor, hx: Optional[torch.Tensor] = None) \ - -> torch.Tensor: - r""" - Forward function of the neural network. Pass the input features through - the hidden layers and return the hidden layer states of the final - layer. - - Parameters - ---------- - input : torch.Tensor, shape = :math:`(N, H_{in})` or :math:`(H_{in})`. - Tensor containing the input features. The shape for unbatched - input is :math:`(H_{in})`. For batched input, the shape is - :math:`(N, H_{in})`. - hx : Optional[torch.Tensor], shape :math:`(H_{out})` - or :math:`(N, H_{out}). - Tensor containing the initial hidden state. The shape for unbatched - input is :math:`(H_{out})`. For batched input, the shape is - :math:`(N, H_{out})`. Defaults to zero if not provided. - - Returns - ------- - output : Tensor, shape = :math:`(H_{out})` or:math:`(N, H_{out})`. - Tensor containing the hidden states `h' `of the RNN. The shape for - unbatched input is :math:`(H_{out})`. For batched input, the shape - is :math:`(N, H_{out})`. - """ - if self.leakage < 1.: - return self._leaky_forward(input=input, hx=hx) - return super().forward(input=input, hx=hx) - - def _leaky_forward(self, input: torch.Tensor, - hx: Optional[torch.Tensor] = None) -> torch.Tensor: - r""" - Leaky integration forward function of the neural network. Pass the - input features through the hidden layers and return the hidden layer - states of the final layer. - - Parameters - ---------- - input : torch.Tensor, shape = :math:`(N, H_{in})` or :math:`(H_{in})`. - Tensor containing the input features. The shape for unbatched - input is :math:`(H_{in})`. For batched input, the shape is - :math:`(N, H_{in})`. - - Returns - ------- - output : torch.Tensor, shape = :math:`(H_{out})` or:math:`(N, H_{out})` - Tensor containing the hidden states `h' `of the RNN. The shape for - unbatched input is :math:`(H_{out})`. For batched input, the shape - is :math:`(N, H_{out})`. - """ - return \ - (1 - self.leakage) * hx + self.leakage * super().forward(input, hx) - - -class IdentityESNCell(ESNCell): - r""" - An Echo State Network cell with :math:`\tanh` or :math:`\text{ReLU}` - non-linearity. - - .. math:: - h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h) - - If :attr:`nonlinearity` is ``'relu'``, then ReLU is used instead of tanh. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - spectral_radius : float, default = 1. - Scales the recurrent weights `w_hh`. - leakage : float, default = 1. - Parameter to determine the degree of leaky integration. - recurrent_sparsity : float, default = 0.9 - Ratio between zero and non-zero values in the recurrent weights `w_hh`. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih : torch.Tensor, shape = `(hidden_size, input_size)`. - The input-hidden weights of shape `(hidden_size, input_size)`. - weight_hh : Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of shape `(hidden_size, hidden_size)`. - bias_ih : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of shape `(hidden_size)`. - """ - - def __init__(self, input_size: int, nonlinearity: str = "tanh", - spectral_radius: float = 1., leakage: float = 1., - recurrent_sparsity: float = 0.9, device: Optional[str] = None, - dtype: Optional = None): - super().__init__( - input_size=input_size, hidden_size=input_size, bias=False, - nonlinearity=nonlinearity, input_scaling=1., - spectral_radius=spectral_radius, bias_scaling=0., bias_shift=0., - leakage=leakage, input_sparsity=1., - recurrent_sparsity=recurrent_sparsity, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for weight in self.parameters(): - weight.requires_grad_(False) - nn.init.eye_(self.weight_ih) - nn.init.sparse_(self.weight_hh, sparsity=self.recurrent_sparsity, - std=1.) - init.spectral_norm_(self.weight_hh) - self.weight_hh *= self.spectral_radius - if self.bias: - nn.init.uniform_(self.bias_ih, - -self.bias_scaling + self.bias_shift, - self.bias_scaling + self.bias_shift) - nn.init.zeros_(self.bias_hh) - - -class IdentityEuSNCell(ESNCell): - r""" - An Euler State Network cell with :math:`\tanh` or :math:`\text{ReLU}` - non-linearity. - - .. math:: - h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h) - - If :attr:`nonlinearity` is ``'relu'``, then ReLU is used instead of tanh. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - spectral_radius : float, default = 1. - Scales the recurrent weights `w_hh`. - leakage : float, default = 1. - Parameter to determine the degree of leaky integration. - recurrent_sparsity : float, default = 0.9 - Ratio between zero and non-zero values in the recurrent weights `w_hh`. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih : torch.Tensor, shape = `(hidden_size, input_size)`. - The input-hidden weights of shape `(hidden_size, input_size)`. - weight_hh : Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of shape `(hidden_size, hidden_size)`. - bias_ih : torch.Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of shape `(hidden_size)`. - """ - - def __init__(self, input_size: int, nonlinearity: str = "tanh", - leakage: float = 1., recurrent_sparsity: float = 0.9, - recurrent_scaling: float = 1., gamma: float = 0.001, - epsilon: float = 0.01, device: Optional[str] = None, - dtype: Optional = None): - self.recurrent_scaling = recurrent_scaling - self.gamma = gamma - self.epsilon = epsilon - super().__init__( - input_size=input_size, hidden_size=input_size, bias=False, - nonlinearity=nonlinearity, input_scaling=1., - spectral_radius=0., bias_scaling=0., bias_shift=0., - leakage=leakage, input_sparsity=1., - recurrent_sparsity=recurrent_sparsity, device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for weight in self.parameters(): - weight.requires_grad_(False) - nn.init.eye_(self.weight_ih) - nn.init.sparse_(self.weight_hh, sparsity=self.recurrent_sparsity, - std=1.) - init.antisymmetric_norm_(self.weight_hh) - init.diffusion_norm_(self.weight_hh, gamma=self.gamma) - if self.bias: - nn.init.uniform_(self.bias_ih, - -self.bias_scaling + self.bias_shift, - self.bias_scaling + self.bias_shift) - nn.init.zeros_(self.bias_hh) - - def forward(self, input: torch.Tensor, hx: Optional[torch.Tensor] = None) \ - -> torch.Tensor: - r""" - Forward function of the neural network. Pass the input features through - the hidden layers and return the hidden layer states of the final - layer. - - Parameters - ---------- - input : torch.Tensor, shape = :math:`(N, H_{in})` or :math:`(H_{in})`. - Tensor containing the input features. The shape for unbatched - input is :math:`(H_{in})`. For batched input, the shape is - :math:`(N, H_{in})`. - hx : Optional[torch.Tensor], shape :math:`(H_{out})` - or :math:`(N, H_{out}). - Tensor containing the initial hidden state. The shape for unbatched - input is :math:`(H_{out})`. For batched input, the shape is - :math:`(N, H_{out})`. Defaults to zero if not provided. - - Returns - ------- - output : Tensor, shape = :math:`(H_{out})` or:math:`(N, H_{out})`. - Tensor containing the hidden states `h' `of the RNN. The shape for - unbatched input is :math:`(H_{out})`. For batched input, the shape - is :math:`(N, H_{out})`. - """ - return self._euler_forward(input=input, hx=hx) - - def _euler_forward(self, input: torch.Tensor, - hx: Optional[torch.Tensor] = None) -> torch.Tensor: - r""" - Euler forward function of the neural network. Pass the input features - through the hidden layers and return the hidden layer states of the - final layer. - - Parameters - ---------- - input : torch.Tensor, shape = :math:`(N, H_{in})` or :math:`(H_{in})`. - Tensor containing the input features. The shape for unbatched - input is :math:`(H_{in})`. For batched input, the shape is - :math:`(N, H_{in})`. - - Returns - ------- - output : torch.Tensor, shape = :math:`(H_{out})` or:math:`(N, H_{out})` - Tensor containing the hidden states `h' `of the RNN. The shape for - unbatched input is :math:`(H_{out})`. For batched input, the shape - is :math:`(N, H_{out})`. - """ - return hx + self.epsilon * super().forward(input, hx) - - -class DelayLineReservoirCell(ESNCell): - r""" - An Echo State Network cell with :math:`\tanh` or :math:`\text{ReLU}` - non-linearity. - - .. math:: - h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h) - - If :attr:`nonlinearity` is ``'relu'``, then ReLU is used instead of tanh. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : float, default = 1. - Scales the input weights `w_ih`. - bias_scaling : float, default = 0. - Scales the bias weights `b_ih`. - bias_shift : float, default = 0. - Shifts the bias weights `b_ih`. - leakage : float, default = 1. - Parameter to determine the degree of leaky integration. - forward_weight : float, default = 1. - Scales the forward weights in `w_hh`. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih : Tensor, shape = `(hidden_size, input_size)`. - The input-hidden weights of shape `(hidden_size, input_size)`. - weight_hh : Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of shape `(hidden_size, hidden_size)`. - bias_ih : Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of shape `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, bias: bool = True, - nonlinearity: str = "tanh", input_scaling: float = 1., - bias_scaling: float = 0., bias_shift: float = 0., - leakage: float = 1., forward_weight: float = .9, - device: Optional[str] = None, dtype: Optional = None): - self.forward_weight = forward_weight - super().__init__( - input_size=input_size, hidden_size=hidden_size, bias=bias, - nonlinearity=nonlinearity, input_scaling=input_scaling, - spectral_radius=0., bias_scaling=bias_scaling, - bias_shift=bias_shift, leakage=leakage, input_sparsity=0., - recurrent_sparsity=0., device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for weight in self.parameters(): - weight.requires_grad_(False) - init.bernoulli_(self.weight_ih, p=.5, std=self.input_scaling) - init.dlr_weights_(self.weight_hh, forward_weight=self.forward_weight) - if self.bias: - init.bernoulli_(self.bias_ih, .5, self.bias_scaling) - nn.init.zeros_(self.bias_hh) - - -class DelayLineReservoirWithFeedbackCell(ESNCell): - r""" - An Echo State Network cell with :math:`\tanh` or :math:`\text{ReLU}` - non-linearity. - - .. math:: - h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h) - - If :attr:`nonlinearity` is ``'relu'``, then ReLU is used instead of tanh. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : float, default = 1. - Scales the input weights `w_ih`. - bias_scaling : float, default = 0. - Scales the bias weights `b_ih`. - bias_shift : float, default = 0. - Shifts the bias weights `b_ih`. - leakage : float, default = 1. - Parameter to determine the degree of leaky integration. - forward_weight : float, default = .9 - Scales the forward weights in `w_hh`. - feedback_weight : float, default = .1 - Scales the feedback weights in `w_hh`. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih : Tensor, shape = `(hidden_size, input_size)`. - The input-hidden weights of shape `(hidden_size, input_size)`. - weight_hh : Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of shape `(hidden_size, hidden_size)`. - bias_ih : Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of shape `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, bias: bool = True, - nonlinearity: str = "tanh", input_scaling: float = 1., - bias_scaling: float = 0., bias_shift: float = 0., - leakage: float = 1., forward_weight: float = .9, - feedback_weight: float = .1, device: Optional[str] = None, - dtype: Optional = None): - self.forward_weight = forward_weight - self.feedback_weight = feedback_weight - super().__init__( - input_size=input_size, hidden_size=hidden_size, bias=bias, - nonlinearity=nonlinearity, input_scaling=input_scaling, - spectral_radius=0., bias_scaling=bias_scaling, - bias_shift=bias_shift, leakage=leakage, input_sparsity=0., - recurrent_sparsity=0., device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for weight in self.parameters(): - weight.requires_grad_(False) - init.bernoulli_(self.weight_ih, p=.5, std=self.input_scaling) - init.dlrb_weights_(self.weight_hh, forward_weight=self.forward_weight, - feedback_weight=self.feedback_weight) - if self.bias: - init.bernoulli_(self.bias_ih, .5, self.bias_scaling) - nn.init.zeros_(self.bias_hh) - - -class SimpleCycleReservoirCell(ESNCell): - r""" - An Echo State Network cell with :math:`\tanh` or :math:`\text{ReLU}` - non-linearity. - - .. math:: - h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h) - - If :attr:`nonlinearity` is ``'relu'``, then ReLU is used instead of tanh. - - Parameters - ---------- - input_size : int - The number of expected features in the input `x`. - hidden_size : int - The number of features in the hidden state `h`. - nonlinearity : str, default = 'tanh' - The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. - bias : bool, default = True - If ``False``, then the layer does not use bias weights `b_ih`. - input_scaling : float, default = 1. - Scales the input weights `w_ih`. - bias_scaling : float, default = 0. - Scales the bias weights `b_ih`. - bias_shift : float, default = 0. - Shifts the bias weights `b_ih`. - leakage : float, default = 1. - Parameter to determine the degree of leaky integration. - forward_weight : float, default = 1. - Scales the forward weights in `w_hh`. - device: Optional[str], default = None - - dtype: Optional, default = None - - - Notes - ----- - Important sizes: - - .. math:: - \begin{aligned} - N ={} & \text{batch size} \\ - L ={} & \text{sequence length} \\ - D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ - H_{in} ={} & \text{input\_size} \\ - H_{out} ={} & \text{hidden\_size} - \end{aligned} - - - Attributes # noqa - ---------- - weight_ih : Tensor, shape = `(hidden_size, input_size)`. - The input-hidden weights of shape `(hidden_size, input_size)`. - weight_hh : Tensor, shape = `(hidden_size, hidden_size)`. - The hidden-hidden weights of shape `(hidden_size, hidden_size)`. - bias_ih : Tensor, shape = `(hidden_size)`. - The learnable input-hidden bias of shape `(hidden_size)`. - """ - - def __init__(self, input_size: int, hidden_size: int, bias: bool = True, - nonlinearity: str = "tanh", input_scaling: float = 1., - bias_scaling: float = 0., bias_shift: float = 0., - leakage: float = 1., forward_weight: float = .9, - device: Optional[str] = None, dtype: Optional = None): - self.forward_weight = forward_weight - super().__init__( - input_size=input_size, hidden_size=hidden_size, bias=bias, - nonlinearity=nonlinearity, input_scaling=input_scaling, - spectral_radius=0., bias_scaling=bias_scaling, - bias_shift=bias_shift, leakage=leakage, input_sparsity=0., - recurrent_sparsity=0., device=device, dtype=dtype) - - def reset_parameters(self) -> None: - """Initialize the weight matrices of the neural network.""" - for weight in self.parameters(): - weight.requires_grad_(False) - init.bernoulli_(self.weight_ih, p=.5, std=self.input_scaling) - init.scr_weights_(self.weight_hh, forward_weight=self.forward_weight) - if self.bias: - init.bernoulli_(self.bias_ih, .5, self.bias_scaling) - nn.init.zeros_(self.bias_hh) diff --git a/src/pyrcn/nn/init.py b/src/pyrcn/nn/init.py deleted file mode 100644 index fafe13f..0000000 --- a/src/pyrcn/nn/init.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Custom weight initialization methods.""" -import torch - - -def antisymmetric_norm_(tensor: torch.Tensor) -> torch.Tensor: - """ - Normalize a given square matrix to be antisymmetric, i.e., by - computing weight - weight.T . - - Parameters - ---------- - tensor : torch.Tensor, shape=(hidden_size, hidden_size) - Tensor (at least 2D) to be normalized. - - Returns - ------- - tensor : torch.Tensor, shape=(hidden_size, hidden_size) - The normalized tensor (at least 2D). - """ - with torch.no_grad(): - tensor = tensor - tensor.T - return tensor - - -def diffusion_norm_(tensor: torch.Tensor, gamma: float) -> torch.Tensor: - r""" - Normalize a given square matrix to be antisymmetric, i.e., by - computing weight - \gamma I . - - Parameters - ---------- - tensor : torch.Tensor, shape=(hidden_size, hidden_size) - Tensor (at least 2D) to be normalized. - - Returns - ------- - tensor : torch.Tensor, shape=(hidden_size, hidden_size) - The normalized tensor (at least 2D). - """ - with torch.no_grad(): - tensor = tensor - gamma * torch.eye(tensor.shape[0]) - return tensor - - -def spectral_norm_(tensor: torch.Tensor) -> torch.Tensor: - """ - Normalize a given square matrix to a unit spectral radius, i.e., to a - maximum absolute eigenvalue of 1. - - Parameters - ---------- - tensor : torch.Tensor, shape=(hidden_size, hidden_size) - Tensor (at least 2D) to be normalized. - - Returns - ------- - tensor : torch.Tensor, shape=(hidden_size, hidden_size) - The normalized tensor (at least 2D). - """ - eigvals = torch.linalg.eigvals(tensor) - with torch.no_grad(): - tensor /= eigvals.abs().max() - return tensor - - -def dlr_weights_(tensor: torch.Tensor, forward_weight: float = 0.9) -> \ - torch.Tensor: - r""" - Fills the 2D input `Tensor` such that the non-zero elements will be a - delay line, and each non-zero element has exactly the same weight value, - as described in `Minimum Complexity Echo State Network` - Rodan, A. (2010). - - Parameters - ---------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`. - forward_weight : float, default = 0.9 - The non-zero weight that is placed in the lower subdiagonal of the - tensor. - - Returns - ------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`, in which the lower subdiagonal is - filled with always the same value. - - Examples - -------- - >>> w = torch.empty(3, 5) - >>> dlr_weights_(w, forward_weight=0.9) - """ - if tensor.ndimension() != 2: - raise ValueError("Only tensors with 2 dimensions are supported") - - rows, cols = tensor.shape - - with torch.no_grad(): - tensor.zero_() - for col_idx in range(1, cols): - tensor[col_idx, col_idx-1] = forward_weight - return tensor - - -def dlrb_weights_(tensor, forward_weight: float = 0.9, - feedback_weight: float = 0.1) -> torch.Tensor: - r""" - Fills the 2D input `Tensor` such that the non-zero elements will be a - delay line with feedback connections, and each non-zero element has - exactly the same weight value, as described in - `Minimum Complexity Echo State Network` - Rodan, A. (2010). - - Parameters - ---------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`. - forward_weight : float, default = 0.9 - The non-zero weight that is placed in the lower subdiagonal of the - tensor. - feedback_weight : float, default = 0.1 - The non-zero weight that is placed in the upper subdiagonal of the - tensor. - - Returns - ------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`, in which the lower and upper - subdiagonals are filled with always the same values. - - Examples - -------- - >>> w = torch.empty(3, 5) - >>> dlrb_weights_(w, forward_weight=0.9, feedback_weight=0.1) - """ - if tensor.ndimension() != 2: - raise ValueError("Only tensors with 2 dimensions are supported") - - rows, cols = tensor.shape - - with torch.no_grad(): - tensor.zero_() - for col_idx in range(1, cols): - tensor[col_idx, col_idx-1] = forward_weight - tensor[col_idx-1, col_idx] = feedback_weight - return tensor - - -def scr_weights_(tensor, forward_weight: float = 0.9) -> torch.Tensor: - r""" - Fills the 2D input `Tensor` such that the non-zero elements will be a - cycle, and each non-zero element has exactly the same weight value, as - described in `Minimum Complexity Echo State Network` - Rodan, A. (2010). - - Parameters - ---------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`. - forward_weight : float, default = 0.9 - The non-zero weight that is placed in the lower subdiagonal of the - tensor. - - Returns - ------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`, in which the lower and upper - subdiagonals are filled with always the same values. - - Examples - -------- - >>> w = torch.empty(3, 5) - >>> scr_weights_(w, forward_weight=0.9) - """ - if tensor.ndimension() != 2: - raise ValueError("Only tensors with 2 dimensions are supported") - - rows, cols = tensor.shape - - with torch.no_grad(): - tensor.zero_() - for col_idx in range(cols): - tensor[col_idx, col_idx-1] = forward_weight - return tensor - - -def bernoulli_(tensor: torch.Tensor, p: float = .5, std: float = 1.) \ - -> torch.Tensor: - r""" - Fills the 2D input `Tensor` as a sparse matrix, where the non-zero elements - will be binary numbers (0 or 1) drawn from a Bernoulli distribution - :math:`\text{Bernoulli}(\texttt{p})` and scaled, as described in - `Minimum Complexity Echo State Network` - Rodan, A. (2010). - - Parameters - ---------- - tensor : torch.Tensor - An n-dimensional `torch.Tensor`. - p : float, default = 0.5 - Probability to be used for drawing the binary random number.The - fraction of elements in each column to be set to zero - std : float, defaul = 1. - The scaling factor for the weight matrix. - - Examples - -------- - >>> w = torch.empty(3, 5) - >>> bernoulli_(w, p=.5, std=1.) - """ - with torch.no_grad(): - tensor.bernoulli_(p) - tensor *= 2*std - tensor -= std - return tensor diff --git a/src/pyrcn/postprocessing/__init__.py b/src/pyrcn/postprocessing/__init__.py index d0ca622..dfc9636 100644 --- a/src/pyrcn/postprocessing/__init__.py +++ b/src/pyrcn/postprocessing/__init__.py @@ -3,7 +3,8 @@ # Authors: Peter Steiner , # License: BSD 3 clause -from ._normal_distribution import NormalDistribution +from __future__ import annotations +from ._normal_distribution import NormalDistribution __all__ = ('NormalDistribution',) diff --git a/src/pyrcn/postprocessing/_normal_distribution.py b/src/pyrcn/postprocessing/_normal_distribution.py index ff1e1fd..2dc780f 100644 --- a/src/pyrcn/postprocessing/_normal_distribution.py +++ b/src/pyrcn/postprocessing/_normal_distribution.py @@ -5,15 +5,15 @@ from __future__ import annotations -from typing import Union, Any +from typing import Any + +import numpy as np import scipy import scipy.stats -import numpy as np - from sklearn.base import BaseEstimator, TransformerMixin -class NormalDistribution(BaseEstimator, TransformerMixin): +class NormalDistribution(TransformerMixin, BaseEstimator): """ Transform an input distribution to a normal distribution. @@ -23,7 +23,7 @@ class NormalDistribution(BaseEstimator, TransformerMixin): Defining number of random variates """ - def __init__(self, size: Union[int, np.integer] = 1): + def __init__(self, size: int | np.integer = 1): """Construct the NormalDistribution.""" self._transformer = scipy.stats.norm self._mean = 0 diff --git a/src/pyrcn/preprocessing/__init__.py b/src/pyrcn/preprocessing/__init__.py index d24151a..c7e4658 100644 --- a/src/pyrcn/preprocessing/__init__.py +++ b/src/pyrcn/preprocessing/__init__.py @@ -3,6 +3,8 @@ # Authors: Peter Steiner , # License: BSD 3 clause +from __future__ import annotations + from ._coates import Coates __all__ = ('Coates',) diff --git a/src/pyrcn/preprocessing/_coates.py b/src/pyrcn/preprocessing/_coates.py index 400b9e4..8a3d860 100644 --- a/src/pyrcn/preprocessing/_coates.py +++ b/src/pyrcn/preprocessing/_coates.py @@ -5,26 +5,22 @@ from __future__ import annotations -import sys -import numpy as np +from collections.abc import Callable +from typing import Literal -from sklearn.base import BaseEstimator, TransformerMixin, ClusterMixin -from sklearn.utils import check_random_state -from sklearn.exceptions import NotFittedError +import numpy as np +from sklearn.base import (BaseEstimator, ClusterMixin, TransformerMixin, + is_clusterer) from sklearn.cluster import KMeans -from sklearn.preprocessing import StandardScaler from sklearn.decomposition import PCA +from sklearn.exceptions import NotFittedError from sklearn.feature_extraction.image import PatchExtractor - -if sys.version_info >= (3, 8): - from typing import Union, Callable, Dict, Tuple, Literal -else: - from typing_extensions import Literal - from typing import Union, Callable, Dict, Tuple +from sklearn.preprocessing import StandardScaler +from sklearn.utils import check_random_state -def inplace_pool_max(X: np.ndarray, axis: Union[None, int, np.integer] = None)\ - -> Union[float, np.ndarray]: +def inplace_pool_max(X: np.ndarray, axis: int | None = None)\ + -> float | np.ndarray: """ Apply max-Pooling on an array. @@ -45,8 +41,8 @@ def inplace_pool_max(X: np.ndarray, axis: Union[None, int, np.integer] = None)\ return np.max(X, axis=axis) -def inplace_pool_min(X: np.ndarray, axis: Union[None, int, np.integer] = None)\ - -> Union[float, np.ndarray]: +def inplace_pool_min(X: np.ndarray, axis: int | None = None)\ + -> float | np.ndarray: """ Apply min-Pooling on an array. @@ -68,8 +64,8 @@ def inplace_pool_min(X: np.ndarray, axis: Union[None, int, np.integer] = None)\ def inplace_pool_average(X: np.ndarray, - axis: Union[None, int, np.integer] = None)\ - -> Union[float, np.ndarray]: + axis: int | None = None)\ + -> float | np.ndarray: """ Apply average-Pooling on an array. @@ -111,13 +107,13 @@ def inplace_pool_mean(X: np.ndarray, axis: None = None) -> np.number: return np.mean(X, axis=axis) -POOLINGS: Dict[str, Callable] = {'max': inplace_pool_max, +POOLINGS: dict[str, Callable] = {'max': inplace_pool_max, 'min': inplace_pool_min, 'average': inplace_pool_average, 'mean': inplace_pool_mean} -class Coates(BaseEstimator, TransformerMixin): +class Coates(TransformerMixin, BaseEstimator): """ Coates Preprocessing. @@ -135,15 +131,15 @@ class Coates(BaseEstimator, TransformerMixin): random_state : Union[None, int, np.random.RandomState], default=None """ - def __init__(self, image_size: Tuple = (), patch_size: Tuple = (), - stride_size: Tuple = (), - n_patches: Union[int, np.integer] = 200, + def __init__(self, image_size: tuple = (), patch_size: tuple = (), + stride_size: tuple = (), + n_patches: int | np.integer = 200, normalize: bool = True, whiten: bool = True, - clusterer: ClusterMixin = KMeans(), + clusterer: ClusterMixin | None = None, pooling_func: Literal['max', 'min', 'average', 'mean'] = 'max', - pooling_size: Tuple = (), - random_state: Union[None, int, np.random.RandomState] = None): + pooling_size: tuple = (), + random_state: None | int | np.random.RandomState = None): """Construct the Coates.""" self.image_size = image_size self.patch_size = patch_size @@ -154,7 +150,7 @@ def __init__(self, image_size: Tuple = (), patch_size: Tuple = (), self.clusterer = clusterer self.pooling_func = pooling_func self.pooling_size = pooling_size - self.random_state = check_random_state(random_state) + self.random_state = random_state self._normalizer = StandardScaler() self._whitener = PCA(whiten=True) @@ -172,6 +168,8 @@ def fit(self, X: np.ndarray, y: None = None) -> Coates: ------- self : returns a trained Coates. """ + if self.clusterer is None: + self.clusterer = KMeans() self._validate_hyperparameters() self.clusterer.fit(self._preprocessing(Coates._extract_random_patches( X, image_size=self.image_size, patch_size=self.patch_size, @@ -192,6 +190,7 @@ def transform(self, X: np.ndarray, y: None = None) -> np.ndarray: ------- features : returns the transformed features. """ + assert self.clusterer is not None # patches[#samples][#patches][#features] patches = Coates._extract_equidistant_patches( X, image_size=self.image_size, patch_size=self.patch_size, @@ -223,6 +222,7 @@ def inverse_transform(self, X: np.ndarray) -> np.ndarray: ------- patches : returns the original features. """ + assert self.clusterer is not None patch_array = Coates._reshape_arrays_to_images(X, image_size=( int(X.shape[-1] / self.clusterer.cluster_centers_.shape[0]), self.clusterer.cluster_centers_.shape[0])) @@ -242,23 +242,23 @@ def _validate_hyperparameters(self) -> None: """ if len(self.patch_size) not in {2, 3}: - raise ValueError('patch_size has invalid format, got {0}' + raise ValueError('patch_size has invalid format, got {}' .format(self.patch_size)) if len(self.stride_size) != len(self.patch_size): - print('stride_size has invalid format, got {0}. ' + print('stride_size has invalid format, got {}. ' 'Set stride_size = patch_size '.format(self.stride_size)) self.stride_size = self.patch_size if any(stride < patch for stride, patch in zip(self.stride_size, self.patch_size)): raise ValueError('stride_size must be greater or equal than ' - 'patch_size, got stride_size = {0}, patch_size ' - '= {1}'.format(self.stride_size, self.patch_size)) + 'patch_size, got stride_size = {}, patch_size ' + '= {}'.format(self.stride_size, self.patch_size)) if self.pooling_func not in POOLINGS: - raise ValueError("The pooling_func '{0}' is not supported. " - "Supported activations are {1}." + raise ValueError("The pooling_func '{}' is not supported. " + "Supported activations are {}." .format(self.pooling_func, POOLINGS)) if any(patches < pool for patches, pool in @@ -266,16 +266,16 @@ def _validate_hyperparameters(self) -> None: self.stride_size), self.pooling_size)): raise ValueError('#patches must be greater or equal than pooling_' - 'size, got patch_size = {0}, pooling_size = {1}' + 'size, got patch_size = {}, pooling_size = {}' .format(self.patch_size, self.pooling_size)) - if getattr(self.clusterer, "_estimator_type", None) != "clusterer": - raise TypeError('clusterer must be of type clusterer, got {0}' + if not is_clusterer(self.clusterer): + raise TypeError('clusterer must be of type clusterer, got {}' .format(self.clusterer)) @staticmethod def _reshape_arrays_to_images(X: np.ndarray, - image_size: Tuple) -> np.ndarray: + image_size: tuple) -> np.ndarray: """ Reshape an array to image. @@ -295,7 +295,7 @@ def _reshape_arrays_to_images(X: np.ndarray, @staticmethod def _reshape_images_to_arrays(X: np.ndarray, - image_size: Tuple) -> np.ndarray: + image_size: tuple) -> np.ndarray: """ Reshape an image to array. @@ -314,7 +314,7 @@ def _reshape_images_to_arrays(X: np.ndarray, return X.reshape(index_dimensions + (int(np.prod(image_size)), )) @staticmethod - def _patches_per_image(image_size: Tuple, stride_size: Tuple) -> Tuple: + def _patches_per_image(image_size: tuple, stride_size: tuple) -> tuple: """ Compute tuple strides fitting in image. @@ -332,11 +332,10 @@ def _patches_per_image(image_size: Tuple, stride_size: Tuple) -> Tuple: return image_size[0] // stride_size[0], image_size[1] // stride_size[1] @staticmethod - def _extract_random_patches(X: np.ndarray, image_size: Tuple, - patch_size: Tuple, - n_patches: Union[int, np.integer], - random_state: Union[ - None, int, np.random.RandomState] = None)\ + def _extract_random_patches( + X: np.ndarray, image_size: tuple, patch_size: tuple, + n_patches: int | np.integer, + random_state: None | int | np.random.RandomState = None)\ -> np.ndarray: """ Extract random patches from image array. @@ -368,9 +367,9 @@ def _extract_random_patches(X: np.ndarray, image_size: Tuple, return Coates._reshape_images_to_arrays(random_patches, patch_size) @staticmethod - def _extract_equidistant_patches(X: np.ndarray, image_size: Tuple, - patch_size: Tuple, - stride_size: Tuple) -> np.ndarray: + def _extract_equidistant_patches(X: np.ndarray, image_size: tuple, + patch_size: tuple, + stride_size: tuple) -> np.ndarray: """ Extract equidistant patches from image array. @@ -465,6 +464,7 @@ def _pooling(self, X: np.ndarray) -> np.ndarray: nm_patches = Coates._patches_per_image(image_size=self.image_size, stride_size=self.stride_size) + assert self.clusterer is not None # feature_pools[#samples][#features][#pools][#pool_features] feature_pools = Coates._extract_equidistant_patches( np.transpose(X, axes=(0, 2, 1)), diff --git a/src/pyrcn/projection/__init__.py b/src/pyrcn/projection/__init__.py index eaaae3b..02c491a 100644 --- a/src/pyrcn/projection/__init__.py +++ b/src/pyrcn/projection/__init__.py @@ -3,7 +3,8 @@ # Authors: Peter Steiner , # License: BSD 3 clause -from ._value_projection import MatrixToValueProjection +from __future__ import annotations +from ._value_projection import MatrixToValueProjection __all__ = ('MatrixToValueProjection',) diff --git a/src/pyrcn/projection/_value_projection.py b/src/pyrcn/projection/_value_projection.py index 538af6c..2718c82 100644 --- a/src/pyrcn/projection/_value_projection.py +++ b/src/pyrcn/projection/_value_projection.py @@ -4,19 +4,14 @@ # License: BSD 3 clause from __future__ import annotations -import sys + +from typing import Literal, cast import numpy as np from sklearn.base import BaseEstimator, TransformerMixin -if sys.version_info >= (3, 8): - from typing import Literal, cast -else: - from typing_extensions import Literal - from typing import cast - -class MatrixToValueProjection(BaseEstimator, TransformerMixin): +class MatrixToValueProjection(TransformerMixin, BaseEstimator): """ Projection of a matrix to any kind of indices, e.g. of the maximum value. diff --git a/src/pyrcn/util/__init__.py b/src/pyrcn/util/__init__.py index 621bfdf..3909332 100644 --- a/src/pyrcn/util/__init__.py +++ b/src/pyrcn/util/__init__.py @@ -4,10 +4,11 @@ # Michael Schindler # License: BSD 3 clause -from ._util import ( - new_logger, get_mnist, argument_parser, concatenate_sequences, - value_to_tuple, batched) +from __future__ import annotations + from ._feature_extractor import FeatureExtractor +from ._util import (argument_parser, batched, concatenate_sequences, get_mnist, + new_logger, value_to_tuple) __all__ = ('new_logger', 'get_mnist', 'argument_parser', 'FeatureExtractor', 'concatenate_sequences', 'value_to_tuple', 'batched') diff --git a/src/pyrcn/util/_feature_extractor.py b/src/pyrcn/util/_feature_extractor.py index ca5067e..159f38c 100644 --- a/src/pyrcn/util/_feature_extractor.py +++ b/src/pyrcn/util/_feature_extractor.py @@ -5,7 +5,7 @@ from __future__ import annotations -from typing import Union, Callable, Dict, Optional +from collections.abc import Callable import numpy as np from sklearn.preprocessing import FunctionTransformer @@ -39,14 +39,14 @@ class FeatureExtractor(FunctionTransformer): """ - def __init__(self, func: Union[Callable, None], - kw_args: Union[Dict, None] = None): + def __init__(self, func: Callable | None, + kw_args: dict | None = None): """Construct the FeatureExtractor.""" super().__init__(func=func, inverse_func=None, validate=False, accept_sparse=False, check_inverse=False, kw_args=kw_args, inv_kw_args=None) - def fit(self, X: Union[str, np.ndarray], y: Optional[np.ndarray] = None)\ + def fit(self, X: str | np.ndarray, y: np.ndarray | None = None)\ -> FeatureExtractor: """ Fit transformer by checking X. @@ -61,7 +61,7 @@ def fit(self, X: Union[str, np.ndarray], y: Optional[np.ndarray] = None)\ super().fit(X=X, y=y) return self - def transform(self, X: Union[str, np.ndarray]) -> np.ndarray: + def transform(self, X: str | np.ndarray) -> np.ndarray: """ Transform X using the forward function. diff --git a/src/pyrcn/util/_util.py b/src/pyrcn/util/_util.py index e53ff49..f6bb835 100644 --- a/src/pyrcn/util/_util.py +++ b/src/pyrcn/util/_util.py @@ -4,20 +4,19 @@ # Michael Schindler # License: BSD 3 clause -import sys -from typing import Union, Tuple, Iterable +from __future__ import annotations -import random -import os -import torch -import logging import argparse -import numpy as np +from collections.abc import Iterable, Iterator from itertools import islice +import logging +import os +import random +import sys -from sklearn.utils import check_X_y, check_consistent_length +import numpy as np from sklearn.datasets import fetch_openml - +from sklearn.utils import check_consistent_length, check_X_y argument_parser = argparse.ArgumentParser( description='Standard input parser for HPC on PyRCN.') @@ -33,7 +32,7 @@ ) -def batched(iterable: Iterable, n: int) -> Tuple: +def batched(iterable: Iterable, n: int) -> Iterator[tuple]: """ Iterate over batches of size n. @@ -64,9 +63,8 @@ def batched(iterable: Iterable, n: int) -> Tuple: yield batch -def value_to_tuple(value: Union[float, int], - size: Union[float, int, Tuple[Union[float, int], ...]]) \ - -> Tuple[Union[float, int], ...]: +def value_to_tuple(value: float | int | tuple[float | int, ...], + size: int) -> tuple[float | int, ...]: """ Convert a value to a tuple of values. @@ -82,10 +80,9 @@ def value_to_tuple(value: Union[float, int], value : Tuple[Union[float, int], ...] Tuple of values. """ - if isinstance(value, float) or isinstance(value, int): + if isinstance(value, (float, int)): return (value, ) * size - elif isinstance(value, Tuple): - return value + return value def seed_everything(seed: int = 42) -> None: @@ -100,10 +97,6 @@ def seed_everything(seed: int = 42) -> None: random.seed(seed) os.environ['PYTHONHASHSEED'] = str(seed) np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False def new_logger(name: str, directory: str = os.getcwd()) -> logging.Logger: @@ -113,13 +106,13 @@ def new_logger(name: str, directory: str = os.getcwd()) -> logging.Logger: formatter = logging.Formatter( fmt='%(asctime)s %(levelname)s %(name)s %(message)s') handler = logging.FileHandler( - os.path.join(directory, '{0}.log'.format(name))) + os.path.join(directory, f'{name}.log')) handler.setFormatter(formatter) logger.addHandler(handler) return logger -def get_mnist(directory: str = os.getcwd()) -> Tuple[np.ndarray, np.ndarray]: +def get_mnist(directory: str = os.getcwd()) -> tuple[np.ndarray, np.ndarray]: """Load the MNIST dataset from harddisk.""" npzfilepath = os.path.join(directory, 'MNIST.npz') @@ -135,10 +128,10 @@ def get_mnist(directory: str = os.getcwd()) -> Tuple[np.ndarray, np.ndarray]: return X, y -def concatenate_sequences(X: Union[list, np.ndarray], - y: Union[list, np.ndarray], +def concatenate_sequences(X: list | np.ndarray, + y: list | np.ndarray, sequence_to_value: bool = False)\ - -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + -> tuple[np.ndarray, np.ndarray, np.ndarray]: """ Concatenate multiple sequences to scikit-learn compatible numpy arrays. diff --git a/tests/test_activations.py b/tests/test_activations.py index b2a1298..95e729f 100644 --- a/tests/test_activations.py +++ b/tests/test_activations.py @@ -1,6 +1,9 @@ """Testing for activation functions in the module (pyrcn.base).""" +from __future__ import annotations + import numpy as np from scipy.special import expit as logistic_sigmoid + from pyrcn.base import (ACTIVATIONS, ACTIVATIONS_INVERSE, ACTIVATIONS_INVERSE_BOUNDS) diff --git a/tests/test_classification_metrics.py b/tests/test_classification_metrics.py index 68b5ed5..f138161 100644 --- a/tests/test_classification_metrics.py +++ b/tests/test_classification_metrics.py @@ -1,12 +1,14 @@ """Testing for Metrics module.""" -import pyrcn.metrics +from __future__ import annotations + +import numpy as np +import pytest import sklearn.metrics from sklearn.datasets import (make_classification, make_multilabel_classification) -import numpy as np -import pytest +import pyrcn.metrics rng_true = np.random.RandomState(42) rng_pred = np.random.RandomState(1234) diff --git a/tests/test_coates.py b/tests/test_coates.py index 5847e48..3738f14 100644 --- a/tests/test_coates.py +++ b/tests/test_coates.py @@ -1,12 +1,12 @@ """Testing for coates preprocessing module (pyrcn.preprocessing.coates).""" -import numpy as np +from __future__ import annotations -from sklearn.datasets import load_digits +import numpy as np from sklearn.cluster import KMeans +from sklearn.datasets import load_digits from pyrcn.preprocessing import Coates - X_digits, y_digits = load_digits(return_X_y=True) diff --git a/tests/test_elm.py b/tests/test_elm.py index 2a2668d..18fe016 100644 --- a/tests/test_elm.py +++ b/tests/test_elm.py @@ -1,19 +1,19 @@ """Testing for Extreme Learning Machine module.""" +from __future__ import annotations + import numpy as np import pytest -from sklearn.datasets import load_iris, load_digits -from sklearn.model_selection import train_test_split -from sklearn.model_selection import GridSearchCV -from sklearn.pipeline import FeatureUnion -from sklearn.linear_model import Ridge -from sklearn.exceptions import NotFittedError from sklearn.base import clone +from sklearn.datasets import load_digits, load_iris +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import Ridge +from sklearn.model_selection import GridSearchCV, train_test_split +from sklearn.pipeline import FeatureUnion from pyrcn.base.blocks import InputToNode -from pyrcn.linear_model import IncrementalRegression from pyrcn.extreme_learning_machine import ELMClassifier, ELMRegressor - +from pyrcn.linear_model import IncrementalRegression X_iris, y_iris = load_iris(return_X_y=True) @@ -64,9 +64,9 @@ def test_elm_regressor_jobs() -> None: elm = GridSearchCV(ELMRegressor(), param_grid) elm.fit(X_train.reshape(-1, 1), y_train, n_jobs=2) y_elm = elm.predict(X_test.reshape(-1, 1)) - print("tests - elm:\n sin | cos \n {0}".format(y_test-y_elm)) - print("best_params_: {0}".format(elm.best_params_)) - print("best_score: {0}".format(elm.best_score_)) + print(f"tests - elm:\n sin | cos \n {y_test-y_elm}") + print(f"best_params_: {elm.best_params_}") + print(f"best_score: {elm.best_score_}") np.testing.assert_allclose(y_test, y_elm, atol=1e-1) @@ -89,15 +89,15 @@ def test_elm_regressor_chunk() -> None: elm = GridSearchCV(ELMRegressor(), param_grid) elm.fit(X_train.reshape(-1, 1), y_train, n_jobs=2) y_elm = elm.predict(X_test.reshape(-1, 1)) - print("tests - elm:\n sin | cos \n {0}".format(y_test-y_elm)) - print("best_params_: {0}".format(elm.best_params_)) - print("best_score: {0}".format(elm.best_score_)) + print(f"tests - elm:\n sin | cos \n {y_test-y_elm}") + print(f"best_params_: {elm.best_params_}") + print(f"best_score: {elm.best_score_}") np.testing.assert_allclose(y_test, y_elm, atol=1e-1) elm.fit(X_train.reshape(-1, 1), y_train) y_elm = elm.predict(X_test.reshape(-1, 1)) - print("tests - elm:\n sin | cos \n {0}".format(y_test-y_elm)) - print("best_params_: {0}".format(elm.best_params_)) - print("best_score: {0}".format(elm.best_score_)) + print(f"tests - elm:\n sin | cos \n {y_test-y_elm}") + print(f"best_params_: {elm.best_params_}") + print(f"best_score: {elm.best_score_}") np.testing.assert_allclose(y_test, y_elm, atol=1e-1) with pytest.raises(ValueError): elm = clone(elm.best_estimator_).set_params(chunk_size=-1) @@ -130,12 +130,12 @@ def test_iris_ensemble_iterative_regression() -> None: y_predicted = cls.predict(X_test) for record in range(len(y_test)): - print('predicted: {0} \ttrue: {1}' + print('predicted: {} \ttrue: {}' .format(y_predicted[record], y_test[record])) - print('score: {0}'.format(cls.score(X_test, y_test))) - print('proba: {0}'.format(cls.predict_proba(X_test))) - print('log_proba: {0}'.format(cls.predict_log_proba(X_test))) + print(f'score: {cls.score(X_test, y_test)}') + print(f'proba: {cls.predict_proba(X_test)}') + print(f'log_proba: {cls.predict_log_proba(X_test)}') assert cls.score(X_test, y_test) >= 4./5. diff --git a/tests/test_esn.py b/tests/test_esn.py index a9dfcc3..41b5d50 100644 --- a/tests/test_esn.py +++ b/tests/test_esn.py @@ -1,16 +1,18 @@ """Testing for Echo State Network module.""" +from __future__ import annotations + import numpy as np import pytest -from pyrcn.datasets import mackey_glass, load_digits -from sklearn.metrics import make_scorer -from sklearn.model_selection import train_test_split -from sklearn.model_selection import GridSearchCV, TimeSeriesSplit -from sklearn.linear_model import Ridge from sklearn.exceptions import NotFittedError +from sklearn.linear_model import Ridge +from sklearn.metrics import make_scorer +from sklearn.model_selection import (GridSearchCV, TimeSeriesSplit, + train_test_split) from pyrcn.base.blocks import InputToNode, NodeToNode +from pyrcn.datasets import load_digits, mackey_glass +from pyrcn.echo_state_network import ESNClassifier, ESNRegressor from pyrcn.linear_model import IncrementalRegression -from pyrcn.echo_state_network import ESNRegressor, ESNClassifier from pyrcn.metrics import mean_squared_error @@ -45,9 +47,9 @@ def test_esn_regressor_jobs() -> None: esn = GridSearchCV(estimator=ESNRegressor(), param_grid=param_grid) esn.fit(X_train.reshape(-1, 1), y_train, n_jobs=2) y_esn = esn.predict(X_test.reshape(-1, 1)) - print("tests - esn:\n sin | cos \n {0}".format(y_test-y_esn)) - print("best_params_: {0}".format(esn.best_params_)) - print("best_score: {0}".format(esn.best_score_)) + print(f"tests - esn:\n sin | cos \n {y_test-y_esn}") + print(f"best_params_: {esn.best_params_}") + print(f"best_score: {esn.best_score_}") np.testing.assert_allclose(1, esn.best_score_, atol=1e-1) diff --git a/tests/test_incremental_regression.py b/tests/test_incremental_regression.py index e42d54f..02dd144 100644 --- a/tests/test_incremental_regression.py +++ b/tests/test_incremental_regression.py @@ -1,16 +1,16 @@ """Testing for Linear model module.""" +from __future__ import annotations + import numpy as np import pytest - from sklearn.base import is_regressor from sklearn.datasets import load_diabetes -from sklearn.model_selection import train_test_split from sklearn.exceptions import NotFittedError - -from pyrcn.linear_model import IncrementalRegression from sklearn.linear_model import Ridge +from sklearn.model_selection import train_test_split +from pyrcn.linear_model import IncrementalRegression X_diabetes, y_diabetes = load_diabetes(return_X_y=True) @@ -50,7 +50,7 @@ def test_postpone_inverse() -> None: reg.partial_fit(X, y) y_reg = reg.predict(X_test) - print("tests: {0}\nregr: {1}".format(y_test, y_reg)) + print(f"tests: {y_test}\nregr: {y_reg}") np.testing.assert_allclose(y_reg, y_test, rtol=.01, atol=.15) @@ -73,7 +73,7 @@ def test_linear() -> None: reg.partial_fit(X[prt, :], y[prt, :]) y_reg = reg.predict(X_test) - print("tests: {0}\nregr: {1}".format(y_test, y_reg)) + print(f"tests: {y_test}\nregr: {y_reg}") np.testing.assert_allclose(y_reg, y_test, rtol=.01, atol=.15) @@ -84,5 +84,5 @@ def test_compare_ridge() -> None: i_reg = IncrementalRegression(alpha=.01).fit(X_train, y_train) ridge = Ridge(alpha=.01, solver='svd').fit(X_train, y_train) - print("incremental: {0} ridge: {1}".format(i_reg.coef_, ridge.coef_)) + print(f"incremental: {i_reg.coef_} ridge: {ridge.coef_}") np.testing.assert_allclose(i_reg.coef_, ridge.coef_, rtol=.0001) diff --git a/tests/test_input_to_node.py b/tests/test_input_to_node.py index 253b5cd..0ecae37 100644 --- a/tests/test_input_to_node.py +++ b/tests/test_input_to_node.py @@ -1,11 +1,13 @@ """Testing for blocks.input_to_node module.""" -import scipy +from __future__ import annotations + import numpy as np import pytest +import scipy from sklearn.utils.extmath import safe_sparse_dot -from pyrcn.base.blocks import (InputToNode, PredefinedWeightsInputToNode, - BatchIntrinsicPlasticity) +from pyrcn.base.blocks import (BatchIntrinsicPlasticity, InputToNode, + PredefinedWeightsInputToNode) def test_input_to_node_invalid_bias_scaling() -> None: diff --git a/tests/test_model_selection.py b/tests/test_model_selection.py index 1e33415..7c60616 100644 --- a/tests/test_model_selection.py +++ b/tests/test_model_selection.py @@ -1,12 +1,15 @@ """Testing for model selection module.""" +from __future__ import annotations + +from collections.abc import Iterable + +import pytest from sklearn import datasets -from sklearn.model_selection import KFold, GridSearchCV, RandomizedSearchCV +from sklearn.model_selection import GridSearchCV, KFold, RandomizedSearchCV from sklearn.svm import SVC -from collections.abc import Iterable from pyrcn.model_selection import SequentialSearchCV, SHGOSearchCV -import pytest def test_sequentialSearchCV_equivalence() -> None: @@ -43,9 +46,9 @@ def test_sequentialSearchCV_equivalence() -> None: @pytest.mark.skip(reason="no way of currently testing this") def test_SHGOSearchCV() -> None: """Test the SHGO search.""" - from sklearn.metrics import accuracy_score - from sklearn.base import clone, BaseEstimator import numpy as np + from sklearn.base import BaseEstimator, clone + from sklearn.metrics import accuracy_score from sklearn.model_selection import StratifiedKFold iris = datasets.load_iris() X = iris.data[:, [0, 2]] diff --git a/tests/test_node_to_node.py b/tests/test_node_to_node.py index 0baf8ae..916aacd 100644 --- a/tests/test_node_to_node.py +++ b/tests/test_node_to_node.py @@ -1,10 +1,12 @@ """Testing for blocks.node_to_node module.""" +from __future__ import annotations + import numpy as np import pytest from sklearn.utils.extmath import safe_sparse_dot -from pyrcn.base.blocks import ( - InputToNode, NodeToNode,PredefinedWeightsNodeToNode, HebbianNodeToNode) +from pyrcn.base.blocks import (HebbianNodeToNode, InputToNode, NodeToNode, + PredefinedWeightsNodeToNode) def test_input_to_node_invalid_spectral_radius() -> None: diff --git a/tests/test_regression_metrics.py b/tests/test_regression_metrics.py index 76c9567..5dc073e 100644 --- a/tests/test_regression_metrics.py +++ b/tests/test_regression_metrics.py @@ -1,11 +1,13 @@ """Testing for Metrics module.""" -import pyrcn.metrics -import sklearn.metrics -from sklearn.datasets import make_regression +from __future__ import annotations + import numpy as np import pytest +import sklearn.metrics +from sklearn.datasets import make_regression +import pyrcn.metrics rng_true = np.random.RandomState(42) rng_pred = np.random.RandomState(1234) diff --git a/tests/test_util.py b/tests/test_util.py index 455cbb5..1210ad0 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,7 +1,11 @@ """Testing for pyrcn.utils module""" +from __future__ import annotations + import os + import pytest -from pyrcn.util import new_logger, argument_parser, get_mnist + +from pyrcn.util import argument_parser, get_mnist, new_logger def test_new_logger() -> None: diff --git a/tests/test_value_projection.py b/tests/test_value_projection.py index 32f3ebb..7d07325 100644 --- a/tests/test_value_projection.py +++ b/tests/test_value_projection.py @@ -1,6 +1,9 @@ """Testing for projection module (pyrcn.projection).""" +from __future__ import annotations + import numpy as np + from pyrcn.projection import MatrixToValueProjection