From b08d142191a3a64def2a2ba7835a8e5162c5ffb4 Mon Sep 17 00:00:00 2001 From: Stefano Braghin <527806+stefano81@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:18:05 +0100 Subject: [PATCH] feat: improve documentation Signed-off-by: Stefano Braghin <527806+stefano81@users.noreply.github.com> --- CONTRIBUTING.md | 99 ++++++++++++ README.md | 30 ++-- pyproject.toml | 7 +- src/risk_assessment/__init__.py | 9 ++ src/risk_assessment/anonymization/__init__.py | 112 +++++++++++++ src/risk_assessment/anonymization/mondrian.py | 81 ++++++++++ .../optimal_lattice_anonymization.py | 82 ++++++++++ src/risk_assessment/masking/__init__.py | 56 +++++++ src/risk_assessment/masking/actions.py | 112 +++++++++++++ src/risk_assessment/metrics/__init__.py | 17 ++ .../metrics/informationloss/__init__.py | 150 ++++++++++++++++++ .../metrics/uniqueness_estimation/__init__.py | 35 ++++ src/risk_assessment/readi/__init__.py | 14 ++ .../readi/sentence_tokenizer.py | 82 +++++++++- src/risk_assessment/readi/text_tokenizer.py | 82 +++++++++- src/risk_assessment/vulnerability/__init__.py | 13 ++ .../vulnerability/brute/__init__.py | 60 ++++++- .../vulnerability/ducc/ducc.py | 53 +++++++ .../ducc/ducc_worker_descending.py | 30 ++++ .../vulnerability/ducc/pruned_graphs.py | 27 ++++ 20 files changed, 1128 insertions(+), 23 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 src/risk_assessment/metrics/__init__.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..64d5615 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,99 @@ +# Contributing to READI + +Thank you for your interest in contributing! This guide covers everything you need to get started. + +--- + +## Development Setup + +**Prerequisites:** Python 3.11+, [uv](https://docs.astral.sh/uv/), Git with [git-lfs](https://git-lfs.com/). + +```bash +git clone https://github.com/IBM/READI.git +cd READI + +# Create virtual environment and install all dependencies (including dev extras) +uv venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +uv pip install -e ".[dev]" + +# Install pre-commit hooks +prek install +``` + +--- + +## Running Tests + +```bash +# Run the full test suite +pytest + +# Run a specific test file +pytest tests/readi/test_analyzer.py -v + +# Run with coverage report +pytest --cov=risk_assessment --cov-report=html +``` + +--- + +## Linting and Formatting + +The project uses [Ruff](https://docs.astral.sh/ruff/) for linting and formatting. + +```bash +# Check for lint errors +ruff check src/ tests/ + +# Auto-fix lint errors +ruff check --fix src/ tests/ + +# Format code +ruff format src/ tests/ +``` + +Pre-commit hooks run these checks automatically on every commit. To run them manually: + +```bash +pre-commit run --all-files +``` + +--- + +## Code Style + +- Follow existing patterns in the surrounding code. +- All public classes, methods, and module-level functions must have docstrings. +- Use Google-style docstrings with `Args:`, `Returns:`, and `Raises:` sections where applicable. +- Type hints are required on all function signatures. +- Line length limit is 120 characters (enforced by Ruff). + +--- + +## Pull Request Process + +1. **Branch** off `main` using a descriptive name, e.g. `feat/italian-fiscal-code` or `fix/email-regex`. +2. **Write tests** for any new functionality. PRs without tests for new features will not be merged. +3. **Ensure all checks pass** locally before opening a PR: + ```bash + prek run --all-files + pytest + ``` +4. **Open a Pull Request** against `main`. Fill in the PR template, linking any related issues. +5. At least one maintainer review is required before merging. + +--- + +## Reporting Issues + +Use [GitHub Issues](https://github.com/IBM/READI/issues) to report bugs or request features. Please include: +- A minimal reproducible example for bugs. +- The Python version and OS you are using. +- The full error traceback if applicable. + +--- + +## License + +By contributing, you agree that your contributions will be licensed under the [Apache License 2.0](LICENSE). diff --git a/README.md b/README.md index 40f81a5..7868afc 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,12 @@ READI augments the functionalities provided by [IBM Data Privacy Toolkit](https: ### Installation -**Recommended: Using uv (10-100x faster)** +**Simplest: install from PyPI** +```bash +pip install readi-privacy +``` + +**Using uv (10-100x faster)** ```bash # Install uv if you haven't already curl -LsSf https://astral.sh/uv/install.sh | sh @@ -46,15 +51,10 @@ uv venv source .venv/bin/activate # On Windows: .venv\Scripts\activate # Install READI -uv pip install git+https://github.com/IBM/READI.git -``` - -**Standard Installation with pip:** -```bash -pip install git+https://github.com/IBM/READI.git +uv pip install readi-privacy ``` -**Clone Repository:** +**Install from source:** ```bash git clone https://github.com/IBM/READI.git cd READI @@ -74,9 +74,8 @@ For contributors and developers: **Recommended: Using uv** ```bash -# Install in editable mode with development dependencies -uv pip install -e . -uv pip install -r requirements-dev.txt +# Install in editable mode with all development dependencies +uv pip install -e ".[dev]" # Set up pre-commit hooks (recommended) pre-commit install @@ -84,9 +83,8 @@ pre-commit install **Alternative: Using pip** ```bash -# Install in editable mode with development dependencies -pip install -e . -pip install -r requirements-dev.txt +# Install in editable mode with all development dependencies +pip install -e ".[dev]" # Set up pre-commit hooks (recommended) pre-commit install @@ -138,7 +136,7 @@ Explore our comprehensive Jupyter notebooks in the [`notebooks/`](./notebooks) d ## 📖 Documentation -For detailed documentation, API references, and advanced usage patterns, please visit our [documentation portal](https://github.com/IBM/READI/docs) *(coming soon)*. +For detailed documentation, API references, and advanced usage patterns, please refer to the [`notebooks/`](./notebooks) directory and the inline docstrings throughout the source code. A hosted documentation portal is planned for a future release. --- @@ -167,7 +165,7 @@ If you use READI in academic work, please cite the most relevant publication fro @software{readi_ibm, title = {READI: Risk Evaluation and De-Identification}, author = {Stefano Braghin and Liubov Nedoshivina and Anisa Halimi and Naoise Holohan and Kieran Fraser}, - year = {2026}, + year = {2025}, url = {https://github.com/IBM/READI} } ``` diff --git a/pyproject.toml b/pyproject.toml index 632eb6c..c407660 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "readi-privacy" dynamic = ["version"] -description = "A collection of funcionality to perform data classification, data privacy risk assessment, and enforce mitigation" +description = "A collection of functionality to perform data classification, data privacy risk assessment, and enforce mitigation" readme = "README.md" requires-python = ">=3.11,<3.14" authors = [ @@ -16,13 +16,14 @@ classifiers = [ "Development Status :: 4 - Beta", # Indicate who your project is intended for "Intended Audience :: Developers", - "Topic :: Software Development :: Build Tools", + "Intended Audience :: Science/Research", + "Topic :: Security", + "Topic :: Scientific/Engineering :: Artificial Intelligence", # Specify the Python versions you support here. "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", ] dependencies = [ "datasets==5.0.0", diff --git a/src/risk_assessment/__init__.py b/src/risk_assessment/__init__.py index e69de29..138149e 100644 --- a/src/risk_assessment/__init__.py +++ b/src/risk_assessment/__init__.py @@ -0,0 +1,9 @@ +"""READI — Risk Evaluation and De-Identification. + +Top-level package exposing the primary public API. For most use-cases, +import directly from the relevant sub-package:: + + from risk_assessment.readi.analyzer import READIAnalyzer + from risk_assessment.anonymization.mondrian import Mondrian, MondrianOptions + from risk_assessment.anonymization import KAnonymity +""" diff --git a/src/risk_assessment/anonymization/__init__.py b/src/risk_assessment/anonymization/__init__.py index a624bc7..b847927 100644 --- a/src/risk_assessment/anonymization/__init__.py +++ b/src/risk_assessment/anonymization/__init__.py @@ -1,3 +1,22 @@ +"""Privacy constraints and base classes for data anonymization algorithms. + +This module provides the foundational abstractions for dataset anonymization, +including the abstract base classes for anonymization algorithms and privacy +constraints, as well as concrete implementations of the most common privacy +models: k-Anonymity, l-Diversity (Distinct and Entropy variants), and +t-Closeness. + +Typical usage:: + + from risk_assessment.anonymization import KAnonymity, TCloseness + from risk_assessment.anonymization.mondrian import Mondrian, MondrianOptions + + constraints = [KAnonymity(k=5), TCloseness(t=0.2)] + options = MondrianOptions(privacy_constraints=constraints) + mondrian = Mondrian(options) + anonymized_df, report = mondrian.anonymize(df, column_information) +""" + import math from abc import ABC, abstractmethod from dataclasses import dataclass @@ -12,35 +31,96 @@ @dataclass class AnonymizationReport: + """Result report produced by an anonymization algorithm. + + Attributes: + anonymized: Whether the dataset was successfully anonymized. + suppression_rate: Fraction of rows suppressed (0.0–100.0). Defaults to 0.0. + generalization_levels: Per-column generalization levels applied, if available. + """ + anonymized: bool suppression_rate: float = 0.0 generalization_levels: list[int] | None = None class AnonymizationAlgorithm(ABC): + """Abstract base class for anonymization algorithms. + + Subclasses implement the ``anonymize`` method to transform a dataset so that + it satisfies the configured privacy constraints. + """ + @abstractmethod def anonymize( self, dataset: DataFrame, column_information: list[ColumnInformation] ) -> tuple[DataFrame, AnonymizationReport]: + """Anonymize the dataset. + + Args: + dataset: The input DataFrame to anonymize. + column_information: Metadata describing each column's type, class, + and associated hierarchy or range. + + Returns: + A tuple of ``(anonymized_dataset, report)`` where ``report`` + summarises whether anonymization succeeded and at what cost. + """ pass class PrivacyConstraint(ABC): + """Abstract base class for privacy constraints. + + A privacy constraint defines the condition that each equivalence class + (partition) in an anonymized dataset must satisfy. + """ + @abstractmethod def check(self, dataset: DataFrame, column_information: list[ColumnInformation]) -> bool: + """Check whether the partition satisfies this constraint. + + Args: + dataset: A single equivalence-class partition of the full dataset. + column_information: Metadata describing each column. + + Returns: + True if the constraint is satisfied, False otherwise. + """ pass @dataclass class KAnonymity(PrivacyConstraint): + """k-Anonymity privacy constraint. + + A partition satisfies k-Anonymity when it contains at least *k* records, + ensuring that each individual is indistinguishable from at least k-1 others + with respect to the quasi-identifier attributes. + + Attributes: + k: Minimum required equivalence-class size. + """ + k: int def check(self, dataset: DataFrame, column_information: list[ColumnInformation]) -> bool: + """Return True if the partition has at least k records.""" return len(dataset) >= self.k @dataclass class DistinctLDiversity(PrivacyConstraint): + """Distinct l-Diversity privacy constraint. + + A partition satisfies Distinct l-Diversity when every sensitive attribute + column contains at least *l* distinct values, limiting the ability to infer + a specific sensitive value for any individual. + + Attributes: + l: Minimum number of distinct sensitive values required per partition. + """ + l: int # noqa: E741 def check(self, dataset: DataFrame, column_information: list[ColumnInformation]) -> bool: @@ -59,6 +139,17 @@ def check(self, dataset: DataFrame, column_information: list[ColumnInformation]) @dataclass class EntropyLDiversity(PrivacyConstraint): + """Entropy l-Diversity privacy constraint. + + A partition satisfies Entropy l-Diversity when, for every sensitive column, + the Shannon entropy of the value distribution is at least log(l). This is + a stronger guarantee than Distinct l-Diversity because it also requires the + values to be roughly evenly distributed. + + Attributes: + l: Minimum entropy threshold expressed as log(l). + """ + l: int # noqa: E741 def check(self, dataset: DataFrame, column_information: list[ColumnInformation]) -> bool: @@ -129,6 +220,19 @@ def _initialize_histograms_and_order( class TCloseness(PrivacyConstraint): + """t-Closeness privacy constraint. + + A partition satisfies t-Closeness when the distribution of each sensitive + attribute within the partition is no further than *t* from the distribution + in the full dataset, measured using the Earth Mover's Distance (categorical) + or the ordered-distance metric (numerical). + + Call :meth:`initialize` with the full dataset before using :meth:`check`. + + Attributes: + t: Maximum allowed distance between the partition and global distributions. + """ + def __init__(self, t: float): self.t = t self.histograms: list[Series | None] | None = None @@ -136,6 +240,14 @@ def __init__(self, t: float): self.total_count: int | None = None def initialize(self, dataset: DataFrame, column_information: list[ColumnInformation]) -> None: + """Pre-compute global histograms and value ordering from the full dataset. + + Must be called once before :meth:`check` is used on individual partitions. + + Args: + dataset: The complete (un-partitioned) dataset. + column_information: Metadata describing each column. + """ (self.histograms, self.orders) = _initialize_histograms_and_order(dataset, column_information) self.total_count = len(dataset) diff --git a/src/risk_assessment/anonymization/mondrian.py b/src/risk_assessment/anonymization/mondrian.py index f0b7a1f..3c45ac0 100644 --- a/src/risk_assessment/anonymization/mondrian.py +++ b/src/risk_assessment/anonymization/mondrian.py @@ -1,3 +1,32 @@ +"""Mondrian multidimensional partitioning anonymization algorithm. + +Mondrian recursively partitions a dataset along quasi-identifier dimensions, +splitting each partition at the median of the widest dimension until no further +split satisfies the configured privacy constraints. The result is a set of +equivalence classes whose quasi-identifier columns are replaced by a +representative value (the "middle"), producing a k-anonymous (or stronger) +dataset. + +Two split strategies are supported: + +- ``ORDER_BASED`` (default): splits at the numerical/index-order median. +- ``HIERARCHY_BASED``: splits according to the children of the current node + in the generalization hierarchy. + +Example:: + + from risk_assessment.anonymization import KAnonymity + from risk_assessment.anonymization.mondrian import Mondrian, MondrianOptions, MondrianSplitStrategy + from risk_assessment.metrics.informationloss import ColumnInformation, ColumnType, ColumnClass + + options = MondrianOptions( + privacy_constraints=[KAnonymity(k=3)], + split_strategy=MondrianSplitStrategy.ORDER_BASED, + ) + mondrian = Mondrian(options) + anonymized_df, report = mondrian.anonymize(df, column_information) +""" + from __future__ import annotations from dataclasses import dataclass @@ -15,22 +44,48 @@ class MondrianSplitStrategy(Enum): + """Strategy used to split a partition along a quasi-identifier dimension. + + Attributes: + HIERARCHY_BASED: Split by descending into the children of the current + generalization-hierarchy node. + ORDER_BASED: Split at the median of the values' index in the hierarchy + (categorical) or of the raw numerical values. + """ + HIERARCHY_BASED = auto() ORDER_BASED = auto() @dataclass class Interval: + """Numeric interval used to track the value range of a partition dimension. + + Attributes: + low: Lower bound of the interval (inclusive). + high: Upper bound of the interval (inclusive). + median: Optional pre-computed median of the interval. + """ + low: float high: float median: float | None = None def range(self) -> float: + """Return the width of the interval (high - low).""" return self.high - self.low @dataclass class MondrianOptions: + """Configuration for the Mondrian anonymization algorithm. + + Attributes: + privacy_constraints: One or more privacy constraints (e.g. KAnonymity) + that each resulting partition must satisfy. + split_strategy: How to split partitions. Defaults to ORDER_BASED. + """ + privacy_constraints: list[PrivacyConstraint] split_strategy: MondrianSplitStrategy = MondrianSplitStrategy.ORDER_BASED @@ -362,12 +417,38 @@ def _create_middles_and_widths( class Mondrian(AnonymizationAlgorithm): + """Mondrian multidimensional partitioning anonymization algorithm. + + Recursively splits the dataset into equivalence classes along the widest + quasi-identifier dimension until no split satisfying the privacy constraints + can be found. + + Args: + options: Algorithm configuration including privacy constraints and + split strategy. + """ + def __init__(self, options: MondrianOptions): self.options = options def anonymize( self, dataset: DataFrame, column_information: list[ColumnInformation] ) -> tuple[DataFrame, AnonymizationReport]: + """Anonymize the dataset using Mondrian partitioning. + + Args: + dataset: The input DataFrame to anonymize. Quasi-identifier columns + will be replaced by their partition representative value. + column_information: Per-column metadata. Length must equal the + number of columns in ``dataset``. + + Returns: + A tuple of ``(anonymized_dataset, report)``. + + Raises: + ValueError: If ``column_information`` length does not match the + number of dataset columns. + """ if len(column_information) != len(dataset.columns): raise ValueError( f"Dataset and column information are inconsisten in shape {len(dataset)} vs {len(column_information)}" diff --git a/src/risk_assessment/anonymization/optimal_lattice_anonymization.py b/src/risk_assessment/anonymization/optimal_lattice_anonymization.py index be63c8f..28003a8 100644 --- a/src/risk_assessment/anonymization/optimal_lattice_anonymization.py +++ b/src/risk_assessment/anonymization/optimal_lattice_anonymization.py @@ -1,3 +1,34 @@ +"""Optimal Lattice Anonymization (OLA) algorithm. + +OLA explores a generalization lattice using binary search to find the +minimally-generalizing solution that satisfies the configured privacy +constraints within the allowed suppression budget. It is more +computationally efficient than exhaustive lattice search because it +narrows the search space by tagging anonymous/non-anonymous regions and +propagating those tags to successors and predecessors. + +The key components are: + +- :class:`OLAOptions` — algorithm configuration. +- :class:`LatticeNode` — a single point in the generalization lattice, + representing a vector of per-column generalization levels. +- :class:`Lattice` — the full generalization lattice with binary-search + exploration logic. +- :class:`AnonymityChecker` — evaluates a lattice node's suppression rate + and information loss. +- :class:`OLA` — the top-level algorithm implementing + :class:`~risk_assessment.anonymization.AnonymizationAlgorithm`. + +Example:: + + from risk_assessment.anonymization import KAnonymity + from risk_assessment.anonymization.optimal_lattice_anonymization import OLA, OLAOptions + + options = OLAOptions(privacy_constraints=[KAnonymity(k=5)], suppression=5.0) + ola = OLA(options) + anonymized_df, report = ola.anonymize(df, column_information) +""" + from __future__ import annotations import math @@ -15,6 +46,17 @@ @dataclass class OLAOptions: + """Configuration for the OLA anonymization algorithm. + + Attributes: + privacy_constraints: One or more constraints every partition must satisfy. + suppression: Maximum percentage of rows that may be suppressed + (0.0 = no suppression allowed). Defaults to 0.0. + information_loss: Callable used to measure information loss between the + original and generalized datasets. Defaults to + :func:`~risk_assessment.metrics.informationloss.categorical_precision`. + """ + privacy_constraints: list[PrivacyConstraint] suppression: float = 0.0 information_loss: Callable[[DataFrame, DataFrame, list[ColumnInformation]], float] = categorical_precision @@ -108,6 +150,19 @@ def _check_constraints(self, dataset: DataFrame) -> bool: @dataclass(eq=True) class LatticeNode: + """A single node in the generalization lattice. + + Each node represents a vector of per-column generalization levels. + A higher level means more generalization for that quasi-identifier column. + + Attributes: + values: Per-column generalization levels (one entry per quasi-identifier). + suppression_rate: Fraction of rows suppressed at this node (set during exploration). + is_anonymous: Whether this node satisfies all privacy constraints (set during exploration). + information_loss: Information loss measured at this node (set during exploration). + tagged: Whether this node has been evaluated. + """ + values: list[int] suppression_rate: float | None = None is_anonymous: bool | None = None @@ -115,6 +170,7 @@ class LatticeNode: tagged: bool = False def sum(self) -> int: + """Return the total generalization level (sum of all per-column levels).""" return sum(self.values) def is_decendent(self, other: LatticeNode) -> bool: @@ -341,12 +397,38 @@ def matches_maximum_exploration_level(node: LatticeNode, maximum_exploration_lev class OLA(AnonymizationAlgorithm): + """Optimal Lattice Anonymization algorithm. + + Finds the generalization with minimal information loss that satisfies all + privacy constraints within the configured suppression budget, using binary + search over the generalization lattice. + + Args: + options: Algorithm configuration. + """ + def __init__(self, options: OLAOptions): self._options = options def anonymize( self, dataset: DataFrame, column_information: list[ColumnInformation] ) -> tuple[DataFrame, AnonymizationReport]: + """Anonymize the dataset using OLA. + + Args: + dataset: The input DataFrame. Quasi-identifier columns will be + generalized according to the best lattice node found. + column_information: Per-column metadata. Length must equal the + number of columns in ``dataset``. + + Returns: + A tuple of ``(anonymized_dataset, report)``. + + Raises: + ValueError: If ``column_information`` length does not match the + number of dataset columns. + RuntimeError: If no suitable generalization can be found. + """ if len(column_information) != len(dataset.columns): raise ValueError( f"Dataset and column information are inconsisten in shape {len(dataset)} vs {len(column_information)}" diff --git a/src/risk_assessment/masking/__init__.py b/src/risk_assessment/masking/__init__.py index 269d661..e1b0a4f 100644 --- a/src/risk_assessment/masking/__init__.py +++ b/src/risk_assessment/masking/__init__.py @@ -1,3 +1,22 @@ +"""Text masking utilities for applying de-identification transformations to DataFrames. + +This module provides tools for replacing detected entities in free-text fields +of a pandas DataFrame with anonymized substitutes. The transformation policy +maps entity types (e.g. ``"Email"``, ``"CreditCardNumber"``) to callables +from :mod:`risk_assessment.masking.actions`. + +Typical usage:: + + from risk_assessment.masking import cleanse_dataframe_field + from risk_assessment.masking.actions import tagging_factory, format_preserving_redact + + policy = { + "Email": tagging_factory(), + "CreditCardNumber": format_preserving_redact, + } + cleansed_df = cleanse_dataframe_field(df, "notes", "./nlp_reports/", transformations=policy) +""" + import json import logging from collections.abc import Callable @@ -14,6 +33,14 @@ @dataclass class NLPReport: + """Parsed NLP report produced by an entity-extraction pipeline. + + Attributes: + extracted_text: The original text that was analysed. + entities: List of detected entities, each represented as + ``[annotation_text, begin, end, entity_type]``. + """ + extracted_text: str entities: list[list[Any]] @@ -42,6 +69,35 @@ def cleanse_dataframe_field( transformations: dict[str, Callable[[str, str], str]] = _default_transformation_policy(), default: Callable[[str, str], str] = tagging_factory(), ) -> DataFrame: + """Apply entity masking to a text column of a DataFrame using pre-generated NLP reports. + + For each row, loads the corresponding NLP report from ``nlp_report_directory``, + verifies the text matches, then replaces every detected entity span using the + appropriate transformation from ``transformations`` (or ``default`` if the + entity type is not in the policy). Replacements are applied in reverse span + order to preserve character offsets. + + Args: + data: The DataFrame to modify in-place. + field_name: Column name of the free-text field to cleanse. + nlp_report_directory: Directory containing one JSON NLP report per row. + nlp_report_file_pattern: ``str.format``-style pattern for the report + filename. ``{}`` is substituted with ``index + 1``. + Defaults to ``"mytext-{}.json"``. + transformations: Mapping from entity type string to a + ``(entity_type, entity_text) -> replacement_text`` callable. + Defaults to a built-in policy covering common entity types. + default: Fallback transformation for entity types not in + ``transformations``. Defaults to :func:`~risk_assessment.masking.actions.tagging_factory`. + + Returns: + The modified DataFrame (also mutated in-place). + + Raises: + ValueError: If the NLP report's extracted text does not match the + DataFrame cell value, or if an entity span does not match the + annotation text. + """ for index in data.index: logger.info("Processing %s", index) report = _load_entities(Path(nlp_report_directory) / Path(nlp_report_file_pattern.format(index + 1))) diff --git a/src/risk_assessment/masking/actions.py b/src/risk_assessment/masking/actions.py index 19b5a4f..711fc58 100644 --- a/src/risk_assessment/masking/actions.py +++ b/src/risk_assessment/masking/actions.py @@ -1,3 +1,24 @@ +"""Masking action callables for de-identifying detected entities. + +Each function (or factory) in this module implements the signature +``(entity_type: str, entity_text: str) -> str`` and can be used directly +as a transformation in :func:`~risk_assessment.masking.cleanse_dataframe_field`. + +Available actions: + +- :func:`tagging_factory` — replaces each unique value with a stable + sequential label (``TYPE-1``, ``TYPE-2``, …). +- :func:`redact_factory` — replaces the entity with a fixed-length redaction + symbol (default ``"XXX"``). +- :func:`tagging_with_hash` — replaces with a hash-based label for + deterministic but non-reversible pseudonymisation. +- :func:`redact_size_preserving` — replaces every character with ``"X"``, + preserving the original length. +- :func:`format_preserving_redact` — replaces alphanumeric characters with + ``"X"`` while keeping punctuation and spaces, preserving the original format. +- :func:`no_action` — returns the entity text unchanged (pass-through). +""" + from abc import ABC, abstractmethod from collections import defaultdict from collections.abc import Callable @@ -5,15 +26,38 @@ class MappingStorage(ABC): + """Abstract storage for a type-keyed entity-to-label mapping. + + Implementations persist the mapping between original entity text and its + assigned anonymized label so that the same value is always replaced + consistently within a session. + """ + @abstractmethod def get_or_create(self, type_name: str, value: str) -> str: + """Return the label for ``value``, creating one if it does not exist. + + Args: + type_name: The entity type (e.g. ``"Email"``). + value: The raw entity text to look up or register. + + Returns: + The anonymized label for this value. + """ pass class InMemoryMappingStorage(MappingStorage): + """In-memory implementation of :class:`MappingStorage`. + + Labels are assigned sequentially per entity type (``TYPE-1``, ``TYPE-2``, + …) and held in a class-level dictionary for the lifetime of the process. + """ + _known_mapping: dict[str, dict[str, str]] = defaultdict(dict) def get_or_create(self, type_name: str, value: str) -> str: + """Return or create a sequential label for ``value`` under ``type_name``.""" type_dict: dict[str, str] = self._known_mapping[type_name] if value not in type_dict: @@ -23,6 +67,20 @@ def get_or_create(self, type_name: str, value: str) -> str: def tagging_factory(storage: MappingStorage = InMemoryMappingStorage()) -> Callable[[str, str], str]: + """Create a tagging transformation that assigns stable sequential labels. + + Each unique ``(entity_type, entity_text)`` pair receives a label of the + form ``TYPE-N`` the first time it is seen; subsequent occurrences receive + the same label. + + Args: + storage: Backing store for the entity-to-label mapping. + Defaults to a shared :class:`InMemoryMappingStorage` instance. + + Returns: + A ``(entity_type, entity_text) -> label`` callable. + """ + def _tagging_operator(entity_type: str, entity_text: str) -> str: return storage.get_or_create(entity_type, entity_text) @@ -30,20 +88,74 @@ def _tagging_operator(entity_type: str, entity_text: str) -> str: def redact_factory(symbol: str = "X", size: int = 3) -> Callable[[str, str], str]: + """Create a fixed-length redaction transformation. + + Args: + symbol: Character used for redaction. Defaults to ``"X"``. + size: Number of times ``symbol`` is repeated. Defaults to 3. + + Returns: + A ``(entity_type, entity_text) -> redacted_text`` callable that always + returns ``symbol * size`` regardless of the input. + """ return lambda x, y: symbol * size def tagging_with_hash(entity_type: str, entity_text: str) -> str: + """Replace entity text with a hash-based deterministic label. + + The label is of the form ``TYPE-`` derived from the SHA-256 + hash of the entity text. The same text always produces the same label, + but the mapping is not reversible. + + Args: + entity_type: The entity type (e.g. ``"SSN"``). + entity_text: The raw sensitive text to pseudonymise. + + Returns: + A string of the form ``"TYPE-xxxxx"``. + """ return f"{entity_type.upper()}-{sha256(entity_text.encode()).hexdigest()[-5:]}" def redact_size_preserving(_: str, entity_text: str) -> str: + """Replace every character in the entity with ``"X"``, preserving length. + + Args: + _: Entity type (unused). + entity_text: The text to redact. + + Returns: + A string of ``"X"`` characters with the same length as ``entity_text``. + """ return "X" * len(entity_text) def format_preserving_redact(_: str, enity_text: str) -> str: + """Redact alphanumeric characters while preserving punctuation and spaces. + + Each letter or digit in ``enity_text`` is replaced by ``"X"``; all other + characters (spaces, hyphens, dots, etc.) are kept unchanged. This keeps + the visual structure of identifiers like phone numbers or credit cards. + + Args: + _: Entity type (unused). + enity_text: The text to redact. + + Returns: + The format-preserving redacted string. + """ return "".join(["X" if c.isalnum() else c for c in enity_text]) def no_action(value: str, _: str) -> str: + """Pass-through transformation — returns the entity text unchanged. + + Args: + value: The entity type (returned as-is). + _: Entity text (unused). + + Returns: + ``value`` unchanged. + """ return value diff --git a/src/risk_assessment/metrics/__init__.py b/src/risk_assessment/metrics/__init__.py new file mode 100644 index 0000000..43d9b1f --- /dev/null +++ b/src/risk_assessment/metrics/__init__.py @@ -0,0 +1,17 @@ +"""Privacy and anonymization quality metrics. + +This package provides two sub-packages: + +- :mod:`~risk_assessment.metrics.informationloss` — information-loss metrics + used to evaluate the quality of an anonymized dataset compared to the + original. Includes categorical precision, discernibility, non-uniform + entropy, generalized loss metric, and global certain penalty. + +- :mod:`~risk_assessment.metrics.uniqueness_estimation` — statistical + estimators for the fraction of records that are unique in the population + based on a sample, using the Zayatz hypergeometric estimator. + +These metrics are consumed internally by the anonymization algorithms +(:mod:`risk_assessment.anonymization`) and can also be used directly to +assess re-identification risk in a dataset. +""" diff --git a/src/risk_assessment/metrics/informationloss/__init__.py b/src/risk_assessment/metrics/informationloss/__init__.py index bab5f7d..78da112 100644 --- a/src/risk_assessment/metrics/informationloss/__init__.py +++ b/src/risk_assessment/metrics/informationloss/__init__.py @@ -1,3 +1,32 @@ +"""Information-loss metrics for evaluating anonymized datasets. + +This module provides a collection of functions and supporting types for +measuring how much information is lost when a dataset is generalized or +suppressed during anonymization. + +Column roles are described by :class:`ColumnType` (normal, direct, quasi, +sensitive) and :class:`ColumnClass` (numeric, categorical). Per-column +metadata is held in :class:`ColumnInformation`. + +Available metrics: + +- :func:`categorical_precision` — weighted average generalization level + across quasi-identifier columns (lower is better). +- :func:`discernibility` — penalty-based metric that penalizes large + equivalence classes and suppressed rows. +- :func:`discernibility_star` — variant of discernibility without suppression + penalty. +- :func:`non_uniform_entropy` — entropy-based metric measuring information + loss per record. +- :func:`non_uniform_entropy_upper_bound` — upper bound for non-uniform entropy. +- :func:`generalized_loss_metric` — loss based on the fraction of the value + range or hierarchy covered by each generalized value. +- :func:`global_certain_penalty` — normalized sum of within-partition + value-range penalties. +- :func:`average_equivalence_class_size` — mean partition size, optionally + normalized by *k*. +""" + from dataclasses import dataclass from enum import Enum, auto from hashlib import md5 @@ -17,21 +46,49 @@ @dataclass class AverageEquivalenceClassSizeOptions: + """Options for the average equivalence-class size metric. + + Attributes: + normalized: When True, the result is divided by *k*. + k: Target equivalence-class size (used for normalization). + """ + normalized: bool k: int @dataclass class DiscernibilityOptions: + """Options for the discernibility metric. + + Attributes: + k: Minimum acceptable equivalence-class size. + """ + k: int @dataclass class NonUniformEntropyOptions: + """Options for the non-uniform entropy metric. + + Attributes: + k: Minimum acceptable equivalence-class size. + """ + k: int class ColumnType(Enum): + """Role of a dataset column in the anonymization process. + + Attributes: + NORMAL: Column carries no special role. + DIRECT: Direct identifier (e.g. name, ID) — typically suppressed entirely. + QUASI: Quasi-identifier used for grouping into equivalence classes. + SENSITIVE: Sensitive attribute whose distribution must be protected. + """ + NORMAL = auto() DIRECT = auto() QUASI = auto() @@ -39,6 +96,14 @@ class ColumnType(Enum): class ColumnClass(Enum): + """Data type of a dataset column. + + Attributes: + NUMERIC: Column contains numeric values. + CATEGORICAL: Column contains categorical (string) values. + NONE: Column class is unspecified or not applicable. + """ + NUMERIC = auto() CATEGORICAL = auto() NONE = auto() @@ -46,6 +111,22 @@ class ColumnClass(Enum): @dataclass class ColumnInformation: + """Metadata describing a single dataset column for anonymization. + + Attributes: + column_type: Role of this column (normal, direct, quasi, sensitive). + column_class: Data type of this column (numeric or categorical). + weight: Relative weight when computing weighted information-loss metrics. + Defaults to 1.0. + hierarchy: Generalization hierarchy for categorical or numeric columns. + Required for quasi-identifier columns. + range: Numerical range used for numeric quasi-identifier columns. + max_level: Maximum allowed generalization level for OLA exploration. + ``-1`` means no limit. + for_linking: When True, this column is used for record linkage in + uniqueness estimation. + """ + column_type: ColumnType = ColumnType.NORMAL column_class: ColumnClass = ColumnClass.NONE weight: float = 1.0 @@ -67,6 +148,17 @@ def average_equivalence_class_size( column_information: list[ColumnInformation], options: AverageEquivalenceClassSizeOptions, ) -> float: + """Compute the average equivalence-class size of an anonymized dataset. + + Args: + original: The original (un-anonymized) DataFrame. + anonymized: The anonymized DataFrame. + column_information: Per-column metadata. + options: Metric options (normalization flag and target *k*). + + Returns: + Average number of records per equivalence class, optionally divided by *k*. + """ partition_sizes = anonymized.groupby(by=_extract_quasi_identifiers(anonymized, column_information)).size() number_equivalence_classes = 0.0 @@ -161,6 +253,21 @@ def categorical_precision( column_information: list[ColumnInformation], transformation_levels: list[int] | None = None, ) -> float: + """Compute the weighted average generalization level across quasi-identifier columns. + + A value of 0.0 means no generalization; 1.0 means full generalization to + the top of every hierarchy. + + Args: + original: The original DataFrame (used for suppression accounting). + anonymized: The anonymized DataFrame. + column_information: Per-column metadata including hierarchies and weights. + transformation_levels: Optional fixed generalization levels per quasi column. + If None, levels are inferred from the anonymized values. + + Returns: + Mean precision loss in the range [0.0, 1.0]. + """ column_results: list[float] = _categorical_precision_report_per_quasi_column( original, anonymized, column_information, transformation_levels ) @@ -175,6 +282,20 @@ def discernibility( column_information: list[ColumnInformation], options: DiscernibilityOptions, ) -> float: + """Compute the discernibility metric for an anonymized dataset. + + Penalizes each record by the size of its equivalence class if it is + anonymous, or by the total number of records if it is suppressed. + + Args: + original: The original DataFrame. + anonymized: The anonymized DataFrame. + column_information: Per-column metadata. + options: Metric options containing the target *k*. + + Returns: + Discernibility penalty (lower is better). + """ partition_sizes = anonymized.groupby(by=_extract_quasi_identifiers(anonymized, column_information)).size() number_of_records = len(original) @@ -256,6 +377,21 @@ def non_uniform_entropy( column_information: list[ColumnInformation], options: NonUniformEntropyOptions, ) -> float: + """Compute the non-uniform entropy information-loss metric. + + Measures entropy-based information loss by comparing the original and + anonymized value distributions per quasi-identifier column within each + equivalence class. + + Args: + original: The original DataFrame (same size and row order as *anonymized*). + anonymized: The anonymized DataFrame. + column_information: Per-column metadata. + options: Metric options containing the target *k*. + + Returns: + Total non-uniform entropy value (lower is better). + """ # assumption: original and anonymized datasets are of the same size and that datasets' records are in the same order original_frequencies = _calculate_frequencies(original, column_information) anonymized_frequencies = _calculate_frequencies(anonymized, column_information) @@ -396,6 +532,20 @@ def _get_value_loss(value: Any, column_information: ColumnInformation) -> float: def generalized_loss_metric( dataset: DataFrame, anonymized: DataFrame, column_information: list[ColumnInformation] ) -> float: + """Compute the generalized loss metric (GLM) for an anonymized dataset. + + For each cell in a quasi-identifier column, GLM computes the fraction of + the hierarchy range (categorical) or numeric range covered by the + generalized value. Suppressed rows are penalized with a loss of 1. + + Args: + dataset: The original DataFrame. + anonymized: The anonymized DataFrame. + column_information: Per-column metadata including hierarchies and weights. + + Returns: + Weighted mean GLM across all columns and records (lower is better). + """ loss_per_column: list[float] = [0.0 for _ in range(len(dataset.columns))] for row in anonymized.iterrows(): diff --git a/src/risk_assessment/metrics/uniqueness_estimation/__init__.py b/src/risk_assessment/metrics/uniqueness_estimation/__init__.py index ca8ec3c..75c66b9 100644 --- a/src/risk_assessment/metrics/uniqueness_estimation/__init__.py +++ b/src/risk_assessment/metrics/uniqueness_estimation/__init__.py @@ -1,3 +1,14 @@ +"""Population uniqueness estimation using the Zayatz hypergeometric estimator. + +Provides a statistical estimate of the proportion of records that are unique +in the full population, based on the equivalence-class size distribution +observed in a sample. This is useful for quantifying re-identification risk +without access to the complete population dataset. + +Reference: Zayatz, L. (1991). *Using the Hypergeometric Model for Disclosure +Avoidance*, US Bureau of the Census Statistical Research Division. +""" + from dataclasses import dataclass from typing import Any @@ -13,12 +24,36 @@ def _for_linking(columns: Index, column_information: list[ColumnInformation]) -> @dataclass class ZayatzEstimatorOptions: + """Options for the Zayatz population uniqueness estimator. + + Attributes: + population_size: Total size of the population from which the sample + was drawn. Used by the hypergeometric distribution. + """ + population_size: int def zayatz_estimator( sample: DataFrame, column_information: list[ColumnInformation], options: ZayatzEstimatorOptions ) -> float: + """Estimate the proportion of population-unique records in *sample*. + + Uses the Zayatz hypergeometric model to infer, from the sample's + equivalence-class distribution, how many records in the full population + have a unique combination of linking attributes. + + Args: + sample: A sample DataFrame. Linking columns are identified via + ``column_information``. + column_information: Per-column metadata; columns with + ``for_linking=True`` are used for grouping. + options: Estimator options including the known population size. + + Returns: + Estimated fraction of records that are unique in the full population. + Returns 0.0 if no singleton equivalence classes are found in the sample. + """ sum = 0.0 equivalence_class_sizes: dict[int, int] = {} diff --git a/src/risk_assessment/readi/__init__.py b/src/risk_assessment/readi/__init__.py index e69de29..7f3a449 100644 --- a/src/risk_assessment/readi/__init__.py +++ b/src/risk_assessment/readi/__init__.py @@ -0,0 +1,14 @@ +"""READI analyzer package. + +Exposes :class:`~risk_assessment.readi.analyzer.READIAnalyzer` as the primary +entry point for detecting PII and PHI in unstructured text:: + + from risk_assessment.readi import READIAnalyzer + + analyzer = READIAnalyzer() + entities = analyzer.detect("Patient John Doe, DOB 01/01/1980") +""" + +from risk_assessment.readi.analyzer import READIAnalyzer + +__all__ = ["READIAnalyzer"] diff --git a/src/risk_assessment/readi/sentence_tokenizer.py b/src/risk_assessment/readi/sentence_tokenizer.py index 8055704..ae040f7 100644 --- a/src/risk_assessment/readi/sentence_tokenizer.py +++ b/src/risk_assessment/readi/sentence_tokenizer.py @@ -1,25 +1,79 @@ +"""Sentence tokenizers for splitting text into sentence-level spans. + +Provides a base class and concrete implementations for splitting raw text +into sentence spans, used by the entity-extraction pipeline to limit the +context window fed to individual extractors. + +Classes: + +- :class:`SentenceTokenizer` — abstract base; override ``span_tokenize`` + and ``sent_tokenize`` in subclasses. +- :class:`JASentenceTokenizerSimple` — regex-based tokenizer supporting + Japanese and Latin end-of-sentence markers. +- :class:`NLTKSentenceTokenizer` — NLTK Punkt-based tokenizer with optional + sentence grouping to respect a maximum character budget per chunk. + +Note: + A Stanza-based tokenizer was considered but is not currently implemented. + The ``import stanza`` line is intentionally commented out as a placeholder. +""" + import re from nltk import PunktSentenceTokenizer -# import stanza - class SentenceTokenizer: + """Abstract base class for sentence tokenizers. + + Subclasses must implement :meth:`span_tokenize` and :meth:`sent_tokenize`. + """ + def __init__(self) -> None: pass def span_tokenize(self, text: str) -> list[tuple[int, int]]: # type: ignore + """Return a list of ``(start, end)`` character spans for each sentence. + + Args: + text: Input text. + + Returns: + List of ``(start, end)`` tuples (end is exclusive). + """ pass def sent_tokenize(self, text: str) -> list[str]: # type: ignore + """Return a list of sentence strings. + + Args: + text: Input text. + + Returns: + List of sentence strings extracted from *text*. + """ pass class JASentenceTokenizerSimple(SentenceTokenizer): + """Regex-based sentence tokenizer supporting Japanese and Latin punctuation. + + Splits text at end-of-sentence markers: ``.``, ``!``, ``?``, ``。``, + newlines, ``？``, and ``！``. + + Attributes: + eos_pattern: Compiled regex pattern used to detect sentence boundaries. + """ + eos_pattern = re.compile(r"\.|\!|\?|\。|\n|\？|\！") def __init__(self, eos_pattern: str | None = r"\.|\!|\?|\。|\n|\？|\！") -> None: + """Initialise the tokenizer with a custom end-of-sentence pattern. + + Args: + eos_pattern: Regex pattern string to detect sentence boundaries. + Pass ``None`` to keep the class-level default. + """ super().__init__() if eos_pattern: self.eos_pattern = re.compile(eos_pattern) @@ -47,7 +101,31 @@ def sent_tokenize(self, text: str) -> list[str]: class NLTKSentenceTokenizer(SentenceTokenizer): + """NLTK Punkt-based sentence tokenizer with optional sentence grouping. + + Texts shorter than *thr* characters are returned as a single span. + Longer texts are tokenized by NLTK's ``PunktSentenceTokenizer`` and + optionally grouped into chunks that each stay within the *thr* character + budget — useful when downstream models have a maximum context length. + + Attributes: + tokenizer: The underlying NLTK Punkt sentence tokenizer. + thr: Character length threshold below which the entire text is one span, + and also the maximum per-group length when grouping is enabled. + group_sentences: When True, adjacent sentences are merged until the + group would exceed *thr* characters. + """ + def __init__(self, group_sentences: bool = True, thr: int = 600) -> None: + """Initialise the tokenizer. + + Args: + group_sentences: Whether to merge short consecutive sentences into + chunks up to *thr* characters. Defaults to True. + thr: Character length threshold. Texts shorter than this are + returned as a single span; grouped chunks will not exceed this + length. Defaults to 600. + """ super().__init__() self.tokenizer = PunktSentenceTokenizer() self.thr = thr diff --git a/src/risk_assessment/readi/text_tokenizer.py b/src/risk_assessment/readi/text_tokenizer.py index ee0bf1f..b3ba002 100644 --- a/src/risk_assessment/readi/text_tokenizer.py +++ b/src/risk_assessment/readi/text_tokenizer.py @@ -1,3 +1,20 @@ +"""Token-level and sentence-level text tokenizers for the READI pipeline. + +This module provides the tokenization layer used by entity extractors that +require text to be split into word/token spans with their positions preserved +relative to the original string. + +Classes: + +- :class:`BaseTokenizer` — abstract interface for span tokenizers. +- :class:`TextTokenizer` — composes a :class:`~risk_assessment.readi.sentence_tokenizer.SentenceTokenizer` + with a span tokenizer to produce token spans aligned to the full document. +- :class:`LMTokenizer` — HuggingFace ``AutoTokenizer``-backed span tokenizer + for language-model based extractors. +- :class:`JapaneseTokenizer` — MeCab-backed morphological tokenizer for + Japanese text. +""" + import warnings from abc import ABC, abstractmethod @@ -11,12 +28,37 @@ class BaseTokenizer(ABC): + """Abstract base class for span tokenizers. + + A span tokenizer maps raw text to a list of ``(start, end)`` character + positions, one per token. + """ + @abstractmethod def span_tokenize(self, text: str) -> list[tuple[int, int]]: + """Tokenize *text* and return character-level spans. + + Args: + text: Input text. + + Returns: + List of ``(start, end)`` tuples (end exclusive) for each token. + """ raise NotImplementedError() class TextTokenizer: + """Two-level tokenizer: first splits text into sentences, then into tokens. + + Combines a :class:`~risk_assessment.readi.sentence_tokenizer.SentenceTokenizer` + with a span tokenizer so that all token spans are expressed as offsets + within the full document rather than within individual sentences. + + Attributes: + sentence_tokenizer: Sentence-level splitter. + span_tokenizer: Token-level splitter applied to each sentence. + """ + def __init__( self, sentence_tokenizer: SentenceTokenizer, span_tokenizer: WordPunctTokenizer | BaseTokenizer ) -> None: @@ -54,11 +96,22 @@ def tokenize_sentence_with_pos_in_text( spans = [(span[0] + sentence_position, span[1] + sentence_position) for span in spans_sentence_level] return sentence_by_token, spans - def tokenize_sentenses( + def tokenize_sentences( self, sentences: list[str], sentence_positions: list[tuple[int, int]], ) -> tuple[list[list[str]], list[list[tuple[int, int]]]]: + """Tokenize a list of sentences, aligning spans to the full document. + + Args: + sentences: Pre-split sentence strings. + sentence_positions: ``(start, end)`` positions of each sentence + within the original document. + + Returns: + A tuple ``(tokens_per_sentence, spans_per_sentence)`` where each + inner list corresponds to one sentence. + """ sentences_by_token: list[list[str]] = [] sentences_by_spans: list[list[tuple[int, int]]] = [] for i, sentence in enumerate(sentences): @@ -79,7 +132,24 @@ def tokenize_text( class LMTokenizer(BaseTokenizer): + """HuggingFace language-model tokenizer that returns character-level spans. + + Wraps an ``AutoTokenizer`` and converts its token-to-character mappings + into ``(start, end)`` spans compatible with the READI pipeline. + + Attributes: + device: Target device string (e.g. ``"cpu"`` or ``"cuda"``). + tokenizer: Loaded HuggingFace tokenizer instance. + """ + def __init__(self, model_name: str = "FacebookAI/roberta-base", device: str = "cpu") -> None: + """Load the tokenizer from a HuggingFace model name or local path. + + Args: + model_name: Model identifier passed to ``AutoTokenizer.from_pretrained``. + Defaults to ``"FacebookAI/roberta-base"``. + device: Device string for downstream model inference. Defaults to ``"cpu"``. + """ self.device = device self.tokenizer = AutoTokenizer.from_pretrained(model_name) # nosec @@ -95,7 +165,17 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]: class JapaneseTokenizer(BaseTokenizer): + """MeCab-based morphological tokenizer for Japanese text. + + Uses MeCab to split Japanese text into morpheme spans aligned to the + original whitespace-separated substrings. + + Attributes: + preprocessor: MeCab tagger instance used for morphological analysis. + """ + def __init__(self) -> None: + """Initialise MeCab tagger.""" self.preprocessor = MeCab.Tagger() def span_tokenize(self, text: str) -> list[tuple[int, int]]: diff --git a/src/risk_assessment/vulnerability/__init__.py b/src/risk_assessment/vulnerability/__init__.py index e69de29..254be71 100644 --- a/src/risk_assessment/vulnerability/__init__.py +++ b/src/risk_assessment/vulnerability/__init__.py @@ -0,0 +1,13 @@ +"""Quasi-identifier vulnerability detection algorithms. + +This package provides two algorithms for discovering columns (or combinations +of columns) in a dataset that act as quasi-identifiers — i.e. that allow +re-identification of individuals when the equivalence-class size falls below a +given *k* threshold: + +- :mod:`~risk_assessment.vulnerability.brute` — exhaustive brute-force search + over all column combinations. +- :mod:`~risk_assessment.vulnerability.ducc` — the DUCC (Descending for Unique + Column Combinations) algorithm, a more efficient graph-pruning approach for + larger attribute sets. +""" diff --git a/src/risk_assessment/vulnerability/brute/__init__.py b/src/risk_assessment/vulnerability/brute/__init__.py index 0895047..5ddd5ed 100644 --- a/src/risk_assessment/vulnerability/brute/__init__.py +++ b/src/risk_assessment/vulnerability/brute/__init__.py @@ -1,3 +1,14 @@ +"""Brute-force quasi-identifier discovery algorithm. + +Exhaustively checks every subset of dataset columns to determine which +combinations act as quasi-identifiers (i.e. produce at least one equivalence +class smaller than *k*). Optional pruning skips supersets of already-found +quasi-identifiers to reduce redundant checks. + +For large attribute sets consider the more efficient DUCC algorithm in +:mod:`risk_assessment.vulnerability.ducc`. +""" + from collections.abc import Iterator, Sequence from itertools import combinations @@ -5,6 +16,20 @@ def is_quasi_identifier(col: list[str], data: DataFrame, k: int) -> bool: + """Check whether a column combination is a quasi-identifier at threshold *k*. + + A combination is a quasi-identifier if any of its equivalence classes + (groups with identical values across all columns in *col*) has fewer than + *k* records. + + Args: + col: Column names to group by. + data: The dataset to analyse. + k: Minimum acceptable equivalence-class size. + + Returns: + True if the combination is a quasi-identifier, False otherwise. + """ occur = data.groupby(col).size() for _, val in occur.items(): if val < k: @@ -13,12 +38,32 @@ def is_quasi_identifier(col: list[str], data: DataFrame, k: int) -> bool: def generate_column_combinations(columns: Sequence[str]) -> Iterator[list[str]]: + """Yield all non-empty subsets of *columns* in order of increasing size. + + Args: + columns: Sequence of column names. + + Yields: + Lists of column names, one for each non-empty subset. + """ for size in range(1, len(columns) + 1): for combination in combinations(columns, size): yield list(combination) def is_known(identifiers: list[list[str]], combination: list[str]) -> bool: + """Check whether *combination* is a superset of any known quasi-identifier. + + Used for pruning: if a subset of *combination* is already known to be a + quasi-identifier, any superset is also a quasi-identifier. + + Args: + identifiers: Previously discovered quasi-identifiers. + combination: The column combination to test. + + Returns: + True if *combination* is a superset of a known quasi-identifier. + """ combination_set: set[str] = set(combination) for identifier in identifiers: if combination_set.issuperset(set(identifier)): @@ -27,9 +72,22 @@ def is_known(identifiers: list[list[str]], combination: list[str]) -> bool: def brute_algorithm(data: DataFrame, k: int, prune: bool) -> list[list[str]]: + """Find all quasi-identifier column combinations in *data* at threshold *k*. + + Args: + data: The dataset to analyse. + k: Minimum acceptable equivalence-class size. Combinations that + produce any class smaller than *k* are quasi-identifiers. + prune: When True, skips supersets of already-found quasi-identifiers + (monotonicity pruning), which can significantly reduce the number + of checks on wide datasets. + + Returns: + A list of minimal quasi-identifier column combinations. + """ quasi_identifiers: list[list[str]] = [] - for column_combination in generate_column_combinations(data.columns): # type: ignore + for column_combination in generate_column_combinations(list(data.columns)): if prune and is_known(quasi_identifiers, column_combination): continue if is_quasi_identifier(column_combination, data, k): diff --git a/src/risk_assessment/vulnerability/ducc/ducc.py b/src/risk_assessment/vulnerability/ducc/ducc.py index b938f4d..e6736ae 100644 --- a/src/risk_assessment/vulnerability/ducc/ducc.py +++ b/src/risk_assessment/vulnerability/ducc/ducc.py @@ -1,3 +1,14 @@ +"""DUCC — Descending for Unique Column Combinations. + +DUCC is a graph-pruning algorithm that efficiently discovers minimal +quasi-identifier sets in a dataset. It builds Partition List Indexes (PLIs) +for each column and uses a descending search strategy together with a pruned +graph to avoid redundant checks. + +Reference: Heise et al., "Scalable Discovery of Unique Column Combinations", +PVLDB 2013. +""" + from itertools import combinations import pandas as pd @@ -8,10 +19,26 @@ class Ducc: + """DUCC quasi-identifier discovery algorithm. + + Attributes: + k: Minimum acceptable equivalence-class size. A column combination is + a quasi-identifier when any group has fewer than *k* records. + """ + def __init__(self, k: int) -> None: self.k = k def apply(self, data: pd.DataFrame) -> list[list[str]]: + """Discover all minimal quasi-identifier column combinations. + + Args: + data: The dataset to analyse. + + Returns: + A list of minimal quasi-identifier combinations (each a list of + column names). + """ pli_repository = self.initialize_pli_repositories(data) graphs = PrunedGraphs() seeds = self.populate_seeds(pli_repository, graphs) @@ -19,6 +46,17 @@ def apply(self, data: pd.DataFrame) -> list[list[str]]: return vulnerabilities def initialize_pli_repositories(self, data: pd.DataFrame) -> dict[str, list[set[int]]]: + """Build Partition List Indexes (PLIs) for every column. + + A PLI maps each column to a list of row-index sets, one set per + distinct value. + + Args: + data: The dataset. + + Returns: + Dictionary mapping column name to its PLI. + """ column_names = list(data.columns) repository: dict[str, list[set[int]]] = {j: [] for j in column_names} for attr in column_names: @@ -29,12 +67,27 @@ def initialize_pli_repositories(self, data: pd.DataFrame) -> dict[str, list[set[ return repository def has_unique(self, sets: list[set[int]]) -> bool: + """Return True if any set in *sets* has fewer than *k* elements.""" for s in sets: if self.k > len(s): return True return False def populate_seeds(self, pli_repository: dict[str, list[set[int]]], graphs: PrunedGraphs) -> list[list[str]]: + """Identify single-column quasi-identifiers and build seed combinations. + + Columns that are quasi-identifiers on their own are removed from the + repository and registered in *graphs*; the remaining columns are + combined into candidate multi-column seeds. + + Args: + pli_repository: PLI repository; single-column quasi-identifiers + are deleted from this dict as a side-effect. + graphs: Pruned-graph state updated with single-column quasi-identifiers. + + Returns: + All multi-column combinations of the non-trivially-unique columns. + """ col_names = list(pli_repository.keys()) for key in col_names: if self.has_unique(pli_repository[key]): diff --git a/src/risk_assessment/vulnerability/ducc/ducc_worker_descending.py b/src/risk_assessment/vulnerability/ducc/ducc_worker_descending.py index bde20bc..419a88f 100644 --- a/src/risk_assessment/vulnerability/ducc/ducc_worker_descending.py +++ b/src/risk_assessment/vulnerability/ducc/ducc_worker_descending.py @@ -1,7 +1,27 @@ +"""Descending worker for the DUCC algorithm. + +Implements the greedy descending search that evaluates candidate column +combinations, records quasi-identifiers in the pruned graph, and removes +subsumed candidates from the seed list. +""" + from .pruned_graphs import PrunedGraphs class DuccWorkerDescending: + """Greedy descending search worker for DUCC. + + Iterates over candidate column combinations (seeds), checks each against + the PLI repository, and updates the pruned graph accordingly. + + Args: + seeds: Initial candidate combinations to evaluate. + pliRepository: Partition List Indexes keyed by column name. + graphs: Shared pruned-graph state tracking known quasi-identifiers + and non-quasi-identifiers. + k: Minimum acceptable equivalence-class size. + """ + def __init__(self, seeds: list[list[str]], pliRepository: dict[str, list[set[int]]], graphs: PrunedGraphs, k: int): self.seeds = seeds self.pliRepository = pliRepository @@ -9,22 +29,26 @@ def __init__(self, seeds: list[list[str]], pliRepository: dict[str, list[set[int self.k = k def remove_super_set(self, item_set: list[str]) -> None: + """Remove all seeds that are proper supersets of *item_set*.""" for elem in self.seeds: if set(item_set).issubset(set(elem)): self.seeds.remove(elem) def remove_sub_sets(self, item_set: list[str]) -> None: + """Remove all seeds that are proper subsets of *item_set*.""" for elem in self.seeds: if set(elem).issubset(set(item_set)): self.seeds.remove(elem) def has_unique(self, sets: list[set[int]]) -> bool: + """Return True if any set in *sets* has fewer than *k* elements.""" for s in sets: if self.k > len(s): return True return False def find_intersect(self, ids1: list[set[int]], ids2: list[set[int]]) -> list[set[int]]: + """Compute the pairwise non-empty intersections of two PLI lists.""" comb_ids = [] for i in ids1: for j in ids2: @@ -34,6 +58,7 @@ def find_intersect(self, ids1: list[set[int]], ids2: list[set[int]]) -> list[set return comb_ids def get_combinations(self, curr_val: list[str]) -> list[set[int]]: + """Return the combined PLI for a multi-column combination.""" if len(curr_val) == 1: return self.pliRepository[curr_val[0]] else: @@ -44,6 +69,11 @@ def get_combinations(self, curr_val: list[str]) -> list[set[int]]: return inter def greedy_step(self) -> list[list[str]]: + """Run the greedy descending search and return all quasi-identifier combinations. + + Returns: + Minimal quasi-identifier column combinations found during the search. + """ itemset_checked: int = 0 start = 0 while len(self.seeds) > 0 and start < len(self.seeds): diff --git a/src/risk_assessment/vulnerability/ducc/pruned_graphs.py b/src/risk_assessment/vulnerability/ducc/pruned_graphs.py index b33dc57..bf53e49 100644 --- a/src/risk_assessment/vulnerability/ducc/pruned_graphs.py +++ b/src/risk_assessment/vulnerability/ducc/pruned_graphs.py @@ -1,24 +1,50 @@ +"""Pruned-graph state for the DUCC algorithm. + +Tracks known quasi-identifier combinations (``uniques``) and known +non-quasi-identifier combinations (``not_uniques``) to avoid redundant +checks via monotonicity-based pruning. +""" + + class PrunedGraphs: + """Monotonicity-based pruning state for the DUCC search. + + Maintains two lists: + + - ``uniques``: minimal quasi-identifier combinations found so far. + Any superset is also a quasi-identifier and can be skipped. + - ``not_uniques``: maximal non-quasi-identifier combinations found so far. + Any subset is also not a quasi-identifier and can be skipped. + + Attributes: + uniques: Known minimal quasi-identifier combinations. + not_uniques: Known maximal non-quasi-identifier combinations. + """ + def __init__(self) -> None: self.uniques: list[list[str]] = [] self.not_uniques: list[list[str]] = [] def is_node_pruned(self, item_set: list[str]) -> bool: + """Return True if *item_set* can be skipped due to prior knowledge.""" return self.is_unsafe(item_set) or self.is_safe(item_set) def is_unsafe(self, item_set: list[str]) -> bool: + """Return True if a subset of *item_set* is a known quasi-identifier.""" for o in self.uniques: if set(o).issubset(set(item_set)): return True return False def is_safe(self, item_set: list[str]) -> bool: + """Return True if *item_set* is a subset of a known non-quasi-identifier.""" for o in self.not_uniques: if set(item_set).issubset(set(o)): return True return False def add_unique(self, item_set: list[str]) -> None: + """Register *item_set* as a quasi-identifier, removing any subsumed entries.""" for o in self.uniques: if set(item_set).issubset(set(o)): self.uniques.remove(o) @@ -27,6 +53,7 @@ def add_unique(self, item_set: list[str]) -> None: self.uniques.append(item_set) def add_not_unique(self, item_set: list[str]) -> None: + """Register *item_set* as a non-quasi-identifier, removing any subsumed entries.""" for o in self.not_uniques: if set(o).issubset(set(item_set)): self.not_uniques.remove(o)