diff --git a/.github/ISSUE_TEMPLATE/datasource_request.yml b/.github/ISSUE_TEMPLATE/datasource_request.yml new file mode 100644 index 000000000..8b6c56ac3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/datasource_request.yml @@ -0,0 +1,100 @@ +name: Data source request +description: Request a new data source for 4CAT. +title: "[Data source] " +labels: ["module-request", "data source"] +body: + - type: markdown + attributes: + value: | + Before you start: + + - Check the [list of available data sources](https://github.com/digitalmethodsinitiative/4cat/wiki) and the open + [data source requests](https://github.com/digitalmethodsinitiative/4cat/issues?q=is%3Aissue+label%3A%22data+source%22) + first. + - Request new [Zeeschuimer data sources on its own GitHub] + (https://github.com/digitalmethodsinitiative/zeeschuimer) instead of here. + + - type: input + id: datasource + attributes: + label: Data source name + description: The name of the datasource (platform, upload, etc) you want to collect data from. + placeholder: e.g. 'BlueSky' + validations: + required: true + + - type: input + id: datasource_url + attributes: + label: URL of data source + placeholder: https://... + validations: + required: false + + - type: dropdown + id: access + attributes: + label: How is the data reachable? + description: Your best guess is fine — say so in the details below if you are unsure. + options: + - It has a documented public API + - It has an API, but it needs approval, an account, or payment + - No API — the data is only visible in the browser + - It offers a bulk export or data archive + - I don't know + validations: + required: true + + - type: textarea + id: access_details + attributes: + label: API documentation and access details + description: > + Link to the API documentation if there is any. Does it need credentials, an approved research application, or a + paid plan? Are there rate limits or caps you know of? If there is no API, describe how you currently get at the + data. + validations: + required: false + + - type: textarea + id: items + attributes: + label: What should one item in the dataset be? + description: > + 4CAT datasets are tables of items. Say what a single item should be for this platform (a post, a comment, a + video, a profile, …) and which fields matter to you — timestamp, author, body text, engagement counts, media + URLs, and so on. + validations: + required: true + + - type: textarea + id: use_case + attributes: + label: What would you use this for? + description: > + A sentence or two on the research you have in mind. This helps us judge which fields and query options actually + need to exist. + validations: + required: true + + - type: textarea + id: sample + attributes: + label: Example data + description: > + If you can share a small sample of the raw data (an API response, an exported file), paste or attach it here. + This is the single most useful thing you can add. Remove anything personal or sensitive first. + validations: + required: false + + - type: checkboxes + id: acknowledgements + attributes: + label: Before submitting + options: + - label: > + I have considered whether collecting this data is compatible with the platform's terms of service and with + the privacy of the people in it, and I am not asking for a way to circumvent access restrictions. + required: true + - label: I searched the existing issues and this data source has not been requested yet. + required: true diff --git a/.github/ISSUE_TEMPLATE/processor_request.yml b/.github/ISSUE_TEMPLATE/processor_request.yml new file mode 100644 index 000000000..73a39d8ec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/processor_request.yml @@ -0,0 +1,92 @@ +name: Processor request +description: Ask for a new analysis step that can be run on a 4CAT dataset. +title: "[Processor] " +labels: ["module-request", "processor"] +body: + - type: markdown + attributes: + value: | + Before you start: + + - Check the [list of available processors](https://github.com/digitalmethodsinitiative/4cat/wiki/Available-processors) + and the open [processor requests](https://github.com/digitalmethodsinitiative/4cat/issues?q=is%3Aissue+label%3Aprocessor) + first — it may already exist. + - Processors take one dataset and produce another. If what you want is a different way of *collecting* data, + open a data source request instead. + - If you want to write it yourself, see + [How to make a processor](https://github.com/digitalmethodsinitiative/4cat/wiki/How-to-make-a-processor). It + does not have to live in 4CAT itself — processors can also be distributed as an extension. + - The clearer the input and output below, the more likely someone can pick this up and build it. + + - type: textarea + id: what + attributes: + label: What should it do? + description: > + One or two sentences, starting with a verb — the way a processor describes itself in the interface. + placeholder: e.g. Calculate the lexical diversity of each post and add it as a column. + validations: + required: true + + - type: textarea + id: input + attributes: + label: What should it run on? + description: > + Which datasets should this be available for — all csv datasets, any dataset with text, only datasets from a + specific data source, only the output of another processor? Name that processor if so. + validations: + required: true + + - type: textarea + id: output + attributes: + label: What should it produce? + description: > + The output format (CSV, NDJSON, an image, a network file, …) and what is in it. For a table, list the columns + and what one row represents. + validations: + required: true + + - type: textarea + id: options + attributes: + label: Options + description: > + Anything the user should be able to configure before running it — thresholds, columns to work on, a language, a + model to use. Include sensible defaults if you have them in mind. + validations: + required: false + + - type: textarea + id: reference + attributes: + label: Reference implementation + description: > + A paper, method, library, script, or an equivalent feature in another tool. If there is code that already does + this, linking to it is the most useful thing you can add. + validations: + required: false + + - type: dropdown + id: requirements + attributes: + label: Does it need anything beyond plain Python? + options: + - "No — it can be computed from the dataset itself" + - "A Python library that 4CAT does not ship yet" + - "A machine learning model (running locally)" + - "A GPU" + - "An external API or paid service" + - "I don't know" + validations: + required: true + + - type: textarea + id: use_case + attributes: + label: What would you use this for? + description: > + A sentence or two on the research you have in mind. This helps us get the output shape right the first time. + validations: + required: true diff --git a/AGENTS.md b/AGENTS.md index b91f37d39..6b7a0461f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,6 +160,7 @@ Invocation (the plain `docker compose` command uses `docker-compose.yml`): - Define reusable Jinja2 components when patterns emerge, but avoid over-engineering for future reuse. - Views are organized by concern in `webtool/views/`. API endpoints are in `api_tool.py` and `api_standalone.py`. - Static assets go in `webtool/static/`; templates in `webtool/templates/`. +- The project uses htmx `4.0.0-beta*`. ## Testing Expectations - Run tests with `pytest` from the repo root. Config is in `pytest.ini`. diff --git a/backend/lib/processor.py b/backend/lib/processor.py index 616ffccaf..5293cfa7f 100644 --- a/backend/lib/processor.py +++ b/backend/lib/processor.py @@ -1,6 +1,7 @@ """ Basic post-processor worker - should be inherited by workers to post-process results """ +from dataclasses import dataclass, field import traceback import inspect as py_inspect import zipfile @@ -30,6 +31,55 @@ # Shared instance for the legacy default `compatibility` _DEFAULT_COMPATIBILITY = Compatibility(top_dataset_only=True) +@dataclass +class ProcessorDescription: + """ + A processor's user-facing description: the information shown about it in + the web interface. A processor can declare one of these directly (as its + `description` attribute) or declare the individual attributes; either way + the result is available via `Processor.get_description()`. + """ + title: str + description: str + category: str = "" # for backwards compatability + tags: typing.List[str] = field(default_factory=list) + references: typing.List[str] = field(default_factory=list) + info: typing.List[str] = field(default_factory=list) + warnings: typing.List[str] = field(default_factory=list) + icon: str = "" + + def __post_init__(self): + if self.category: + self.category = self.category[0].upper() + self.category[1:] + self.tags = [tag.strip() for tag in self.tags] + + # `category` is kept as the first entry of `tags` (as its lower-case + # tag form), so 4CAT can move to tags (which allow several per processor + # and can be filtered on) while `category` keeps working. Derive + # whichever is missing; when both are given, make sure the category + # leads the tag list. + category_tag = self.category.lower() + if self.category and not self.tags: + self.tags = [category_tag] + elif self.tags and not self.category: + self.category = self.tags[0][0].upper() + self.tags[0][1:] + elif self.category and self.tags: + self.tags = [category_tag] + [tag for tag in self.tags if tag != category_tag] + + +class _DescriptionField: + """ + Exposes one ProcessorDescription field as an attribute on the processor, + e.g. `Processor.title`. A plain `property` only runs on instance access; + this descriptor also runs on class access (`owner` is the class in both + cases), so `Processor.title` and `self.title` both return the value from + the processor's ProcessorDescription. + """ + def __init__(self, name): + self.name = name + + def __get__(self, obj, owner): + return getattr(owner._processor_description, self.name) class BasicProcessor(FourcatModule, BasicWorker, metaclass=abc.ABCMeta): """ @@ -81,11 +131,30 @@ def is_compatible_with(cls, module=None, config=None): #: The file that is being processed source_file = None - #: Processor description, which will be displayed in the web interface - description = "No description available" - - #: Category identifier, used to group processors in the web interface - category = "Other" + #: The processor's user-facing description (title, category, description + #: text and references) as a single object. A processor may set this + #: directly, or set the individual attributes below; both are normalised + #: into `_processor_description` when the class is defined. + _processor_description = ProcessorDescription( + title="", + description="No description available", + references=[], + info=[], + warnings=[], + icon="" + ) + + #: Title, category, description text and references, read from the + #: processor's ProcessorDescription. Defined as descriptors so that both + #: `Processor.title` and `self.title` resolve to the stored value. + title = _DescriptionField("title") + category = _DescriptionField("category") + description = _DescriptionField("description") + references = _DescriptionField("references") + info = _DescriptionField("info") + warnings = _DescriptionField("warnings") + tags = _DescriptionField("tags") + icon = _DescriptionField("icon") #: Extension of the file created by the processor extension = "csv" @@ -991,6 +1060,18 @@ def _validate_map_item_post_run(self): except Exception: pass + @classmethod + def get_repo_link(cls, config): + """ + Get a link to the processor's source code repository + + :param ConfigManager config: Configuration reader + :return str: URL to the processor's source code repository + """ + repo_url = config.get("4cat.github_url") + path = cls.filepath.replace("\\", "/").lstrip("/") + return f"{repo_url.rstrip('/')}/blob/master/{path}" + @classmethod def is_compatible_with(cls, module=None, config=None): """ @@ -1026,10 +1107,10 @@ def is_filter(cls): Filters do not produce their own dataset but replace the source_dataset dataset instead. - :todo: Make this a bit more robust than sniffing the processor category + :todo: Make this a bit more robust than sniffing the processor tags :return bool: """ - return (hasattr(cls, "category") and cls.category and "filter" in cls.category.lower()) or (hasattr(cls, "filter") and cls.filter) + return (hasattr(cls, "tags") and cls.tags and "filtering" in [tag.lower() for tag in cls.tags]) or (hasattr(cls, "filter") and cls.filter) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: @@ -1134,6 +1215,58 @@ def exclude_followup_processors(cls, processor_type=None): return True return False + def __init_subclass__(cls, **kwargs): + """ + Normalise a processor's description when its class is defined. + + A processor may declare its description either as a ProcessorDescription + object (assigned to `description`) or as the individual attributes + (title, category, description, references, info, warnings, icon, tags). + Either way it is folded into a single `_processor_description` object + here, and the raw attributes are removed so the descriptors on + BasicProcessor provide access to them. + """ + super().__init_subclass__(**kwargs) + + # the description inherited from the nearest ancestor (cls has none yet) + inherited = getattr(cls, "_processor_description", None) + + # read the raw class-body value, bypassing the descriptor + declared = cls.__dict__.get("description") + if isinstance(declared, ProcessorDescription): + description = declared + else: + # build from the flat attributes, falling back to inherited values + # so a legacy subclass keeps anything an ancestor set + description = ProcessorDescription( + title=cls.__dict__.get("title", inherited.title), + category=cls.__dict__.get("category", inherited.category), + description=cls.__dict__.get("description", inherited.description), + references=list(cls.__dict__.get("references", inherited.references)), + info=list(cls.__dict__.get("info", inherited.info)), + warnings=list(cls.__dict__.get("warnings", inherited.warnings)), + # tags default to this class's category (see __post_init__), so + # don't inherit them — re-derive from the resolved category + tags=list(cls.__dict__.get("tags", [])), + icon=cls.__dict__.get("icon", inherited.icon), + ) + + # remove raw attributes so the inherited descriptors govern access + for name in ("title", "category", "description", "references", "info", "warnings", "tags", "icon"): + if name in cls.__dict__: + delattr(cls, name) + + cls._processor_description = description + + @classmethod + def get_description(cls): + """ + Get the processor's user-facing description + + :return ProcessorDescription: Description of this processor + """ + return cls._processor_description + @abc.abstractmethod def process(self): """ @@ -1162,4 +1295,4 @@ def is_preset(): :return: False """ - return False + return False \ No newline at end of file diff --git a/backend/lib/search.py b/backend/lib/search.py index c79849f51..562296f01 100644 --- a/backend/lib/search.py +++ b/backend/lib/search.py @@ -13,6 +13,7 @@ from backend.lib.processor import BasicProcessor from common.lib.helpers import strip_tags, dict_search_and_update, remove_nuls, HashCache, format_import_item from common.lib.exceptions import WorkerInterruptedException, ProcessorInterruptedException, MapItemException +from common.lib.outputs import Datasource class Search(BasicProcessor, ABC): @@ -32,6 +33,15 @@ class Search(BasicProcessor, ABC): #: backwards-compatibility reasons. For example, `instagram-search`. type = "abstract-search" + #: Default output shape: a collected, top-level dataset whose extension is this + #: worker's own (ndjson for most, csv/zip for some). A data source that produces + #: media (an uploaded archive) or whose shape is only known at run time overrides + #: this with a MediaArchive or a per-worker Output. + output = Datasource() + + # generic icon + icon = "comments" + #: Amount of workers of this type that can run in parallel. Be careful with #: this, because values higher than 1 will mean that e.g. API rate limits #: are easily violated. @@ -50,6 +60,33 @@ class Search(BasicProcessor, ABC): import_error_count = 0 import_warning_count = 0 + @classmethod + def get_variants(cls, config=None): + """ + Variants of this data source, to offer as separate cards + + Some data sources are one interface onto a number of distinct + collections - a database server hosting several corpora, say. Instead of + having the user select the data source and then pick the collection from + a dropdown, such a worker can return one entry per collection here, and + the create-dataset page shows one card per entry. + + Variants are not modules: they share this worker's type, data source ID + and settings, and none of the module machinery knows about them. What is + picked is passed to `get_options()` as `variant` and stored in the + dataset's `variant` parameter, so `validate_query()` and `get_items()` + can read it back. + + A worker returning variants MUST accept a `variant` keyword argument in + `get_options()`. Returning nothing (the default, and what every core data + source does) means this data source is a single card, as before. + + :param config: Configuration reader + :return dict: `{variant ID: {"title": str, "description": str, "tags": + list}}`, all keys optional; empty if this data source has no variants + """ + return {} + def process(self): """ Create 4CAT dataset from a data source diff --git a/backend/workers/datasource_metrics.py b/backend/workers/datasource_metrics.py index 464e99328..836ec8152 100644 --- a/backend/workers/datasource_metrics.py +++ b/backend/workers/datasource_metrics.py @@ -10,8 +10,6 @@ """ import os -from datetime import datetime, time, timezone - from backend.lib.worker import BasicWorker @@ -38,7 +36,6 @@ def ensure_job(cls, config=None): def work(self): self.general_stats() - self.data_stats() @staticmethod def folder_size(path='.'): @@ -77,137 +74,4 @@ def general_stats(self): "datasource": "4cat", "board": "", "date": "now" - }, constraints=["metric", "datasource", "board", "date"]) - - def data_stats(self): - """ - Go through all local datasources, and update the posts per day - if they haven't been calculated yet. These data can then be used - to calculate e.g. posts per month. - :return: - """ - - # Get a list of all database tables - all_tables = [row["tablename"] for row in self.db.fetchall( - "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema';")] - - # Check if the metrics table is already present - metrics_exists = True if "metrics" in all_tables else False - - # If not, make it. - if not metrics_exists: - self.db.execute(""" - CREATE TABLE IF NOT EXISTS metrics ( - metric text, - datasource text, - board text, - date text, - count integer - ); - - """) - - added_datasources = [row["datasource"] for row in self.db.fetchall("SELECT DISTINCT(datasource) FROM metrics")] - enabled_datasources = self.config.get("datasources.enabled", {}) - - for datasource_id in self.modules.datasources: - if datasource_id not in enabled_datasources: - continue - - datasource = self.modules.workers.get(datasource_id + "-search") - if not datasource: - continue - - # Database IDs may be different from the Datasource ID (e.g. the datasource "4chan" became "fourchan" but the database ID remained "4chan") - database_db_id = datasource.prefix if hasattr(datasource, "prefix") else datasource_id - - is_local = True if hasattr(datasource, "is_local") and datasource.is_local else False - is_static = True if hasattr(datasource, "is_static") and datasource.is_static else False - - # Only update local datasources - if is_local: - - # Some translating.. - settings_id = datasource_id - if datasource_id == "4chan": - settings_id = "fourchan" - elif datasource_id == "8chan": - settings_id = "eightchan" - - boards = [b for b in self.config.get(settings_id + "-search.boards", [])] - - # If a datasource is static (so not updated) and it - # is already present in the metrics table, we don't - # need to update its metrics anymore. - if is_static and datasource_id in added_datasources: - continue - else: - - # ------------------------- - # Posts per day metric - # ------------------------- - - # Get the name of the posts table for this datasource - posts_table = datasource_id if "posts_" + database_db_id not in all_tables else "posts_" + database_db_id - - # Count and update for every board individually - for board in boards: - - if not board: - board_sql = " board = '' OR board = NULL" - else: - board_sql = " board='" + board + "'" - - # Midnight of this day in UTC epoch timestamp - midnight = int( - datetime.combine(datetime.today(), time.min).replace(tzinfo=timezone.utc).timestamp()) - - # We only count passed days - time_sql = "timestamp < " + str(midnight) - - # If the datasource is dynamic, we also only update days - # that haven't been added yet - these are heavy queries. - if not is_static: - days_added = self.db.fetchall( - "SELECT date FROM metrics WHERE datasource = '%s' AND board = '%s' AND metric = 'posts_per_day';" % ( - database_db_id, board)) - - if days_added: - - last_day_added = max([row["date"] for row in days_added]) - last_day_added = datetime.strptime(last_day_added, '%Y-%m-%d').replace( - tzinfo=timezone.utc) - - # If the last day added is today, there's no need to update yet - if last_day_added.date() == datetime.today().replace(tzinfo=timezone.utc).date(): - self.log.info( - "No new posts per day to count for %s%s" % (datasource_id, "/" + board)) - continue - - # Change to UTC epoch timestamp for postgres query - after_timestamp = int(last_day_added.timestamp()) - - time_sql += " AND timestamp > " + str(after_timestamp) + " " - - self.log.info( - "Calculating metric posts_per_day for datasource %s%s" % (datasource_id, "/" + board)) - - # Get those counts - query = """ - SELECT 'posts_per_day' AS metric, '%s' AS datasource, board, to_char(to_timestamp(timestamp), 'YYYY-MM-DD') AS date, count(*)COUNT - FROM %s - WHERE %s AND %s - GROUP BY metric, datasource, board, date; - """ % (database_db_id, posts_table, board_sql, time_sql) - # Add to metrics table - rows = [dict(row) for row in self.db.fetchall(query)] - - if rows: - for row in rows: - self.db.upsert("metrics", row, constraints=["metric", "datasource", "board", "date"]) - - # ------------------------------- - # no other metrics added yet - # ------------------------------- - - self.job.finish() + }, constraints=["metric", "datasource", "board", "date"]) \ No newline at end of file diff --git a/common/lib/annotation.py b/common/lib/annotation.py index 07ce6762b..dcb138d9b 100644 --- a/common/lib/annotation.py +++ b/common/lib/annotation.py @@ -3,11 +3,28 @@ """ +import math import time import json from common.lib.database import Database from common.lib.exceptions import AnnotationException +from common.lib.user_input import UserInput + +# Which UserInput option each annotation field type corresponds to. +ANNOTATION_TYPES = { + "text": UserInput.OPTION_TEXT, + "textarea": UserInput.OPTION_TEXT_LARGE, + "integer": UserInput.OPTION_TEXT, + "float": UserInput.OPTION_TEXT, + "dropdown": UserInput.OPTION_CHOICE, + "checkbox": UserInput.OPTION_MULTI, +} + +# Field types whose value is a number rather than the text it was typed as. +NUMERIC_TYPES = ("integer", "float") +# Field types whose value is a single piece of free text. +TEXT_TYPES = ("text", "textarea") class Annotation: @@ -61,6 +78,16 @@ def __init__(self, data=None, annotation_id=None, db=None): self.db = db + # a numeric annotation is held as a number rather than as the text it + # was typed as, so that a value just submitted and one read from the + # database compare as the same value + if data and data.get("type") in NUMERIC_TYPES and "value" in data: + number = Annotation.parse_number(data["type"], data["value"]) + if number is None: + raise AnnotationException("'%s' is not a valid %s value" + % (data["value"], data["type"])) + data["value"] = number + new_or_updated = False if annotation_id is not None or "id" in data: @@ -169,8 +196,7 @@ def get_by_id(self, annotation_id: int): if not data: return {} - if data["type"] == "checkbox": - data["value"] = data["value"].split(",") + data["value"] = Annotation.parse_stored_value(data["type"], data["value"]) data["metadata"] = json.loads(data["metadata"]) return data @@ -192,8 +218,7 @@ def get_by_field(self, dataset_key: str, item_id: str, field_id: str) -> dict: if not data: return {} - if data["type"] == "checkbox": - data["value"] = data["value"].split(",") + data["value"] = Annotation.parse_stored_value(data["type"], data["value"]) data["metadata"] = json.loads(data["metadata"]) return data @@ -209,6 +234,14 @@ def write_to_db(self): db_data["metadata"] = json.dumps(m) if db_data["type"] == "checkbox": db_data["value"] = ",".join(db_data["value"]) + elif db_data["type"] in NUMERIC_TYPES: + # every write passes through here, so this is where a value that is + # not a number is refused, whatever route it took to get here + number = Annotation.parse_number(db_data["type"], db_data["value"]) + if number is None: + raise AnnotationException("'%s' is not a valid %s value" + % (db_data["value"], db_data["type"])) + db_data["value"] = str(number) return self.db.upsert("annotations", data=db_data, constraints=["field_id", "dataset", "item_id"]) @@ -218,6 +251,102 @@ def delete(self): """ return self.db.delete("annotations", {"id": self.id}) + @staticmethod + def parse_number(annotation_type: str, value): + """ + Read a value as the number an annotation field of this type holds + + Whatever the value was typed or stored as, since the database column is + text and a value made by a processor may be a number already. + + :param str annotation_type: `integer` or `float` + :param value: The value to read + + :return: The number, an empty string if there is no value at all, or + `None` if the value cannot be read as a number of this type + """ + if value is None: + return "" + + if isinstance(value, str): + value = value.strip() + if not value: + return "" + + try: + number = float(value) + except (TypeError, ValueError): + return None + + # a dataset column is no place for infinity or a NaN + if not math.isfinite(number): + return None + + if annotation_type != "integer": + return number + + # an integer that came in as one is kept as it is, since going through + # a float would round the very large ones + if isinstance(value, int): + return int(value) + + # rounded half away from zero (5.5 -> 6, -5.5 -> -6), which is what + # people expect of a number; Python's own round() rounds half to even + return int(number + 0.5) if number >= 0 else -int(-number + 0.5) + + @staticmethod + def parse_stored_value(annotation_type: str, value): + """ + Read a value as it is stored in the database into what it stands for + + Checkbox values are stored as a comma-separated list and numeric values + as text; everything else is the string it says it is. + + :param str annotation_type: The type of the field the value belongs to + :param value: The value as the database holds it + + :return: The value as this type of field holds it + """ + if annotation_type == "checkbox": + return value.split(",") + + if annotation_type in NUMERIC_TYPES: + number = Annotation.parse_number(annotation_type, value) + # a stored value that is not a number reads as no annotation rather + # than breaking everything that reads the dataset + return "" if number is None else number + + return value + + @staticmethod + def type_change_effect(old_type: str, new_type: str) -> str: + """ + What changing a field from one type to another does to its annotations + + The one place that decides this, so that what a change is announced to + do and what it then does cannot drift apart. + + :param str old_type: The type the field has now + :param str new_type: The type it would get + + :return str: `keep` if the values survive as they are, `convert` if + they can be read as values of the new type - the ones that + cannot are deleted - and `delete` if none of them survive + """ + if old_type == new_type: + return "keep" + + # any text that reads as a number is kept as one, and an integer and a + # float can always be read as each other + if new_type in NUMERIC_TYPES and old_type in (*TEXT_TYPES, *NUMERIC_TYPES): + return "convert" + + # text stays text however long it may be, and a number reads as text + if old_type in (*TEXT_TYPES, *NUMERIC_TYPES) and new_type in TEXT_TYPES: + return "keep" + + return "delete" + @staticmethod def get_annotations_for_dataset(db: Database, dataset_key: str, item_id=None, before=0) -> list: @@ -258,8 +387,7 @@ def get_annotations_for_dataset(db: Database, dataset_key: str, item_id=None, be return [] for i in range(len(data)): - if data[i]["type"] == "checkbox": - data[i]["value"] = data[i]["value"].split(",") + data[i]["value"] = Annotation.parse_stored_value(data[i]["type"], data[i]["value"]) data[i]["metadata"] = json.loads(data[i]["metadata"]) return [Annotation(data=d, db=db) for d in data] @@ -305,14 +433,13 @@ def update_annotations_via_fields(dataset_key: str, old_fields: dict, new_fields :returns int: How many records were affected. """ - text_fields = ["textarea", "text"] - # If old and new fields are identical, do nothing. if old_fields == new_fields: return 0 fields_to_delete = set() # Delete all annotations with this field ID fields_to_update = {} # Update values of annotations with this field ID + fields_to_convert = {} # Read values of annotations with this field ID as another type old_options = {} # Loop through the old annotation fields @@ -326,12 +453,17 @@ def update_annotations_via_fields(dataset_key: str, old_fields: dict, new_fields field_id = old_field_id new_field = new_fields[field_id] - # If the annotation type has changed, also delete existing annotations, - # except between text and textarea, where we can just change the type and keep the text. + # If the annotation type has changed, what happens to the values + # that were annotated with it depends on what it changed into: they + # may survive as they are, be readable as the new type, or be + # nothing the new type can hold at all. if old_field["type"] != new_field["type"]: - if old_field["type"] not in text_fields and new_field["type"] not in text_fields: + effect = Annotation.type_change_effect(old_field["type"], new_field["type"]) + if effect == "delete": fields_to_delete.add(field_id) continue + elif effect == "convert": + fields_to_convert[field_id] = new_field["type"] # Loop through all the key/values in the new field settings # and update in case it's different from the old values. @@ -373,8 +505,24 @@ def update_annotations_via_fields(dataset_key: str, old_fields: dict, new_fields if fields_to_delete: Annotation.delete_many(db, field_id=list(fields_to_delete)) - # Write changes to fields to database count = 0 + + # Read the values of fields that became numeric as numbers. What can be + # read as one is kept (an integer field rounds what it is given); what + # cannot is not a value the field can hold anymore, so it goes. + for field_id, new_type in fields_to_convert.items(): + if field_id in fields_to_delete: + continue + + for annotation in db.fetchall("SELECT id, value FROM annotations WHERE dataset = %s AND field_id = %s", + (dataset_key, field_id)): + number = Annotation.parse_number(new_type, annotation["value"]) + if number is None: + db.delete("annotations", {"id": annotation["id"]}) + else: + count += db.update("annotations", {"value": str(number)}, where={"id": annotation["id"]}) + + # Write changes to fields to database if fields_to_update: for field_id, updates in fields_to_update.items(): diff --git a/common/lib/compatibility.py b/common/lib/compatibility.py index 279c78474..9eb5f2b4e 100644 --- a/common/lib/compatibility.py +++ b/common/lib/compatibility.py @@ -1,75 +1,71 @@ """ Declarative processor compatibility. -A Compatibility object describes the conditions under which a processor can run -on a dataset: - -* the data shape it consumes -- the dataset's type, file extension, media type, - datasource, and any columns it needs; -* the environment it needs -- external executables and 4CAT configuration - settings; -* the follow-up processors that are most relevant for its output, and any that - should never be offered. - -A processor declares one as its `compatibility` class attribute, for example:: - - compatibility = Compatibility( - media_types={"video"}, - type_prefixes={"video-downloader"}, - required_settings={("video-downloader.ffmpeg_path", is_executable)}, - ) - -BasicProcessor.is_compatible_with() evaluates it. A processor whose -requirements cannot be expressed this way -- for example one that must inspect -a dataset's ancestry -- may override is_compatible_with() instead; the override -is used in preference to the attribute. - -`_maybe_call`: a utility function to safely read attributes or call methods on a - module, handling cases where the attribute or method might not exist or raise an exception. -Normally a `module` is a DataSet, but the values read here (its type, extension, media type -and so on) are also available on a processor class, so a processor can be checked even when -no dataset exists yet. +A processor says what it accepts as a `Compatibility` -- the dataset type, file +extension, media type, datasource, columns it needs, and the settings/executables its +environment needs. `Compatibility.check(subject)` compares that against a `subject`, +which is one of two things: + +* a live dataset (wrapped in `DatasetShape`), when 4CAT decides at run time whether a + processor can run on a dataset; +* a processor's declared output (a `Shape`, built in outputs.py from its `Output`), + when the processor map works out which processors can follow which without running + anything. + +Both answer the same handful of questions -- extension, media, columns, and so on -- +each a real value or `UNKNOWN` when a declared output has not pinned it down. `check` +returns "yes", "no" or "maybe": a live dataset knows all its values so the answer is +only ever yes/no, but a declared output can leave things open, which is where "maybe" +comes from. There is one comparison, written once, so the run-time check and the map +can never disagree. + +A processor whose acceptance can't be expressed this way (e.g. it must walk a dataset's +ancestry) keeps a custom `is_compatible_with` method instead; that wins at run time. """ from __future__ import annotations import shutil from dataclasses import dataclass -from typing import Iterable, List, Optional +from functools import cached_property +from typing import Iterable, Optional -def _maybe_call(module, method, **kwargs): - """ - Read `module.method` without assuming it exists. +# Marks a property a declared output has not pinned down (a filter's extension, a deep +# child's datasource, ...). It is *only* ever produced by a declared output -- a live +# dataset always has a concrete value -- so it is what turns a "no" into a "maybe". +UNKNOWN = object() + +# The columns that make a CSV rankable; three of them must be present (see get_columns +# / is_rankable on DataSet). Rankability is worked out from the columns and extension a +# subject has, so it never needs declaring separately. +_RANK_COLUMNS = {"date", "value", "item"} + - Calls it and returns the result when it is a method, returns the value when - it is a plain attribute, and returns None when it is missing or raises. A - DataSet exposes these as methods; a processor class exposes some of them as - well, and this keeps the same check working for both. Any keyword arguments - are forwarded to the call (e.g. is_rankable(multiple_items=False)). +def _maybe_call(subject, method, **kwargs): """ - attr = getattr(module, method, None) + Read `subject.method` without assuming it exists: call it when it is a method, + return it when it is a plain attribute, and return None when it is missing or + raises. A DataSet exposes these as methods, so this keeps DatasetShape simple. + """ + attr = getattr(subject, method, None) if attr is None: return None if callable(attr): try: return attr(**kwargs) except Exception: - # deliberately swallowed: a failing get_columns()/is_rankable() reads - # as "cannot determine"; could pass logger here to debug in case there is - # a real bug, but this caller is expected to handle None as "not met" + # a failing get_columns()/get_extension() reads as "no value", never a crash return None return attr -# TODO: memoize shutil.which() (used by is_executable / ExecutableSibling) -- its -# result is constant per process, so a cached wrapper (positive results only, to -# avoid stale negatives if an executable is installed without a restart) would -# avoid repeated $PATH scans on video-heavy pages. +# TODO: memoize shutil.which() (used by is_executable / ExecutableSibling) -- its result +# is constant per process, so a cached wrapper (positive results only) would avoid +# repeated $PATH scans on video-heavy pages. def is_executable(path): """ - Matcher for `required_settings`: the setting's value must point to an - executable found on the system (resolved with `shutil.which`). An unset or - empty value fails safely, e.g.:: + Matcher for `required_settings`: the setting's value must point to an executable + found on the system (resolved with `shutil.which`). An unset value fails safely:: required_settings={("video-downloader.ffmpeg_path", is_executable)} """ @@ -78,16 +74,12 @@ def is_executable(path): class ExecutableSibling: """ - Matcher for `required_settings`: the configured executable must resolve (via - `shutil.which`) AND a sibling executable must exist next to it, found by - swapping the name in the resolved path. For tools that ship together, e.g. - ffprobe alongside ffmpeg:: + Matcher for `required_settings`: the configured executable must resolve AND a sibling + executable must exist next to it, found by swapping the name in the resolved path. + For tools that ship together, e.g. ffprobe alongside ffmpeg:: required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))} - - The matcher protocol is a one-argument callable, so arguments are passed via - the constructor; `name`/`sibling` stay readable for a future UI. None-safe. """ def __init__(self, name, sibling): @@ -98,9 +90,8 @@ def __call__(self, path): resolved = shutil.which(path) if path else None if not resolved: return False - # if `name` is not in the resolved path, the rsplit/join below leaves it - # unchanged and we would re-check the same executable -- a false positive - # that never actually locates the sibling. Fail instead. + # if `name` is not in the resolved path the swap below is a no-op and we would + # re-check the same executable -- a false positive that never finds the sibling if self.name not in resolved: return False return shutil.which(self.sibling.join(resolved.rsplit(self.name, 1))) is not None @@ -111,16 +102,13 @@ class Compatibility: """ Declarative compatibility specification for a processor. - Any axis left unset (None, or empty) is not checked. - - The four identity axes -- types, type_prefixes, media_types and - datasources -- describe what kind of dataset the processor accepts. If any - of them are set, the module must match at least one (they are OR-ed). - Every other axis is an additional requirement that must also hold (they are - AND-ed). + Any axis left unset (None, or empty) is not checked. The four identity axes -- + types, type_prefixes, media_types and datasources -- describe what kind of input the + processor accepts; if any are set, the subject must match at least one (they are + OR-ed). Every other axis is an additional requirement that must also hold (AND-ed). """ - # --- Identity axes: consumed data shape (the module must match one of these) --- + # --- Identity axes: what kind of input (match at least one of these) --- # Dataset types the processor accepts, matched exactly. types: Optional[Iterable[str]] = None # Dataset type prefixes the processor accepts, matched with str.startswith. @@ -129,233 +117,412 @@ class Compatibility: media_types: Optional[Iterable[str]] = None # Datasources the processor accepts, e.g. {"4chan", "reddit"}. datasources: Optional[Iterable[str]] = None - # Collectors types (a dataset whose type ends in -search or -import) - # Compare with top_dataset_only, which reads key_parent; the two cover nearly - # the same datasets but differ in role (identity/OR vs gate/AND) + # Accepts a collector's output (a dataset whose type ends in -search or -import). is_collector: bool = False - # --- Shape gates (structural checks) --- - # Parent dataset types this processor CANNOT run on -- a hard gate - # (is_compatible_with returns False). Use when the processor would fail or - # produce garbage on that type (e.g. download_videos on telegram-search). - # For a soft filter use excluded_followups on the producer instead. + # --- Gates: extra conditions that must all hold --- + # Dataset types this processor CANNOT run on -- a hard veto. Use when it would fail + # or produce garbage on that type (e.g. download_videos on telegram-search). For a + # soft "don't suggest" use excluded_followups on the producer instead. excluded_types: Iterable[str] = () - - # --- DataSet required gates --- - # The ground truth requires an existing DataSet to read its produced data - # TODO: processors could declare these attributes more explicitly - # When True, the processor only accepts a top-level dataset (one with no parent). + # Accepts only a top-level dataset (no parent) / only a non-top-level (child) one. top_dataset_only: bool = False - # When True, the processor only accepts a non-top-level (child) dataset -- the - # inverse of top_dataset_only. child_only: bool = False # Result-file extensions the processor accepts, e.g. {"csv", "ndjson"}. extensions: Optional[Iterable[str]] = None - # --- Result file gates (reading the actual file is required) - # TODO: processors again could point to these attributes more explicitly if known - # dataset and cannot be resolved from a processor class -- see requires_dataset) --- - # When set, is_rankable() must equal this (read from the result file). None = not checked. + # --- Column/rankability gates (read from the produced data) --- + # When set, the subject's rankability must equal this. Worked out from its columns + # and extension (a CSV with date/value/item), so it is never declared directly. rankable: Optional[bool] = None - # Forwarded to is_rankable(multiple_items=...) when `rankable` is set. False - # restricts to single-value rankings (rejecting multi-column word_1/word_2/... rankings). + # Forwarded to the rank check: False rejects multi-value word_1/word_2/... rankings. rankable_multiple_items: bool = True - # Columns that must ALL be present in the dataset, read from its columns. + # Columns that must ALL be present / of which AT LEAST ONE must be present. requires_all_columns: Iterable[str] = () - # Columns of which AT LEAST ONE must be present, read from its columns. requires_any_columns: Iterable[str] = () - # --- Environment requirements --- - # Executables that must be found on the system path (checked with shutil.which). + # --- Environment requirements (about the machine, not the dataset) --- + # Executables that must be on the system path (checked with shutil.which). required_packages: Iterable[str] = () - # Configuration the processor needs. Each entry is either a setting key, - # which must resolve to a truthy value, or a (key, expected) pair. The - # expected part may be a single value the setting must equal, a collection - # the setting's value must be in, or a function that receives the value and - # returns whether it is acceptable. + # Configuration the processor needs. Each entry is a setting key (which must be + # truthy) or a (key, expected) pair; `expected` may be a value the setting must + # equal, a collection it must be in, or a function that validates it. required_settings: Iterable = () - # --- Follow-up processors --- + # --- Follow-up hints (not a requirement -- they describe this processor's output) --- # Processor types to recommend first as next steps for this processor's output. preferred_followups: Iterable[str] = () - # Processor types never SUGGESTED as follow-ups after this one -- a soft - # filter (affects the suggestion list only; is_compatible_with is unchanged, - # so they can still be run directly). Use for "a more specific processor is - # preferred here" (e.g. tiktok-search excludes the generic video-downloader). - # For "that processor would fail on this output", use its excluded_types (hard). + # Processor types never suggested after this one (a soft filter; they can still be + # run directly). For "would fail on this output" use that processor's excluded_types. excluded_followups: Iterable[str] = () - @property - def requires_dataset_result_file(self) -> bool: + # -- the one comparison -- + + def check(self, subject) -> str: """ - Whether fully evaluating this spec needs the dataset's produced data. - - True when `rankable`, `requires_all_columns`, or `requires_any_columns` is - set: all are read from the produced result file, so they cannot be - resolved from a processor class alone. (Shape axes such as - top_dataset/extension also read instance state, but they are recoverable - from a dataset's shape; only these need the produced data, so only they - are counted here.) A consumer that reasons about processors without real - datasets (e.g. a processor map) can use this to mark those axes as - undecided rather than treating them as failed. + Compare `subject` against this spec, returning "yes", "no" or "maybe". + + `subject` is a live dataset (DatasetShape) or a declared output (Shape); both + answer the same questions, a real value or UNKNOWN when an output has not said. + "no" means a condition is definitely unmet; "maybe" means everything known + passes but the output left something we check undefined; "yes" means all met. A + live dataset knows all its values, so for a dataset the answer is only yes/no. + + Only the data shape is compared here; the environment is environment_ok(). """ - return (self.rankable is not None - or bool(self.requires_all_columns) - or bool(self.requires_any_columns)) + maybe = False + + # identity -- if we name any kind of input, the subject must be one of them + if self._accepts_a_kind(): + kind = self._kind_result(subject) + if kind == "no": + return "no" + if kind == "maybe": + maybe = True + + # a type we refuse outright + if subject.type in set(self.excluded_types): + return "no" + + # structural position + if self.top_dataset_only: + if subject.top_level is UNKNOWN: + maybe = True + elif not subject.top_level: + return "no" + if self.child_only: + if subject.top_level is UNKNOWN: + maybe = True + elif subject.top_level: + return "no" + + # file extension + if self.extensions: + if subject.extension is UNKNOWN: + maybe = True + elif subject.extension not in set(self.extensions): + return "no" + + # columns the processor needs to read + if self.requires_all_columns or self.requires_any_columns: + columns = self._columns_result(subject) + if columns == "no": + return "no" + if columns == "maybe": + maybe = True + + # rankable (a CSV with date/value/item columns) + if self.rankable is not None: + rankable = _is_rankable(subject, self.rankable_multiple_items) + if rankable is UNKNOWN: + maybe = True + elif rankable != self.rankable: + return "no" + + return "maybe" if maybe else "yes" def is_compatible_with(self, module, config=None) -> bool: """ - Return whether `module` meets every requirement in this specification. - - `module` is normally a DataSet but may be a processor class. `config` - is the configuration reader, or None when none is available. + Whether a real dataset can be run on. A live dataset knows all its values, so + check() is only ever "yes"/"no" here. `module` is normally a DataSet. """ - return not self.unmet_requirements(module, config=config) + return self.check(DatasetShape(module)) == "yes" and self.environment_ok(config) - def unmet_requirements(self, module, config=None, first_only=True) -> List[str]: + def environment_ok(self, config=None) -> bool: """ - Return the requirements `module` does not meet, as readable strings. - - An empty list means `module` is compatible. Each string names one thing - that is missing -- a wrong dataset type, an absent column, a setting - that is not configured, and so on. - - The checks run in three tiers, cheapest first so the short-circuit can - skip later work once something fails: - - 1. structural -- the dataset's shape (type, extension, parent, - datasource); cheap, no result-file read. (Several still read instance - state -- is_top_dataset() -> key_parent, get_extension(), parameters -- - so on a bare processor class they return a stub, not a real answer.) - 2. dataset-required -- `rankable`, `requires_all_columns`, and - `requires_any_columns`, read from the produced result file, so they - need a materialized DataSet (see `requires_dataset_result_file`); - 3. environment -- configuration settings and system executables. - - By default the method returns as soon as one requirement is unmet -- - enough for the yes/no `is_compatible_with`. Pass `first_only=False` to - collect every unmet requirement -- used to explain why a module is not - compatible. + Whether the system has the settings and executables the processor needs. This is + about the machine, not the dataset, so the map surfaces it as a note on the + processor rather than removing a connection. """ - reasons: List[str] = [] - if module is None: - return ["no dataset provided"] - - # --- tier 1: structural shape (cheap; no result-file read) --- - - # if the processor names the kinds of dataset it accepts, the module - # must be one of them - if self._identity_declared() and not self._identity_matches(module): - reasons.append("dataset type/media is not accepted") - if first_only: - return reasons - - if self.excluded_types and getattr(module, "type", None) in set(self.excluded_types): - reasons.append("does not run on dataset type: %s" % getattr(module, "type", None)) - if first_only: - return reasons - - if self.top_dataset_only and not _maybe_call(module, "is_top_dataset"): - reasons.append("requires a top-level dataset") - if first_only: - return reasons - - if self.child_only and _maybe_call(module, "is_top_dataset"): - reasons.append("requires a child (non-top-level) dataset") - if first_only: - return reasons - - if self.extensions is not None: - extension = _maybe_call(module, "get_extension") - if extension not in set(self.extensions): - reasons.append("requires extension: %s" % ", ".join(self.extensions)) - if first_only: - return reasons - - # --- tier 2: dataset-required (read from the result file; cannot be - # resolved from a processor class -- see requires_dataset_result_file) --- - - if self.rankable is not None: - if bool(_maybe_call(module, "is_rankable", multiple_items=self.rankable_multiple_items)) != self.rankable: - reasons.append( - "requires a rankable dataset" if self.rankable - else "requires a non-rankable dataset" - ) - if first_only: - return reasons - - if self.requires_all_columns or self.requires_any_columns: - columns = _maybe_call(module, "get_columns") or [] - missing = [column for column in self.requires_all_columns if column not in columns] - if missing: - reasons.append("requires all column(s): %s" % ", ".join(missing)) - if first_only: - return reasons - if self.requires_any_columns and not any(column in columns for column in self.requires_any_columns): - reasons.append("requires any of column(s): %s" % ", ".join(self.requires_any_columns)) - if first_only: - return reasons - - # --- tier 3: environment (needs config/system, not a DataSet; the - # executable matchers here can be expensive, so this tier runs last. - # TODO: cheap setting reads could be split out ahead of those matchers) --- for requirement in self.required_settings: key, expected = (requirement, None) if isinstance(requirement, str) else requirement value = config.get(key) if config is not None else None - # no expected value; just check that the setting is truthy if expected is None: met = bool(value) - # a function that validates the value (e.g. is_executable / ExecutableSibling) elif callable(expected): met = bool(expected(value)) - # a collection of acceptable values elif isinstance(expected, (set, frozenset, list, tuple)): met = value in expected - # a single expected value else: met = value == expected if not met: - reasons.append("requires setting: %s" % key) - if first_only: - return reasons - + return False for package in self.required_packages: if not shutil.which(package): - reasons.append("requires package: %s" % package) - if first_only: - return reasons + return False + return True + + @property + def requires_dataset_result_file(self) -> bool: + """ + Whether fully deciding this spec needs the produced data -- true when it gates on + rankability or columns, both read from the result file. The map exposes this so + the UI can say "you'll only know for certain once it runs". + """ + return (self.rankable is not None + or bool(self.requires_all_columns) + or bool(self.requires_any_columns)) + + # -- helpers for check(); each returns "yes"/"no"/"maybe" -- + + def _accepts_a_kind(self) -> bool: + """Whether the spec names any kind of input it accepts.""" + return self.is_collector or any(axis is not None for axis in + (self.types, self.type_prefixes, self.media_types, self.datasources)) + + def _kind_result(self, subject) -> str: + """Identity is OR: "yes" if the subject is any kind we accept, "no" only if it is + definitely none, else "maybe".""" + maybe = False + for outcome in self._kind_checks(subject): + if outcome == "yes": + return "yes" + if outcome == "maybe": + maybe = True + return "maybe" if maybe else "no" + + def _kind_checks(self, subject): + """One outcome per identity axis the spec declares, cheapest first.""" + if self.types is not None: + yield "yes" if subject.type in set(self.types) else "no" + if self.type_prefixes is not None: + yield "yes" if (subject.type and subject.type.startswith(tuple(self.type_prefixes))) else "no" + if self.media_types is not None: + yield _media_result(set(self.media_types), subject.media) + if self.datasources is not None: + if subject.datasource is UNKNOWN: + yield "maybe" + else: + yield "yes" if subject.datasource in set(self.datasources) else "no" + if self.is_collector: + if subject.from_collector is UNKNOWN: + yield "maybe" + else: + yield "yes" if subject.from_collector else "no" + + def _columns_result(self, subject) -> str: + """Does the subject have the columns we need? A missing column is a definite "no" + only when we know its columns are the complete set (a real dataset, or an output + that has none); an output floor might still gain the column, so it is "maybe".""" + columns = subject.columns + if columns is UNKNOWN: + return "maybe" + complete = subject.columns_are_all + outcome = "yes" + if self.requires_all_columns and not set(self.requires_all_columns) <= columns: + if complete: + return "no" + outcome = "maybe" + if self.requires_any_columns and not (columns & set(self.requires_any_columns)): + if complete: + return "no" + outcome = "maybe" + return outcome + + +def _media_result(accepted, media) -> str: + """Compare a set of accepted media against a subject's media, which may be one value, + a set of possible values (a media archive that could hold image OR video), or + UNKNOWN.""" + if media is UNKNOWN: + return "maybe" + values = set(media) if isinstance(media, (set, frozenset)) else {media} + if values <= accepted: + return "yes" + if values & accepted: + return "maybe" + return "no" + + +def _is_rankable(subject, multiple_items): + """Whether the subject is rankable: a CSV with at least three of the ranking columns. + UNKNOWN when the output has not pinned its extension or columns.""" + ranking = _RANK_COLUMNS | ({"word_1"} if multiple_items else set()) + extension, columns = subject.extension, subject.columns + if extension is UNKNOWN: + return UNKNOWN # can't tell whether it's a CSV + if extension != "csv": + return False # a known non-CSV is never rankable + if columns is UNKNOWN: + return UNKNOWN # a CSV, but its columns aren't declared + if len(columns & ranking) >= 3: + return True + return False if subject.columns_are_all else UNKNOWN + + +class DatasetShape: + """ + A live dataset, presented as the plain properties check() reads. Every value comes + from the dataset's own methods, so it is always concrete -- a real dataset is never + "maybe". Read lazily and cached, so the slower reads (columns) happen only for a spec + that actually needs them. + """ - return reasons + columns_are_all = True # a real dataset's columns are all the columns it has - def _identity_declared(self) -> bool: - """Whether the processor names any kind of dataset it accepts.""" - return self.is_collector or any( - axis is not None - for axis in (self.types, self.type_prefixes, self.media_types, self.datasources) - ) + def __init__(self, dataset): + self._dataset = dataset - def _identity_matches(self, module) -> bool: - """Whether the module is one of the kinds of dataset the processor accepts.""" - module_type = getattr(module, "type", None) + @property + def type(self): + return getattr(self._dataset, "type", None) - if self.types is not None and module_type in set(self.types): - return True + @cached_property + def extension(self): + return _maybe_call(self._dataset, "get_extension") - if self.type_prefixes is not None and module_type is not None \ - and any(module_type.startswith(prefix) for prefix in self.type_prefixes): - return True + @cached_property + def media(self): + return _maybe_call(self._dataset, "get_media_type") or getattr(self._dataset, "media_type", None) - if self.media_types is not None: - media = _maybe_call(module, "get_media_type") or getattr(module, "media_type", None) - if media in set(self.media_types): - return True + @cached_property + def datasource(self): + parameters = getattr(self._dataset, "parameters", None) or {} + return parameters.get("datasource") if isinstance(parameters, dict) else None + + @cached_property + def top_level(self): + return bool(_maybe_call(self._dataset, "is_top_dataset")) + + @cached_property + def from_collector(self): + return bool(_maybe_call(self._dataset, "is_from_collector")) + + @cached_property + def columns(self): + columns = _maybe_call(self._dataset, "get_columns") + return frozenset(columns) if columns else frozenset() + + +@dataclass(frozen=True) +class Shape: + """ + A processor's declared output, as the plain properties check() reads. A value is + UNKNOWN when the processor's Output leaves it open (a filter's extension, say). + `columns_are_all` is True only when the output has NO columns; a declared column set + is a floor (at least these), so it is False. Built from an Output in outputs.py, or + inferred from the class for a processor that declares none. + """ + + type: Optional[str] + extension: object = UNKNOWN # a str, or UNKNOWN + media: object = UNKNOWN # a str, a set of str, or UNKNOWN + datasource: object = UNKNOWN # a str, or UNKNOWN + top_level: object = UNKNOWN # a bool, or UNKNOWN + from_collector: object = UNKNOWN # a bool, or UNKNOWN + columns: object = UNKNOWN # a frozenset, or UNKNOWN + columns_are_all: bool = False + produces_file: bool = True - if self.datasources is not None: - parameters = getattr(module, "parameters", None) or {} - if isinstance(parameters, dict) and parameters.get("datasource") in set(self.datasources): - return True - if self.is_collector and _maybe_call(module, "is_from_collector"): +# --------------------------------------------------------------------------- +# Helpers shared with outputs.py (which builds a Shape and needs to know a processor's +# collector-ness), and the spec serialiser the catalogue displays. +# --------------------------------------------------------------------------- + +def _is_collector_type(type_id) -> bool: + """A collector/datasource output: its type ends in `-search` or `-import`.""" + return bool(type_id) and (type_id.endswith("-search") or type_id.endswith("-import")) + + +def _declared_class_value(processor, name): + """ + The value of class attribute `name` if the processor (or a parent class below + BasicProcessor) actually sets it, otherwise None. This tells a real choice apart from + a value merely inherited from BasicProcessor (the `extension = "csv"` default every + processor gets for free). A shared parent such as Search or a base filter counts as a + real choice; BasicProcessor itself, and anything above it, does not. + """ + cls = processor if isinstance(processor, type) else type(processor) + try: + from backend.lib.processor import BasicProcessor + except Exception: + BasicProcessor = None + for klass in cls.__mro__: + if klass is object or klass is BasicProcessor or klass.__name__ == "BasicProcessor": + break + if name in vars(klass): + return vars(klass)[name] + return None + + +def is_collector(processor) -> bool: + """ + Whether a processor starts a chain -- a Search subclass, or (fallback) a + -search/-import type. This is about its role as a starting point, separate from + whether its *output* counts as a collector's (a filter's result can be made to look + like one, so that is left unknown for a filter). + """ + try: + from backend.lib.search import Search + if isinstance(processor, type) and issubclass(processor, Search): return True + except Exception: + pass + return _is_collector_type(getattr(processor, "type", None)) + + +def is_declaratively_compatible(processor) -> bool: + """ + Whether the processor relies solely on its declared `compatibility` -- i.e. it does + NOT keep a custom `is_compatible_with`. When False it escapes into runtime logic the + specs can't see, so a map built from specs alone can only approximate edges into it. + """ + try: + from backend.lib.processor import BasicProcessor + except Exception: + return True + own = getattr(getattr(processor, "is_compatible_with", None), "__func__", None) + base = getattr(BasicProcessor.is_compatible_with, "__func__", None) + has_override = own is not None and base is not None and own is not base + return not has_override + + +def describe_spec(spec) -> Optional[dict]: + """ + Serialise a Compatibility to a dict of only its *declared* axes, for display and as + the "requirement" label in the map. Defaults and empties are omitted, so what shows + is exactly what the processor opted into. Returns None for an undeclared spec. + + (test_describe_spec_covers_every_compatibility_axis asserts every axis appears here.) + """ + if spec is None: + return None - return False + def norm(value): + if isinstance(value, (set, frozenset)): + return sorted(value) + if isinstance(value, (list, tuple)): + return list(value) + return value + + declared = {} + for axis in ("types", "type_prefixes", "media_types", "datasources"): + value = getattr(spec, axis, None) + if value: + declared[axis] = norm(value) + if getattr(spec, "is_collector", False): + declared["is_collector"] = True + if getattr(spec, "extensions", None): + declared["extensions"] = norm(spec.extensions) + for gate in ("top_dataset_only", "child_only"): + if getattr(spec, gate, False): + declared[gate] = True + if getattr(spec, "excluded_types", None): + declared["excluded_types"] = norm(spec.excluded_types) + if getattr(spec, "rankable", None) is not None: + declared["rankable"] = spec.rankable + if getattr(spec, "requires_all_columns", None): + declared["requires_all_columns"] = norm(spec.requires_all_columns) + if getattr(spec, "requires_any_columns", None): + declared["requires_any_columns"] = norm(spec.requires_any_columns) + if getattr(spec, "required_settings", None): + keys = [r if isinstance(r, str) else r[0] for r in spec.required_settings] + declared["required_settings"] = sorted(keys) + if getattr(spec, "required_packages", None): + declared["required_packages"] = norm(spec.required_packages) + for followup in ("preferred_followups", "excluded_followups"): + value = getattr(spec, followup, None) + if value: + declared[followup] = norm(value) + return declared diff --git a/common/lib/config_definition.py b/common/lib/config_definition.py index e133966a7..ea905e7ac 100644 --- a/common/lib/config_definition.py +++ b/common/lib/config_definition.py @@ -390,12 +390,6 @@ "global": True }, # Explorer settings - "explorer.basic-explanation": { - "type": UserInput.OPTION_INFO, - "help": "4CAT's Explorer feature lets you navigate and annotate datasets as if they " - "appared on their original platform. This is intended to facilitate qualitative " - "exploration and manual coding." - }, "explorer.max_posts": { "type": UserInput.OPTION_TEXT, "default": 100000, @@ -411,19 +405,6 @@ "coerce_type": int, "tooltip": "Number of items to display per page" }, - "explorer.config_explanation": { - "type": UserInput.OPTION_INFO, - "help": "Data sources use Explorer templates that determine how they look and what information is " - "displayed. Explorer templates consist of [custom HTML templates](https://github.com/" - "digitalmethodsinitiative/4cat/tree/master/webtool/templates/explorer/datasource-templates) and " - "[custom CSS files](https://github.com/digitalmethodsinitiative/4cat/tree/master/webtool/static/css/" - "explorer). If no template is available for a data source, a generic template is used " - "made of [this HTML file](https://github.com/digitalmethodsinitiative/4cat/blob/master/webtool/" - "templates/explorer/datasource-templates/generic.html) and [this CSS file](https://github.com/" - "digitalmethodsinitiative/4cat/tree/master/webtool/static/css/explorer/generic.css).\n\n" - "You can request a new data source Explorer template by [creating a GitHub issue](https://github.com/" - "digitalmethodsinitiative/4cat/issues) or adding them yourself and opening a pull request." - }, # Web tool settings # These are used by the FlaskConfig class in config.py # Flask may require a restart to update them @@ -675,12 +656,6 @@ "help": "4CAT home page", "default": "about" }, - "ui.inline_preview": { - "type": UserInput.OPTION_TOGGLE, - "help": "Show inline preview", - "default": False, - "tooltip": "Show main dataset preview directly on dataset pages, instead of behind a 'preview' button" - }, "ui.offer_anonymisation": { "type": UserInput.OPTION_TOGGLE, "help": "Offer anonymisation options", diff --git a/common/lib/dataset.py b/common/lib/dataset.py index f29081399..737a162c0 100644 --- a/common/lib/dataset.py +++ b/common/lib/dataset.py @@ -2378,6 +2378,28 @@ def get_media_type(self): # Default to text return self.parameters.get("media_type", "text") + def set_media_type(self, media_type): + """ + Set the media type of this dataset's file. + + For processors whose output media is only known while running (for example, + from the type of an uploaded file). The processor's declared `output` states + the media it can produce; this pins the one it actually produced. Stored on + the instance for the current run and in the parameters so later reads (after + a reload) return it too. + + :param str media_type: Media type, e.g. "image" + :return str: The media type that was set + """ + self.media_type = media_type + self.parameters["media_type"] = media_type + self.db.update( + "datasets", + data={"parameters": json.dumps(self.parameters)}, + where={"key": self.key}, + ) + return media_type + def get_media_from_children(self, item_ids=[]) -> dict: """ Returns a list of media filenames that have been downloaded via video or image download child processors @@ -2753,10 +2775,11 @@ def save_annotation_fields(self, new_fields: dict, add=False) -> int: % field_id ) - # Check if fields are removed + # Check if fields are removed or changed into another type; both mean + # the annotations made with them have to be deleted or rewritten if not add and old_fields: - for field_id in old_fields.keys(): - if field_id not in new_fields: + for field_id, old_field in old_fields.items(): + if field_id not in new_fields or old_field.get("type") != new_fields[field_id].get("type"): changes = True # Make sure to do nothing to processor-generated annotations; these must remain 'traceable' to their origin diff --git a/common/lib/module_map.py b/common/lib/module_map.py new file mode 100644 index 000000000..e85a26d30 --- /dev/null +++ b/common/lib/module_map.py @@ -0,0 +1,527 @@ +""" +Query layer for the module map: turn the declarative Compatibility + Output specs +into the questions a user-facing catalogue asks -- browse/search, "how do I run this +one" (what it accepts + where to start), and "what can run on its output". + +The matcher it builds on lives in common/lib/compatibility.py and the output shapes in +common/lib/outputs.py; this module only builds the producer->consumer graph and reads +it. Computed purely by inspection -- no datasets, no database. Each link carries a +yes/maybe answer. + +Two ideas keep the answers readable: + +* The declared spec is the label. Rather than re-deriving why a producer matched, + "how to run" shows the processor's own `describe_spec` (its declared requirement) + and lists the producers flat, split only on the one honest distinction: a data + source you start from vs. another processor you run first. +* Filters are transparent. A filter runs on almost any dataset and keeps its format, + so it never changes what you can run next and can be inserted anywhere. Filters are + therefore kept out of the normal producer lists and noted separately -- their own + group under "what can run on this", their own short "how to run". +""" +import logging + +from collections import defaultdict + +from common.lib.compatibility import ( + Compatibility, + describe_spec, + is_collector, + is_declaratively_compatible, + UNKNOWN, +) +from common.lib.outputs import describe_output, Filter + +_DEFAULT_SPEC = Compatibility(top_dataset_only=True) + +#: Each condition a processor can declare, as a short plain phrase. A key not +#: listed here falls back to a generic "key: value" (see describe_requirements). +_REQUIREMENT_LABELS = { + "types": lambda value: "type is %s" % _listed(value), + "type_prefixes": lambda value: "type starts with %s" % _listed(value), + "media_types": lambda value: "media is %s" % _listed(value), + "datasources": lambda value: "from data source %s" % _listed(value), + "is_collector": lambda value: "must be a data source", + "extensions": lambda value: "format is %s" % _listed(value), + "top_dataset_only": lambda value: "top-level dataset only", + "child_only": lambda value: "must be a derived dataset", + "excluded_types": lambda value: "not %s" % _listed(value), + "rankable": lambda value: "must be rankable" if value else "must not be rankable", + "requires_all_columns": lambda value: "has columns %s" % _listed(value), + "requires_any_columns": lambda value: "has a column %s" % _listed(value), + "required_settings": lambda value: "setting: %s" % _listed(value), + "required_packages": lambda value: "package: %s" % _listed(value), +} + + +def _listed(value): + """Render a requirement's value for display, joining a list into a phrase.""" + if isinstance(value, (list, tuple, set, frozenset)): + return ", ".join(str(item) for item in value) + + return str(value) + + +def describe_requirements(requirement): + """ + The conditions a processor puts on its input, as plain phrases + + The keys of a `how_to_run` requirement are terse spec field names; this + turns them into something readable, so a front-end can list them without + knowing what the fields mean. + + :param dict requirement: A requirement as returned by `ModuleMap.how_to_run` + :return list: Phrases, empty when the processor declares no conditions + """ + if not requirement: + return [] + + return [_REQUIREMENT_LABELS[key](value) if key in _REQUIREMENT_LABELS + else "%s: %s" % (key.replace("_", " "), _listed(value)) + for key, value in requirement.items()] + + +def _is_filter(processor): + """ + Whether a processor is a filter -- it runs on almost anything, keeps its input's + format, and can be inserted anywhere in a chain. True when it declares a Filter + output or reports is_filter() (the latter catches filters in the Filtering + category that do not declare a Filter output). + """ + if isinstance(getattr(processor, "output", None), Filter): + return True + is_filter = getattr(processor, "is_filter", None) + try: + return bool(is_filter()) if callable(is_filter) else bool(is_filter) + except Exception: + return False + + +def _required_columns(spec): + """ + Columns a spec needs present -- the one prerequisite that can't be met by naming a + producer (any dataset with the columns works). Empty when not column-gated. + """ + if spec is None: + return [] + return sorted(set(getattr(spec, "requires_all_columns", None) or []) + | set(getattr(spec, "requires_any_columns", None) or [])) + + +def _shape_dict(shape): + """A producer's output shape as a JSON-friendly dict ('unknown' for what it left open).""" + def show(value): + if value is UNKNOWN: + return "unknown" + if isinstance(value, (set, frozenset)): + return sorted(str(item) for item in value) + return value + + def show_columns(shape): + if shape.columns is UNKNOWN: + return "unknown" + if not shape.columns and shape.columns_are_all: + return "none" + return sorted(shape.columns) + + return { + "type": shape.type, + "extension": show(shape.extension), + "media_type": show(shape.media), + "datasource": show(shape.datasource), + "top_level": show(shape.top_level), + "from_collector": show(shape.from_collector), + "columns": show_columns(shape), + "produces_file": shape.produces_file, + } + + +class ModuleMap: + """ + Built once from the loaded modules; answers the catalogue's questions about how + processors connect, from their declared compatibility + output alone. + """ + + def __init__(self, modules, config=None, logger=None): + self.config = config + # The frontend passes its Logger (g.log) so a miscalibrated spec reaches the + # 4CAT log and, at ERROR, Slack; falls back to stdlib when none is injected. + self.log = logger or logging.getLogger(__name__) + self.processors = {} + self._spec = {} # declared spec (None when undeclared) -- for display + self._match_spec = {} # effective spec used for matching (default if None) + self._shapes = {} # producer output shape + self._collector = {} + self._declarative = {} + self._filter = {} # runs on anything, keeps format, inserted anywhere + + # Build per-processor so one bad processor can't take down the whole map: a + # `compatibility` that is not a Compatibility (likely a custom extension) is + # treated as undeclared and logged at ERROR; one that raises is dropped. + for ptype, processor in (getattr(modules, "processors", {}) or {}).items(): + try: + spec = getattr(processor, "compatibility", None) + if spec is not None and not isinstance(spec, Compatibility): + self.log.error( + "module map: processor '%s' has a 'compatibility' that is " + "%s, not a Compatibility -- treating it as undeclared. The " + "spec needs fixing (most likely in a custom extension)." + % (ptype, type(spec).__name__)) + spec = None + shape = describe_output(processor) + collector = is_collector(processor) + declarative = is_declaratively_compatible(processor) + is_filter = _is_filter(processor) + except Exception as e: + self.log.error("module map: processor '%s' could not be read and " + "is omitted from the map: %s" % (ptype, e)) + continue + self.processors[ptype] = processor + self._spec[ptype] = spec + self._match_spec[ptype] = spec if spec is not None else _DEFAULT_SPEC + self._shapes[ptype] = shape + self._collector[ptype] = collector + self._declarative[ptype] = declarative + self._filter[ptype] = is_filter + + # which processors can follow which (datasources never follow anything). Each + # link keeps its answer; whether a link is also *approximate* depends on the + # following processor (it keeps a custom is_compatible_with), tracked per processor. + self._succ = defaultdict(list) + self._pred = defaultdict(list) + self._edge = {} # (producer, consumer) -> MatchResult + for ptype in self.processors: + shape = self._shapes[ptype] + # a processor that writes no result file (e.g. only annotates its parent) + # produces nothing for another processor to run on + if not shape.produces_file: + continue + for qtype in self.processors: + if qtype == ptype or self._collector[qtype]: + continue + try: + outcome = self._match_spec[qtype].check(shape) # "yes" / "maybe" / "no" + except Exception as e: + self.log.warning("module map: edge %s -> %s skipped: %s" % (ptype, qtype, e)) + continue + if outcome != "no": + self._succ[ptype].append(qtype) + self._pred[qtype].append(ptype) + self._edge[(ptype, qtype)] = "definite" if outcome == "yes" else "maybe" + + # -- catalogue / search -- + + def _entry(self, ptype): + processor = self.processors[ptype] + spec = self._spec[ptype] + return { + "type": ptype, + "title": getattr(processor, "title", ptype), + "category": getattr(processor, "category", None), + "description": getattr(processor, "description", None), + "tags": list(getattr(processor, "tags", []) or []), + "info": list(getattr(processor, "info", []) or []), + "warnings": list(getattr(processor, "warnings", []) or []), + "references": list(getattr(processor, "references", []) or []), + "icon": getattr(processor, "icon", "") or "", + "repo_url": self._repo_link(processor), + "is_datasource": self._collector[ptype], + "is_filter": self._filter[ptype], + "has_override": not self._declarative[ptype], + "requires_dataset_result_file": bool( + spec is not None and getattr(spec, "requires_dataset_result_file", False)), + } + + def _repo_link(self, processor): + """ + Link to a module's source code, when the map was built with a config + + :return str|None: URL to the module's file in the configured repository + """ + if self.config is None: + return None + + try: + return processor.get_repo_link(self.config) + except Exception: + return None + + def catalogue(self): + """Every processor (and datasource), each with display metadata + flags.""" + return [self._entry(ptype) for ptype in self.processors] + + def categories(self): + """{category: [sorted types]} for grouped browsing.""" + groups = defaultdict(list) + for ptype in self.processors: + groups[getattr(self.processors[ptype], "category", None) or "(uncategorised)"].append(ptype) + return {category: sorted(types) for category, types in sorted(groups.items())} + + def search(self, query): + """Catalogue entries whose type/title/category/description contains `query`.""" + needle = (query or "").strip().lower() + if not needle: + return [] + hits = [] + for ptype in self.processors: + entry = self._entry(ptype) + haystack = " ".join(str(entry.get(field) or "") for field in + ("type", "title", "category", "description")).lower() + if needle in haystack: + hits.append(entry) + return hits + + # -- one processor -- + + def module(self, ptype): + """Full bundle for one processor: metadata, spec, how-to-run, follow-ups.""" + if ptype not in self.processors: + return None + return { + **self._entry(ptype), + "output_shape": _shape_dict(self._shapes[ptype]), + "compatibility": describe_spec(self._spec[ptype]), + "how_to_run": self.how_to_run(ptype), + "followups": self.followups(ptype), + } + + def _title(self, ptype): + return getattr(self.processors[ptype], "title", ptype) if ptype in self.processors else ptype + + def _step(self, ptype, certainty=None): + """A producer step, optionally annotated with the certainty of its link.""" + step = {"type": ptype, "title": self._title(ptype), + "is_datasource": self._collector.get(ptype, False)} + if certainty is not None: + step["certainty"] = certainty + return step + + def _producers(self, ptype, certainty=None): + """The processors whose output `ptype` accepts; with `certainty` set + ("definite" or "maybe"), only the producers whose link has that certainty.""" + producers = [] + for producer in self._pred.get(ptype, []): + edge = self._edge.get((producer, ptype)) + if certainty is None or edge == certainty: + producers.append(producer) + return producers + + # -- how to run one processor -- + + def _accepts(self, ptype): + """ + What `ptype` runs on directly. The declared requirement is the label (its own + `describe_spec`); the confirmed producers are listed flat, split only on the + one honest distinction -- a data source you start from vs. another processor + you run first. Filters are left out: they are transparent (see followups). + """ + confirmed = self._producers(ptype, "definite") + datasources = sorted(p for p in confirmed if self._collector[p]) + from_processors = sorted(p for p in confirmed + if not self._collector[p] and not self._filter[p]) + # the declared spec is the label, but only its *input* conditions -- the + # follow-up hints describe this processor's output, not what it accepts + requirement = describe_spec(self._spec[ptype]) or {} + requirement = {key: value for key, value in requirement.items() + if key not in ("preferred_followups", "excluded_followups")} + return { + "requirement": requirement, + "datasources": [self._step(p, self._edge.get((p, ptype))) for p in datasources], + "from_processors": [self._step(p, self._edge.get((p, ptype))) for p in from_processors], + } + + def _confirmed_producers(self, ptype): + """Processors and data sources whose output `ptype` definitely accepts, filters + excluded (they are transparent -- see followups).""" + return [p for p in self._producers(ptype, "definite") if not self._filter[p]] + + def _match_strength(self, consumer_type, producer_type): + """ + How fully a producer's output matches what `consumer_type` says it accepts: the + number of "what kind of input" axes (type, type-prefix, media, datasource) the + producer definitely satisfies. Used only to order example paths, so a producer built + for the job (an image downloader for an image step) is preferred over one that + matches only incidentally (a chart that merely happens to be an image). + """ + spec = self._match_spec.get(consumer_type) + shape = self._shapes.get(producer_type) + if spec is None or shape is None: + return 0 + strength = 0 + if spec.types and shape.type in set(spec.types): + strength += 1 + if spec.type_prefixes and shape.type and shape.type.startswith(tuple(spec.type_prefixes)): + strength += 1 + if spec.media_types and shape.media is not UNKNOWN: + have = shape.media if isinstance(shape.media, (set, frozenset)) else {shape.media} + if have and have <= set(spec.media_types): + strength += 1 + if spec.datasources and shape.datasource is not UNKNOWN and shape.datasource in set(spec.datasources): + strength += 1 + return strength + + def _examples(self, ptype, count=3): + """ + A few concrete example paths from a data source to `ptype` -- illustrations of how + you might reach it, NOT a complete or curated list. Each example is a full chain (a + data source, the processors to run in order, then `ptype`) and shows a different + thing to run `ptype` on: one example per direct producer, via the shortest way to + reach that producer. Producers are ordered by how fully they match `ptype`'s stated + input (see _match_strength), so a processor built for the job leads and a merely + incidental match (a chart that happens to be an image, for an image step) shows only + if nothing better exists. One-per-producer also stops a processor with a single real + recipe being padded out with longer, roundabout ones. Confirmed links only, filters + skipped. Empty when nothing reaches `ptype` by confirmed steps (a likely spec gap). + """ + # Walk producers backward from `ptype`, level by level so the shortest path to each + # producer surfaces first. `path` is stored tail-first ([node, ..., ptype]); a + # finished chain (one that reached a data source) drops `ptype` off the tail. + shortest = {} # direct producer of ptype -> shortest complete chain ending in it + frontier = [[ptype]] + for _ in range(4): # depth cap; deeper chains are not useful examples + if not frontier or len(shortest) >= 40: + break + nxt = [] + for path in frontier: + for producer in self._confirmed_producers(path[0]): + if producer in path: + continue # don't loop back on a processor already in this path + if self._collector[producer]: + chain = [producer] + path[:-1] # reached a data source: chain complete + direct = chain[-1] # the producer `ptype` runs on directly + if direct not in shortest or len(chain) < len(shortest[direct]): + shortest[direct] = chain + else: + nxt.append([producer] + path) + frontier = nxt[:400] # bound the search; plenty for a few short examples + + ranked = sorted(shortest.values(), + key=lambda chain: (-self._match_strength(ptype, chain[-1]), + len(chain), [self._title(step) for step in chain])) + return [{"datasource": chain[0], "title": self._title(chain[0]), + "then": [{"type": step, "title": self._title(step)} for step in chain[1:]]} + for chain in ranked[:count]] + + def how_to_run(self, ptype): + """ + How to produce a dataset `ptype` can run on: + + * a filter answers with a single note -- it runs on almost anything and can be + inserted anywhere, so listing producers would be noise; + * otherwise `accepts` is what it runs on directly (its declared requirement + + the confirmed data sources and processors), and `examples` gives a few of the + shortest full paths from a data source -- concrete illustrations, not a curated + or complete list of ways to get here; + * `notes` carries the data-source, filter, column and override caveats. + """ + if self._filter[ptype]: + return { + "type": ptype, + "is_filter": True, + "notes": ["This is a filter: it runs on almost any dataset and can be " + "inserted at any point in a chain. Its output keeps the same " + "format, so it does not change what you can run next."], + } + + columns = _required_columns(self._spec[ptype]) + notes = [] + if self._collector[ptype]: + notes.append("This is a data source: it collects data directly (from an upload " + "or a query), so it does not run on another dataset.") + else: + notes.append("Filters can be applied at any earlier point -- they keep the " + "format, so they do not change what this accepts.") + if columns: + notes.append("Needs a dataset with these columns (%s); the producing processor " + "can't be named from the specs alone." % ", ".join(columns)) + if not self._declarative[ptype]: + notes.append("Keeps a custom is_compatible_with, so these connections are approximate.") + + return { + "type": ptype, + "accepts": self._accepts(ptype), + "examples": self._examples(ptype), + "notes": notes, + } + + def _effective_followups(self, ptype): + """ + The consumers that can run on `ptype`'s output, as {consumer: certainty}. + + For an ordinary processor this is just its outgoing edges. For a filter it is + resolved by propagation instead: a filter's output has the same shape as its + input, so anything that runs on what fed the filter also runs on the filtered + result. Because a filter accepts almost anything, every data source or processor + that can reach it is one of its direct producers, so reading its non-filter + producers (and what runs on each of them) resolves the followups without a walk. + This turns a filter's followups from the blanket "maybe" its own unknown shape + gives into the concrete, mostly-definite set it really has. + """ + if not self._filter[ptype]: + return {consumer: self._edge[(ptype, consumer)] + for consumer in self._succ.get(ptype, [])} + resolved = {} + for producer in self._pred.get(ptype, []): + if self._filter[producer]: + continue # another filter gives no concrete shape to propagate + into_filter = self._edge[(producer, ptype)] + for consumer in self._succ.get(producer, []): + if consumer == ptype: + continue + certainty = ("definite" if into_filter == "definite" + and self._edge[(producer, consumer)] == "definite" else "maybe") + if resolved.get(consumer) != "definite": + resolved[consumer] = certainty + return resolved + + def followups(self, ptype): + """ + What can run on `ptype`'s output: curated `preferred` first, then `filters` + (kept separate -- a filter can be applied to narrow the data without changing + its format), then the real analysis steps grouped by category. For a filter the + set is resolved by propagation (see _effective_followups) and a note explains it. + """ + spec = self._spec[ptype] + preferred = [followup for followup in (getattr(spec, "preferred_followups", None) or []) + if followup in self.processors] + preferred_set = set(preferred) + filters = [] + grouped = defaultdict(list) + for qtype, certainty in self._effective_followups(ptype).items(): + if qtype in preferred_set: + continue + item = { + "type": qtype, + "title": getattr(self.processors[qtype], "title", qtype), + "certainty": certainty, + "approximate": not self._declarative[qtype], + } + if self._filter[qtype]: + filters.append(item) + else: + category = getattr(self.processors[qtype], "category", None) or "(uncategorised)" + grouped[category].append(item) + result = { + "preferred": [self._entry(followup) for followup in preferred], + "filters": sorted(filters, key=lambda item: item["type"]), + "others_by_category": {category: sorted(items, key=lambda item: item["type"]) + for category, items in sorted(grouped.items())}, + } + if self._filter[ptype]: + result["note"] = ("A filter keeps its input's format, so anything you could run on " + "the dataset you filtered you can still run here.") + return result + + def graph(self): + """ + The whole map as {nodes, edges} -- the producer->consumer backbone the query + methods traverse, exposed for graph-drawing and debugging. Nodes carry + `is_root` (alias of is_datasource); edges carry the outcome certainty and + whether the consumer is approximate (keeps an override). + """ + nodes = [{**self._entry(ptype), "is_root": self._collector[ptype]} for ptype in self.processors] + edges = [{"from": producer, "to": consumer, + "certainty": self._edge[(producer, consumer)], + "approximate": not self._declarative[consumer]} + for producer, consumers in self._succ.items() for consumer in consumers] + return {"nodes": nodes, "edges": edges} diff --git a/common/lib/outputs.py b/common/lib/outputs.py new file mode 100644 index 000000000..6d3e81810 --- /dev/null +++ b/common/lib/outputs.py @@ -0,0 +1,273 @@ +""" +What a processor produces. + +A processor states the shape of its output as an `output` class attribute -- an Output, +usually one of the archetypes below. This is the counterpart to Compatibility (what a +processor accepts): the two together let the processor map decide which processors can +follow which, without running anything. + +Most processors never write one by hand -- the base classes set a sensible default (a +data source produces an ndjson table, a filter passes its parent's shape through) and a +processor overrides only the field that differs, e.g. `output = Table(columns={"date", +"item", "value"})`. The archetypes: + + Datasource a collected, top-level table (its own extension, columns from its items) + Table a derived table (csv/ndjson) + Filter same shape as the parent (extension, media and columns pass through) + Network a single graph file (gexf), no columns + Render a single image (svg/png), no columns + MediaArchive a zip of media files, no columns + Archive a zip of data files, no columns + File a single json/txt/html file, no columns + Delegated a preset whose real output is its pipeline's last step (unknown here) + NoOutput writes no result file (e.g. only adds annotations to its parent) + +`describe_output(processor)` turns the declaration into a Shape -- the plain properties +Compatibility.check reads -- filling in UNKNOWN for anything left open. A processor that +declares nothing falls back to inferring a Shape from its class. +""" +from __future__ import annotations + +from common.lib.compatibility import ( + Shape, + UNKNOWN, + _is_collector_type, + is_collector, + _maybe_call, + _declared_class_value, +) + + +class _Sentinel: + def __init__(self, name): + self._name = name + + def __repr__(self): + return self._name + + +# Use for any field whose value is the parent dataset's (a filter's extension, media or +# columns) -- unknown until the chain is known, so it only ever softens an answer. +PASSTHROUGH = _Sentinel("PASSTHROUGH") +# Use as an extension when the processor's own `extension` class attribute is the real, +# trusted value (a data source whose subclasses each set their own). +CLASS_EXTENSION = _Sentinel("CLASS_EXTENSION") +# Use as `columns` when the output has no column table at all (a network, image, or media +# archive). A consumer that needs a column then gets a definite "no". +NO_COLUMNS = _Sentinel("NO_COLUMNS") + + +def _extension_value(value, processor): + if value is PASSTHROUGH or value is None: + return UNKNOWN + if value is CLASS_EXTENSION: + return getattr(processor, "extension", None) or UNKNOWN + return value # a plain extension string + + +def _media_value(value): + if value is None or value is PASSTHROUGH: + return UNKNOWN + if isinstance(value, (set, frozenset, list, tuple)): + return set(value) # a bounded set: could be image OR video OR ... + return value # a single media string + + +def _datasource_value(value, collector, ptype): + if value is not None and value is not PASSTHROUGH: + return value + # a collector carries its datasource in its type; anything else passes it through + if collector and _is_collector_type(ptype): + return ptype.rsplit("-", 1)[0] + return UNKNOWN + + +def _top_level_value(position): + if position == "top": + return True + if position == "child": + return False + return UNKNOWN + + +def _columns_value(value): + """Returns (columns, columns_are_all) the way Shape wants them.""" + if value is None or value is PASSTHROUGH: + return UNKNOWN, False + if value is NO_COLUMNS: + return frozenset(), True # definitely no columns + return frozenset(value), False # a floor: at least these + + +class Output: + """ + The shape a processor produces. Subclass it for the common archetypes; the fields are + kept as plain values and turned into a Shape by to_shape. Any field left None is + "unknown" -- it can never cause a false "no", only soften an answer to "maybe". + `produces_file` is False for a processor that writes no result file. + """ + + def __init__(self, *, extension=None, media="text", columns=None, + position="child", collector=False, datasource=None, produces_file=True): + self.extension = extension # str | set | PASSTHROUGH | CLASS_EXTENSION | None + self.media = media # str | set | PASSTHROUGH | None + self.columns = columns # set | NO_COLUMNS | PASSTHROUGH | None + self.position = position # "top" | "child" | None + self.collector = collector # bool | None + self.datasource = datasource # str | PASSTHROUGH | None + self.produces_file = produces_file + + def to_shape(self, processor) -> Shape: + """Turn this into the Shape Compatibility.check reads. `processor` supplies the + type (and, for a collector, the datasource it carries).""" + ptype = getattr(processor, "type", None) + columns, columns_are_all = _columns_value(self.columns) + return Shape( + type=ptype, + extension=_extension_value(self.extension, processor), + media=_media_value(self.media), + datasource=_datasource_value(self.datasource, self.collector, ptype), + top_level=_top_level_value(self.position), + from_collector=UNKNOWN if self.collector is None else bool(self.collector), + columns=columns, + columns_are_all=columns_are_all, + produces_file=self.produces_file, + ) + + +# Default archetypes for the common processor types. A processor can override any field +# by declaring its own Output (or subclass) as its `output` class attribute. + +class Datasource(Output): + """A collected, top-level dataset. Its extension is its own (ndjson for most, csv or + zip for some), trusted because it drives the result file. Defaults to text items; + pass `columns` to sharpen. A data source producing media uses MediaArchive.""" + + def __init__(self, *, extension=CLASS_EXTENSION, columns=None, media="text", datasource=None): + super().__init__(extension=extension, media=media, columns=columns, + position="top", collector=True, datasource=datasource) + + +class Table(Output): + """A derived table. Defaults to a csv of text; pass `columns` to sharpen.""" + + def __init__(self, *, columns=None, media="text", extension="csv"): + super().__init__(extension=extension, media=media, columns=columns, + position="child", collector=False) + + +class Filter(Output): + """A filter: extension, media and columns all follow the parent, and its position and + collector-ness are unknown (its result may be made standalone).""" + + def __init__(self): + super().__init__(extension=PASSTHROUGH, media=PASSTHROUGH, columns=PASSTHROUGH, + position=None, collector=None, datasource=PASSTHROUGH) + + +class Network(Output): + """A single graph file (gexf) with no column table.""" + + def __init__(self, *, extension="gexf"): + super().__init__(extension=extension, media="text", columns=NO_COLUMNS, + position="child", collector=False) + + +class Render(Output): + """A single rendered image (svg by default, or png) with no column table.""" + + def __init__(self, extension="svg", *, media="image"): + super().__init__(extension=extension, media=media, columns=NO_COLUMNS, + position="child", collector=False) + + +class MediaArchive(Output): + """A zip archive of media files with no column table. `media` is the kind of media in + it (a single value, or a set when it varies per file).""" + + def __init__(self, *, media=None, collector=False, position="child"): + if media is None: + media = {"image", "video", "audio", "file"} + super().__init__(extension="zip", media=media, columns=NO_COLUMNS, + position=position, collector=collector) + + +class Archive(Output): + """A zip archive of data files with no column table (tokens, embeddings, an export + bundle). For an archive of media files use MediaArchive.""" + + def __init__(self, *, media="text", collector=False, position="child"): + super().__init__(extension="zip", media=media, columns=NO_COLUMNS, + position=position, collector=collector) + + +class File(Output): + """A single non-tabular file (e.g. json, txt, html) with no column table.""" + + def __init__(self, extension, *, media="text"): + super().__init__(extension=extension, media=media, columns=NO_COLUMNS, + position="child", collector=False) + + +class Delegated(Output): + """A preset that writes no file of its own; its real output is whatever the last + processor in its pipeline produces, so the shape is unknown here. `terminal` names + that last processor when it is known.""" + + def __init__(self, terminal=None): + super().__init__(extension=PASSTHROUGH, media=None, columns=None, + position=None, collector=False) + self.terminal = terminal + + +class NoOutput(Output): + """Produces no result file (for example, only adds annotations to its parent), so + nothing can run on it.""" + + def __init__(self): + super().__init__(extension=None, media=None, columns=NO_COLUMNS, + position=None, collector=False, produces_file=False) + + +def _infer_output(processor) -> Shape: + """ + Work out a processor's output from its class -- the fallback for a processor that + declares no `output` (today only third-party extensions). It synthesises a plain + Output from what the class safely promises and hands it to to_shape, so the + class-to-shape conversion lives in one place. A filter is passthrough; a collector is + top-level with its datasource; anything else is a child with its trusted class + extension (an inherited default stays unknown, never a false claim). + """ + collector = is_collector(processor) + is_filter = bool(_maybe_call(processor, "is_filter")) + declared_media = _declared_class_value(processor, "media_type") + + if collector: + position, collector_flag = "top", True + elif is_filter: + position, collector_flag = None, None + else: + position, collector_flag = "child", False + + extension = PASSTHROUGH if is_filter else _declared_class_value(processor, "extension") + + return Output( + extension=extension, + media=declared_media or None, + columns=None, + position=position, + collector=collector_flag, + ).to_shape(processor) + + +def describe_output(processor) -> Shape: + """ + A processor's output as a Shape Compatibility.check can read. Reads the processor's + declared `output` when it has one; otherwise falls back to inferring it from the + class. The single place that chooses between the two, so everything downstream reads + one kind of shape regardless. + """ + declared = getattr(processor, "output", None) + if isinstance(declared, Output): + return declared.to_shape(processor) + return _infer_output(processor) diff --git a/common/lib/user_input.py b/common/lib/user_input.py index 55aac477f..df7ce5edb 100644 --- a/common/lib/user_input.py +++ b/common/lib/user_input.py @@ -407,12 +407,15 @@ def parse_value(settings, choice, other_input=None, silently_correct=True): elif input_type in (UserInput.OPTION_MULTI, UserInput.OPTION_ANNOTATIONS): # any number of values out of a list of possible values - # comma-separated during input, returned as a list of valid options + # the form posts one value per picked option, which parse_all() + # collects into a list; the API sends them comma-separated if not choice: return settings.get("default", []) - chosen = choice.split(",") - return [item for item in chosen if item in settings.get("options", [])] + if type(choice) is str: + choice = choice.split(",") + + return [item for item in choice if item in settings.get("options", [])] elif input_type == UserInput.OPTION_MULTI_SELECT: # multiple number of values out of a dropdown list of possible values diff --git a/datasources/audio_to_text/audio_to_text.py b/datasources/audio_to_text/audio_to_text.py index ead409088..cb1dffca4 100644 --- a/datasources/audio_to_text/audio_to_text.py +++ b/datasources/audio_to_text/audio_to_text.py @@ -8,13 +8,18 @@ from datasources.media_import.import_media import SearchMedia from processors.machine_learning.audio_to_text import AudioToText +from common.lib.outputs import MediaArchive class AudioUploadToText(SearchMedia): type = "upload-audio-to-text-search" # job ID - category = "Search" # category title = "Convert speech to text" # title displayed in UI description = "Upload your own audio and use OpenAI's Whisper or GPT models to create transcripts" # description displayed in UI + icon = "closed-captioning" + tags = ["audio", "media", "upload"] + + # only audio is accepted here, so the output media is narrower than SearchMedia's + output = MediaArchive(media="audio", collector=True, position="top") # reuse the AudioToText processor's compatibility -- this datasource runs it on uploaded audio compatibility = AudioToText.compatibility diff --git a/datasources/bsky/search_bsky.py b/datasources/bsky/search_bsky.py index 3454bc426..3438199e3 100644 --- a/datasources/bsky/search_bsky.py +++ b/datasources/bsky/search_bsky.py @@ -17,18 +17,20 @@ from common.lib.helpers import timify from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem +from common.lib.outputs import Datasource class SearchBluesky(Search): """ Search for posts in Bluesky """ type = "bsky-search" # job ID - category = "Search" # category title = "Bluesky Search" # title displayed in UI description = "Collects Bluesky posts via its API." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"tags"}) + icon = "brand-bluesky" + tags = ["API"] config = { "bsky-search.max_results": { diff --git a/datasources/dmi-tcat/search_tcat.py b/datasources/dmi-tcat/search_tcat.py index 44a646bbf..82659ae31 100644 --- a/datasources/dmi-tcat/search_tcat.py +++ b/datasources/dmi-tcat/search_tcat.py @@ -13,6 +13,7 @@ from common.lib.user_input import UserInput from common.lib.helpers import sniff_encoding from common.lib.item_mapping import MappedItem +from common.lib.outputs import Datasource from datasources.twitterv2.search_twitter import SearchWithTwitterAPIv2 @@ -26,7 +27,9 @@ class SearchWithinTCATBins(Search): """ type = "dmi-tcat-search" # job ID extension = "ndjson" + output = Datasource() title = "TCAT Search (HTTP)" + icon = "brand-twitter" # TCAT has a few fields that do not exist in APIv2 additional_TCAT_fields = ["to_user_name", "filter_level", "favorite_count", "truncated", "from_user_favourites_count", "from_user_lang", "from_user_utcoffset", diff --git a/datasources/dmi-tcatv2/search_tcat_v2.py b/datasources/dmi-tcatv2/search_tcat_v2.py index 5d5ef362e..734b8bb25 100644 --- a/datasources/dmi-tcatv2/search_tcat_v2.py +++ b/datasources/dmi-tcatv2/search_tcat_v2.py @@ -11,6 +11,7 @@ from backend.lib.search import Search from common.lib.exceptions import QueryParametersException from common.lib.user_input import UserInput +from common.lib.outputs import Datasource from backend.lib.database_mysql import MySQLDatabase @@ -23,7 +24,9 @@ class SearchWithinTCATBinsV2(Search): """ type = "dmi-tcatv2-search" # job ID extension = "csv" + output = Datasource() title = "TCAT Search (SQL)" + icon = "brand-twitter" config = { "dmi-tcatv2-search.database_instances": { diff --git a/datasources/douban/search_douban.py b/datasources/douban/search_douban.py index 37ff7bbb0..e012f1b24 100644 --- a/datasources/douban/search_douban.py +++ b/datasources/douban/search_douban.py @@ -11,6 +11,7 @@ from backend.lib.search import Search from common.lib.helpers import convert_to_int, strip_tags, UserInput from common.lib.exceptions import QueryParametersException, ProcessorInterruptedException +from common.lib.outputs import Datasource class SearchDouban(Search): @@ -20,12 +21,13 @@ class SearchDouban(Search): Defines methods that are used to query Douban data from the site directly """ type = "douban-search" # job ID - category = "Search" # category + title = "Douban Search" # title displayed in UI description = "Scrapes group posts from Douban for a given set of groups" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + output = Datasource() + icon = "comment" + tags = ["scraping"] # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/douyin/search_douyin.py b/datasources/douyin/search_douyin.py index 3ccf08c68..c95ce6fd7 100644 --- a/datasources/douyin/search_douyin.py +++ b/datasources/douyin/search_douyin.py @@ -8,17 +8,21 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem, MissingMappedField from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchDouyin(Search): """ Import scraped Douyin data """ type = "douyin-search" # job ID - category = "Search" # category + title = "Import scraped Douyin data" # title displayed in UI - description = "Import Douyin data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Douyin data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True + icon = "brand-tiktok" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/eightchan/search_8chan.py b/datasources/eightchan/search_8chan.py index 2a079e29d..6a4955d5e 100644 --- a/datasources/eightchan/search_8chan.py +++ b/datasources/eightchan/search_8chan.py @@ -4,6 +4,7 @@ from datasources.fourchan.search_4chan import Search4Chan from common.lib.helpers import UserInput +from common.lib.outputs import Datasource class Search8Chan(Search4Chan): @@ -16,11 +17,10 @@ class Search8Chan(Search4Chan): most methods are inherited from there. """ type = "eightchan-search" + output = Datasource() sphinx_index = "8chan" title = "8chan search" prefix = "8chan" - is_local = True # Whether this datasource is locally scraped - is_static = True # Whether this datasource is still updated config = { "eightchan-search.autoscrape": { diff --git a/datasources/eightkun/__init__.py b/datasources/eightkun/__init__.py index daf4a57e5..5159a5377 100644 --- a/datasources/eightkun/__init__.py +++ b/datasources/eightkun/__init__.py @@ -4,5 +4,4 @@ # Internal identifier for this data source DATASOURCE = "eightkun" -NAME = "8kun" -IS_LOCAL = True \ No newline at end of file +NAME = "8kun" \ No newline at end of file diff --git a/datasources/eightkun/search_8kun.py b/datasources/eightkun/search_8kun.py index dab7c6fbb..2cd65b9d2 100644 --- a/datasources/eightkun/search_8kun.py +++ b/datasources/eightkun/search_8kun.py @@ -4,6 +4,7 @@ from datasources.fourchan.search_4chan import Search4Chan from common.lib.helpers import UserInput +from common.lib.outputs import Datasource class Search8Kun(Search4Chan): @@ -16,11 +17,10 @@ class Search8Kun(Search4Chan): most methods are inherited from there. """ type = "eightkun-search" + output = Datasource() sphinx_index = "8kun" title = "8kun search" prefix = "8kun" - is_local = True # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated config = { "eightkun-search.autoscrape": { diff --git a/datasources/facebook/search_facebook.py b/datasources/facebook/search_facebook.py index 3a16f4074..01184bbf1 100644 --- a/datasources/facebook/search_facebook.py +++ b/datasources/facebook/search_facebook.py @@ -9,6 +9,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem +from common.lib.outputs import Datasource class SearchFacebook(Search): @@ -16,10 +17,11 @@ class SearchFacebook(Search): Import scraped 9gag data """ type = "facebook-search" # job ID - category = "Search" # category + title = "Import scraped Facebook data" # title displayed in UI - description = "Import Facebook data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Facebook data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + output = Datasource() is_from_zeeschuimer = True # not available as a processor for existing datasets diff --git a/datasources/fourcat_import/import_4cat.py b/datasources/fourcat_import/import_4cat.py index 8b6cadb33..777689e7c 100644 --- a/datasources/fourcat_import/import_4cat.py +++ b/datasources/fourcat_import/import_4cat.py @@ -13,6 +13,7 @@ DataSetException) from common.lib.helpers import UserInput, get_software_version from common.lib.dataset import DataSet +from common.lib.outputs import Output class FourcatImportException(FourcatException): @@ -21,11 +22,14 @@ class FourcatImportException(FourcatException): class SearchImportFromFourcat(BasicProcessor): type = "import_4cat-search" # job ID - category = "Search" # category + title = "Import 4CAT dataset and analyses" # title displayed in UI description = "Import a dataset from another 4CAT server or from a zip file (exported from a 4CAT server)" # description displayed in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + tags = ["upload"] + # a top-level import; extension, media and columns are copied from whatever dataset + # is imported, so none of them are known ahead of the run + output = Output(extension=None, media=None, columns=None, position="top", collector=True) + icon = "file-import" max_workers = 1 # this cannot be more than 1, else things get VERY messy diff --git a/datasources/fourchan/search_4chan.py b/datasources/fourchan/search_4chan.py index abc9e722c..60d102757 100644 --- a/datasources/fourchan/search_4chan.py +++ b/datasources/fourchan/search_4chan.py @@ -9,6 +9,7 @@ from backend.lib.database_mysql import MySQLDatabase from common.lib.helpers import UserInput +from common.lib.outputs import Datasource from backend.lib.search import SearchWithScope from common.lib.exceptions import QueryParametersException, ProcessorInterruptedException @@ -20,11 +21,10 @@ class Search4Chan(SearchWithScope): Defines methods that are used to query the 4chan data indexed and saved. """ type = "fourchan-search" # job ID + output = Datasource() title = "4chan search" sphinx_index = "4chan" # sphinx index name; this should match the index name in sphinx.conf prefix = "4chan" # table identifier for this datasource; see below for usage - is_local = True # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated # Columns to return in csv return_cols = ['thread_id', 'id', 'timestamp', 'board', 'body', 'subject', 'author', 'image_file', 'image_4chan', diff --git a/datasources/gab/search_gab.py b/datasources/gab/search_gab.py index 2cdd35c3b..6975cdccd 100644 --- a/datasources/gab/search_gab.py +++ b/datasources/gab/search_gab.py @@ -5,6 +5,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem, MissingMappedField +from common.lib.outputs import Datasource from common.lib.helpers import normalize_url_encoding @@ -13,10 +14,12 @@ class SearchGab(Search): Import scraped gab data """ type = "gab-search" # job ID - category = "Search" # category + title = "Import scraped Gab data" # title displayed in UI - description = "Import Gab data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Gab data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"tags"}) is_from_zeeschuimer = True fake = "" diff --git a/datasources/imgur/search_imgur.py b/datasources/imgur/search_imgur.py index f9d87ca9f..129894051 100644 --- a/datasources/imgur/search_imgur.py +++ b/datasources/imgur/search_imgur.py @@ -8,6 +8,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem +from common.lib.outputs import Datasource from common.lib.helpers import normalize_url_encoding class SearchImgur(Search): @@ -15,10 +16,11 @@ class SearchImgur(Search): Import scraped Imgur data """ type = "imgur-search" # job ID - category = "Search" # category + title = "Import scraped Imgur data" # title displayed in UI - description = "Import Imgur data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Imgur data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + output = Datasource() is_from_zeeschuimer = True # not available as a processor for existing datasets diff --git a/datasources/instagram/instagram-explorer.css b/datasources/instagram/instagram-explorer.css index 5d9264a39..48899e012 100644 --- a/datasources/instagram/instagram-explorer.css +++ b/datasources/instagram/instagram-explorer.css @@ -45,7 +45,7 @@ .items .external-url { position: absolute; - bottom: 0; + top: 0; right: 0; padding: 10px; } diff --git a/datasources/instagram/search_instagram.py b/datasources/instagram/search_instagram.py index 0f76b66f1..29227844f 100644 --- a/datasources/instagram/search_instagram.py +++ b/datasources/instagram/search_instagram.py @@ -9,6 +9,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem, MissingMappedField +from common.lib.outputs import Datasource from common.lib.exceptions import MapItemException from common.lib.helpers import normalize_url_encoding @@ -18,11 +19,14 @@ class SearchInstagram(Search): Import scraped Instagram data """ type = "instagram-search" # job ID - category = "Search" # category + title = "Import scraped Instagram data" # title displayed in UI - description = "Import Instagram data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Instagram data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True + icon = "brand-instagram" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/linkedin/search_linkedin.py b/datasources/linkedin/search_linkedin.py index 2c5884ec6..774abbc03 100644 --- a/datasources/linkedin/search_linkedin.py +++ b/datasources/linkedin/search_linkedin.py @@ -12,17 +12,21 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchLinkedIn(Search): """ Import scraped LinkedIn data """ type = "linkedin-search" # job ID - category = "Search" # category + title = "Import scraped LinkedIn data" # title displayed in UI - description = "Import LinkedIn data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import LinkedIn data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True + icon = "brand-linkedin" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/media_import/import_media.py b/datasources/media_import/import_media.py index f510caf29..917b6da4f 100644 --- a/datasources/media_import/import_media.py +++ b/datasources/media_import/import_media.py @@ -9,15 +9,20 @@ from common.lib.exceptions import QueryParametersException, QueryNeedsExplicitConfirmationException from common.lib.user_input import UserInput from common.lib.helpers import andify +from common.lib.outputs import MediaArchive class SearchMedia(BasicProcessor): type = "media-import-search" # job ID - category = "Search" # category + title = "Upload Media" # title displayed in UI description = "Upload your own audio, video, or image files to be used as a dataset" # description displayed in UI extension = "zip" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + icon = "images" + tags = ["media", "upload"] + + # A top-level archive of uploaded media; one upload is a single media type, but + # which one is only known at run time, so the output media is that bounded set. + output = MediaArchive(media={"image", "video", "audio"}, collector=True, position="top") max_workers = 1 diff --git a/datasources/ninegag/search_9gag.py b/datasources/ninegag/search_9gag.py index 8c029cc53..0b00faf7a 100644 --- a/datasources/ninegag/search_9gag.py +++ b/datasources/ninegag/search_9gag.py @@ -9,6 +9,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchNineGag(Search): @@ -16,10 +17,12 @@ class SearchNineGag(Search): Import scraped 9gag data """ type = "ninegag-search" # job ID - category = "Search" # category + title = "Import scraped 9gag data" # title displayed in UI - description = "Import 9gag data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import 9gag data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"tags"}) is_from_zeeschuimer = True # not available as a processor for existing datasets diff --git a/datasources/pinterest/search_pinterest.py b/datasources/pinterest/search_pinterest.py index 853ad875c..6995feda2 100644 --- a/datasources/pinterest/search_pinterest.py +++ b/datasources/pinterest/search_pinterest.py @@ -9,6 +9,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem, MissingMappedField from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchPinterest(Search): @@ -16,11 +17,13 @@ class SearchPinterest(Search): Import scraped Pinterest data """ type = "pinterest-search" # job ID - category = "Search" # category + title = "Import scraped Pinterest data" # title displayed in UI - description = "Import Pinterest data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Pinterest data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + output = Datasource() is_from_zeeschuimer = True + icon = "brand-pinterest" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/telegram/search_telegram.py b/datasources/telegram/search_telegram.py index 5f80d11a5..20a6358a3 100644 --- a/datasources/telegram/search_telegram.py +++ b/datasources/telegram/search_telegram.py @@ -14,6 +14,7 @@ QueryNeedsFurtherInputException from common.lib.helpers import convert_to_int, UserInput from common.lib.item_mapping import MappedItem, MissingMappedField +from common.lib.outputs import Datasource from datetime import datetime from telethon import TelegramClient, utils @@ -30,12 +31,13 @@ class SearchTelegram(Search): Search Telegram via API """ type = "telegram-search" # job ID - category = "Search" # category + title = "Telegram API search" # title displayed in UI description = "Scrapes messages from open Telegram groups via its API." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + output = Datasource() + icon = "brand-telegram" + tags = ["API"] # cache details_cache = None @@ -158,7 +160,7 @@ def get_options(cls, parent_dataset=None, config=None): }, "save-session": { "type": UserInput.OPTION_TOGGLE, - "help": "Save session:", + "help": "Save session", "default": False }, "resolve-entities-intro": { diff --git a/datasources/telegram/telegram-explorer.css b/datasources/telegram/telegram-explorer.css index 8fa3a73b7..cb3342132 100644 --- a/datasources/telegram/telegram-explorer.css +++ b/datasources/telegram/telegram-explorer.css @@ -8,8 +8,6 @@ } * { - font-size: 15px; - line-height: 1.4; } .explorer-content { @@ -32,15 +30,20 @@ margin: 0 auto; } +.items .item .item-container { + display: flex; + flex-direction: row; +} + .items .item .item-container.new-group { margin-top: 6px; } /* Profile picture */ .items .item .profile-picture-container { - display: inline-block; - width: 60px; - vertical-align: top; + position: relative; + flex: 0 0 12%; + max-width: 12%; } .profile-picture { @@ -49,7 +52,6 @@ width: 50px; height: 50px; line-height: 53px; - float: left; text-align: center; } @@ -62,13 +64,11 @@ /* item content */ .items .item .item-content { - display: inline-block; - max-width: 80%; + flex: 1 1 auto; + min-width: 0; list-style-type: none; background-color: white; border-radius: 5px 20px 20px 5px; - padding: 12px 17px; - z-index: -1; overflow: hidden; } @@ -84,12 +84,16 @@ } .bubble-left { - position: relative; - margin-right: -5px; - float: right; + position: absolute; + top: 0; + right: -1px; z-index: 0; } +.item-content > div:not(.carousel-wrapper, .media-wrapper) { + padding: 12px 17px; +} + .author, .author a, .author a:hover { margin-bottom: 5px; color: #2984cd; @@ -97,12 +101,6 @@ text-decoration: none; } -.items .item .body { - display: inline; - padding-top: 5px; - padding-bottom: 5px; -} - .items .item .body a { color: #2984cd; } @@ -119,7 +117,6 @@ /* Media carousel */ .carousel-wrapper { position: relative; - margin: -12px -17px 10px; overflow: hidden; /* Ensures large media doesn't spill out */ } @@ -196,18 +193,20 @@ .item-content .media-wrapper .no-media { padding: 20px; - color: var(--gray-darker); - background-color: var(--gray); + background-color: var(--yellow-light); + color: var(--yellow); } .item-content .media-wrapper .no-media a { - color: var(--gray-darker); + background-color: var(--yellow-light); + color: var(--yellow); text-decoration: none; font-weight: bold; } /* Emoji reaction counts */ .reactions { + padding: 0 3px; margin-top: 3px; margin-bottom: 3px; } @@ -272,7 +271,7 @@ background-image: linear-gradient(#389ed5, #59c8e2); color: white; border-radius: 5px 20px 20px 5px; - margin-left: 63px; + margin-bottom: var(--spacing-regular); } .item-annotation { diff --git a/datasources/test/search_test.py b/datasources/test/search_test.py index 2fca9b0a6..684365a30 100644 --- a/datasources/test/search_test.py +++ b/datasources/test/search_test.py @@ -23,6 +23,7 @@ from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.exceptions import ProcessorInterruptedException, ProcessorException +from common.lib.outputs import Datasource # only make this worker available when explicitly enabled, so it never loads on # a normal/production instance (the datasource folder is always discovered, but @@ -36,10 +37,11 @@ class SearchTest(Search): Dummy search worker for exercising the worker/queue status pages """ type = "test-search" # job ID - category = "Search" # category + title = "Test datasource (dev only)" # title displayed in UI description = "Development-only datasource that creates dummy datasets in various states (complete, forever, crash) to exercise admin status pages." extension = "ndjson" # extension of result file + output = Datasource() # not offered as a processor for existing datasets accepts = [None] diff --git a/datasources/threads/search_threads.py b/datasources/threads/search_threads.py index 38f8eb946..fba666b82 100644 --- a/datasources/threads/search_threads.py +++ b/datasources/threads/search_threads.py @@ -11,6 +11,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchThreads(Search): @@ -18,11 +19,14 @@ class SearchThreads(Search): Import scraped Threads data """ type = "threads-search" # job ID - category = "Search" # category + title = "Import scraped Threads data" # title displayed in UI - description = "Import Threads data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Threads data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True + icon = "brand-threads" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/tiktok/search_tiktok.py b/datasources/tiktok/search_tiktok.py index 4c1bd4ada..f2809d2bb 100644 --- a/datasources/tiktok/search_tiktok.py +++ b/datasources/tiktok/search_tiktok.py @@ -10,6 +10,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchTikTok(Search): @@ -17,11 +18,14 @@ class SearchTikTok(Search): Import scraped TikTok data """ type = "tiktok-search" # job ID - category = "Search" # category + title = "Import scraped Tiktok data" # title displayed in UI - description = "Import Tiktok data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Tiktok data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True + icon = "brand-tiktok" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/tiktok/tiktok-explorer.css b/datasources/tiktok/tiktok-explorer.css index 8b07e38ef..7c407cb95 100644 --- a/datasources/tiktok/tiktok-explorer.css +++ b/datasources/tiktok/tiktok-explorer.css @@ -7,6 +7,7 @@ font-family: Arial, sans-serif; font-size: 15px; width: 580px; + padding: 20px; margin: 0 auto; background-color: white; list-style-type: none; @@ -81,7 +82,7 @@ .metrics { display: table-cell; - width: 5%; + width: 10%; vertical-align: top; margin-top: 40px; } diff --git a/datasources/tiktok_comments/search_tiktok_comments.py b/datasources/tiktok_comments/search_tiktok_comments.py index 081abb8fd..12d0ef48c 100644 --- a/datasources/tiktok_comments/search_tiktok_comments.py +++ b/datasources/tiktok_comments/search_tiktok_comments.py @@ -9,6 +9,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem, MissingMappedField from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchTikTokComments(Search): @@ -16,11 +17,13 @@ class SearchTikTokComments(Search): Import scraped TikTok comment data """ type = "tiktok-comments-search" # job ID - category = "Search" # category + title = "Import scraped Tiktok comment data" # title displayed in UI - description = "Import Tiktok comment data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Tiktok comment data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + output = Datasource() is_from_zeeschuimer = True + icon = "brand-tiktok" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/tiktok_urls/search_tiktok_urls.py b/datasources/tiktok_urls/search_tiktok_urls.py index 472128779..de2a71396 100644 --- a/datasources/tiktok_urls/search_tiktok_urls.py +++ b/datasources/tiktok_urls/search_tiktok_urls.py @@ -15,6 +15,7 @@ from backend.lib.search import Search from common.lib.helpers import UserInput +from common.lib.outputs import Datasource from common.lib.exceptions import WorkerInterruptedException, QueryParametersException, ProcessorException from datasources.tiktok.search_tiktok import SearchTikTok as SearchTikTokByImport @@ -29,12 +30,14 @@ class SearchTikTokByID(Search): Import scraped TikTok data """ type = "tiktok-urls-search" # job ID - category = "Search" # category + title = "Search TikTok by post URL" # title displayed in UI description = "Retrieve metadata for TikTok post URLs." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) + icon = "brand-tiktok" + tags = ["scraping"] # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/truth/search_truth.py b/datasources/truth/search_truth.py index 6693dd614..5aabae418 100644 --- a/datasources/truth/search_truth.py +++ b/datasources/truth/search_truth.py @@ -6,6 +6,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchGab(Search): @@ -13,10 +14,12 @@ class SearchGab(Search): Import scraped truth social data """ type = "truthsocial-search" # job ID - category = "Search" # category + title = "Import scraped Truth Social data" # title displayed in UI - description = "Import Truth Social data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import Truth Social data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True fake = "" diff --git a/datasources/tumblr/search_tumblr.py b/datasources/tumblr/search_tumblr.py index d87531f03..29d2f41a8 100644 --- a/datasources/tumblr/search_tumblr.py +++ b/datasources/tumblr/search_tumblr.py @@ -20,6 +20,7 @@ from common.lib.helpers import UserInput, strip_tags from common.lib.exceptions import QueryParametersException, ProcessorInterruptedException, ConfigException from common.lib.item_mapping import MappedItem +from common.lib.outputs import Datasource __author__ = "Sal Hagen" __credits__ = ["Sal Hagen", "Tumblr API (api.tumblr.com)"] @@ -32,12 +33,15 @@ class SearchTumblr(Search): Tumblr data filter module. """ type = "tumblr-search" # job ID - category = "Search" # category + title = "Search Tumblr" # title displayed in UI description = "Retrieve Tumblr posts by tags or blogs." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"tags"}) + tags = ["API"] + + icon = "brand-tumblr" # not available as a processor for existing datasets accepts = [None] diff --git a/datasources/tumblr/tumblr-explorer.css b/datasources/tumblr/tumblr-explorer.css index 799c0efd7..3c0185e43 100644 --- a/datasources/tumblr/tumblr-explorer.css +++ b/datasources/tumblr/tumblr-explorer.css @@ -270,7 +270,7 @@ footer { } .note-counts { - padding-top: 19px; + padding: 19px 0; } .note-count { @@ -326,22 +326,16 @@ footer { .item-annotations { background-color: #7c5cff; color: white; - border-radius: 0px 0px 8px 8px; } .item-annotation { padding: 15px; } -.item-annotation input { - border-radius: 5px; -} - .item-annotation > .annotation-label { display: inline-block; vertical-align: middle; text-align: right; - min-width: 150px; margin-right: 5px; line-height: 1.6em; overflow-x: hidden; diff --git a/datasources/twitter-import/search_twitter.py b/datasources/twitter-import/search_twitter.py index f5de495ea..ce4de8a1c 100644 --- a/datasources/twitter-import/search_twitter.py +++ b/datasources/twitter-import/search_twitter.py @@ -11,6 +11,7 @@ from common.lib.helpers import strip_tags from common.lib.item_mapping import MappedItem from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchTwitterViaZeeschuimer(Search): @@ -18,11 +19,14 @@ class SearchTwitterViaZeeschuimer(Search): Import scraped X/Twitter data """ type = "twitter-import" # job ID - category = "Search" # category + title = "Import scraped X/Twitter data" # title displayed in UI - description = "Import X/Twitter data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import X/Twitter data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True + icon = "brand-x-twitter" # not available as a processor for existing datasets accepts = [] diff --git a/datasources/twitterv2/search_twitter.py b/datasources/twitterv2/search_twitter.py index a813e3d50..894f61c72 100644 --- a/datasources/twitterv2/search_twitter.py +++ b/datasources/twitterv2/search_twitter.py @@ -12,6 +12,7 @@ from common.lib.exceptions import QueryParametersException, ProcessorInterruptedException, QueryNeedsExplicitConfirmationException from common.lib.helpers import convert_to_int, UserInput, timify from common.lib.item_mapping import MappedItem, MissingMappedField +from common.lib.outputs import Datasource class SearchWithTwitterAPIv2(Search): @@ -21,8 +22,12 @@ class SearchWithTwitterAPIv2(Search): type = "twitterv2-search" # job ID title = "X/Twitter API (v2)" extension = "ndjson" - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) + + icon = "brand-twitter" + + tags = ["API"] previous_request = 0 import_issues = True diff --git a/datasources/upload/import_csv.py b/datasources/upload/import_csv.py index a76081475..7acd1ad17 100644 --- a/datasources/upload/import_csv.py +++ b/datasources/upload/import_csv.py @@ -14,6 +14,7 @@ from datetime import datetime from backend.lib.processor import BasicProcessor +from common.lib.outputs import Datasource from common.lib.exceptions import QueryParametersException, QueryNeedsFurtherInputException, \ QueryNeedsExplicitConfirmationException, CsvDialectException from common.lib.helpers import strip_tags, sniff_encoding, UserInput, HashCache @@ -21,12 +22,15 @@ class SearchCustom(BasicProcessor): type = "upload-search" # job ID - category = "Search" # category + title = "Custom Dataset Upload" # title displayed in UI description = "Upload your own CSV file to be used as a dataset" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + # a collected, top-level csv; the columns depend on the uploaded file, so unknown here + output = Datasource() + icon = "file-import" + + tags = ["upload"] max_workers = 1 diff --git a/datasources/vk/search_vk.py b/datasources/vk/search_vk.py index 7fcbebe66..8e82688b0 100644 --- a/datasources/vk/search_vk.py +++ b/datasources/vk/search_vk.py @@ -9,6 +9,7 @@ from common.lib.exceptions import QueryParametersException, ProcessorInterruptedException, ProcessorException from common.lib.helpers import UserInput from common.lib.item_mapping import MappedItem +from common.lib.outputs import Datasource class SearchVK(Search): @@ -18,8 +19,10 @@ class SearchVK(Search): type = "vk-search" # job ID title = "VK" extension = "ndjson" - is_local = False # Whether this datasource is locally scraped - is_static = False # Whether this datasource is still updated + output = Datasource() + icon = "brand-vk" + + tags = ["API"] previous_request = 0 import_issues = True diff --git a/datasources/xiaohongshu/search_rednote.py b/datasources/xiaohongshu/search_rednote.py index 27c5c9305..47ee7114f 100644 --- a/datasources/xiaohongshu/search_rednote.py +++ b/datasources/xiaohongshu/search_rednote.py @@ -12,6 +12,7 @@ from common.lib.exceptions import MapItemException from common.lib.item_mapping import MappedItem, MissingMappedField from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchRedNote(Search): @@ -19,10 +20,12 @@ class SearchRedNote(Search): Import scraped RedNote/Xiaohongshu/XSH data """ type = "xiaohongshu-search" # job ID - category = "Search" # category + title = "Import scraped RedNote data" # title displayed in UI - description = "Import RedNote data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import RedNote data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + # the tag column the co-tag and hashtag networks look for + output = Datasource(columns={"hashtags"}) is_from_zeeschuimer = True # not available as a processor for existing datasets diff --git a/datasources/xiaohongshu_comments/search_rednote_comments.py b/datasources/xiaohongshu_comments/search_rednote_comments.py index 655b4417b..8a9427055 100644 --- a/datasources/xiaohongshu_comments/search_rednote_comments.py +++ b/datasources/xiaohongshu_comments/search_rednote_comments.py @@ -9,6 +9,7 @@ from backend.lib.search import Search from common.lib.item_mapping import MappedItem, MissingMappedField from common.lib.helpers import normalize_url_encoding +from common.lib.outputs import Datasource class SearchRedNoteComments(Search): @@ -16,10 +17,11 @@ class SearchRedNoteComments(Search): Import scraped RedNote/Xiaohongshu/XSH comment data """ type = "xiaohongshu-comments-search" # job ID - category = "Search" # category + title = "Import scraped RedNote comment data" # title displayed in UI - description = "Import RedNote comment data collected with an external tool such as Zeeschuimer." # description displayed in UI + description = "Import RedNote comment data collected with Zeeschuimer." # description displayed in UI extension = "ndjson" # extension of result file, used internally and in UI + output = Datasource() is_from_zeeschuimer = True # not available as a processor for existing datasets diff --git a/processors/audio/audio_extractor.py b/processors/audio/audio_extractor.py index 637798836..bc87d37a0 100644 --- a/processors/audio/audio_extractor.py +++ b/processors/audio/audio_extractor.py @@ -9,8 +9,9 @@ from pathlib import Path import oslex -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility, is_executable +from common.lib.outputs import MediaArchive from common.lib.exceptions import ProcessorInterruptedException __author__ = "Dale Wahl" @@ -20,7 +21,6 @@ from common.lib.user_input import UserInput - class AudioExtractor(BasicProcessor): """ Audio from video Extractor @@ -28,14 +28,19 @@ class AudioExtractor(BasicProcessor): Uses ffmpeg to extract audio from videos and saves them in an archive. """ type = "audio-extractor" # job type ID - category = "Audio" # category - title = "Extract audio from videos" # title displayed in UI - description = "Create audio files per video" # description displayed in UI + description = ProcessorDescription( + title="Extract audio from videos", + tags=["audio", "download media"], + description="Extract the audio track from each video and save the results as WAV files in a ZIP archive. Uses ffmpeg to convert audio to 16 kHz.", + icon="closed-captioning", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of media files + output = MediaArchive(media="audio") media_type = "audio" # Allow on video datasets when ffmpeg is available - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}, preferred_followups=["audio-to-text"]) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}, preferred_followups=["audio-to-text"]) @classmethod def get_options(cls, parent_dataset=None, config=None): diff --git a/processors/conversion/clarifai_to_csv.py b/processors/conversion/clarifai_to_csv.py index 6aeb08168..f39c6ccf7 100644 --- a/processors/conversion/clarifai_to_csv.py +++ b/processors/conversion/clarifai_to_csv.py @@ -3,8 +3,9 @@ """ import csv -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -23,11 +24,17 @@ class ConvertClarifaiOutputToCSV(BasicProcessor): information to allow 'flattening' the output to a simple CSV file. """ type = "convert-clarifai-vision-to-csv" # job type ID - category = "Conversion" # category - title = "Convert Clarifai results to CSV" # title displayed in UI - description = "Convert the Clarifai API output to a simplified CSV file." # description displayed in UI + description = ProcessorDescription( + title="Convert Clarifai results to CSV", + tags=["visual", "annotation", "machine learning", "classification", "external service"], + description="Convert the Clarifai API output to a simplified CSV file.", + icon="file-csv", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow processor on Clarifai API output compatibility = Compatibility(types={"clarifai-api"}) diff --git a/processors/conversion/consolidate_urls.py b/processors/conversion/consolidate_urls.py index e3de3d5b6..8b2cf7400 100644 --- a/processors/conversion/consolidate_urls.py +++ b/processors/conversion/consolidate_urls.py @@ -7,8 +7,9 @@ from processors.conversion.extract_urls import ExtractURLs from common.lib.exceptions import ProcessorInterruptedException -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.helpers import UserInput, split_urls __author__ = "Dale Wahl" @@ -24,10 +25,18 @@ class ConsolidateURLs(BasicProcessor): """ type = "consolidate-urls" # job type ID - category = "Conversion" # category - title = "Consolidate URLs" # title displayed in UI - description = "Retain only the domain (and optionally path) of URLs; used for custom networks (e.g. author + domains)" + description = ProcessorDescription( + title="Consolidate URLs", + tags=["conversion", "urls"], + description="Reduce URLs in a column to a shorter form, keeping only the domain or applying per-site rules for Facebook, Instagram, YouTube, and other platforms. Optionally expand shortened URLs first. Useful for building networks that link authors to the domains they share.", + warnings=[ + "Expanding shortened URLs is slow and not recommended for datasets larger than 10,000 items.", + ], + icon="globe", + ) extension = "csv" + # a derived table + output = Table() # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/conversion/convert_text.py b/processors/conversion/convert_text.py index 007c245e6..6663c4c18 100644 --- a/processors/conversion/convert_text.py +++ b/processors/conversion/convert_text.py @@ -5,8 +5,9 @@ import csv from common.lib.exceptions import ProcessorInterruptedException -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.helpers import UserInput __author__ = "Sal Hagen" @@ -20,12 +21,24 @@ class ConvertText(BasicProcessor): Retain only posts matching a given lexicon """ type = "convert-text" # job type ID - category = "Conversion" # category - title = "Replace text" # title displayed in UI - description = ("Find text in selected fields, replace parts of it, and write to a new dataset. Converted texts can " - "also be added to the original dataset as annotations.") # description displayed in UI + description = ProcessorDescription( + title="Find and replace text", + tags=["conversion", "preprocessing"], + description="Find text in selected columns, replace matching parts, and write the result to a new dataset.", + info=[ + "Matches can be plain text or a regular expression.", + "Replaced text can also be added back to the original dataset as annotations." + ], + warnings=[ + "Reading a single JSON file will use a lot of memory for large datasets. You may wish to download the dataset as a CSV or NDJSON instead." + ], + icon="arrows-turn-to-dots", + ) extension = "csv" + # a derived table + output = Table() + # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/conversion/csv_to_json.py b/processors/conversion/csv_to_json.py index 418bfd05e..28048e30c 100644 --- a/processors/conversion/csv_to_json.py +++ b/processors/conversion/csv_to_json.py @@ -3,8 +3,9 @@ """ import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import File __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -16,10 +17,16 @@ class ConvertCSVToJSON(BasicProcessor): Convert a CSV file to JSON """ type = "convert-csv" # job type ID - category = "Conversion" # category - title = "Convert to JSON" # title displayed in UI - description = "Change a CSV file to a JSON file" # description displayed in UI + description = ProcessorDescription( + title="Convert to JSON", + tags=["conversion"], + description="Convert a CSV file to a JSON file, writing one JSON list with one object per row.", + warnings=["This processor will read the entire CSV file and write it to a single JSON file. This may take a long time and use a lot of memory for large datasets."], + icon="square-js", + ) extension = "json" # extension of result file, used internally and in UI + # a single json file + output = File("json") # Allow on CSV datasets compatibility = Compatibility(extensions={"csv"}) diff --git a/processors/conversion/export_datasets.py b/processors/conversion/export_datasets.py index 2dc12308e..24d5ecbea 100644 --- a/processors/conversion/export_datasets.py +++ b/processors/conversion/export_datasets.py @@ -5,10 +5,11 @@ import json import datetime -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.dataset import DataSet from common.lib.exceptions import DataSetException from common.lib.compatibility import Compatibility +from common.lib.outputs import Archive __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -22,11 +23,18 @@ class ExportDatasets(BasicProcessor): Export a dataset and all its children to a ZIP file """ type = "export-datasets" # job type ID - category = "Conversion" # category - title = "Export dataset and processor results" # title displayed in UI - description = ("Creates a ZIP file containing the dataset and all processor results. This can also be uploaded to " - "another 4CAT instance. Filters are not included. Results expire after one day.") # description displayed in UI + description = ProcessorDescription( + title="Export dataset and processor results", + tags=["conversion", "metadata"], + description="Create a ZIP file containing the dataset and all of its processor results. The ZIP can be uploaded to another 4CAT instance. Filters are not included.", + warnings=[ + "This dataset expires after one day. You will need to run this processor again to get a new export file.", + ], + icon="file-export", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of data files + output = Archive() # coarse map spec; is_compatible_with (below) is the runtime truth -- it also checks # the requesting user owns the dataset (is_accessible_by), which is per-user, not shape diff --git a/processors/conversion/extract_urls.py b/processors/conversion/extract_urls.py index d95600874..e369bf01a 100644 --- a/processors/conversion/extract_urls.py +++ b/processors/conversion/extract_urls.py @@ -10,9 +10,10 @@ from ural import urls_from_text from common.lib.exceptions import ProcessorInterruptedException -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Stijn Peeters", "Dale Wahl", "Sal Hagen"] @@ -27,10 +28,18 @@ class ExtractURLs(BasicProcessor): Retain only posts where a given column matches a given value """ type = "extract-urls-filter" # job type ID - category = "Conversion" # category - title = "Extract and expand URLs" # title displayed in UI - description = "Extract any URLs from selected column(s) with the option to expand shortened URLs." + description = ProcessorDescription( + title="Extract URLs", + tags=["conversion", "urls"], + description="Extract URLs from selected columns into a new table listing each item's unique URLs. Optionally expand shortened URLs to their final destination, and resolve CrowdTangle's inline links.", + warnings=[ + "Expanding shortened URLs sends a request per URL and is slow; it is not recommended on datasets larger than 10,000 items.", + ], + icon="globe", + ) extension = "csv" + # a derived table + output = Table() # any csv/ndjson dataset, except this processor's own filter output compatibility = Compatibility(extensions={"csv", "ndjson"}, excluded_types={"extract-urls-filter"}) diff --git a/processors/conversion/hash_images.py b/processors/conversion/hash_images.py index 13b0414ca..a2a4e0760 100644 --- a/processors/conversion/hash_images.py +++ b/processors/conversion/hash_images.py @@ -6,10 +6,11 @@ from PIL import UnidentifiedImageError -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput, hash_image, stringify_hash from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from processors.metrics.group_hashes import HashGrouper @@ -24,18 +25,24 @@ class ImageHasher(BasicProcessor): Hash images """ type = "image-hasher" # job type ID - category = "Conversion" # category - title = "Hash images" # title displayed in UI - description = "Convert images to text hashes for comparison and similarity detection." # description displayed in UI + description = ProcessorDescription( + title="Hash images", + tags=["conversion", "visual"], + description="Calculate a perceptual hash for each image so near-duplicate images can be compared. Choose a perceptual, wavelet, or crop-resistant hash, and optionally group visually similar images together using a similarity threshold.", + references=[ + "[Imagehash library](https://github.com/JohannesBuchner/imagehash?tab=readme-ov-file)", + "Explainer: [Perceptual hashing](https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html)", + ], + info=[ + "Smaller hash sizes are faster; larger sizes are more accurate. Lower the similarity threshold for stricter grouping.", + ], + ) extension = "csv" + # a derived table + output = Table() # image datasets: image archives, image-downloader output, or extracted video frames - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}) - - references = [ - "[Imagehash library](https://github.com/JohannesBuchner/imagehash?tab=readme-ov-file)", - "Explainer: [Perceptual hashing](https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html)", - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}) # "phash": "Perceptual (DCT) hash: strong general-purpose near-duplicate. Robust to resize, JPEG, small blur/contrast tweaks; weaker to large crop/rotation. Hamming; size=16≈256-bit (use ~20–40 threshold), size=32≈1024-bit.", # "whash-haar": "Wavelet (Haar) hash: similar to pHash but often more stable to brightness/exposure shifts. Robust to resize/compression/exposure; weaker to large crop/rotation. Hamming; size=16≈256-bit.", diff --git a/processors/conversion/item_to_annotation.py b/processors/conversion/item_to_annotation.py index bd91871a6..58f966a89 100644 --- a/processors/conversion/item_to_annotation.py +++ b/processors/conversion/item_to_annotation.py @@ -2,8 +2,9 @@ Change a dataset item to an annotation """ from common.lib.exceptions import ProcessorInterruptedException -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import NoOutput from common.lib.helpers import UserInput __author__ = "Sal Hagen" @@ -17,12 +18,16 @@ class ItemToAnnotation(BasicProcessor): Change a dataset item to an annotation """ type = "item-to-annotation" # job type ID - category = "Conversion" # category filter = True # to indicate we're filtering the top dataset - title = "Convert items to annotations" # title displayed in UI - description = ("Convert a regular dataset item to an annotation. This will show it as a separate value in the " - "Explorer. Item values must be numbers or strings.") # description displayed in UI + description = ProcessorDescription( + title="Convert items to annotations", + tags=["conversion", "annotation", "override"], + description="Convert values from selected columns into annotations on the parent dataset, shown as separate values in the Explorer. Only values that are numbers or strings are converted.", + icon="tags", + ) extension = "csv" + # writes annotations to its parent, no result file of its own + output = NoOutput() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/conversion/merge_datasets.py b/processors/conversion/merge_datasets.py index a2022b57e..8f59d19b0 100644 --- a/processors/conversion/merge_datasets.py +++ b/processors/conversion/merge_datasets.py @@ -4,12 +4,13 @@ import csv import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.dataset import DataSet from common.lib.exceptions import ProcessorInterruptedException, DataSetException from common.lib.helpers import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility +from common.lib.outputs import Filter import ural __author__ = "Stijn Peeters" @@ -25,10 +26,17 @@ class DatasetMerger(BasicProcessor): Merge two datasets """ type = "merge-datasets" # job type ID - category = "Conversion" # category - title = "Merge datasets" # title displayed in UI - description = "Merge this dataset with other datasets of the same format. A new dataset is " \ - "created containing a combination of items from the original datasets." # description displayed in UI + description = ProcessorDescription( + title="Merge datasets", + tags=["merging"], + description="Combine this dataset with other datasets of the same format into a new dataset. Provide the URLs of the datasets to merge, and choose whether to keep or remove items that share an item ID across datasets.", + warnings=[ + "All datasets must have the same format and columns, or the merge fails.", + ], + icon="arrows-to-dot", + ) + # keeps the (primary parent's) shape; the merged datasets share its format + output = Filter() # a collector's csv or ndjson output compatibility = Compatibility(is_collector=True, extensions={"csv", "ndjson"}) diff --git a/processors/conversion/ndjson_to_csv.py b/processors/conversion/ndjson_to_csv.py index 0782e0954..3d58b3963 100644 --- a/processors/conversion/ndjson_to_csv.py +++ b/processors/conversion/ndjson_to_csv.py @@ -5,8 +5,9 @@ import json from common.lib.helpers import flatten_dict -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException __author__ = "Dale Wahl" @@ -21,11 +22,15 @@ class ConvertNDJSONtoCSV(BasicProcessor): Convert a NDJSON file to CSV """ type = "convert-ndjson-csv" # job type ID - category = "Conversion" # category - title = "Convert NDJSON file to CSV" # title displayed in UI - description = "Create a CSV file from an NDJSON file. Note that some data may be lost as CSV files cannot " \ - "contain nested data." # description displayed in UI + description = ProcessorDescription( + title="Convert NDJSON file to CSV", + tags=["conversion"], + description="Create a CSV file from an NDJSON dataset, flattening nested fields into separate columns.", + icon="file-csv", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow on NDJSON datasets compatibility = Compatibility(extensions={"ndjson"}) diff --git a/processors/conversion/remove_author_info.py b/processors/conversion/remove_author_info.py index ad1e8daf1..2671fdfe0 100644 --- a/processors/conversion/remove_author_info.py +++ b/processors/conversion/remove_author_info.py @@ -9,8 +9,9 @@ import json import csv -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Filter from common.lib.helpers import dict_search_and_update, UserInput, HashCache __author__ = "Stijn Peeters" @@ -26,20 +27,31 @@ class AuthorInfoRemover(BasicProcessor): Retain only posts where a given column matches a given value """ type = "author-info-remover" # job type ID - category = "Conversion" # category filter = True # to indicate we're filtering the top dataset - title = "Pseudonymise or anonymise" # title displayed in UI - description = "Removes or replaces data from the dataset in fields identified as containing personal information" + description = ProcessorDescription( + title="Pseudonymise or anonymise", + tags=["authors", "override"], + description="Remove or replace values in fields that hold personal information, such as author and user columns. Choose to replace values with 'REDACTED' or with a unique identifier that hides the value while still marking equal values as equal.", + info=[ + "This processor targets fields that contain \"author\" or \"user\" in their name, but you can also specify other fields to process.", + "See references for more information on hashing and salting used to create unique identifiers." + ], + references=[ + "[What is a hash?](https://techterms.com/definition/hash)", + "[What is a salt?](https://en.wikipedia.org/wiki/Salt_(cryptography))", + "[What is Blake2?](https://en.wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2)" + ], + warnings=[ + "This overwrites the original dataset in place and cannot be undone.", + ], + icon="user-secret", + ) + # keeps the parent's shape, with personal-information fields removed or replaced + output = Filter() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) - references = [ - "[What is a hash?](https://techterms.com/definition/hash)", - "[What is a salt?](https://en.wikipedia.org/wiki/Salt_(cryptography))", - "[What is Blake2?](https://en.wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2)" - ] - @classmethod def get_options(cls, parent_dataset=None, config=None): options = { diff --git a/processors/conversion/split_by_thread.py b/processors/conversion/split_by_thread.py index a53f8e55e..098c4ca05 100644 --- a/processors/conversion/split_by_thread.py +++ b/processors/conversion/split_by_thread.py @@ -3,8 +3,9 @@ """ import csv -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Archive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -21,11 +22,17 @@ class ThreadSplitter(BasicProcessor): containing only the posts in that thread. """ type = "split-threads" # job type ID - category = "Conversion" # category - title = "Split by thread" # title displayed in UI - description = "Split the dataset per thread. The result is a ZIP archive containing separate CSV files." # description displayed in UI + description = ProcessorDescription( + title="Split by thread", + tags=["conversion"], + description="Split the dataset into one file per thread, keeping only the posts in each thread.", + icon="scissors", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of data files + output = Archive() + # datasets with a thread structure (4chan/8chan, reddit, breitbart) compatibility = Compatibility(datasources={"fourchan", "eightchan", "reddit", "breitbart"}) diff --git a/processors/conversion/stringify.py b/processors/conversion/stringify.py index 95ed4b2d2..213387e23 100644 --- a/processors/conversion/stringify.py +++ b/processors/conversion/stringify.py @@ -4,8 +4,9 @@ import re import string -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import File from common.lib.helpers import UserInput __author__ = "Sal Hagen" @@ -18,10 +19,18 @@ class Stringify(BasicProcessor): Merge post body into one long string """ type = "stringify-posts" # job type ID - category = "Conversion" # category - title = "Merge texts" # title displayed in UI - description = "Merges the data from the body column into a single text file. The result can be used for word clouds, word trees, etc." # description displayed in UI + description = ProcessorDescription( + title="Merge texts", + tags=["conversion", "preprocessing"], + description="Merge the text from the body column of every item into a single continuous text file. Optionally strip URLs, numbers, or punctuation, and convert the text to lowercase.", + info=[ + "The output works well as input for word clouds, word trees, and similar text visualisations.", + ], + icon="file-lines", + ) extension = "txt" # extension of result file, used internally and in UI + # a single txt file + output = File("txt") # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/conversion/tcat_auto_upload.py b/processors/conversion/tcat_auto_upload.py index afdc2d521..7e7152de2 100644 --- a/processors/conversion/tcat_auto_upload.py +++ b/processors/conversion/tcat_auto_upload.py @@ -6,10 +6,11 @@ import json from urllib.parse import urlparse -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.user_input import UserInput from common.lib.helpers import get_last_line from common.lib.compatibility import Compatibility +from common.lib.outputs import File __author__ = "Dale Wahl" __credits__ = ["Dale Wahl", "Stijn Peeters"] @@ -23,10 +24,19 @@ class FourcatToDmiTcatUploader(BasicProcessor): File to be imported by TCAT's import-jsondump.php """ type = "tcat-auto-upload" # job type ID - category = "Conversion" # category - title = "Upload to DMI-TCAT" # title displayed in UI - description = "Send a TCAT-ready JSON file to a particular DMI-TCAT server." # description displayed in UI + description = ProcessorDescription( + title="Upload to DMI-TCAT", + tags=["conversion", "external service"], + description="Send a TCAT-ready JSON file to a configured DMI-TCAT server, where it is imported as a new tweet bin. The result is an HTML page that redirects to the dataset on the TCAT server.", + warnings=[ + "The dataset is sent to an external DMI-TCAT server, which must be configured in the settings.", + "Tweets larger than 40 KB are dropped, because TCAT rejects them on import.", + ], + icon="brand-twitter", + ) extension = "html" # extension of result file, used internally and in UI + # a single html file + output = File("html") # the TCAT converter's output, when a TCAT server is configured compatibility = Compatibility(types={"convert-ndjson-for-tcat"}, required_settings={"tcat-auto-upload.server_url", "tcat-auto-upload.token", "tcat-auto-upload.username", "tcat-auto-upload.password"}) diff --git a/processors/conversion/twitter_ndjson_to_tcat_json.py b/processors/conversion/twitter_ndjson_to_tcat_json.py index 44f4f0b04..0c3044c56 100644 --- a/processors/conversion/twitter_ndjson_to_tcat_json.py +++ b/processors/conversion/twitter_ndjson_to_tcat_json.py @@ -3,8 +3,9 @@ """ import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import File __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -16,10 +17,15 @@ class ConvertNDJSONToJSON(BasicProcessor): Convert a Twitter NDJSON file to be importable by TCAT's import-jsondump.php """ type = "convert-ndjson-for-tcat" # job type ID - category = "Conversion" # category - title = "Convert to TCAT JSON" # title displayed in UI - description = "Convert a Twitter dataset to a TCAT-compatible format. This file can then be uploaded to TCAT." # description displayed in UI + description = ProcessorDescription( + title="Convert to TCAT JSON", + tags=["conversion"], + description="Convert a Twitter/X (API v2) dataset to the JSON format that DMI-TCAT's import-jsondump.php can read. The result can then be uploaded to a TCAT server.", + icon="square-js", + ) extension = "json" # extension of result file, used internally and in UI + # a single json file + output = File("json") # Allow processor on Twitter/X (API v2) datasets compatibility = Compatibility(types={"twitterv2-search"}, preferred_followups=["tcat-auto-upload"]) diff --git a/processors/conversion/upload_annotations.py b/processors/conversion/upload_annotations.py index 958fbfa15..f7472c1d1 100644 --- a/processors/conversion/upload_annotations.py +++ b/processors/conversion/upload_annotations.py @@ -6,8 +6,9 @@ from flask import g -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException, DataSetException from common.lib.helpers import UserInput from common.lib.dataset import DataSet @@ -23,12 +24,19 @@ class UploadAnnotations(BasicProcessor): Upload annotations for a dataset """ type = "upload-annotations" # job type ID - category = "Conversion" # category - title = "Upload annotations" # title displayed in UI - description = ("Upload annotations for this dataset via a CSV file or by pasting text data. " - "The first column should contain item IDs; subsequent columns become annotation fields. " - "For CSV file uploads, comma is used as the separator. For text input, a custom separator can be specified.") + description = ProcessorDescription( + title="Upload annotations", + tags=["conversion", "annotation", "override"], + description="Add annotations to the dataset from a CSV file or pasted text. The first column holds item IDs matching items in the dataset, and each further column becomes an annotation field.", + warnings=[ + "Only rows whose item ID matches an item in the dataset are used; other rows are skipped.", + "At most 20 new annotation fields per upload are allowed, and field names must not clash with existing ones.", + ], + icon="tags", + ) extension = "csv" + # a derived table + output = Table() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/conversion/view_metadata.py b/processors/conversion/view_metadata.py index 8d09afb44..742fbd92c 100644 --- a/processors/conversion/view_metadata.py +++ b/processors/conversion/view_metadata.py @@ -6,8 +6,9 @@ import json import zipfile -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.user_input import UserInput __author__ = "Dale Wahl" @@ -23,10 +24,15 @@ class ViewMetadata(BasicProcessor): Reformats the .metadata.json file and calculates some basic analytics """ type = "metadata-viewer" # job type ID - category = "Conversion" # category - title = "View media metadata" # title displayed in UI - description = "Reformats the .metadata.json file and calculates analytics" # description displayed in UI + description = ProcessorDescription( + title="Extract media metadata", + tags=["conversion"], + description="Read the .metadata.json file produced by an image or video downloader and turn it into a flat table, with one row per downloaded item. Optionally include items whose download failed.", + icon="circle-info", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow on downloaded media datasets compatibility = Compatibility(type_prefixes={"video-downloader", "image-downloader"}) diff --git a/processors/conversion/vision_api_to_csv.py b/processors/conversion/vision_api_to_csv.py index 578563156..0f677ed2b 100644 --- a/processors/conversion/vision_api_to_csv.py +++ b/processors/conversion/vision_api_to_csv.py @@ -3,8 +3,9 @@ """ import csv -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.helpers import UserInput __author__ = "Stijn Peeters" @@ -24,11 +25,18 @@ class ConvertVisionOutputToCSV(BasicProcessor): information to allow 'flattening' the output to a simple CSV file. """ type = "convert-google-vision-to-csv" # job type ID - category = "Conversion" # category - title = "Convert Vision results to CSV" # title displayed in UI - description = ("Convert the Vision API output to a simplified CSV file. Also allows writing results as annotations " - "to the original dataset.") # description displayed in UI + description = ProcessorDescription( + title="Convert Google Vision results to CSV", + tags=["conversion"], + description="Convert the Google Vision API output from NDJSON to a simplified CSV file, flattening detected labels, logos, landmarks, objects, and text into columns. Optionally write the results back as annotations on the original dataset.", + info=[ + "Some detail is lost when flattening, so keep the original NDJSON dataset if you need the full output.", + ], + icon="file-csv", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Google Vision API output compatibility = Compatibility(types={"google-vision-api"}) diff --git a/processors/filtering/accent_fold.py b/processors/filtering/accent_fold.py index aed4de800..49c2c3d1c 100644 --- a/processors/filtering/accent_fold.py +++ b/processors/filtering/accent_fold.py @@ -5,8 +5,9 @@ import csv from unidecode import unidecode -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Output, PASSTHROUGH from common.lib.helpers import UserInput __author__ = "Stijn Peeters" @@ -22,11 +23,17 @@ class AccentFoldingFilter(BasicProcessor): Fold accents, case, and diacritics. """ type = "accent-folder" # job type ID - category = "Filtering" # category - title = "Convert accented and non-Latin characters" # title displayed in UI - description = ("Replaces or transliterates non-Latin characters with the closest ASCII equivalent, converting e.g. " - "'á' to 'a', 'ç' to 'c', etc. This creates a new dataset.") + description = ProcessorDescription( + title="Convert accents and non-Latin characters", + tags=["filtering", "preprocessing"], + description="Replace accented and non-Latin characters with their closest ASCII equivalent, converting for example 'á' to 'a' and 'ç' to 'c'. Fold only accented Latin characters, or transliterate all non-ASCII characters. Optionally convert all text to lowercase. This creates a new dataset.", + icon="language", + ) extension = "csv" # extension of result file, used internally and in UI + # writes a new csv keeping the parent's columns; made standalone, so its position + # and collector-ness take on the original dataset's and are unknown here + output = Output(extension="csv", columns=PASSTHROUGH, position=None, collector=None, + datasource=PASSTHROUGH) # Allow on top-level CSV datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv"}) diff --git a/processors/filtering/base_filter.py b/processors/filtering/base_filter.py index 0e0e95cf6..621f204cc 100644 --- a/processors/filtering/base_filter.py +++ b/processors/filtering/base_filter.py @@ -6,8 +6,9 @@ import json import shutil -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Filter __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -22,12 +23,19 @@ class BaseFilter(BasicProcessor): Retain only posts where a given column matches a given value """ type = "column-filter" # job type ID - category = "Filtering" # category - title = "Base Filter" # title displayed in UI - description = "This should not be available." + description = ProcessorDescription( + title="Base filter", + tags=["filtering", "internal"], + description="Abstract base class for filters that re-emit a parent dataset's rows. Not runnable on its own.", + icon="filter", + ) item_ids = [] + # A filter re-emits its parent's rows, so its extension, media and columns are + # the parent's. Subclasses keep this; one that adds a column declares its own. + output = Filter() + # Abstract base filter; not runnable on its own (empty type set never matches) compatibility = Compatibility(types=set()) diff --git a/processors/filtering/column_filter.py b/processors/filtering/column_filter.py index c681f3ca7..73caa87dd 100644 --- a/processors/filtering/column_filter.py +++ b/processors/filtering/column_filter.py @@ -4,7 +4,7 @@ import re import datetime -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.dataset import StatusType from processors.filtering.base_filter import BaseFilter from common.lib.helpers import UserInput, convert_to_int @@ -21,10 +21,11 @@ class ColumnFilter(BaseFilter): Retain only posts where a given column matches a given value """ type = "column-filter" # job type ID - category = "Filtering" # category - title = "Filter by value" # title displayed in UI - description = ("A flexible and customizable filter that lets you retain items in selected column that match a " - "custom requirement. This creates a new dataset.") + description = ProcessorDescription( + title="Filter by value", + tags=["filtering"], + description="Retain items whose value in a chosen column matches a requirement, such as equals, contains, is before or after a date, is greater or less than a number, or is in the top or bottom results. Match against one or several comma-separated values. This creates a new dataset.", + ) # top-level csv/ndjson datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) @@ -316,9 +317,11 @@ class ColumnProcessorFilter(ColumnFilter): Retain only posts where a given column matches a given value """ type = "column-processor-filter" # job type ID - category = "Filtering" # category - title = "Filter by value" # title displayed in UI - description = "A generic filter that checks whether a value in a selected column matches a custom requirement. " + description = ProcessorDescription( + title="Filter by value", + tags=["filtering"], + description="Retain items whose value in a chosen column matches a requirement, such as equals, contains, is before or after a date, is greater or less than a number, or is in the top or bottom results. Match against one or several comma-separated values. This creates a new dataset.", + ) # child (non-top-level) csv/ndjson datasets compatibility = Compatibility(child_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/filtering/date_filter.py b/processors/filtering/date_filter.py index b82e31ff6..6befff295 100644 --- a/processors/filtering/date_filter.py +++ b/processors/filtering/date_filter.py @@ -5,6 +5,7 @@ from dateutil.parser import ParserError from datetime import datetime +from backend.lib.processor import ProcessorDescription from processors.filtering.base_filter import BaseFilter from common.lib.compatibility import Compatibility from common.lib.helpers import UserInput @@ -21,9 +22,14 @@ class DateFilter(BaseFilter): Retain only posts between specific dates """ type = "date-filter" # job type ID - category = "Filtering" # category - title = "Filter by date" # title displayed in UI - description = "Retains posts between given dates. This creates a new dataset." + description = ProcessorDescription( + title="Filter by date", + tags=["filtering", "time series"], + description="Retain items whose timestamp falls between a given start and end date. Choose which column holds the date; values can be Unix timestamps or 'YYYY-MM-DD HH:MM:SS' strings. This creates a new dataset.", + info=[ + "Items with a missing or unreadable date are skipped and counted as invalid.", + ], + ) # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/filtering/lexical_filter.py b/processors/filtering/lexical_filter.py index 4c0129f65..ec3d7fb1c 100644 --- a/processors/filtering/lexical_filter.py +++ b/processors/filtering/lexical_filter.py @@ -4,6 +4,7 @@ import re from processors.filtering.base_filter import BaseFilter +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility from common.lib.helpers import UserInput @@ -18,18 +19,23 @@ class LexicalFilter(BaseFilter): Retain only posts matching a given lexicon """ type = "lexical-filter" # job type ID - category = "Filtering" # category - title = "Filter by words or phrases" # title displayed in UI - description = "Retains posts that contain selected words or phrases, including preset word lists. " \ - "This creates a new dataset." # description displayed in UI + description = ProcessorDescription( + title="Filter by words or phrases", + tags=["filtering"], + description=("Retain only items whose text contains one of the given words or phrases. Accepts a custom " + "comma-separated list, built-in word lists, and regular expressions."), + references=[ + "[Regex101](https://regex101.com/)", + ], + warnings=[ + ("With the regular-expression option the word list is read as a single Python regular expression, so a " + "malformed expression may match nothing."), + ], + ) # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) - references = [ - "[Regex101](https://regex101.com/)" - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/filtering/random_filter.py b/processors/filtering/random_filter.py index 6f13f88d1..660518d1f 100644 --- a/processors/filtering/random_filter.py +++ b/processors/filtering/random_filter.py @@ -3,7 +3,7 @@ """ import random -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from processors.filtering.base_filter import BaseFilter from common.lib.compatibility import Compatibility from common.lib.helpers import UserInput @@ -20,9 +20,11 @@ class RandomFilter(BaseFilter): Retain a pseudo-random amount of posts """ type = "random-filter" # job type ID - category = "Filtering" # category - title = "Random sample" # title displayed in UI - description = "Retain a random sample of items from the dataset. Creates a new dataset containing the sampled items." # description displayed in UI + description = ProcessorDescription( + title="Create random sample", + tags=["filtering"], + description="Retain a pseudo-random sample of a chosen number of items from the dataset. This creates a new dataset containing the sampled items.", + ) # Allow on top-level CSV/NDJSON/ZIP datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson", "zip"}) @@ -127,9 +129,11 @@ class RandomProcessorFilter(RandomFilter): Retain only posts where a given column matches a given value """ type = "random-processor-filter" # job type ID - category = "Filtering" # category - title = "Random sample" # title displayed in UI - description = "Retain a random sample of items from the dataset. Creates a new dataset containing the sampled items." + description = ProcessorDescription( + title="Random sample", + tags=["filtering", "sampling"], + description="Retain a pseudo-random sample of a chosen number of items from the dataset. This creates a new dataset containing the sampled items.", + ) # child (non-top-level) csv/ndjson/zip datasets compatibility = Compatibility(child_only=True, extensions={"csv", "ndjson", "zip"}) diff --git a/processors/filtering/tiktok_refresh.py b/processors/filtering/tiktok_refresh.py index a4cb52458..25b67b8b2 100644 --- a/processors/filtering/tiktok_refresh.py +++ b/processors/filtering/tiktok_refresh.py @@ -5,8 +5,9 @@ import json from datasources.tiktok_urls.search_tiktok_urls import TikTokScraper -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Output, PASSTHROUGH __author__ = "Dale Wahl" @@ -17,10 +18,20 @@ class UpdateTikTok(BasicProcessor): type = "tiktok-update-filter" # job type ID - category = "Filtering" # category - title = "Recollect TikTok data" # title displayed in UI - description = "Re-query TikTok URLs to update the dataset, e.g. to refresh video URLs or like counts." + description = ProcessorDescription( + title="Recollect TikTok data", + tags=["filtering", "override", "external service"], + description="Re-query the TikTok URLs in the dataset to refresh their metadata, such as video URLs or like counts.", + warnings=[ + "This re-fetches every item from TikTok, so it can be slow on large datasets.", + ], + icon="brand-tiktok", + ) extension = "ndjson" + # re-collects the parent's TikTok data as ndjson; made standalone and adopts the + # collector's type, so its position and collector-ness are unknown here + output = Output(extension="ndjson", columns=PASSTHROUGH, position=None, collector=None, + datasource=PASSTHROUGH) # Allow processor on TikTok datasets compatibility = Compatibility(types={"tiktok-search", "tiktok-urls-search"}) diff --git a/processors/filtering/unique_filter.py b/processors/filtering/unique_filter.py index 8a82dfc47..080dbca58 100644 --- a/processors/filtering/unique_filter.py +++ b/processors/filtering/unique_filter.py @@ -4,6 +4,7 @@ import json from processors.filtering.base_filter import BaseFilter +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility from common.lib.helpers import UserInput @@ -18,9 +19,14 @@ class UniqueFilter(BaseFilter): Retain only posts matching a given lexicon """ type = "unique-filter" # job type ID - category = "Filtering" # category - title = "Filter for unique items" # title displayed in UI - description = "Only keeps the first encounter of an item. This creates a new dataset." # description displayed in UI + description = ProcessorDescription( + title="Filter for unique items", + tags=["filtering"], + description="Keep only the first item for each unique combination of the selected columns, removing later duplicates. This creates a new dataset.", + info=[ + "Choose which columns define uniqueness, and whether to treat items as duplicates when all selected values match or when any single value matches.", + ], + ) # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/filtering/unique_images.py b/processors/filtering/unique_images.py index 8f5b448eb..266dd517c 100644 --- a/processors/filtering/unique_images.py +++ b/processors/filtering/unique_images.py @@ -4,10 +4,11 @@ import shutil import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput, hash_file from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -20,21 +21,27 @@ class UniqueImageFilter(BasicProcessor): Retain only unique images, by a user-defined metric """ type = "image-downloader-unique" # job type ID - category = "Visualisation" # category - title = "Filter for unique images" # title displayed in UI - description = "Only keeps one instance per image using various detection methods." # description displayed in UI + description = ProcessorDescription( + title="Filter for unique images", + tags=["filtering", "visual"], + description="Keep only one copy of each image, detecting duplicates with an exact file hash or with a perceptual, colour, average, or difference hash. The remaining images are saved as a new image archive.", + info=[ + "The file hash only matches byte-for-byte identical files; the perceptual, colour, average, and difference hashes also match visually similar images such as crops or re-saves.", + ], + references=[ + "[Imagehash library](https://github.com/JohannesBuchner/imagehash?tab=readme-ov-file)", + "Explainer: [Average hash](https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html)", + "Explainer: [Perceptual hashing](https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html)", + "Explainer: [Difference hash](https://www.hackerfactor.com/blog/index.php?/archives/529-Kind-of-Like-That.html)", + ], + ) extension = "zip" + media_type = "image" # the retained files are images; set so the map and runtime agree + # a zip archive of image files + output = MediaArchive(media="image") # image datasets: image archives, image-downloader output, or extracted video frames - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}) - - references = [ - "[Imagehash library](https://github.com/JohannesBuchner/imagehash?tab=readme-ov-file)", - "Explainer: [Average hash](https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html)", - "Explainer: [Perceptual hashing](https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html)", - "Explainer: [Difference hash](https://www.hackerfactor.com/blog/index.php?/archives/529-Kind-of-Like-That.html)", - - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: diff --git a/processors/machine_learning/audio_to_text.py b/processors/machine_learning/audio_to_text.py index ed4078ba0..70785bbca 100644 --- a/processors/machine_learning/audio_to_text.py +++ b/processors/machine_learning/audio_to_text.py @@ -6,8 +6,9 @@ import openai from requests.exceptions import ConnectionError -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput @@ -24,23 +25,30 @@ class AudioToText(BasicProcessor): Convert audio to text with Whisper / GPT models, locally or through the OpenAI API """ type = "audio-to-text" # job type ID - category = "Audio" # category - title = "Audio to text" # title displayed in UI - description = ("Detect speech and other sounds in audio and convert to text with either OpenAI's Whisper or " - " GPT models (GPT only via API).") # description displayed in UI + description = ProcessorDescription( + title="Transcribe audio to text", + tags=["audio", "machine learning", "transcribe", "external service"], + description="Transcribe speech in audio files to text with OpenAI's Whisper or GPT models. Run Whisper locally through the DMI Service Manager, or send the audio to the OpenAI API, which can also translate to English or separate speakers.", + warnings=[ + "Using an OpenAI model sends your audio to OpenAI, a paid external service that bills the owner of the API key.", + "The local Whisper models run through the DMI Service Manager and need a GPU to run at a reasonable speed.", + ], + references=[ + "[OpenAI Whisper blog](https://openai.com/research/whisper)", + "[OpenAI speech to text](https://github.com/openai/whisper/blob/248b6cb124225dd263bb9bd32d060b6517e067f8/whisper" + "/transcribe.py#LL374C3-L374C3)", + "[Whisper paper: Robust Speech Recognition via Large-Scale Weak Supervision](https://arxiv.org/abs/2212.04356)", + "[OpenAI Whisper statistics & code](https://github.com/openai/whisper#whisper)", + "[How to use prompts](https://platform.openai.com/docs/guides/speech-to-text/prompting)", + ], + icon="closed-captioning", + ) extension = "ndjson" # extension of result file, used internally and in UI + # a derived table + output = Table(extension="ndjson") # Allow on audio datasets - compatibility = Compatibility(media_types={"audio"}, type_prefixes={"audio-extractor"}) - - references = [ - "[OpenAI Whisper blog](https://openai.com/research/whisper)", - "[OpenAI speech to text](https://github.com/openai/whisper/blob/248b6cb124225dd263bb9bd32d060b6517e067f8/whisper" - "/transcribe.py#LL374C3-L374C3)", - "[Whisper paper: Robust Speech Recognition via Large-Scale Weak Supervision](https://arxiv.org/abs/2212.04356)", - "[OpenAI Whisper statistics & code](https://github.com/openai/whisper#whisper)", - "[How to use prompts](https://platform.openai.com/docs/guides/speech-to-text/prompting)", - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"audio"}, type_prefixes={"audio-extractor"}) config = { "dmi-service-manager.bb_whisper-intro-1": { diff --git a/processors/machine_learning/blip2_image_caption.py b/processors/machine_learning/blip2_image_caption.py index 109d811d2..cd0c45b06 100644 --- a/processors/machine_learning/blip2_image_caption.py +++ b/processors/machine_learning/blip2_image_caption.py @@ -4,12 +4,13 @@ import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory, DsmConnectionError from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -22,19 +23,26 @@ class CategorizeImagesCLIP(BasicProcessor): Caption Images with OpenAI BLIP2 """ type = "image-captions" # job type ID - category = "Visual" # category - title = "Generate image captions using OpenAI's BLIP2 model" # title displayed in UI - description = "The BLIP2 model uses a pretrained image encoder combined with an LLM to generate image captions. The model can also be prompted and uses the image plus prompt to generate text responses." # description displayed in UI + description = ProcessorDescription( + title="Caption images with BLIP-2", + tags=["visual", "classification", "transcribe", "external service"], + description="Generate a caption for each image with the BLIP-2 model, which combines an image encoder with a language model. You can also supply a prompt to get a text response about each image instead of a plain caption.", + warnings=[ + "This runs the BLIP-2 model through the DMI Service Manager, which must be set up with a GPU by an administrator.", + ], + references=[ + "[OpenAI CLIP blog](https://openai.com/research/clip)", + "[BLIP-2 paper: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597)", + "[BLIP-2 documentation](https://huggingface.co/docs/transformers/main/model_doc/blip-2)", + ], + icon="eye", + ) extension = "ndjson" # extension of result file, used internally and in UI + # a derived table + output = Table(extension="ndjson") # image datasets (image archives or image-downloader output), when BLIP2 is enabled - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.fc_blip2_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=["image-text-wall"]) - - references = [ - "[OpenAI CLIP blog](https://openai.com/research/clip)", - "[BLIP-2 paper: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597)", - "[BLIP-2 documentation](https://huggingface.co/docs/transformers/main/model_doc/blip-2)", - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.fc_blip2_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=["image-text-wall"]) config = { "dmi-service-manager.fb_blip2-intro-1": { diff --git a/processors/machine_learning/clarifai_api.py b/processors/machine_learning/clarifai_api.py index 1b6dfaab1..62e5a027f 100644 --- a/processors/machine_learning/clarifai_api.py +++ b/processors/machine_learning/clarifai_api.py @@ -9,8 +9,9 @@ from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel from common.lib.helpers import UserInput, convert_to_int -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -27,21 +28,29 @@ class ClarifaiAPIFetcher(BasicProcessor): Request tags and labels from the Clarifai API for a given set of images """ type = "clarifai-api" # job type ID - category = "Machine learning" # category - title = "Clarifai analysis" # title displayed in UI - description = "Use the Clarifai API to annotate images with tags and labels identified via machine learning. " \ - "One request will be made per image per annotation type. Note that this is NOT a free service and " \ - "requests will be credited by Clarifai to the owner of the API token you provide." # description displayed in UI + description = ProcessorDescription( + title="Label images with Clarifai", + tags=["machine learning", "visual", "classification", "external service"], + description="Use the Clarifai API to tag and label images with machine learning. One request is made per " + "image for each model you select.", + references=[ + "[Clarifai](https://www.clarifai.com/)", + "[Clarifai API pricing and free usage limits](https://www.clarifai.com/pricing)", + "[Clarifai model browser](https://clarifai.com/clarifai/main/models)", + ], + warnings=[ + "This is a paid service. Clarifai bills the owner of the API key you provide, and every image and every " + "selected model adds to the number of requests made.", + "Images are sent to Clarifai, an external service, to be analysed.", + ], + icon="eye", + ) extension = "ndjson" # extension of result file, used internally and in UI + # a derived table + output = Table(extension="ndjson") # Allow on image sets - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}, preferred_followups=["convert-clarifai-vision-to-csv", "clarifai-bipartite-network"]) - - references = [ - "[Clarifai](https://www.clarifai.com/)", - "[Clarifai API Pricing & Free Usage Limits](https://www.clarifai.com/pricing)", - "[Clarifai model browser](https://clarifai.com/clarifai/main/models)" - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}, preferred_followups=["convert-clarifai-vision-to-csv", "clarifai-bipartite-network"]) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: diff --git a/processors/machine_learning/clip_categorize_images.py b/processors/machine_learning/clip_categorize_images.py index 99aa084d1..ea2a6af55 100644 --- a/processors/machine_learning/clip_categorize_images.py +++ b/processors/machine_learning/clip_categorize_images.py @@ -5,12 +5,13 @@ import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory, DsmConnectionError from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -23,21 +24,31 @@ class CategorizeImagesCLIP(BasicProcessor): Categorize Images with OpenAI CLIP """ type = "image-to-categories" # job type ID - category = "Visual" # category - title = "Categorize images with CLIP" # title displayed in UI - description = ("Provide a list of categories and classify images with OpenAI's CLIP models. This will estimate " - "the likelihood an image belongs to a category (total of all category values will be 100%).") # description displayed in UI + description = ProcessorDescription( + title="Categorize images with CLIP", + tags=["visual", "classification", "external service"], + description="Classify images into your own list of categories with OpenAI's CLIP model. For each image it estimates the likelihood of each category, with the values across all categories adding up to 100%.", + warnings=[ + "This runs the CLIP model through the DMI Service Manager, which must be set up with a GPU by an administrator.", + ], + info=[ + "Categories can be plain words or phrases, including proper nouns and contrasts. Unique categories may work better such as 'animal' versus 'object' than 'animal' versus 'not animal'.", + ], + references=[ + "[OpenAI CLIP blog](https://openai.com/research/clip)", + "[CLIP paper: Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/pdf/2103.00020.pdf)", + "[OpenAI CLIP code](https://github.com/openai/CLIP/#clip)", + "[Model comparison](https://arxiv.org/pdf/2103.00020.pdf#page=40&zoom=auto,-457,754)", + ], + icon="eye", + ) extension = "ndjson" # extension of result file, used internally and in UI + # a derived table + output = Table(extension="ndjson") + # image datasets (image archives or image-downloader output), when CLIP is enabled - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.cc_clip_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=["image-category-wall"]) - - references = [ - "[OpenAI CLIP blog](https://openai.com/research/clip)", - "[CLIP paper: Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/pdf/2103.00020.pdf)", - "[OpenAI CLIP code](https://github.com/openai/CLIP/#clip)", - "[Model comparison](https://arxiv.org/pdf/2103.00020.pdf#page=40&zoom=auto,-457,754)", - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.cc_clip_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=["image-category-wall"]) config = { "dmi-service-manager.cb_clip-intro-1": { diff --git a/processors/machine_learning/generate_images.py b/processors/machine_learning/generate_images.py index 273d0835c..fcdcb36d2 100644 --- a/processors/machine_learning/generate_images.py +++ b/processors/machine_learning/generate_images.py @@ -7,11 +7,12 @@ import json import re -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from processors.visualisation.download_images import ImageDownloader from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory from common.lib.user_input import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -24,19 +25,28 @@ class StableDiffusionImageGenerator(BasicProcessor): Generate images with Stable Diffusion """ type = "image-downloader-stable-diffusion" # job type ID - category = "Visual" # category - title = "Generate images from text prompts" # title displayed in UI - description = "Given a list of prompts, generates images using the Stable Diffusion XL image model." # description displayed in UI + description = ProcessorDescription( + title="Generate images from text prompts", + tags=["visual", "machine learning", "genAI", "external service"], + description="Generate images from a column of text prompts using the Stable Diffusion XL model. Each prompt produces one image, with an optional negative prompt to steer the model away from unwanted content.", + references=[ + "[Stable Diffusion XL 1.0 model card](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)", + ], + warnings=[ + "Images are generated by a self-hosted DMI Service Manager, which must have a Stable Diffusion image built and a GPU available.", + "Prompts are truncated to 70 characters before generation.", + ], + icon="images", + ) extension = "zip" # extension of result file, used internally and in UI + media_type = "image" # the generated files are images; set so the map and runtime agree + # a zip archive of image files + output = MediaArchive(media="image") # coarse map spec; is_compatible_with (below) is the runtime truth -- it also requires the # dataset to have columns (a prompt source), which can't be declared statically compatibility = Compatibility(required_settings={"dmi-service-manager.sd_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=ImageDownloader.followups) - references = [ - "[Stable Diffusion XL 1.0 model card](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)" - ] - config = { "dmi-service-manager.sd_intro-1": { "type": UserInput.OPTION_INFO, diff --git a/processors/machine_learning/google_vision_api.py b/processors/machine_learning/google_vision_api.py index 73cae0d2a..42275eaee 100644 --- a/processors/machine_learning/google_vision_api.py +++ b/processors/machine_learning/google_vision_api.py @@ -9,8 +9,9 @@ from pathlib import Path from common.lib.helpers import UserInput, convert_to_int -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException __author__ = "Stijn Peeters" @@ -28,21 +29,27 @@ class GoogleVisionAPIFetcher(BasicProcessor): Request tags and labels from the Google Vision API for a given set of images """ type = "google-vision-api" # job type ID - category = "Machine learning" # category - title = "Google Vision analysis" # title displayed in UI - description = "Use the Google Vision API to annotate images with tags and labels identified via machine learning. " \ - "One request will be made per image per annotation type. Note that this is not a free service and " \ - "requests will be credited by Google to the owner of the API token you provide. Requires billing " \ - "and Google Vision API enabled (this may take a few minutes)." # description displayed in UI + description = ProcessorDescription( + title="Label images with Google Vision", + tags=["visual", "classification", "machine learning", "external service"], + description="Use the Google Vision API to detect labels, text, faces, landmarks, logos, and other features in images. One request is made per image for each feature type you select.", + references=[ + "[Google Vision API Documentation](https://cloud.google.com/vision/docs)", + "[Google Vision API Pricing & Free Usage Limits](https://cloud.google.com/vision/pricing)", + ], + warnings=[ + "This is a paid service. Google bills the owner of the API key you provide, which requires billing and the Vision API to be enabled.", + "Images are sent to Google, an external service, to be analysed.", + ], + icon="eye", + ) extension = "ndjson" # extension of result file, used internally and in UI - # Allow on image sets - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}, preferred_followups=["convert-google-vision-to-csv", "vision-bipartite-network", "vision-label-network"]) + # a derived table + output = Table(extension="ndjson") - references = [ - "[Google Vision API Documentation](https://cloud.google.com/vision/docs)", - "[Google Vision API Pricing & Free Usage Limits](https://cloud.google.com/vision/pricing)" - ] + # Allow on image sets + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, types={"video-frames"}, preferred_followups=["convert-google-vision-to-csv", "vision-bipartite-network", "vision-label-network"]) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: diff --git a/processors/machine_learning/llm_prompter.py b/processors/machine_learning/llm_prompter.py index f2332ae40..26b6978c2 100644 --- a/processors/machine_learning/llm_prompter.py +++ b/processors/machine_learning/llm_prompter.py @@ -19,33 +19,41 @@ from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException, QueryNeedsExplicitConfirmationException from common.lib.helpers import UserInput, nthify, andify, remove_nuls, flatten_dict from common.lib.llm.adapter import LLMAdapter -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table class LLMPrompter(BasicProcessor): """ Prompt various LLMs, locally or through APIs """ type = "llm-prompter" # job type ID - category = "Machine learning" # category - title = "LLM prompting" # title displayed in UI - description = ("Use LLMs to analyze a dataset per item, via APIs or locally. Suitable for a wide arrange of tasks like " - "classification, entity extraction, or OCR. Supported APIs include OpenAI, Google, Anthropic, " - "Mistral, and DeepSeek.") + description = ProcessorDescription( + title="Prompt a large language model", + tags=["text analysis", "visual", "audio", "classification", "genAI", "annotations"], + description="Run a prompt against a large language model for each item in a dataset, using a local model or a third-party API such as OpenAI, Google, Anthropic, Mistral, or DeepSeek. Insert column values into the prompt with brackets, attach images or other media, and optionally return structured JSON.", + references=[ + "[Törnberg, Petter. 2023. 'How to Use LLMs for Text Analysis.' arXiv:2307.13106.](https://arxiv.org/pdf/2307.13106)", + "[Karjus, Andres. 2023. 'Machine-assisted mixed methods: augmenting humanities and social sciences with artificial intelligence.' arXiv preprint arXiv:2309.14379.](https://arxiv.org/abs/2309.14379)", + ], + warnings=[ + "Third-party API models send your data to an external provider and usually incur costs; consider anonymising your data or using a local model instead.", + "Test your prompt on a small sample first, as results depend heavily on the prompt and the chosen model.", + ], + info=[ + "Batching several items per prompt can be faster but may reduce accuracy and needs a model that supports structured output.", + ], + icon="robot", + ) extension = "ndjson" # extension of result file, used internally and in UI. In this case it's variable! + # a derived table + output = Table(extension="ndjson") + # coarse map spec; is_compatible_with (below) is the runtime truth -- it accepts csv/ndjson # tables, OR zip archives of image/video/audio media (_almost_ all zips but not) compatibility = Compatibility(extensions={"csv", "ndjson", "zip"}) - references = [ - "[Törnberg, Petter. 2023. 'How to Use LLMs for Text Analysis.' arXiv:2307.13106.](https://arxiv.org/pdf/2307." - "13106)", - "[Karjus, Andres. 2023. 'Machine-assisted mixed methods: augmenting humanities and social sciences " - "with artificial intelligence.' arXiv preprint arXiv:2309.14379.]" - "(https://arxiv.org/abs/2309.14379)" - ] - @classmethod def get_queue_id(cls, remote_id, details, dataset) -> str: """ diff --git a/processors/machine_learning/perspective.py b/processors/machine_learning/perspective.py index 4b88035d3..aa821ecd7 100644 --- a/processors/machine_learning/perspective.py +++ b/processors/machine_learning/perspective.py @@ -6,32 +6,43 @@ from googleapiclient.errors import HttpError from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from googleapiclient import discovery from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility +from common.lib.outputs import Table class Perspective(BasicProcessor): """ Score items with toxicity and other scores through Google Jigsaw's Perspective API. """ type = "perspective" # job type ID - category = "Machine learning" # category - title = "Toxicity scores" # title displayed in UI - description = ("Use the Perspective API to score text with attributes on toxicity, " - "including 'toxicity', 'insult', and 'profanity'.") # description displayed in UI + description = ProcessorDescription( + title="Score text toxicity with Perspective", + tags=["machine learning", "classification", "external service", "annotation"], + description="Use Google Jigsaw's Perspective API to score the 'body' text of each item on attributes such as toxicity, severe toxicity, identity attack, insult, profanity, and threat. Each attribute is returned as a value between 0 and 1.", + references=[ + "[Perspective API documentation](https://developers.perspectiveapi.com/s/about-the-api)", + "[Rieder, Bernhard, and Yarden Skop. 2021. 'The fabrics of machine moderation: Studying the technical, " + "normative, and organizational structure of Perspective API.' Big Data & Society, 8(2).]" + "(https://doi.org/10.1177/20539517211046181)", + ], + warnings=[ + "Text is sent to Google's Perspective API, an external service, to be scored.", + ], + info=[ + "Enable 'toxicity scores' annotations to write the attribute scores back to the parent dataset.", + ], + icon="hand-middle-finger", + ) extension = "ndjson" # extension of result file, used internally and in UI + # a derived table + output = Table(extension="ndjson") + # top-level text datasets (scores text columns via the Perspective API) compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) - references = [ - "[Perspective API documentation](https://developers.perspectiveapi.com/s/about-the-api)", - "[Rieder, Bernhard, and Yarden Skop. 2021. 'The fabrics of machine moderation: Studying the technical, " - "normative, and organizational structure of Perspective API.' Big Data & Society, 8(2).]" - "(https://doi.org/10.1177/20539517211046181)" - ] - config = { "api.google.api_key": { "type": UserInput.OPTION_TEXT, diff --git a/processors/machine_learning/pix-plot.py b/processors/machine_learning/pix-plot.py index 7d0a6b991..a276aefd6 100644 --- a/processors/machine_learning/pix-plot.py +++ b/processors/machine_learning/pix-plot.py @@ -12,8 +12,9 @@ from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException from common.lib.helpers import UserInput, ellipsiate -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import File __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -28,19 +29,26 @@ class PixPlotGenerator(BasicProcessor): Create an PixPlot from the downloaded images in the dataset """ type = "pix-plot" # job type ID - category = "Visual" # category - title = "Create PixPlot visualisation" # title displayed in UI - description = "Put all images from an archive into a PixPlot visualisation: an explorable map of images " \ - "algorithmically grouped by similarity." + description = ProcessorDescription( + title="Create PixPlot visualisation", + tags=["visual", "external service"], + description="Arrange the images in an archive into a PixPlot: an explorable map where images are grouped by visual similarity. Use the neighbours and minimum distance options to control how tightly images cluster.", + references=[ + "[PixPlot](https://pixplot.io/)", + "[Parameter documentation](https://pixplot.io/docs/api/parameters.html)", + ], + warnings=[ + "The visualisation is built by a self-hosted DMI Service Manager, which must have PixPlot enabled.", + "At least 12 images are needed, and large numbers of images can make the processor run for a long time.", + ], + icon="images", + ) extension = "html" # extension of result file, used internally and in UI + # a single html file + output = File("html") # image datasets (image archives or image-downloader output), when PixPlot is enabled - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.db_pixplot_enabled", "dmi-service-manager.ab_server_address"}) - - references = [ - "[PixPlot](https://pixplot.io/)", - "[Parameter documentation](https://pixplot.io/docs/api/parameters.html)" - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.db_pixplot_enabled", "dmi-service-manager.ab_server_address"}) # PixPlot requires a minimum number of photos to be created # This is currently due to the clustering algorithm which creates 12 clusters diff --git a/processors/machine_learning/text_from_image.py b/processors/machine_learning/text_from_image.py index 0edce68a9..df6a27d50 100644 --- a/processors/machine_learning/text_from_image.py +++ b/processors/machine_learning/text_from_image.py @@ -10,10 +10,11 @@ from common.lib.dmi_service_manager import DmiServiceManager, DsmOutOfMemory, DmiServiceManagerException, DsmConnectionError from common.lib.helpers import UserInput, hash_to_md5 -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.exceptions import ProcessorInterruptedException from common.lib.item_mapping import MappedItem from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -25,29 +26,25 @@ class ImageTextDetector(BasicProcessor): Send images to DMI OCR server for OCR analysis """ type = "text-from-images" # job type ID - category = "Conversion" # category - title = "Extract text from images" # title displayed in UI - description = """ - Uses optical character recognition (OCR) to extract text from images via machine learning. - - This processor first detects areas of an image that may contain text with the pretrained - Character-Region Awareness For Text (CRAFT) detection model and then attempts to predict the - text inside each area using Keras' implementation of a Convolutional Recurrent Neural - Network (CRNN) for text recognition. Once words are predicted, an algorithm attempts to - sort them into likely groupings based on locations within the original image. - """ + description = ProcessorDescription( + title="Extract text from images", + tags=["machine learning", "visual", "transcribe", "external service"], + description="Use optical character recognition (OCR) to detect and read text in images. Detected words are grouped by their position in the image and returned per image.", + references=[ + "[DMI OCR Server](https://github.com/digitalmethodsinitiative/ocr_server#readme)", + "[Paddle OCR model](https://github.com/PaddlePaddle/PaddleOCR#readme)", + ], + warnings=[ + "Images are sent to a self-hosted DMI Service Manager running the OCR server, which must be enabled.", + ], + icon="language", + ) extension = "ndjson" # extension of result file, used internally and in UI + # a derived table + output = Table(extension="ndjson") # image datasets (image archives or image-downloader output), when the OCR server is enabled - compatibility = Compatibility(media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.eb_ocr_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=["image-text-wall"]) - - references = [ - "[DMI OCR Server](https://github.com/digitalmethodsinitiative/ocr_server#readme)", - "[Paddle OCR model](https://github.com/PaddlePaddle/PaddleOCR#readme)" - #"[Keras OCR model]( https://keras-ocr.readthedocs.io/en/latest/)", - #"[CRAFT text detection model](https://github.com/clovaai/CRAFT-pytorch)", - #"[Keras CRNN text recognition model](https://github.com/kurapan/CRNN)" - ] + compatibility = Compatibility(extensions={"zip"}, media_types={"image"}, type_prefixes={"image-downloader"}, required_settings={"dmi-service-manager.eb_ocr_enabled", "dmi-service-manager.ab_server_address"}, preferred_followups=["image-text-wall"]) config = { "dmi-service-manager.ea_ocr-intro-1": { diff --git a/processors/metrics/annotation_metadata.py b/processors/metrics/annotation_metadata.py index 21fb091ba..cc21e2e19 100644 --- a/processors/metrics/annotation_metadata.py +++ b/processors/metrics/annotation_metadata.py @@ -2,8 +2,9 @@ Retrieves metadata on annotations for this dataset. """ -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from datetime import datetime @@ -12,11 +13,18 @@ class AnnotationMetadata(BasicProcessor): Download annotation metadata from this dataset """ type = "annotation-metadata" # job type ID - category = "Conversion" # category - title = "Export annotations" # title displayed in UI - description = ("Download annotations and their metadata for this dataset. " - "Includes annotation author, timestamp, type, etc.") # description displayed in UI + description = ProcessorDescription( + title="Export annotations", + tags=["conversion", "annotation", "metadata"], + description="Export the annotations made on this dataset along with their metadata, such as the annotation author, timestamp, and type.", + info=[ + "Only datasets that have annotations can be processed.", + ], + icon="circle-info", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # coarse map spec (accepts any dataset); is_compatible_with (below) is the runtime # truth -- it requires the dataset to actually have annotations (annotation_fields) diff --git a/processors/metrics/count_posts.py b/processors/metrics/count_posts.py index e30cabe1c..05b96fc37 100644 --- a/processors/metrics/count_posts.py +++ b/processors/metrics/count_posts.py @@ -3,8 +3,9 @@ """ from common.lib.helpers import UserInput, pad_interval, get_interval_descriptor -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -17,10 +18,18 @@ class CountPosts(BasicProcessor): """ type = "count-posts" # job type ID - category = "Metrics" # category - title = "Count items per date" # title displayed in UI - description = "Counts how many items are in the dataset per date (or overall)." # description displayed in UI + description = ProcessorDescription( + title="Count items per date", + tags=["counting", "time series"], + description="Count how many items are in the dataset, grouped by date or counted overall.", + info=[ + "Enable 'Include dates with zero items' to keep the timeline continuous when some dates have no data.", + ], + icon="list-ol", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}, preferred_followups=["histogram"]) diff --git a/processors/metrics/debate_metrics.py b/processors/metrics/debate_metrics.py index d784fb848..81dfff191 100644 --- a/processors/metrics/debate_metrics.py +++ b/processors/metrics/debate_metrics.py @@ -4,8 +4,9 @@ import datetime import time -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Sal Hagen" __credits__ = ["Sal Hagen"] @@ -26,11 +27,17 @@ class DebateMetrics(BasicProcessor): """ type = "debate_metrics" # job type ID - category = "Thread metrics" # category - title = "Debate metrics" # title displayed in UI - description = "Returns a csv with meta-metrics per thread." # description displayed in UI + description = ProcessorDescription( + title="Get debate metrics", + tags=["counting", "metadata"], + description="Calculate debate metrics for each thread in the dataset, such as the number of posts, the number of images, and the length of the opening post.", + icon="circle-info", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # chan datasets (thread-level debate metrics) compatibility = Compatibility(datasources={"fourchan", "eightchan", "eightkun"}) diff --git a/processors/metrics/group_hashes.py b/processors/metrics/group_hashes.py index 7712aa4fa..b48eb6eb2 100644 --- a/processors/metrics/group_hashes.py +++ b/processors/metrics/group_hashes.py @@ -2,8 +2,9 @@ import json import imagehash -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput, normalize_crhash_components @@ -17,10 +18,15 @@ class HashGrouper(BasicProcessor): Group hashes """ type = "image-hash-grouper" # job type ID - category = "Conversion" # category - title = "Group similar hashes" # title displayed in UI - description = "Calculate groups of similar hashes from a CSV file." # description displayed in UI + description = ProcessorDescription( + title="Group similar hashes", + tags=["counting", "visual", "preprocessing"], + description="Group image hashes into clusters of visually similar images, based on a similarity threshold you set. Runs on the output of the image hasher and rewrites the 'group' column with the new clusters.", + icon="hashtag", + ) extension = "csv" + # a derived table + output = Table() # Allow processor on image-hasher output (could also work on any CSV with the right fields) compatibility = Compatibility(types={"image-hasher"}) diff --git a/processors/metrics/most_quoted.py b/processors/metrics/most_quoted.py index 6bb49fe4b..1599a2ff7 100644 --- a/processors/metrics/most_quoted.py +++ b/processors/metrics/most_quoted.py @@ -4,8 +4,9 @@ import csv import re -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -19,10 +20,15 @@ class QuoteRanker(BasicProcessor): Rank posts by most-quoted """ type = "quote-ranker" # job type ID - category = "Metrics" # category - title = "Sort by most replied-to" # title displayed in UI - description = "Sort posts by how often they were replied to by other posts in the dataset." # description displayed in UI + description = ProcessorDescription( + title="Sort by most replied-to", + tags=["counting"], + description="Sort posts by how often they were replied to by other posts in the dataset, adding a column with the number of replies each post received.", + icon="comments", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # chan datasets (posts reply to / quote each other) compatibility = Compatibility(datasources={"fourchan", "eightchan", "eightkun"}) diff --git a/processors/metrics/rank_attribute.py b/processors/metrics/rank_attribute.py index a47c350dd..d7840abab 100644 --- a/processors/metrics/rank_attribute.py +++ b/processors/metrics/rank_attribute.py @@ -7,8 +7,9 @@ from collections import OrderedDict from itertools import islice, chain -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.helpers import UserInput, convert_to_int, get_interval_descriptor __author__ = "Stijn Peeters" @@ -26,16 +27,22 @@ class AttributeRanker(BasicProcessor): most-occurring country codes per month; overall top host names, etc """ type = "attribute-frequencies" # job type ID - category = "Metrics" # category - title = "Count values" # title displayed in UI - description = "Count values in a dataset column, like URLs or hashtags (overall or per timeframe)" # description displayed in UI + description = ProcessorDescription( + title="Count values", + tags=["counting", "time series"], + description="Count how often values occur in one or more dataset columns, overall or per timeframe. Optionally extract URLs, domain names, hashtags, or emoji from the column before counting, and filter values with a regular expression.", + references=[ + "[regex010](https://regex101.com/)", + ], + icon="list-ol", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) - references = ["[regex010](https://regex101.com/)"] - include_missing_data = True @classmethod diff --git a/processors/metrics/thread_metadata.py b/processors/metrics/thread_metadata.py index 86b487d5a..7fde0be69 100644 --- a/processors/metrics/thread_metadata.py +++ b/processors/metrics/thread_metadata.py @@ -4,8 +4,9 @@ import datetime import math -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Sal Hagen" __credits__ = ["Sal Hagen"] @@ -18,14 +19,20 @@ class ThreadMetadata(BasicProcessor): """ type = "thread-metadata" # job type ID - category = "Metrics" # category - title = "Thread metadata" # title displayed in UI - description = ( - "Extract various metadata on the threads in the dataset, including time data and post counts. Note " - "that this extracted only on the basis of the items present this dataset." - ) # description displayed in UI + description = ProcessorDescription( + title="Calculate thread metadata", + tags=["counting", "metadata"], + description="Extract metadata for each thread in the dataset, such as the first and last post timestamps, thread age, subject, author, and post and image counts.", + info=[ + "Metadata is derived only from the items present in the dataset, so incomplete threads yield partial figures.", + ], + icon="circle-info", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/metrics/top_images.py b/processors/metrics/top_images.py index a6d5a8d87..3aa19e2bc 100644 --- a/processors/metrics/top_images.py +++ b/processors/metrics/top_images.py @@ -4,9 +4,10 @@ import re from collections import Counter, OrderedDict -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -21,10 +22,15 @@ class TopImageCounter(BasicProcessor): Collects all images used in a data set, and sorts by most-used. """ type = "top-images" # job type ID - category = "Metrics" # category - title = "Rank image URLs" # title displayed in UI - description = "Collect all image URLs and sort by most-occurring." # description displayed in UI + description = ProcessorDescription( + title="Rank image URLs", + tags=["urls", "counting", "annotation"], + description="Extract all image URLs from the dataset and rank them by how often they occur. Optionally save the extracted URLs back to the source dataset as annotations.", + icon="arrow-up-1-9", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # top-level csv/ndjson datasets, except Telegram (which has its own image logic) compatibility = Compatibility(top_dataset_only=True, excluded_types={"telegram-search"}, extensions={"csv", "ndjson"}, preferred_followups=["image-downloader"]) diff --git a/processors/metrics/url_titles.py b/processors/metrics/url_titles.py index a50abec33..9eed4d514 100644 --- a/processors/metrics/url_titles.py +++ b/processors/metrics/url_titles.py @@ -3,8 +3,9 @@ """ import csv -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from backend.lib.proxied_requests import FailedProxiedRequest from common.lib.helpers import UserInput from common.lib.exceptions import ProcessorInterruptedException @@ -26,11 +27,18 @@ class URLFetcher(BasicProcessor): Retrieve HTML title (and other metadata) for URLs """ type = "url-metadata" # job type ID - category = "Metrics" # category - title = "Fetch URL metadata" # title displayed in UI - description = ("Fetches the page title and other metadata for URLs referenced in the dataset. Makes a request to " - "each URL, optionally following HTTP redirects.") # description displayed in UI + description = ProcessorDescription( + title="Fetch URL metadata", + tags=["urls", "counting", "metadata"], + description="Fetch the page title, final URL, domain name, and HTTP status for each URL referenced in the dataset. Make one request per URL, optionally following HTTP redirects.", + warnings=[ + "This visits every URL in the dataset live, which can be slow for large datasets." + ], + icon="globe", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/metrics/vocabulary_overtime.py b/processors/metrics/vocabulary_overtime.py index eae9b2712..4461a3c75 100644 --- a/processors/metrics/vocabulary_overtime.py +++ b/processors/metrics/vocabulary_overtime.py @@ -3,8 +3,9 @@ """ import re -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.helpers import UserInput, get_interval_descriptor __author__ = "Stijn Peeters" @@ -18,18 +19,22 @@ class OvertimeAnalysis(BasicProcessor): Show overall activity levels for Telegram datasets """ type = "overtime-vocabulary" # job type ID - category = "Metrics" # category - title = "Over-time word counts" # title displayed in UI - description = "Determines the counts over time of particular set of words or phrases." # description displayed in UI + description = ProcessorDescription( + title="Count words over time", + tags=["counting", "time series"], + description="Count how often a chosen set of words or phrases occurs in the dataset over time. Use the built-in OILab extreme speech lexicons or supply your own comma-separated word list, per year, month, week, or day.", + references=[ + "[\"Salvaging the Internet Hate Machine: Using the discourse of radical online subcultures to identify emergent extreme speech\" - Unblished paper detailing the OILab extreme speech lexigon](https://oilab.eu/texts/4CAT_Hate_Speech_WebSci_paper.pdf)", + ], + icon="chart-line", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) - references = [ - "[\"Salvaging the Internet Hate Machine: Using the discourse of radical online subcultures to identify emergent extreme speech\" - Unblished paper detailing the OILab extreme speech lexigon](https://oilab.eu/texts/4CAT_Hate_Speech_WebSci_paper.pdf)", - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/metrics/youtube_metadata.py b/processors/metrics/youtube_metadata.py index 6dacf0fe3..211d7fd78 100644 --- a/processors/metrics/youtube_metadata.py +++ b/processors/metrics/youtube_metadata.py @@ -9,9 +9,10 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Sal Hagen" __credits__ = ["Sal Hagen"] @@ -31,11 +32,25 @@ class YouTubeMetadata(BasicProcessor): """ type = "youtube-metadata" # job type ID - category = "Metrics" # category - title = "Fetch YouTube metadata from URLs" # title displayed in UI - description = ("Collect metadata from YouTube videos, channels, and playlists that are linked to in the dataset. " - "Uses the YouTube API.") # description displayed in UI + description = ProcessorDescription( + title="Fetch YouTube metadata", + tags=["API", "urls", "metadata", "external service"], + description="Collect metadata from YouTube videos, channels, and playlists linked to in the dataset, using the YouTube Data API. Return one row per link with details such as title, view count, and upload date, and optionally save these as annotations on the source dataset.", + references=[ + "[YouTube v3 API documentation](https://developers.google.com/youtube/v3)", + "[4chan’s YouTube: A Fringe Perspective on YouTube’s Great Purge of 2019 - OILab.eu](https://oilab.eu/4chans-youtube-a-fringe-perspective-on-youtubes-great-purge-of-2019/)" + ], + warnings=[ + "This requires a YouTube Data API key and is subject to that key's daily quota; large datasets may hit the limit before all links are retrieved.", + ], + info=[ + "Run 'Extract URLs' first if your YouTube links are inside text columns rather than a dedicated URL column.", + ], + icon="brand-youtube", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # collector output or extract-urls-filter output, as csv/ndjson (may contain youtube links) compatibility = Compatibility(is_collector=True, types={"extract-urls-filter"}, extensions={"csv", "ndjson"}, preferred_followups=["youtube-thumbnails"]) @@ -49,11 +64,6 @@ class YouTubeMetadata(BasicProcessor): client = None - references = [ - "[YouTube v3 API documentation](https://developers.google.com/youtube/v3)", - "[4chan’s YouTube: A Fringe Perspective on YouTube’s Great Purge of 2019 - OILab.eu](https://oilab.eu/4chans-youtube-a-fringe-perspective-on-youtubes-great-purge-of-2019/)" - ] - config = { "api.youtube.key": { "type": UserInput.OPTION_TEXT, diff --git a/processors/networks/clarifai_bipartite_network.py b/processors/networks/clarifai_bipartite_network.py index 35bbdcd2f..7585dc0a5 100644 --- a/processors/networks/clarifai_bipartite_network.py +++ b/processors/networks/clarifai_bipartite_network.py @@ -1,8 +1,9 @@ """ Google Vision API co-label network """ -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.helpers import UserInput __author__ = "Stijn Peeters" @@ -18,12 +19,15 @@ class VisionTagBiPartiteNetworker(BasicProcessor): Google Vision API co-label network """ type = "clarifai-bipartite-network" # job type ID - category = "Networks" # category - title = "Clarifai Bipartite Annotation Network" # title displayed in UI - description = "Create a GEXF network file comprised of all annotations returned by the Clarifai API. Labels " \ - "returned by the API, and image file names, are nodes. Edges are created between file names and " \ - "labels if the label occurs for the image with that file name." + description = ProcessorDescription( + title="Create Clarifai bipartite network", + tags=["networks"], + description="Build a bipartite network from Clarifai image annotations. Image file names and Clarifai labels are the nodes, and an edge connects a file name to a label when Clarifai assigned that label to that image. Use the minimum confidence option to drop low-confidence labels.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # Allow processor to run on Clarifai API data compatibility = Compatibility(types={"clarifai-api"}) diff --git a/processors/networks/colink_urls.py b/processors/networks/colink_urls.py index ff192d150..6d6930567 100644 --- a/processors/networks/colink_urls.py +++ b/processors/networks/colink_urls.py @@ -7,8 +7,9 @@ import psutil -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput @@ -28,11 +29,15 @@ class URLCoLinker(BasicProcessor): Generate URL co-link network """ type = "url-network" # job type ID - category = "Networks" # category - title = "URL co-occurence network" # title displayed in UI - description = "Create a GEXF network file comprised of URLs appearing together (in a post or thread). " \ - "Edges are weighted by amount of co-links." # description displayed in UI + description = ProcessorDescription( + title="Create URL co-link network", + tags=["urls", "networks"], + description="Build a network of URLs that appear together in the same post or thread. Each URL is a node, and edges connect URLs that co-occur. Choose whether to use full URLs or only domain names, and whether to count co-occurrence per post or per thread.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/networks/cotag_network.py b/processors/networks/cotag_network.py index 6e4ec3816..0568d5bb5 100644 --- a/processors/networks/cotag_network.py +++ b/processors/networks/cotag_network.py @@ -3,8 +3,10 @@ """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -17,12 +19,15 @@ class CoTaggerPreset(ProcessorPreset): Generate co-tag network of co-occurring (hash)tags in items """ type = "preset-cotag-network" # job type ID - category = "Networks" # category - title = "Co-tag network" # title displayed in UI - description = "Create a GEXF network file of tags co-occurring in a posts. " \ - "Edges are weighted by the amount of tag co-occurrences; nodes " \ - "are weighted by how often the tag appears in the dataset." # description displayed in UI + description = ProcessorDescription( + title="Co-tag network", + tags=["hashtags", "networks"], + description="Create a network of tags co-occurring in items. Edges are weighted by how often two tags co-occur; nodes are weighted by how often a tag appears in the dataset.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a preset; its output is its last step's + output = Delegated() possible_tag_columns = {"tags", "hashtags", "groups"} diff --git a/processors/networks/coword_network.py b/processors/networks/coword_network.py index 14be06c44..e212d1043 100644 --- a/processors/networks/coword_network.py +++ b/processors/networks/coword_network.py @@ -3,7 +3,9 @@ """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated __author__ = "Sal Hagen" __credits__ = ["Sal Hagen"] @@ -16,13 +18,17 @@ class CowordNetworker(ProcessorPreset): Generate co-word network """ type = "preset-coword-network" # job type ID - category = "Networks" # category - title = "Co-word network" # title displayed in UI - description = "Create a GEXF network file of word co-occurences. Edges denote " \ - "words that appear close to each other. Edges and nodes are weighted by the " \ - "amount of co-word occurrences." # description displayed in UI + description = ProcessorDescription( + title="Co-word network", + tags=["networks", "text analysis"], + description="Create a network of word co-occurrences. Edges connect words that appear close to each other. Edges and nodes are weighted by how often the words co-occur.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a preset; its output is its last step's + output = Delegated() + # Allow processor to run on collocations compatibility = Compatibility(types={"collocations"}) diff --git a/processors/networks/gexf_to_csv.py b/processors/networks/gexf_to_csv.py index 729ca7b8b..1534b1730 100644 --- a/processors/networks/gexf_to_csv.py +++ b/processors/networks/gexf_to_csv.py @@ -1,8 +1,9 @@ """ Convert a GEXF network file to a CSV file """ -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table import networkx as nx import csv @@ -19,10 +20,15 @@ class GexfToCsv(BasicProcessor): Convert a GEXF network file to a CSV file """ type = "gexf-to-csv" - category = "Networks" - title = "Export Network as CSV Spreadsheet" - description = "Convert a GEXF network file to a CSV spreadsheet" + description = ProcessorDescription( + title="Convert network to CSV", + tags=["conversion", "networks"], + description="Convert a GEXF network file to a CSV file, with one row per edge. Each row lists the source and target nodes, their attributes, and the edge attributes. Edges are sorted by weight, from most to least frequent.", + icon="file-csv", + ) extension = "csv" + # a derived table + output = Table() # Allow on GEXF datasets compatibility = Compatibility(extensions={"gexf"}) diff --git a/processors/networks/google_vision_bipartite_network.py b/processors/networks/google_vision_bipartite_network.py index aef0f3c19..cbea2b229 100644 --- a/processors/networks/google_vision_bipartite_network.py +++ b/processors/networks/google_vision_bipartite_network.py @@ -1,8 +1,9 @@ """ Google Vision API co-label network """ -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.helpers import UserInput from common.lib.exceptions import ProcessorInterruptedException @@ -19,12 +20,15 @@ class VisionTagBiPartiteNetworker(BasicProcessor): Google Vision API co-label network """ type = "vision-bipartite-network" # job type ID - category = "Networks" # category - title = "Google Vision bipartite annotation network" # title displayed in UI - description = "Create a GEXF network file comprised of all annotations returned by the Google Vision API. Labels " \ - "returned by the API, and image file names, are nodes. Edges are created between file names and " \ - "labels if the label occurs for the image with that file name." + description = ProcessorDescription( + title="Google Vision bipartite annotation network", + tags=["networks", "visual"], + description="Create a network from annotations returned by the Google Vision API. Image file names and the labels returned for them are nodes. An edge connects a file name to a label when that label occurs for that image.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # Allow processor to run on Google Vision API data compatibility = Compatibility(types={"google-vision-api"}) diff --git a/processors/networks/google_vision_network.py b/processors/networks/google_vision_network.py index 9ddf0da8d..f72e3df37 100644 --- a/processors/networks/google_vision_network.py +++ b/processors/networks/google_vision_network.py @@ -1,8 +1,9 @@ """ Google Vision API co-label network """ -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.helpers import UserInput from common.lib.exceptions import ProcessorInterruptedException @@ -19,12 +20,15 @@ class VisionTagNetworker(BasicProcessor): Google Vision API co-label network """ type = "vision-label-network" # job type ID - category = "Networks" # category - title = "Google Vision co-Label network" # title displayed in UI - description = "Create a GEXF network file comprised of all annotations returned by the" \ - "Google Vision API. Labels returned by the API are nodes. Labels occurring on the same image form" \ - "edges." + description = ProcessorDescription( + title="Google Vision co-label network", + tags=["networks", "visual"], + description="Create a network from annotations returned by the Google Vision API. Labels returned by the API are nodes. An edge connects two labels when they occur on the same image, weighted by how often they co-occur.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # Allow processor to run on Google Vision API data compatibility = Compatibility(types={"google-vision-api"}) diff --git a/processors/networks/hash_similarity_network.py b/processors/networks/hash_similarity_network.py index 845c91814..809c6f339 100644 --- a/processors/networks/hash_similarity_network.py +++ b/processors/networks/hash_similarity_network.py @@ -6,8 +6,9 @@ import networkx as nx import numpy as np -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.exceptions import ProcessorException from common.lib.helpers import UserInput @@ -23,13 +24,23 @@ class HashSimilarityNetworker(BasicProcessor): Compare hashes and generate a network based on similarity """ type = "hash-similarity-network" - category = "Networks" - title = "Hash similarity network" - description = "Calculate similarity of hashes and create a GEXF network file. Can identify near duplicate hashes." + description = ProcessorDescription( + title="Hash similarity network", + tags=["networks"], + description="Compare bit hashes and create a network linking similar items. Each pair of hashes is compared bit by bit, and an edge is added when they are at least as similar as the chosen threshold. Useful for finding near-duplicate images or videos.", + warnings=[ + "Only bit hashes are supported, such as those produced by the video hasher.", + "Every pair of hashes is compared, so this can be slow on large datasets.", + ], + icon="circle-nodes", + ) extension = "gexf" + # a graph file, no column table + output = Network() - # Currently only allowed on video-hashes, though any row of bit hashes would work. - compatibility = Compatibility(types={"video-hashes"}) + # Runs on the video hasher's output (like the other video-hash networks); any + # dataset of bit-hash rows would work. + compatibility = Compatibility(types={"video-hasher-1"}) @classmethod def get_options(cls, parent_dataset=None, config=None): diff --git a/processors/networks/image-network.py b/processors/networks/image-network.py index ee573604b..31f1849c1 100644 --- a/processors/networks/image-network.py +++ b/processors/networks/image-network.py @@ -3,7 +3,7 @@ """ import json -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import hash_file import networkx as nx @@ -16,6 +16,7 @@ from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Network class ImageGrapher(BasicProcessor): @@ -26,13 +27,18 @@ class ImageGrapher(BasicProcessor): images were sourced from """ type = "image-bipartite-network" # job type ID - category = "Networks" - title = "Bipartite image-item network" # title displayed in UI - description = ("Create a GEXF network file with a bipartite network of " - "images and some data field (e.g. author) of the dataset " - "the images were sourced from. Suitable for use with Gephi's " - "'Image Preview' plugin.") + description = ProcessorDescription( + title="Bipartite image-item network", + tags=["networks", "visual"], + description="Create a network with a bipartite network of images and a data field (for example author) of the dataset the images were sourced from. Suitable for use with Gephi's Image Preview plugin.", + info=[ + "Optionally merge similar images into a single node using file hashing or perceptual hashing.", + ], + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # coarse map spec; is_compatible_with (below) is the runtime truth -- it also walks the # genealogy to find an image-downloader root (get_root_dataset) diff --git a/processors/networks/quote_network.py b/processors/networks/quote_network.py index 98ebf2979..85a0ef549 100644 --- a/processors/networks/quote_network.py +++ b/processors/networks/quote_network.py @@ -3,8 +3,9 @@ """ import re -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network import networkx as nx @@ -20,11 +21,15 @@ class QuoteNetworkGrapher(BasicProcessor): Creates a network of posts quoting each other """ type = "quote-network" # job type ID - category = "Networks" - title = "Reply network" # title displayed in UI - description = "Create a GEXF network file of posts replying to each other. " \ - "Each reference to another post creates an edge between posts. " # description displayed in UI + description = ProcessorDescription( + title="Reply network", + tags=["networks"], + description="Create a network of posts replying to each other. Each reference to another post creates an edge between the two posts.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # chan datasets (posts reply to / quote each other) compatibility = Compatibility(datasources={"fourchan", "eightchan", "eightkun"}) diff --git a/processors/networks/two-column-network.py b/processors/networks/two-column-network.py index f79cb6f85..dc7db334a 100644 --- a/processors/networks/two-column-network.py +++ b/processors/networks/two-column-network.py @@ -4,8 +4,9 @@ from dateutil.relativedelta import relativedelta from functools import partial -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.helpers import UserInput, get_interval_descriptor import networkx as nx @@ -22,19 +23,23 @@ class ColumnNetworker(BasicProcessor): Generate network of values from two columns """ type = "column-network" - category = "Networks" - title = "Custom network" - description = "Create a GEXF network file comprised of linked values between a custom set of columns " \ - "(e.g. 'author' and 'subreddit'). Nodes and edges are weighted by frequency." + description = ProcessorDescription( + title="Custom network", + tags=["networks"], + description="Create a network of linked values between two chosen columns (for example 'author' and 'subreddit'). Nodes and edges are weighted by frequency. Optionally make the network dynamic over time and detect communities.", + references=[ + "Utilises [Networkx](https://networkx.org/).", + "Networkx built-in [Louvain](https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.community.louvain.louvain_communities.html#networkx.algorithms.community.louvain.louvain_communities) and [greedy modularity](https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.community.modularity_max.greedy_modularity_communities.html#networkx.algorithms.community.modularity_max.greedy_modularity_communities) community detection algorithms.", + ], + icon="circle-nodes", + ) extension = "gexf" + # a graph file, no column table + output = Network() # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) - references = [ - "Utilises [Networkx](https://networkx.org/)' built-in [Louvain](https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.community.louvain.louvain_communities.html#networkx.algorithms.community.louvain.louvain_communities) and [greedy modularity](https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.community.modularity_max.greedy_modularity_communities.html#networkx.algorithms.community.modularity_max.greedy_modularity_communities) community detection algorithms." - ] - @classmethod def get_options(cls, parent_dataset=None, config=None): """ diff --git a/processors/networks/user_hashtag_network.py b/processors/networks/user_hashtag_network.py index b21366cf7..ee42e13a8 100644 --- a/processors/networks/user_hashtag_network.py +++ b/processors/networks/user_hashtag_network.py @@ -2,8 +2,10 @@ Generate bipartite user-hashtag graph of posts """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.user_input import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated __author__ = "Stijn Peeters" @@ -17,10 +19,15 @@ class HashtagUserBipartiteGrapherPreset(ProcessorPreset): Generate bipartite user-hashtag graph of posts """ type = "preset-bipartite-user-tag-network" # job type ID - category = "Networks" # category - title = "Author-tag Network" # title displayed in UI - description = "Produces a bipartite graph based on co-occurence of (hash)tags and authors. If someone wrote a post with a certain tag, there will be a link between that person and the tag. The more often they appear together, the stronger the link. Tag nodes are weighed on how often they occur. User nodes are weighed on how many posts they've made." # description displayed in UI + description = ProcessorDescription( + title="Author-tag network", + tags=["networks", "authors", "hashtags"], + description="Create a bipartite network of authors and the (hash)tags they use, based on co-occurrence. An author and a tag are linked when the author wrote a post with that tag, and the link grows stronger the more often they appear together. Tag nodes are weighted by how often they occur, and author nodes by how many posts they made.", + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a preset; its output is its last step's + output = Delegated() # datasets with at least one tag-like column compatibility = Compatibility(requires_any_columns={"tags", "hashtags", "groups"}) diff --git a/processors/networks/wikipedia_network.py b/processors/networks/wikipedia_network.py index e69a00045..c0d8f32c3 100644 --- a/processors/networks/wikipedia_network.py +++ b/processors/networks/wikipedia_network.py @@ -8,8 +8,9 @@ from io import StringIO import networkx as nx -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Network from common.lib.exceptions import ProcessorInterruptedException __author__ = "Stijn Peeters" @@ -23,10 +24,19 @@ class WikiURLCoLinker(BasicProcessor): Generate URL co-link network """ type = "wiki-category-network" # job type ID - category = "Networks" # category - title = "Wikipedia category network" # title displayed in UI - description = "Create a GEXF network file comprised network comprised of linked-to Wikipedia pages, linked to the categories they are part of. English Wikipedia only. Will only fetch the first 10,000 links." # description displayed in UI + description = ProcessorDescription( + title="Wikipedia category network", + tags=["networks", "urls"], + description="Create a network of linked-to Wikipedia pages connected to the categories they belong to. Wikipedia links are extracted from the post body and looked up through Wikipedia to find their categories.", + warnings=[ + "Only English Wikipedia is supported, and only the first 10,000 links found are used.", + "Page categories are fetched from Wikipedia, an external service.", + ], + icon="circle-nodes", + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/presets/annotate-images.py b/processors/presets/annotate-images.py index 829c5d8ac..b29330d5a 100644 --- a/processors/presets/annotate-images.py +++ b/processors/presets/annotate-images.py @@ -2,7 +2,9 @@ Annotate top images """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated from common.lib.helpers import UserInput, convert_to_int @@ -12,20 +14,27 @@ class AnnotateImages(ProcessorPreset): Run processor pipeline to annotate images """ type = "preset-annotate-images" # job type ID - category = "Combined processors" # category. 'Combined processors' are always listed first in the UI. - title = "Annotate images with Google Vision" # title displayed in UI - description = "Use the Google Vision API to extract labels detected in the most-linked images from the dataset. Note that " \ - "this is a paid service and will count towards your API credit." + description = ProcessorDescription( + title="Annotate images with Google Vision", + tags=["combined", "classification", "transcribe", "external service"], + description="Download the most-linked images from the dataset and use the Google Vision API to detect labels, text, faces, landmarks, logos, and other features in them.", + references=[ + "[Google Vision API Documentation](https://cloud.google.com/vision/docs)", + "[Google Vision API Pricing & Free Usage Limits](https://cloud.google.com/vision/pricing)", + ], + warnings=[ + "This is a paid service. Google bills the owner of the API key you provide.", + "Images are sent to Google Vision, an external service, to be analysed.", + ], + icon="tags", + ) extension = "csv" + # a preset; its output is its last step's + output = Delegated() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) - references = [ - "[Google Vision API Documentation](https://cloud.google.com/vision/docs)", - "[Google Vision API Pricing & Free Usage Limits](https://cloud.google.com/vision/pricing)" - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/presets/monthly-histogram.py b/processors/presets/monthly-histogram.py index a67464b5a..e563d92d8 100644 --- a/processors/presets/monthly-histogram.py +++ b/processors/presets/monthly-histogram.py @@ -2,7 +2,9 @@ Extract neologisms """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated from processors.metrics.count_posts import CountPosts @@ -11,11 +13,17 @@ class MonthlyHistogramCreator(ProcessorPreset): Run processor pipeline to extract neologisms """ type = "preset-histogram" # job type ID - category = "Combined processors" # category. 'Combined processors' are always listed first in the UI. - title = "Histogram" # title displayed in UI - description = "Create a histogram that shows the number of items over time." # description displayed in UI + description = ProcessorDescription( + title="Create a histogram of items over time", + tags=["combined", "time series", "counting", "chart"], + description="Count items per day, week, month, or year and render the totals as a bar chart in an SVG file.", + icon="square-poll-vertical", + ) extension = "svg" + # a preset; its output is its last step's + output = Delegated() + # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/presets/neologisms.py b/processors/presets/neologisms.py index 0b1e0179d..dd89265b7 100644 --- a/processors/presets/neologisms.py +++ b/processors/presets/neologisms.py @@ -2,7 +2,9 @@ Extract neologisms """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated from common.lib.helpers import UserInput @@ -12,18 +14,25 @@ class NeologismExtractor(ProcessorPreset): Run processor pipeline to extract neologisms """ type = "preset-neologisms" # job type ID - category = "Combined processors" # category. 'Combined processors' are always listed first in the UI. - title = "Extract neologisms" # title displayed in UI - description = ("Retrieve uncommon terms by deleting all words that appears in dictionary lists. Assumes English-" - "language data. Uses stopwords-iso as a stopword filter.") + description = ProcessorDescription( + title="Extract neologisms", + tags=["combined", "text analysis"], + description="Find uncommon terms by removing every word that appears in dictionary and stopword lists. Uses a Google Books English word list and the stopwords-iso English list as filters.", + warnings=[ + "This assumes the data is in English; for other languages start with the Tokenise posts processor and use a different word list.", + ], + references=[ + "Van Soest, Jeroen. 2019. 'Language Innovation Tracker: Detecting language innovation in online discussion fora.' (MA thesis), Beuls, K. (Promotor), Van Eecke, P. (Advisor).", + ], + icon="comment-medical", + ) extension = "csv" + # a preset; its output is its last step's + output = Delegated() # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) - references = [ - "Van Soest, Jeroen. 2019. 'Language Innovation Tracker: Detecting language innovation in online discussion fora.' (MA thesis), Beuls, K. (Promotor), Van Eecke, P. (Advisor).'"] - @classmethod def get_options(cls, parent_dataset=None, config=None): """ diff --git a/processors/presets/similar-words.py b/processors/presets/similar-words.py index 5c172776e..ed55e1e38 100644 --- a/processors/presets/similar-words.py +++ b/processors/presets/similar-words.py @@ -4,7 +4,9 @@ from nltk.stem.snowball import SnowballStemmer from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated from common.lib.helpers import UserInput @@ -14,11 +16,18 @@ class SimilarWords(ProcessorPreset): Run processor pipeline to find similar words """ type = "preset-similar-words" # job type ID - category = "Combined processors" # category. 'Combined processors' are always listed first in the UI. - title = "Find similar words" # title displayed in UI - description = ("Create a word2vec model to find words used in a similar context as the queried word(s). Only works " - "with large datasets (e.g. 100,000+ items).") + description = ProcessorDescription( + title="Find similar words", + tags=["combined", "text analysis", "machine learning"], + description="Train a word2vec model on the dataset to find words used in a context similar to the words you enter.", + warnings=[ + "This only produces useful results on large datasets, roughly 100,000 items or more.", + ], + icon="comments", + ) extension = "csv" + # a preset; its output is its last step's + output = Delegated() # Allow on top-level CSV/NDJSON datasets compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"}) diff --git a/processors/presets/top-hashtags.py b/processors/presets/top-hashtags.py index dc79a784e..13e11de23 100644 --- a/processors/presets/top-hashtags.py +++ b/processors/presets/top-hashtags.py @@ -2,9 +2,11 @@ Find most-used hashtags in a dataset """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.helpers import UserInput from processors.networks.cotag_network import CoTaggerPreset from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated class TopHashtags(ProcessorPreset): @@ -12,10 +14,15 @@ class TopHashtags(ProcessorPreset): Run processor pipeline to find top hashtags """ type = "preset-top-hashtags" # job type ID - category = "Combined processors" # category. 'Combined processors' are always listed first in the UI. - title = "Top hashtags" # title displayed in UI - description = "Count how often each hashtag occurs in the dataset and sort by this value" + description = ProcessorDescription( + title="Top hashtags", + tags=["combined", "hashtags", "counting"], + description="Count how often each hashtag occurs in the dataset and rank them from most to least frequent.", + icon="hashtag", + ) extension = "csv" + # a preset; its output is its last step's + output = Delegated() # datasets with at least one tag-like column compatibility = Compatibility(requires_any_columns=CoTaggerPreset.possible_tag_columns) diff --git a/processors/presets/upload-to-dmi-tcat.py b/processors/presets/upload-to-dmi-tcat.py index 7e6004415..00b754035 100644 --- a/processors/presets/upload-to-dmi-tcat.py +++ b/processors/presets/upload-to-dmi-tcat.py @@ -2,18 +2,28 @@ Upload Twitter dataset to DMI-TCAT instance """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility +from common.lib.outputs import Delegated class FourcatToDmiTcatConverterAndUploader(ProcessorPreset): """ Run processor pipeline to extract neologisms """ type = "preset-upload-tcat" # job type ID - category = "Combined processors" # category. 'Combined processors' are always listed first in the UI. - title = "Upload to DMI-TCAT" # title displayed in UI - description = "Convert the dataset to a TCAT-compatible format and upload it to an available TCAT server." # description displayed in UI + description = ProcessorDescription( + title="Upload to DMI-TCAT", + tags=["combined", "conversion", "external service"], + description="Convert the dataset to a DMI-TCAT-compatible format and upload it to a configured DMI-TCAT server.", + warnings=[ + "This sends the dataset to a DMI-TCAT server, an external service.", + ], + icon="brand-twitter", + ) extension = "html" + # a preset; its output is its last step's + output = Delegated() # Twitter v2 search results, when a TCAT server is configured compatibility = Compatibility(types={"twitterv2-search"}, required_settings={"tcat-auto-upload.server_url", "tcat-auto-upload.token", "tcat-auto-upload.username", "tcat-auto-upload.password"}) diff --git a/processors/presets/video-scene-timelines.py b/processors/presets/video-scene-timelines.py index 406c2a07e..d46f9a640 100644 --- a/processors/presets/video-scene-timelines.py +++ b/processors/presets/video-scene-timelines.py @@ -3,7 +3,9 @@ """ from backend.lib.preset import ProcessorPreset +from backend.lib.processor import ProcessorDescription from common.lib.compatibility import Compatibility, is_executable +from common.lib.outputs import Delegated class VideoSceneTimelineCreator(ProcessorPreset): @@ -11,15 +13,19 @@ class VideoSceneTimelineCreator(ProcessorPreset): Run processor pipeline to create video scene timelines """ type = "preset-scene-timelines" # job type ID - category = "Visual" # category. 'Combined processors' are always listed first in the UI. - title = "Create scene-by-scene timelines" # title displayed in UI - description = "Creates a 'timeline' for each video, a horizontal collage of sequential frames. Each 'scene' in " \ - "the video is visualised as a single frame. Scenes are detected algorithmically. The timelines " \ - "for all videos are then stacked vertically and rendered as a single SVG file." + description = ProcessorDescription( + title="Create scene-by-scene timelines", + tags=["video", "visual", "time series"], + description="Build a horizontal timeline for each video, showing one frame per detected scene. Scenes are detected automatically, and the per-video timelines are stacked into a single SVG file.", + icon="film", + ) extension = "svg" + # a preset; its output is its last step's + output = Delegated() + # Allow on video datasets when ffmpeg is available - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}) def get_processor_pipeline(self): """ diff --git a/processors/statistics/classification_evaluation.py b/processors/statistics/classification_evaluation.py index ae01ecbc7..59f93c3aa 100644 --- a/processors/statistics/classification_evaluation.py +++ b/processors/statistics/classification_evaluation.py @@ -4,8 +4,9 @@ from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput, andify -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from sklearn.preprocessing import MultiLabelBinarizer from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, cohen_kappa_score @@ -21,13 +22,20 @@ class ClassificationEvaluation(BasicProcessor): Generate accuracy, F1, recall, and precision scores for labels in two columns. """ type = "classification_evaluation" # job type ID - category = "Statistics" # category - title = "Classification evaluation" # title displayed in UI - description = ("Use calculate evaluation metrics (accuracy, precision, recall, F1, " - "and Cohen's Kappa) with labels from two columns. Produces overall and per-label metrics. " - "Also supports multi-label values.") + description = ProcessorDescription( + title="Evaluate classification labels", + tags=["classification", "statistics", "counting"], + description="Compare true and predicted labels in two columns and calculate accuracy, precision, recall, F1, and Cohen's Kappa. Produces overall and per-label metrics, and supports multiple labels per cell.", + info=[ + "For multiple labels per cell, enable the multi-label option and separate the labels with commas.", + ], + icon="table-columns", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/statistics/confusion_matrix.py b/processors/statistics/confusion_matrix.py index 00e4c521e..51af834ae 100644 --- a/processors/statistics/confusion_matrix.py +++ b/processors/statistics/confusion_matrix.py @@ -3,8 +3,9 @@ """ from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay import matplotlib @@ -21,10 +22,18 @@ class ConfusionMatrix(BasicProcessor): Create a confusion matrix with values from two columns """ type = "confusion-matrix" # job type ID - category = "Statistics" # category - title = "Confusion matrix" # title displayed in UI - description = "Create a confusion matrix with data from two columns." # description displayed in UI + description = ProcessorDescription( + title="Create a confusion matrix", + tags=["statistics", "chart", "classification", "counting"], + description="Build a confusion matrix comparing true labels and predicted labels from two columns. The result is a rendered image cross-tabulating how often each true category was predicted as each category. Supports up to 500 unique labels.", + info=[ + "Best used to evaluate a classifier by comparing its predictions against known correct labels.", + ], + icon="table-cells", + ) extension = "png" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render("png") # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/statistics/descriptive_statistics.py b/processors/statistics/descriptive_statistics.py index 6156ecfac..64511fec1 100644 --- a/processors/statistics/descriptive_statistics.py +++ b/processors/statistics/descriptive_statistics.py @@ -4,8 +4,9 @@ from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table import numpy as np @@ -20,10 +21,15 @@ class DescriptiveStatistics(BasicProcessor): Generate descriptive statistics for numerical columns. """ type = "descriptive_statistics" # job type ID - category = "Statistics" # category - title = "Descriptive statistics" # title displayed in UI - description = "Calculate descriptive statistics (mean, median, std dev, etc.) for numerical columns." + description = ProcessorDescription( + title="Calculate descriptive statistics", + tags=["statistics", "counting", "metadata"], + description="Calculate descriptive statistics for selected numerical columns, including count, mean, standard deviation, minimum, maximum, range, quartiles, interquartile range, variance, median, and mode. Rows with missing or non-numeric values can be skipped or treated as an error.", + icon="table-columns", + ) extension = "csv" # extension of result file, used internally in UI + # a derived table + output = Table() # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/statistics/regression-evaluation.py b/processors/statistics/regression-evaluation.py index 09fc65da9..4b2dc3ebc 100644 --- a/processors/statistics/regression-evaluation.py +++ b/processors/statistics/regression-evaluation.py @@ -4,8 +4,9 @@ from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score import numpy as np @@ -21,11 +22,17 @@ class RegressionEvaluation(BasicProcessor): Generate MAE, MSE, R2, and RMSE scores for numerical predictions. """ type = "regression_evaluation" # job type ID - category = "Statistics" # category - title = "Regression evaluation" # title displayed in UI - description = "Calculate regression metrics (MAE, MSE, R2, RMSE) between two numerical columns." + description = ProcessorDescription( + title="Evaluate regression predictions", + tags=["statistics", "counting"], + description="Compare true and predicted numerical values in two columns and report regression error metrics. Calculates mean absolute error, mean squared error, root mean squared error, and R-squared, according to which metrics are selected. Rows with missing or non-numeric values can be skipped or treated as an error.", + icon="table-columns", + ) extension = "csv" # extension of result file, used internally in UI + # a derived table + output = Table() + # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/text-analysis/collocations.py b/processors/text-analysis/collocations.py index 147608334..bb66e8a80 100644 --- a/processors/text-analysis/collocations.py +++ b/processors/text-analysis/collocations.py @@ -9,8 +9,9 @@ from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table class GetCollocations(BasicProcessor): @@ -18,11 +19,17 @@ class GetCollocations(BasicProcessor): Generates word collocations from input tokens """ type = "collocations" # job type ID - category = "Text analysis" # category - title = "Extract co-words" # title displayed in UI - description = "Extracts words appearing close to each other from a set of tokens." # description displayed in UI + description = ProcessorDescription( + title="Extract co-words", + tags=["text analysis", "counting"], + description="Find pairs or triplets of words that appear close together in a set of tokens, along with how often each combination occurs. A window size sets how near words must be to count as co-words. Results can be limited to combinations containing a required word, filtered by a minimum frequency, and optionally saved as annotations.", + icon="timeline", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow processor on token sets compatibility = Compatibility(types={"tokenise-posts"}, preferred_followups=["preset-coword-network", "wordcloud"]) diff --git a/processors/text-analysis/documents_per_topic.py b/processors/text-analysis/documents_per_topic.py index 80375c339..854c4d3c5 100644 --- a/processors/text-analysis/documents_per_topic.py +++ b/processors/text-analysis/documents_per_topic.py @@ -2,8 +2,9 @@ Extracts topics per model and top associated words """ -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException import json @@ -20,11 +21,17 @@ class TopicModelWordExtractor(BasicProcessor): Extracts topics per model and top associated words """ type = "document_count" # job type ID - category = "Text analysis" # category - title = "Count documents per topic" # title displayed in UI - description = "Uses the LDA model to predict to which topic each item or sentence belongs and counts as belonging to whichever topic has the highest probability." # description displayed in UI + description = ProcessorDescription( + title="Count documents per topic", + tags=["text analysis", "counting"], + description="Assign each item or sentence to the topic model topic it fits best and count how many documents fall under each topic per time interval. Each document is placed in the topic with the highest probability, and documents that fit two topics equally are skipped. The result also lists the top five words for each topic.", + icon="file-circle-question", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow processor on topic models compatibility = Compatibility(types={"topic-modeller"}) diff --git a/processors/text-analysis/generate_embeddings.py b/processors/text-analysis/generate_embeddings.py index ea352895b..513fd6bd8 100644 --- a/processors/text-analysis/generate_embeddings.py +++ b/processors/text-analysis/generate_embeddings.py @@ -9,8 +9,9 @@ from gensim.models.phrases import Phrases, Phraser from common.lib.helpers import UserInput, convert_to_int -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Archive from common.lib.exceptions import ProcessorInterruptedException __author__ = "Sal Hagen" @@ -24,25 +25,29 @@ class GenerateWordEmbeddings(BasicProcessor): Generate Word Embeddings """ type = "generate-embeddings" # job type ID - category = "Text analysis" # category - title = "Generate word embedding models" # title displayed in UI - description = "Generates word2vec or FastText word embedding models (overall or per timeframe). " \ - "These calculate coordinates (vectors) per word on the basis of their context. The " \ - "coordinates are positioned in a \"vector space\" with a large amount of dimensions (so a coordinate can " \ - "e.g. exist of 100 numbers). These numeric word representations can be used to extract words with similar contexts. " \ - "Note that good models require a lot of data." # description displayed in UI + description = ProcessorDescription( + title="Generate word embedding models", + tags=["text analysis", "machine learning"], + description="Train word2vec or FastText word embedding models from tokens, either for the whole dataset or per time interval. Each word is assigned a position in a multi-dimensional vector space based on the words it appears alongside, so words used in similar contexts end up close together. These models can then be used to find words with similar contexts or to track how word usage shifts over time.", + references=[ + "word2vec: [Mikolov, Tomas, Ilya Sutskever, Kai Chen, Greg Corrado, and Jeffrey Dean. 2013. “Distributed Representations of Words and Phrases and Their Compositionality.” 8Advances in Neural Information Processing Systems*, 2013: 3111-3119.](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf)", + "word2vec: [Mikolov, Tomas, Kai Chen, Greg Corrado, and Jeffrey Dean. 2013. “Efficient Estimation of Word Representations in Vector Space.” *ICLR Workshop Papers*, 2013: 1-12.](https://arxiv.org/pdf/1301.3781.pdf)", + "word2vec: [A Beginner's Guide to Word Embedding with Gensim Word2Vec Model - Towards Data Science](https://towardsdatascience.com/a-beginners-guide-to-word-embedding-with-gensim-word2vec-model-5970fa56cc92)", + "FastText: [Bojanowski, P., Grave, E., Joulin, A., & Mikolov, T. (2017). Enriching word vectors with subword information. *Transactions of the Association for Computational Linguistics*, 5, 135-146.](https://www.mitpressjournals.org/doi/abs/10.1162/tacl_a_00051)", + ], + warnings=[ + "Models trained on small amounts of text are unreliable; good results need a large corpus.", + ], + icon="cube", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of data files + output = Archive() + # Allow processor on token sets compatibility = Compatibility(types={"tokenise-posts"}, preferred_followups=["similar-word2vec", "histwords-vectspace"]) - references = [ - "word2vec: [Mikolov, Tomas, Ilya Sutskever, Kai Chen, Greg Corrado, and Jeffrey Dean. 2013. “Distributed Representations of Words and Phrases and Their Compositionality.” 8Advances in Neural Information Processing Systems*, 2013: 3111-3119.](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf)", - "word2vec: [Mikolov, Tomas, Kai Chen, Greg Corrado, and Jeffrey Dean. 2013. “Efficient Estimation of Word Representations in Vector Space.” *ICLR Workshop Papers*, 2013: 1-12.](https://arxiv.org/pdf/1301.3781.pdf)", - "word2vec: [A Beginner's Guide to Word Embedding with Gensim Word2Vec Model - Towards Data Science](https://towardsdatascience.com/a-beginners-guide-to-word-embedding-with-gensim-word2vec-model-5970fa56cc92)", - "FastText: [Bojanowski, P., Grave, E., Joulin, A., & Mikolov, T. (2017). Enriching word vectors with subword information. *Transactions of the Association for Computational Linguistics*, 5, 135-146.](https://www.mitpressjournals.org/doi/abs/10.1162/tacl_a_00051)" - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/text-analysis/post_topic_matrix.py b/processors/text-analysis/post_topic_matrix.py index 190abcef6..0c4692177 100644 --- a/processors/text-analysis/post_topic_matrix.py +++ b/processors/text-analysis/post_topic_matrix.py @@ -3,8 +3,9 @@ """ from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException import csv @@ -22,13 +23,18 @@ class TopicModelWordExtractor(BasicProcessor): Extracts topics per model and top associated words """ type = "document_topic_matrix" # job type ID - category = "Text analysis" # category - title = "Post/topic matrix" # title displayed in UI - description = ("Predict which item or sentence belong to which topics using LDA. Creates a CSV file where " - "each line represents one 'document'. If tokens are grouped per 'item' and only one column is used " - "(e.g. only the 'body' column), there is one row per post/item, otherwise a post may be represented " - "by multiple rows (for each sentence and/or column used).") # description displayed in UI + description = ProcessorDescription( + title="Match items to topics", + tags=["text analysis", "machine learning", "classification"], + description="Use Latent Dirichlet Allocation to predict which topics each item or sentence belongs to. " + "Produce a table where each row is one document. If tokens are grouped per item and only one " + "column is used, there is one row per item, otherwise an item spans several rows, one per " + "sentence or column used.", + icon="table-cells", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on topic models compatibility = Compatibility(types={"topic-modeller"}) diff --git a/processors/text-analysis/similar_words.py b/processors/text-analysis/similar_words.py index 7db6573c6..967f3fdf9 100644 --- a/processors/text-analysis/similar_words.py +++ b/processors/text-analysis/similar_words.py @@ -6,8 +6,9 @@ from gensim.models import KeyedVectors from common.lib.helpers import UserInput, convert_to_int, convert_to_float -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException __author__ = "Sal Hagen" @@ -21,10 +22,16 @@ class SimilarWord2VecWords(BasicProcessor): Find similar words based on word2vec modeling """ type = "similar-word2vec" # job type ID - category = "Text analysis" # category - title = "Extract similar words" # title displayed in UI - description = "Uses a word2vec model to find words used in a similar context" # description displayed in UI + description = ProcessorDescription( + title="Extract similar words", + tags=["text analysis", "machine learning"], + description="Use a word2vec model to find words that appear in similar contexts to the words you provide. " + "Set a similarity threshold and a crawl depth to also follow the neighbours of neighbours.", + icon="language", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # Allow processor on word embedding models compatibility = Compatibility(types={"generate-embeddings"}, preferred_followups=["wordcloud"]) diff --git a/processors/text-analysis/split_sentences.py b/processors/text-analysis/split_sentences.py index e5a5cc712..137d7e597 100644 --- a/processors/text-analysis/split_sentences.py +++ b/processors/text-analysis/split_sentences.py @@ -5,8 +5,9 @@ from nltk.tokenize import sent_tokenize, word_tokenize from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -19,11 +20,19 @@ class SplitSentences(BasicProcessor): Split sentences """ type = "sentence-split" # job type ID - category = "Text analysis" # category - title = "Split text into sentences" # title displayed in UI - description = "Split a body of posts into discrete sentences. Output file has one row per sentence, containing the sentence and item ID." # description displayed in UI + description = ProcessorDescription( + title="Split text into sentences", + tags=["text analysis", "preprocessing", "conversion"], + description="Split the text in a chosen column into separate sentences. The output has one row per sentence, " + "with the sentence and its item ID. Sentences shorter than a chosen number of words can be " + "dropped.", + icon="arrows-left-right-to-line", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow on CSV/NDJSON datasets compatibility = Compatibility(extensions={"csv", "ndjson"}) diff --git a/processors/text-analysis/tf_idf.py b/processors/text-analysis/tf_idf.py index 4e7378c4f..fef1efaaf 100644 --- a/processors/text-analysis/tf_idf.py +++ b/processors/text-analysis/tf_idf.py @@ -8,8 +8,9 @@ import itertools from common.lib.helpers import UserInput, convert_to_int -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from sklearn.feature_extraction.text import TfidfVectorizer from gensim.models import TfidfModel @@ -27,10 +28,30 @@ class TfIdf(BasicProcessor): """ type = "tfidf" # job type ID - category = "Text analysis" # category - title = "Tf-idf" # title displayed in UI - description = "Get the tf-idf values of tokenised text. Works better with more documents (e.g. time-separated)." # description displayed in UI + description = ProcessorDescription( + title="Calculate tf-idf", + tags=["text analysis", "counting"], + description="Calculate the tf-idf (term frequency-inverse document frequency) value of tokenised text, a " + "measure of how distinctive each word is to a document. Choose between the scikit-learn and gensim " + "libraries and return the top-scoring words per timeframe. Works better with more documents.", + info=[ + "Use gensim rather than scikit-learn for large datasets, as it uses less memory.", + ], + references=[ + "[Spärck Jones, Karen. 1972. \"A statistical interpretation of term specificity and its application in retrieval.\" *Journal of Documentation* (28), 1: 11–21.](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.115.8343&rep=rep1&type=pdf)", + "[Robertson, Stephen. 2004. \"Understanding Inverse Document value: On Theoretical arguments for IDF.\" *Journal of Documentation* (60), 5: 503–520](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.438.2284&rep=rep1&type=pdf)", + "[Spärck Jones, Karen. 2004. \"IDF term weighting and IR research lessons\". *Journal of Communication* (60), 5: 521-523.](https://www.staff.city.ac.uk/~sb317/idfpapers/ksj_reply.pdf)", + "[Gensim tf-idf documentation.](https://radimrehurek.com/gensim/models/tfidfmodel.html)", + "[Scikit learn tf-idf documentation.](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html)", + "[Tf-idf - Wikipedia.](https://en.wikipedia.org/wiki/Tf%E2%80%93idf)", + "[What is tf-idf? - William Scott](https://towardsdatascience.com/tf-idf-for-document-ranking-from-scratch-in-python-on-real-world-dataset-796d339a4089)", + "[SMART Information Retrieval System](https://en.wikipedia.org/wiki/SMART_Information_Retrieval_System)", + ], + icon="ranking-star", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # Allow processor on token sets compatibility = Compatibility( @@ -42,17 +63,6 @@ class TfIdf(BasicProcessor): ], ) - references = [ - "[Spärck Jones, Karen. 1972. \"A statistical interpretation of term specificity and its application in retrieval.\" *Journal of Documentation* (28), 1: 11–21.](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.115.8343&rep=rep1&type=pdf)", - "[Robertson, Stephen. 2004. \"Understanding Inverse Document value: On Theoretical arguments for IDF.\" *Journal of Documentation* (60), 5: 503–520](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.438.2284&rep=rep1&type=pdf)", - "[Spärck Jones, Karen. 2004. \"IDF term weighting and IR research lessons\". *Journal of Communication* (60), 5: 521-523.](https://www.staff.city.ac.uk/~sb317/idfpapers/ksj_reply.pdf)", - "[Gensim tf-idf documentation.](https://radimrehurek.com/gensim/models/tfidfmodel.html)", - "[Scikit learn tf-idf documentation.](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html)", - "[Tf-idf - Wikipedia.](https://en.wikipedia.org/wiki/Tf%E2%80%93idf)", - "[What is tf-idf? - William Scott](https://towardsdatascience.com/tf-idf-for-document-ranking-from-scratch-in-python-on-real-world-dataset-796d339a4089)", - "[SMART Information Retrieval System](https://en.wikipedia.org/wiki/SMART_Information_Retrieval_System)" - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/text-analysis/tokenise.py b/processors/text-analysis/tokenise.py index 577900efa..231463d38 100644 --- a/processors/text-analysis/tokenise.py +++ b/processors/text-analysis/tokenise.py @@ -16,8 +16,9 @@ from razdel.substring import Substring from common.lib.helpers import UserInput, get_interval_descriptor -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Archive __author__ = ["Stijn Peeters", "Sal Hagen"] __credits__ = ["Stijn Peeters", "Sal Hagen"] @@ -30,23 +31,31 @@ class Tokenise(BasicProcessor): Tokenize posts """ type = "tokenise-posts" # job type ID - category = "Text analysis" # category - title = "Tokenise" # title displayed in UI - description = "Splits item texts into separate tokens. This data can then be used for text analysis. " \ - "The output is a list of lists, each list representing all item tokens or " \ - "tokens per sentence." # description displayed in UI + description = ProcessorDescription( + title="Tokenise text", + tags=["text analysis", "preprocessing"], + description="Split item texts into separate tokens (words) for use in later text analysis. Optionally stem " + "or lemmatise tokens, remove stop words, and group tokens per item or per sentence and per " + "timeframe. The output is a list of lists, each list holding the tokens for one item or sentence.", + info=[ + "The TweetTokenizer works well for social media text; use the language-specific options for other text.", + ], + references=[ + "[NLTK tokenizer documentation](https://www.nltk.org/api/nltk.tokenize.html)", + "[Different types of tokenizers in NLTK](https://chendianblog.wordpress.com/2016/11/25/different-types-of-tokenizers-in-nltk/)", + "[Words in stopwords-iso word list](https://github.com/stopwords-iso/stopwords-iso/blob/master/stopwords-iso.json)", + "[Words in Google Books word list](https://github.com/hackerb9/gwordlist)", + "[Words in cracklib word list](https://github.com/cracklib/cracklib/tree/master/words)", + "[Words in OpenTaal word list](https://github.com/OpenTaal/opentaal-wordlist)", + ], + icon="pallet", + ) extension = "zip" # extension of result file, used internally and in UI - compatibility = Compatibility(extensions={"csv", "ndjson"}, preferred_followups=["collocations", "vectorise-tokens", "generate-embeddings", "tfidf", "topic-modeller", ]) + # a zip archive of data files + output = Archive() - references = [ - "[NLTK tokenizer documentation](https://www.nltk.org/api/nltk.tokenize.html)", - "[Different types of tokenizers in NLTK](https://chendianblog.wordpress.com/2016/11/25/different-types-of-tokenizers-in-nltk/)", - "[Words in stopwords-iso word list](https://github.com/stopwords-iso/stopwords-iso/blob/master/stopwords-iso.json)", - "[Words in Google Books word list](https://github.com/hackerb9/gwordlist)", - "[Words in cracklib word list](https://github.com/cracklib/cracklib/tree/master/words)", - "[Words in OpenTaal word list](https://github.com/OpenTaal/opentaal-wordlist)" - ] + compatibility = Compatibility(extensions={"csv", "ndjson"}, preferred_followups=["collocations", "vectorise-tokens", "generate-embeddings", "tfidf", "topic-modeller", ]) @classmethod def get_options(cls, parent_dataset=None, config=None): diff --git a/processors/text-analysis/top_vectors.py b/processors/text-analysis/top_vectors.py index a04d33d3c..0e9f04535 100644 --- a/processors/text-analysis/top_vectors.py +++ b/processors/text-analysis/top_vectors.py @@ -6,8 +6,9 @@ import json from common.lib.helpers import UserInput, convert_to_int -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -19,11 +20,16 @@ class VectorRanker(BasicProcessor): Rank vectors over time """ type = "vector-ranker" # job type ID - category = "Metrics" # category - title = "Extract top words" # title displayed in UI - description = "Ranks most used tokens per token set (overall or per timeframe). " \ - "Limited to 100 most-used tokens." # description displayed in UI + description = ProcessorDescription( + title="Extract top words", + tags=["counting", "text analysis", "time series"], + description="Rank the most-used tokens per token set, either overall or per timeframe. Return up to 100 of the " + "most-used tokens.", + icon="ranking-star", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) # Allow processor on token vectors compatibility = Compatibility(types={"vectorise-tokens"}, preferred_followups=["wordcloud"]) diff --git a/processors/text-analysis/topic_modeling.py b/processors/text-analysis/topic_modeling.py index d9070ab67..f90fc5a05 100644 --- a/processors/text-analysis/topic_modeling.py +++ b/processors/text-analysis/topic_modeling.py @@ -3,8 +3,9 @@ """ from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Archive from common.lib.exceptions import ProcessorInterruptedException import json @@ -25,21 +26,24 @@ class TopicModeler(BasicProcessor): Generate topic models """ type = "topic-modeller" # job type ID - category = "Text analysis" # category - title = "Generate topic models" # title displayed in UI - description = "Creates topic models per token set using Latent Dirichlet Allocation (LDA). " \ - "For a given number of topics, tokens are assigned a relevance weight per topic, " \ - "which can be used to find clusters of related words." # description displayed in UI + description = ProcessorDescription( + title="Generate topic models", + tags=["text analysis", "machine learning"], + description="Create topic models per token set using Latent Dirichlet Allocation (LDA). For a given number of topics, tokens are assigned a relevance weight per topic. Use these weights to find clusters of related words.", + references=[ + 'Blei, David M., Andrew Y. Ng, and Michael I. Jordan (2003). "Latent dirichlet allocation." the *Journal of machine Learning research* 3: 993-1022.', + 'Blei, David M. (2003). "Topic Modeling and Digital Humanities." *Journal of Digital Humanities* 2(1).', + ], + icon="boxes-stacked", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of data files + output = Archive() + # Allow processor on token sets compatibility = Compatibility(types={"tokenise-posts"}, preferred_followups=["document_count", "document_topic_matrix", "topic-model-words"]) - references = [ - 'Blei, David M., Andrew Y. Ng, and Michael I. Jordan (2003). "Latent dirichlet allocation." the *Journal of machine Learning research* 3: 993-1022.', - 'Blei, David M. (2003). "Topic Modeling and Digital Humanities." *Journal of Digital Humanities* 2(1).' - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/text-analysis/topic_words.py b/processors/text-analysis/topic_words.py index 91d37d382..7895468cc 100644 --- a/processors/text-analysis/topic_words.py +++ b/processors/text-analysis/topic_words.py @@ -3,8 +3,9 @@ """ from common.lib.helpers import UserInput -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException import pickle @@ -20,10 +21,15 @@ class TopicModelWordExtractor(BasicProcessor): Extracts topics per model and top associated words """ type = "topic-model-words" # job type ID - category = "Text analysis" # category - title = "Top words per topic" # title displayed in UI - description = "Creates a CSV file with the top tokens (words) per topic in the generated topic model, and their associated weights." # description displayed in UI + description = ProcessorDescription( + title="Top words per topic", + tags=["text analysis", "machine learning"], + description="Extract the top tokens (words) per topic from a topic model, along with their weights. Use the 'Tokens per topic' option to set how many words are kept for each topic.", + icon="ranking-star", + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on topic models compatibility = Compatibility(types={"topic-modeller"}, preferred_followups=["wordcloud"]) diff --git a/processors/text-analysis/vectorise.py b/processors/text-analysis/vectorise.py index 0b591afba..b3bac1359 100644 --- a/processors/text-analysis/vectorise.py +++ b/processors/text-analysis/vectorise.py @@ -5,8 +5,9 @@ import pickle import itertools -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Archive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -18,10 +19,15 @@ class Vectorise(BasicProcessor): Creates word vectors from tokens """ type = "vectorise-tokens" # job type ID - category = "Text analysis" # category - title = "Count words" # title displayed in UI - description = "Counts how often a token appears in the dataset. This creates a bag of words." # description displayed in UI + description = ProcessorDescription( + title="Count words", + tags=["counting", "text analysis"], + description="Count how often each token appears in the dataset, producing a bag of words per token set. The counts are sorted from most to least frequent.", + icon="list-ol", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of data files + output = Archive() # Allow processor on token sets compatibility = Compatibility(types={"tokenise-posts"}, preferred_followups=["vector-ranker"]) diff --git a/processors/text-analysis/vectorise_by_cat.py b/processors/text-analysis/vectorise_by_cat.py index eacf9a2a1..fafc053e5 100644 --- a/processors/text-analysis/vectorise_by_cat.py +++ b/processors/text-analysis/vectorise_by_cat.py @@ -5,8 +5,9 @@ import json import pickle -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.helpers import UserInput __author__ = "Dale Wahl" @@ -19,11 +20,17 @@ class VectoriseByCategory(BasicProcessor): Creates word vectors from tokens and organises them by category. """ type = "vectorise-tokens-by-category" # job type ID - category = "Text analysis" # category - title = "Count words by category" # title displayed in UI - description = "Counts all tokens per category." # description displayed in UI + description = ProcessorDescription( + title="Count words by category", + tags=["counting", "text analysis"], + description="Count how often each token appears within each category, using a chosen column of the parent dataset as the category. Optionally split multi-value categories by comma, separate counts per time interval, and filter by word or number of occurrences.", + icon="list-ol", + ) extension = "csv" # extension of result file, used internally and in UI + # a ranking table (date/item/value), so ranking visualisations can run on it + output = Table(columns={"date", "item", "value"}) + # Allow processor on token sets compatibility = Compatibility( types={"tokenise-posts"}, diff --git a/processors/twitter/aggregate_stats.py b/processors/twitter/aggregate_stats.py index c07cc9aac..795e5e079 100644 --- a/processors/twitter/aggregate_stats.py +++ b/processors/twitter/aggregate_stats.py @@ -8,6 +8,7 @@ from common.lib.helpers import UserInput, pad_interval, get_interval_descriptor from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility +from common.lib.outputs import Render, Table from common.lib.exceptions import ProcessorException __author__ = "Dale Wahl" @@ -21,10 +22,12 @@ class TwitterAggregatedStats(BasicProcessor): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-aggregated-stats" # job type ID - category = "Twitter analysis" # category + tags = ["counting", "metadata"] # tags title = "Aggregated statistics" # title displayed in UI description = "Group tweets by category and count tweets per timeframe and then calculate aggregate group statistics (i.e. min, max, average, Q1, median, Q3, and trimmed mean): number of tweets, urls, hashtags, mentions, etc. \nUse for example to find the distribution of the number of tweets per author and compare across time." # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) @@ -253,10 +256,12 @@ class TwitterAggregatedStatsVis(TwitterAggregatedStats): Collect Twitter statistics and create boxplots to visualise. """ type = "twitter-aggregated-stats-vis" # job type ID - category = "Twitter Analysis" # category + tags = ["statistics", "chart"] # category title = "Aggregated Statistics Visualization" # title displayed in UI description = "Gathers Aggregated Statistics data and creates Box Plots visualising the spread of intervals. A large number of intervals will not properly display. " # description displayed in UI extension = "png" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render("png") references = [ "[matplotlib.pyplot.boxplot documentation](https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.boxplot.html)" diff --git a/processors/twitter/base_twitter_stats.py b/processors/twitter/base_twitter_stats.py index ab3d37e0a..b18c48b5b 100644 --- a/processors/twitter/base_twitter_stats.py +++ b/processors/twitter/base_twitter_stats.py @@ -20,7 +20,7 @@ class TwitterStatsBase(BasicProcessor): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-stats-base" # job type ID - category = "Twitter analysis" # category + tags = ["counting", "metadata"] # tags title = "Twitter base statistics" # title displayed in UI description = "This is a class to help other twitter classes" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI diff --git a/processors/twitter/custom_stats.py b/processors/twitter/custom_stats.py index e37cf73b9..7bd4842f5 100644 --- a/processors/twitter/custom_stats.py +++ b/processors/twitter/custom_stats.py @@ -5,6 +5,7 @@ from common.lib.helpers import UserInput from processors.twitter.base_twitter_stats import TwitterStatsBase from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -17,10 +18,12 @@ class TwitterCustomStats(TwitterStatsBase): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-1-custom-stats" # job type ID - category = "Twitter analysis" # category + tags = ["counting", "metadata"] # tags title = "Custom statistics" # title displayed in UI description = "Group tweets by category and count tweets per timeframe to collect aggregate group statistics.\nFor retweets and quotes, hashtags, mentions, URLs, and images from the original tweet are included in the retweet/quote. Data on public metrics (e.g., number of retweets or likes of tweets) are as of the time the data was collected." # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/twitter/hashtag_stats.py b/processors/twitter/hashtag_stats.py index 52bff79f1..f206221bc 100644 --- a/processors/twitter/hashtag_stats.py +++ b/processors/twitter/hashtag_stats.py @@ -4,6 +4,7 @@ from common.lib.helpers import UserInput from processors.twitter.base_twitter_stats import TwitterStatsBase from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -16,10 +17,12 @@ class TwitterHashtagStats(TwitterStatsBase): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-hashtag-stats" # job type ID - category = "Twitter analysis" # category + tags = ["hashtags", "counting", "statistics"] # tags title = "Hashtag statistics" # title displayed in UI description = "Lists by hashtag how many tweets contain hashtags, how many times those tweets have been retweeted/replied to/liked/quoted, and information about unique users and hashtags used alongside each hashtag.\nFor retweets and quotes, hashtags from the original tweet are included in the retweet/quote." # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/twitter/identical_tweets.py b/processors/twitter/identical_tweets.py index 9dc923fe5..811458a94 100644 --- a/processors/twitter/identical_tweets.py +++ b/processors/twitter/identical_tweets.py @@ -4,6 +4,7 @@ from common.lib.helpers import UserInput from processors.twitter.base_twitter_stats import TwitterStatsBase from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -16,11 +17,14 @@ class TwitterIdenticalTweets(TwitterStatsBase): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-identical-tweets" # job type ID - category = "Twitter analysis" # category + tags = ["counting"] # tags title = "Identical tweet frequency" # title displayed in UI description = "Groups tweets by text and counts the number of times they have been (re)tweeted indentically." # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/twitter/mention_export.py b/processors/twitter/mention_export.py index 834ee9287..971c76cfc 100644 --- a/processors/twitter/mention_export.py +++ b/processors/twitter/mention_export.py @@ -5,6 +5,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorException, ProcessorInterruptedException __author__ = "Dale Wahl" @@ -18,10 +19,12 @@ class TwitterMentionsExport(BasicProcessor): Collect User stats as both author and mention. """ type = "twitter-mentions-export" # job type ID - category = "Twitter analysis" # category + tags = ["metadata"] # tags title = "Mentions export" # title displayed in UI description = "Identifies mentions types and creates mentions table (tweet id, from author id, from username, to user id, to username, mention type)" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Twitter/X (API v2) datasets compatibility = Compatibility(types={"twitterv2-search"}) @@ -139,10 +142,12 @@ class TCATMentionsExport(BasicProcessor): Collect User stats as both author and mention. """ type = "tcat-mentions-export" # job type ID - category = "Twitter Analysis" # category + tags = ["metadata", "counting"] # category title = "Mentions Export" # title displayed in UI description = "Identifies mentions types and creates mentions table (tweet id, from author id, from username, to username)" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on imported TCAT datasets compatibility = Compatibility(types={"dmi-tcat-search"}) diff --git a/processors/twitter/source_stats.py b/processors/twitter/source_stats.py index e5f25e00c..723feaa8c 100644 --- a/processors/twitter/source_stats.py +++ b/processors/twitter/source_stats.py @@ -4,6 +4,7 @@ from common.lib.helpers import UserInput from processors.twitter.base_twitter_stats import TwitterStatsBase from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -16,11 +17,14 @@ class TwitterHashtagStats(TwitterStatsBase): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-source-stats" # job type ID - category = "Twitter analysis" # category + tags = ["counting", "statistics", "metadata"] # tags title = "Source statistics" # title displayed in UI description = "Lists by source of tweet how many tweets contain hashtags, how many times those tweets have been retweeted/replied to/liked/quoted, and information about unique users and hashtags used alongside each hashtag.\nFor retweets and quotes, hashtags from the original tweet are included in the retweet/quote." # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/twitter/twitter_stats.py b/processors/twitter/twitter_stats.py index 4fe18d6df..454528fd1 100644 --- a/processors/twitter/twitter_stats.py +++ b/processors/twitter/twitter_stats.py @@ -4,6 +4,7 @@ from common.lib.helpers import UserInput from processors.twitter.base_twitter_stats import TwitterStatsBase from common.lib.compatibility import Compatibility +from common.lib.outputs import Table __author__ = "Dale Wahl" __credits__ = ["Dale Wahl"] @@ -16,10 +17,12 @@ class TwitterStats(TwitterStatsBase): Collect Twitter statistics. Built to emulate TCAT statistic. """ type = "twitter-0-stats" # job type ID - category = "Twitter analysis" # category + tags = ["statistics", "counting", "metadata"] # tags title = "Twitter statistics" # title displayed in UI description = "Contains the number of tweets, number of tweets with links, number of tweets with hashtags, number of tweets with mentions, number of retweets, and number of replies" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/twitter/user_stats_individual.py b/processors/twitter/user_stats_individual.py index ee9cfdf91..b1a21532b 100644 --- a/processors/twitter/user_stats_individual.py +++ b/processors/twitter/user_stats_individual.py @@ -4,6 +4,7 @@ from common.lib.helpers import UserInput from processors.twitter.base_twitter_stats import TwitterStatsBase from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorException __author__ = "Dale Wahl" @@ -17,10 +18,12 @@ class TwitterStats(TwitterStatsBase): Collect Twitter statistics. Build to emulate TCAT statistic. """ type = "twitter-user-stats-individual" # job type ID - category = "Twitter analysis" # category + tags = ["counting", "statistics", "authors", "metadata"] # tags title = "Individual user statistics" # title displayed in UI description = "Lists users and their number of tweets, number of followers, number of friends, how many times they are listed, their UTC time offset, whether the user has a verified account and how many times they appear in the data set." # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/twitter/user_visibility.py b/processors/twitter/user_visibility.py index 36f458a15..1c4ef6bb8 100644 --- a/processors/twitter/user_visibility.py +++ b/processors/twitter/user_visibility.py @@ -6,6 +6,7 @@ from common.lib.helpers import get_interval_descriptor from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput @@ -20,11 +21,14 @@ class TwitterUserVisibility(BasicProcessor): Collect User stats as both author and mention. """ type = "twitter-user-visibility" # job type ID - category = "Twitter analysis" # category + tags = ["counting", "authors", "metadata"] # tags title = "User visibility" # title displayed in UI description = "Collects usernames and totals how many tweets are authored by the user and how many tweets mention the user" # description displayed in UI extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() + # Allow processor on Twitter/X datasets (API v2 or imported TCAT) compatibility = Compatibility(types={"twitterv2-search", "dmi-tcat-search"}) diff --git a/processors/visualisation/download-telegram-images.py b/processors/visualisation/download-telegram-images.py index d69c74f89..e67cd547e 100644 --- a/processors/visualisation/download-telegram-images.py +++ b/processors/visualisation/download-telegram-images.py @@ -6,9 +6,11 @@ for video documents and webpage previews. """ from common.lib.helpers import UserInput +from backend.lib.processor import ProcessorDescription from processors.visualisation.download_images import ImageDownloader from processors.visualisation.download_telegram_videos import TelegramVideoDownloader from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -25,12 +27,22 @@ class TelegramImageDownloader(TelegramVideoDownloader): `TelegramVideoDownloader`; only the media-specific bits are overridden. """ type = "image-downloader-telegram" # job type ID - category = "Visual" - title = "Download Telegram images" - description = "Download images and store in a ZIP file. Downloads through the Telegram API might take a while. " \ - "Note that not always all images can be retrieved. A JSON metadata file is included in the output " \ - "archive." + description = ProcessorDescription( + title="Download Telegram images", + tags=["visual", "download media", "external service"], + description="Download the images attached to Telegram messages and store them in a ZIP file. Video and link preview thumbnails can optionally be included.", + info=[ + "A JSON metadata file recording the download outcome per message is included in the archive." + ], + warnings=[ + "Images are fetched through the Telegram API, so this can take a long time and some images may fail to download.", + "Channels with 'Restrict Saving Content' enabled will refuse the download; those images cannot be retrieved.", + ], + icon="images", + ) extension = "zip" + # a zip archive of media files + output = MediaArchive(media="image") media_type = "image" # coarse map spec; is_compatible_with (below) is the runtime truth (Telegram API creds) diff --git a/processors/visualisation/download_images.py b/processors/visualisation/download_images.py index bd13b21c7..401539d82 100644 --- a/processors/visualisation/download_images.py +++ b/processors/visualisation/download_images.py @@ -11,10 +11,11 @@ from requests.structures import CaseInsensitiveDict from common.lib.helpers import UserInput, url_to_filename -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from backend.lib.proxied_requests import FailedProxiedRequest from common.lib.exceptions import ProcessorInterruptedException, FourcatException from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -34,17 +35,24 @@ class ImageDownloader(BasicProcessor): """ type = "image-downloader" # job type ID - category = "Visual" # category - title = "Download images" # title displayed in UI - description = ( - "Download images and store in a a ZIP file. May take a while to complete as images are retrieved " - "externally. Note that not always all images can be saved. For imgur galleries, only the first " - "image is saved. For animations (GIFs), only the first frame is saved if available. A JSON metadata file " - "is included in the output archive." - ) # description displayed in UI + description = ProcessorDescription( + title="Download images", + tags=["visual", "download media", "urls"], + description="Extract image URLs from a chosen column and download the images into a ZIP file. For Imgur galleries only the first image is saved, and for animated GIFs only the first frame.", + info=[ + "A JSON metadata file recording the download outcome per image is included in the archive." + ], + warnings=[ + "Images are retrieved from external servers, so this can take a long time and some images may fail to download.", + ], + icon="images", + ) extension = "zip" # extension of result file, used internally and in UI media_type = "image" # media type of the dataset + # a zip archive of media files + output = MediaArchive(media="image") + # Shared list -- other download_* processors reuse this as ImageDownloader.followups # (and preferred_followups below reuses it), so it stays a named attribute. followups = [ diff --git a/processors/visualisation/download_telegram_files.py b/processors/visualisation/download_telegram_files.py index 20870953d..ece4e7977 100644 --- a/processors/visualisation/download_telegram_files.py +++ b/processors/visualisation/download_telegram_files.py @@ -13,8 +13,10 @@ from telethon import utils as telethon_utils from common.lib.helpers import UserInput +from backend.lib.processor import ProcessorDescription from processors.visualisation.download_telegram_videos import TelegramVideoDownloader from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -33,14 +35,22 @@ class TelegramFileDownloader(TelegramVideoDownloader): that key off "video" or "image" will not pick this up. """ type = "file-downloader-telegram" # job type ID - category = "Visual" - title = "Download Telegram files" - description = ("Download audio, documents, stickers, and other non-video / non-photo file " - "attachments and store in a ZIP file. Note that not every channel allows " - "downloads; if 'chat_noforwards' is set in the source dataset, the channel " - "owner has asked clients not to save its content and Telegram will refuse " - "the file fetch.") + description = ProcessorDescription( + title="Download Telegram files", + tags=["visual", "download media", "external service"], + description="Download the audio, documents, stickers, and other non-video, non-photo file attachments of Telegram messages and store them in a ZIP file.", + info=[ + "A JSON metadata file recording the download outcome per message is included in the archive." + ], + warnings=[ + "Files are fetched through the Telegram API, so this can take a long time and some files may fail to download.", + "Channels with 'Restrict Saving Content' enabled will refuse the download; those files cannot be retrieved.", + ], + icon="file", + ) extension = "zip" + # a zip archive of media files + output = MediaArchive(media="file") media_type = "file" # coarse map spec; is_compatible_with (below) is the runtime truth (Telegram API creds). diff --git a/processors/visualisation/download_telegram_videos.py b/processors/visualisation/download_telegram_videos.py index a463d0126..045c8522f 100644 --- a/processors/visualisation/download_telegram_videos.py +++ b/processors/visualisation/download_telegram_videos.py @@ -16,13 +16,14 @@ class attributes to switch behavior for a different media type. from telethon import TelegramClient from telethon.errors import FloodError, BadRequestError -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.exceptions import ProcessorInterruptedException from datasources.telegram.search_telegram import SearchTelegram from processors.visualisation.download_videos import VideoDownloaderPlus from common.lib.helpers import UserInput, timify from common.lib.dataset import DataSet from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters", "Dale Wahl"] @@ -38,15 +39,26 @@ class TelegramVideoDownloader(BasicProcessor): Also serves as the base class for `TelegramImageDownloader`. """ type = "video-downloader-telegram" # job type ID - category = "Visual" # category - title = "Download Telegram videos" # title displayed in UI - description = "Download videos and store in a ZIP file. Downloads through the Telegram API might take a while. " \ - "Note that not always all videos can be retrieved. A JSON metadata file is included in the output " \ - "archive." # description displayed in UI + description = ProcessorDescription( + title="Download Telegram videos", + tags=["visual", "download media", "external service"], + description="Download the videos attached to Telegram messages and store them in a ZIP file.", + info=[ + "A JSON metadata file recording the download outcome per message is included in the archive." + ], + warnings=[ + "Videos are fetched through the Telegram API, so this can take a long time and some videos may fail to download.", + "Channels with 'Restrict Saving Content' enabled will refuse the download; those videos cannot be retrieved.", + ], + icon="film", + ) extension = "zip" # extension of result file, used internally and in UI media_type = "video" # media type of the result + # a zip archive of media files + output = MediaArchive(media="video") flawless = True + # coarse map spec; is_compatible_with (below) is the runtime truth -- it also checks the # source dataset carries Telegram API credentials, which are read from the dataset compatibility = Compatibility(types={"telegram-search"}, required_settings={"video-downloader-telegram.allow_videos"}, preferred_followups=VideoDownloaderPlus.followups) diff --git a/processors/visualisation/download_tiktok.py b/processors/visualisation/download_tiktok.py index 158637da1..f0a0ec0e8 100644 --- a/processors/visualisation/download_tiktok.py +++ b/processors/visualisation/download_tiktok.py @@ -15,8 +15,9 @@ from datasources.tiktok_urls.search_tiktok_urls import TikTokScraper from datasources.tiktok.search_tiktok import SearchTikTok as SearchTikTokByImport from processors.visualisation.download_images import ImageDownloader -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive __author__ = "Dale Wahl" @@ -26,10 +27,19 @@ class TikTokImageDownloader(BasicProcessor): type = "image-downloader-tiktok" # job type ID - category = "Visual" # category - title = "Download TikTok images" # title displayed in UI - description = "Downloads video/music thumbnails for TikTok; refreshes TikTok data if URLs have expired" + description = ProcessorDescription( + title="Download TikTok images", + tags=["visual", "download media"], + description="Download video thumbnails, music thumbnails, or author avatars for TikTok posts. Refreshes TikTok data through the scraper when a thumbnail URL has expired. Saves the images to a zip archive.", + info=[ + "Choose which image to download per post: video thumbnail, music thumbnail, or author avatar.", + "A JSON metadata file recording the download outcome per post is included in the archive." + ], + icon="images", + ) extension = "zip" + # a zip archive of media files + output = MediaArchive(media="image") media_type = "image" # Allow processor on TikTok datasets diff --git a/processors/visualisation/download_tiktok_video.py b/processors/visualisation/download_tiktok_video.py index 97a809602..b280f8db0 100644 --- a/processors/visualisation/download_tiktok_video.py +++ b/processors/visualisation/download_tiktok_video.py @@ -10,9 +10,10 @@ from backend.lib.proxied_requests import FailedProxiedRequest from common.lib.helpers import UserInput from processors.visualisation.download_videos import VideoDownloaderPlus -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from datasources.tiktok_urls.search_tiktok_urls import TikTokScraper from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive, Table class TikTokVideoDownloader(ProcessorPreset): """ @@ -21,11 +22,19 @@ class TikTokVideoDownloader(ProcessorPreset): This is a Preset that runs the VideoDownloaderPlus with set parameters """ type = "video-downloader-tiktok" # job type ID - category = "Visual" # category - title = "Download TikTok videos" # title displayed in UI - description = "Downloads full videos for TikTok" + description = ProcessorDescription( + title="Download TikTok videos", + tags=["video", "download media"], + description="Download the full videos for TikTok posts and store them in a zip archive. Retrieves fresh video URLs before downloading, so it also works on older datasets whose links have expired.", + info=[ + "A JSON metadata file recording the download outcome per post is included in the archive." + ], + icon="film", + ) extension = "zip" media_type = "video" + # its pipeline downloads the videos into a zip archive + output = MediaArchive(media="video") # coarse map spec; is_compatible_with (below) is the runtime truth -- it also accepts # tiktok uploads, which depends on the dataset label and can't be declared statically @@ -120,11 +129,15 @@ class TikTokVideoMetadata(BasicProcessor): requests to download videos. Otherwise all videos would be sent to YT-DLP (not currently asynchronous). """ type = "tiktok-video-downloader-metadata" # job type ID - category = "Visual" # category - title = "TikTok Video URLs Updater" # title displayed in UI - description = "Retrieves updated video URLs from TikTok" + description = ProcessorDescription( + title="Update TikTok video URLs", + tags=["urls", "internal"], + description="Retrieve fresh video download URLs from TikTok for a set of post IDs. Used internally as a step before downloading TikTok videos.", + ) extension = "csv" media_type = "text" + # a derived table + output = Table() consecutive_failures = None diff --git a/processors/visualisation/download_videos.py b/processors/visualisation/download_videos.py index 552eb4be8..b074e3a7a 100644 --- a/processors/visualisation/download_videos.py +++ b/processors/visualisation/download_videos.py @@ -18,9 +18,10 @@ from yt_dlp import DownloadError from yt_dlp.utils import ExistingVideoReached -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from backend.lib.proxied_requests import FailedProxiedRequest from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive from common.lib.dataset import DataSet from common.lib.exceptions import ProcessorInterruptedException, ProcessorException, DataSetException from common.lib.helpers import UserInput, sets_to_lists, url_to_filename @@ -89,13 +90,29 @@ class VideoDownloaderPlus(BasicProcessor): which attempts to keep up with a plethora of sites: https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md """ type = "video-downloader" # job type ID - category = "Visual" # category - title = "Download videos" # title displayed in UI - description = "Download videos from URLs and store in a zip file. May take a while to complete as videos are " \ - "retrieved externally." # description displayed in UI + description = ProcessorDescription( + title="Download videos", + tags=["video", "download media", "urls"], + description="Find video links in a column and download the videos to a zip archive. Tries a direct download first and falls back to yt-dlp, which supports YouTube and many other video hosts.", + info=[ + "A JSON metadata file recording the download outcome per video is included in the archive.", + ], + references=[ + "[yt-dlp python package](https://github.com/yt-dlp/yt-dlp/#readme)", + "[Supported sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)", + ], + warnings=[ + "This can be slow and produce large datasets, as each video is retrieved from an external site.", + "Downloads only directly linked videos unless indirect links are enabled by an administrator." + ], + icon="film", + ) extension = "zip" # extension of result file, used internally and in UI media_type = "video" # media type of the processor + # a zip archive of media files + output = MediaArchive(media="video") + # Shared list -- other download_* processors reuse this as VideoDownloaderPlus.followups # (and preferred_followups below reuses it), so it stays a named attribute. followups = ["audio-extractor", "metadata-viewer", "video-scene-detector", "preset-scene-timelines", "video-stack", "preset-video-hashes", "video-hasher-1", "video-frames"] @@ -103,11 +120,6 @@ class VideoDownloaderPlus(BasicProcessor): # any collector's csv/ndjson output (except sources with their own downloaders), plus the tiktok-metadata helper compatibility = Compatibility(is_collector=True, types={"tiktok-video-downloader-metadata"}, excluded_types={"tiktok-search", "tiktok-urls-search", "telegram-search"}, extensions={"csv", "ndjson"}, preferred_followups=followups) - references = [ - "[YT-DLP python package](https://github.com/yt-dlp/yt-dlp/#readme)", - "[Supported sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)", - ] - known_channels = ['youtube.com/c/', 'youtube.com/channel/'] # Some datasets have known mixed media types; do not stop due to many "Not a video" errors diff --git a/processors/visualisation/histwords.py b/processors/visualisation/histwords.py index 1fbe4302c..1a1da10b5 100644 --- a/processors/visualisation/histwords.py +++ b/processors/visualisation/histwords.py @@ -11,8 +11,9 @@ from gensim.models import KeyedVectors -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas, convert_to_float from common.lib.exceptions import ProcessorInterruptedException @@ -37,22 +38,27 @@ class HistWordsVectorSpaceVisualiser(BasicProcessor): dimensions, plots them, and highlights given words and their neighbours. """ type = "histwords-vectspace" # job type ID - category = "Visual" # category - title = "Chart diachronic nearest neighbours" # title displayed in UI - description = "Visualise nearest neighbours of a given query across all models and show the closest neighbours per model in one combined graph. Based on the 'HistWords' algorithm by Hamilton et al." # description displayed in UI + description = ProcessorDescription( + title="Chart diachronic nearest neighbours", + tags=["text analysis", "chart", "machine learning"], + description="Chart how the nearest neighbours of a query word shift across a set of word embedding models. Reduces the word vectors to two dimensions with t-SNE, PCA, or truncated SVD, plots the neighbours per model, and links each query word's positions across models. Based on the 'HistWords' algorithm by Hamilton et al.", + references=[ + "HistWords: [Hamilton, W. L., Leskovec, J., & Jurafsky, D. (2016). Diachronic word embeddings reveal statistical laws of semantic change. *arXiv preprint** arXiv:1605.09096.](https://arxiv.org/pdf/1605.09096.pdf)", + "HistWords: [William L. Hamilton, Jure Leskovec, and Dan Jurafsky. HistWords: Word Embeddings for Historical Text](https://nlp.stanford.edu/projects/histwords/)", + "t-SNE: [Maaten, L. V. D., & Hinton, G. (2008). Visualizing data using t-SNE. *Journal of machine learning research*, 9(Nov), 2579-2605.](https://www.jmlr.org/papers/v9/vandermaaten08a.html)", + "PCA: [Joliffe, I. T., & Morgan, B. J. T. (1992). Principal component analysis and exploratory factor analysis. *Statistical methods in medical research*, 1(1), 69-95.](https://journals.sagepub.com/doi/abs/10.1177/096228029200100105)", + "Truncated SVD: [Manning, C. D., Raghavan, P., & Schütze, H. (2008). Matrix decompositions and latent semantic indexing. *Introduction to information retrieval*, 403-417.](http://nlp.stanford.edu/IR-book/pdf/18lsi.pdf)", + ], + icon="diagram-project", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() + # Allow processor on word embedding models compatibility = Compatibility(types={"generate-embeddings"}) - references = [ - "HistWords: [Hamilton, W. L., Leskovec, J., & Jurafsky, D. (2016). Diachronic word embeddings reveal statistical laws of semantic change. *arXiv preprint** arXiv:1605.09096.](https://arxiv.org/pdf/1605.09096.pdf)", - "HistWords: [William L. Hamilton, Jure Leskovec, and Dan Jurafsky. HistWords: Word Embeddings for Historical Text](https://nlp.stanford.edu/projects/histwords/)", - "t-SNE: [Maaten, L. V. D., & Hinton, G. (2008). Visualizing data using t-SNE. *Journal of machine learning research*, 9(Nov), 2579-2605.](https://www.jmlr.org/papers/v9/vandermaaten08a.html)", - "PCA: [Joliffe, I. T., & Morgan, B. J. T. (1992). Principal component analysis and exploratory factor analysis. *Statistical methods in medical research*, 1(1), 69-95.](https://journals.sagepub.com/doi/abs/10.1177/096228029200100105)", - "Truncated SVD: [Manning, C. D., Raghavan, P., & Schütze, H. (2008). Matrix decompositions and latent semantic indexing. *Introduction to information retrieval*, 403-417.](http://nlp.stanford.edu/IR-book/pdf/18lsi.pdf)" - ] - @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: """ diff --git a/processors/visualisation/image_category_wall.py b/processors/visualisation/image_category_wall.py index 7dd9df5c2..b9ba810d9 100644 --- a/processors/visualisation/image_category_wall.py +++ b/processors/visualisation/image_category_wall.py @@ -15,9 +15,10 @@ from PIL import Image from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.exceptions import ProcessorInterruptedException from common.lib.compatibility import Compatibility +from common.lib.outputs import Render __author__ = "Dale Wahl" __credits__ = ["Dale Wahl", "Stijn Peeters"] @@ -32,11 +33,17 @@ class ImageCategoryWallGenerator(BasicProcessor): Create an image wall from the top images in the dataset """ type = "image-category-wall" # job type ID - category = "Visual" # category - title = "Visualise images by category" # title displayed in UI - description = "Combine images into a single image arranged by category" # description displayed in UI + description = ProcessorDescription( + title="Visualise images by category", + tags=["visual", "chart"], + description="Arrange downloaded images into a single wall grouped by the values in a category column. Each category becomes a row, images are sorted within it, and numeric categories are grouped into ranges. Runs on datasets that pair images with a category, such as image classification results.", + icon="panorama", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() + # image-category, image-downloader, or video-hash datasets (except screenshot downloads) compatibility = Compatibility(type_prefixes={"image-to-categories", "image-downloader", "video-hasher-1", "video-hash-similarity-matrix"}, excluded_types={"image-downloader-screenshots-search"}) diff --git a/processors/visualisation/image_wall.py b/processors/visualisation/image_wall.py index ebbb25f2b..7ac4f6fa4 100644 --- a/processors/visualisation/image_wall.py +++ b/processors/visualisation/image_wall.py @@ -5,6 +5,8 @@ from sklearn.cluster import KMeans from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility, ExecutableSibling +from common.lib.outputs import Render +from backend.lib.processor import ProcessorDescription import colorsys import copy @@ -25,13 +27,18 @@ class ImageWallGenerator(VideoWallGenerator): images just as well as videos! """ type = "image-wall" - category = "Visual" - title = "Image wall" - description = "Put all images in a single combined image, side by side. Images can be sorted and resized." + description = ProcessorDescription( + title="Image wall", + tags=["visual", "chart"], + description="Combine all images into a single wall, placed side by side. Images can be sorted by dominant or average colour and resized to a set height. Uses ffmpeg to assemble the wall, and can take the first frame of each video when run on a video dataset.", + icon="panorama", + ) extension = "png" + # a rendered image, no column table + output = Render("png") # Allow on image/video datasets when ffmpeg and ffprobe are available - compatibility = Compatibility(media_types={"video", "image"}, type_prefixes={"image-downloader"}, types={"video-frames"}, required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))}) + compatibility = Compatibility(extensions={"zip"}, media_types={"video", "image"}, type_prefixes={"image-downloader"}, types={"video-frames"}, required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))}) @classmethod def get_options(cls, parent_dataset=None, config=None): diff --git a/processors/visualisation/image_wall_w_text.py b/processors/visualisation/image_wall_w_text.py index 37d5aa24a..3e0e70c63 100644 --- a/processors/visualisation/image_wall_w_text.py +++ b/processors/visualisation/image_wall_w_text.py @@ -16,9 +16,10 @@ from PIL import Image, ImageOps, UnidentifiedImageError from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.exceptions import ProcessorInterruptedException from common.lib.compatibility import Compatibility +from common.lib.outputs import Render __author__ = "Dale Wahl" __credits__ = ["Dale Wahl", "Stijn Peeters"] @@ -31,11 +32,17 @@ class ImageTextWallGenerator(BasicProcessor): Image wall with text generator """ type = "image-text-wall" # job type ID - category = "Visual" # category - title = "Image wall with captions" # title displayed in UI - description = "Combine images into a single image including text" # description displayed in UI + description = ProcessorDescription( + title="Image wall with captions", + tags=["visual", "chart"], + description="Combine downloaded images and their captions into a single wall-like image. Each image is tiled and labelled with its caption text below it. Works with datasets that pair images and captions, such as generated images with prompts or images with their extracted text.", + icon="panorama", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() + image_datasets = ["image-downloader", "video-hasher-1", "media-import-search"] caption_datasets = ["image-captions", "text-from-images", "llm-prompter"] combined_dataset = ["image-downloader-stable-diffusion"] diff --git a/processors/visualisation/isoviz.py b/processors/visualisation/isoviz.py index a1199abda..2c48bbf11 100644 --- a/processors/visualisation/isoviz.py +++ b/processors/visualisation/isoviz.py @@ -4,9 +4,10 @@ import csv import re -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput, convert_to_int, pad_interval, get_4cat_canvas from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from calendar import month_abbr from math import sin, cos, tan, degrees, radians, copysign @@ -31,10 +32,15 @@ class IsometricMultigraphRenderer(BasicProcessor): attributes in a data set over time. """ type = "render-graphs-isometric" # job type ID - category = "Visual" # category - title = "Side-by-side area graphs" # title displayed in UI - description = "Generate area graphs showing prevalence per item over time. These are visualised side-by-side on an isometric plane for easy comparison." # description displayed in UI + description = ProcessorDescription( + title="Side-by-side area graphs", + tags=["time series", "chart"], + description="Generate an area graph per item showing its prevalence over time. The graphs are projected side by side on an isometric plane for comparison. Values can optionally be normalised to 0-100% and smoothed into curves.", + icon="chart-area", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() # rankable datasets with a single value per item (multiple_items=False) compatibility = Compatibility(rankable=True, rankable_multiple_items=False) diff --git a/processors/visualisation/rankflow.py b/processors/visualisation/rankflow.py index 99397321d..60aec93d3 100644 --- a/processors/visualisation/rankflow.py +++ b/processors/visualisation/rankflow.py @@ -4,10 +4,11 @@ import colorsys import csv -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput, get_4cat_canvas from common.lib.exceptions import ProcessorInterruptedException from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from svgwrite.shapes import Rect from svgwrite.path import Path @@ -34,21 +35,22 @@ class RankFlowRenderer(BasicProcessor): """ type = "render-rankflow" # job type ID - category = "Visual" # category - title = "RankFlow diagram" # title displayed in UI - description = ( - "Create a diagram showing changes in prevalence over time for ranked lists (following " - "Bernhard Rieder's RankFlow." - ) # description displayed in UI + description = ProcessorDescription( + title="RankFlow diagram", + tags=["time series", "counting", "chart"], + description="Create an interactive RankFlow diagram showing how the rank and prevalence of items change over time. Each period is drawn as a column of ranked boxes, connected by flows to the same items in adjacent periods. Boxes can be coloured and sized by value or by change between periods.", + references=[ + "[Rieder, B. RankFlow. *The Politics of Systems*](https://labs.polsys.net/tools/rankflow/)", + ], + icon="shuffle", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() # rankable datasets, including multi-column rankings (e.g. top vectors per interval) compatibility = Compatibility(rankable=True) - references = [ - "[Rieder, B. RankFlow. *The Politics of Systems*](https://labs.polsys.net/tools/rankflow/)" - ] - # 25-colour palette via https://medialab.github.io/iwanthue/ palette = [ [0.081, 1.0, 0.902], diff --git a/processors/visualisation/vector_histogram.py b/processors/visualisation/vector_histogram.py index e01f2ba22..e72d98a90 100644 --- a/processors/visualisation/vector_histogram.py +++ b/processors/visualisation/vector_histogram.py @@ -10,9 +10,10 @@ from svgwrite.path import Path as SVGPath from svgwrite.text import Text -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput, pad_interval, get_4cat_canvas from common.lib.compatibility import Compatibility +from common.lib.outputs import Render __author__ = "Stijn Peeters" __credits__ = ["Stijn Peeters"] @@ -24,11 +25,17 @@ class SVGHistogramRenderer(BasicProcessor): Generate activity histogram """ type = "histogram" # job type ID - category = "Visual" # category - title = "Histogram" # title displayed in UI - description = "Generates a histogram from time frequencies." # description displayed in UI + description = ProcessorDescription( + title="Histogram", + tags=["time series", "counting", "chart"], + description="Generate a bar chart showing how a value changes over time, using an over-time frequency analysis as input. Each interval becomes one bar, sized by its value. Intervals without a date, such as \"unknown_date\", are dropped.", + icon="square-poll-vertical", + ) extension = "svg" + # a rendered image, no column table + output = Render() + # rankable datasets with a single value per item (multiple_items=False) compatibility = Compatibility(rankable=True, rankable_multiple_items=False) diff --git a/processors/visualisation/video_frames.py b/processors/visualisation/video_frames.py index 9a24621fa..2ad047303 100644 --- a/processors/visualisation/video_frames.py +++ b/processors/visualisation/video_frames.py @@ -7,8 +7,9 @@ import shutil import oslex -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility, is_executable +from common.lib.outputs import MediaArchive from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from processors.visualisation.download_videos import VideoDownloaderPlus @@ -26,13 +27,19 @@ class VideoFrames(BasicProcessor): Uses ffmpeg to extract a certain number of frames per second at different sizes and saves them in an archive. """ type = "video-frames" # job type ID - category = "Visual" # category - title = "Extract frames from videos" # title displayed in UI - description = "Extract frames from videos" # description displayed in UI + description = ProcessorDescription( + title="Extract frames from videos", + tags=["video", "visual", "download media"], + description="Use ffmpeg to extract still frames from each video and save them as images in an archive. You can set how many frames to capture per second and resize them to a fixed dimension. Use a frame interval of 0 to capture only the first frame of each video.", + icon="photo-film", + ) extension = "zip" # extension of result file, used internally and in UI + media_type = "image" # the extracted frames are images; set so the map and runtime agree + # a zip archive of image files + output = MediaArchive(media="image") # Allow on video datasets when ffmpeg is available - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}, preferred_followups=["video-timelines"] + VideoDownloaderPlus.followups) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}, preferred_followups=["video-timelines"] + VideoDownloaderPlus.followups) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: diff --git a/processors/visualisation/video_hasher.py b/processors/visualisation/video_hasher.py index 8271e7b69..9dc53aff8 100644 --- a/processors/visualisation/video_hasher.py +++ b/processors/visualisation/video_hasher.py @@ -14,9 +14,10 @@ from videohash import VideoHash from videohash.exceptions import FFmpegNotFound, FFmpegFailedToExtractFrames -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from backend.lib.preset import ProcessorAdvancedPreset from common.lib.compatibility import Compatibility, is_executable +from common.lib.outputs import Network, MediaArchive, Table, Delegated from common.lib.exceptions import ProcessorInterruptedException, ProcessorException from common.lib.user_input import UserInput @@ -31,13 +32,21 @@ class VideoHasherPreset(ProcessorAdvancedPreset): Run processor pipeline to create video hashes """ type = "preset-video-hashes" # job type ID - category = "Visual" # category. 'Combined processors' are always listed first in the UI. - title = "Create video hashes to identify near duplicate videos" # title displayed in UI - description = "Creates video hashes (64 bits/identifiers) to identify near duplicate videos in a dataset based on hash similarity. Uses video only. This process can take a long time depending on video length, amount, and frames per second." + description = ProcessorDescription( + title="Find near-duplicate videos with hashes", + tags=["visual", "video", "networks"], + description="Run the full pipeline to detect near-duplicate videos: extract frames, build a 64-bit hash per video, and produce a similarity network and matrix. Two videos are linked when their hashes are at least the chosen percentage similar. Only the video content is used.", + warnings=[ + "This can take a very long time depending on the number of videos, their length, and the frames per second used.", + ], + icon="square-binary", + ) extension = "gexf" + # a preset; its output is its last step's + output = Delegated() # video datasets, when ffmpeg is available - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}) @classmethod def get_options(cls, parent_dataset=None, config=None): @@ -128,14 +137,21 @@ class VideoHasher(BasicProcessor): "scene" changes) and have lead to unwanted collision in tests. """ type = "video-hasher-1" # job type ID - category = "Visual" # category - title = "Create video collages" # title displayed in UI - description = "Creates collages from video frames. Can be used to create video hashes to detect similar videos." # description displayed in UI + description = ProcessorDescription( + title="Create video collages", + tags=["visual", "chart"], + description="Sample frames from each video with ffmpeg and combine them into a single collage image, then derive a 64-bit hash from the collage. These hashes can be compared to find near-duplicate videos. Optionally save each video's hash back to the source dataset as an annotation.", + warnings=[ + "For short videos, a higher frame rate reduces false matches but takes longer to process.", + ], + ) extension = "zip" # extension of result file, used internally and in UI media_type = "image" # media type of the result + # a zip archive of image files (video-frame collages) + output = MediaArchive(media="image") # video datasets (collages are made from video frames) - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, preferred_followups=["video-hash-network", "video-hash-similarity-matrix"]) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, preferred_followups=["video-hash-network", "video-hash-similarity-matrix"]) @classmethod def get_options(cls, parent_dataset=None, config=None): @@ -334,18 +350,21 @@ class VideoHashNetwork(BasicProcessor): This creates a network graph of the video hashes similarity """ type = "video-hash-network" # job type ID - category = "Visual" # category - title = "Create Video hashes network" # title displayed in UI - description = "Creates hashes network to identify duplicate or similar videos." # description displayed in UI + description = ProcessorDescription( + title="Create video hash network", + tags=["networks", "visual"], + description="Build a network of videos linked by the similarity of their hashes. Each video is a node, and an edge is drawn between two videos when their hashes are at least the chosen percentage similar. Run this on the output of 'Create video collages'.", + references=[ + "[Video Hash](https://github.com/akamhy/videohash#readme)", + ], + ) extension = "gexf" # extension of result file, used internally and in UI + # a graph file, no column table + output = Network() # Allow on video hasher compatibility = Compatibility(types={"video-hasher-1"}) - references = [ - "[Video Hash](https://github.com/akamhy/videohash#readme)", - ] - @classmethod def get_options(cls, parent_dataset=None, config=None): return {"percent": { @@ -448,18 +467,21 @@ class VideoHashSimilarities(BasicProcessor): This creates a network graph of the video hashes similarity """ type = "video-hash-similarity-matrix" # job type ID - category = "Visual" # category - title = "Calculates hashes and similarity groups" # title displayed in UI - description = "Creates CSV with hashes and groups videos above similarity value." # description displayed in UI + description = ProcessorDescription( + title="Group videos by hash similarity", + tags=["visual", "classification"], + description="Compare video hashes and assign each video to a similarity group. Videos are grouped when their hashes are at least the chosen percentage similar, and groups can chain together when videos overlap. Run this on the output of 'Create video collages'.", + references=[ + "[Video Hash](https://github.com/akamhy/videohash#readme)", + ], + ) extension = "csv" # extension of result file, used internally and in UI + # a derived table + output = Table() # Allow on video hasher compatibility = Compatibility(types={"video-hasher-1"}) - references = [ - "[Video Hash](https://github.com/akamhy/videohash#readme)", - ] - @classmethod def get_options(cls, parent_dataset=None, config=None): return {"percent": { diff --git a/processors/visualisation/video_scene_frames.py b/processors/visualisation/video_scene_frames.py index 65e136fee..0c478b81e 100644 --- a/processors/visualisation/video_scene_frames.py +++ b/processors/visualisation/video_scene_frames.py @@ -11,8 +11,9 @@ from packaging import version -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility, is_executable +from common.lib.outputs import MediaArchive from common.lib.user_input import UserInput from common.lib.helpers import get_ffmpeg_version @@ -29,10 +30,16 @@ class VideoSceneFrames(BasicProcessor): Uses ffmpeg to extract a certain number of frames per second at different sizes and saves them in an archive. """ type = "video-scene-frames" # job type ID - category = "Visual" # category - title = "Extract key frames from each scene" # title displayed in UI - description = "For each scene identified, extracts a key frame (e.g. the first frame)." # description displayed in UI + description = ProcessorDescription( + title="Extract key frames from each scene", + tags=["visual", "video", "download media"], + description="Extract one key frame from each detected scene and save the frames as an image archive. Choose the first, middle, or last frame of each scene, optionally resized to a fixed size.", + icon="photo-film", + ) extension = "zip" # extension of result file, used internally and in UI + media_type = "image" # the extracted frames are images; set so the map and runtime agree + # a zip archive of image files + output = MediaArchive(media="image") # Allow on detected video scenes when ffmpeg is available compatibility = Compatibility(types={"video-scene-detector"}, required_settings={("video-downloader.ffmpeg_path", is_executable)}, preferred_followups=["video-timelines"]) diff --git a/processors/visualisation/video_scene_identifier.py b/processors/visualisation/video_scene_identifier.py index 7fa6c03b8..acd7908a4 100644 --- a/processors/visualisation/video_scene_identifier.py +++ b/processors/visualisation/video_scene_identifier.py @@ -9,8 +9,9 @@ from scenedetect import open_video, SceneManager, VideoOpenFailure, FrameTimecode -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Table from common.lib.exceptions import ProcessorInterruptedException, ProcessorException from common.lib.user_input import UserInput @@ -34,20 +35,24 @@ class VideoSceneDetector(BasicProcessor): start and end times. """ type = "video-scene-detector" # job type ID - category = "Visual" # category - title = "Detect scenes in video" # title displayed in UI - description = "Detect distinct 'scenes' in videos based on various parameters (e.g. change in color and " \ - "intensity or cuts and fades to black) and extract the scene metadata." # description displayed in UI + description = ProcessorDescription( + title="Detect scenes in video", + tags=["video", "visual", "time series", "machine learning"], + description="Detect distinct scenes in videos and record their boundaries as a table. Scenes are found with an ffmpeg threshold filter or one of PySceneDetect's content, adaptive, or threshold detectors, based on changes in colour, intensity, or cuts and fades to black. Each row lists a scene's start and end frame, timecode, and duration.", + references=[ + "[PySceneDetect](https://github.com/Breakthrough/PySceneDetect)", + "[Detection Algorithms](https://scenedetect.com/projects/Manual/en/latest/api/detectors.html)", + "ffmpeg's scene/shot detection algorithm is based on [ShotDetect](https://github.com/johmathe/Shotdetect) (see [here](https://github.com/FFmpeg/FFmpeg/commit/7286814))", + ], + icon="clapperboard", + ) extension = "csv" # extension of result file, used internally and in UI - # Allow on video datasets - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, preferred_followups=["video-scene-frames", "video-timelines"]) + # a derived table + output = Table() - references = [ - "[PySceneDetect](https://github.com/Breakthrough/PySceneDetect)", - "[Detection Algorithms](https://scenedetect.com/projects/Manual/en/latest/api/detectors.html)", - "ffmpeg's scene/shot detection algorithm is based on [ShotDetect](https://github.com/johmathe/Shotdetect) (see [here](https://github.com/FFmpeg/FFmpeg/commit/7286814))" - ] + # Allow on video datasets + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, preferred_followups=["video-scene-frames", "video-timelines"]) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: diff --git a/processors/visualisation/video_stack.py b/processors/visualisation/video_stack.py index a2f2fcac5..a479e8156 100644 --- a/processors/visualisation/video_stack.py +++ b/processors/visualisation/video_stack.py @@ -11,8 +11,9 @@ from packaging import version -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility, ExecutableSibling +from common.lib.outputs import Render from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.helpers import get_ffmpeg_version @@ -30,15 +31,22 @@ class VideoStack(BasicProcessor): Use ffmpeg to render multiple videos into one combined video in which they are overlaid. """ type = "video-stack" # job type ID - category = "Visual" # category - title = "Stack videos" # title displayed in UI - description = "Create a video stack from the videos in the dataset. Videos are layered on top of each other " \ - "transparently to help visualise similarities. Does not work well with more than a dozen or so " \ - "videos. Videos are stacked by length, i.e. the longest video is at the 'bottom' of the stack." # description displayed in UI + description = ProcessorDescription( + title="Stack videos", + tags=["video", "chart"], + description="Layer the videos in the dataset on top of each other transparently into a single combined video to reveal similarities. Videos are ordered by length, with the longest at the bottom of the stack.", + warnings=[ + "This works best with a dozen or fewer videos; more than that becomes hard to read.", + "Requires ffmpeg and ffprobe to be installed on the server.", + ], + icon="layer-group", + ) extension = "mp4" # extension of result file, used internally and in UI + # a rendered video, no column table + output = Render("mp4", media="video") # Allow on video datasets when ffmpeg and ffprobe are available - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))}) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))}) @classmethod def get_options(cls, parent_dataset=None, config=None) -> dict: diff --git a/processors/visualisation/video_timelines.py b/processors/visualisation/video_timelines.py index e2036c4bc..259bd6344 100644 --- a/processors/visualisation/video_timelines.py +++ b/processors/visualisation/video_timelines.py @@ -13,8 +13,9 @@ from ural import is_url -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from common.lib.exceptions import ProcessorInterruptedException from common.lib.user_input import UserInput from common.lib.helpers import get_4cat_canvas @@ -33,11 +34,15 @@ class VideoTimelines(BasicProcessor): Takes a set of folders containing video frames and renders them as a horizontal collage per video """ type = "video-timelines" # job type ID - category = "Visual" # category - title = "Create video timelines" # title displayed in UI - description = "For each video for which frames were extracted, create a video timeline (i.e. a horizontal " \ - "collage of sequential frames). Timelines are then vertically stacked." # description displayed in UI + description = ProcessorDescription( + title="Create video timelines", + tags=["video", "visual", "chart"], + description="Arrange extracted video frames into a timeline for each video, laying the frames out in sequence as a horizontal strip. The per-video timelines are stacked vertically into a single image, with each video labelled underneath.", + icon="photo-film", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() # Compatible with extracted video frames (or anything that stores related # images in separate folders within a zip archive). diff --git a/processors/visualisation/video_wall.py b/processors/visualisation/video_wall.py index 74800834c..49754cbc5 100644 --- a/processors/visualisation/video_wall.py +++ b/processors/visualisation/video_wall.py @@ -11,8 +11,9 @@ from packaging import version from common.lib.helpers import UserInput, get_ffmpeg_version, convert_to_int -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility, ExecutableSibling +from common.lib.outputs import Render from common.lib.exceptions import ProcessorInterruptedException, MediaSignatureException __author__ = "Stijn Peeters" @@ -30,13 +31,23 @@ class VideoWallGenerator(BasicProcessor): or a collage of images and videos combined. """ type = "video-wall" # job type ID - category = "Visual" # category - title = "Video wall" # title displayed in UI - description = "Put all videos in a single combined video, side by side. Videos can be sorted and resized." + description = ProcessorDescription( + title="Create video wall", + tags=["video", "visual", "chart"], + description="Combine the videos in the dataset into a single grid, playing side by side. Videos can be sorted by length or at random, resized to a chosen tile size, and arranged to a chosen aspect ratio.", + warnings=[ + "Rendering can be slow and heavy for large datasets or long videos; set a length limit to keep run times reasonable.", + "Requires ffmpeg and ffprobe to be installed on the server.", + ], + icon="panorama", + ) extension = "mp4" # extension of result file, used internally and in UI + # a rendered video, no column table + output = Render("mp4", media="video") + # Allow on video datasets when ffmpeg and ffprobe are available - compatibility = Compatibility(media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))}) + compatibility = Compatibility(extensions={"zip"}, media_types={"video"}, type_prefixes={"video-downloader"}, required_settings={("video-downloader.ffmpeg_path", ExecutableSibling("ffmpeg", "ffprobe"))}) # videos will be arranged and resized to fit these image wall dimensions # note that video aspect ratio may not allow for a precise fit diff --git a/processors/visualisation/word-cloud.py b/processors/visualisation/word-cloud.py index be922a0a5..a24caaa07 100644 --- a/processors/visualisation/word-cloud.py +++ b/processors/visualisation/word-cloud.py @@ -5,8 +5,9 @@ from wordcloud import WordCloud -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from common.lib.helpers import UserInput __author__ = "Sal Hagen" @@ -20,10 +21,18 @@ class MakeWordCloud(BasicProcessor): Generate activity histogram """ type = "wordcloud" # job type ID - category = "Visual" # category - title = "Word cloud" # title displayed in UI - description = "Generates a word cloud with words sized on occurrence." # description displayed in UI + description = ProcessorDescription( + title="Create word cloud", + tags=["text analysis", "chart"], + description="Draw a word cloud from a word column and a count column, sizing each word by its count. Optionally lower-case the words and limit how many are shown.", + info=[ + "This works best on the output of a word frequency processor, such as tf-idf, collocations, or word counts.", + ], + icon="cloud", + ) extension = "svg" + # a rendered image, no column table + output = Render() # Allow processor on rankable items compatibility = Compatibility(types={ diff --git a/processors/visualisation/word-trees.py b/processors/visualisation/word-trees.py index f1b493570..836df09de 100644 --- a/processors/visualisation/word-trees.py +++ b/processors/visualisation/word-trees.py @@ -6,10 +6,11 @@ import jieba import re -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas from common.lib.exceptions import QueryParametersException from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from nltk.tokenize import word_tokenize, TweetTokenizer @@ -194,20 +195,27 @@ class MakeWordtree(BasicProcessor): """ type = "word-trees" # job type ID - category = "Visual" # category - title = "Word tree" # title displayed in UI - description = "Generates a word tree for a given query, a \"graphical version of the traditional 'keyword-in-context' method\" (Wattenberg & Viégas, 2008)." # description displayed in UI + description = ProcessorDescription( + title="Word tree", + tags=["text analysis", "chart"], + description="Build a word tree around a search phrase, a graphical version of the keyword-in-context method (Wattenberg & Viégas, 2008). Words that follow or precede the phrase branch out into a tree, sized by how often they occur. You can set the phrase, window size, tokeniser, and how many branches to show per level.", + references=[ + "Wattenberg, M., & Viégas, F. B. (2008). [The Word Tree, an Interactive Visual Concordance](https://doi.org/10.1109/TVCG.2008.172). IEEE Transactions on Visualization and Computer Graphics, 14(6), 1221–1228.", + "[NLTK tokenizer documentation](https://www.nltk.org/api/nltk.tokenize.html)", + "[Different types of tokenizers in NLTK](https://chendianblog.wordpress.com/2016/11/25/different-types-of-tokenizers-in-nltk/)", + ], + info=[ + "Use a wildcard in the root phrase (for example 'politic*') to match several words at once, at the cost of a slower run.", + ], + icon="tree", + ) extension = "svg" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render() # any csv or ndjson dataset compatibility = Compatibility(extensions={"csv", "ndjson"}) - references = [ - "Wattenberg, M., & Viégas, F. B. (2008). [The Word Tree, an Interactive Visual Concordance](https://doi.org/10.1109/TVCG.2008.172). IEEE Transactions on Visualization and Computer Graphics, 14(6), 1221–1228.", - "[NLTK tokenizer documentation](https://www.nltk.org/api/nltk.tokenize.html)", - "[Different types of tokenizers in NLTK](https://chendianblog.wordpress.com/2016/11/25/different-types-of-tokenizers-in-nltk/)", - ] - # can be changed FONT_SIZE = 14 # in px FONT_FACTOR_MAX = 3 # how big can the font get? diff --git a/processors/visualisation/youtube_imagewall.py b/processors/visualisation/youtube_imagewall.py index 94e9c969d..f5cd62d42 100644 --- a/processors/visualisation/youtube_imagewall.py +++ b/processors/visualisation/youtube_imagewall.py @@ -9,8 +9,9 @@ from collections import Counter from PIL import Image, ImageOps, ImageDraw, ImageFont -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import Render from common.lib.helpers import UserInput, convert_to_int __author__ = "Sal Hagen" @@ -28,10 +29,15 @@ class YouTubeImageWall(BasicProcessor): """ type = "youtube-imagewall" # job type ID - category = "Visualisation" # category - title = "YouTube thumbnails image wall" # title displayed in UI - description = "Make an image wall from YouTube video thumbnails." # description displayed in UI + description = ProcessorDescription( + title="Make image wall from YouTube thumbnails", + tags=["visual", "chart"], + description="Arrange downloaded YouTube video thumbnails into a single grid image. Optionally overlay each thumbnail with a colour for its video category and add a category legend.", + icon="panorama", + ) extension = "png" # extension of result file, used internally and in UI + # a rendered image, no column table + output = Render("png") # Allow processor on YouTube thumbnail sets compatibility = Compatibility(types={"youtube-thumbnails"}) diff --git a/processors/visualisation/youtube_thumbnails.py b/processors/visualisation/youtube_thumbnails.py index 111cd6a43..fa5f1059c 100644 --- a/processors/visualisation/youtube_thumbnails.py +++ b/processors/visualisation/youtube_thumbnails.py @@ -6,8 +6,9 @@ from apiclient.discovery import build -from backend.lib.processor import BasicProcessor +from backend.lib.processor import BasicProcessor, ProcessorDescription from common.lib.compatibility import Compatibility +from common.lib.outputs import MediaArchive from common.lib.exceptions import ProcessorInterruptedException from common.lib.helpers import get_yt_compatible_ids, UserInput @@ -25,10 +26,18 @@ class YouTubeThumbnails(BasicProcessor): """ type = "youtube-thumbnails" # job type ID - category = "Cross-platform" # category - title = "Download YouTube thumbnails" # title displayed in UI - description = "Downloads the thumbnails of YouTube videos and stores it in a zip archive." # description displayed in UI + description = ProcessorDescription( + title="Download YouTube thumbnails", + tags=["download media", "external service"], + description="Download the thumbnail image of each YouTube video in the dataset through the YouTube Data API.", + warnings=[ + "This uses the YouTube Data API, which requires an API key and counts against your daily quota.", + ], + icon="images", + ) extension = "zip" # extension of result file, used internally and in UI + # a zip archive of media files + output = MediaArchive(media="image") media_type = "image" # media type of the result # Allow processor on YouTube metadata sets diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..50e269a0a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,61 @@ +""" +Shared pytest fixtures. + +These mirror the fixtures in test_modules.py so that other test modules (e.g. +test_module_map.py) can build the real module set without a database. Defining +them here makes them available session-wide; test_modules.py keeps its own local +copies, which simply override these for its own tests (standard pytest behaviour), +so nothing about that file changes. +""" +import os +from pathlib import Path +from unittest.mock import patch, MagicMock + +import pytest + +PATH_ROOT = Path(os.path.abspath(os.path.dirname(__file__))).joinpath("..").resolve() + + +@pytest.fixture +def mock_database(): + """Mock the database connection.""" + with patch("common.config_manager.Database") as mock_cfg_db, \ + patch("backend.lib.worker.Database") as mock_worker_db: + mock_database_instance = MagicMock() + mock_cfg_db.return_value = mock_database_instance + mock_worker_db.return_value = mock_database_instance + yield mock_database_instance + + +@pytest.fixture +def mock_basic_config(tmp_path, mock_database): + """Set up a config reader without connecting it to the database.""" + class mocked_config: + pass + + mocked_basic_config = mocked_config() + mocked_basic_config.get = MagicMock(side_effect=lambda key, default=None, is_json=False, user=None, tags=None: { + "PATH_ROOT": PATH_ROOT, + "PATH_DATA": PATH_ROOT, + "PATH_LOGS": PATH_ROOT / "logs", + "PATH_EXTENSIONS": PATH_ROOT / "config/extensions", + "extensions.enabled": {}, + }.get(key, default)) + mocked_basic_config.load_user_settings = MagicMock() + (tmp_path / "logs").mkdir(parents=True, exist_ok=True) + yield mocked_basic_config + + +@pytest.fixture +def logger(mock_basic_config): + """Initialize the Logger and return it.""" + from common.lib.logger import Logger + return Logger(logger_name="pytest", output=True, + log_path=mock_basic_config.get("PATH_LOGS").joinpath("test.log"), log_level='DEBUG') + + +@pytest.fixture +def fourcat_modules(mock_basic_config): + """The real loaded module set, built with a mocked config (no database).""" + from common.lib.module_loader import ModuleCollector + return ModuleCollector(config=mock_basic_config) diff --git a/tests/test_module_map.py b/tests/test_module_map.py new file mode 100644 index 000000000..898f85bad --- /dev/null +++ b/tests/test_module_map.py @@ -0,0 +1,387 @@ +""" +Tests for the compatibility check, the output-description helper, and the module map +query layer. + +Two halves: + +* plain unit tests on `Compatibility.check` -- comparing a spec against a subject + (a live dataset or a declared output), and the rule that an output leaving a value + UNKNOWN can only soften an answer to "maybe", never produce a false "no". No app + context needed. +* tests against the real loaded modules (the `fourcat_modules` fixture from + test_modules.py): every processor declares an output, declarations agree with the + class attributes, and the map builds and answers questions. +""" +import pytest + +from common.lib.compatibility import Compatibility, UNKNOWN, Shape, DatasetShape + + +# --- helpers --------------------------------------------------------------- + +def shape(type=None, extension=UNKNOWN, media=UNKNOWN, datasource=UNKNOWN, + top_level=False, from_collector=False, columns=UNKNOWN, columns_are_all=False): + """A declared output; anything left as UNKNOWN reads as 'the output didn't say'.""" + return Shape(type=type, extension=extension, media=media, datasource=datasource, + top_level=top_level, from_collector=from_collector, + columns=columns, columns_are_all=columns_are_all) + + +class FakeModule: + """A minimal live dataset exposing what DatasetShape reads.""" + + def __init__(self, type=None, extension=None, media_type=None, datasource=None, + top=True, from_collector=False): + self.type = type + self._extension = extension + self._media_type = media_type + self.parameters = {"datasource": datasource} if datasource else {} + self._top = top + self._from_collector = from_collector + + def get_extension(self): + return self._extension + + def get_media_type(self): + return self._media_type + + def is_top_dataset(self): + return self._top + + def is_from_collector(self): + return self._from_collector + + +# --- check() against a declared output ------------------------------------- + +def test_check_definite_yes_and_no_on_known_media(): + assert Compatibility(media_types={"audio"}).check(shape(media="audio")) == "yes" + assert Compatibility(media_types={"image"}).check(shape(media="audio")) == "no" + + +def test_unknown_value_is_maybe_never_false_no(): + # import_media sets its media only when it runs, so its output leaves media UNKNOWN; + # a processor wanting images then gets "maybe", never a false "no" + spec = Compatibility(media_types={"image"}) + assert spec.check(shape(media=UNKNOWN, from_collector=True, top_level=True)) == "maybe" + + +def test_unknown_extension_is_maybe_but_wrong_extension_is_no(): + spec = Compatibility(extensions={"ndjson"}) + assert spec.check(shape(extension=UNKNOWN)) == "maybe" # the output didn't say + assert spec.check(shape(extension="csv")) == "no" # definitely the wrong one + assert spec.check(shape(extension="ndjson")) == "yes" + + +def test_column_requirement_softens_to_maybe(): + spec = Compatibility(media_types={"audio"}, requires_all_columns={"author"}) + # media matches, but the output hasn't declared its columns -> maybe + assert spec.check(shape(media="audio")) == "maybe" + + +def test_filter_position_is_maybe(): + # a filter's result may or may not be top-level -> top_level UNKNOWN -> maybe + assert Compatibility(top_dataset_only=True).check(shape(top_level=UNKNOWN)) == "maybe" + + +def test_columns_requirement_resolved_against_declared_columns(): + spec = Compatibility(requires_all_columns={"author", "body"}) + # the output guarantees both -> yes + assert spec.check(shape(columns=frozenset({"author", "body", "id"}))) == "yes" + # a column not in the floor might still appear at run time -> maybe, never a false no + assert spec.check(shape(columns=frozenset({"author"}))) == "maybe" + # an output with no columns at all (columns_are_all) can never satisfy it -> no + assert spec.check(shape(columns=frozenset(), columns_are_all=True)) == "no" + # nothing declared -> maybe + assert spec.check(shape(columns=UNKNOWN)) == "maybe" + + +def test_columns_any_requirement(): + spec = Compatibility(requires_any_columns={"image", "video"}) + assert spec.check(shape(columns=frozenset({"video"}))) == "yes" + assert spec.check(shape(columns=frozenset(), columns_are_all=True)) == "no" + + +def test_rankable_derived_from_columns_and_extension(): + spec = Compatibility(rankable=True) + # a csv guaranteeing the ranking columns is rankable + assert spec.check(shape(extension="csv", columns=frozenset({"date", "value", "item"}))) == "yes" + # a non-csv (a network) is definitely not rankable + assert spec.check(shape(extension="gexf", columns=frozenset(), columns_are_all=True)) == "no" + # columns unknown -> rankability unknown -> maybe + assert spec.check(shape(extension="csv")) == "maybe" + + +# --- check() against a real dataset is only ever yes/no -------------------- + +@pytest.mark.parametrize("spec, module, compatible", [ + (Compatibility(types={"a"}), FakeModule(type="a"), True), + (Compatibility(types={"a"}), FakeModule(type="b"), False), + (Compatibility(media_types={"image"}), FakeModule(type="x", media_type="image"), True), + (Compatibility(media_types={"image"}), FakeModule(type="x", media_type="text"), False), + (Compatibility(extensions={"csv"}), FakeModule(type="x", extension="csv"), True), + (Compatibility(extensions={"csv"}), FakeModule(type="x", extension="ndjson"), False), + (Compatibility(top_dataset_only=True), FakeModule(type="x", top=True), True), + (Compatibility(top_dataset_only=True), FakeModule(type="x", top=False), False), + (Compatibility(child_only=True), FakeModule(type="x", top=False), True), + (Compatibility(child_only=True), FakeModule(type="x", top=True), False), + (Compatibility(excluded_types={"x"}), FakeModule(type="x"), False), + (Compatibility(datasources={"4chan"}), FakeModule(type="x", datasource="4chan"), True), + (Compatibility(datasources={"4chan"}), FakeModule(type="x", datasource="reddit"), False), + (Compatibility(is_collector=True), FakeModule(type="x", from_collector=True), True), + (Compatibility(is_collector=True), FakeModule(type="x", from_collector=False), False), +]) +def test_live_path_behaviour_preserved(spec, module, compatible): + assert spec.is_compatible_with(module) is compatible + + +def test_live_check_is_never_maybe(): + # a real dataset knows all its values, so check() is only ever yes/no for a dataset + spec = Compatibility(media_types={"image"}, extensions={"csv"}, top_dataset_only=True) + for module in (FakeModule(type="x", media_type="image", extension="csv", top=True), + FakeModule(type="x", media_type="text", extension="ndjson", top=False)): + assert spec.check(DatasetShape(module)) in ("yes", "no") + + +# --- the output description is honest about what it cannot know ------------ + +def test_infer_output_honour_traps(logger, fourcat_modules): + """ + The fallback inference must stay honest on every real processor class: a value not + set on the class comes out UNKNOWN (never guessed). Tested against _infer_output + directly, so a processor that declares its own output does not hide a dishonest + inference. + """ + from common.lib.outputs import _infer_output + from common.lib.compatibility import _declared_class_value, _maybe_call, is_collector as is_collector_fn + + for ptype, processor in fourcat_modules.processors.items(): + shape = _infer_output(processor) + + # media: known iff declared on the class (below BasicProcessor), else UNKNOWN + declared_media = _declared_class_value(processor, "media_type") + if declared_media: + assert shape.media == declared_media, f"{ptype}: declared media lost" + else: + assert shape.media is UNKNOWN, f"{ptype}: undeclared media is not UNKNOWN" + + # extension: a filter passes its parent's through (UNKNOWN); a value set on the + # class is trusted; an inherited default stays UNKNOWN, never a confident guess + is_filter = bool(_maybe_call(processor, "is_filter")) + declared_ext = _declared_class_value(processor, "extension") + if is_filter: + assert shape.extension is UNKNOWN, f"{ptype}: filter extension is not UNKNOWN" + elif declared_ext: + assert shape.extension == declared_ext, f"{ptype}: declared extension lost" + else: + assert shape.extension is UNKNOWN, f"{ptype}: inherited extension treated as fact" + + # position/collector-ness: a collector is top-level; a filter's result can be made + # top-level and take on a -search type, so both are UNKNOWN; else a child + if is_collector_fn(processor): + assert shape.top_level is True and shape.from_collector is True, f"{ptype}: collector not top" + elif is_filter: + assert shape.top_level is UNKNOWN and shape.from_collector is UNKNOWN, f"{ptype}: filter not UNKNOWN" + else: + assert shape.top_level is False and shape.from_collector is False, f"{ptype}: not a child" + + +def test_describe_output_reads_declared_media(logger, fourcat_modules): + """Cross-check: a media_type set directly on the class must show up as known.""" + from common.lib.outputs import describe_output + + checked = 0 + for processor in fourcat_modules.processors.values(): + own_media = vars(processor).get("media_type") + if own_media: + media = describe_output(processor).media + assert own_media == media or (isinstance(media, set) and own_media in media) + checked += 1 + logger.info(f"verified {checked} processor(s) that declare media_type directly") + + +# --- every processor declares its output, and the declaration is truthful --- + +def test_every_processor_declares_output(logger, fourcat_modules): + """The coverage gate: every processor should declare an `output` (an Output, usually + inherited from a base class). Lists any that still fall back to class inference.""" + from common.lib.outputs import Output + + missing = sorted(ptype for ptype, cls in fourcat_modules.processors.items() + if not isinstance(getattr(cls, "output", None), Output)) + logger.info(f"{len(fourcat_modules.processors) - len(missing)} of " + f"{len(fourcat_modules.processors)} processors declare an output") + if missing: + pytest.fail(f"{len(missing)} processor(s) declare no output:\n" + "\n".join(missing)) + + +def test_output_matches_class_attributes(logger, fourcat_modules): + """Where a processor declares both an `output` and the legacy class attributes, a + fixed output extension/media must equal the class one -- keeping the declaration + honest while the legacy attributes still exist (the eventual source to derive from).""" + from common.lib.outputs import Output + from common.lib.compatibility import _declared_class_value + + mismatches = [] + for ptype, cls in fourcat_modules.processors.items(): + out = getattr(cls, "output", None) + if not isinstance(out, Output): + continue + shape = out.to_shape(cls) + + declared_ext = _declared_class_value(cls, "extension") + if declared_ext and isinstance(shape.extension, str) and shape.extension != declared_ext: + mismatches.append(f"{ptype}: output extension {shape.extension!r} != class {declared_ext!r}") + + declared_media = _declared_class_value(cls, "media_type") + if declared_media and isinstance(shape.media, str) and shape.media != declared_media: + mismatches.append(f"{ptype}: output media {shape.media!r} != class {declared_media!r}") + + if mismatches: + pytest.fail("output does not match class attributes:\n" + "\n".join(sorted(mismatches))) + + +def test_describe_spec_covers_every_compatibility_axis(): + """ + Every axis a Compatibility can declare must round-trip through describe_spec -- the + dict the catalogue displays and the map uses as the "requirement" label. Lockstep + insurance: add an axis without teaching describe_spec and this fails, rather than the + axis silently vanishing. A field is exempt only when it modifies another axis. + """ + import dataclasses + from common.lib.compatibility import describe_spec + + modifiers = {"rankable_multiple_items"} # tunes `rankable`; not a standalone axis + + def sample(name): + if name in ("rankable", "is_collector", "top_dataset_only", "child_only"): + return True + if name == "required_settings": + return ["some.setting"] + return {"x"} + + missing = [field.name for field in dataclasses.fields(Compatibility) + if field.name not in modifiers + and field.name not in (describe_spec(Compatibility(**{field.name: sample(field.name)})) or {})] + assert not missing, ("describe_spec drops these Compatibility axes (add them to " + "describe_spec, or to the modifiers exemption): %s" % missing) + + +# --- the map builds and answers -------------------------------------------- + +def test_module_map_builds_and_answers(logger, fourcat_modules): + from common.lib.module_map import ModuleMap + + pmap = ModuleMap(fourcat_modules, config=None, logger=logger) + + assert set(pmap.processors) == set(fourcat_modules.processors) # nothing silently dropped + + catalogue = pmap.catalogue() + assert len(catalogue) == len(fourcat_modules.processors) + for entry in catalogue: + assert {"type", "title", "is_datasource", "is_filter", "has_override"} <= set(entry) + assert "output_shape" not in entry # browse rows stay light; shape is on the processor view + + total_edges = sum(len(consumers) for consumers in pmap._succ.values()) + assert total_edges > 0, "no edges -- the specs produced an empty map" + + graph = pmap.graph() + assert graph["nodes"] and isinstance(graph["edges"], list) + for edge in graph["edges"]: + assert edge["certainty"] in ("definite", "maybe") + + sample = next(iter(pmap.processors)) + info = pmap.module(sample) + assert {"how_to_run", "followups", "compatibility", "output_shape"} <= set(info) + how_to_run = info["how_to_run"] + assert "notes" in how_to_run + if not how_to_run.get("is_filter"): + assert "accepts" in how_to_run and "examples" in how_to_run + assert {"preferred", "filters", "others_by_category"} <= set(info["followups"]) + + assert isinstance(pmap.search("data"), list) + + +def test_datasources_are_roots_not_consumers(logger, fourcat_modules): + """Collectors produce but never consume -- they have no incoming edges.""" + from common.lib.module_map import ModuleMap + + pmap = ModuleMap(fourcat_modules, config=None, logger=logger) + for ptype, is_root in pmap._collector.items(): + if is_root: + assert not pmap._pred.get(ptype), f"datasource {ptype} has incoming edges" + + +# --- the author-facing archetypes ------------------------------------------ + +class FakeProcessor: + """A stand-in processor class with just the attributes an Output reads.""" + + def __init__(self, type=None, extension="csv"): + self.type = type + self.extension = extension + + +def test_datasource_archetype_uses_class_extension_and_text_media(): + from common.lib.outputs import Datasource + shape = Datasource().to_shape(FakeProcessor(type="bsky-search", extension="ndjson")) + assert shape.extension == "ndjson" + assert shape.media == "text" + assert shape.top_level is True + assert shape.from_collector is True + assert shape.datasource == "bsky" # a collector carries its datasource in its type + assert shape.produces_file + + +def test_datasource_archetype_can_declare_columns(): + from common.lib.outputs import Datasource + shape = Datasource(columns={"id", "body", "author"}).to_shape(FakeProcessor(type="x-search")) + assert shape.columns == frozenset({"id", "body", "author"}) + assert Compatibility(requires_all_columns={"author"}).check(shape) == "yes" + + +def test_filter_archetype_is_passthrough_everywhere(): + from common.lib.outputs import Filter + shape = Filter().to_shape(FakeProcessor(type="x-filter")) + assert shape.extension is UNKNOWN + assert shape.media is UNKNOWN + assert shape.top_level is UNKNOWN + assert shape.from_collector is UNKNOWN + assert shape.columns is UNKNOWN + + +def test_render_and_network_have_no_columns(): + from common.lib.outputs import Render, Network + render = Render("svg").to_shape(FakeProcessor(type="x")) + assert render.extension == "svg" + assert render.media == "image" + assert render.columns == frozenset() and render.columns_are_all + network = Network().to_shape(FakeProcessor(type="x")) + assert network.extension == "gexf" + assert network.columns == frozenset() and network.columns_are_all + + +def test_media_archive_bounded_media_set(): + from common.lib.outputs import MediaArchive + shape = MediaArchive(media={"image", "video", "audio"}).to_shape(FakeProcessor(type="x")) + assert shape.extension == "zip" + assert shape.media == {"image", "video", "audio"} + # wanting image gets "maybe" (some, not all, of the set matches); wanting text is a no + assert Compatibility(media_types={"image"}).check(shape) == "maybe" + assert Compatibility(media_types={"text"}).check(shape) == "no" + + +def test_no_output_produces_no_file(): + from common.lib.outputs import NoOutput + assert NoOutput().to_shape(FakeProcessor(type="item-to-annotation")).produces_file is False + + +def test_describe_output_prefers_declared_over_inference(): + from common.lib.outputs import describe_output, Table + + class Declared(FakeProcessor): + output = Table(columns={"date", "value", "item"}) + + shape = describe_output(Declared(type="x", extension="csv")) + assert shape.columns == frozenset({"date", "value", "item"}) + assert Compatibility(rankable=True).check(shape) == "yes" # rankability falls out of the columns diff --git a/tests/test_modules.py b/tests/test_modules.py index 5b6e50e55..447ebaee9 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -205,7 +205,7 @@ def test_processors(logger, fourcat_modules, mock_job, mock_job_queue, mock_data assert issubclass(processor_class, BasicProcessor), f"{processor_name} is not a subclass of BasicProcessor" # Check if required attributes are implemented - required_attributes = ["type", "category", "title", "description", "extension"] + required_attributes = ["type", "tags", "title", "description", "extension"] for attr in required_attributes: assert hasattr(processor_class, attr), f"{processor_name} is missing required attribute: {attr}" assert getattr(processor_class, attr), f"{processor_name} has an empty value for attribute: {attr}" diff --git a/webtool/__init__.py b/webtool/__init__.py index f7cd24b8c..bdbc7166a 100644 --- a/webtool/__init__.py +++ b/webtool/__init__.py @@ -126,7 +126,8 @@ def time_this(func): "HOSTNAME_WHITELIST": config.get("flask.autologin.hostnames"), "HOSTNAME_WHITELIST_NAME": config.get("flask.autologin.name"), "HOSTNAME_WHITELIST_API": config.get("flask.autologin.api"), - "PREFERRED_URL_SCHEME": "https" if config.get("flask.https") else "http" + "PREFERRED_URL_SCHEME": "https" if config.get("flask.https") else "http", + "TEMPLATES_AUTO_RELOAD": True }) # Set number of form parts to accept (default is 1000; affects number of files that can be uploaded) @@ -178,6 +179,7 @@ def time_this(func): import webtool.views.views_explorer # noqa: E402 import webtool.views.api_standalone # noqa: E402 import webtool.views.api_tool # noqa: E402 + import webtool.views.api_module_map # noqa: E402 app.register_blueprint(webtool.views.views_restart.component) app.register_blueprint(webtool.views.views_admin.component) @@ -189,6 +191,7 @@ def time_this(func): app.register_blueprint(webtool.views.views_explorer.component) app.register_blueprint(webtool.views.api_standalone.component) app.register_blueprint(webtool.views.api_tool.component) + app.register_blueprint(webtool.views.api_module_map.component) @app.before_request def before_request(): diff --git a/webtool/lib/helpers.py b/webtool/lib/helpers.py index 586d2ab56..9975e11c0 100644 --- a/webtool/lib/helpers.py +++ b/webtool/lib/helpers.py @@ -4,6 +4,7 @@ import markdown2 import colorsys import csv +import json import re from functools import wraps @@ -13,6 +14,10 @@ from flask import (current_app, request, jsonify, g) from PIL import Image, ImageColor, ImageOps +from common.lib.helpers import hash_to_md5 +from common.lib.module_map import ModuleMap +from common.lib.user_input import UserInput + csv.field_size_limit(1024 * 1024 * 1024) class Pagination(object): @@ -80,6 +85,233 @@ def iter_pages(self, left_edge=2, left_current=2, right_current=5, right_edge=2) last = num +# Cache the built module map per modules object. `g.modules` +# (app.fourcat_modules) is a single process-global, replaced only on a full +# reload, so its identity is a safe cache key -- a new identity forces a +# rebuild. +# NOTE: config is NOT part of the key: the ModuleMap does not gate edges on +# per-user config today. Per-user maps would need the config (or user) in the key. +_MODULE_MAP_CACHE = {} # id(modules) -> (modules, ModuleMap) + + +def module_map(): + """ + The module map for the currently loaded modules + + Built lazily and cached, since building it walks every processor's + compatibility spec. + + :return ModuleMap: + """ + modules = g.modules + cached = _MODULE_MAP_CACHE.get(id(modules)) + if cached is None or cached[0] is not modules: + _MODULE_MAP_CACHE.clear() + _MODULE_MAP_CACHE[id(modules)] = (modules, ModuleMap(modules, g.config, logger=g.log)) + + return _MODULE_MAP_CACHE[id(modules)][1] + + +def collect_grid_tags(grid_sections): + """ + Every tag present in a module grid, for its tag filter + + The modules in a grid are whatever the grid's view put there - processor + classes in the processor grid, plain dicts elsewhere - so both are read. + + :param list grid_sections: `grid_sections` as passed to components/module-grid.html + :return list: Sorted, de-duplicated tags + """ + tags = set() + for section in grid_sections: + for module in section["modules"].values(): + module_tags = module.get("tags") if isinstance(module, dict) else getattr(module, "tags", None) + tags.update(module_tags or []) + + return sorted(tags) + + +def datasource_variants(worker, config): + """ + The variants a data source's search worker offers, if any + + Variants let one data source appear as several cards on the create-dataset + page - see `Search.get_variants`. Only search workers can declare them, and + declaring them is optional, so anything that does not is simply a data + source without variants. + + Whatever this reads is extension code that may be talking to a server that + is not answering, so a worker that raises is treated as having no variants + rather than taking the page down with it. + + :param worker: Search worker class + :param config: Configuration reader + :return dict: Variants by ID; empty if there are none + """ + getter = getattr(worker, "get_variants", None) + if not getter: + return {} + + try: + return getter(config=config) or {} + except Exception as e: + g.log.warning("Could not read variants for data source worker %s (%s: %s)" % + (getattr(worker, "type", "unknown"), type(e).__name__, e)) + return {} + + +def datasource_worker_options(worker, config, variant=None): + """ + A data source's dataset parameters, for the variant that was picked + + The `variant` argument is only passed on to workers that actually declare + variants, so a data source that knows nothing about them keeps the + `get_options()` signature it always had. + + :param worker: Search worker class + :param config: Configuration reader + :param str variant: Variant ID, or None + :return dict: Options, as `get_options()` returns them + """ + if variant and datasource_variants(worker, config): + return worker.get_options(None, config, variant=variant) + + return worker.get_options(None, config) + + +def module_request_url(kind): + """ + Link to the GitHub issue form for requesting a new module + + Follows the configured repository, so an instance running its own fork + sends requests there rather than upstream. Returns None if no repository is + configured, in which case the front-end omits the link rather than + rendering a broken one. + + :param str kind: `datasource` or `processor`; matches the file name of the + issue form in .github/ISSUE_TEMPLATE/. None where both + kinds are in view, which lands on the form picker instead. + :return str|None: URL to the issue form + """ + repository = g.config.get("4cat.github_url") + if not repository: + return None + + if not kind: + return "%s/issues/new/choose" % repository.rstrip("/") + + return "%s/issues/new?template=%s_request.yml" % (repository.rstrip("/"), kind) + + +def can_annotate_dataset(dataset): + """ + Whether the current user may write annotations on a dataset + + Reading a dataset is enough to see its annotations; writing them needs the + same standing as running a processor on it. + + :param dataset: The DataSet in question + :return bool: + """ + return bool( + g.config.get("privileges.can_run_processors") + and g.config.get("privileges.can_use_explorer") + and (g.config.get("privileges.admin.can_manipulate_all_datasets") + or dataset.is_accessible_by(current_user, "owner")) + ) + + +def annotation_watch_state(dataset): + """ + Whether a processor could still be writing annotations, and what there is now + + Processors write their annotations to the dataset they were ultimately run + on, which is the dataset whose page this is. That happens in the background, + so the page has to go and look: it polls for as long as an analysis is + running (`running`), and compares what it has (`state`) with what is there. + + `state` stands for every annotation of this dataset at once rather than for + any one of them, since the page only needs to know *whether* it is behind, + not what changed. Both are strings so a rendered page can carry them back + unaltered. + + :param dataset: The DataSet being watched + :return dict: `running`, the keys of its unfinished analyses, and `state` + """ + # every analysis below this dataset, however deep, in one query rather than + # one per level of a tree that can be several deep + running = [row["key"] for row in g.db.fetchall(""" + WITH RECURSIVE analyses AS ( + SELECT key, is_finished FROM datasets WHERE key_parent = %s + UNION ALL + SELECT child.key, child.is_finished FROM datasets child, analyses + WHERE child.key_parent = analyses.key + ) + SELECT key FROM analyses WHERE NOT is_finished ORDER BY key + """, (dataset.key,))] + + # the annotations themselves are counted rather than read: what matters is + # whether they are still the ones the page was rendered with + totals = g.db.fetchone("SELECT COUNT(*) AS count, COALESCE(MAX(timestamp), 0) AS latest " + "FROM annotations WHERE dataset = %s", (dataset.key,)) + + state = "%s-%s-%s" % ( + hash_to_md5(json.dumps(dataset.annotation_fields, sort_keys=True))[:12], + totals["count"], totals["latest"]) + + return {"running": ",".join(running), "state": state} + + +def annotation_context(dataset): + """ + Context for anything rendering a dataset's annotation fields + + The fields as the Explorer shows them - so without the ones processors keep + to themselves - plus, for processor-generated fields, the dataset that + generated them, since those are shown attributed to their origin. + + Both the dataset page and the Explorer's own endpoints render annotation + fields, so both need this. + + :param dataset: The DataSet whose fields to describe + :return dict: Template context + """ + # imported here: common.lib.dataset imports from this module's package, and + # only this function needs it + from common.lib.dataset import DataSet + from common.lib.exceptions import DataSetException + + annotation_fields = { + field_id: field + for field_id, field in dataset.annotation_fields.items() + if not field.get("hide_in_explorer") + } + + from_datasets = {} + for field in annotation_fields.values(): + if field.get("from_dataset"): + child_key = field["from_dataset"] + try: + from_datasets[child_key] = DataSet(key=child_key, db=g.db, modules=g.modules) + except DataSetException: + # can be absent if this dataset is a filter and the original was deleted + from_datasets[child_key] = "deleted" + + return { + "annotation_fields": annotation_fields, + "from_datasets": from_datasets, + "can_annotate": can_annotate_dataset(dataset), + "annotation_watch": annotation_watch_state(dataset), + # which fields the reader has folded away in the items below. Kept in the + # address so that it survives a refresh and can be linked to, and read + # back here so the items are rendered folded rather than folded by script + # once they are already on screen + "hidden_fields": { + field_id for field_id in request.args.get("hidden", "").split(",") if field_id + }, + } + + def error(code=200, **kwargs): """ Custom HTTP response @@ -93,6 +325,86 @@ def error(code=200, **kwargs): return response +def common_dataset_options(config, user=None): + """ + Standard dataset creation options shown for every data source + + These are the controls that are not data source-specific but are offered + whenever a dataset is created: pseudonymisation, privacy, e-mail + notification and the dataset label. They are returned as a `UserInput` + options dict so they can be rendered through the same user-input + components as data source options (see `components/form-options.html`). + + Which options are offered depends on the instance configuration. + + Note the `name` overrides: unlike data source options these are read + directly by `toolapi.queue_dataset` under their bare names (not + `option-`-prefixed), which keeps them out of the data source's + `UserInput.parse_all` whitelist and avoids colliding with option names. + + :param config: Configuration reader (e.g. `g.config`) + :param user: The current user, used to pre-fill the e-mail address + :return dict: Option name -> settings, ready for `form-options.html` + """ + options = {} + + if config.get("ui.offer_hashing"): + options["pseudonymise-info"] = { + "type": UserInput.OPTION_INFO, + "help": "4CAT can remove information it identifies as relating to an item's author, or " + "replace it with a [hashed](https://techterms.com/definition/hash) value. Other " + "personal information may persist; it is your responsibility to further anonymise " + "data where appropriate." + } + options["pseudonymise"] = { + "type": UserInput.OPTION_CHOICE, + "name": "pseudonymise", + "help": "Pseudonymise", + "default": "pseudonymise", + "options": { + "anonymise": "Replace author information with 'REDACTED'", + "pseudonymise": "Replace author information with hashed values", + "none": "Leave author information as-is", + } + } + + if config.get("ui.offer_private"): + options["make-private"] = { + "type": UserInput.OPTION_TOGGLE, + "name": "make-private", + "help": "Make dataset private", + "default": True, + "tooltip": "This will only hide your dataset from other users. It will NOT encrypt your " + "data and server administrators will still be able to view it. If you are working " + "with sensitive data, you should consider running your own 4CAT instance." + } + + if config.get("ui.option_email") in ("both", "datasources_only") and config.get("mail.server"): + options["email-complete"] = { + "type": UserInput.OPTION_TOGGLE, + "name": "email-complete", + "help": "Receive e-mail on completion", + "default": False, + } + options["email-user"] = { + "type": UserInput.OPTION_TEXT, + "name": "email-user", + "help": "E-mail address", + "default": user.get_name() if user and user.is_authenticated else "", + "requires": "email-complete==true", + "tooltip": "This will only function if your username is your e-mail address." + } + + options["label"] = { + "type": UserInput.OPTION_TEXT, + "name": "label", + "help": "Dataset name", + "tooltip": "A name will be generated automatically if you do not provide one.", + } + + return options + + def pad_interval(intervals, first_interval=None, last_interval=None): """ Pad an interval so all intermediate intervals are filled diff --git a/webtool/lib/template_filters.py b/webtool/lib/template_filters.py index afc1b17e5..5ef7c5466 100644 --- a/webtool/lib/template_filters.py +++ b/webtool/lib/template_filters.py @@ -1,6 +1,7 @@ import datetime import json +import time import ural import uuid import math @@ -267,6 +268,33 @@ def _jinja2_filter_extension_to_noun(ext): def _jinja2_filter_ellipsiate(*args, **kwargs): return ellipsiate(*args, **kwargs) +@current_app.template_filter("previewable") +def _jinja2_filter_previewable(dataset): + """ + Can this dataset be previewed inline? + + Mirrors what views_dataset.preview_items() knows how to render. Templates + use this to decide whether to offer the 'view' button and render + components/preview.html. + + :param DataSet dataset: Dataset to check + :return bool: Whether an inline preview is available + """ + if not dataset.is_finished() or not dataset.num_rows: + return False + + extension = dataset.get_extension() + if extension == "zip": + # media archives are previewed as a carousel + return dataset.get_media_type() in ("image", "video", "audio") + + if extension in ("html", "gexf", "csv", "svg", "jpeg", "jpg", "png", "gif", "webp", "mp4"): + return True + + # anything else needs to be mappable to be rendered as a table + processor = dataset.get_own_processor() + return bool(processor and getattr(processor, "map_item", None) and callable(processor.map_item)) + @current_app.template_filter('chan_image') def _jinja2_filter_chan_image(tim, ext, board): @@ -402,6 +430,40 @@ def _jinja2_filter_media_url_from_filepath(filepath): # Convert to forward slashes for URL (works on both Windows and Linux) return relative_path.as_posix() +@current_app.template_filter("visible_parameters") +def _jinja2_filter_visible_parameters(parameters): + """ + Filter parameters dictionary to only retain those that should be visbile to a user + + :param dict parameters: + :return dict: + """ + result = {} + for key, value in parameters.items(): + if key in ("copied_from", "copied_at", "next", "attach_to", "frontend-confirm"): + # for internal 4CAT preset/dataset linking/parsing + continue + + elif key in ("pseudonymise", "user", "board", "datasource", "type", "label", "header", "expires-after", "email-complete", "original_timestamp", "session_id"): + # these are used, but not directly in the dataset parameter list + continue + + elif key in ("search-scope", "search_scope", "random_amount", "scope_length", "scope_density", "country_name"): + # ancient 4chan-related things + continue + + elif key in ("jst", "mst"): + # deprecated data sources used these, they contained cookie values (i.e. senstive data) + continue + + elif key.startswith("api_"): + # api keys etc, handled separately + continue + + result[key] = value + + return result + @current_app.template_filter('parameter_str') def _jinja2_filter_parameter_str(url): @@ -456,6 +518,21 @@ def explorer_css(datasource, scope_class="explorer-content-container"): return f".{scope_class} {{\n{css_content}\n}}" +@current_app.template_filter('idify') +def _jinja2_filter_idify(value): + """ + Turn string into safe ID string + + :param str value: + :return str: + """ + value = str(value).lower() + + value = re.sub(r"\s+", "-", value) + value = re.sub(r"[^a-z0-9-]", "", value) + + return value + @current_app.template_filter('hasattr') def _jinja2_filter_hasattr(obj, attribute): return hasattr(obj, attribute) @@ -507,6 +584,7 @@ def uniqid(): return { "__has_https": g.config.get("flask.https"), "__datenow": datetime.datetime.utcnow(), + "__now": time.time(), "__notifications": current_user.get_notifications(), "__user_config": lambda setting: g.config.get(setting), "__config": g.config, diff --git a/webtool/static/css/animations.css b/webtool/static/css/animations.css new file mode 100644 index 000000000..419e57743 --- /dev/null +++ b/webtool/static/css/animations.css @@ -0,0 +1,60 @@ +.slide-enter { + transition: transform 0.2s linear; + will-change: transform; + backface-visibility: hidden; +} + +.slide-from { + transform: translateY(100vh); +} + +.slide-to { + transform: translateY(0); +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +.spinner { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + color: var(--semidark); + pointer-events: none; +} + +.spinner .spinner-icon { + font-size: 2.5em; + animation: spin 0.8s linear infinite; +} + +.htmx-indicator { + opacity: 0; + transition: opacity 0.2s ease-in; +} + +.htmx-request .htmx-indicator, +.htmx-request.htmx-indicator { + opacity: 1; +} + +/* Reveal for content that appears while the user is watching, rather than on + page load - e.g. the analysis tree the moment a dataset finishes collecting. */ +.reveal-enter { + transition: opacity 0.4s ease-in, transform 0.4s ease-out; +} + +.reveal-from { + opacity: 0; + transform: translateY(-0.5rem); +} + +.reveal-to { + opacity: 1; + transform: translateY(0); +} diff --git a/webtool/static/css/base.css b/webtool/static/css/base.css new file mode 100644 index 000000000..a52c875a3 --- /dev/null +++ b/webtool/static/css/base.css @@ -0,0 +1,63 @@ +:root { + interpolate-size: allow-keywords; +} + +html { + width: 100%; + background: var(--bright); + font-size: 16px; +} + +body { + margin: 0 auto; + font-family: Chivo, serif; + font-size: var(--font-body); + border-top: 0; + border-bottom: 0; + padding-bottom: calc(var(--spacing-xlarge) * 2); +} + +a { + color: inherit; + text-decoration: none; +} + +ul, ol, dt, dd { + padding: 0; + margin: 0; +} + +ul, ol, dl { + list-style: none; +} + +nav li { + display: inline-block; +} + +.center-stage { + width: var(--max-width); + margin: 0 auto; +} + +pre { + overflow: auto; + font-family: monospace; + padding: var(--spacing-regular); + background: var(--panel-bg-0); +} + +button { + font: inherit; +} + +.sr-only { + /** content hidden, but not for screen readers */ + clip: rect(0 0 0 0); + clip-path: inset(50%); + height: 1px; + overflow: hidden; + position: absolute; + white-space: nowrap; + width: 1px; +} \ No newline at end of file diff --git a/webtool/static/css/colours.css b/webtool/static/css/colours.css new file mode 100644 index 000000000..fb262baa0 --- /dev/null +++ b/webtool/static/css/colours.css @@ -0,0 +1,22 @@ +/** --------------------- * + Colour definitions + * --------------------- */ +:root { + --accent: #CE411A; + --highlight: #FF5121; + --accent-alternate: #8EE51D; + --accent-okay: #26b018; + --accent-okay-light: #bee0ba; + --accent-warning: #ffcf13; + --accent-error: #ff2231; + --gray-dark: #DFDFDF; + --gray-darker: #6C6C6C; + --gray: #E9E9E9; + --gray-light: #EEE; + --contrast-bright: #F5F5F5; + --contrast-dark: #21202E; + --always-white: #FFF; + --text: #37373C; + --warning: var(--accent); + +} \ No newline at end of file diff --git a/webtool/static/css/components/analysis-tree.css b/webtool/static/css/components/analysis-tree.css new file mode 100644 index 000000000..3ef9a8102 --- /dev/null +++ b/webtool/static/css/components/analysis-tree.css @@ -0,0 +1,420 @@ +:root { + --journey-width: 54px; +} + +/** base tree container **/ +.analysis-results { + position: relative; + padding-left: var(--journey-width); + box-sizing: border-box; +} + +/** very first item connects to the header box **/ +.analysis-results > ul > li:first-child { + padding-top: var(--spacing-xlarge); +} + +.analysis-results > ul > li:first-child::before { + /** funnel **/ + content: ''; + position: absolute; + top: 0; + left: calc((var(--journey-width) * 0.5) + (var(--stroke-journey) * 1.25)); + width: calc(var(--journey-width) - (var(--stroke-journey) * 2)); + height: calc(var(--journey-width) / 2); + background: var(--semidark); + mask: radial-gradient(#0000 71%, #F09 1%) 10000% 10000%/99.5% 200%; +} + +/** very last item of analysis tree root level **/ +.analysis-results > ul > li:last-child { + border-color: transparent; +} + +/** any item **/ +.analysis-results .tree > li { + border-left: var(--stroke-journey) solid var(--semidark); + padding-left: calc(var(--spacing-xlarge) / 2); +} + +/** last item of sub-tree **/ +.analysis-results .tree .tree > li:last-child { + border-left-color: transparent; +} + +/** any processor card wrapper **/ +.analysis-results article { + border-left: var(--stroke-journey) solid transparent; + padding-left: calc(var(--spacing-xlarge) / 2); +} + +/** processor card wrappers of processors with children **/ +.analysis-results li.with-children > article { + border-left-color: var(--semidark); +} + +/** processor datetime **/ +.analysis-results .subway-station { + position: absolute; + left: 0; + font-size: var(--font-body); +} + +.analysis-results .subway-station .date { + font-weight: bold; +} + +.analysis-results .subway-station .time { + color: var(--stroke-bright) +} + +.analysis-results .subway-station span { + display: block; + text-align: right; + text-transform: uppercase; +} + +.analysis-results .subway-station .new-notice { + position: absolute; + top: 0; + left: calc(-3.8rem - var(--spacing-small)); + text-align: right; + width: 3rem; + padding-right: calc(var(--font-normal)); +} + +.analysis-results .subway-station .new-notice::before { + content: ''; + display: block; + position: absolute; + top: 25%; + right: 0; + background: var(--red); + width: calc(var(--font-normal) * 0.5); + height: calc(var(--font-normal) * 0.5); + border-radius: 50%; +} + +.journey-marker { + /** big circle before each dataset **/ + font-family: 'Font Awesome 7 Free', monospace; + font-weight: 900; + background: var(--dark); + color: var(--bright); + border: 0; + border-radius: var(--font-body); + width: calc(var(--font-body) * 2); + height: calc(var(--font-body) * 2); + display: block; + text-align: center; + + position: absolute; + top: calc((var(--font-body) * 2.25) + (var(--stroke-journey) / 2) - var(--font-body)); + left: calc((-0.5 * var(--spacing-xlarge)) - (var(--stroke-journey) / 2) - var(--font-body)); + z-index: var(--z-journey-marker); + + /** this is to get the glyph in exactly the center **/ + padding: 0; + padding-top: 1px; +} + +.journey-marker.end::after { + content: ''; + height: var(--stroke-journey); + background: var(--dark); + width: 100%; + position: absolute; + left: 0; + bottom: 0; +} + +.in-progress .journey-marker.end::after { + top: 100%; + width: calc(100% + (2 * var(--stroke-journey))); + left: calc(var(--stroke-journey) * -1); +} + +.journey-marker .status-indicator { + /** coloured dot in middle of journey indicator **/ + position: absolute; + top: 50%; + left: 50%; + overflow: hidden; + height: calc(var(--font-body) * 0.75); + width: calc(var(--font-body) * 0.75); + border-radius: calc(var(--font-body) * 0.375); + margin-top: calc(var(--font-body) * -0.375); + margin-left: calc(var(--font-body) * -0.375); + border: 0; + text-indent: -10em; + animation-timing-function: linear; +} + +.in-progress > article > div > .journey-marker { + animation: pulse-colour 1.25s infinite alternate; + border: var(--stroke-journey) solid var(--semidark); +} + +.in-progress > article > div > .journey-marker > * { + animation: status-spin 1.25s infinite alternate; + color: var(--bright); +} + +@keyframes status-pulse { + from { + height: calc(var(--font-body) * 0.75); + width: calc(var(--font-body) * 0.75); + border-radius: calc(var(--font-body) * 0.375); + margin-top: calc(var(--font-body) * -0.375); + margin-left: calc(var(--font-body) * -0.375); + } + to { + width: calc((var(--font-body) * 2) - var(--stroke-journey) * 2); + height: calc((var(--font-body) * 2) - var(--stroke-journey) * 2); + border-radius: var(--font-body); + margin-top: calc(((var(--font-body) * 2) - var(--stroke-journey) * 2) * -0.5); + margin-left: calc(((var(--font-body) * 2) - var(--stroke-journey) * 2) * -0.5); + } +} + +@keyframes status-spin { + from { + transform:rotate(0deg); + color: var(--bright); + } + to { + transform:rotate(360deg); + color: var(--fourcat-accent); + } +} + +@keyframes pulse-colour { + from { + border-color: var(--semidark); + background: var(--semidark); + } + to { + border-color: var(--fourcat-accent); + background: var(--bright); + } +} + +/** journey marker of first item in tree, and any item with children **/ +:first-child > article > .module-card > .journey-marker::before, .with-children > article > .module-card > .journey-marker::before { + /** block top of border that extends above journey marker **/ + content: ''; + display: block; + position: absolute; + top: calc(-1 * (var(--spacing-regular) + 0.5em)); + left: calc(50% - var(--stroke-journey)); + background: var(--bright); + width: calc((var(--stroke-journey) * 2) + var(--stroke-light)); + height: calc(var(--spacing-regular) + 0.5em); +} + +.analysis-results .module-card { + padding: var(--spacing-regular) 0; +} + +.analysis-results .module-card::before { + /** line connecting card to journey marker. A single box rather than a pair + of half-width borders, so that the stroke is snapped to the pixel grid + in one piece instead of each half rounding on its own **/ + content: ''; + display: block; + position: absolute; + top: calc(var(--spacing-regular) + var(--font-body) + (var(--stroke) / 2)); + left: calc(-1 * var(--spacing-regular)); + width: var(--spacing-regular); + border: var(--stroke) solid var(--semidark); + border-width: calc(var(--stroke-journey) / 2) 0; +} + +.analysis-results .module-card > .card-header::before { + /** curve towards journey marker **/ + border: var(--stroke-journey) solid var(--semidark); + border-bottom-left-radius: var(--double-radius); + border-top-width: 0; + border-right-width: 0; + width: calc(var(--font-body) * 1.5); + height: calc(var(--font-body) * 2.25); + outline: var(--stroke-journey) solid var(--bright); + position: absolute; + top: 0; + background: transparent; + display: block; + content: ''; + left: calc((-1 * var(--spacing-xlarge)) - (2 * var(--stroke-journey))); +} + +.analysis-results .module-card > .card-header::after, .big-module-button::after { + /** cover for outline on top of curve (break in stroke otherwise) **/ + content: ''; + position: absolute; + top: calc(var(--stroke-journey) * -1); + background: transparent; + display: block; + left: calc((-1 * var(--spacing-xlarge)) - (2 * var(--stroke-journey))); + width: var(--stroke-journey); + height: var(--stroke-journey); + background: var(--semidark); +} + +/** 'run processor' button/icon **/ +.run-processor-button, button.run-processor-button { + color: var(--fourcat-accent); + display: inline-block; + border: var(--stroke-journey) solid var(--fourcat-accent); + border-radius: 50%; + width: calc(var(--font-body) * 2); + height: calc(var(--font-body) * 2); + font-size: calc(var(--font-body) * 1.1); + line-height: calc(var(--font-body) * 1.6); +} + +button.run-processor-button { + outline: var(--stroke-journey) solid var(--bright) +} + +aside.run-child-processor { + position: absolute; + top: calc(var(--font-body) * 3); + left: calc((-0.5 * var(--spacing-xlarge)) - (var(--stroke-journey) / 2) - var(--font-body)); + color: var(--fourcat-accent); + background: var(--bright); + font-size: calc(var(--font-body) * 2); + padding-top: var(--spacing-regular); +} + +aside.run-child-processor::before { + position: absolute; + top: 0; + left: calc(50% - var(--stroke-journey) + 1px); + height: var(--spacing-regular); + border: var(--stroke) dotted var(--stroke-bright); + content: ''; +} + +aside.run-child-processor button:hover { + font-weight: 900; + cursor: pointer; + background: var(--fourcat-accent); + color: var(--bright); +} + +/** any badge list in a processor card that is not the last one **/ +.analysis-results .module-card .badge-list:not(:last-child) { + border-bottom: var(--stroke-light) dashed var(--panel-bg-2); +} + +/** 'run new processor' button at bottom **/ +.big-button-wrap { + position: relative; + padding-left: calc(var(--spacing-xlarge) + (2 * var(--stroke-journey))); + box-sizing: border-box; + margin-left: calc((var(--spacing-xlarge) / 2 * -1) - var(--stroke-journey)); +} + +.big-button-wrap em { + color: var(--fourcat-accent); + display: block; + font-style: normal; +} + +.big-module-button, .run-hint p { + border-radius: var(--radius); + border: var(--stroke-light) solid var(--dark); + border-bottom-width: var(--stroke-journey); + background: transparent; + padding: var(--spacing-small); + font-size: var(--font-h1); + text-align: center; + width: 100%; +} + +.big-module-button:hover { + cursor: pointer; + background: var(--button-bg-regular); +} + +.big-module-button:hover, .big-module-button:hover em { + color: var(--bright); +} + +.big-module-button .byline { + margin: var(--spacing-tiny) 0 0 0; + font-size: var(--font-normal) +} + +.run-processor-button.inline { + width: calc(var(--font-body) * 1.45); + height: calc(var(--font-body) * 1.45); + line-height: calc(var(--font-body) * 1.5); +} + +.big-module-button:hover .run-processor-button { + border-color: var(--bright); + color: var(--bright); +} + +.big-module-button::before { + /** curve towards button **/ + content: ''; + display: block; + position: absolute; + left: 0; + top: 0; + border: var(--stroke-journey) dotted var(--semidark); + border-bottom-left-radius: var(--double-radius); + border-top-width: 0; + border-right-width: 0; + width: calc(var(--spacing-xlarge)); + height: calc((100% - var(--stroke-journey)) / 2); + outline: var(--stroke-journey) solid var(--bright); +} + +.big-module-button::after { + left: 0; +} + +.run-hint { + border-left: var(--stroke-journey) solid var(--semidark); + position: relative; + display: none; +} + +.tree > li:not(.with-children) > .run-hint { + border-left-color: transparent; + height: 2rem; + padding-top: calc(2.5rem + var(--spacing-regular)); + margin-top: calc(-1 * (6rem - var(--spacing-xlarge) - var(--stroke-journey))); +} + +.run-hint::before { + content: ''; + display: block; + position: absolute; + left: calc(var(--stroke-journey) * -1); + top: calc(-100%); + border: 0 dotted var(--semidark); + border-bottom-left-radius: var(--double-radius); + border-left-width: var(--stroke-journey); + border-bottom-width: var(--stroke-journey); + width: var(--spacing-xlarge); + height: calc(var(--spacing-xlarge) + var(--spacing-small)); +} + +.tree > li:not(.with-children) > .run-hint::before { + top: 0; + height: 5.8em; + z-index: var(--z-run-hint); + border-left: var(--stroke-journey) dotted var(--semidark); +} + +.run-hint p { + width: max-content; + font-size: var(--font-body); + margin: 0; + margin-left: var(--spacing-xlarge); + padding: var(--spacing-small) var(--spacing-large); +} \ No newline at end of file diff --git a/webtool/static/css/components/annotations.css b/webtool/static/css/components/annotations.css new file mode 100644 index 000000000..66c181ada --- /dev/null +++ b/webtool/static/css/components/annotations.css @@ -0,0 +1,257 @@ +/* Annotations: the fields a dataset can be annotated with, shown as a row in + its metadata box, and the inputs for them on the items in the Explorer. + + Annotation inputs save themselves as they are edited, so the chip next to + one is the only feedback there is that a value has landed - it is styled to + be noticeable without being loud, since it appears on every keystroke pause. + + What the inputs themselves look like is not decided here: the boxes that + hold them carry .form-controls, so they get the same chrome as any other + 4CAT input (see 'shared input chrome' in forms.css). This file only says how + those controls are laid out and sized inside an annotation box. */ + +/* --- the fields row in the dataset metadata box --- */ +.badge-list .annotation-fields-box { + flex: 1 1 auto; + min-width: 0; + grid-template-columns: auto 1fr; + align-items: start; +} + +.annotation-fields-box > dd { + min-width: 0; + white-space: normal; + background: none; +} + +/* the toggle fills its cell, so the whole of the left-hand column responds to + the pointer rather than just the words in it */ +.annotation-fields-box > dt { + padding: 0; + margin-right: var(--spacing-small); +} + +.annotation-fields-box .annotation-fields-summary { + display: block; + width: 100%; + font: inherit; + color: inherit; + background: none; + border: 0; + padding: var(--spacing-tiny); + text-transform: inherit; + text-align: left; + cursor: pointer; +} + +.annotation-fields-box .annotation-fields-summary:hover i { + color: var(--fourcat-accent); +} + +.annotation-fields-empty { + color: var(--semidark); + font-style: italic; +} + +.annotation-fields-editor-host { + position: relative; +} + +/* --- the field editor --- */ + +.annotation-fields-editor { + padding-top: var(--spacing-regular); + color: var(--dark); +} + +.annotation-field-list { + display: flex; + flex-direction: column; + gap: var(--spacing-small); +} + +.annotation-field { + padding: var(--spacing-small); + background: var(--brighten-12); + border-radius: var(--radius); +} + +.annotation-field-main { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--spacing-small); +} + +/* the label takes whatever room the type picker leaves. Both set a flex basis, + which is what decides their width - the shared chrome's width:100% only + applies where a control is not a flex item with a basis of its own */ +.annotation-field-label, .annotation-field-type { + flex: 1 1 12rem; + min-width: 0; + max-width: 12rem; +} + +.annotation-field-origin { + margin: 0; + color: var(--semidark); + font-size: var(--font-body); +} + +/* the row's controls sit at its end, past the label and the type picker */ +.annotation-field-controls { + display: flex; + flex: 0 0 auto; + gap: var(--spacing-tiny); + margin-left: auto; +} + +.annotation-field-controls button:disabled { + opacity: 0.3; + cursor: default; +} + +.annotation-field-options { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--spacing-tiny); + margin: var(--spacing-small) 0 0 var(--spacing-regular); + row-gap: var(--spacing-regular); +} + +.annotation-field-options dd { + background: none; +} + +/* an option is a text box plus its remove button. The option box gets the + width, so the input inside it has something definite to fill */ +.annotation-field-option { + display: flex; + flex: 0 1 10rem; + align-items: center; + gap: var(--spacing-tiny); +} + +.annotation-field-option input { + flex: 1 1 auto; + min-width: 0; +} + +/* add/save stay together at the left edge */ +.annotation-fields-actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--spacing-small); + margin-top: var(--spacing-small); +} + +.annotation-fields-impact { + margin: 0 0 var(--spacing-regular) var(--spacing-regular); + list-style: disc; +} + +.annotation-fields-note { + margin: var(--spacing-small) 0 0 0; + color: var(--semidark); + font-style: italic; +} + +.annotation-fields-saved, .annotation-fields-warning { + margin-top: var(--spacing-small); +} + +/* --- annotations on an item --- */ + +.item-annotations { + display: flex; + flex-direction: column; + gap: var(--spacing-small); +} + +.item-annotation { + display: flex; + align-items: center; + gap: var(--spacing-small); + font-size: var(--font-body); +} + +/* folded away by the eye button on that field in the editor. A way of reading + the items: the annotation is still there and still saved, just not shown */ +.item-annotation.is-hidden { + display: none; +} + +.annotation-label { + flex: 0 0 auto; + padding-top: var(--spacing-tiny); + max-width: 25%; +} + +.item-annotation-value { + display: flex; + flex: 1 1 0; + align-items: center; + gap: var(--spacing-small); + min-width: 0; +} + +.item-annotation-input { + flex: 1 1 16rem; + min-width: 0; + max-width: 100%; +} + +textarea.item-annotation-input { + min-height: 4em; +} + +.item-annotation-input.is-readonly { + flex: 1 1 auto; + color: var(--dark); +} + +.item-annotation-info-buttons { + margin-left: auto; +} + +.item-annotation-info-buttons div { + display: inline-block; +} + +.item-annotation-info { + flex: 0 0 auto; +} + +/* a checkbox field renders as .option-tags (forms.css); it only needs to be + told to take the room the input of any other field type would */ +.item-annotation-value > .option-tags { + flex: 1 1 16rem; + min-width: 0; +} + +/* where a value stands: appears while saving, then settles into a tick that + stays, so it is clear the value is safe without a save button existing */ +.annotation-state { + flex: 0 0 auto; + font-size: var(--font-body); + color: var(--semidark); +} + +.annotation-state.is-saved { + color: var(--green); +} + +.annotation-state.is-refused { + color: var(--yellow); +} + +.annotation-state.is-error { + color: var(--red); +} + +#explorer-spinner { + margin-top: var(--spacing-large); + border: none; +} \ No newline at end of file diff --git a/webtool/static/css/components/badges.css b/webtool/static/css/components/badges.css new file mode 100644 index 000000000..fb9412658 --- /dev/null +++ b/webtool/static/css/components/badges.css @@ -0,0 +1,279 @@ +.badge-list { + display: flex; + font-size: var(--font-body); + position: relative; + column-gap: var(--spacing-tiny); +} + +.badge-list.no-overflow { + flex-wrap: wrap; + row-gap: var(--spacing-small); +} + +.badge-list.with-overflow { + overflow-y: auto; + white-space: nowrap; +} + +.badge-list.with-overflow::before { + display: block; + content: ''; + position: absolute; + top: 0; + right: 0; + width: 50%; + height: 100%; + background: linear-gradient(90deg, transparent 0%, transparent 80%, var(--bright) 100%); +} + +.dark .badge-list.with-overflow::before { + background: linear-gradient(90deg, transparent 0%, transparent 80%, var(--panel-bg-3) 100%); +} + +/* the wrapper around a dt/dd pair. Direct children only: this used to catch + every nested div too, which meant any component rendered inside a badge list + had its own layout silently replaced by this grid */ +.badge-list > div { + display: grid; + grid-template-columns: auto auto; + align-items: center; +} + +.badge-list dt, .badge-list dd { + padding: var(--spacing-tiny); + white-space: nowrap; +} + +.badge-list dt { + background: transparent; + text-transform: uppercase; +} + +.badge-list dd { + background: var(--brighten-12); + padding: var(--spacing-tiny) var(--spacing-small); +} + +.badge-list.dataset-status dt { + align-self: start; +} + +.badge-list.dataset-status dd { + white-space: wrap; +} + +.status-badge dt { + display: none; +} + +.status-badge dd { + text-transform: uppercase; +} + +.dark .badge-list .status-badge-annotated dd { + background: transparent; +} + +.status-badge.status-good dd { + background: var(--green); + color: var(--green-light); +} + +.status-badge.status-bad dd { + background: var(--red); + color: var(--red-light); +} + +.status-badge.status-neutral dd { + background: var(--blue); + color: var(--blue-light); +} + +.status-badge.status-warning dd { + background: var(--yellow); + color: var(--yellow-light); +} + +.badge-list .status-badge-annotated.status-good dt, .status-indicator.status-good { + border: var(--stroke) solid var(--green-light); + background: var(--green-light); + color: var(--green); +} + +.badge-list .status-badge-annotated.status-good dd { + border-color: var(--green-light); + color: var(--green); +} + +.dark .badge-list .status-badge-annotated.status-good dd { + color: var(--green-light); + border: var(--stroke) solid var(--green-light); +} + +.badge-list .status-badge-annotated.status-bad dt, .status-indicator.status-bad { + border: var(--stroke) solid var(--red-light); + background: var(--red-light); + color: var(--red); +} + +.badge-list .status-badge-annotated.status-bad dd { + border-color: var(--red-light); + color: var(--red); +} + +.dark .badge-list .status-badge-annotated.status-bad dd { + color: var(--red-light); + border: var(--stroke) solid var(--red-light); +} + +.badge-list .status-badge-annotated.status-neutral dt, .status-indicator.status-neutral { + border: var(--stroke) solid var(--blue-light); + background: var(--blue-light); + color: var(--blue); +} + +.badge-list .status-badge-annotated.status-neutral dd { + border-color: var(--blue-light); + color: var(--blue); +} + +.dark .badge-list .status-badge-annotated.status-neutral dd { + color: var(--blue-light); + border: var(--stroke) solid var(--blue-light); +} + +.badge-list .status-badge-annotated.status-warning dt, .status-indicator.status-warning { + border: var(--stroke) solid var(--yellow-light); + background: var(--yellow-light); + color: var(--yellow); +} + +.badge-list .status-badge-annotated.status-warning dd { + border-color: var(--yellow-light); + color: var(--yellow); +} + +.dark .badge-list .status-badge-annotated.status-warning dd { + color: var(--yellow-light); + border: var(--stroke) solid var(--yellow-light); +} + +/* Tags for modules. The single definition of what a tag looks like, wherever it + is rendered: as a
on a module card (components/module-tags.html) or as a + - - {% endif %} - - {# Store some invisible data here to we can retrieve in with JS #} - - - {% endif %} +{# The annotations of one item. + + One block per annotation field. Human-editable fields render an input that + saves itself; processor-generated ones render their value as read-only text, + attributed to the dataset that produced it, and are left out entirely when + that processor had nothing to say about this item. + + Expects: item, annotation_fields, annotations, from_datasets, processors, + dataset, can_annotate +#} +{% if annotation_fields %} + {% set item_id = item.id|string %} + {% set item_annotations = annotations.get(item_id, {}) %} +
+ {% for field_id, field in annotation_fields.items() %} + {% set annotation = item_annotations.get(field_id) %} + {% if not (field.from_dataset and not (annotation and annotation.value)) %} + {% include "explorer/annotation-input.html" %} {% endif %} - {% endfor %} - {% endif %} -
\ No newline at end of file + {% endfor %} + +{% endif %} diff --git a/webtool/templates/explorer/items.html b/webtool/templates/explorer/items.html new file mode 100644 index 000000000..9c566dc67 --- /dev/null +++ b/webtool/templates/explorer/items.html @@ -0,0 +1,44 @@ +{# One page of dataset items, with the pagination that goes with it. + + Swapped into #explorer-body when paging or sorting, and included directly by + explorer/pane.html for the first page. Both the list and the pagination + change per page, so they travel together and nothing needs an out-of-band + swap. + + The pagination is shown twice, above the items and below them, so a page + that is longer than the screen can be paged without scrolling back. The one + above sits directly under the controls header. + + Expects the context built by explorer.explorer_pane. +#} +{% if item_count > max_items %} +

+ + Large dataset — only the first {{ max_items|commafy }} items can be shown here. Use a filter processor to + narrow it down. +

+{% endif %} + +{% with above = True, label = "Item pages, above the items" %} + {% include "explorer/pagination.html" %} +{% endwith %} + +{# .explorer-content-container / .explorer-content / .items is the structure + every data source's Explorer stylesheet is written against - see + datasources/*/[name]-explorer.css #} +
+
+
    + {% for item in items %} + {% set item_index = loop.index - 1 %} + {% if not (datasource == "media-import" and item.id == ".metadata.json") %} + {% include "explorer/item.html" %} + {% endif %} + {% endfor %} +
+
+
+ +
+ {% include "explorer/pagination.html" %} +
diff --git a/webtool/templates/explorer/pagination.html b/webtool/templates/explorer/pagination.html index 06adde569..1c241a945 100644 --- a/webtool/templates/explorer/pagination.html +++ b/webtool/templates/explorer/pagination.html @@ -1,70 +1,30 @@ - \ No newline at end of file +{# Page links for the Explorer. + + The bar itself is the shared one every paged listing renders + (components/pagination.html); only what a link *is* belongs here. Each one + swaps the item list in place rather than loading a page, and pushes its own + URL, so a page can still be linked to and the back button still works. The + href is what it pushes, and what happens without JavaScript. + + Included above and below the items, so a long page can be paged from either + end; both are inside #explorer-body, so both are replaced along with them. + + Expects: dataset, page, item_count, items_per_page, max_items, sort, reverse + label (optional) what to call this bar, for screen readers + above (optional) whether this is the copy above the items +#} +{% from "components/pagination.html" import page_links %} +{% set shown_count = item_count if item_count < max_items else max_items %} +{% set pages = ((shown_count / items_per_page)|round(0, 'ceil'))|int %} +{# every window onto the dataset keeps the sort it was opened with #} +{% set order = "reverse" if reverse else "regular" %} +{% call(number, link_label) page_links(page, pages, label|default("Item pages"), + "pagination above" if above else "pagination") %} + {{ link_label }} +{% endcall %} diff --git a/webtool/templates/explorer/pane.html b/webtool/templates/explorer/pane.html new file mode 100644 index 000000000..8042ed757 --- /dev/null +++ b/webtool/templates/explorer/pane.html @@ -0,0 +1,25 @@ +{# The Explorer pane on the dataset page. + + Swapped into #explorer-pane the first time the 'Annotate & Explore' toggle is + flipped. Holds the controls, the items and the pagination; paging and sorting + afterwards replace only the last two (see explorer/items.html), so the + controls and the data source's stylesheet below survive them. + + Expects the context built by explorer.explorer_pane. +#} +{# The data source's own item styling, scoped to .explorer-content-container by + the filter - the structure every data source's stylesheet is written against, + so nothing here may rename it. It lives in this partial rather than in the + items one so that paging does not re-inject it on every swap. #} + +{% if datasource == "4chan" %} + +{% endif %} + +{% include "explorer/controls.html" %} + +
+ {% include "explorer/items.html" %} +
diff --git a/webtool/templates/layout.html b/webtool/templates/layout.html index 269e804ab..36e1d95f0 100644 --- a/webtool/templates/layout.html +++ b/webtool/templates/layout.html @@ -1,25 +1,28 @@ {% set navigation = namespace(current="") %}{% block breadcrumbs %}{% endblock %}{% block subbreadcrumbs %}{% endblock %} - {% block title %}{{ __user_config("4cat.name_long") }}{% endblock %} • {{ __user_config("4cat.name") }} + {% block title %}{{ __user_config("4cat.name_long") }}{% endblock %} • {{ __user_config("4cat.name") }} - - + + + + + + + + + + + + + + + - - - - - - - {% if navigation.current == "about" %} - - {% endif %} - @@ -31,70 +34,86 @@ - - -

- - {% block site_header %}{{ __user_config("4cat.name_long") }}{% endblock %} -

- + + + + + +
{% block pre_body %} {% endblock %} - {% block body %} + {% block body %}

Welcome.

{% endblock %} +
- + - - + + \ No newline at end of file diff --git a/webtool/templates/layout.old.html b/webtool/templates/layout.old.html new file mode 100644 index 000000000..691d66bdd --- /dev/null +++ b/webtool/templates/layout.old.html @@ -0,0 +1,100 @@ +{% set navigation = namespace(current="") %}{% block breadcrumbs %}{% endblock %}{% block subbreadcrumbs %}{% endblock %} + + + {% block title %}{{ __user_config("4cat.name_long") }}{% endblock %} • {{ __user_config("4cat.name") }} + + + + + + + + + + + + + + + {% if navigation.current == "about" %} + + {% endif %} + + + + + + + + + + + + + + + +

+ + {% block site_header %}{{ __user_config("4cat.name_long") }}{% endblock %} +

+ + + {% block pre_body %} + {% endblock %} + + {% block body %} +

Welcome.

+ {% endblock %} + + + + + diff --git a/webtool/templates/module-catalog.html b/webtool/templates/module-catalog.html new file mode 100644 index 000000000..5bd2a8ab5 --- /dev/null +++ b/webtool/templates/module-catalog.html @@ -0,0 +1,56 @@ +{% extends "layout.html" %} + +{% block title %}{% if detail %}{{ detail.module.title }}{% else %}Module catalog{% endif %}{% endblock %} +{% block body_class %}plain-page module-catalog{% endblock %} +{% block breadcrumbs %}{% set navigation.current = "datasources" %}{% endblock %} + +{% block body %} + {# Every module 4CAT knows, as the same module cards used in the slideout. + Selecting one loads its full detail into the pane at the top of the page + and puts it in the address bar, so a view can be linked to. #} +
+
+
+ {% if detail %} + {# same context the htmx endpoint renders this partial with #} + {% with module = detail.module, module_type = detail.module_type, + requirements = detail.requirements, datasource = detail.datasource %} + {% include "components/module-detail.html" %} + {% endwith %} + {% else %} +

+ Select a modules below to get more information and see how to run it. +

+ {% endif %} +
+ {% with spinner_id = "module-detail-spinner" %}{% include "components/spinner.html" %}{% endwith %} +
+ + {# the same panel the slideout shows this grid in: a header holding the + search box, over a body holding the tag filter and the cards #} +
+
+
+
    +

  • Browse modules

    • +
+
+
+
    + {% with search_placeholder = "Search modules…", + search_label = "Search modules by title, description or tag" %} + {% include "components/module-search.html" %} + {% endwith %} +
+
+
+ +
+ {% with card_template = "components/module-catalog-card.html", noun = "modules", + request_label = "Missing a module? Request a new one here." %} + {% include "components/module-grid.html" %} + {% endwith %} +
+
+
+{% endblock %} diff --git a/webtool/templates/preview/partials/csv.html b/webtool/templates/preview/partials/csv.html new file mode 100644 index 000000000..b8d0e05b5 --- /dev/null +++ b/webtool/templates/preview/partials/csv.html @@ -0,0 +1,33 @@ +{# Tabulated preview fragment. Rendered on its own (inline embed) or, for + legacy consumers, inside an iframe/standalone page — hence the stylesheet + link, which is a cache hit when injected into an already-styled page. #} + +
+{% if dataset.num_rows > max_items %} +

Note: only the first {{ "{:,}".format(max_items) }} of {{ "{:,}".format(dataset.num_rows) }} total items of this dataset are shown in this preview (~{{ ((max_items / dataset.num_rows) * 100)|round(0)|int }}%). Download the dataset file for the rest of the data.

+{% endif %} + + {% set ns = namespace(links=[]) %} + {% for row in rows %} + {% set outer_loop = loop %} + + {% for column, cell in row.items() %} + {% set inner_loop = loop %} + {% if outer_loop.index == 1 %} + {% if "link" in cell or "url" in cell %} + {% set ns.links = ns.links + [inner_loop.index] %} + {% endif %} + {% endif %} + + {% autoescape false %} + {{ cell|e|replace("\n","
\n")|add_ahref(ellipsiate=50)|add_colour|safe }} + {% endautoescape %} + + {% endfor %} + + {% endfor %} +
+
diff --git a/webtool/templates/preview/partials/image.html b/webtool/templates/preview/partials/image.html new file mode 100644 index 000000000..ee19af2e5 --- /dev/null +++ b/webtool/templates/preview/partials/image.html @@ -0,0 +1,17 @@ +{# Image/video preview fragment. See csv.html partial for why the stylesheet + link is included. Zoom is a pure-CSS checkbox toggle, so no JS is needed. #} + +
+
+ + +
+
diff --git a/webtool/templates/preview/partials/json.html b/webtool/templates/preview/partials/json.html new file mode 100644 index 000000000..e8fa4c7f8 --- /dev/null +++ b/webtool/templates/preview/partials/json.html @@ -0,0 +1,9 @@ +{# JSON/NDJSON preview fragment. See csv.html partial for why the stylesheet + link is included. #} + +
+{% if truncated %} +

Note: only the first {{ "{:,}".format(truncated) }} of {{ "{:,}".format(dataset.num_rows) }} total items of this dataset are shown in this preview (~{{ ((truncated / dataset.num_rows) * 100)|round(0)|int }}%)

+{% endif %} +
{{ json|safe }}
+
diff --git a/webtool/templates/preview/partials/media.html b/webtool/templates/preview/partials/media.html new file mode 100644 index 000000000..840356de4 --- /dev/null +++ b/webtool/templates/preview/partials/media.html @@ -0,0 +1,56 @@ +{# Media archive preview fragment: a carousel of the first items in an + image/video/audio zip. See the csv.html partial for why the stylesheet link + is included. Each slide streams its file on demand via the get_result + endpoint's zip_member support. #} + +
+{% if not members %} +

No media items could be found in this archive.

+{% else %} + {% if dataset.num_rows > max_items %} +

Showing the first {{ members|length }} of {{ "{:,}".format(dataset.num_rows) }} items in this archive. Download the dataset file for the rest.

+ {% endif %} +
+ + +
+ {% for member in members %} + {% set file_ext = (member.rsplit('.', 1)[-1]|lower) if '.' in member else '' %} + {% set src = url_for('dataset.get_result', dataset_key=dataset.key, zip_member=member) %} +
+ {% if file_ext in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg', 'tiff', 'ico', 'heic', 'heif'] %} + {{ member }} + {% elif file_ext in ['mp4', 'avi', 'mov', 'wmv', 'flv', 'webm', 'mkv', 'm4v'] %} + + {% elif file_ext in ['mp3', 'wav', 'ogg', 'flac', 'aac', 'm4a', 'wma'] %} + + {% else %} +
+ +
+ {% endif %} +
{{ member }}
+
+ {% endfor %} +
+ + + +

+ / {{ members|length }} +

+
+{% endif %} +
diff --git a/webtool/templates/result.html b/webtool/templates/result.html deleted file mode 100644 index 4a4ee2297..000000000 --- a/webtool/templates/result.html +++ /dev/null @@ -1,9 +0,0 @@ -{% extends "layout.html" %} - -{% block title %}{% if dataset.query %}Dataset: {{ dataset.get_label() }}{% else %}Dataset{% endif %}{% endblock %} -{% block body_class %}plain-page result-page{% endblock %} -{% block breadcrumbs %}{% set navigation.current = "dataset" %}{% endblock %} - -{% block body %} -{% include "components/result-details.html" %} -{% endblock %} \ No newline at end of file diff --git a/webtool/templates/results.html b/webtool/templates/results.html index 2a2354b9e..9380fe23a 100644 --- a/webtool/templates/results.html +++ b/webtool/templates/results.html @@ -1,93 +1,100 @@ {% extends "layout.html" %} -{% block title %}Datasets & previous results{% endblock %} -{% block body_class %}result-list plain-page{% endblock %} +{% block title %}My datasets{% endblock %} +{% block body_class %}result-list{% endblock %} {% block breadcrumbs %}{% set navigation.current = "dataset" %}{% endblock %} {% block body %} -
-
-

Recently created datasets

- - {% if datasets %} -
    - {% for dataset in datasets %}{% if dataset.key_parent in ("", None) %} -
  1. - -
    -
    - show result - {% if __user_config("ui.show_datasource") %}{{ dataset.parameters.datasource if "datasource" in dataset.parameters else "4chan" }}{% if "board" in dataset.parameters and dataset.parameters.board %}/{{ dataset.parameters.board }}/{% endif %}{% endif %} -

    {{ dataset.get_label() }}

    - {% if dataset.is_finished() %} - ({{ "{:,}".format(dataset.num_rows) }} item{% if dataset.num_rows != 1 %}s{% endif %}) - {% endif %} -
    -
    {% include 'components/result-metadata.html' %}
    -
    +
    +

    Datasets

    -
    - {% if not dataset.is_finished() or dataset.num_rows == 0 %} -

    {% include "components/result-status.html" %}

    - {% else %} - {% if dataset.get_results_path().exists() %} - {% if dataset.get_own_processor().map_item or dataset.annotation_fields %} - Download csv - {% else %} - Download {{ dataset.result_file.split(".")[-1] }} - {% endif %} - {% endif %} + {# Which datasets are being looked at, over how they are narrowed + down. The first row are links, so picking a set takes effect at + once; it carries the filters along, so they keep applying within + the set that was picked. The second row needs submitting, so the + button doing that closes it off. #} +
    +
  2. My + own datasets
    3. +
  3. Favourites + only
    4. + - {% if dataset.is_finished() and dataset.num_rows > 0 %} -
    - Analysis{% if dataset.get_children()|length > 0 %} (+{{ dataset.get_children()|length }}){% endif %} -
    +
      +
    • + +
      • + {% if current_user.is_admin %} +
    • + +
      • +
    • + +
      • {% endif %} +
    • + {# only what there is something to filter to: one option + per data source these datasets were collected from, + or per variant of one where it has them #} + +
      • +
    • + +
      • +
    • + +
      • +
    + + + {% if datasets %} +
      + {% for dataset in datasets %}{% if dataset.key_parent in ("", None) %} +
    1. + {% with lite_header=True %} + {% include "dataset-page/dataset-card.html" %} + {% endwith %}
      2. {% endif %}{% endfor %}
    {% else %} -

    No datasets available.{% if depth == "favourites" %} You can mark datasets as 'favourite' to easily find them in this view.{% endif %}

    {% endif %}
- {% include "components/pagination.html" %} + {% from "components/pagination.html" import page_links %} +{% call(number, label) page_links(pagination.page, pagination.pages, "Pages", "pagination", True) %} + {{ label }} +{% endcall %} {% endblock %} diff --git a/webtool/views/api_module_map.py b/webtool/views/api_module_map.py new file mode 100644 index 000000000..b49266ca1 --- /dev/null +++ b/webtool/views/api_module_map.py @@ -0,0 +1,67 @@ +""" +Module map API. + +Thin JSON endpoints over `common.lib.module_map` -- each just builds the +ModuleMap and calls one method, so the data layer stays in common/lib and any +UI can be built against these without touching it. 'Module' here covers both +processors and data sources: a data source's search worker is a processor too, +and the map flags it with `is_datasource`. + +Login-gated. Demonstrates what the declarative Compatibility specs make computable +(search, "how to run this", shape buckets, follow-ups) with no datasets and no +database. +""" +from flask import Blueprint, current_app, jsonify, request +from flask_login import login_required + +from webtool.lib.helpers import error, module_map as _module_map + +component = Blueprint("modulemap", __name__) +api_ratelimit = current_app.limiter.shared_limit("3 per second", scope="api") + + +@component.route("/api/module-map/catalogue") +@api_ratelimit +@login_required +def module_map_catalogue(): + """Every module - processor or data source - with display metadata and flags.""" + return jsonify({"modules": _module_map().catalogue()}) + + +@component.route("/api/module-map/categories") +@api_ratelimit +@login_required +def module_map_categories(): + """{category: [types]} for grouped browsing.""" + return jsonify(_module_map().categories()) + + +@component.route("/api/module-map/search") +@api_ratelimit +@login_required +def module_map_search(): + """Find modules by a substring of type/title/category/description.""" + query = request.args.get("q", "")[:200] # substring search + return jsonify({"query": query, "results": _module_map().search(query)}) + + +@component.route("/api/module-map/module/") +@api_ratelimit +@login_required +def module_map_node(module_type): + """ + One module in full: metadata, declared compatibility, how-to-run (the + prerequisite chain + datasources + shape buckets) and available follow-ups. + """ + info = _module_map().module(module_type) + if info is None: + return error(404, message="Module '%s' does not exist" % module_type) + return jsonify(info) + + +@component.route("/api/module-map/graph") +@api_ratelimit +@login_required +def module_map_graph(): + """The whole graph as {nodes, edges} -- low-level/debug backbone.""" + return jsonify(_module_map().graph()) diff --git a/webtool/views/api_tool.py b/webtool/views/api_tool.py index 903f5fb75..95f216c29 100644 --- a/webtool/views/api_tool.py +++ b/webtool/views/api_tool.py @@ -13,7 +13,8 @@ get_flashed_messages, send_from_directory, stream_with_context, g from flask_login import login_required, current_user -from webtool.lib.helpers import error, setting_required, parse_markdown +from webtool.lib.helpers import (error, setting_required, parse_markdown, datasource_variants, + datasource_worker_options) from common.lib.exceptions import QueryParametersException, JobNotFoundException, \ QueryNeedsExplicitConfirmationException, QueryNeedsFurtherInputException, DataSetException @@ -265,12 +266,18 @@ def datasource_form(datasource_id): If the data source has no search worker or its search worker does not have any parameters defined, this returns a 404 Not Found status. + A data source offering variants (see `Search.get_variants`) takes a + `variant` query argument saying which one the options are wanted for. It is + returned as given, so that whatever renders the form can send it back when + the form is submitted. + :param datasource_id: Data source ID, as specified in the data source and config.py - :return: A JSON object with the `html` of the template, a `status` code and - the `datasource` ID. + :request-param str ?variant: Variant of the data source to get options for + :return: A JSON object with the `html` of the template, a `status` code, the + `datasource` ID and the `variant`. - :return-error 404: If the datasource does not exist. + :return-error 404: If the datasource or variant does not exist. """ if datasource_id not in g.modules.datasources: return error(404, message="Datasource '%s' does not exist" % datasource_id) @@ -284,28 +291,27 @@ def datasource_form(datasource_id): if not worker_class: return error(404, message="Datasource '%s' has no search worker" % datasource_id) - worker_options = worker_class.get_options(None, g.config) + variant = request.args.get("variant") or None + if variant and variant not in datasource_variants(worker_class, g.config): + return error(404, message="Datasource '%s' has no variant '%s'" % (datasource_id, variant)) + + worker_options = datasource_worker_options(worker_class, g.config, variant) if not worker_options: return error(404, message="Datasource '%s' has no dataset parameter options defined" % datasource_id) # Status labels to display in query form labels = [] - is_local = "local" if hasattr(worker_class, "is_local") and worker_class.is_local else "external" - is_static = True if hasattr(worker_class, "is_static") and worker_class.is_static else False - - labels.append(is_local) - if is_static: - labels.append("static") status = worker_class.get_status() if status: labels.append(status) - form = render_template("components/create-dataset-option.html", options=worker_options, labels=labels) + form = render_template("components/form-options.html", options=worker_options) html = render_template_string(form, datasource_id=datasource_id, datasource=datasource) return jsonify({ "status": "success", "datasource": datasource_id, + "variant": variant, "type": labels, "html": html, # "options": worker_options @@ -314,7 +320,7 @@ def datasource_form(datasource_id): @component.route("/api/import-dataset/", methods=["POST"]) @login_required -@current_app.limiter.limit("5 per minute") +@current_app.limiter.limit("10 per minute") @current_app.openapi.endpoint("tool") @setting_required("privileges.can_create_dataset") def import_dataset(): @@ -415,7 +421,7 @@ def import_dataset(): @component.route("/api/queue-query/", methods=["POST"]) @login_required @setting_required("privileges.can_create_dataset") -@current_app.limiter.limit("5 per minute") +@current_app.limiter.limit("10 per minute") @current_app.openapi.endpoint("tool") def queue_dataset(): """ @@ -443,6 +449,12 @@ def queue_dataset(): search_worker = g.modules.workers[search_worker_id] + # which variant of the data source this is for, if it has any; the form + # carries it because the options depend on it, and so does the query + variant = request.form.get("variant") or None + if variant and variant not in datasource_variants(search_worker, g.config): + return error(404, message="Datasource '%s' has no variant '%s'" % (datasource_id, variant)) + # handle confirmation outside of parameter parsing, since it is not data # source specific has_confirm = bool(request.form.get("frontend-confirm", False)) @@ -452,27 +464,37 @@ def queue_dataset(): # just in case try: # first sanitise values - sanitised_query = UserInput.parse_all(search_worker.get_options(None, g.config), request.form, silently_correct=False) + sanitised_query = UserInput.parse_all(datasource_worker_options(search_worker, g.config, variant), + request.form, silently_correct=False) # then validate for this particular datasource sanitised_query = {"frontend-confirm": has_confirm, **sanitised_query} + if variant: + # only when there is one: this is echoed back to the front-end + # in `keep` below, and a null would be re-submitted as "null" + sanitised_query["variant"] = variant + sanitised_query = search_worker.validate_query(sanitised_query, request, g.config) except QueryNeedsFurtherInputException as e: # ask the user for more input by returning a HTML snippet # containing form fields to be added to the form before it is # re-submitted - form = render_template("components/create-dataset-option.html", options=e.config) + form = render_template("components/form-options.html", options=e.config) return jsonify({"status": "extra-form", "html": form}) except QueryParametersException as e: # parameters need amending - return jsonify({"status": "error", "message": "Cannot create a dataset with these parameters. %s" % e}) + message = "Cannot create a dataset with these parameters. %s" % e + return jsonify({"status": "error", "message": message, + "html": render_template("components/form-notice.html", message=message)}) except QueryNeedsExplicitConfirmationException as e: # parameters are OK, but we need to be sure the user wants this # (because it will e.g. take a long time) - return jsonify({"status": "confirm", "message": str(e)}) + return jsonify({"status": "confirm", "message": str(e), + "html": render_template("components/form-notice.html", message=str(e), + needs_confirmation=True)}) else: raise NotImplementedError("Data sources MUST sanitise input values with validate_query") @@ -489,7 +511,8 @@ def queue_dataset(): # those to "[object Object],..." on the for-real re-submission. # (Other list/dict fields, e.g. parsed URL lists are not touched.) json_option_keys = { - option for option, settings in search_worker.get_options(None, g.config).items() + option for option, settings in + datasource_worker_options(search_worker, g.config, variant).items() if settings.get("type") == UserInput.OPTION_TEXT_JSON } keep = { @@ -500,6 +523,10 @@ def queue_dataset(): sanitised_query["datasource"] = datasource_id sanitised_query["type"] = search_worker_id + if variant: + # re-asserted after validation, since validate_query may return a dict + # it built from scratch + sanitised_query["variant"] = variant if request.form.to_dict().get("pseudonymise") in ("pseudonymise", "anonymise"): sanitised_query["pseudonymise"] = request.form.to_dict().get("pseudonymise") @@ -538,7 +565,8 @@ def queue_dataset(): new_job = Job.get_by_remote_ID(dataset.key, g.db) dataset.link_job(new_job) - return jsonify({"status": "success", "message": "", "key": dataset.key}) + return jsonify({"status": "success", "message": "", "key": dataset.key, + "url": url_for("dataset.show_result", key=dataset.key)}) @component.route('/api/check-query/') @@ -1235,17 +1263,12 @@ def queue_processor(key=None, processor=None): # ask the user for more input by returning a HTML snippet # containing form fields to be added to the form before it is # re-submitted - form = "\n".join( - [ - render_template( - "components/processor-option.html", - option_override={k: v}, - option=k, - dataset=dataset, - processor=processor_worker, - ) - for k, v in e.config.items() - ] + form = render_template( + "components/form-options.html", + options=e.config, + delegated=True, + dataset=dataset, + processor=processor_worker, ) return jsonify({"status": "extra-form", "html": form}) diff --git a/webtool/views/views_dataset.py b/webtool/views/views_dataset.py index 650794a29..695780aba 100644 --- a/webtool/views/views_dataset.py +++ b/webtool/views/views_dataset.py @@ -6,13 +6,17 @@ import io import json_stream import mimetypes +import zipfile +from natsort import natsorted from pathlib import Path from flask import (Blueprint, current_app, render_template, request, redirect, send_from_directory, flash, - get_flashed_messages, url_for, stream_with_context, g) + get_flashed_messages, url_for, stream_with_context, g, make_response) from flask_login import login_required, current_user -from webtool.lib.helpers import Pagination, error, setting_required -from webtool.views.api_tool import toggle_favourite, toggle_private, queue_processor +from webtool.lib.helpers import (Pagination, annotation_context, error, setting_required, + common_dataset_options, collect_grid_tags, module_request_url, + datasource_variants) +from webtool.views.api_tool import toggle_favourite, toggle_private, queue_processor, datasource_form from common.lib.dataset import DataSet from common.lib.exceptions import DataSetException @@ -22,6 +26,100 @@ csv.field_size_limit(1024 * 1024 * 1024) +def available_datasources(): + """ + Enabled data sources, whether or not a dataset can be created from them + + :return dict: Data source metadata, by data source ID + """ + return {datasource: metadata for datasource, metadata in g.modules.datasources.items() if + metadata["has_worker"] and datasource in g.config.get("datasources.enabled", {})} + + +def split_datasource_filter(value): + """ + Read a data source filter value into what it selects + + The dataset overview's data source filter is one `