From 37e3954baaad8a3ee65dd9c2538f9263a4433e26 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 01/20] chore(deps): atualiza packtools de 4.15.0 para 4.16.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: disponibilizar os novos métodos do packtools utilizados pelas refatorações do pid_provider (get_article_data, get_complete_publication_date, deprecated_sps_pkg_name_list, get_body_fragment). Solução técnica: bump de versão fixado via git+https no requirements/base.txt. --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index afc93ff4e..0849c45de 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -91,7 +91,7 @@ opensearch-py==3.2.0 # https://github.com/opensearch-project/opensearch-py # ------------------------------------------------------------------------------ tornado>=6.5.2 # not directly required, pinned by Snyk to avoid a vulnerability lxml==6.0.2 # https://github.com/lxml/lxml -git+https://git@github.com/scieloorg/packtools@4.15.0#egg=packtools +git+https://git@github.com/scieloorg/packtools@4.16.8#egg=packtools # pymongo # ------------------------------------------------------------------------------ From 6dc4e08dac195d47c157d0f1080b2dd3a2ca252e Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 02/20] feat(pid_provider): adiciona choices de status para XMLURL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: padronizar os valores possíveis do campo status do modelo XMLURL, substituindo texto livre por opções controladas. Solução técnica: cria XMLURL_STATUS_SUCCESS, XMLURL_STATUS_XML_FETCH_FAILED, XMLURL_STATUS_PID_PROVIDER_XML_FAILED e a tupla XMLURL_STATUS. --- pid_provider/choices.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pid_provider/choices.py b/pid_provider/choices.py index 9415dab9b..2d3127e76 100644 --- a/pid_provider/choices.py +++ b/pid_provider/choices.py @@ -2,6 +2,15 @@ ENDPOINTS = (("fix-pid-v2", "fix-pid-v2"),) +XMLURL_STATUS_SUCCESS = "success" +XMLURL_STATUS_XML_FETCH_FAILED = "xml_fetch_failed" +XMLURL_STATUS_PID_PROVIDER_XML_FAILED = "pid_provider_xml_failed" +XMLURL_STATUS = ( + (XMLURL_STATUS_SUCCESS, _("Success")), + (XMLURL_STATUS_XML_FETCH_FAILED, _("XML fetch failed")), + (XMLURL_STATUS_PID_PROVIDER_XML_FAILED, _("PID provider XML failed")), +) + PPXML_STATUS_WAIT = "WAIT" PPXML_STATUS_IGNORED = "IGNORE" PPXML_STATUS_TODO = "TODO" From dbec565eb3b474869fd21651ea6d0367f62cfa81 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 03/20] =?UTF-8?q?feat(pid=5Fprovider):=20adiciona=20exce?= =?UTF-8?q?=C3=A7=C3=A3o=20SkipSavePidProviderXML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: permitir que is_updated() sinalize, via exceção, que o registro deve ser ignorado (equal ou já atualizado), em vez de retornar dados silenciosamente. Solução técnica: cria a classe SkipSavePidProviderXML(Exception). --- pid_provider/exceptions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pid_provider/exceptions.py b/pid_provider/exceptions.py index d91e40792..a8ff9624e 100644 --- a/pid_provider/exceptions.py +++ b/pid_provider/exceptions.py @@ -1,3 +1,6 @@ +class SkipSavePidProviderXML(Exception): ... + + class APIPidProviderPostError(Exception): ... From b50deb3787ff6ec09708a6544a10726c2318a9da Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 04/20] =?UTF-8?q?refactor(pid=5Fprovider):=20reescreve=20Q?= =?UTF-8?q?ueryBuilderPidProviderXML=20e=20cria=20compara=C3=A7=C3=A3o=20p?= =?UTF-8?q?or=20similaridade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: substituir o cálculo de score aditivo por campo por comparação percentual baseada em similaridade textual, e simplificar o builder de queries centralizando o acesso aos dados do adaptador. Solução técnica: adiciona compare/compare_items/compare_lists usando how_similar; QueryBuilderPidProviderXML passa a usar adapter_data, compare_data e xml_with_pre_data como fonte única, com validate_input_data() e article_location_params novos; remove as antigas cached_property espelhando atributos do xml_adapter. --- pid_provider/query_params.py | 383 +++++++++++++++-------------------- 1 file changed, 162 insertions(+), 221 deletions(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index bfb6901bd..b4dbc1f9c 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -3,10 +3,55 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ -from core.utils.profiling_tools import profile_function +from core.utils.similarity import how_similar from pid_provider import exceptions +def compare(registered_items, input_data): + """ + """ + total_score = 0 + items = [] + for label, registered_item in registered_items.items(): + result = compare_items(label, registered_item, input_data.get(label)) + items.append(result) + total_score += result["score"] + return { + "items": items, + "total_score": total_score, + "percentual_score": total_score / len(items) + } + + +def compare_lists(registered, xml_adapter_titles): + if xml_adapter_titles == registered: + return 1 + if not xml_adapter_titles: + return 0 + if not registered: + return 0 + words1 = set() + for item in xml_adapter_titles: + words1.update(item.split()) + words2 = set() + for item in registered: + words2.update(item.split()) + return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + + +def compare_items(label, registered, input_data): + if isinstance(registered, list): + score = compare_lists(registered, input_data) + elif (input_data or None) == (registered or None): + score = 1 + else: + score = how_similar(input_data, registered) + response = {"label": label, "score": score} + if score != 1: + response["registered"] = registered + return response + + def get_score(registered, xml_data, min_value, max_value): if registered == xml_data: if registered: @@ -35,7 +80,7 @@ class QueryBuilderPidProviderXML: def __init__(self, xml_adapter): """ - Inicializa o construtor de queries. + Inicializa o construtor de queries obtendo os dicionários de dados do adaptador. Parameters ---------- @@ -43,269 +88,165 @@ def __init__(self, xml_adapter): Adaptador com dados do XML para busca """ self.xml_adapter = xml_adapter - - # ========== Cached Properties para Atributos do XML Adapter ========== - - @cached_property - def v3(self): - """PID v3 do documento.""" - return self.xml_adapter.v3 - - @cached_property - def v2(self): - """PID v2 do documento.""" - return self.xml_adapter.v2 - - @cached_property - def aop_pid(self): - """PID AOP (Ahead of Print) do documento.""" - return self.xml_adapter.aop_pid - - @cached_property - def pkg_name(self): - """Nome do pacote do documento, parâmtro usado ao instanciar XMLAdapter""" - return self.xml_adapter.pkg_name - - @cached_property - def sps_pkg_name(self): - """Nome do pacote do documento (deprecated).""" - return self.xml_adapter.sps_pkg_name + # Centraliza o acesso aos dados brutos e normalizados (hashes de 64 chars) + self.adapter_data = xml_adapter.data + self.compare_data = xml_adapter.get_data_to_compare() + self.xml_with_pre_data = xml_adapter.xml_with_pre.get_article_data(300) - @cached_property - def deprecated_sps_pkg_name(self): - """Nome do pacote do documento (deprecated).""" - return self.xml_adapter.sps_pkg_name - - @cached_property - def main_doi(self): - """DOI principal do documento.""" - return self.xml_adapter.main_doi - - @cached_property - def journal_issn_electronic(self): - """ISSN eletrônico do periódico.""" - return self.xml_adapter.journal_issn_electronic - - @cached_property - def journal_issn_print(self): - """ISSN impresso do periódico.""" - return self.xml_adapter.journal_issn_print - - @cached_property - def elocation_id(self): - """Identificador de localização eletrônica.""" - return self.xml_adapter.elocation_id - - @cached_property - def fpage(self): - """Primeira página do artigo.""" - return self.xml_adapter.fpage - - @cached_property - def fpage_seq(self): - """Sequência da primeira página.""" - return self.xml_adapter.fpage_seq - - @cached_property - def lpage(self): - """Última página do artigo.""" - return self.xml_adapter.lpage - - @cached_property - def pub_year(self): - """Ano de publicação.""" - return self.xml_adapter.pub_year - - @cached_property - def volume(self): - """Volume da publicação.""" - return self.xml_adapter.volume - - @cached_property - def number(self): - """Número/fascículo da publicação.""" - return self.xml_adapter.number - - @cached_property - def suppl(self): - """Suplemento da publicação.""" - return self.xml_adapter.suppl - - @cached_property - def z_surnames(self): - """Sobrenomes dos autores concatenados.""" - return self.xml_adapter.z_surnames - - @cached_property - def z_collab(self): - """Colaborações do artigo.""" - return self.xml_adapter.z_collab - - @cached_property - def z_links(self): - """Links relacionados ao artigo.""" - return self.xml_adapter.z_links - - @cached_property - def z_partial_body(self): - """Conteúdo parcial do corpo do artigo.""" - return self.xml_adapter.z_partial_body + @property + def pkg_name_list(self): + # --- Resolução Consolidada de Package Names --- + pkg_names = set() + # 1. Nome enviado originalmente via parâmetro no construtor + if self.xml_adapter.pkg_name: + pkg_names.add(self.xml_adapter.pkg_name) + # 2. Nome oficial atual gerado pelo motor de cálculo do XML + if self.xml_adapter.sps_pkg_name: + pkg_names.add(self.xml_adapter.sps_pkg_name) + # 3. Consolida todas as listas de nomes depreciados/alternativos + pkg_names.update(self.xml_adapter.xml_with_pre.deprecated_sps_pkg_name_list) + return set(item for item in pkg_names if item) + + def validate_input_data(self): + if not self.adapter_data.get("pub_year"): + raise exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError() + issn_electronic = self.adapter_data.get("issn_electronic") + issn_print = self.adapter_data.get("issn_print") + if not issn_electronic and not issn_print: + raise exceptions.RequiredISSNErrorToGetPidProviderXMLError() + items = list(self.article_location_params.values()) + if any(items): + return + article_titles = (self.xml_with_pre_data.get("article_titles") or []) + article_titles = [x for x in article_titles if x] + items = [ + article_titles, + self.xml_with_pre_data.get("surnames"), + self.xml_with_pre_data.get("collab"), + self.xml_with_pre_data.get("links"), + self.xml_with_pre_data.get("partial_body"), + ] + if any(items): + return + raise exceptions.NotEnoughParametersToGetPidProviderXMLError() - @cached_property - def order(self): - """Conteúdo parcial do corpo do artigo.""" - return self.xml_adapter.order - # ========== Queries Construídas ========== - @cached_property + @property def identifier_queries(self): """ Constrói queries para busca por identificadores (v3, v2, aop_pid, pkg_name, DOI). - - Busca em múltiplos campos incluindo other_pid para garantir - compatibilidade com diferentes formatos de PIDs. - - Returns - ------- - Q - Query object combinando buscas por v3, v2, aop_pid, pkg_name e main_doi """ q = Q() + other_pids = set() + # PIDs diretos do xml_adapter (não envelopados no data dict) + v3 = self.xml_adapter.v3 + v2 = self.xml_adapter.v2 + aop_pid = self.xml_adapter.aop_pid + # PID v3 - máxima prioridade - if self.v3: - q |= Q(v3=self.v3) + if v3: + q |= Q(v3=v3) # PID v2 - if self.v2: - q |= Q(v2=self.v2) + if v2: + q |= Q(v2=v2) # AOP PID - if self.aop_pid: - q |= Q(v2=self.aop_pid) | Q(aop_pid=self.aop_pid) + if aop_pid: + q |= Q(v2=aop_pid) | Q(aop_pid=aop_pid) - # Package name - pkg_names = set() - if self.pkg_name: - pkg_names.add(self.pkg_name) - if self.sps_pkg_name: - pkg_names.add(self.sps_pkg_name) - if self.deprecated_sps_pkg_name: - pkg_names.add(self.deprecated_sps_pkg_name) + # Package names históricos e atuais + pkg_names = self.pkg_name_list if pkg_names: q |= Q(pkg_name__in=pkg_names) - # # DOI principal - # if self.main_doi: - # q |= Q(main_doi=self.main_doi) - + main_doi = self.adapter_data.get("main_doi") + if main_doi: + q |= Q(main_doi=main_doi) + return q - @cached_property + @property def issn_query(self): """ Constrói query base para busca por ISSN (eletrônico ou impresso). - - Returns - ------- - Q - Query object combinando ISSN eletrônico e impresso com operador OR - - Raises - ------ - RequiredISSNErrorToGetPidProviderXMLError - Se nenhum ISSN (eletrônico ou impresso) estiver disponível """ q = Q() + issn_electronic = self.adapter_data.get("issn_electronic") + issn_print = self.adapter_data.get("issn_print") - if not self.journal_issn_electronic and not self.journal_issn_print: + if not issn_electronic and not issn_print: raise exceptions.RequiredISSNErrorToGetPidProviderXMLError( _("Required Print or Electronic ISSN to identify XML {}").format( - self.pkg_name, + self.xml_adapter.pkg_name, ) ) - if self.journal_issn_electronic: - q |= Q(issn_electronic=self.journal_issn_electronic) + if issn_electronic: + q |= Q(issn_electronic=issn_electronic) - if self.journal_issn_print: - q |= Q(issn_print=self.journal_issn_print) + if issn_print: + q |= Q(issn_print=issn_print) return q - @cached_property + @property def issue_params(self): """ Constrói dicionário com metadados do fascículo e paginação do artigo. - - Retorna todos os campos sem verificar presença, permitindo - que o ORM do Django filtre automaticamente valores None. - - Returns - ------- - dict - Dicionário com elocation_id, fpage, fpage_seq, lpage, - pub_year, volume, number e suppl + """ + return { + "pub_year": self.adapter_data.get("pub_year"), + "volume": self.adapter_data.get("volume"), + "number": self.adapter_data.get("number"), + "suppl": self.adapter_data.get("suppl"), + } + + @property + def article_location_params(self): + """ + Constrói dicionário com metadados de localização do artigo. """ data = { - "elocation_id": self.elocation_id, - "fpage": self.fpage, - "fpage_seq": self.fpage_seq, - "lpage": self.lpage, - "pub_year": self.pub_year, - "volume": self.volume, - "number": self.number, - "suppl": self.suppl, + "elocation_id": self.adapter_data.get("elocation_id"), + "fpage": self.adapter_data.get("fpage"), + "fpage_seq": self.adapter_data.get("fpage_seq"), + "lpage": self.adapter_data.get("lpage"), } - if self.order: - data["v2__endswith"] = self.order - elif not self.elocation_id and not self.fpage and self.main_doi: - data["main_doi__iexact"] = self.main_doi + order = self.xml_adapter.order + if order: + data["v2__endswith"] = order return data - @cached_property + @property def article_data_query(self): """ - Constrói query para busca por dados textuais do artigo. - - Combina buscas por sobrenomes de autores, colaborações, - links e conteúdo parcial do corpo do artigo. - - Returns - ------- - Q or None - Query object combinando z_surnames, z_collab, z_links e z_partial_body, - ou None se nenhum dado textual estiver disponível + Constrói query para busca por dados textuais codificados (hashes sha256). """ - # Verifica se há algum dado textual disponível - if not any([ - self.z_surnames, - self.z_collab, - self.z_links, - self.z_partial_body, - ]): - return Q( - z_surnames=self.z_surnames, - z_collab=self.z_collab, - z_links=self.z_links, - z_partial_body=self.z_partial_body, - ) - - q = Q() - - # Adiciona query para sobrenomes se disponível - if self.z_surnames: - q |= Q(z_surnames=self.z_surnames) - - # Adiciona queries para outros campos textuais - if self.z_collab: - q |= Q(z_collab=self.z_collab) - - if self.z_links: - q |= Q(z_links=self.z_links) - - if self.z_partial_body: - q |= Q(z_partial_body=self.z_partial_body) - - return q \ No newline at end of file + z_surnames = self.adapter_data.get("z_surnames") + z_collab = self.adapter_data.get("z_collab") + z_links = self.adapter_data.get("z_links") + z_partial_body = self.adapter_data.get("z_partial_body") + + # Se houver qualquer dado textual disponível, constrói query com OR (|) + if z_surnames or z_partial_body or z_collab or z_links: + q = Q() + if z_surnames: + q |= Q(z_surnames=z_surnames) + if z_collab: + q |= Q(z_collab=z_collab) + if z_links: + q |= Q(z_links=z_links) + if z_partial_body: + q |= Q(z_partial_body=z_partial_body) + return q + + # Caso contrário, retorna os campos (geralmente None neste ponto) com AND + return Q( + z_surnames=z_surnames, + z_collab=z_collab, + z_links=z_links, + z_partial_body=z_partial_body, + ) & Q(**self.article_location_params) \ No newline at end of file From 0cc4a2da7afce9c0c7417723b13f23f54ba18013 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 05/20] =?UTF-8?q?refactor(pid=5Fprovider):=20reescreve=20f?= =?UTF-8?q?luxo=20de=20correspond=C3=AAncia/registro=20de=20XML=20e=20cria?= =?UTF-8?q?=20auditoria=20por=20documento?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: tornar o processo de identificação de documentos já registrados mais preciso (comparação por similaridade em estágios) e garantir rastreabilidade de toda tentativa de registro (sucesso, erro, conflito, skip), substituindo o modelo de eventos XMLEvent. Solução técnica: - get_records/get_record/best_matches substituídos por select_records (generator em estágios: ids, journal-issue-article, journal-article), select_record e get_best_match, usando percentual_score. - PidProviderXML.register() reescrito com try/except/finally, sempre gravando o evento via PidProviderXMLRegistration.record(). - is_updated() passa a levantar SkipSavePidProviderXML em vez de retornar registered.data. - Novo PidProviderXMLManager aplicando select_related('current_version') por padrão. - Novo campo readable_data em PidProviderXML. - add_collections extraído de _save(), com fallback via FieldError entre scielojournal e journalproc (compatibilidade upload/core). - Corrige bug em merge_records (other_pid.version, não other_pid.current_version) e em mark_items_as_invalid (persiste o status calculado via bulk_update). - XMLURL ganha os campos detail e is_public e o classmethod record(). - Remove modelo XMLEvent e o método add_event(). --- pid_provider/models.py | 874 +++++++++++++++++++++++++---------------- 1 file changed, 544 insertions(+), 330 deletions(-) diff --git a/pid_provider/models.py b/pid_provider/models.py index cb67abda7..782351f35 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -1,28 +1,27 @@ import io -import json import logging import os import sys import traceback import zipfile from datetime import datetime -from functools import lru_cache, cached_property +from functools import cached_property from zlib import crc32 from django.core.files.base import ContentFile +from django.core.exceptions import FieldError from django.db import IntegrityError, models -from django.db.models import Q, Count, Min +from django.db.models import Prefetch, Q, Count from django.utils.translation import gettext_lazy as _ from modelcluster.fields import ParentalKey from modelcluster.models import ClusterableModel from packtools.sps.pid_provider import v3_gen, xml_sps_adapter from packtools.sps.pid_provider.xml_sps_lib import XMLWithPre from wagtail.admin.panels import FieldPanel, InlinePanel, ObjectList, TabbedInterface -from wagtail.fields import RichTextField -from wagtail.models import Orderable from wagtailautocomplete.edit_handlers import AutocompletePanel from collection.models import Collection +from core.widgets import ReadOnlyPrettyJSONWidget from core.forms import CoreAdminModelForm from core.models import CommonControlField from core.utils.profiling_tools import ( # ajuste o import conforme sua estrutura @@ -31,14 +30,15 @@ profile_property, profile_staticmethod, ) -from core.utils.similarity import how_similar from pid_provider import choices, exceptions from pid_provider.query_params import ( - get_score, zero_to_none, + compare, QueryBuilderPidProviderXML, ) -from tracker.models import BaseEvent, UnexpectedEvent +from tracker.models import UnexpectedEvent + +PARTIAL_BODY_MAX = 300 try: from django_prometheus.models import ExportModelOperationsMixin @@ -136,14 +136,10 @@ def create( obj.pid_provider_xml = pid_provider_xml obj.finger_print = xml_with_pre.finger_print obj.creator = user - # Salvar primeiro sem arquivo para obter o PK obj.save() - # save_file já faz self.file.save() que persiste o campo file, - # mas precisamos persistir o registro completo com o path do arquivo obj.save_file( f"{pid_provider_xml.v3}.xml", xml_with_pre.tostring(pretty_print=True) ) - # Único save final após salvar o arquivo obj.save() return obj except IntegrityError: @@ -348,6 +344,24 @@ def created_updated(self): return self.updated or self.created +class PidProviderXMLManager(models.Manager): + """ + Manager customizado: aplica select_related("current_version") em toda + consulta de PidProviderXML.objects, evitando repetir esse select_related + manualmente em cada classmethod (get_xml_with_pre, get_record_by_pid_v3, + select_records, public_items, mark_items_as_invalid, get_by_pid_v3, etc). + + Nota: prefetch_related("collections") NÃO entra aqui de propósito — + prefetch_related sempre dispara uma query extra, mesmo quando + "collections" não é usado (ex.: em _is_registered_pid, que só faz + .exists()). Por isso ele é aplicado pontualmente em get_queryset(), + que é o método de listagem que de fato usa collection_list. + """ + + def get_queryset(self): + return super().get_queryset().select_related("current_version") + + class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): """ Tem responsabilidade de garantir a atribuição do PID da versão 3, @@ -414,6 +428,13 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): registered_in_core = models.BooleanField(default=False) collections = models.ManyToManyField(Collection, blank=True) + # dados legíveis para facilitar a análise + readable_data = models.JSONField( + _("Readable data"), null=True, blank=True + ) + + objects = PidProviderXMLManager() + base_form_class = CoreAdminModelForm panel_a = [ @@ -436,13 +457,11 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): InlinePanel("other_pid", label=_("Other PID")), ] panel_c = [ - FieldPanel("z_surnames"), - FieldPanel("z_collab"), - FieldPanel("z_links"), - FieldPanel("z_partial_body"), - ] - panels_event = [ - InlinePanel("events", label=_("Events")), + FieldPanel("z_surnames", read_only=True), + FieldPanel("z_collab", read_only=True), + FieldPanel("z_links", read_only=True), + FieldPanel("z_partial_body", read_only=True), + FieldPanel("readable_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), ] edit_handler = TabbedInterface( @@ -450,7 +469,6 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): ObjectList(panel_a, heading=_("Identification")), ObjectList(panel_b, heading=_("Other PIDs")), ObjectList(panel_c, heading=_("Data")), - ObjectList(panels_event, heading=_("Events")), ] ) @@ -527,19 +545,6 @@ class Meta: def __str__(self): return f"{self.pkg_name} {self.v3}" - - @property - def article_pid_suffix_source(self): - try: - return self.xml_with_pre.get_article_pid_suffix_source() - except AttributeError: - return self.elocation_id or self.fpage or self.xml_with_pre.order - - def get_article_pid_suffix(self): - data = self.article_pid_suffix_source - if not data: - data = self.pkg_name.split("-")[-1] - return string_to_5_digits(data) @property def collection_list(self): @@ -571,13 +576,22 @@ def get_queryset( params["pub_year__lte"] = until_pub_year if proc_status_list: params["proc_status__in"] = proc_status_list - return cls.objects.filter(q, **params) + # select_related("current_version") já vem do manager; + # prefetch_related("collections") é aplicado aqui pois este método + # é usado em listagens que iteram collection_list. + return cls.objects.prefetch_related("collections").filter(q, **params) + + @classmethod + def delete_queryset(cls, qs): + OtherPid.objects.filter(pid_provider_xml__in=qs).delete() + qs.delete() @classmethod @profile_classmethod def public_items(cls, from_date): now = datetime.utcnow().isoformat()[:10] - return cls.objects.select_related("current_version").filter( + # select_related("current_version") já vem do manager + return cls.objects.filter( (Q(available_since__isnull=True) | Q(available_since__lte=now)) & (Q(created__gte=from_date) | Q(updated__gte=from_date)), current_version__pid_provider_xml__v3__isnull=False, @@ -607,8 +621,8 @@ def data(self): @profile_classmethod def get_xml_with_pre(cls, v3): try: - # Usar select_related para evitar query extra ao acessar current_version - return cls.objects.select_related("current_version").get(v3=v3).xml_with_pre + # select_related("current_version") já vem do manager + return cls.objects.get(v3=v3).xml_with_pre except cls.DoesNotExist: return None except Exception: @@ -635,6 +649,20 @@ def is_aop(self): return False return True + @property + def data_to_compare(self): + readable = self.readable_data or {} + titles = readable.get("article_titles") + body_fragment = readable.get("body_fragment") + return { + "article_titles": titles or self.xml_with_pre.article_titles_texts, + "z_surnames": self.z_surnames, + "z_collab": self.z_collab, + "z_links": self.z_links, + "z_partial_body": self.z_partial_body, + "body_fragment": body_fragment or self.xml_with_pre.get_body_fragment(PARTIAL_BODY_MAX), + } + @classmethod @profile_classmethod def register( @@ -695,80 +723,116 @@ def register( Parâmetros insuficientes para identificar documento """ try: + # outputs + response = {} + registered = None + event_status = None + error_type = None + select_record_response = None + + # inputs + pkg_name = filename input_data = None xml_adapter_data = None - response = {} - response["input_data"] = xml_with_pre.data - response["input_data"].update({"origin": origin}) + input_data = {} + input_data.update(xml_with_pre.data) + input_data.update(xml_with_pre.get_article_data()) + input_data["origin"] = origin + response["input_data"] = input_data # adaptador do xml with pre xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) - response["xml_adapter_data"] = xml_adapter.data + xml_adapter_data = xml_adapter.data + response["xml_adapter_data"] = xml_adapter_data # consulta se documento já está registrado try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + records = cls.select_records(xml_adapter) + select_record_response = cls.select_record(xml_adapter, records) + try: + registered = select_record_response.pop("registered") + except KeyError: + unmatched_items = select_record_response.get("unmatched_items") + if unmatched_items: + raise exceptions.UnmatchedPidProviderXMLError + raise cls.DoesNotExist + event_status = "updated" + if select_record_response.get("matched_items"): + response["select_record_response"] = select_record_response except cls.DoesNotExist as exc: registered = None + event_status = "created" except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - response["records"] = [item.data for item in records] + event_status = "unmatched" + response["select_record_response"] = select_record_response raise exceptions.QueryDocumentMultipleObjectsReturnedError(exc) except ( - exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError, + exceptions.RequiredISSNErrorToGetPidProviderXMLError, + exceptions.NotEnoughParametersToGetPidProviderXMLError, ) as exc: - raise exc - except exceptions.RequiredISSNErrorToGetPidProviderXMLError as exc: - raise exc - except exceptions.NotEnoughParametersToGetPidProviderXMLError as exc: + event_status = "bad_request" raise exc # valida os PIDs do XML # - não podem ter conflito com outros registros # - identifica mudança - response["xml_changed"] = cls.complete_missing_xml_pids( - xml_adapter, registered, auto_solve_pid_conflict - ) + try: + response["xml_changed"] = cls.complete_missing_xml_pids( + xml_adapter, registered, auto_solve_pid_conflict + ) + except PidProviderXMLPidV3ConflictError as exc: + event_status = "conflict" + raise exc # analisa se continua o registro - updated_data = cls.is_updated( - xml_with_pre, - registered, - force_update, - origin_date, - registered_in_core, - ) - if updated_data: - response["skip_update"] = True - response.update(updated_data) - return response - - # cria ou atualiza registro - registered = cls._save( - registered, - xml_adapter, - user, - origin_date, - available_since, - registered_in_core, - ) - - # data to return - response.update(registered.data) - return response - - except Exception as e: + try: + PidProviderXML.is_updated( + xml_with_pre, + registered, + force_update, + origin_date, + registered_in_core, + ) + registered = cls._save( + registered, + xml_adapter, + user, + origin_date, + available_since, + registered_in_core, + ) + # data to return + response.update(registered.data) + except exceptions.ForbiddenPidProviderXMLRegistrationError: + event_status = "forbidden" + raise + except exceptions.SkipSavePidProviderXML: + event_status = "skipped" + response["skipped"] = True + response.update(registered.data) + # do not raise + except Exception as exc: + event_status = event_status or "error" exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.register", - exception=e, - exc_traceback=exc_traceback, - detail=response, - ) - response.update({"error_msg": str(e), "error_type": str(type(e))}) - return response + error_type = str(type(exc)) + response.update({ + "error_msg": str(exc), + "error_type": error_type, + "traceback": traceback.format_exc() + }) + finally: + response["event_status"] = event_status + if error_type or (select_record_response or {}).get("matched_items"): + PidProviderXMLRegistration.record( + user=user, + pid_provider_xml=registered, + pkg_name=pkg_name, + event_status=event_status, + detail=response, + ) + return response @classmethod @profile_classmethod @@ -814,15 +878,19 @@ def get_valid_pid_v3( if xml_pid and xml_pid != registered_pid: # Verifica se o XML PID já está em uso por outro documento. try: - # garantir que xml_adapter.v3 não tenha conflito + # verificar se xml_adapter.v3 pertence a outro xml cls.get_record_by_pid_v3(xml_adapter) + # pertence a xml_adapter return xml_pid except cls.DoesNotExist: + # não pertence a nenhum xml return xml_pid except PidProviderXMLPidV3ConflictError: + # pertence a um xml diferente de xml_adapter if not auto_solve_pid_conflict: + # rejeita o uso deste pid raise - + # ignora # XML PID não fornecido, ou igual ao registrado # ou em conflito sem exceção # retorna o PID registrado ou gera um novo. @@ -840,7 +908,6 @@ def _save( registered_in_core=None, ): if registered: - # obtém os dados de substituição para registrar em other_pid registered_changed = registered.check_registered_pids_changed( xml_adapter.xml_with_pre ) @@ -849,51 +916,44 @@ def _save( registered = cls() registered.creator = user registered_changed = None - + registered.proc_status = choices.PPXML_STATUS_TODO registered._add_dates(xml_adapter, origin_date, available_since) registered._add_data(xml_adapter, registered_in_core) registered._add_journal(xml_adapter) registered._add_issue(xml_adapter) - - # Primeiro save: necessário para obter PK (se novo) antes de criar XMLVersion / OtherPid - if registered.pk is None: - registered.save() - + + registered.save() + if registered_changed: registered._add_other_pid(registered_changed, user) registered._add_current_version(xml_adapter.xml_with_pre, user) + + registered.add_collections(xml_adapter) + return registered - # Save final consolidado: persiste current_version e other_pid_count - registered.save() + def add_collections(self, xml_adapter): q = Q() - if COLLECTION_PREFIX == "scielojournal": - if xml_adapter.journal_issn_print: - q |= Q( - scielojournal__journal__official__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - scielojournal__journal__official__issn_electronic=xml_adapter.journal_issn_electronic - ) - else: - if xml_adapter.journal_issn_print: - q |= Q( - journalproc__journal__official_journal__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - journalproc__journal__official_journal__issn_electronic=xml_adapter.journal_issn_electronic - ) + issn_print = xml_adapter.journal_issn_print + issn_electronic = xml_adapter.journal_issn_electronic + + try: + Collection.objects.filter(scielojournal__isnull=True).exists() + issn_path = "scielojournal__journal__official" + except FieldError: + issn_path = "journalproc__journal__official_journal" + + if issn_print: + q |= Q(**{f"{issn_path}__issn_print": issn_print}) + if issn_electronic: + q |= Q(**{f"{issn_path}__issn_electronic": issn_electronic}) for collection in Collection.objects.filter(q): - registered.collections.add(collection) - return registered + self.collections.add(collection) - @classmethod - @profile_classmethod + @staticmethod def is_updated( - cls, xml_with_pre, registered, force_update, origin_date, registered_in_core + xml_with_pre, registered, force_update, origin_date, registered_in_core ): """ XML é versão AOP, mas @@ -919,7 +979,7 @@ def is_updated( if registered.is_equal_to(xml_with_pre): # XML fornecido é igual ao registrado, não precisa continuar logging.info(f"Skip update: equal") - return registered.data + raise exceptions.SkipSavePidProviderXML if xml_with_pre.is_aop and registered and not registered.is_aop: logging.info(f"Skip update: forbidden") @@ -935,9 +995,7 @@ def is_updated( and registered.origin_date and registered.origin_date > origin_date ): - # retorna item registrado que está mais atualizado - logging.info(f"Skip update: is already up-to-date") - return registered.data + raise exceptions.SkipSavePidProviderXML @profile_method def is_equal_to(self, xml_with_pre): @@ -947,23 +1005,78 @@ def is_equal_to(self, xml_with_pre): @classmethod @profile_classmethod - def get_records(cls, xml_adapter): + def select_records(cls, xml_adapter): + """ + Gera pares (label, lista_de_candidatos) para cada estratégia de + correspondência, do mais específico ao mais genérico. + + Cada branch é materializada (list(...)) uma única vez aqui, para + que o consumidor (select_record) nunca precise avaliar a queryset + mais de uma vez (evita repetir .exists() + .count() + iteração, + que geram queries separadas no banco). Por ser um generator, uma + branch só é construída e avaliada quando o consumidor de fato + solicita o próximo item — se a primeira branch já resolver, as + demais nunca chegam a rodar no banco. + """ qbuilder = QueryBuilderPidProviderXML(xml_adapter) - q_ids = qbuilder.identifier_queries - q_journal = qbuilder.issn_query - q_issue = Q(**qbuilder.issue_params) - return cls.objects.filter(q_ids | (q_journal & q_issue)).distinct() + qbuilder.validate_input_data() - @classmethod - @profile_classmethod - def get_record(cls, xml_adapter, records): - results = records - if not results.exists(): - raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - raise cls.DoesNotExist - return cls.objects.get(id=sorted(matched)[-1][-1]) + # select_related("current_version") já vem do manager + objects = cls.objects.all() + + # 1) correspondência direta por identificadores + yield "ids", list(objects.filter(qbuilder.identifier_queries)) + + selected_journal = objects.filter(qbuilder.issn_query) + + # 2) journal + issue + dados do artigo + yield ( + "journal-issue-article", + list( + selected_journal.filter( + Q(**qbuilder.issue_params) & qbuilder.article_data_query + ) + ), + ) + + # 3) journal + dados do artigo + yield "journal-article", list(selected_journal.filter(qbuilder.article_data_query)) + + @staticmethod + def select_record(xml_adapter, selection_results): + """ + Consome os pares (label, lista_de_candidatos) produzidos por + select_records. As listas já vêm materializadas, então aqui só + checamos truthiness (nunca .exists()/.count() sobre queryset). + """ + unmatched_items = {} + xml_adapter_data_to_compare = xml_adapter.get_data_to_compare() + for label, results in selection_results: + if not results: + continue + + result = PidProviderXML.get_best_match(results, xml_adapter_data_to_compare) + + matched = result.get("matched") + unmatched = result.get("unmatched") + registered = result.get("registered") + if registered: + response = { + "total_results": len(results), + "registered": registered, + } + if matched: + response["matched_items"] = {label: matched} + if unmatched: + response["unmatched_items"] = {label: unmatched} + return response + + if unmatched: + unmatched_items[label] = unmatched + + if unmatched_items: + return {"unmatched_items": unmatched_items} + return {} @classmethod @profile_classmethod @@ -972,106 +1085,84 @@ def get_record_by_pid_v3(cls, xml_adapter): if not xml_adapter.v3: raise ValueError("get_record_by_pid_v3: XML has not pid v3") xml_pid_v3 = xml_adapter.v3 - results = ( - cls.objects.filter(Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3)) + # select_related("current_version") já vem do manager + results = cls.objects.filter( + Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3) ) if not results.exists(): + # pid v3 é inédito raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.get_record_by_pid_v3", - exception=PidProviderXMLPidV3ConflictError, - detail={"xml_adapter": xml_adapter.data, "results": [i.data for i in results]}, - ) + + xml_adapter_data_to_compare = xml_adapter.get_data_to_compare() + result = PidProviderXML.get_best_match(results, xml_adapter_data_to_compare) + registered = result.get("registered") + if not registered: + xml_data = xml_adapter.xml_with_pre.get_article_data(PARTIAL_BODY_MAX) + items = [item.data for item in results] raise PidProviderXMLPidV3ConflictError( - _("No matching record found for the provided XML data.") + _(f"{xml_pid_v3} belongs to {items}, not to {xml_data}") ) - return cls.objects.get(id=sorted(matched)[-1][-1]) + return registered - @profile_method - def match(self, xml_adapter): - """ + @staticmethod + def get_best_match(results, xml_adapter_data): """ - labels = [] - score = self.title_similarity(xml_adapter) * 100 - if score > 50: - labels.append("title") - if score_item := get_score(self.z_surnames, xml_adapter.z_surnames, 10, 100): - labels.append("z_surnames") - score += score_item - if score_item := get_score(self.z_collab, xml_adapter.z_collab, 10, 100): - labels.append("z_collab") - score += score_item - if score_item := get_score(self.z_links, xml_adapter.z_links, 10, 100): - labels.append("z_links") - score += score_item - if score_item := get_score(self.z_partial_body, xml_adapter.z_partial_body, 10, 100): - labels.append("z_partial_body") - score += score_item - return {"score": score, "labels": labels} - - def title_similarity(self, xml_adapter): - try: - registered = self.xml_with_pre.article_titles_texts - except Exception: - registered = [] - xml_adapter_titles = xml_adapter.xml_with_pre.article_titles_texts - if xml_adapter_titles == registered: - return 1 - if not xml_adapter_titles: - return 0 - if not registered: - return 0 - words1 = set() - for item in xml_adapter_titles: - words1.update(item.split()) - words2 = set() - for item in registered: - words2.update(item.split()) - return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + Compara uma lista de candidatos (PidProviderXML) com os dados do XML + recebido e classifica os candidatos por similaridade. - @classmethod - def best_matches(cls, results, xml_adapter): - data = [] + Parameters + ---------- + results : list[PidProviderXML] + Lista JÁ MATERIALIZADA (não queryset) de candidatos a comparar. + xml_adapter_data : dict + Dados de comparação do XML de entrada, ou seja, o retorno de + ``xml_adapter.get_data_to_compare()``. + + Returns + ------- + dict + Todas as chaves abaixo são OPCIONAIS — só aparecem quando há + conteúdo para elas. Use ``.get(...)`` ou ``"chave" in result`` + ao consumir o retorno, nunca acesso direto. + + - ``"unmatched"``: presente apenas se houver ao menos 1 + candidato com ``percentual_score`` <= 0.6. Lista de + ``item.data`` desses candidatos. + - ``"registered"``: presente apenas se houver ao menos 1 + candidato aprovado (score > 0.6). Contém o OBJETO + ``PidProviderXML`` (não o dict ``.data``) do candidato com + maior score — em caso de empate, o critério de desempate é + ``updated`` mais recente e, em seguida, maior ``id``. + - ``"matched"``: presente apenas se houver 2 OU MAIS candidatos + aprovados. Contém ``item.data`` dos candidatos aprovados + EXCLUINDO o que já está em ``"registered"`` (ou seja, é a + lista de aprovados a partir do 2º colocado), na mesma ordem + de score decrescente. + """ + detail = {} + found = [] + items = {} + for item in results: + item_data = item.data_to_compare + response = compare(item_data, xml_adapter_data) + items[item.id] = item + found.append((response["percentual_score"], item.updated.isoformat(), item.id)) + + found = sorted(found, reverse=True) matched = [] - for item in results.select_related("current_version").iterator(): - response = item.match(xml_adapter) - score = response["score"] - - if xml_adapter.v2: - if item.v2 == xml_adapter.v2: - score += 100 - elif xml_adapter.order and item.v2 and item.v2.endswith(xml_adapter.order): - score += 100 - if item.v3 == xml_adapter.v3: - score += 100 - if item.pkg_name == xml_adapter.pkg_name: - score += 100 - if item.main_doi == xml_adapter.main_doi: - score += 100 - - _data = response - _data.update(item.data) - data.append(_data) - - if score > 50: - matched.append((score, item.updated.isoformat(), item.id)) - - if results.count() > 1 or not matched: - detail = { - "xml_adapter_data": xml_adapter.data, - "data": data, - "matched": matched, - } - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.best_matches", - exception=cls.MultipleObjectsReturned, - detail=detail, - ) - return matched + unmatched = [] + for percentual_score, updated, item_id in found: + if percentual_score > 0.6: + matched.append(items[item_id].data) + else: + unmatched.append(items[item_id].data) + if matched: + detail["registered"] = items[found[0][-1]] + if len(matched) > 1: + detail["matched"] = matched[1:] + if unmatched: + detail["unmatched"] = unmatched + return detail @profile_method def _add_data(self, xml_adapter, registered_in_core): @@ -1095,12 +1186,18 @@ def _add_data(self, xml_adapter, registered_in_core): self.z_links = xml_adapter.z_links self.z_partial_body = xml_adapter.z_partial_body + self.readable_data = xml_adapter.xml_with_pre.get_article_data() + @profile_method def _add_dates(self, xml_adapter, origin_date, available_since): # evita que artigos WIP fique disponíveis antes de estarem públicos try: + # Usa get_complete_publication_date para evitar logs de erro do + # packtools quando a data de publicação no XML é incompleta + # (ex.: apenas com e , sem mes/dia). + # Mesmo padrão adotado em proc/models.py e package/models.py. self.available_since = available_since or ( - xml_adapter.xml_with_pre.article_publication_date + xml_adapter.xml_with_pre.get_complete_publication_date() ) except Exception as e: # packtools error @@ -1128,7 +1225,7 @@ def _add_current_version(self, xml_with_pre, user, delete=False): pass self.current_version = XMLVersion.get_or_create(user, self, xml_with_pre) - # Não faz save() aqui; chamador é responsável por consolidar o save + self.save() @profile_method def check_registered_pids_changed(self, xml_with_pre): @@ -1164,20 +1261,15 @@ def check_registered_pids_changed(self, xml_with_pre): @profile_method def _add_other_pid(self, registered_changed, user): - # registrados passam a ser other pid - # os pids do XML passam a ser os vigentes if not registered_changed: return for change_args in registered_changed: - change_args["pid_in_xml"] = change_args.pop("registered") - change_args["user"] = user change_args["pid_provider_xml"] = self - OtherPid.get_or_create(**change_args) self.other_pid_count = self.other_pid.count() - # Não é necessário save() aqui; será consolidado no _save() pai + self.save(update_fields=["other_pid_count"]) @classmethod @profile_classmethod @@ -1229,58 +1321,52 @@ def is_registered( """ try: + select_record_response = None response = {} response["input_data"] = xml_with_pre.data - xml_adapter_data = None - xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) - xml_adapter_data = xml_adapter.data - response["xml_adapter_data"] = xml_adapter_data + xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) + response["xml_adapter_data"] = xml_adapter.data try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + records = cls.select_records(xml_adapter) + select_record_response = cls.select_record(xml_adapter, records) + try: + registered = select_record_response.pop("registered") + except KeyError: + unmatched_items = select_record_response.get("unmatched_items") + if unmatched_items: + raise exceptions.UnmatchedPidProviderXMLError + raise cls.DoesNotExist + matched_items = select_record_response.get("matched_items") + if matched_items: + response["select_record_response"] = select_record_response except cls.DoesNotExist as exc: response.update( {"filename": xml_with_pre.filename, "registered": False} ) return response except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - exc_type, exc_value, exc_traceback = sys.exc_info() - response["records"] = [item.data for item in records] - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.is_registered", - exception=exc, - exc_traceback=exc_traceback, - detail=response, - ) - response.update({"error_msg": str(exc), "error_type": str(type(exc))}) - return response + response["select_record_response"] = select_record_response + raise except ( - exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError, + exceptions.RequiredISSNErrorToGetPidProviderXMLError, + exceptions.NotEnoughParametersToGetPidProviderXMLError, ) as exc: raise exc - except exceptions.RequiredISSNErrorToGetPidProviderXMLError as exc: - raise exc - except exceptions.NotEnoughParametersToGetPidProviderXMLError as exc: - raise exc response["registered"] = True response.update(registered.data) response["is_equal"] = registered.is_equal_to(xml_with_pre) return response except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.is_registered", - exception=e, - exc_traceback=exc_traceback, - detail=response, - ) - response.update({"error_msg": str(e), "error_type": str(type(e))}) + response.update({ + "error_msg": str(e), + "error_type": str(type(e)), + "traceback": traceback.format_exc() + }) return response - return {} @classmethod def get_by_pid_v3(cls, pid_v3, partial_pid_v2=None, pid_v2=None): @@ -1291,6 +1377,7 @@ def get_by_pid_v3(cls, pid_v3, partial_pid_v2=None, pid_v2=None): params["v2"] = pid_v2 if partial_pid_v2: params["v2__contains"] = partial_pid_v2 + # select_related("current_version") já vem do manager try: return cls.objects.get(**params) except cls.MultipleObjectsReturned as e: @@ -1333,13 +1420,22 @@ def mark_as_done(self): @classmethod @profile_classmethod def mark_items_as_invalid(cls, issns): - for item in cls.objects.filter( + # select_related("current_version") já vem do manager + # (necessário aqui pois o loop acessa item.xml_with_pre, que usa + # self.current_version) + items = cls.objects.filter( Q(issn_print__in=issns) | Q(issn_electronic__in=issns), - ).iterator(): + ) + items_to_update = [] + for item in items.iterator(): try: - invalid = bool(item.xml_with_pre) + valid = bool(item.xml_with_pre) except Exception as e: - invalid = True + valid = False + if not valid: + item.proc_status = choices.PPXML_STATUS_INVALID + items_to_update.append(item) + cls.objects.bulk_update(items_to_update, ["proc_status"], batch_size=100) @classmethod @profile_classmethod @@ -1403,7 +1499,19 @@ def fix_duplicated_pkg_name(cls, pkg_name, user): int: Número de items atualizados. """ try: - items = cls.objects.filter(pkg_name=pkg_name) + # select_related("current_version") já vem do manager. + # prefetch_related com Prefetch + to_attr é necessário aqui + # porque o loop chama item.other_pid.filter(pid_type="pid_v3"), + # e um .filter() sobre manager relacionado ignora o cache do + # prefetch_related simples (só .all() usa o cache) — por isso + # a filtragem precisa estar dentro do próprio Prefetch. + items = cls.objects.prefetch_related( + Prefetch( + "other_pid", + queryset=OtherPid.objects.filter(pid_type="pid_v3"), + to_attr="pid_v3_others", + ) + ).filter(pkg_name=pkg_name) if items.count() <= 1: return 0 @@ -1419,12 +1527,16 @@ def fix_duplicated_pkg_name(cls, pkg_name, user): most_recent_item.save() for item in items.exclude(id=most_recent_item.id): - for other_pid in item.other_pid.filter(pid_type="pid_v3"): + for other_pid in item.pid_v3_others: OtherPid.get_or_create( user=user, pid_type=other_pid.pid_type, pid_in_xml=other_pid.pid_in_xml, - version=other_pid.current_version, + # Nota: OtherPid não tem campo current_version, e + # sim `version` — corrigido aqui (era + # other_pid.current_version, que não existe no + # modelo e lançaria AttributeError). + version=other_pid.version, pid_provider_xml=most_recent_item, ) OtherPid.get_or_create( @@ -1451,11 +1563,6 @@ def fix_pkg_name(self, pkg_name): self.save() return True return False - - def add_event(self, name, proc_status, detail=None, errors=None, exceptions=None): - self.proc_status = proc_status - self.save() - return XMLEvent.register(self, name, detail=detail, errors=errors, exceptions=exceptions) class FixPidV2(CommonControlField): @@ -1618,23 +1725,25 @@ def xml_url_zipfile_path(instance, filename): class XMLURL(CommonControlField): """ Model to store URLs that experienced failures and should be retried in the future. - + This model tracks URLs that failed during processing, along with their status and associated article PID, enabling retry mechanisms to reprocess them later. - + Fields: url: URLField - The URL that needs to be retried status: CharField - To control the request status (e.g., "pending", "failed", "retrying") pid: CharField - Article PID associated with this URL zipfile: FileField - Compressed XML content retrieved from the URL - exceptions: CharField - Exception traceback information (truncated to 255 chars if needed) + detail: JSONField + is_public: BooleanField - Whether the document is public (derived from item status) """ url = models.URLField( _("URL"), max_length=500, null=False, blank=False ) status = models.CharField( - _("Status"), max_length=50, null=True, blank=True + _("Status"), max_length=50, null=True, blank=True, + choices=choices.XMLURL_STATUS, ) pid = models.CharField( _("Article PID"), max_length=23, null=True, blank=True @@ -1642,8 +1751,12 @@ class XMLURL(CommonControlField): zipfile = models.FileField( _("ZIP File"), upload_to=xml_url_zipfile_path, null=True, blank=True, max_length=300, ) - exceptions = models.CharField( - _("Exceptions"), max_length=255, null=True, blank=True + exceptions = models.CharField(_("Exceptions"), max_length=255, null=True, blank=True) + detail = models.JSONField( + _("Detail"), null=True, blank=True + ) + is_public = models.BooleanField( + _("Is Public"), null=True, blank=True, default=None ) base_form_class = CoreAdminModelForm @@ -1653,7 +1766,9 @@ class XMLURL(CommonControlField): FieldPanel("status"), FieldPanel("pid"), FieldPanel("zipfile"), + FieldPanel("detail", widget=ReadOnlyPrettyJSONWidget()), FieldPanel("exceptions"), + FieldPanel("is_public"), ] class Meta: @@ -1665,6 +1780,7 @@ class Meta: models.Index(fields=["url"]), models.Index(fields=["status"]), models.Index(fields=["pid"]), + models.Index(fields=["is_public"], name="pid_provide_is_public_idx"), ] def __str__(self): @@ -1683,14 +1799,16 @@ def create( url=None, status=None, pid=None, - exceptions=None, + detail=None, + is_public=None, ): try: obj = cls() obj.url = url obj.status = status obj.pid = pid - obj.exceptions = exceptions + obj.detail = detail + obj.is_public = is_public obj.creator = user obj.save() return obj @@ -1704,7 +1822,8 @@ def create_or_update( url=None, status=None, pid=None, - exceptions=None, + detail=None, + is_public=None, ): try: obj = cls.get(url=url) @@ -1713,8 +1832,10 @@ def create_or_update( obj.status = status if pid is not None: obj.pid = pid - if exceptions is not None: - obj.exceptions = exceptions + if detail is not None: + obj.detail = detail + if is_public is not None: + obj.is_public = is_public obj.save() return obj except cls.DoesNotExist: @@ -1723,17 +1844,18 @@ def create_or_update( url, status, pid, - exceptions, + detail, + is_public=is_public, ) def save_file(self, xml_content, filename=None): """ Create a zip file from XML content and save it to the zipfile field. - + Args: xml_content: str or bytes - The XML content to compress filename: str - Optional filename for the XML inside the zip (defaults to 'content.xml') - + Returns: bool - True if file was saved successfully, False otherwise """ @@ -1741,53 +1863,145 @@ def save_file(self, xml_content, filename=None): # Convert string to bytes if needed if isinstance(xml_content, str): xml_content = xml_content.encode('utf-8') - + # Create in-memory zip file zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file: # Use provided filename or default xml_filename = filename or 'content.xml' zip_file.writestr(xml_filename, xml_content) - + # Save the zip file to the model zip_filename = f"{self.pid or 'unknown'}_{self.pk or 'new'}.zip" self.zipfile.save(zip_filename, ContentFile(zip_buffer.getvalue()), save=True) - + return True except Exception as e: logging.error(f"Error saving zip file for XMLURL {self.url}: {e}") return False + + @classmethod + def record(cls, user, url, status, document_item, *, exception=None, response=None, xml_with_pre=None, name=None): + detail = {"document_item": document_item} + if exception is not None: + detail["exceptions"] = traceback.format_exc() + if response is not None: + detail["response"] = response + + pid = response.get("v3") if response else None + + is_public = None + if document_item: + doc_status = document_item.get("status") + if doc_status is not None: + is_public = doc_status != "false" + + xmlurl_obj = cls.create_or_update(user=user, url=url, status=status, pid=pid, detail=detail, is_public=is_public) + + if xml_with_pre is not None: + filename = name or pid or "content.xml" + xmlurl_obj.save_file(xml_with_pre.tostring(), filename=filename) + + return xmlurl_obj + + +# ----------------------------------------------------------------------------- +# [models.py] MODELO NOVO — PidProviderXMLRegistration +# Auditoria por documento. Grava SEMPRE (created/updated/skipped/forbidden/ +# conflict/unmatched/error). FK nullable (unmatched/error podem não ter PPX). +# ----------------------------------------------------------------------------- +class PidProviderXMLRegistration(CommonControlField): + LIGHTWEIGHT_STATUSES = {"created", "updated", "skip_update"} + + EVENT_CREATED = "created" + EVENT_UPDATED = "updated" + EVENT_SKIPPED = "skipped" + EVENT_FORBIDDEN = "forbidden" + EVENT_CONFLICT = "conflict" + EVENT_UNMATCHED = "unmatched" + EVENT_ERROR = "error" + EVENT_BAD_REQUEST = "bad_request" + + EVENT_STATUS_CHOICES = ( + (EVENT_CREATED, "created"), + (EVENT_UPDATED, "updated"), + (EVENT_SKIPPED, "skipped"), + (EVENT_FORBIDDEN, "forbidden"), + (EVENT_CONFLICT, "conflict"), + (EVENT_UNMATCHED, "unmatched"), + (EVENT_BAD_REQUEST, "bad_request"), + (EVENT_ERROR, "error"), + ) - -class XMLEvent(BaseEvent, CommonControlField): - """ - Model to log events related to XML processing in the PID Provider system. - - This model captures various events that occur during the processing of XML data, - such as registration attempts, validation errors, and other significant actions, - along with relevant details for debugging and monitoring purposes. - - Attributes: - name (CharField): Name of the event. - detail (JSONField): Detailed information about the event. - created (DateTimeField): Timestamp when the event was created. - completed (BooleanField): Indicates if the event has been completed. - ppxml (ParentalKey): Reference to the related PidProviderXML instance. - - Methods: - data (property): Returns a dictionary with the event's name, detail, and creation timestamp. - create (classmethod): Creates and saves a new XMLEvent instance. - finish: Marks the event as completed and optionally updates details, errors, or exceptions. - """ - ppxml = ParentalKey( - PidProviderXML, on_delete=models.CASCADE, related_name="events" + pid_provider_xml = models.ForeignKey( + PidProviderXML, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="registration_events", + ) + pkg_name = models.CharField( + _("Package name"), max_length=100, null=True, blank=True + ) + event_status = models.CharField( + _("Event status"), + max_length=15, + null=True, + blank=True, + choices=EVENT_STATUS_CHOICES, ) + detail = models.JSONField(_("Detail"), null=True, blank=True) + + base_form_class = CoreAdminModelForm + + panels = [ + FieldPanel("event_status", read_only=True), + FieldPanel("pkg_name", read_only=True), + AutocompletePanel("pid_provider_xml", read_only=True), + FieldPanel("detail", widget=ReadOnlyPrettyJSONWidget(), read_only=True), + ] + + class Meta: + ordering = ["-created"] + verbose_name = _("PidProviderXML Registration") + verbose_name_plural = _("PidProviderXML Registrations") + indexes = [ + models.Index(fields=["pkg_name"]), + models.Index(fields=["event_status"]), + models.Index(fields=["-created"]), + models.Index(fields=["pid_provider_xml"]), + ] + + def __str__(self): + return f"{self.pkg_name} {self.event_status} {self.created}" + + @staticmethod + def _serialize_detail(detail): + """ + O detail do detail contém o objeto PidProviderXML em + detail['registered']. Para gravar em JSON, troca pelo v3/id. + """ + if not detail: + return None + data = dict(detail) + registered = data.get("registered") + if registered is not None and hasattr(registered, "v3"): + data["registered"] = {"id": registered.id, "v3": registered.v3} + return data @classmethod - def register(cls, ppxml, name, detail=None, errors=None, exceptions=None): - obj = cls() - obj.ppxml = ppxml - obj.name = name - completed = bool(not errors and not exceptions) - obj.finish(completed=completed, detail=detail, errors=errors, exceptions=exceptions) - return obj \ No newline at end of file + def record(cls, user, event_status, pid_provider_xml=None, pkg_name=None, + detail=None): + try: + obj = cls() + obj.creator = user + obj.pid_provider_xml = pid_provider_xml + obj.pkg_name = pkg_name or (pid_provider_xml and pid_provider_xml.pkg_name) + obj.event_status = event_status + if event_status not in cls.LIGHTWEIGHT_STATUSES: + obj.detail = cls._serialize_detail(detail) + obj.save() + return obj + except Exception as e: + logging.exception(f"Unable to record PidProviderXMLRegistration: {e}") + return None From 8104e8a132a112442d56f710dbc8205d1b1da172 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 06/20] =?UTF-8?q?feat(pid=5Fprovider):=20migra=C3=A7=C3=A3?= =?UTF-8?q?o=20para=20PidProviderXMLRegistration=20e=20remo=C3=A7=C3=A3o?= =?UTF-8?q?=20de=20XMLEvent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: aplicar no banco de dados as mudanças de modelo introduzidas na refatoração do pid_provider. Solução técnica: cria o modelo PidProviderXMLRegistration com seus índices (pid_provide_pkg_nam_2db0b2_idx, pid_provide_event_s_3c9ae7_idx, pid_provide_created_94fb08_idx, pid_provide_pid_pro_c9fb0e_idx); remove os campos creator, ppxml e updated_by de XMLEvent e em seguida exclui o modelo XMLEvent; adiciona readable_data em PidProviderXML; adiciona detail e is_public em XMLURL, altera o campo status (com choices) e cria o índice pid_provide_is_public_idx. --- ...ration_remove_xmlevent_creator_and_more.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py diff --git a/pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py b/pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py new file mode 100644 index 000000000..838efd843 --- /dev/null +++ b/pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py @@ -0,0 +1,190 @@ +# Generated by Django 5.2.7 on 2026-07-20 18:35 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pid_provider", "0016_alter_pidproviderxml_proc_status_xmlevent"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="PidProviderXMLRegistration", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pkg_name", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Package name", + ), + ), + ( + "event_status", + models.CharField( + blank=True, + choices=[ + ("created", "created"), + ("updated", "updated"), + ("skipped", "skipped"), + ("forbidden", "forbidden"), + ("conflict", "conflict"), + ("unmatched", "unmatched"), + ("bad_request", "bad_request"), + ("error", "error"), + ], + max_length=15, + null=True, + verbose_name="Event status", + ), + ), + ( + "detail", + models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + ], + options={ + "verbose_name": "PidProviderXML Registration", + "verbose_name_plural": "PidProviderXML Registrations", + "ordering": ["-created"], + }, + ), + migrations.RemoveField( + model_name="xmlevent", + name="creator", + ), + migrations.RemoveField( + model_name="xmlevent", + name="ppxml", + ), + migrations.RemoveField( + model_name="xmlevent", + name="updated_by", + ), + migrations.AddField( + model_name="pidproviderxml", + name="readable_data", + field=models.JSONField(blank=True, null=True, verbose_name="Readable data"), + ), + migrations.AddField( + model_name="xmlurl", + name="detail", + field=models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + migrations.AddField( + model_name="xmlurl", + name="is_public", + field=models.BooleanField( + blank=True, default=None, null=True, verbose_name="Is Public" + ), + ), + migrations.AlterField( + model_name="xmlurl", + name="status", + field=models.CharField( + blank=True, + choices=[ + ("success", "Success"), + ("xml_fetch_failed", "XML fetch failed"), + ("pid_provider_xml_failed", "PID provider XML failed"), + ], + max_length=50, + null=True, + verbose_name="Status", + ), + ), + migrations.AddIndex( + model_name="xmlurl", + index=models.Index(fields=["is_public"], name="pid_provide_is_public_idx"), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="creator", + field=models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="pid_provider_xml", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="registration_events", + to="pid_provider.pidproviderxml", + ), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="updated_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + migrations.DeleteModel( + name="XMLEvent", + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["pkg_name"], name="pid_provide_pkg_nam_2db0b2_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["event_status"], name="pid_provide_event_s_3c9ae7_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["-created"], name="pid_provide_created_94fb08_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["pid_provider_xml"], name="pid_provide_pid_pro_c9fb0e_idx" + ), + ), + ] From 01fbaa8e1d04b73a1f87817d8d33dad3d82ce237 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 15:42:37 -0300 Subject: [PATCH 07/20] feat(pid_provider): registra XMLURLViewSet e PidProviderXMLRegistrationViewSet no Wagtail admin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: permitir consulta, pela interface administrativa, das URLs com falha de processamento (XMLURL) e dos eventos de auditoria de registro de XML (PidProviderXMLRegistration). Solução técnica: cria XMLURLViewSet e PidProviderXMLRegistrationViewSet (list_display, list_filter, search_fields e select_related no get_queryset) e os inclui em PidProviderViewSetGroup. --- pid_provider/wagtail_hooks.py | 64 +++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/pid_provider/wagtail_hooks.py b/pid_provider/wagtail_hooks.py index 228742585..2c0f586f8 100644 --- a/pid_provider/wagtail_hooks.py +++ b/pid_provider/wagtail_hooks.py @@ -1,12 +1,10 @@ -from django.http import HttpResponseRedirect from django.utils.translation import gettext_lazy as _ -from wagtail import hooks from wagtail.snippets.models import register_snippet from wagtail.snippets.views.snippets import SnippetViewSetGroup from config.menu import get_menu_order from core.views import CommonControlFieldViewSet -from pid_provider.models import XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML +from pid_provider.models import XMLURL, XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML, PidProviderXMLRegistration class PidProviderXMLViewSet(CommonControlFieldViewSet): @@ -178,6 +176,64 @@ class XMLVersionViewSet(CommonControlFieldViewSet): "available_since", ) +class XMLURLViewSet(CommonControlFieldViewSet): + model = XMLURL + menu_label = _("XML URLs") + menu_icon = "folder" + menu_order = 300 + add_to_settings_menu = False + list_per_page = 10 + + # Configuração de listagem + list_display = [ + "url", + "status", + "pid", + ] + list_filter = { + "status": ["exact"], + "is_public": ["exact"], + } + search_fields = ( + "url", + "status", + "pid", + ) + + +class PidProviderXMLRegistrationViewSet(CommonControlFieldViewSet): + model = PidProviderXMLRegistration + icon = "doc-empty-inverse" + menu_label = _("PID Registration Events") + menu_name = "pid_provider_xml_registration" + + # ordenação na listagem + ordering = ["-created"] + + # colunas da listagem + list_display = ( + "pkg_name", + "event_status", + "pid_provider_xml", + "created", + ) + + # filtros laterais + list_filter = ("event_status", "created") + + # busca + search_fields = ("pkg_name", "pid_provider_xml__v2", "pid_provider_xml__v3") + + # paginação (tabela cresce em volume) + list_per_page = 50 + + def get_queryset(self, request): + # super = CommonControlFieldViewSet + queryset = super().get_queryset(request) + if queryset is None: + queryset = self.model._default_manager.all() + return queryset.select_related("pid_provider_xml") + # Grupo de ViewSets class PidProviderViewSetGroup(SnippetViewSetGroup): @@ -190,6 +246,8 @@ class PidProviderViewSetGroup(SnippetViewSetGroup): FixPidV2ViewSet, PidProviderConfigViewSet, XMLVersionViewSet, + XMLURLViewSet, + PidProviderXMLRegistrationViewSet, ) From 7430d9917a061d38c1c69806276b83b7e58659f1 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 20 Jul 2026 16:45:22 -0300 Subject: [PATCH 08/20] Apaga testes desatualizados --- pid_provider/test_controller.py | 87 --- pid_provider/test_models.py | 1029 ------------------------------- pid_provider/tests.py | 3 - 3 files changed, 1119 deletions(-) delete mode 100644 pid_provider/test_controller.py delete mode 100644 pid_provider/test_models.py delete mode 100644 pid_provider/tests.py diff --git a/pid_provider/test_controller.py b/pid_provider/test_controller.py deleted file mode 100644 index 3ff177825..000000000 --- a/pid_provider/test_controller.py +++ /dev/null @@ -1,87 +0,0 @@ -from unittest.mock import ANY, Mock, patch - -from django.contrib.auth import get_user_model -from django.test import TestCase - -from pid_provider.controller import PidProvider -from pid_provider.models import PidProviderXML - -User = get_user_model() - - -# def get_mock_config(): -# config = object() -# config.host = '' -# config.access_key = '' -# config.secret_key = '' -# config.bucket_root = '' -# config.bucket_app_subdir = 'bucket-app-subdir' -# config.secure = '' -# return config - - -class PidProviderTest(TestCase): - @patch("pid_provider.models.XMLSPS.save") - @patch("pid_provider.models.XMLVersion.save") - @patch("pid_provider.models.XMLIssue.save") - @patch("pid_provider.models.XMLJournal.save") - @patch("pid_provider.models.PidProviderXML.save") - @patch( - "pid_provider.models.PidProviderXML._get_unique_v3", - return_value="SJLD63mRxz9nTXtyMj7SLwk", - ) - @patch( - "pid_provider.models.PidProviderXML._get_unique_v2", - return_value="S2236-89062022061645340", - ) - @patch("pid_provider.controller.PidProviderConfig.get_or_create") - def test_provide_pid_for_xml_zip( - self, - mock_pid_provider_config, - mock_get_unique_v2, - mock_get_unique_v3, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xmlsps_save, - ): - pid_provider = PidProvider() - result = pid_provider.provide_pid_for_xml_zip( - zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", - user=User.objects.first(), - ) - result = list(result) - self.assertEqual("SJLD63mRxz9nTXtyMj7SLwk", result[0]["v3"]) - self.assertEqual("S2236-89062022061645340", result[0]["v2"]) - self.assertIsNone(result[0]["aop_pid"]) - self.assertIsNotNone(result[0]["created"]) - self.assertIsNone(result[0]["updated"]) - self.assertEqual("2236-8906-hoehnea-49-e1082020.xml", result[0]["filename"]) - self.assertEqual("created", result[0]["record_status"]) - self.assertEqual(True, result[0]["xml_changed"]) - - @patch("pid_provider.models.PidProviderXML._query_document") - @patch("pid_provider.models.PidProviderXML.is_equal_to", return_value=True) - def test_provide_pid_for_xml_with_pre_do_nothing_because_it_is_already_updated( - self, - mock_is_equal, - mock_query_document, - ): - # dubla o registro encontrado - pid_provider_xml = Mock(PidProviderXML) - pid_provider_xml.data = {"v3": ""} - mock_query_document.return_value = pid_provider_xml - - pid_provider_ = PidProvider() - result = pid_provider_.provide_pid_for_xml_zip( - zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", - user=User.objects.first(), - ) - result = list(result) - expected = { - "filename": "2236-8906-hoehnea-49-e1082020.xml", - "v3": "", - "xml_with_pre": ANY, - } - self.assertDictEqual(result[0], expected) diff --git a/pid_provider/test_models.py b/pid_provider/test_models.py deleted file mode 100644 index 09626540c..000000000 --- a/pid_provider/test_models.py +++ /dev/null @@ -1,1029 +0,0 @@ -import logging -from datetime import datetime -from unittest import mock -from unittest.mock import ANY, MagicMock, Mock, call, patch - -from django.contrib.auth import get_user_model -from django.test import TestCase -from lxml import etree -from xmlsps.xml_sps_lib import XMLWithPre - -from pid_provider import exceptions, models -from pid_provider.xml_sps_adapter import PidProviderXMLAdapter - -User = get_user_model() - - -def _get_xml_adapter_from_file(path): - for xml_with_pre in XMLWithPre.create(path=path): - obj = PidProviderXMLAdapter(xml_with_pre) - return obj - - -def _get_xml_with_pre(xml=None): - xml = xml or "
" - return XMLWithPre("", etree.fromstring(xml)) - - -def _get_xml_adapter(xml=None): - xml = xml or "
" - xml_with_pre = XMLWithPre("", etree.fromstring(xml)) - obj = PidProviderXMLAdapter(xml_with_pre) - return obj - - -def _get_xml_adapter_with_issue_data(): - xml_adapter = _get_xml_adapter() - xml_adapter.journal_issn_electronic = "data-issn-e" - xml_adapter.journal_issn_print = "data-issn-p" - xml_adapter.volume = "data-vol" - xml_adapter.number = "data-num" - xml_adapter.suppl = "data-suppl" - xml_adapter.pub_year = "data-year" - xml_adapter.issue = models.XMLIssue.get_or_create( - models.XMLJournal.get_or_create("data-issn-e", "data-issn-p"), - "data-vol", - "data-num", - "data-suppl", - "data-year", - ) - xml_adapter.fpage = "data-fpage" - xml_adapter.fpage_seq = "data-fpage-seq" - xml_adapter.lpage = "data-lpage" - - xml_adapter.article_pub_year = "data-pub-year" - xml_adapter.v3 = "123456789012345678901v3" - xml_adapter.v2 = "123456789012345678901v2" - xml_adapter.aop_pid = "12345678901234567890aop" - - xml_adapter.main_doi = "data-main_doi" - xml_adapter.elocation_id = "data-elocation_id" - return xml_adapter - - -class PidProviderXMLValidateQueryParamsTest(TestCase): - def setUp(self): - self.article_params = { - "z_collab": "VALUE", - "z_links": "Links", - "z_partial_body": "Body", - "z_surnames": "Z_SURNAMES", - "article_pub_year": "2020", - "elocation_id": "e19347", - "journal__issn_electronic": "issn electronic", - "journal__issn_print": "issn print", - "main_doi": "DOI", - "pkg_name": "pkgName", - } - - self.issue_params = { - "issue__pub_year": "year", - "issue__volume": "vol", - "issue__number": "num", - "issue__suppl": "suppl", - "fpage": "1", - "fpage_seq": "a", - "lpage": "11", - } - - def test_validate_query_params_all_present(self): - params = self.article_params - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_all_present_plus_issue_params(self): - params = self.article_params - params.update(self.issue_params) - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_issue_params_only(self): - params = {} - params.update(self.issue_params) - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_journal_issns_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["journal__issn_print"] - del params["journal__issn_electronic"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_pub_year_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["article_pub_year"] - del params["issue__pub_year"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_main_doi_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_fpage_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["fpage"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_elocation_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["elocation_id"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_main_doi_fpage_elocation_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_surnames_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_surnames"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_collab"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_links"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["pkg_name"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["pkg_name"] - del params["z_surnames"] - del params["z_collab"] - del params["z_links"] - - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - -@patch( - "pid_provider.xml_sps_adapter.PidProviderXMLAdapter.query_list", - new_callable=mock.PropertyMock, -) -@patch( - "pid_provider.models.PidProviderXML.validate_query_params", - return_value=True, -) -@patch("pid_provider.models.PidProviderXML.objects.get") -class PidProviderXMLQueryDocumentTest(TestCase): - def test_query_document_is_called_with_query_params( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - """ - PidProviderXML._query_document is called with parameters returned by - PidProviderXML.query_list - """ - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.DoesNotExist - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - mock_get.assert_called_once_with(**{"key": "value"}) - - def test_query_document_returns_none_if_document_does_not_exist( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.DoesNotExist - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertIsNone(result) - - def test_query_document_returns_found_document( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.return_value = models.PidProviderXML() - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertEqual(models.PidProviderXML, type(result)) - - def test_query_document_returns_found_item_at_the_second_round( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - {"key": "value2"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = [ - models.PidProviderXML.DoesNotExist, - models.PidProviderXML(), - ] - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertEqual(models.PidProviderXML, type(result)) - - def test_query_document_raises_query_document_error_because_multiple_objects_returned( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.MultipleObjectsReturned - with self.assertRaises( - exceptions.QueryDocumentMultipleObjectsReturnedError - ) as exc: - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - - def test_query_document_raises_error( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - """ - PidProviderXML._query_document is called with parameters returned by - PidProviderXML.query_list - """ - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_validate_params.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError - ) - - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - - -@patch("pid_provider.models.PidProviderXML._query_document") -class PidProviderXMLGetRegisteredTest(TestCase): - def setUp(self): - self.xml_with_pre = _get_xml_with_pre() - - def test_get_registered_returns_dict_with_registered_data( - self, - mock_query_document, - ): - pid_req_xml = models.PidProviderXML() - pid_req_xml.pkg_name = "registered_pkg_name" - pid_req_xml.v2 = "registered_v2" - pid_req_xml.v3 = "registered_v3" - pid_req_xml.aop_pid = "registered_aop_pid" - pid_req_xml.created = datetime(2023, 2, 20) - pid_req_xml.updated = datetime(2023, 2, 20) - - mock_query_document.return_value = pid_req_xml - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - expected = { - "v3": "registered_v3", - "v2": "registered_v2", - "aop_pid": "registered_aop_pid", - "pkg_name": "registered_pkg_name", - "created": "2023-02-20T00:00:00", - "updated": "2023-02-20T00:00:00", - "record_status": "updated", - } - self.assertDictEqual(expected, result) - - def test_get_registered_returns_none( - self, - mock_query_document, - ): - mock_query_document.return_value = None - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIsNone(result) - - def test_get_registered_returns_error_multiple_return( - self, - mock_query_document, - ): - mock_query_document.side_effect = ( - exceptions.QueryDocumentMultipleObjectsReturnedError - ) - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIn("error_type", result.keys()) - self.assertIn("error_msg", result.keys()) - - def test_get_registered_returns_error_not_enough_params( - self, - mock_query_document, - ): - mock_query_document.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError - ) - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIn("error_type", result.keys()) - self.assertIn("error_msg", result.keys()) - - -class PidProviderXMLEvaluateRegistrationTest(TestCase): - def setUp(self): - self.xml_adapter = _get_xml_adapter() - - def test_evaluate_registration_accepts_xml_is_aop_and_registered_is_aop(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = True - - self.xml_adapter.is_aop = True - - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - self.assertTrue(result) - - def test_evaluate_registration_accepts_xml_is_not_aop_and_registered_is_aop(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = True - - self.xml_adapter.is_aop = False - - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - self.assertTrue(result) - - def test_evaluate_registration_raises_error(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = False - - self.xml_adapter.is_aop = True - - with self.assertRaises(exceptions.ForbiddenPidProviderXMLRegistrationError): - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - - -@patch("pid_provider.models.PidProviderXML._get_unique_v2") -class PidProviderXMLAddV2Test(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - # TODO - # def test_add_pid_v2_uses_registered_pid_v2( - # self, - # mock_get_unique_v2, - # ): - # found = models.PidProviderXML() - # found.v2 = "registered_v2" - - # xml_adapter = self._get_xml_adapter(v2='xml_v2') - - # mock_get_unique_v2.return_value = "generated_v2" - - # models.PidProviderXML._add_pid_v2(xml_adapter, found) - # self.assertEqual("registered_v2", xml_adapter.v2) - - def test_add_pid_v2_replace_xml_v2_because_its_value_is_invalid_length_is_not_23( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter(v2="bad_size_not_23") - - mock_get_unique_v2.return_value = "S1806-37132022000201100" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000201100", xml_adapter.v2) - - def test_add_pid_v2_keeps_xml_v2( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter(v2="S1806-37132022000199999") - - mock_get_unique_v2.return_value = "S1806-37132022000300001" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000199999", xml_adapter.v2) - - def test_add_pid_v2_uses_unique_v2( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter() - - mock_get_unique_v2.return_value = "S1806-37132022000201100" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000201100", xml_adapter.v2) - - -class PidProviderXMLAddAopPidTest(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - def test_add_aop_pid_uses_registered_aop_pid( - self, - ): - found = models.PidProviderXML() - found.aop_pid = "12345678901234567890aop" - - xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") - - models.PidProviderXML._add_aop_pid(xml_adapter, found) - self.assertEqual("12345678901234567890aop", xml_adapter.aop_pid) - - def test_add_aop_pid_does_not_replace_by_none( - self, - ): - found = models.PidProviderXML() - found.aop_pid = None - - xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") - - models.PidProviderXML._add_aop_pid(xml_adapter, found) - self.assertEqual("xml_aop_pid", xml_adapter.aop_pid) - - -@patch("pid_provider.models.PidProviderXML._is_registered_pid") -@patch("pid_provider.models.PidProviderXML._get_unique_v3") -class PidProviderXMLAddPidV3Test(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - def test_add_pid_v3_uses_registered_v3( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - found = models.PidProviderXML() - found.v3 = "123456789012345678901v3" - - xml_adapter = self._get_xml_adapter(v3="xml_v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("123456789012345678901v3", xml_adapter.v3) - - def test_add_pid_v3_replaced_by_generated( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - mock__is_registered_pid.return_value = True - mock__get_unique_v3.return_value = "gen456789012345678901v3" - - found = None - - xml_adapter = self._get_xml_adapter(v3="xml_v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("gen456789012345678901v3", xml_adapter.v3) - - def test_add_pid_v3_keeps_xml_v3( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - mock__is_registered_pid.return_value = False - mock__get_unique_v3.return_value = "gen456789012345678901v3" - - found = None - - xml_adapter = self._get_xml_adapter(v3="xml456789012345678901v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("xml456789012345678901v3", xml_adapter.v3) - - -class PidProviderXMLIsEqualToTest(TestCase): - def test_is_equal_to_returns_false(self): - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - registered = models.PidProviderXML() - - result = registered.is_equal_to(xml_adapter) - self.assertFalse(result) - - -class PidProviderXMLAddDataForRegularArticleTest(TestCase): - def setUp(self): - user = User() - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - self.registered = models.PidProviderXML() - self.registered._add_data(xml_adapter, user) - - def test_v3(self): - self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", self.registered.v3) - - def test_v2(self): - self.assertEqual("S1413-41522020000400627", self.registered.v2) - - def test_aop_pid(self): - self.assertEqual("S1413-41522020005000111", self.registered.aop_pid) - - def test_main_doi(self): - self.assertEqual("10.1590/S1413-4152202020180029", self.registered.main_doi) - - def test_fpage(self): - self.assertEqual("627", self.registered.fpage) - - def test_fpage_seq(self): - self.assertEqual(None, self.registered.fpage_seq) - - def test_lpage(self): - self.assertEqual("634", self.registered.lpage) - - def test_elocation_id(self): - self.assertEqual(None, self.registered.elocation_id) - - def test_article_pub_year(self): - self.assertEqual("2020", self.registered.article_pub_year) - - def test_z_surnames(self): - self.assertEqual( - "544700df348a47fdd7c55713054e12663a0c530e60e7a166395a496f77de9d36", - self.registered.z_surnames, - ) - - def test_z_collab(self): - self.assertIsNone(self.registered.z_collab) - - def test_z_links(self): - self.assertIsNone(self.registered.z_links) - - def test_z_partial_body(self): - self.assertEqual( - "2e07675bfe91c65e1544ada450ff2e956fef9b492d30e997ebd47687e0f7afa2", - self.registered.z_partial_body, - ) - - -@patch( - "pid_provider.models.utcnow", - side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], -) -@patch("pid_provider.models.XMLVersion.save_file") -@patch("pid_provider.models.XMLVersion.save") -@patch("pid_provider.models.XMLIssue.save") -@patch("pid_provider.models.XMLJournal.save") -@patch("pid_provider.models.PidProviderXML.save") -@patch("pid_provider.models.PidRequest.save") -class PidProviderXMLRegisterTest(TestCase): - def test_register_returns_error( - self, - mock_pid_request_save, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xml_version_save_file, - mock_now, - ): - expected = { - "result_type": "", - "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", - "origin": "filename.xml", - "xml": "
", - } - - user = User() - xml_with_pre = _get_xml_with_pre() - result = models.PidProviderXML.register( - xml_with_pre=xml_with_pre, - filename="filename.xml", - user=user, - ) - print(result) - self.assertEqual(expected["result_type"], result["result_type"]) - self.assertIsNotNone(result["result_msg"]) - # self.assertEqual(expected["result_message"], result["result_msg"]) - self.assertEqual(expected["origin"], result["origin"]) - self.assertEqual(expected["xml"], result["detail"]["xml"]) - mock_pid_provider_xml_save.assert_not_called() - mock_pid_request_save.assert_called_once_with() - - -@patch( - "pid_provider.models.utcnow", - side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], -) -@patch("pid_provider.models.XMLSPS.save") -@patch("pid_provider.models.XMLVersion.save_file") -@patch("pid_provider.models.XMLVersion.save") -@patch("pid_provider.models.XMLIssue.save") -@patch("pid_provider.models.XMLJournal.save") -@patch("pid_provider.models.PidProviderXML.save") -@patch("pid_provider.models.PidRequest.save") -class PidProviderXMLRegisterTest(TestCase): - def test_register_with_success( - self, - mock_pid_request_save, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xml_version_save_file, - mock_xml_sps_save, - mock_now, - ): - expected = { - "result_type": "", - "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", - "origin": "filename.xml", - "xml": "
", - } - - user = User() - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - result = models.PidProviderXML.register( - xml_with_pre=xml_adapter.xml_with_pre, - filename="ex-aop.xml", - user=user, - ) - self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", result["v3"]) - self.assertEqual("S1413-41522020000400627", result["v2"]) - self.assertEqual("S1413-41522020005000111", result["aop_pid"]) - self.assertEqual("1809-4457-esa-25-04-627", result["pkg_name"]) - self.assertEqual(False, result["xml_changed"]) - self.assertEqual("created", result["record_status"]) - self.assertIsNone(result["updated"]) - self.assertIsNotNone(result["created"]) - mock_pid_request_save.assert_not_called() - - -class XMLURLTest(TestCase): - """Tests for XMLURL model""" - - def setUp(self): - self.user = User.objects.create_user(username="testuser", password="testpass") - self.test_url = "http://example.com/article.xml" - self.test_pid = "ABC123XYZ456" - - def test_create_xmlurl(self): - """Test creating a new XMLURL instance""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - self.assertEqual(xmlurl.status, "pending") - self.assertEqual(xmlurl.pid, self.test_pid) - self.assertEqual(xmlurl.creator, self.user) - - def test_get_xmlurl(self): - """Test getting an XMLURL by URL""" - models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xmlurl = models.XMLURL.get(url=self.test_url) - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - - def test_create_or_update_existing(self): - """Test updating an existing XMLURL""" - # Create initial record - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=None, - ) - - # Update it - updated_xmlurl = models.XMLURL.create_or_update( - user=self.user, - url=self.test_url, - status="success", - pid=self.test_pid, - ) - - self.assertEqual(updated_xmlurl.id, xmlurl.id) - self.assertEqual(updated_xmlurl.status, "success") - self.assertEqual(updated_xmlurl.pid, self.test_pid) - self.assertEqual(updated_xmlurl.updated_by, self.user) - - def test_create_or_update_new(self): - """Test creating a new XMLURL when it doesn't exist""" - xmlurl = models.XMLURL.create_or_update( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - self.assertEqual(xmlurl.status, "pending") - - def test_save_file_with_string_content(self): - """Test save_file method with string XML content""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = "
Test Article
" - result = xmlurl.save_file(xml_content, filename="test.xml") - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_save_file_with_bytes_content(self): - """Test save_file method with bytes XML content""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = b"
Test Article
" - result = xmlurl.save_file(xml_content, filename="test.xml") - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_save_file_default_filename(self): - """Test save_file method with default filename""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = "
Test Article
" - result = xmlurl.save_file(xml_content) - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_str_method(self): - """Test __str__ method""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - expected_str = f"{self.test_url} - pending" - self.assertEqual(str(xmlurl), expected_str) - - -class BasePidProviderXMLURITest(TestCase): - """Tests for BasePidProvider.provide_pid_for_xml_uri method""" - - def setUp(self): - self.user = User.objects.create_user(username="testuser", password="testpass") - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - def test_provide_pid_for_xml_uri_fetch_failure(self, mock_create): - """Test exception type a) - Failure to obtain XML""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create to raise an exception - mock_create.side_effect = Exception("Connection timeout") - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article.xml", - name="test.xml", - user=self.user, - ) - - # Should return error details - self.assertIn("error_msg", result) - self.assertIn("error_type", result) - - # Should create XMLURL with failed status - xmlurl = models.XMLURL.get(url="http://example.com/article.xml") - self.assertEqual(xmlurl.status, "xml_fetch_failed") - self.assertIsNone(xmlurl.pid) - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - @patch.object(models.PidProviderXML, "register") - def test_provide_pid_for_xml_uri_success(self, mock_register, mock_create): - """Test successful processing with XMLURL creation""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create - xml_with_pre = _get_xml_with_pre("
Test
") - mock_create.return_value = [xml_with_pre] - - # Mock successful registration - mock_register.return_value = { - "v3": "test_v3_pid", - "v2": "test_v2_pid", - "created": datetime.now(), - } - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article.xml", - name="test.xml", - user=self.user, - ) - - # Should return success response - self.assertEqual(result.get("v3"), "test_v3_pid") - - # Should create XMLURL with success status - xmlurl = models.XMLURL.get(url="http://example.com/article.xml") - self.assertEqual(xmlurl.status, "success") - self.assertEqual(xmlurl.pid, "test_v3_pid") - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - @patch.object(models.PidProviderXML, "register") - def test_provide_pid_for_xml_uri_registration_failure(self, mock_register, mock_create): - """Test exception type b) - XML obtained but registration failed""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create - xml_with_pre = _get_xml_with_pre("
Test
") - mock_create.return_value = [xml_with_pre] - - # Mock failed registration - mock_register.return_value = { - "error_type": "ValidationError", - "error_msg": "Invalid XML structure", - "v3": "test_v3_pid", - } - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article2.xml", - name="test2.xml", - user=self.user, - ) - - # Should return error response - self.assertIn("error_type", result) - - # Should create XMLURL with failed status and save zipfile - xmlurl = models.XMLURL.get(url="http://example.com/article2.xml") - self.assertEqual(xmlurl.status, "pid_provider_xml_failed") - self.assertEqual(xmlurl.pid, "test_v3_pid") diff --git a/pid_provider/tests.py b/pid_provider/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/pid_provider/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. From 5a396beb61b02bb454f772733092e21fb1b4ee93 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 21 Jul 2026 08:08:07 -0300 Subject: [PATCH 09/20] Adiciona ReadOnlyPrettyJSONWidget --- core/widgets.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 core/widgets.py diff --git a/core/widgets.py b/core/widgets.py new file mode 100644 index 000000000..1ea6893b8 --- /dev/null +++ b/core/widgets.py @@ -0,0 +1,28 @@ +import json +from django import forms + +class ReadOnlyPrettyJSONWidget(forms.Textarea): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.attrs.update({ + 'readonly': True, + 'rows': 20, + 'style': ( + 'font-family: monospace;' + ), + }) + + def format_value(self, value): + if not value: + return '' + try: + parsed = json.loads(value) if isinstance(value, str) else value + return json.dumps( + parsed, + indent=2, + ensure_ascii=False, + sort_keys=True, + default=str, # fallback: converte qualquer coisa para str + ) + except (json.JSONDecodeError, TypeError): + return value \ No newline at end of file From f75a6a1084af74e478bcc21d606551b5fa083f7c Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 21 Jul 2026 11:33:01 -0300 Subject: [PATCH 10/20] =?UTF-8?q?pid=5Fprovider:=20restaura=20model=20XMLE?= =?UTF-8?q?vent=20e=20m=C3=A9todo=20add=5Fevent=20em=20PidProviderXML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Reverter a remoção do model XMLEvent feita em refatoração anterior — o rastreio de eventos por documento (registration attempts, validation errors, etc.) via XMLEvent ainda é necessário e não deve ser substituído pelo novo PidProviderXMLRegistration, que serve a um propósito complementar (auditoria agregada de register()), não ao histórico de eventos por instância de PidProviderXML. Solução técnica: Reintroduzido o import de BaseEvent em 'from tracker.models import BaseEvent, UnexpectedEvent'. Restaurado o método PidProviderXML.add_event(name, proc_status, detail=None, errors=None, exceptions=None), que atualiza proc_status, salva a instância e delega o registro do evento a XMLEvent.register(). Restaurado o model XMLEvent(BaseEvent, CommonControlField), com o campo ppxml (ParentalKey para PidProviderXML, related_name='events') e o classmethod register(), que cria a instância, marca completed conforme presença de errors/exceptions e delega a finalização a BaseEvent.finish(). --- ...017_pidproviderxmlregistration_and_more.py | 175 ++++++++++++++++++ pid_provider/models.py | 40 +++- 2 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py diff --git a/pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py b/pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py new file mode 100644 index 000000000..df583db64 --- /dev/null +++ b/pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py @@ -0,0 +1,175 @@ +# Generated by Django 5.2.7 on 2026-07-21 14:25 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pid_provider", "0016_alter_pidproviderxml_proc_status_xmlevent"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="PidProviderXMLRegistration", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pkg_name", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Package name", + ), + ), + ( + "event_status", + models.CharField( + blank=True, + choices=[ + ("created", "created"), + ("updated", "updated"), + ("skipped", "skipped"), + ("forbidden", "forbidden"), + ("conflict", "conflict"), + ("unmatched", "unmatched"), + ("bad_request", "bad_request"), + ("error", "error"), + ], + max_length=15, + null=True, + verbose_name="Event status", + ), + ), + ( + "detail", + models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + ], + options={ + "verbose_name": "PidProviderXML Registration", + "verbose_name_plural": "PidProviderXML Registrations", + "ordering": ["-created"], + }, + ), + migrations.AddField( + model_name="pidproviderxml", + name="readable_data", + field=models.JSONField(blank=True, null=True, verbose_name="Readable data"), + ), + migrations.AddField( + model_name="xmlurl", + name="detail", + field=models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + migrations.AddField( + model_name="xmlurl", + name="is_public", + field=models.BooleanField( + blank=True, default=None, null=True, verbose_name="Is Public" + ), + ), + migrations.AlterField( + model_name="xmlurl", + name="status", + field=models.CharField( + blank=True, + choices=[ + ("success", "Success"), + ("xml_fetch_failed", "XML fetch failed"), + ("pid_provider_xml_failed", "PID provider XML failed"), + ], + max_length=50, + null=True, + verbose_name="Status", + ), + ), + migrations.AddIndex( + model_name="xmlurl", + index=models.Index(fields=["is_public"], name="pid_provide_is_public_idx"), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="creator", + field=models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="pid_provider_xml", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="registration_events", + to="pid_provider.pidproviderxml", + ), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="updated_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["pkg_name"], name="pid_provide_pkg_nam_2db0b2_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["event_status"], name="pid_provide_event_s_3c9ae7_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["-created"], name="pid_provide_created_94fb08_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["pid_provider_xml"], name="pid_provide_pid_pro_c9fb0e_idx" + ), + ), + ] diff --git a/pid_provider/models.py b/pid_provider/models.py index 782351f35..0fea7e48a 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -36,7 +36,7 @@ compare, QueryBuilderPidProviderXML, ) -from tracker.models import UnexpectedEvent +from tracker.models import BaseEvent, UnexpectedEvent PARTIAL_BODY_MAX = 300 @@ -1564,6 +1564,11 @@ def fix_pkg_name(self, pkg_name): return True return False + def add_event(self, name, proc_status, detail=None, errors=None, exceptions=None): + self.proc_status = proc_status + self.save() + return XMLEvent.register(self, name, detail=detail, errors=errors, exceptions=exceptions) + class FixPidV2(CommonControlField): """ @@ -1905,6 +1910,39 @@ def record(cls, user, url, status, document_item, *, exception=None, response=No return xmlurl_obj +class XMLEvent(BaseEvent, CommonControlField): + """ + Model to log events related to XML processing in the PID Provider system. + + This model captures various events that occur during the processing of XML data, + such as registration attempts, validation errors, and other significant actions, + along with relevant details for debugging and monitoring purposes. + + Attributes: + name (CharField): Name of the event. + detail (JSONField): Detailed information about the event. + created (DateTimeField): Timestamp when the event was created. + completed (BooleanField): Indicates if the event has been completed. + ppxml (ParentalKey): Reference to the related PidProviderXML instance. + + Methods: + data (property): Returns a dictionary with the event's name, detail, and creation timestamp. + create (classmethod): Creates and saves a new XMLEvent instance. + finish: Marks the event as completed and optionally updates details, errors, or exceptions. + """ + ppxml = ParentalKey( + PidProviderXML, on_delete=models.CASCADE, related_name="events" + ) + + @classmethod + def register(cls, ppxml, name, detail=None, errors=None, exceptions=None): + obj = cls() + obj.ppxml = ppxml + obj.name = name + completed = bool(not errors and not exceptions) + obj.finish(completed=completed, detail=detail, errors=errors, exceptions=exceptions) + return obj + # ----------------------------------------------------------------------------- # [models.py] MODELO NOVO — PidProviderXMLRegistration # Auditoria por documento. Grava SEMPRE (created/updated/skipped/forbidden/ From 7ca767207beacfde636723eea42a5fefd82edeb7 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 21 Jul 2026 11:33:01 -0300 Subject: [PATCH 11/20] =?UTF-8?q?pid=5Fprovider:=20remove=20migration=20qu?= =?UTF-8?q?e=20exclu=C3=ADa=20o=20model=20XMLEvent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Consistência com a reversão feita em models.py — como o model XMLEvent voltou a existir no código, a migration 0017 (que removia seus campos creator/ppxml/updated_by e por fim o próprio model via DeleteModel) deixou de refletir o estado real dos models e precisa ser descartada. Solução técnica: Excluído o arquivo pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py. A criação de PidProviderXMLRegistration e os demais campos que essa migration também introduzia (readable_data, XMLURL.detail/is_public, choices de XMLURL.status, índices) precisam ser recriados em uma nova migration, desta vez sem a remoção de XMLEvent — rodar 'python manage.py makemigrations pid_provider' para gerá-la a partir do estado atual dos models. --- ...ration_remove_xmlevent_creator_and_more.py | 190 ------------------ 1 file changed, 190 deletions(-) delete mode 100644 pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py diff --git a/pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py b/pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py deleted file mode 100644 index 838efd843..000000000 --- a/pid_provider/migrations/0017_pidproviderxmlregistration_remove_xmlevent_creator_and_more.py +++ /dev/null @@ -1,190 +0,0 @@ -# Generated by Django 5.2.7 on 2026-07-20 18:35 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("pid_provider", "0016_alter_pidproviderxml_proc_status_xmlevent"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="PidProviderXMLRegistration", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "created", - models.DateTimeField( - auto_now_add=True, verbose_name="Creation date" - ), - ), - ( - "updated", - models.DateTimeField( - auto_now=True, verbose_name="Last update date" - ), - ), - ( - "pkg_name", - models.CharField( - blank=True, - max_length=100, - null=True, - verbose_name="Package name", - ), - ), - ( - "event_status", - models.CharField( - blank=True, - choices=[ - ("created", "created"), - ("updated", "updated"), - ("skipped", "skipped"), - ("forbidden", "forbidden"), - ("conflict", "conflict"), - ("unmatched", "unmatched"), - ("bad_request", "bad_request"), - ("error", "error"), - ], - max_length=15, - null=True, - verbose_name="Event status", - ), - ), - ( - "detail", - models.JSONField(blank=True, null=True, verbose_name="Detail"), - ), - ], - options={ - "verbose_name": "PidProviderXML Registration", - "verbose_name_plural": "PidProviderXML Registrations", - "ordering": ["-created"], - }, - ), - migrations.RemoveField( - model_name="xmlevent", - name="creator", - ), - migrations.RemoveField( - model_name="xmlevent", - name="ppxml", - ), - migrations.RemoveField( - model_name="xmlevent", - name="updated_by", - ), - migrations.AddField( - model_name="pidproviderxml", - name="readable_data", - field=models.JSONField(blank=True, null=True, verbose_name="Readable data"), - ), - migrations.AddField( - model_name="xmlurl", - name="detail", - field=models.JSONField(blank=True, null=True, verbose_name="Detail"), - ), - migrations.AddField( - model_name="xmlurl", - name="is_public", - field=models.BooleanField( - blank=True, default=None, null=True, verbose_name="Is Public" - ), - ), - migrations.AlterField( - model_name="xmlurl", - name="status", - field=models.CharField( - blank=True, - choices=[ - ("success", "Success"), - ("xml_fetch_failed", "XML fetch failed"), - ("pid_provider_xml_failed", "PID provider XML failed"), - ], - max_length=50, - null=True, - verbose_name="Status", - ), - ), - migrations.AddIndex( - model_name="xmlurl", - index=models.Index(fields=["is_public"], name="pid_provide_is_public_idx"), - ), - migrations.AddField( - model_name="pidproviderxmlregistration", - name="creator", - field=models.ForeignKey( - editable=False, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="%(class)s_creator", - to=settings.AUTH_USER_MODEL, - verbose_name="Creator", - ), - ), - migrations.AddField( - model_name="pidproviderxmlregistration", - name="pid_provider_xml", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="registration_events", - to="pid_provider.pidproviderxml", - ), - ), - migrations.AddField( - model_name="pidproviderxmlregistration", - name="updated_by", - field=models.ForeignKey( - blank=True, - editable=False, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="%(class)s_last_mod_user", - to=settings.AUTH_USER_MODEL, - verbose_name="Updater", - ), - ), - migrations.DeleteModel( - name="XMLEvent", - ), - migrations.AddIndex( - model_name="pidproviderxmlregistration", - index=models.Index( - fields=["pkg_name"], name="pid_provide_pkg_nam_2db0b2_idx" - ), - ), - migrations.AddIndex( - model_name="pidproviderxmlregistration", - index=models.Index( - fields=["event_status"], name="pid_provide_event_s_3c9ae7_idx" - ), - ), - migrations.AddIndex( - model_name="pidproviderxmlregistration", - index=models.Index( - fields=["-created"], name="pid_provide_created_94fb08_idx" - ), - ), - migrations.AddIndex( - model_name="pidproviderxmlregistration", - index=models.Index( - fields=["pid_provider_xml"], name="pid_provide_pid_pro_c9fb0e_idx" - ), - ), - ] From 1231ff1636c02c5fe62812e67fc1893ddd09c6d7 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 17:56:12 -0300 Subject: [PATCH 12/20] =?UTF-8?q?Corrige=20compara=C3=A7=C3=A3o=20com=20No?= =?UTF-8?q?ne=20e=20reduz=20ambiguidade=20na=20query=20de=20correspond?= =?UTF-8?q?=C3=AAncia=20de=20artigos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: A query anterior usava OR (|) entre z_surnames, z_collab, z_links e z_partial_body, o que permitia que documentos totalmente diferentes fossem retornados como correspondência só por coincidirem em z_partial_body — campo que, no formato antigo, era o hash do primeiro parágrafo e acabava coincidindo com títulos de seção repetidos entre artigos distintos. Isso gerava falsos positivos no select_records. Solução técnica: - compare_items: usa 'input_data or ""' e 'registered or ""' antes de chamar how_similar, evitando passar None para a função de similaridade. - article_data_query: removida a lógica OR entre os campos textuais; agora sempre retorna um Q com AND entre z_surnames, z_collab, z_links e z_partial_body, exigindo que todos os campos disponíveis coincidam simultaneamente, eliminando a ambiguidade do OR. - Novo método get_article_data_query(issue), separando a combinação da query de dados do artigo conforme o escopo da busca: - issue=True: article_data_query & issue_params & article_location_params (correspondência dentro de um fascículo específico). - issue=False: article_data_query & filtro garantindo volume/number/suppl/elocation_id/fpage/lpage nulos (correspondência apenas por journal + dados do artigo, sem fascículo, evitando confundir com artigos de fascículos diferentes). --- pid_provider/query_params.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index b4dbc1f9c..81cb08a4b 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -45,7 +45,7 @@ def compare_items(label, registered, input_data): elif (input_data or None) == (registered or None): score = 1 else: - score = how_similar(input_data, registered) + score = how_similar(input_data or "", registered or "") response = {"label": label, "score": score} if score != 1: response["registered"] = registered @@ -230,23 +230,29 @@ def article_data_query(self): z_links = self.adapter_data.get("z_links") z_partial_body = self.adapter_data.get("z_partial_body") - # Se houver qualquer dado textual disponível, constrói query com OR (|) - if z_surnames or z_partial_body or z_collab or z_links: - q = Q() - if z_surnames: - q |= Q(z_surnames=z_surnames) - if z_collab: - q |= Q(z_collab=z_collab) - if z_links: - q |= Q(z_links=z_links) - if z_partial_body: - q |= Q(z_partial_body=z_partial_body) - return q - # Caso contrário, retorna os campos (geralmente None neste ponto) com AND return Q( z_surnames=z_surnames, z_collab=z_collab, z_links=z_links, z_partial_body=z_partial_body, - ) & Q(**self.article_location_params) \ No newline at end of file + ) + + def get_article_data_query(self, issue): + if issue: + return ( + self.article_data_query & + Q(**self.issue_params) & + Q(**self.article_location_params) + ) + return ( + self.article_data_query & + Q( + volume__isnull=True, + number__isnull=True, + suppl__isnull=True, + elocation_id__isnull=True, + fpage__isnull=True, + lpage__isnull=True, + ) + ) \ No newline at end of file From 6be07e898ba7886d885919838e58bb25a9b1fc5e Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 17:56:41 -0300 Subject: [PATCH 13/20] Adota body_fragment_fingerprint e usa get_article_data_query no select_records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Migrar o fingerprint principal de correspondência de artigos de z_partial_body para body_fragment_fingerprint, disponibilizar os dados legíveis do artigo com fallback ao XML, e centralizar a lógica de query de seleção de registros no QueryBuilderPidProviderXML. Solução técnica: - Adicionado comentário documentando a migração de z_partial_body (hash do primeiro parágrafo, ambíguo) para body_fragment_fingerprint (hash de 300 caracteres); registros antigos mantêm o valor legado e a query de match passa a cobrir ambos os formatos sem exigir backfill. - Novo método get_readable_data(): retorna self.readable_data se já existir; caso contrário, calcula via self.xml_with_pre.get_article_data(); retorna {} se não houver XML disponível. - as_dict() passa a incluir os dados de get_readable_data() no dicionário retornado. - data_to_compare agora usa get_readable_data() em vez de acessar readable_data diretamente, garantindo dados mesmo antes da persistência. - select_records: substituídas as queries inline (Q(**qbuilder.issue_params) & qbuilder.article_data_query e qbuilder.article_data_query) pelas chamadas qbuilder.get_article_data_query(issue=True) e qbuilder.get_article_data_query(issue=False), centralizando a lógica de filtro no QueryBuilder. - z_partial_body passa a ser preenchido a partir de xml_adapter.xml_with_pre.body_fragment_fingerprint, em vez de xml_adapter.z_partial_body. --- pid_provider/models.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pid_provider/models.py b/pid_provider/models.py index 0fea7e48a..7532bd77f 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -412,6 +412,12 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): z_surnames = models.CharField(_("surnames"), max_length=64, null=True, blank=True) z_collab = models.CharField(_("collab"), max_length=64, null=True, blank=True) z_links = models.CharField(_("links"), max_length=64, null=True, blank=True) + + # NOTA: a partir de então body_fragment_fingerprint (hash 300 chars), não mais + # z_partial_body (hash do primeiro parágrafo que apresentou muita ambiguidade). + # Registros antigos mantêm o valor legado; + # a query de match (article_data_query) compara com ambos os + # candidatos para cobrir os dois formatos sem exigir backfill. z_partial_body = models.CharField( _("partial_body"), max_length=64, null=True, blank=True ) @@ -615,6 +621,7 @@ def data(self): "record_status": "updated" if self.updated else "created", "registered_in_core": self.registered_in_core, } + _data.update(self.get_readable_data()) return _data @classmethod @@ -649,9 +656,16 @@ def is_aop(self): return False return True + def get_readable_data(self): + if self.readable_data: + return self.readable_data + if self.xml_with_pre: + return self.xml_with_pre.get_article_data() + return {} + @property def data_to_compare(self): - readable = self.readable_data or {} + readable = self.get_readable_data() titles = readable.get("article_titles") body_fragment = readable.get("body_fragment") return { @@ -1034,13 +1048,15 @@ def select_records(cls, xml_adapter): "journal-issue-article", list( selected_journal.filter( - Q(**qbuilder.issue_params) & qbuilder.article_data_query + qbuilder.get_article_data_query(issue=True) ) ), ) # 3) journal + dados do artigo - yield "journal-article", list(selected_journal.filter(qbuilder.article_data_query)) + yield "journal-article", list( + selected_journal.filter(qbuilder.get_article_data_query(issue=False)) + ) @staticmethod def select_record(xml_adapter, selection_results): @@ -1184,7 +1200,7 @@ def _add_data(self, xml_adapter, registered_in_core): self.z_surnames = xml_adapter.z_surnames self.z_collab = xml_adapter.z_collab self.z_links = xml_adapter.z_links - self.z_partial_body = xml_adapter.z_partial_body + self.z_partial_body = xml_adapter.xml_with_pre.body_fragment_fingerprint self.readable_data = xml_adapter.xml_with_pre.get_article_data() @@ -1943,6 +1959,7 @@ def register(cls, ppxml, name, detail=None, errors=None, exceptions=None): obj.finish(completed=completed, detail=detail, errors=errors, exceptions=exceptions) return obj + # ----------------------------------------------------------------------------- # [models.py] MODELO NOVO — PidProviderXMLRegistration # Auditoria por documento. Grava SEMPRE (created/updated/skipped/forbidden/ From a245d2b6eb6b6b9311c182ff81790f8990a9f7ca Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:25:34 -0300 Subject: [PATCH 14/20] Suporta dois formatos de hash em z_partial_body via novo partial_body_query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Registros gravados antes desta correção têm z_partial_body como hash do primeiro parágrafo não vazio do corpo (formato legado, sujeito a colisão entre artigos diferentes que compartilham o mesmo texto de seção, ex.: rótulos genéricos como "ARTIGO DE REVISÃO"). A partir de agora, o campo passa a ser preenchido com o fingerprint do corpo INTEIRO do artigo (xml_with_pre.body_fragment_fingerprint), mais robusto. É preciso que a query de correspondência aceite ambos os formatos simultaneamente, sem exigir migração/backfill dos registros antigos. Solução técnica: - __init__: adiciona self.z_body_fragment, obtido diretamente de xml_adapter.xml_with_pre.body_fragment_fingerprint, sem exigir mudanças no packtools nem no PidProviderXMLAdapter. - Novo property partial_body_query: - Reúne em candidates os hashes não vazios entre adapter_data['z_partial_body'] (formato legado) e self.z_body_fragment (formato atual). - Se houver candidatos, usa Q(z_partial_body__in=candidates) para casar com registros gravados em qualquer um dos dois formatos. - Se não houver nenhum candidato (ambos None), usa Q(z_partial_body__isnull=True) explicitamente — evita usar z_partial_body__in=(None, None), que em SQL nunca retorna resultados porque IN é uma cadeia de igualdades e NULL = NULL é UNKNOWN, não True. Isso preserva o comportamento equivalente ao antigo Q(z_partial_body=None), que o Django traduz para IS NULL. - article_data_query: remove o acesso direto a z_partial_body e a comparação Q(z_partial_body=z_partial_body); passa a compor o Q final com '& self.partial_body_query', delegando a lógica de correspondência do campo ao novo property. --- pid_provider/query_params.py | 47 ++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index 81cb08a4b..fd6061944 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -89,6 +89,14 @@ def __init__(self, xml_adapter): """ self.xml_adapter = xml_adapter # Centraliza o acesso aos dados brutos e normalizados (hashes de 64 chars) + # z_body_fragment: fingerprint sha256 do corpo INTEIRO do artigo + # (XMLWithPre.body_fragment_fingerprint), acessado direto do + # xml_with_pre — não requer nenhuma mudança no packtools nem no + # PidProviderXMLAdapter. É mais robusto que z_partial_body (que + # é só o primeiro parágrafo não vazio e pode colidir entre + # artigos diferentes, ex.: rótulos de seção genéricos como + # "ARTIGO DE REVISÃO"). + self.z_body_fragment = xml_adapter.xml_with_pre.body_fragment_fingerprint self.adapter_data = xml_adapter.data self.compare_data = xml_adapter.get_data_to_compare() self.xml_with_pre_data = xml_adapter.xml_with_pre.get_article_data(300) @@ -219,7 +227,39 @@ def article_location_params(self): if order: data["v2__endswith"] = order return data - + + @property + def partial_body_query(self): + """ + Constrói a query para o campo z_partial_body, que hoje armazena + dois formatos possíveis de hash, dependendo de quando o registro + foi salvo: + + - legado: hash de z_partial_body (primeiro parágrafo não vazio + do corpo, via xml_adapter.z_partial_body); + - atual: fingerprint do corpo INTEIRO do artigo + (xml_with_pre.body_fragment_fingerprint), gravado no mesmo + campo z_partial_body a partir desta correção (sem necessidade + de migração/backfill). + + Usa IN com os hashes disponíveis do XML de entrada para casar + com candidatos em qualquer um dos dois formatos. + + Quando o XML de entrada não tem NENHUM dos dois hashes + calculados (ambos None), não é seguro usar + `z_partial_body__in=(None, None)`: em SQL, `IN` é uma cadeia de + igualdades e `NULL = NULL` é UNKNOWN (nunca True), então essa + forma jamais encontraria candidatos com z_partial_body nulo. + Nesse caso, usamos `z_partial_body__isnull=True` explicitamente, + preservando o comportamento equivalente ao antigo + `Q(z_partial_body=None)` (que o Django traduz para IS NULL). + """ + z_partial_body = self.adapter_data.get("z_partial_body") + candidates = set(v for v in (z_partial_body, self.z_body_fragment) if v) + if candidates: + return Q(z_partial_body__in=candidates) + return Q(z_partial_body__isnull=True) + @property def article_data_query(self): """ @@ -228,15 +268,12 @@ def article_data_query(self): z_surnames = self.adapter_data.get("z_surnames") z_collab = self.adapter_data.get("z_collab") z_links = self.adapter_data.get("z_links") - z_partial_body = self.adapter_data.get("z_partial_body") - # Caso contrário, retorna os campos (geralmente None neste ponto) com AND return Q( z_surnames=z_surnames, z_collab=z_collab, z_links=z_links, - z_partial_body=z_partial_body, - ) + ) & self.partial_body_query def get_article_data_query(self, issue): if issue: From 04fccd9bdbce99e8c44ebae7cc4fbe125bcee41d Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:32:27 -0300 Subject: [PATCH 15/20] Adiciona __init__.py ao pacote de testes do pid_provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Tornar o diretório de testes reconhecido como pacote Python, permitindo que os módulos de teste (test_query_params, test_get_best_match, test_select_record, test_select_records, test_register) sejam descobertos e importados corretamente pelo test runner. Solução técnica: - Arquivo vazio, apenas com a função de marcar o diretório como pacote. --- pid_provider/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pid_provider/tests/__init__.py diff --git a/pid_provider/tests/__init__.py b/pid_provider/tests/__init__.py new file mode 100644 index 000000000..e69de29bb From bc68b7f17eb31c25c33a854471ecfdbf4e79440c Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:33:00 -0300 Subject: [PATCH 16/20] =?UTF-8?q?Adiciona=20testes=20unit=C3=A1rios=20para?= =?UTF-8?q?=20QueryBuilderPidProviderXML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Cobrir com testes unitários a lógica de construção de queries em QueryBuilderPidProviderXML, incluindo article_data_query, partial_body_query e get_article_data_query, garantindo que as correções recentes de ambiguidade (OR entre campos textuais, suporte a dois formatos de hash em z_partial_body e tratamento de valores nulos) se comportem conforme esperado. Solução técnica: - Casos de teste cobrindo cenários com e sem candidatos de hash disponíveis (z_partial_body legado, body_fragment_fingerprint atual, ambos ausentes). - Verificação de que get_article_data_query(issue=True/False) monta corretamente a combinação com issue_params e article_location_params ou com o filtro de campos de fascículo nulos. --- pid_provider/tests/test_query_params.py | 556 ++++++++++++++++++++++++ 1 file changed, 556 insertions(+) create mode 100644 pid_provider/tests/test_query_params.py diff --git a/pid_provider/tests/test_query_params.py b/pid_provider/tests/test_query_params.py new file mode 100644 index 000000000..dbf9ae059 --- /dev/null +++ b/pid_provider/tests/test_query_params.py @@ -0,0 +1,556 @@ +""" +Testes para QueryBuilderPidProviderXML e as funções de comparação +(compare, compare_lists, compare_items, get_score, zero_to_none). + +Atualizado para cobrir a correção do falso-match na branch journal-article: +- QueryBuilderPidProviderXML agora também lê + `xml_adapter.xml_with_pre.body_fragment_fingerprint` (fingerprint do + corpo INTEIRO do artigo) diretamente do xml_with_pre — sem depender de + mudança no PidProviderXMLAdapter/packtools. +- `article_data_query` não usa mais z_partial_body isolado: delega ao + novo `partial_body_query`, que monta `z_partial_body__in=[...]` com os + hashes disponíveis (legado + novo fingerprint) ou, quando nenhum dos + dois existe no XML de entrada, `z_partial_body__isnull=True` — nunca + `z_partial_body__in=(None, None)`, que em SQL jamais casaria com + candidatos NULL (NULL = NULL é UNKNOWN, não True). + +ATENÇÃO: ajuste o caminho de import abaixo (`pid_provider.query_params`) +para o módulo real onde essas classes/funções estão definidas no projeto, +caso seja diferente. +""" +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase +from django.db.models import Q + +from pid_provider import exceptions +from pid_provider.query_params import ( + QueryBuilderPidProviderXML, + compare, + compare_items, + compare_lists, + get_score, + zero_to_none, +) + + +def make_xml_adapter( + data=None, + v3=None, + v2=None, + aop_pid=None, + pkg_name=None, + sps_pkg_name=None, + deprecated_sps_pkg_name_list=None, + order=None, + article_titles=None, + surnames=None, + collab=None, + links=None, + partial_body=None, + body_fragment_fingerprint=None, +): + """ + Monta um mock de xml_adapter com a forma esperada por + QueryBuilderPidProviderXML. + + body_fragment_fingerprint: valor de + xml_adapter.xml_with_pre.body_fragment_fingerprint, o novo sinal + (hash do corpo inteiro do artigo) usado em partial_body_query. + """ + adapter = MagicMock() + adapter.data = data or {} + adapter.get_data_to_compare.return_value = {} + adapter.v3 = v3 + adapter.v2 = v2 + adapter.aop_pid = aop_pid + adapter.pkg_name = pkg_name + adapter.sps_pkg_name = sps_pkg_name + adapter.order = order + adapter.xml_with_pre.deprecated_sps_pkg_name_list = deprecated_sps_pkg_name_list or [] + adapter.xml_with_pre.body_fragment_fingerprint = body_fragment_fingerprint + adapter.xml_with_pre.get_article_data.return_value = { + "article_titles": article_titles or [], + "surnames": surnames, + "collab": collab, + "links": links, + "partial_body": partial_body, + } + return adapter + + +class ValidateInputDataTests(SimpleTestCase): + + def test_raises_when_pub_year_missing(self): + adapter = make_xml_adapter(data={}) + qbuilder = QueryBuilderPidProviderXML(adapter) + with self.assertRaises( + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + ): + qbuilder.validate_input_data() + + def test_raises_when_issn_missing(self): + adapter = make_xml_adapter(data={"pub_year": "2026"}) + qbuilder = QueryBuilderPidProviderXML(adapter) + with self.assertRaises(exceptions.RequiredISSNErrorToGetPidProviderXMLError): + qbuilder.validate_input_data() + + def test_passes_when_location_params_present(self): + """Se houver dado de localização do artigo, retorna sem checar dados textuais.""" + adapter = make_xml_adapter( + data={ + "pub_year": "2026", + "issn_print": "1234-5678", + "fpage": "10", + }, + article_titles=[], + surnames=None, + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + qbuilder.validate_input_data() # não deve levantar + + def test_passes_when_textual_data_present(self): + adapter = make_xml_adapter( + data={"pub_year": "2026", "issn_electronic": "0000-1111"}, + article_titles=["Título do artigo"], + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + qbuilder.validate_input_data() # não deve levantar + + def test_passes_when_only_surnames_present(self): + adapter = make_xml_adapter( + data={"pub_year": "2026", "issn_electronic": "0000-1111"}, + surnames="Silva", + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + qbuilder.validate_input_data() # não deve levantar + + def test_raises_not_enough_parameters_when_all_empty(self): + adapter = make_xml_adapter( + data={"pub_year": "2026", "issn_electronic": "0000-1111"}, + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + qbuilder.validate_input_data() + + def test_raises_not_enough_parameters_when_titles_are_blank(self): + """Lista de títulos só com valores falsy deve ser tratada como vazia.""" + adapter = make_xml_adapter( + data={"pub_year": "2026", "issn_electronic": "0000-1111"}, + article_titles=["", None], + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + qbuilder.validate_input_data() + + +class PkgNameListTests(SimpleTestCase): + + def test_combines_all_sources_and_drops_falsy(self): + adapter = make_xml_adapter( + data={}, + pkg_name="pkg-a", + sps_pkg_name="pkg-b", + deprecated_sps_pkg_name_list=["pkg-c", "", None, "pkg-a"], + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.pkg_name_list, {"pkg-a", "pkg-b", "pkg-c"}) + + def test_empty_when_no_names_available(self): + adapter = make_xml_adapter(data={}, pkg_name=None, sps_pkg_name=None) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.pkg_name_list, set()) + + +class IdentifierQueriesTests(SimpleTestCase): + + def test_empty_when_nothing_set(self): + adapter = make_xml_adapter(data={}) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.identifier_queries, Q()) + + def test_v3_only(self): + adapter = make_xml_adapter(data={}, v3="V3-123") + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.identifier_queries, Q(v3="V3-123")) + + def test_v2_and_aop_pid_combine_with_or(self): + adapter = make_xml_adapter(data={}, v2="V2-1", aop_pid="AOP-1") + qbuilder = QueryBuilderPidProviderXML(adapter) + expected = Q(v2="V2-1") | (Q(v2="AOP-1") | Q(aop_pid="AOP-1")) + self.assertEqual(qbuilder.identifier_queries, expected) + + def test_includes_pkg_names_and_main_doi(self): + adapter = make_xml_adapter( + data={"main_doi": "10.1234/xyz"}, + pkg_name="pkg-a", + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + expected = Q(pkg_name__in={"pkg-a"}) | Q(main_doi="10.1234/xyz") + self.assertEqual(qbuilder.identifier_queries, expected) + + +class IssnQueryTests(SimpleTestCase): + + def test_raises_when_no_issn(self): + adapter = make_xml_adapter(data={}) + qbuilder = QueryBuilderPidProviderXML(adapter) + with self.assertRaises(exceptions.RequiredISSNErrorToGetPidProviderXMLError): + qbuilder.issn_query + + def test_electronic_only(self): + adapter = make_xml_adapter(data={"issn_electronic": "0000-1111"}) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.issn_query, Q(issn_electronic="0000-1111")) + + def test_print_only(self): + adapter = make_xml_adapter(data={"issn_print": "1234-5678"}) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.issn_query, Q(issn_print="1234-5678")) + + def test_both_issn_combine_with_or(self): + adapter = make_xml_adapter( + data={"issn_electronic": "0000-1111", "issn_print": "1234-5678"} + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + expected = Q(issn_electronic="0000-1111") | Q(issn_print="1234-5678") + self.assertEqual(qbuilder.issn_query, expected) + + +class IssueParamsTests(SimpleTestCase): + + def test_returns_expected_keys(self): + adapter = make_xml_adapter( + data={"pub_year": "2026", "volume": "10", "number": "2", "suppl": "1"} + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual( + qbuilder.issue_params, + {"pub_year": "2026", "volume": "10", "number": "2", "suppl": "1"}, + ) + + def test_missing_values_are_none(self): + adapter = make_xml_adapter(data={}) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual( + qbuilder.issue_params, + {"pub_year": None, "volume": None, "number": None, "suppl": None}, + ) + + +class ArticleLocationParamsTests(SimpleTestCase): + + def test_without_order(self): + adapter = make_xml_adapter( + data={"elocation_id": "e123", "fpage": "10", "lpage": "20"}, + order=None, + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + params = qbuilder.article_location_params + self.assertEqual( + params, + { + "elocation_id": "e123", + "fpage": "10", + "fpage_seq": None, + "lpage": "20", + }, + ) + self.assertNotIn("v2__endswith", params) + + def test_with_order_adds_v2_endswith(self): + adapter = make_xml_adapter(data={}, order="00003") + qbuilder = QueryBuilderPidProviderXML(adapter) + params = qbuilder.article_location_params + self.assertEqual(params["v2__endswith"], "00003") + + +class PartialBodyQueryTests(SimpleTestCase): + """ + Cobre especificamente o fix do incidente: z_partial_body agora aceita + dois formatos de hash (legado e fingerprint do corpo inteiro), e o + caso "nenhum dos dois presente" precisa cair em isnull=True, nunca em + __in=(None, None). + """ + + def test_uses_in_with_only_legacy_partial_body(self): + adapter = make_xml_adapter( + data={"z_partial_body": "hash-legado"}, + body_fragment_fingerprint=None, + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual( + qbuilder.partial_body_query, Q(z_partial_body__in=["hash-legado"]) + ) + + def test_uses_in_with_only_body_fragment_fingerprint(self): + adapter = make_xml_adapter( + data={}, + body_fragment_fingerprint="hash-corpo-inteiro", + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual( + qbuilder.partial_body_query, + Q(z_partial_body__in=["hash-corpo-inteiro"]), + ) + + def test_uses_in_with_both_hashes_when_both_present_and_different(self): + adapter = make_xml_adapter( + data={"z_partial_body": "hash-legado"}, + body_fragment_fingerprint="hash-corpo-inteiro", + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual( + qbuilder.partial_body_query, + Q(z_partial_body__in=["hash-legado", "hash-corpo-inteiro"]), + ) + + def test_deduplicates_when_both_hashes_are_equal(self): + adapter = make_xml_adapter( + data={"z_partial_body": "hash-igual"}, + body_fragment_fingerprint="hash-igual", + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual( + qbuilder.partial_body_query, Q(z_partial_body__in=["hash-igual"]) + ) + + def test_uses_isnull_when_neither_hash_is_present(self): + """ + Regressão do incidente: quando o XML de entrada não tem nenhum + hash de corpo, a query deve usar isnull=True (equivalente ao + antigo Q(z_partial_body=None)), e JAMAIS __in=(None, None), que + em SQL nunca casaria com candidatos cujo z_partial_body é NULL + (NULL = NULL é UNKNOWN, não True). + """ + adapter = make_xml_adapter(data={}, body_fragment_fingerprint=None) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertEqual(qbuilder.partial_body_query, Q(z_partial_body__isnull=True)) + self.assertNotEqual( + qbuilder.partial_body_query, Q(z_partial_body__in=(None, None)) + ) + + +class ArticleDataQueryTests(SimpleTestCase): + """ + article_data_query agora delega o campo z_partial_body inteiramente a + partial_body_query (ver PartialBodyQueryTests) e mantém AND puro para + z_surnames/z_collab/z_links. + """ + + def test_combines_textual_fields_with_partial_body_query_both_hashes(self): + adapter = make_xml_adapter( + data={ + "z_surnames": "Silva", + "z_collab": None, + "z_links": None, + "z_partial_body": "hash-legado", + }, + body_fragment_fingerprint="hash-corpo-inteiro", + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + expected = Q(z_surnames="Silva", z_collab=None, z_links=None) & Q( + z_partial_body__in=["hash-legado", "hash-corpo-inteiro"] + ) + self.assertEqual(qbuilder.article_data_query, expected) + + def test_falls_back_to_isnull_when_no_body_hash_available(self): + adapter = make_xml_adapter(data={}, body_fragment_fingerprint=None) + qbuilder = QueryBuilderPidProviderXML(adapter) + expected = Q(z_surnames=None, z_collab=None, z_links=None) & Q( + z_partial_body__isnull=True + ) + self.assertEqual(qbuilder.article_data_query, expected) + + def test_two_different_articles_produce_different_queries(self): + """ + Regressão conceitual do incidente: dois artigos com hashes de + corpo diferentes (mesmo que ambos tenham, no passado, colidido + via z_partial_body legado genérico) agora produzem queries IN + distintas, pois o fingerprint do corpo inteiro entra na + composição. + """ + adapter_a = make_xml_adapter( + data={"z_partial_body": "rotulo-generico-artigo-revisao"}, + body_fragment_fingerprint="hash-corpo-artigo-a", + ) + adapter_b = make_xml_adapter( + data={"z_partial_body": "rotulo-generico-artigo-revisao"}, + body_fragment_fingerprint="hash-corpo-artigo-b", + ) + qbuilder_a = QueryBuilderPidProviderXML(adapter_a) + qbuilder_b = QueryBuilderPidProviderXML(adapter_b) + self.assertNotEqual( + qbuilder_a.article_data_query, qbuilder_b.article_data_query + ) + + +class GetArticleDataQueryTests(SimpleTestCase): + """Método usado em select_records (models.py).""" + + def test_issue_true_combines_article_data_issue_and_location_params(self): + adapter = make_xml_adapter( + data={ + "z_surnames": "Silva", + "pub_year": "2026", + "volume": "10", + "number": "2", + "suppl": None, + "elocation_id": "e1", + "fpage": "10", + "lpage": "20", + }, + body_fragment_fingerprint=None, + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + result = qbuilder.get_article_data_query(issue=True) + expected = ( + qbuilder.article_data_query + & Q(**qbuilder.issue_params) + & Q(**qbuilder.article_location_params) + ) + self.assertEqual(result, expected) + + def test_issue_false_requires_issue_and_location_fields_null(self): + adapter = make_xml_adapter( + data={"z_surnames": "Silva"}, body_fragment_fingerprint=None + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + result = qbuilder.get_article_data_query(issue=False) + expected = qbuilder.article_data_query & Q( + volume__isnull=True, + number__isnull=True, + suppl__isnull=True, + elocation_id__isnull=True, + fpage__isnull=True, + lpage__isnull=True, + ) + self.assertEqual(result, expected) + + def test_issue_true_and_false_produce_different_queries(self): + adapter = make_xml_adapter( + data={"z_surnames": "Silva", "pub_year": "2026"}, + body_fragment_fingerprint=None, + ) + qbuilder = QueryBuilderPidProviderXML(adapter) + self.assertNotEqual( + qbuilder.get_article_data_query(issue=True), + qbuilder.get_article_data_query(issue=False), + ) + + +class ZeroToNoneTests(SimpleTestCase): + + def test_returns_none_for_falsy_input(self): + self.assertIsNone(zero_to_none(None)) + self.assertIsNone(zero_to_none("")) + + def test_returns_none_when_digit_zero(self): + self.assertIsNone(zero_to_none("0")) + + def test_returns_data_when_non_digit(self): + self.assertEqual(zero_to_none("abc"), "abc") + + def test_returns_data_when_digit_nonzero(self): + self.assertEqual(zero_to_none("5"), "5") + + +class GetScoreTests(SimpleTestCase): + + def test_equal_and_truthy_returns_max(self): + self.assertEqual(get_score("a", "a", min_value=0, max_value=10), 10) + + def test_equal_and_falsy_returns_min(self): + self.assertEqual(get_score(None, None, min_value=1, max_value=10), 1) + + def test_different_returns_zero(self): + self.assertEqual(get_score("a", "b", min_value=0, max_value=10), 0) + + +class CompareListsTests(SimpleTestCase): + + def test_identical_lists_return_one(self): + self.assertEqual(compare_lists(["a", "b"], ["a", "b"]), 1) + + def test_empty_xml_adapter_titles_returns_zero(self): + self.assertEqual(compare_lists(["a"], []), 0) + + def test_empty_registered_returns_zero(self): + self.assertEqual(compare_lists([], ["a"]), 0) + + @patch("pid_provider.query_params.how_similar") + def test_delegates_to_how_similar_when_different(self, mock_how_similar): + mock_how_similar.return_value = 0.75 + result = compare_lists(["Título Um"], ["Titulo Dois"]) + self.assertEqual(result, 0.75) + mock_how_similar.assert_called_once() + + +class CompareItemsTests(SimpleTestCase): + + def test_list_field_uses_compare_lists(self): + result = compare_items("titles", ["a", "b"], ["a", "b"]) + self.assertEqual(result, {"label": "titles", "score": 1}) + + def test_equal_scalars_score_one_without_registered_key(self): + result = compare_items("z_surnames", "Silva", "Silva") + self.assertEqual(result, {"label": "z_surnames", "score": 1}) + + def test_none_and_falsy_are_treated_as_equal(self): + result = compare_items("z_collab", None, "") + self.assertEqual(result, {"label": "z_collab", "score": 1}) + + @patch("pid_provider.query_params.how_similar") + def test_different_scalars_uses_how_similar_and_includes_registered( + self, mock_how_similar + ): + mock_how_similar.return_value = 0.4 + result = compare_items("z_surnames", "Silva", "Souza") + self.assertEqual( + result, {"label": "z_surnames", "score": 0.4, "registered": "Silva"} + ) + mock_how_similar.assert_called_once_with("Souza", "Silva") + + @patch("pid_provider.query_params.how_similar") + def test_none_input_data_falls_back_to_empty_string_for_how_similar( + self, mock_how_similar + ): + mock_how_similar.return_value = 0.2 + result = compare_items("z_links", "algum-link", None) + self.assertEqual( + result, {"label": "z_links", "score": 0.2, "registered": "algum-link"} + ) + mock_how_similar.assert_called_once_with("", "algum-link") + + @patch("pid_provider.query_params.how_similar") + def test_none_registered_falls_back_to_empty_string_for_how_similar( + self, mock_how_similar + ): + mock_how_similar.return_value = 0.3 + result = compare_items("z_links", None, "algum-link") + self.assertEqual(result, {"label": "z_links", "score": 0.3, "registered": None}) + mock_how_similar.assert_called_once_with("algum-link", "") + + +class CompareTests(SimpleTestCase): + + @patch("pid_provider.query_params.how_similar") + def test_aggregates_scores_from_all_items(self, mock_how_similar): + mock_how_similar.return_value = 0.5 + registered_items = {"title": "Título A", "z_surnames": "Silva"} + input_data = {"title": "Título A", "z_surnames": "Souza"} + + result = compare(registered_items, input_data) + + self.assertEqual(len(result["items"]), 2) + self.assertEqual(result["total_score"], 1.5) # 1 (match) + 0.5 (mocked) + self.assertEqual(result["percentual_score"], 0.75) + + def test_missing_input_key_is_treated_as_none(self): + registered_items = {"z_collab": None} + input_data = {} + + result = compare(registered_items, input_data) + + self.assertEqual(result["total_score"], 1) + self.assertEqual(result["percentual_score"], 1) \ No newline at end of file From 265231d8a0ba6efef5c4027a37991b6fe20ca3cc Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:33:32 -0300 Subject: [PATCH 17/20] =?UTF-8?q?Adiciona=20testes=20unit=C3=A1rios=20para?= =?UTF-8?q?=20get=5Fbest=5Fmatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Cobrir com testes unitários a lógica de seleção do melhor candidato dentre múltiplos registros retornados pela query de correspondência, validando o cálculo de similaridade e o critério de desempate. Solução técnica: - Casos de teste com mocks simulando diferentes níveis de similaridade entre o XML de entrada e os candidatos registrados. - Verificação do comportamento em cenários de match exato, parcial e ausência de candidatos. --- pid_provider/tests/test_get_best_match.py | 139 ++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 pid_provider/tests/test_get_best_match.py diff --git a/pid_provider/tests/test_get_best_match.py b/pid_provider/tests/test_get_best_match.py new file mode 100644 index 000000000..1d83e94e0 --- /dev/null +++ b/pid_provider/tests/test_get_best_match.py @@ -0,0 +1,139 @@ +from django.test import TestCase +from unittest.mock import MagicMock, patch +from pid_provider.models import PidProviderXML + + +class PidProviderXMLBestMatchesTests(TestCase): + + def setUp(self): + # xml_adapter_data é o dado já processado (ex.: xml_adapter.get_data_to_compare()), + # e é passado direto para compare() -- não precisa simular atributos internos. + self.xml_adapter_data_mock = {"title": "Titulo Original", "z_surnames": "Silva; Santos"} + + @patch("pid_provider.models.compare") + def test_get_best_match_single_match_does_not_expose_matched_key(self, mock_compare): + """Com apenas 1 item aprovado (>0.6), 'registered' deve existir mas 'matched' NÃO deve ser exposto.""" + + item_bom = MagicMock(spec=PidProviderXML) + item_bom.id = 101 + item_bom.updated.isoformat.return_value = "2026-06-27T12:00:00" + item_bom.data_to_compare = {"title": "Titulo Original", "z_surnames": "Silva; Santos"} + item_bom.data = {"id": 101, "title": "Titulo Original", "z_surnames": "Silva; Santos"} + + item_ruim = MagicMock(spec=PidProviderXML) + item_ruim.id = 102 + item_ruim.updated.isoformat.return_value = "2026-06-27T13:00:00" + item_ruim.data_to_compare = {"title": "Outro Titulo Completamente Diferente", "z_surnames": "Alves"} + item_ruim.data = {"id": 102, "title": "Outro Titulo Completamente Diferente", "z_surnames": "Alves"} + + def side_effect_compare(item_data, xml_adapter_data): + if item_data["title"] == "Titulo Original": + return {"percentual_score": 0.95} + return {"percentual_score": 0.20} + + mock_compare.side_effect = side_effect_compare + + # Enviados fora de ordem propositalmente + candidates = [item_ruim, item_bom] + result = PidProviderXML.get_best_match(candidates, self.xml_adapter_data_mock) + + # Apenas 1 item passou do corte -> "matched" não deve aparecer + self.assertNotIn("matched", result) + + # "registered" deve existir e ser o OBJETO item de maior score + self.assertEqual(result["registered"], item_bom) + + # "unmatched" sempre é exposto + self.assertEqual(len(result["unmatched"]), 1) + self.assertEqual(result["unmatched"][0]["id"], 102) + + @patch("pid_provider.models.compare") + def test_get_best_match_no_candidates_approved(self, mock_compare): + """Quando nenhum candidato atinge score > 0.6, nem 'registered' nem 'matched' devem existir.""" + + item_fraco = MagicMock(spec=PidProviderXML) + item_fraco.id = 201 + item_fraco.updated.isoformat.return_value = "2026-06-27T14:00:00" + item_fraco.data_to_compare = {"title": "Quase igual, mas nao o suficiente"} + item_fraco.data = {"id": 201, "title": "Quase igual, mas nao o suficiente"} + + mock_compare.return_value = {"percentual_score": 0.48} + + result = PidProviderXML.get_best_match([item_fraco], self.xml_adapter_data_mock) + + self.assertNotIn("registered", result) + self.assertNotIn("matched", result) + self.assertEqual(len(result["unmatched"]), 1) + self.assertEqual(result["unmatched"][0]["id"], 201) + + @patch("pid_provider.models.compare") + def test_get_best_match_two_matches_excludes_registered_from_matched(self, mock_compare): + """Com 2 itens aprovados, 'registered' recebe o de maior score e 'matched' deve conter só o restante (matched[1:]).""" + + item_antigo = MagicMock(spec=PidProviderXML) + item_antigo.id = 301 + item_antigo.updated.isoformat.return_value = "2026-01-01T00:00:00" + item_antigo.data_to_compare = {"title": "Clone"} + item_antigo.data = {"id": 301, "title": "Clone"} + + item_recente = MagicMock(spec=PidProviderXML) + item_recente.id = 302 + item_recente.updated.isoformat.return_value = "2026-06-27T00:00:00" # Mais recente + item_recente.data_to_compare = {"title": "Clone"} + item_recente.data = {"id": 302, "title": "Clone"} + + # Mesmo score alto para os dois -> desempate por 'updated' + mock_compare.return_value = {"percentual_score": 0.90} + + result = PidProviderXML.get_best_match([item_antigo, item_recente], self.xml_adapter_data_mock) + + # reverse=True em (score, updated.isoformat(), id); + # "2026-06-27..." > "2026-01-01..." lexicograficamente, então item_recente vem primeiro (registered). + self.assertEqual(result["registered"], item_recente) + + # "matched" agora é matched[1:] -> exclui o item que virou "registered" + self.assertIn("matched", result) + self.assertEqual(len(result["matched"]), 1) + self.assertEqual(result["matched"][0]["id"], 301) + + self.assertNotIn("unmatched", result) + + @patch("pid_provider.models.compare") + def test_get_best_match_three_matches_only_secondary_items_in_matched(self, mock_compare): + """Com 3+ itens aprovados, 'registered' fica com o 1º colocado e 'matched' com os demais, na mesma ordem de score.""" + + item_1 = MagicMock(spec=PidProviderXML) + item_1.id = 401 + item_1.updated.isoformat.return_value = "2026-06-01T00:00:00" + item_1.data_to_compare = {"title": "A"} + item_1.data = {"id": 401, "title": "A"} + + item_2 = MagicMock(spec=PidProviderXML) + item_2.id = 402 + item_2.updated.isoformat.return_value = "2026-06-01T00:00:00" + item_2.data_to_compare = {"title": "B"} + item_2.data = {"id": 402, "title": "B"} + + item_3 = MagicMock(spec=PidProviderXML) + item_3.id = 403 + item_3.updated.isoformat.return_value = "2026-06-01T00:00:00" + item_3.data_to_compare = {"title": "C"} + item_3.data = {"id": 403, "title": "C"} + + def side_effect_compare(item_data, xml_adapter_data): + scores = {"A": 0.95, "B": 0.85, "C": 0.75} + return {"percentual_score": scores[item_data["title"]]} + + mock_compare.side_effect = side_effect_compare + + result = PidProviderXML.get_best_match([item_3, item_1, item_2], self.xml_adapter_data_mock) + + # item_1 (0.95) é o de maior score -> vira "registered" e some da lista "matched" + self.assertEqual(result["registered"], item_1) + + # "matched" deve conter apenas item_2 (0.85) e item_3 (0.75), nessa ordem + self.assertEqual(len(result["matched"]), 2) + self.assertEqual(result["matched"][0]["id"], 402) + self.assertEqual(result["matched"][1]["id"], 403) + + self.assertNotIn("unmatched", result) \ No newline at end of file From b256d74f5f1d3757b0e1bfbe48a362d33f5e1ae1 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:34:35 -0300 Subject: [PATCH 18/20] =?UTF-8?q?Adiciona=20testes=20unit=C3=A1rios=20para?= =?UTF-8?q?=20select=5Frecord?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Cobrir com testes unitários o fluxo de seleção de um único registro correspondente ao XML de entrada, incluindo a integração com o resultado de get_best_match. Solução técnica: - Casos de teste simulando diferentes conjuntos de resultados retornados pela query de seleção, verificando o registro escolhido em cada cenário. --- pid_provider/tests/test_select_record.py | 216 +++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 pid_provider/tests/test_select_record.py diff --git a/pid_provider/tests/test_select_record.py b/pid_provider/tests/test_select_record.py new file mode 100644 index 000000000..52ed0e9d4 --- /dev/null +++ b/pid_provider/tests/test_select_record.py @@ -0,0 +1,216 @@ +from django.test import TestCase +from unittest.mock import MagicMock, patch +from pid_provider.models import PidProviderXML + + +class PidProviderXMLSelectRecordTests(TestCase): + """ + Testes de select_record com get_best_match totalmente mockado + (get_best_match já tem cobertura própria em outro arquivo). + + IMPORTANTE: select_record recebe pares (label, lista_de_candidatos) + já materializados por select_records (list(...), não QuerySet). + O código só faz truthiness (`if not results`) e `len(results)` — + NUNCA chama `.exists()` ou `.count()`. Por isso os "candidatos" + aqui são listas Python simples, não MagicMock simulando QuerySet. + """ + + def _make_results(self, count): + """Simula a lista de candidatos já materializada para um label.""" + return [MagicMock(name=f"candidate_{i}") for i in range(count)] + + def _make_xml_adapter(self, data_to_compare=None): + xml_adapter = MagicMock() + xml_adapter.get_data_to_compare.return_value = data_to_compare or {} + return xml_adapter + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_returns_empty_dict_when_no_selection_results(self, mock_get_best_match): + """Sem nenhum label/lista, retorna dict vazio e nem chama get_best_match.""" + + xml_adapter = self._make_xml_adapter() + + result = PidProviderXML.select_record(xml_adapter, []) + + self.assertEqual(result, {}) + mock_get_best_match.assert_not_called() + xml_adapter.get_data_to_compare.assert_called_once() + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_skips_falsy_empty_lists(self, mock_get_best_match): + """Listas vazias (falsy) devem ser puladas via `if not results`, sem chamar get_best_match.""" + + empty_list_1 = self._make_results(0) # [] + empty_list_2 = self._make_results(0) # [] + + xml_adapter = self._make_xml_adapter() + selection_results = [ + ("empty_label", empty_list_1), + ("also_empty_label", empty_list_2), + ] + + result = PidProviderXML.select_record(xml_adapter, selection_results) + + self.assertEqual(result, {}) + mock_get_best_match.assert_not_called() + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_uses_matched_list_as_is_no_double_slice(self, mock_get_best_match): + """ + CORRIGIDO: matched_items agora usa a lista "matched" tal como veio de + get_best_match, sem fatiar de novo -- nenhum item deve se perder. + """ + + candidates = self._make_results(5) + xml_adapter = self._make_xml_adapter() + + mock_get_best_match.return_value = { + "registered": "ITEM_1", + "matched": ["ITEM_2_DATA", "ITEM_3_DATA"], + # sem "unmatched": todos os candidatos foram aprovados + } + + result = PidProviderXML.select_record(xml_adapter, [("journal", candidates)]) + + # total_results = len(results), NÃO .count() + self.assertEqual(result["total_results"], 5) + self.assertEqual(result["registered"], "ITEM_1") + # Sem re-fatiamento: os 2 itens de "matched" continuam intactos + self.assertEqual(result["matched_items"], {"journal": ["ITEM_2_DATA", "ITEM_3_DATA"]}) + self.assertNotIn("unmatched_items", result) + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_single_approved_item_returns_response_without_matched_key(self, mock_get_best_match): + """ + CORRIGIDO: com apenas 1 candidato aprovado, get_best_match não retorna "matched", + só "registered". Antes isso caía (erroneamente) no branch de unmatched_items; + agora o gatilho é "if registered:", então a resposta correta é retornada mesmo + sem a chave "matched_items". + """ + + candidates = self._make_results(1) + xml_adapter = self._make_xml_adapter() + + mock_get_best_match.return_value = { + "registered": "ITEM_1", + # sem "matched": só havia 1 candidato aprovado + } + + result = PidProviderXML.select_record(xml_adapter, [("journal", candidates)]) + + self.assertEqual(result["total_results"], 1) + self.assertEqual(result["registered"], "ITEM_1") + self.assertNotIn("matched_items", result) + self.assertNotIn("unmatched_items", result) + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_includes_unmatched_items_alongside_matched(self, mock_get_best_match): + """Quando há "registered"/"matched" E "unmatched" no mesmo label, ambos aparecem na resposta.""" + + candidates = self._make_results(4) + xml_adapter = self._make_xml_adapter() + + mock_get_best_match.return_value = { + "registered": "ITEM_1", + "matched": ["ITEM_2_DATA", "ITEM_3_DATA"], + "unmatched": ["ITEM_4_DATA"], + } + + result = PidProviderXML.select_record(xml_adapter, [("journal", candidates)]) + + self.assertEqual(result["matched_items"], {"journal": ["ITEM_2_DATA", "ITEM_3_DATA"]}) + self.assertEqual(result["unmatched_items"], {"journal": ["ITEM_4_DATA"]}) + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_no_registered_stores_actual_unmatched_list(self, mock_get_best_match): + """ + CORRIGIDO: quando get_best_match não retorna "registered" (nenhum candidato + aprovado) mas retorna "unmatched", unmatched_items[label] agora recebe a lista + real ["ITEM_X_DATA"], e não mais uma auto-referência ao dicionário acumulador. + """ + + candidates = self._make_results(1) + xml_adapter = self._make_xml_adapter() + + mock_get_best_match.return_value = { + "unmatched": ["ITEM_X_DATA"], + # sem "registered": nenhum candidato passou do corte + } + + result = PidProviderXML.select_record(xml_adapter, [("journal", candidates)]) + + self.assertEqual(result, {"unmatched_items": {"journal": ["ITEM_X_DATA"]}}) + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_returns_on_first_label_with_registered_ignoring_earlier_unmatched(self, mock_get_best_match): + """ + Ao encontrar o primeiro label com "registered", a função retorna imediatamente -- + o que foi acumulado em unmatched_items para labels anteriores é descartado. + """ + + candidates_1 = self._make_results(1) + candidates_2 = self._make_results(3) + xml_adapter = self._make_xml_adapter() + + mock_get_best_match.side_effect = [ + {"unmatched": ["LABEL1_UNMATCHED_DATA"]}, # label1: sem "registered" + { + "registered": "LABEL2_ITEM_1", + "matched": ["LABEL2_ITEM_2_DATA", "LABEL2_ITEM_3_DATA"], + }, # label2: com "registered" + ] + + selection_results = [ + ("label1", candidates_1), + ("label2", candidates_2), + ] + + result = PidProviderXML.select_record(xml_adapter, selection_results) + + self.assertEqual(result["total_results"], 3) + self.assertEqual(result["registered"], "LABEL2_ITEM_1") + self.assertEqual(result["matched_items"], {"label2": ["LABEL2_ITEM_2_DATA", "LABEL2_ITEM_3_DATA"]}) + self.assertNotIn("unmatched_items", result) + self.assertNotIn("label1", result) + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_accumulates_actual_unmatched_lists_across_labels_when_none_registered(self, mock_get_best_match): + """ + CORRIGIDO: quando nenhum label produz "registered", a função percorre todos e + retorna {"unmatched_items": unmatched_items} ao final, com cada label apontando + para sua própria lista de não aprovados (não mais para o dict acumulador). + """ + + candidates_1 = self._make_results(1) + candidates_2 = self._make_results(1) + xml_adapter = self._make_xml_adapter() + + mock_get_best_match.side_effect = [ + {"unmatched": ["L1_DATA"]}, + {"unmatched": ["L2_DATA"]}, + ] + + selection_results = [ + ("label1", candidates_1), + ("label2", candidates_2), + ] + + result = PidProviderXML.select_record(xml_adapter, selection_results) + + self.assertEqual( + result, + {"unmatched_items": {"label1": ["L1_DATA"], "label2": ["L2_DATA"]}}, + ) + + @patch("pid_provider.models.PidProviderXML.get_best_match") + def test_select_record_passes_candidates_and_comparison_data_to_get_best_match(self, mock_get_best_match): + """get_best_match deve ser chamado com a lista de candidatos do label e os dados já processados do xml_adapter.""" + + candidates = self._make_results(1) + xml_adapter = self._make_xml_adapter(data_to_compare={"title": "Foo"}) + + mock_get_best_match.return_value = {"unmatched": ["ITEM_DATA"]} + + PidProviderXML.select_record(xml_adapter, [("journal", candidates)]) + + mock_get_best_match.assert_called_once_with(candidates, {"title": "Foo"}) From 17226d6469add6bda0ee2d201cf8bf39bf8b95e0 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:34:35 -0300 Subject: [PATCH 19/20] =?UTF-8?q?Adiciona=20testes=20unit=C3=A1rios=20para?= =?UTF-8?q?=20select=5Frecords?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Cobrir com testes unitários o pipeline de seleção de múltiplos candidatos (por journal-issue-article e journal-article), validando a integração com QueryBuilderPidProviderXML.get_article_data_query nos dois escopos (issue=True/False). Solução técnica: - Casos de teste com mocks para os diferentes yields do gerador de seleção de registros, cobrindo cenários com e sem dados de fascículo. --- pid_provider/tests/test_select_records.py | 180 ++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 pid_provider/tests/test_select_records.py diff --git a/pid_provider/tests/test_select_records.py b/pid_provider/tests/test_select_records.py new file mode 100644 index 000000000..b73e0134e --- /dev/null +++ b/pid_provider/tests/test_select_records.py @@ -0,0 +1,180 @@ +from unittest.mock import MagicMock, patch + +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.test import TestCase + +from pid_provider.models import PidProviderXML + + +User = get_user_model() + + +def build_get_article_data_query_side_effect(issue_true_query, issue_false_query): + """ + Constrói o side_effect para `qbuilder.get_article_data_query(issue)`, + já que agora é ele quem decide a query final (antes, select_records + montava `Q(**issue_params) & article_data_query` diretamente). + """ + def _side_effect(issue): + return issue_true_query if issue else issue_false_query + return _side_effect + + +class PidProviderXMLSelectRecordsTests(TestCase): + """ + select_records agora é um generator: apenas yield-a tuplas + (label, lista_de_candidatos_materializada) com os candidatos de + cada estratégia de busca. Cada branch é convertida com list(...) + dentro do próprio método (ver docstring de select_records), então + o que chega aqui NÃO é mais um QuerySet — é uma list — e portanto + não expõe métodos como .count() ou .filter(). + Ele NÃO chama mais best_matches nem levanta DoesNotExist — + essa orquestração ficou fora deste método. + + IMPORTANTE (pós-diff): as branches 2 e 3 não usam mais + `qbuilder.issue_params` e `qbuilder.article_data_query` diretamente — + passaram a usar `qbuilder.get_article_data_query(issue=True)` (branch + "journal-issue-article") e `qbuilder.get_article_data_query(issue=False)` + (branch "journal-article"). Por isso o mock precisa configurar + `get_article_data_query` (não mais `issue_params`/`article_data_query` + isolados). + """ + + def setUp(self): + self.user = User.objects.create_user(username="testuser", password="password") + + self.xml_adapter_mock = MagicMock() + self.xml_adapter_mock.xml_with_pre.article_titles_texts = "Titulo de Teste" + self.xml_adapter_mock.z_surnames = "Silva" + self.xml_adapter_mock.z_collab = None + self.xml_adapter_mock.z_links = None + self.xml_adapter_mock.z_partial_body = "Corpo parcial do artigo" + self.xml_adapter_mock.sps_pkg_name = "test_package" + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + def test_select_records_yields_three_labeled_lists_in_order(self, mock_qbuilder_cls): + """O generator deve produzir, nesta ordem: ids, journal-issue-article, journal-article.""" + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="12345") + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + + # issue=True -> exige pub_year=2026 (equivalente ao antigo issue_params) + # issue=False -> ignora pub_year, só olha z_surnames (equivalente ao + # antigo article_data_query "puro") + mock_qbuilder.get_article_data_query.side_effect = ( + build_get_article_data_query_side_effect( + issue_true_query=Q(pub_year=2026) & Q(z_surnames="Silva"), + issue_false_query=Q(z_surnames="Silva"), + ) + ) + + record_by_id = PidProviderXML.objects.create( + creator=self.user, v3="12345", registered_in_core=True + ) + record_by_journal_issue_article = PidProviderXML.objects.create( + creator=self.user, + issn_print="1234-5678", + pub_year=2026, + z_surnames="Silva", + ) + record_by_journal_article_only = PidProviderXML.objects.create( + creator=self.user, + issn_print="1234-5678", + pub_year=1999, + z_surnames="Silva", + ) + + results = list(PidProviderXML.select_records(self.xml_adapter_mock)) + + mock_qbuilder.validate_input_data.assert_called_once() + + self.assertEqual(len(results), 3) + + labels = [label for label, _ in results] + self.assertEqual(labels, ["ids", "journal-issue-article", "journal-article"]) + + # cada branch já vem materializada como list (não QuerySet) + for _label, candidates in results: + self.assertIsInstance(candidates, list) + + # get_article_data_query deve ter sido chamado com issue=True e + # depois issue=False, nesta ordem + calls = [c.args[0] if c.args else c.kwargs.get("issue") for c in mock_qbuilder.get_article_data_query.call_args_list] + self.assertEqual(calls, [True, False]) + + # 1) ids: só o registro com v3 correspondente + ids_list = results[0][1] + self.assertIn(record_by_id, ids_list) + self.assertNotIn(record_by_journal_issue_article, ids_list) + self.assertNotIn(record_by_journal_article_only, ids_list) + + # 2) journal + issue + artigo: só o que bate no pub_year certo + journal_issue_article_list = results[1][1] + self.assertIn(record_by_journal_issue_article, journal_issue_article_list) + self.assertNotIn(record_by_journal_article_only, journal_issue_article_list) + + # 3) journal + artigo (ignora issue): pega os dois do mesmo issn/z_surnames + journal_article_list = results[2][1] + self.assertIn(record_by_journal_issue_article, journal_article_list) + self.assertIn(record_by_journal_article_only, journal_article_list) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + def test_select_records_empty_lists_when_no_match(self, mock_qbuilder_cls): + """Sem nenhum registro correspondente, cada lista yield deve vir vazia (sem levantar exceção).""" + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="nao_existe") + mock_qbuilder.issn_query = Q(issn_print="0000-0000") + mock_qbuilder.get_article_data_query.side_effect = ( + build_get_article_data_query_side_effect( + issue_true_query=Q(pub_year=1900) & Q(z_surnames="Ninguem"), + issue_false_query=Q(z_surnames="Ninguem"), + ) + ) + + results = list(PidProviderXML.select_records(self.xml_adapter_mock)) + + self.assertEqual(len(results), 3) + for _label, candidates in results: + self.assertIsInstance(candidates, list) + # listas usam len(), não .count() (que é método de QuerySet) + self.assertEqual(len(candidates), 0) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + def test_select_records_is_lazy_until_iterated(self, mock_qbuilder_cls): + """ + Por ser generator, nada é executado na chamada da função: + QueryBuilderPidProviderXML(...) e validate_input_data() só + rodam quando o generator é de fato consumido (primeiro next()). + get_article_data_query só é chamado a partir do 2º/3º next(), + já que a 1ª branch ("ids") não depende dele. + """ + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="qualquer") + mock_qbuilder.issn_query = Q(issn_print="0000-0000") + mock_qbuilder.get_article_data_query.side_effect = ( + build_get_article_data_query_side_effect( + issue_true_query=Q(), + issue_false_query=Q(), + ) + ) + + gen = PidProviderXML.select_records(self.xml_adapter_mock) + + # nada foi executado ainda + mock_qbuilder_cls.assert_not_called() + mock_qbuilder.validate_input_data.assert_not_called() + + next(gen) # yield "ids" + + mock_qbuilder_cls.assert_called_once_with(self.xml_adapter_mock) + mock_qbuilder.validate_input_data.assert_called_once() + # "ids" não usa get_article_data_query + mock_qbuilder.get_article_data_query.assert_not_called() + + next(gen) # yield "journal-issue-article" + mock_qbuilder.get_article_data_query.assert_called_once_with(issue=True) + + next(gen) # yield "journal-article" + self.assertEqual(mock_qbuilder.get_article_data_query.call_count, 2) + mock_qbuilder.get_article_data_query.assert_called_with(issue=False) \ No newline at end of file From 6cdd5c5c05b45968b6b83f34af9594bcba42a73e Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 26 Jul 2026 18:34:35 -0300 Subject: [PATCH 20/20] =?UTF-8?q?Adiciona=20testes=20unit=C3=A1rios=20para?= =?UTF-8?q?=20o=20fluxo=20de=20registro=20(register)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propósito: Cobrir com testes unitários o fluxo completo de registro de um XML, incluindo a seleção de registros existentes, atualização ou criação de PidProviderXML e a gravação dos campos derivados (z_surnames, z_collab, z_links, z_partial_body via body_fragment_fingerprint). Solução técnica: - Casos de teste simulando cenários de criação, atualização e registros já existentes, com mocks para as dependências externas (xml_adapter, xml_with_pre). - Verificação de que os campos textuais e o novo fingerprint de corpo são persistidos corretamente conforme a lógica revisada em models.py e query_params.py. --- pid_provider/tests/test_register.py | 511 ++++++++++++++++++++++++++++ 1 file changed, 511 insertions(+) create mode 100644 pid_provider/tests/test_register.py diff --git a/pid_provider/tests/test_register.py b/pid_provider/tests/test_register.py new file mode 100644 index 000000000..5a09550a2 --- /dev/null +++ b/pid_provider/tests/test_register.py @@ -0,0 +1,511 @@ +""" +Testes unitários para PidProviderXML.register. + +Estratégia +---------- +register() é orquestrador: delega a select_record (NÃO select_records), +complete_missing_xml_pids, is_updated e _save. Os testes isolam register +desses colaboradores via mock e verificam, para cada caminho, os contratos +relevantes: valor de response["event_status"], conteúdo do response, e SE/COMO +PidProviderXMLRegistration.record foi chamado. + +MUDANÇAS DE CONTRATO EM RELAÇÃO À VERSÃO ANTERIOR DESTE ARQUIVO +---------------------------------------------------------------- +1. `select_records` (plural) é um gerador (tem `yield`) que apenas monta + querysets candidatos; quem de fato resolve o "melhor match" e decide se + há registro é `select_record` (singular, @staticmethod), que recebe esse + gerador. Mockar `select_records` sozinho não controla o fluxo — é preciso + mockar `select_record`, que é o que register() efetivamente consome. + +2. `_save` agora retorna o objeto PidProviderXML salvo diretamente + (não mais uma tupla `(objeto, status)`). + +3. **register() NÃO grava auditoria sempre.** No `finally`, a chamada a + `PidProviderXMLRegistration.record` só acontece se: + error_type (uma exceção foi capturada) OR + select_record_response.get("matched_items") (havia ambiguidade) + Ou seja: um "created"/"updated"/"skipped" limpo, sem matches concorrentes + e sem erro, NÃO gera registro de auditoria. Isso é uma mudança de + comportamento relevante e é testado explicitamente abaixo. + +4. `is_updated` deixou de retornar um dict "já atualizado" — agora ela + LEVANTA exceções para sinalizar o que aconteceu: + - `exceptions.SkipSavePidProviderXML`: quando o XML é igual ao + registrado, ou quando `origin_date` do registrado é mais recente. + - `exceptions.ForbiddenPidProviderXMLRegistrationError`: quando o XML + é AOP mas o registrado já é VoR (comportamento antigo, inalterado). + Essas exceções agora são capturadas no lugar certo: um `try/except` que + envolve a chamada a `is_updated()` E a chamada a `_save()` juntas: + - `except ForbiddenPidProviderXMLRegistrationError`: seta + `event_status = "forbidden"` e RE-LEVANTA (propaga para o + `except Exception` externo, vira erro registrado com esse status). + Uma rodada anterior deste diff havia adicionado essa exceção ao + tuple de "bad_request" (que só envolve a chamada a `select_record`, + não `is_updated`), o que não tinha efeito nenhum — isso foi + corrigido: agora forbidden é de fato capturado onde é levantado e + gravado como "forbidden", não mais como "updated"/"bad_request". + - `except SkipSavePidProviderXML`: seta `event_status = "skipped"`, + `response["skipped"] = True`, `response.update(registered.data)` + usando o objeto JÁ EXISTENTE (pois `_save` nunca chega a rodar + nesse caminho) — e NÃO relevanta, então o fluxo segue normalmente + até o `finally` (sem passar por `except Exception`). + +5. `event_status = event_status or "error"` foi adicionado no `except` + externo. Isso significa: se uma exceção genérica ocorre ANTES de + qualquer `event_status` ter sido atribuído, o valor gravado agora é + `"error"` (não mais `None`). Se a exceção ocorre DEPOIS que algum status + já existia (ex.: "created", "forbidden"), esse status é preservado — + `x or "error"` só substitui quando `x` é falsy. `UnexpectedEvent.create` + continua sem ser chamado dentro de register(). + +6. Os caminhos "conflict" e "unmatched" continuam setando `event_status` + explicitamente antes de re-levantar a exceção. + +Ajuste os caminhos de import (PATCH_BASE) conforme a estrutura do seu projeto. +""" + +from unittest.mock import patch, MagicMock + +from django.test import TestCase + +from pid_provider import exceptions +from pid_provider.models import ( + PidProviderXML, + PidProviderXMLPidV3ConflictError, +) + +# Caminho do módulo onde register está definido (para os patches "where used"). +PATCH_BASE = "pid_provider.models" + + +def make_xml_with_pre(**overrides): + """ + XMLWithPre falso, com os atributos que register/build_readable_data tocam. + """ + m = MagicMock(name="xml_with_pre") + m.data = {"pid_v3": overrides.get("v3"), "sps_pkg_name": "pkg-fake"} + m.sps_pkg_name = overrides.get("sps_pkg_name", "pkg-fake") + m.authors = {"person": [{"surname": "SILVA"}]} + m.collab = None + m.links = [] + m.article_titles_texts = ["Some title"] + m.partial_body = "corpo parcial" + return m + + +class RegisterTestBase(TestCase): + """ + Mocka os colaboradores de register e o gravador de auditoria. + Cada teste configura os side_effects/returns conforme o caminho. + """ + + def setUp(self): + self.user = MagicMock(name="user") + self.xml = make_xml_with_pre(v3="ABCDEFGHIJKLMNOPQRSTUVW") + + # patch do adapter para não depender de packtools real + self.p_adapter = patch( + "packtools.sps.pid_provider.xml_sps_adapter.PidProviderXMLAdapter" + ) + self.m_adapter_cls = self.p_adapter.start() + self.m_adapter = self.m_adapter_cls.return_value + self.m_adapter.data = {"pkg_name": "pkg-fake"} + self.m_adapter.sps_pkg_name = "pkg-fake" + self.m_adapter.xml_with_pre = self.xml + self.addCleanup(self.p_adapter.stop) + + # select_records é um gerador que apenas monta querysets candidatos; + # quem register() efetivamente usa para decidir o fluxo é + # select_record (singular). Deixamos select_records "inofensivo" + # (não é consumido, pois select_record está sempre mockado abaixo). + self.p_select_records = patch(f"{PATCH_BASE}.PidProviderXML.select_records") + self.m_select_records = self.p_select_records.start() + self.m_select_records.return_value = iter([]) + self.addCleanup(self.p_select_records.stop) + + # patch do gravador de auditoria — ponto central de verificação + self.p_record = patch(f"{PATCH_BASE}.PidProviderXMLRegistration.record") + self.m_record = self.p_record.start() + self.addCleanup(self.p_record.stop) + + # -- helpers de asserção -------------------------------------------- + def assert_recorded_status(self, expected_status): + self.assertTrue( + self.m_record.called, "PidProviderXMLRegistration.record não foi chamado" + ) + kwargs = self.m_record.call_args.kwargs + self.assertEqual(kwargs.get("event_status"), expected_status) + return kwargs + + def assert_not_recorded(self): + self.m_record.assert_not_called() + + +# --------------------------------------------------------------------------- +# Caminhos "limpos" (sem ambiguidade, sem erro) -> NÃO devem gravar auditoria +# --------------------------------------------------------------------------- +class CreatedPathTest(RegisterTestBase): + def test_created_when_no_existing_record_and_no_ambiguity(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + # nenhum "registered" no retorno -> pop KeyError -> sem + # unmatched_items -> DoesNotExist -> event_status="created" + m_select.return_value = {} + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "created"} + m_save.return_value = saved # objeto direto, não mais tupla + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(response.get("event_status"), "created") + self.assertEqual(response.get("v3"), "ABC") + self.assertNotIn("error_msg", response) + # created "limpo": sem matched_items e sem erro -> não grava auditoria + self.assert_not_recorded() + + +class UpdatedPathTest(RegisterTestBase): + def test_updated_when_existing_record_and_no_ambiguity(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = {"registered": existing} # sem matched_items + m_cmp.return_value = {"pid_v3": "NEW"} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "updated"} + m_save.return_value = saved + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(response.get("event_status"), "updated") + self.assertIn("xml_changed", response) + # updated "limpo" (match único, sem ambiguidade) -> não grava auditoria + self.assert_not_recorded() + + +class SkippedPathTest(RegisterTestBase): + def test_skipped_returns_data_and_does_not_log_when_clean(self): + """ + MUDANÇA DE CONTRATO: skip agora é sinalizado por is_updated + LEVANTANDO exceptions.SkipSavePidProviderXML, não mais retornando + um dict truthy. register() captura essa exceção, seta + event_status="skipped" e usa registered.data do objeto JÁ + EXISTENTE (pois _save nunca roda nesse caminho). + """ + existing = MagicMock(name="existing_ppx") + existing.data = {"v3": "ABC", "record_status": "updated"} + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = {"registered": existing} # sem matched_items + m_cmp.return_value = {} + m_upd.side_effect = exceptions.SkipSavePidProviderXML + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + self.assertEqual(response.get("event_status"), "skipped") + self.assertTrue(response.get("skipped")) + self.assertEqual(response.get("v3"), "ABC") # veio de existing.data + self.assertNotIn("error_msg", response) + # skip "limpo" (sem ambiguidade, sem erro) -> não grava auditoria + self.assert_not_recorded() + + +# --------------------------------------------------------------------------- +# Ambiguidade (matched_items presentes) -> DEVE gravar auditoria mesmo sem erro +# --------------------------------------------------------------------------- +class MatchedItemsLoggingTest(RegisterTestBase): + def test_updated_with_matched_items_logs_audit(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = { + "registered": existing, + "matched_items": [{"id": 2, "v3": "OTHER"}], + } + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "updated"} + m_save.return_value = saved + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + kwargs = self.assert_recorded_status("updated") + self.assertIs(kwargs.get("pid_provider_xml"), saved) + self.assertIn("select_record_response", response) + + def test_skipped_with_matched_items_still_logs_audit(self): + existing = MagicMock(name="existing_ppx") + existing.data = {"v3": "ABC", "record_status": "updated"} + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = { + "registered": existing, + "matched_items": [{"id": 2, "v3": "OTHER"}], + } + m_cmp.return_value = {} + m_upd.side_effect = exceptions.SkipSavePidProviderXML + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + # mesmo em skip, se havia ambiguidade, a auditoria é gravada + self.assert_recorded_status("skipped") + + +# --------------------------------------------------------------------------- +# Caminhos de erro -> sempre logam (error_type setado no except externo) +# --------------------------------------------------------------------------- +class ConflictPathTest(RegisterTestBase): + def test_conflict_when_pid_v3_conflict(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = {"registered": existing} + m_cmp.side_effect = PidProviderXMLPidV3ConflictError("conflict!") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + self.assert_recorded_status("conflict") + self.assertIn("error_msg", response) + self.assertIn("error_type", response) + + +class ForbiddenPathTest(RegisterTestBase): + def test_forbidden_when_aop_over_vor(self): + """ + CORRIGIDO em relação à rodada anterior: agora existe um + `try/except ForbiddenPidProviderXMLRegistrationError` envolvendo + `is_updated()` + `_save()`, que seta event_status="forbidden" e + RE-LEVANTA a exceção (propaga para o except Exception externo, + que grava o erro mas preserva event_status="forbidden", já que + `event_status or "error"` só substitui valores falsy). + """ + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = {"registered": existing} + m_cmp.return_value = {} + m_upd.side_effect = ( + exceptions.ForbiddenPidProviderXMLRegistrationError("forbidden") + ) + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + kwargs = self.assert_recorded_status("forbidden") + self.assertIn("error_msg", response) + self.assertEqual(response.get("event_status"), "forbidden") + self.assertIs(kwargs.get("pid_provider_xml"), existing) + + +class UnmatchedPathTest(RegisterTestBase): + def test_unmatched_when_select_record_raises_unmatched(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.side_effect = exceptions.UnmatchedPidProviderXMLError("unmatched") + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + self.assertIn("error_msg", response) + + def test_unmatched_when_unmatched_items_without_registered(self): + # select_record retorna dict com unmatched_items e sem "registered" + # -> register() levanta UnmatchedPidProviderXMLError internamente + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.return_value = {"unmatched_items": [{"id": 1}]} + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + + def test_multiple_objects_returned_is_unmatched(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.side_effect = PidProviderXML.MultipleObjectsReturned() + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + + +class BadRequestPathTest(RegisterTestBase): + """ + As exceções de bad_request continuam setando event_status="bad_request" + explicitamente antes de re-levantar, então esse contrato NÃO mudou. + """ + + def test_required_issn_becomes_response_not_raise(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.side_effect = ( + exceptions.RequiredISSNErrorToGetPidProviderXMLError("no issn") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + self.assertIn("error_msg", response) + + def test_required_pub_year_becomes_response(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.side_effect = ( + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError("no year") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + + def test_not_enough_parameters_becomes_response(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.side_effect = ( + exceptions.NotEnoughParametersToGetPidProviderXMLError("not enough") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + + +class UnexpectedErrorPathTest(RegisterTestBase): + """ + ATENÇÃO — MUDANÇA DE CONTRATO: + Agora existe `event_status = event_status or "error"` no `except` + externo. Se nenhum event_status tinha sido atribuído ainda quando a + exceção genérica ocorre, o valor gravado passa a ser "error" (fallback). + Se um status já existia (ex.: "created"), ele é preservado — + `UnexpectedEvent.create` continua sem ser chamado dentro de register(). + """ + + def test_unexpected_exception_before_any_status_set_falls_back_to_error(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select: + m_select.side_effect = ValueError("falha totalmente inesperada") + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + # fallback: nenhum status tinha sido setado -> "error" + self.assert_recorded_status("error") + self.assertIn("error_msg", response) + + def test_unexpected_exception_after_created_keeps_created_status(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp: + + m_select.return_value = {} # -> DoesNotExist -> event_status="created" + m_cmp.side_effect = ValueError("algo inesperado depois de created") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + # event_status permanece "created" (fallback só entra quando é falsy) + self.assert_recorded_status("created") + self.assertIn("error_msg", response) + self.assertEqual(response.get("event_status"), "created") + + +# --------------------------------------------------------------------------- +# Invariante: quando aplicável, record() é chamado no máximo 1 vez +# (nunca duplicado por causa de except + finally) +# --------------------------------------------------------------------------- +class RecordInvocationInvariantTest(RegisterTestBase): + def test_record_not_called_on_clean_success(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = {} + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock() + saved.data = {"v3": "ABC"} + m_save.return_value = saved + + PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(self.m_record.call_count, 0) + + def test_record_called_exactly_once_on_conflict(self): + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp: + m_select.return_value = {"registered": MagicMock()} + m_cmp.side_effect = PidProviderXMLPidV3ConflictError("x") + PidProviderXML.register(self.xml, "file.xml", self.user) + + # antes havia risco de gravar 2x (except interno + finally); deve ser 1 + self.assertEqual(self.m_record.call_count, 1) + + def test_record_called_exactly_once_when_matched_items_present(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_select.return_value = { + "registered": existing, + "matched_items": [{"id": 2}], + } + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock() + saved.data = {"v3": "ABC"} + m_save.return_value = saved + + PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(self.m_record.call_count, 1) + + def test_record_called_exactly_once_on_forbidden(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd: + + m_select.return_value = {"registered": existing} + m_cmp.return_value = {} + m_upd.side_effect = ( + exceptions.ForbiddenPidProviderXMLRegistrationError("x") + ) + PidProviderXML.register(self.xml, "file.xml", self.user) + + # o except específico de Forbidden seta status e re-levanta; o + # except Exception externo não deve gravar de novo + self.assertEqual(self.m_record.call_count, 1) + + def test_record_called_exactly_once_on_skip(self): + existing = MagicMock(name="existing_ppx") + existing.data = {"v3": "ABC"} + with patch(f"{PATCH_BASE}.PidProviderXML.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd: + + m_select.return_value = { + "registered": existing, + "matched_items": [{"id": 2}], # força log para poder contar + } + m_cmp.return_value = {} + m_upd.side_effect = exceptions.SkipSavePidProviderXML + PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(self.m_record.call_count, 1)