From fb8a4f04b93a8d3aa34cbeb264dca00ba08d6418 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Fri, 26 Jun 2026 13:53:59 -0300 Subject: [PATCH 01/13] =?UTF-8?q?Refactor:=20Modulariza=20l=C3=B3gica=20de?= =?UTF-8?q?=20compara=C3=A7=C3=A3o=20de=20metadados=20e=20otimiza=20builde?= =?UTF-8?q?rs=20de=20query?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Centralização da lógica de comparação: Criação das funções 'compare', 'compare_lists' e 'compare_items' para isolar a responsabilidade de matching de dados e cálculo de score percentual. * Otimização do 'identifier_queries': Ativação do filtro por 'main_doi' que estava comentado e inclusão da busca otimizada de 'other_pids' utilizando o operador '__in' em um conjunto ('set') coletado dinamicamente. * Segregação de parâmetros de busca: Desacoplamento dos parâmetros de paginação do artigo ('article_location_params') dos dados gerais do fascículo ('issue_params'), permitindo buscas granulares no banco de dados. Por que: Melhora a manutenibilidade ao remover o acoplamento de cálculos estatísticos dentro da classe do modelo e otimiza a performance das consultas iniciais por identificadores únicos no banco. --- pid_provider/query_params.py | 87 +++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 12 deletions(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index bfb6901bd..8f4dabfa6 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -3,10 +3,55 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ -from core.utils.profiling_tools import profile_function +from core.utils.similarity import how_similar from pid_provider import exceptions +def compare(registered_items, input_data): + """ + """ + total_score = 0 + items = [] + for label, registered_item in registered_items.items(): + result = compare_items(label, registered_item, input_data[label]) + items.append(result) + total_score += result["score"] + return { + "items": items, + "total_score": total_score, + "percentual_score": total_score * 100 / len(items) + } + + +def compare_lists(registered, xml_adapter_titles): + if xml_adapter_titles == registered: + return 1 + if not xml_adapter_titles: + return 0 + if not registered: + return 0 + words1 = set() + for item in xml_adapter_titles: + words1.update(item.split()) + words2 = set() + for item in registered: + words2.update(item.split()) + return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + + +def compare_items(label, registered, input_data): + if isinstance(registered, list): + score = compare_lists(registered, input_data) + elif (input_data or None) == (registered or None): + score = 1 + else: + score = 0 + response = {"label": label, "score": score} + if score != 1: + response["registered"] = registered + return response + + def get_score(registered, xml_data, min_value, max_value): if registered == xml_data: if registered: @@ -173,17 +218,21 @@ def identifier_queries(self): """ q = Q() + other_pids = set() # PID v3 - máxima prioridade if self.v3: q |= Q(v3=self.v3) + other_pids.add(self.v3) # PID v2 if self.v2: q |= Q(v2=self.v2) + other_pids.add(self.v2) # AOP PID if self.aop_pid: q |= Q(v2=self.aop_pid) | Q(aop_pid=self.aop_pid) + other_pids.add(self.aop_pid) # Package name pkg_names = set() @@ -196,10 +245,11 @@ def identifier_queries(self): if pkg_names: q |= Q(pkg_name__in=pkg_names) - # # DOI principal - # if self.main_doi: - # q |= Q(main_doi=self.main_doi) + if self.main_doi: + q |= Q(main_doi=self.main_doi) + if other_pids: + q |= Q(other_pid__pid_in_xml=other_pids) return q @cached_property @@ -245,23 +295,36 @@ def issue_params(self): Returns ------- dict - Dicionário com elocation_id, fpage, fpage_seq, lpage, - pub_year, volume, number e suppl + Dicionário com pub_year, volume, number e suppl + """ + return { + "pub_year": self.pub_year, + "volume": self.volume, + "number": self.number, + "suppl": self.suppl, + } + + @cached_property + def article_location_params(self): + """ + Constrói dicionário com metadados do fascículo e paginação do artigo. + + Retorna todos os campos sem verificar presença, permitindo + que o ORM do Django filtre automaticamente valores None. + + Returns + ------- + dict + Dicionário com elocation_id, fpage, fpage_seq, lpage, v2__endswith """ data = { "elocation_id": self.elocation_id, "fpage": self.fpage, "fpage_seq": self.fpage_seq, "lpage": self.lpage, - "pub_year": self.pub_year, - "volume": self.volume, - "number": self.number, - "suppl": self.suppl, } if self.order: data["v2__endswith"] = self.order - elif not self.elocation_id and not self.fpage and self.main_doi: - data["main_doi__iexact"] = self.main_doi return data @cached_property From 57ddbeab19bf4e3f07eeb30509d0b88e917b13c9 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Fri, 26 Jun 2026 13:53:59 -0300 Subject: [PATCH 02/13] =?UTF-8?q?Refactor:=20Otimiza=20estrat=C3=A9gia=20d?= =?UTF-8?q?e=20busca=20de=20registros=20e=20introduz=20cache=20de=20propri?= =?UTF-8?q?edade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Otimização de Performance com Cache: Aplicação do decorador '@cached_property' na propriedade 'xml_with_pre' para evitar leituras redundantes e reprocessamento do XML. * Nova Estratégia de Curto-Circuito (Short-Circuit Queries): Reformulação do método 'get_records' para realizar buscas em cascata (IDs diretos -> Journal + Issue + Article -> Journal + Article), retornando os resultados imediatamente ao encontrar um match ideal, mitigando queries pesadas desnecessárias. * Simplificação e Limpeza do Modelo: Remoção dos métodos depreciados 'article_pid_suffix_source', 'get_article_pid_suffix', 'match' e 'title_similarity' (agora delegados ao 'query_params'). * Ajuste no Fluxo de Exceções: Substituição do método 'get_record' por um mapeamento direto através do 'best_matches', tratando de forma transparente erros de 'MultipleObjectsReturned' e unificando as respostas de erro para o client. Por que: Reduz drasticamente o consumo de CPU e acessos ao banco de dados através do cache de propriedades e impede a execução de queries complexas de fallback quando uma correspondência exata por ID já foi estabelecida. --- pid_provider/models.py | 221 ++++++++++++++++++----------------------- 1 file changed, 97 insertions(+), 124 deletions(-) diff --git a/pid_provider/models.py b/pid_provider/models.py index c46016e61..3ddd12807 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -29,14 +29,13 @@ profile_property, profile_staticmethod, ) -from core.utils.similarity import how_similar from pid_provider import choices, exceptions from pid_provider.query_params import ( - get_score, + compare, zero_to_none, QueryBuilderPidProviderXML, ) -from tracker.models import BaseEvent, EventSaveError, UnexpectedEvent +from tracker.models import UnexpectedEvent try: from django_prometheus.models import ExportModelOperationsMixin @@ -518,19 +517,6 @@ class Meta: def __str__(self): return f"{self.pkg_name} {self.v3}" - @property - def article_pid_suffix_source(self): - try: - return self.xml_with_pre.get_article_pid_suffix_source() - except AttributeError: - return self.elocation_id or self.fpage or self.xml_with_pre.order - - def get_article_pid_suffix(self): - data = self.article_pid_suffix_source - if not data: - data = self.pkg_name.split("-")[-1] - return string_to_5_digits(data) - @property def collection_list(self): return "|".join(c.acron3 for c in self.collections.all()) @@ -609,7 +595,7 @@ def get_xml_with_pre(cls, v3): except Exception: return None - @property + @cached_property @profile_property def xml_with_pre(self): try: @@ -703,13 +689,16 @@ def register( # consulta se documento já está registrado try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + get_records_result = cls.get_records(xml_adapter) + registered = get_records_result.get("registered") + if not registered and get_records_result.get("failed"): + raise exceptions.UnmatchedPidProviderXMLError except cls.DoesNotExist as exc: registered = None except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - response["records"] = [item.data for item in records] - raise exceptions.QueryDocumentMultipleObjectsReturnedError(exc) + response["get_records_result"] = get_records_result + response.update({"error_msg": str(exc), "error_type": str(type(exc))}) + return response except ( exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError ) as exc: @@ -939,21 +928,33 @@ def is_equal_to(self, xml_with_pre): @profile_classmethod def get_records(cls, xml_adapter): qbuilder = QueryBuilderPidProviderXML(xml_adapter) + + # correspondencia direta com ids q_ids = qbuilder.identifier_queries + qs = cls.objects.filter(q_ids).distinct() + if qs.exists(): + best_matches_results = cls.best_matches(qs, xml_adapter) + if best_matches_results.get("registered"): + return best_matches_results + + q_journal = qbuilder.issn_query - q_issue = Q(**qbuilder.issue_params) - return cls.objects.filter(q_ids | (q_journal & q_issue)).distinct() - - @classmethod - @profile_classmethod - def get_record(cls, xml_adapter, records): - results = records - if not results.exists(): - raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - raise cls.DoesNotExist - return cls.objects.get(id=sorted(matched)[-1][-1]) + selected_journal = cls.objects.filter(q_journal) + + # caminho com dados de journal, issue e do article + qs = selected_journal.filter(Q(**qbuilder.issue_params) & qbuilder.article_data_query).distinct() + if qs.exists(): + best_matches_results = cls.best_matches(qs, xml_adapter) + if best_matches_results.get("registered"): + return best_matches_results + + # caminho com dados de journal e do article + qs = selected_journal.filter(qbuilder.article_data_query).distinct() + if qs.exists(): + best_matches_results = cls.best_matches(qs, xml_adapter) + if best_matches_results.get("registered"): + return best_matches_results + raise cls.DoesNotExist @classmethod @profile_classmethod @@ -963,105 +964,76 @@ def get_record_by_pid_v3(cls, xml_adapter): raise ValueError("get_record_by_pid_v3: XML has not pid v3") xml_pid_v3 = xml_adapter.v3 results = ( - cls.objects.filter(Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3)) + cls.objects.filter(Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3)).distinct() ) if not results.exists(): raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.get_record_by_pid_v3", - exception=PidProviderXMLPidV3ConflictError, - detail={"xml_adapter": xml_adapter.data, "results": [i.data for i in results]}, - ) - raise PidProviderXMLPidV3ConflictError( - _("No matching record found for the provided XML data.") - ) - return cls.objects.get(id=sorted(matched)[-1][-1]) - - @profile_method - def match(self, xml_adapter): - """ - """ - labels = [] - score = self.title_similarity(xml_adapter) * 100 - if score > 50: - labels.append("title") - if score_item := get_score(self.z_surnames, xml_adapter.z_surnames, 10, 100): - labels.append("z_surnames") - score += score_item - if score_item := get_score(self.z_collab, xml_adapter.z_collab, 10, 100): - labels.append("z_collab") - score += score_item - if score_item := get_score(self.z_links, xml_adapter.z_links, 10, 100): - labels.append("z_links") - score += score_item - if score_item := get_score(self.z_partial_body, xml_adapter.z_partial_body, 10, 100): - labels.append("z_partial_body") - score += score_item - return {"score": score, "labels": labels} - - def title_similarity(self, xml_adapter): + best_matches_results = cls.best_matches(results, xml_adapter) try: - registered = self.xml_with_pre.article_titles_texts - except Exception: - registered = [] - xml_adapter_titles = xml_adapter.xml_with_pre.article_titles_texts - if xml_adapter_titles == registered: - return 1 - if not xml_adapter_titles: - return 0 - if not registered: - return 0 - words1 = set() - for item in xml_adapter_titles: - words1.update(item.split()) - words2 = set() - for item in registered: - words2.update(item.split()) - return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + return best_matches_results["registered"] + except KeyError: + if best_matches_results["failed"]: + raise PidProviderXMLPidV3ConflictError( + _("No matching record found for the provided XML data.") + ) + raise cls.DoesNotExist + + @property + def data_to_compare(self): + return { + "title": self.xml_with_pre.article_titles_texts, + "z_surnames": self.z_surnames, + "z_collab": self.z_collab, + "z_links": self.z_links, + "z_partial_body": self.z_partial_body, + } @classmethod def best_matches(cls, results, xml_adapter): - data = [] - matched = [] + input_data = { + "title": xml_adapter.xml_with_pre.article_titles_texts, + "z_surnames": xml_adapter.z_surnames, + "z_collab": xml_adapter.z_collab, + "z_links": xml_adapter.z_links, + "z_partial_body": xml_adapter.z_partial_body, + } + detail = { + "total_results": results.count(), + "input_data": input_data, + } + responses = {} + found = [] + items = {} for item in results.iterator(): - response = item.match(xml_adapter) - score = response["score"] - - if xml_adapter.v2: - if item.v2 == xml_adapter.v2: - score += 100 - elif xml_adapter.order and item.v2 and item.v2.endswith(xml_adapter.order): - score += 100 - if item.v3 == xml_adapter.v3: - score += 100 - if item.pkg_name == xml_adapter.pkg_name: - score += 100 - if item.main_doi == xml_adapter.main_doi: - score += 100 - - _data = response - _data.update(item.data) - data.append(_data) - - if score > 50: - matched.append((score, item.updated.isoformat(), item.id)) - - if results.count() > 1 or not matched: - detail = { - "xml_adapter_data": xml_adapter.data, - "data": data, - "matched": matched, - } + item_data = item.data_to_compare + response = compare(item_data, input_data) + responses[item.id] = response + items[item.id] = item + found.append((response["percentual_score"], item.updated.isoformat(), item.id)) + + found = sorted(found, reverse=True) + + ok = [] + failed = [] + for item in found: + percentual_score, updated, item_id = item + if percentual_score > 0.5: + ok.append(responses[item_id]) + else: + failed.append(responses[item_id]) + + detail["ok"] = ok + detail["failed"] = failed + if ok: + detail["registered"] = items.get(ok[0]["id"]) + elif failed: UnexpectedEvent.create( item=xml_adapter.sps_pkg_name, action="PidProviderXML.best_matches", - exception=cls.MultipleObjectsReturned, + exception=exceptions.UnmatchedPidProviderXMLError, detail=detail, ) - return matched + return detail @profile_method def _add_data(self, xml_adapter, registered_in_core): @@ -1232,8 +1204,10 @@ def is_registered( response["xml_adapter_data"] = xml_adapter_data try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + get_records_result = cls.get_records(xml_adapter) + registered = get_records_result.get("registered") + if not registered and get_records_result.get("failed"): + raise exceptions.UnmatchedPidProviderXMLError except cls.DoesNotExist as exc: response.update( {"filename": xml_with_pre.filename, "registered": False} @@ -1241,13 +1215,12 @@ def is_registered( return response except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: exc_type, exc_value, exc_traceback = sys.exc_info() - response["records"] = [item.data for item in records] UnexpectedEvent.create( item=xml_with_pre.sps_pkg_name, action="PidProviderXML.is_registered", exception=exc, exc_traceback=exc_traceback, - detail=response, + detail=get_records_result, ) response.update({"error_msg": str(exc), "error_type": str(type(exc))}) return response From c43373fdecae71ac03993d5a055de47c858a33a9 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Fri, 26 Jun 2026 23:06:49 -0300 Subject: [PATCH 03/13] Refatora montagem de queries em QueryBuilderPidProviderXML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Corrige lookup de 'other_pids' para utilizar o operador '__in', evitando erros de query quando múltiplos PIDs são fornecidos. * Simplifica e otimiza a lógica do método 'article_data_query': remove blocos condicionais redundantes e limpa o fluxo de checagem dos campos de texto (surnames, collab, links, partial_body). * Melhora a legibilidade do código reduzindo aninhamentos desnecessários na construção do objeto Q. --- pid_provider/query_params.py | 53 +++++++++++++++++------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index 8f4dabfa6..cfcd0fda6 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -249,7 +249,7 @@ def identifier_queries(self): q |= Q(main_doi=self.main_doi) if other_pids: - q |= Q(other_pid__pid_in_xml=other_pids) + q |= Q(other_pid__pid_in_xml__in=other_pids) return q @cached_property @@ -342,33 +342,30 @@ def article_data_query(self): ou None se nenhum dado textual estiver disponível """ # Verifica se há algum dado textual disponível - if not any([ - self.z_surnames, - self.z_collab, - self.z_links, - self.z_partial_body, - ]): - return Q( - z_surnames=self.z_surnames, - z_collab=self.z_collab, - z_links=self.z_links, - z_partial_body=self.z_partial_body, - ) - - q = Q() - - # Adiciona query para sobrenomes se disponível - if self.z_surnames: - q |= Q(z_surnames=self.z_surnames) - - # Adiciona queries para outros campos textuais - if self.z_collab: - q |= Q(z_collab=self.z_collab) + if self.z_surnames or self.z_partial_body or self.z_collab or self.z_links: + q = Q() - if self.z_links: - q |= Q(z_links=self.z_links) + # Adiciona query para sobrenomes se disponível + if self.z_surnames: + q |= Q(z_surnames=self.z_surnames) + + # Adiciona queries para outros campos textuais + if self.z_collab: + q |= Q(z_collab=self.z_collab) + + if self.z_links: + q |= Q(z_links=self.z_links) + + if self.z_partial_body: + q |= Q(z_partial_body=self.z_partial_body) + + return q - if self.z_partial_body: - q |= Q(z_partial_body=self.z_partial_body) + return Q( + z_surnames=self.z_surnames, + z_collab=self.z_collab, + z_links=self.z_links, + z_partial_body=self.z_partial_body, + ) - return q \ No newline at end of file + \ No newline at end of file From f6681c087c034298b9174177f38edeab74eb415c Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Fri, 26 Jun 2026 23:06:50 -0300 Subject: [PATCH 04/13] Implementa modelo de auditoria e otimiza o ciclo de registro do PidProviderXML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Desempenho e Memória: Modifica 'get_records' e 'best_matches' para materializar QuerySets em listas e utilizar 'select_related("current_version")', eliminando consultas redundantes ao banco de dados durante loops. * Desempenho e Redundância: Substitui iterações individuais por 'bulk_update' com batch_size em 'mark_items_as_invalid'. Otimiza '_add_other_pid' para salvar apenas o campo modificado utilizando 'update_fields=["other_pid_count"]'. * Desempenho e Cache: Introduz o campo 'readable_data' (JSON) no 'PidProviderXML' para armazenar metadados cacheados (sobrenomes, títulos, links), evitando a leitura custosa de arquivos XML em disco no método 'data_to_compare'. * Robustez e Arquitetura: Simplifica a busca por '.latest()', removendo condicionais obsoletas de tratamento de exceção. Refatora o método 'add_collections' para inferir dinamicamente o path do campo via 'FieldError', isolando regras legadas de prefixo. * Auditoria: Cria o modelo 'PidProviderXMLRegistration' e integra-o via bloco 'finally' no método 'register', garantindo que todo histórico de importação (sucessos, pulados, conflitos e falhas) seja persistido de forma resiliente. --- pid_provider/models.py | 494 ++++++++++++++++++++++++++--------------- 1 file changed, 313 insertions(+), 181 deletions(-) diff --git a/pid_provider/models.py b/pid_provider/models.py index 3ddd12807..c2c41bc87 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -9,6 +9,7 @@ from zlib import crc32 from django.core.files.base import ContentFile +from django.core.exceptions import FieldError from django.db import IntegrityError, models from django.db.models import Q, Count from django.utils.translation import gettext_lazy as _ @@ -37,6 +38,9 @@ ) from tracker.models import UnexpectedEvent + +PARTIAL_BODY_MAX = 300 + try: from django_prometheus.models import ExportModelOperationsMixin @@ -185,12 +189,10 @@ def get(cls, pid_provider_xml, finger_print): raise XMLVersionGetError( "XMLVersion.get requires pid_provider_xml and xml_with_pre parameters" ) - found = cls.objects.filter( + # .latest() já levanta DoesNotExist se vazio + return cls.objects.filter( pid_provider_xml=pid_provider_xml, finger_print=finger_print ).latest("created") - if found: - return found - raise cls.DoesNotExist(f"{pid_provider_xml} {finger_print}") @classmethod @profile_classmethod @@ -406,6 +408,13 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): other_pid_count = models.PositiveIntegerField(default=0) registered_in_core = models.BooleanField(default=False) collections = models.ManyToManyField(Collection, blank=True) + + # dados legíveis para facilitar a análise + readable_data = models.JSONField( + _("Readable data"), null=True, blank=True + ) + + base_form_class = CoreAdminModelForm @@ -433,6 +442,7 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): FieldPanel("z_collab"), FieldPanel("z_links"), FieldPanel("z_partial_body"), + FieldPanel("readable_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), ] edit_handler = TabbedInterface( @@ -616,6 +626,37 @@ def is_aop(self): return False return True + @staticmethod + def build_readable_data(xml_with_pre): + try: + persons = xml_with_pre.authors.get("person") or [] + surnames = [p.get("surname") for p in persons if p.get("surname")] + except Exception: + surnames = [] + partial = (xml_with_pre.partial_body or "")[: PidProviderXML.PARTIAL_BODY_MAX] + return { + "surnames": surnames, + "collab": xml_with_pre.collab, + "links": xml_with_pre.links, + "article_titles": xml_with_pre.article_titles_texts, + "partial_body": partial or None, + } + + @property + def data_to_compare(self): + readable = self.readable_data or {} + titles = readable.get("article_titles") + if titles is None: + # legado: sem readable_data -> recai no XML (caro, lê arquivo) + titles = self.xml_with_pre.article_titles_texts + return { + "title": titles, + "z_surnames": self.z_surnames, + "z_collab": self.z_collab, + "z_links": self.z_links, + "z_partial_body": self.z_partial_body, + } + @classmethod @profile_classmethod def register( @@ -631,105 +672,78 @@ def register( registered_in_core=None, auto_solve_pid_conflict=True, ): - """ - Registra documento XML no sistema de PIDs, retornando PIDs v3, v2 e aop_pid. - - Parameters - ---------- - xml_with_pre : XMLWithPre - Dados XML preprocessados - filename : str - Nome do arquivo XML - user : User - Usuário responsável pelo registro - origin_date : datetime, optional - Data de origem do documento - force_update : bool, optional - Força atualização mesmo sem alterações - is_published : bool, default False - Status de publicação - available_since : datetime, optional - Data de disponibilização - origin : str, optional - Origem do documento - registered_in_core : bool, optional - Se já registrado no sistema core - auto_solve_pid_conflict : bool, default False - Resolve conflitos de PID automaticamente + response = {} + readable_input = None + pkg_name = filename + error_type = None + event_status = None + get_records_result = None + registered = None - Returns - ------- - dict - Sucesso: {"v3", "v2", "aop_pid", "xml_uri", "article", "created", - "updated", "xml_changed", "record_status"} - Erro: {"error_type", "error_message", "id", "filename"} - - Raises - ------ - QueryDocumentMultipleObjectsReturnedError - Múltiplos documentos encontrados - RequiredPublicationYearErrorToGetPidProviderXMLError - Ano de publicação obrigatório ausente - RequiredISSNErrorToGetPidProviderXMLError - ISSN obrigatório ausente - NotEnoughParametersToGetPidProviderXMLError - Parâmetros insuficientes para identificar documento - """ try: - input_data = None - xml_adapter_data = None - - response = {} response["input_data"] = xml_with_pre.data response["input_data"].update({"origin": origin}) - # adaptador do xml with pre xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) response["xml_adapter_data"] = xml_adapter.data + # dados legíveis do XML entrando (mesmo formato do readable_data) + readable_input = cls.build_readable_data(xml_with_pre) + pkg_name = xml_adapter.sps_pkg_name + # consulta se documento já está registrado try: + get_records_result = cls.get_records(xml_adapter) registered = get_records_result.get("registered") if not registered and get_records_result.get("failed"): raise exceptions.UnmatchedPidProviderXMLError - except cls.DoesNotExist as exc: - registered = None - except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - response["get_records_result"] = get_records_result - response.update({"error_msg": str(exc), "error_type": str(type(exc))}) - return response + except cls.DoesNotExist: + pass except ( - exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + cls.MultipleObjectsReturned, + exceptions.UnmatchedPidProviderXMLError, ) as exc: + event_status = "unmatched" raise exc - except exceptions.RequiredISSNErrorToGetPidProviderXMLError as exc: + except ( + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError, + exceptions.RequiredISSNErrorToGetPidProviderXMLError, + exceptions.NotEnoughParametersToGetPidProviderXMLError, + ) as exc: + event_status = "bad_request" raise exc - except exceptions.NotEnoughParametersToGetPidProviderXMLError as exc: + + # valida/condiciona PIDs do XML (pode levantar conflito) + try: + response["xml_changed"] = cls.complete_missing_xml_pids( + xml_adapter, registered, auto_solve_pid_conflict + ) + except PidProviderXMLPidV3ConflictError as exc: + event_status = "conflict" raise exc - # valida os PIDs do XML - # - não podem ter conflito com outros registros - # - identifica mudança - response["xml_changed"] = cls.complete_missing_xml_pids( - xml_adapter, registered, auto_solve_pid_conflict - ) + # analisa se continua o registro (skip) + try: + updated_data = PidProviderXML.is_updated( + xml_with_pre, + registered, + force_update, + origin_date, + registered_in_core, + ) + except exceptions.ForbiddenPidProviderXMLRegistrationError as exc: + event_status = "forbidden" + raise exc - # analisa se continua o registro - updated_data = cls.is_updated( - xml_with_pre, - registered, - force_update, - origin_date, - registered_in_core, - ) if updated_data: response["skip_update"] = True response.update(updated_data) - return response + event_status = "skipped" + raise Exception(f"{pkg_name} is already up to date") # cria ou atualiza registro - registered = cls._save( + registered, event_status = cls._save( registered, xml_adapter, user, @@ -737,22 +751,25 @@ def register( available_since, registered_in_core, ) - - # data to return response.update(registered.data) - return response - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.register", - exception=e, - exc_traceback=exc_traceback, - detail=response, + except Exception as exc: + if event_status != "skipped": + exc_type, exc_value, exc_traceback = sys.exc_info() + error_type = str(type(exc)) + response.update({"error_msg": str(exc), "error_type": error_type}) + + finally: + PidProviderXMLRegistration.record( + user=user, + pid_provider_xml=registered, + pkg_name=pkg_name, + event_status=event_status, + input_data=readable_input, + best_matches=get_records_result, + error_type=error_type, ) - response.update({"error_msg": str(e), "error_type": str(type(e))}) - return response + return response @classmethod @profile_classmethod @@ -824,55 +841,54 @@ def _save( registered_in_core=None, ): if registered: - # obtém os dados de substituição para registrar em other_pid + event_status = "updated" registered_changed = registered.check_registered_pids_changed( xml_adapter.xml_with_pre ) registered.updated_by = user else: + event_status = "created" registered = cls() registered.creator = user registered_changed = None - + registered.proc_status = choices.PPXML_STATUS_TODO registered._add_dates(xml_adapter, origin_date, available_since) registered._add_data(xml_adapter, registered_in_core) registered._add_journal(xml_adapter) registered._add_issue(xml_adapter) - + registered.save() - + if registered_changed: registered._add_other_pid(registered_changed, user) registered._add_current_version(xml_adapter.xml_with_pre, user) + + registered.add_collections(xml_adapter) + return registered, event_status + + def add_collections(self, xml_adapter): q = Q() - if COLLECTION_PREFIX == "scielojournal": - if xml_adapter.journal_issn_print: - q |= Q( - scielojournal__journal__official__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - scielojournal__journal__official__issn_electronic=xml_adapter.journal_issn_electronic - ) - else: - if xml_adapter.journal_issn_print: - q |= Q( - journalproc__journal__official_journal__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - journalproc__journal__official_journal__issn_electronic=xml_adapter.journal_issn_electronic - ) + issn_print = xml_adapter.journal_issn_print + issn_electronic = xml_adapter.journal_issn_electronic + + try: + Collection.objects.filter(scielojournal__isnull=True).exists() + issn_path = "scielojournal__journal__official" + except FieldError: + issn_path = "journalproc__journal__official_journal" + + if issn_print: + q |= Q(**{f"{issn_path}__issn_print": issn_print}) + if issn_electronic: + q |= Q(**{f"{issn_path}__issn_electronic": issn_electronic}) for collection in Collection.objects.filter(q): - registered.collections.add(collection) - return registered + self.collections.add(collection) - @classmethod - @profile_classmethod + @staticmethod def is_updated( - cls, xml_with_pre, registered, force_update, origin_date, registered_in_core + xml_with_pre, registered, force_update, origin_date, registered_in_core ): """ XML é versão AOP, mas @@ -928,32 +944,35 @@ def is_equal_to(self, xml_with_pre): @profile_classmethod def get_records(cls, xml_adapter): qbuilder = QueryBuilderPidProviderXML(xml_adapter) - - # correspondencia direta com ids - q_ids = qbuilder.identifier_queries - qs = cls.objects.filter(q_ids).distinct() - if qs.exists(): - best_matches_results = cls.best_matches(qs, xml_adapter) - if best_matches_results.get("registered"): - return best_matches_results - - - q_journal = qbuilder.issn_query - selected_journal = cls.objects.filter(q_journal) - - # caminho com dados de journal, issue e do article - qs = selected_journal.filter(Q(**qbuilder.issue_params) & qbuilder.article_data_query).distinct() - if qs.exists(): - best_matches_results = cls.best_matches(qs, xml_adapter) - if best_matches_results.get("registered"): - return best_matches_results - - # caminho com dados de journal e do article - qs = selected_journal.filter(qbuilder.article_data_query).distinct() - if qs.exists(): - best_matches_results = cls.best_matches(qs, xml_adapter) - if best_matches_results.get("registered"): - return best_matches_results + + def _try(qs): + candidates = list(qs.select_related("current_version").distinct()) + if not candidates: + return None + result = cls.best_matches(candidates, xml_adapter) + return result if result.get("registered") else None + + # 1) correspondência direta por identificadores + result = _try(cls.objects.filter(qbuilder.identifier_queries)) + if result: + return result + + selected_journal = cls.objects.filter(qbuilder.issn_query) + + # 2) journal + issue + dados do artigo + result = _try( + selected_journal.filter( + Q(**qbuilder.issue_params) & qbuilder.article_data_query + ) + ) + if result: + return result + + # 3) journal + dados do artigo + result = _try(selected_journal.filter(qbuilder.article_data_query)) + if result: + return result + raise cls.DoesNotExist @classmethod @@ -963,10 +982,14 @@ def get_record_by_pid_v3(cls, xml_adapter): if not xml_adapter.v3: raise ValueError("get_record_by_pid_v3: XML has not pid v3") xml_pid_v3 = xml_adapter.v3 - results = ( - cls.objects.filter(Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3)).distinct() + results = list( + cls.objects.filter( + Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3) + ) + .select_related("current_version") + .distinct() ) - if not results.exists(): + if not results: raise cls.DoesNotExist best_matches_results = cls.best_matches(results, xml_adapter) try: @@ -977,19 +1000,10 @@ def get_record_by_pid_v3(cls, xml_adapter): _("No matching record found for the provided XML data.") ) raise cls.DoesNotExist - - @property - def data_to_compare(self): - return { - "title": self.xml_with_pre.article_titles_texts, - "z_surnames": self.z_surnames, - "z_collab": self.z_collab, - "z_links": self.z_links, - "z_partial_body": self.z_partial_body, - } @classmethod def best_matches(cls, results, xml_adapter): + # results agora é uma LISTA materializada (não queryset) input_data = { "title": xml_adapter.xml_with_pre.article_titles_texts, "z_surnames": xml_adapter.z_surnames, @@ -998,41 +1012,38 @@ def best_matches(cls, results, xml_adapter): "z_partial_body": xml_adapter.z_partial_body, } detail = { - "total_results": results.count(), + "total_results": len(results), "input_data": input_data, } responses = {} found = [] items = {} - for item in results.iterator(): + for item in results: item_data = item.data_to_compare response = compare(item_data, input_data) responses[item.id] = response items[item.id] = item - found.append((response["percentual_score"], item.updated.isoformat(), item.id)) - + found.append( + (response["percentual_score"], item.updated.isoformat(), item.id) + ) + found = sorted(found, reverse=True) - + ok = [] failed = [] - for item in found: - percentual_score, updated, item_id = item + for percentual_score, updated, item_id in found: if percentual_score > 0.5: ok.append(responses[item_id]) else: failed.append(responses[item_id]) - + detail["ok"] = ok detail["failed"] = failed if ok: detail["registered"] = items.get(ok[0]["id"]) - elif failed: - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.best_matches", - exception=exceptions.UnmatchedPidProviderXMLError, - detail=detail, - ) + # NOTA: removido o UnexpectedEvent daqui (disciplina item 4). + # O caso "failed sem ok" passa a ser refletido no PidProviderXMLRegistration + # com event_status="unmatched" no register/is_registered. return detail @profile_method @@ -1057,6 +1068,10 @@ def _add_data(self, xml_adapter, registered_in_core): self.z_links = xml_adapter.z_links self.z_partial_body = xml_adapter.z_partial_body + # NOVO: dados legíveis (somente inspeção) + self.readable_data = self.build_readable_data(xml_adapter.xml_with_pre) + + @profile_method def _add_dates(self, xml_adapter, origin_date, available_since): # evita que artigos WIP fique disponíveis antes de estarem públicos @@ -1130,20 +1145,15 @@ def check_registered_pids_changed(self, xml_with_pre): @profile_method def _add_other_pid(self, registered_changed, user): - # registrados passam a ser other pid - # os pids do XML passam a ser os vigentes if not registered_changed: return for change_args in registered_changed: - change_args["pid_in_xml"] = change_args.pop("registered") - change_args["user"] = user change_args["pid_provider_xml"] = self - OtherPid.get_or_create(**change_args) self.other_pid_count = self.other_pid.count() - self.save() + self.save(update_fields=["other_pid_count"]) @classmethod @profile_classmethod @@ -1300,13 +1310,19 @@ def mark_as_done(self): @classmethod @profile_classmethod def mark_items_as_invalid(cls, issns): - for item in cls.objects.filter( + items = cls.objects.filter( Q(issn_print__in=issns) | Q(issn_electronic__in=issns), - ).iterator(): + ) + items_to_update = [] + for item in items.iterator(): try: - invalid = bool(item.xml_with_pre) + valid = bool(item.xml_with_pre) except Exception as e: - invalid = True + valid = False + if not valid: + item.proc_status = choices.PPXML_STATUS_INVALID + items_to_update.append(item) + cls.objects.bulk_update(items_to_update, ["proc_status"], batch_size=100) @classmethod @profile_classmethod @@ -1758,3 +1774,119 @@ def record(cls, user, url, status, document_item, *, exception=None, response=No xmlurl_obj.save_file(xml_with_pre.tostring(), filename=filename) return xmlurl_obj + + +# ----------------------------------------------------------------------------- +# [models.py] MODELO NOVO — PidProviderXMLRegistration +# Auditoria por documento. Grava SEMPRE (created/updated/skipped/forbidden/ +# conflict/unmatched/error). FK nullable (unmatched/error podem não ter PPX). +# ----------------------------------------------------------------------------- +class PidProviderXMLRegistration(CommonControlField): + EVENT_CREATED = "created" + EVENT_UPDATED = "updated" + EVENT_SKIPPED = "skipped" + EVENT_FORBIDDEN = "forbidden" + EVENT_CONFLICT = "conflict" + EVENT_UNMATCHED = "unmatched" + EVENT_ERROR = "error" + EVENT_BAD_REQUEST = "bad_request" + + EVENT_STATUS_CHOICES = ( + (EVENT_CREATED, "created"), + (EVENT_UPDATED, "updated"), + (EVENT_SKIPPED, "skipped"), + (EVENT_FORBIDDEN, "forbidden"), + (EVENT_CONFLICT, "conflict"), + (EVENT_UNMATCHED, "unmatched"), + (EVENT_BAD_REQUEST, "bad_request"), + (EVENT_ERROR, "error"), + ) + + pid_provider_xml = models.ForeignKey( + PidProviderXML, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="registration_events", + ) + pkg_name = models.CharField( + _("Package name"), max_length=100, null=True, blank=True + ) + event_status = models.CharField( + _("Event status"), + max_length=15, + null=True, + blank=True, + choices=EVENT_STATUS_CHOICES, + ) + input_data = models.JSONField(_("Input data (readable)"), null=True, blank=True) + best_matches = models.JSONField(_("Best matches detail"), null=True, blank=True) + error_type = models.CharField( + _("Error type"), max_length=255, null=True, blank=True + ) + + base_form_class = CoreAdminModelForm + + panels = [ + FieldPanel("event_status", read_only=True), + FieldPanel("pkg_name", read_only=True), + AutocompletePanel("pid_provider_xml", read_only=True), + FieldPanel("error_type", read_only=True), + FieldPanel("input_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), + FieldPanel("best_matches", widget=ReadOnlyPrettyJSONWidget(), read_only=True), + ] + + class Meta: + ordering = ["-created"] + verbose_name = _("PidProviderXML Registration") + verbose_name_plural = _("PidProviderXML Registrations") + indexes = [ + models.Index(fields=["pkg_name"]), + models.Index(fields=["event_status"]), + models.Index(fields=["-created"]), + models.Index(fields=["pid_provider_xml"]), + ] + + def __str__(self): + return f"{self.pkg_name} {self.event_status} {self.created}" + + @staticmethod + def _serialize_best_matches(best_matches): + """ + O detail do best_matches contém o objeto PidProviderXML em + detail['registered']. Para gravar em JSON, troca pelo v3/id. + """ + if not best_matches: + return None + data = dict(best_matches) + registered = data.get("registered") + if registered is not None and hasattr(registered, "v3"): + data["registered"] = {"id": registered.id, "v3": registered.v3} + return data + + @classmethod + def record( + cls, + user, + event_status, + pid_provider_xml=None, + pkg_name=None, + input_data=None, + best_matches=None, + error_type=None, + ): + try: + obj = cls() + obj.creator = user + obj.pid_provider_xml = pid_provider_xml + obj.pkg_name = pkg_name or (pid_provider_xml and pid_provider_xml.pkg_name) + obj.event_status = event_status + obj.input_data = input_data + obj.best_matches = cls._serialize_best_matches(best_matches) + obj.error_type = error_type + obj.save() + return obj + except Exception as e: + # registro de auditoria nunca deve derrubar o register + logging.exception(f"Unable to record PidProviderXMLRegistration: {e}") + return None From 879d283879ca90ee69b85fdd6471e23006d0ddae Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Fri, 26 Jun 2026 23:06:50 -0300 Subject: [PATCH 05/13] Registra o novo modelo PidProviderXMLRegistration no Wagtail Admin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adiciona o 'PidProviderXMLRegistrationViewSet' ao ecossistema do Wagtail. * Configura colunas de exibição (list_display), filtros laterais (list_filter) e campos de busca por nome do pacote, tipo de erro e v3 do PID. * Otimiza o desempenho da listagem administrativa forçando o uso de 'select_related("pid_provider_xml")' no queryset para mitigar problemas de query N+1 na UI. --- pid_provider/wagtail_hooks.py | 36 +++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/pid_provider/wagtail_hooks.py b/pid_provider/wagtail_hooks.py index 550d167eb..d90bb6b3b 100644 --- a/pid_provider/wagtail_hooks.py +++ b/pid_provider/wagtail_hooks.py @@ -6,8 +6,7 @@ from config.menu import get_menu_order from core.views import CommonControlFieldViewSet -from pid_provider import choices -from pid_provider.models import XMLURL, XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML +from pid_provider.models import XMLURL, XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML, PidProviderXMLRegistration class PidProviderXMLViewSet(CommonControlFieldViewSet): @@ -204,6 +203,39 @@ class XMLURLViewSet(CommonControlFieldViewSet): ) +class PidProviderXMLRegistrationViewSet(CommonControlFieldViewSet): + model = PidProviderXMLRegistration + icon = "doc-empty-inverse" + menu_label = _("PID Registration Events") + menu_name = "pid_provider_xml_registration" + add_to_admin_menu = True + + # ordenação na listagem + ordering = ["-created"] + + # colunas da listagem + list_display = ( + "pkg_name", + "event_status", + "pid_provider_xml", + "error_type", + "created", + ) + + # filtros laterais + list_filter = ("event_status", "error_type", "created") + + # busca + search_fields = ("pkg_name", "error_type", "pid_provider_xml__v3") + + # paginação (tabela cresce em volume) + list_per_page = 50 + + # ---- READ ONLY: auditoria não é editada pela UI ---- + def get_queryset(self, request): + return super().get_queryset(request).select_related("pid_provider_xml") + + # Grupo de ViewSets class PidProviderViewSetGroup(SnippetViewSetGroup): menu_label = _("Pid Provider") From 59db2a929e7cf443891b50d851d81348e0cc7fcc Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 17:13:19 -0300 Subject: [PATCH 06/13] refactor(query_params): previne excecoes ao acessar chaves ausentes no input_data - Substituida a busca direta por chave por 'input_data.get(label)' dentro da funcao de comparacao. - Otimizacao/Seguranca: Evita falhas de execucao (KeyError) caso o dicionario de entrada nao contenha todas as chaves esperadas, tornando o codigo mais resiliente sem a necessidade de blocos try/except adicionais. --- pid_provider/query_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index cfcd0fda6..f8e71ec4a 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -13,7 +13,7 @@ def compare(registered_items, input_data): total_score = 0 items = [] for label, registered_item in registered_items.items(): - result = compare_items(label, registered_item, input_data[label]) + result = compare_items(label, registered_item, input_data.get(label)) items.append(result) total_score += result["score"] return { From 8e2d068dd2115a6ecd49e95f1acb887b3cea202d Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 17:13:19 -0300 Subject: [PATCH 07/13] fix(models): ajusta nota de corte de score, melhora tratamento de erros e adiciona telemetria - Removida a redundancia de escopo 'PidProviderXML.PARTIAL_BODY_MAX' para 'PARTIAL_BODY_MAX' na propriedade partial, otimizando o acesso ao atributo. - Adicionados logs de depuracao (print) para rastreamento do fluxo de dados das variaveis 'response' e 'readable_input'. - Refatorado o bloco except para capturar o traceback via 'sys.exc_info()', registrar a excecao nativamente com 'logging.exception(exc)' e tratar de forma explicita cenarios onde o status do evento e 'skipped' ou 'unmatched'. - Atualizada a regra de negocio de correspondencia (score minimo) de 0.5 para 0.6 para aumentar a precisao dos resultados homologados. - Injetado o ID do item diretamente no dicionario de resposta para facilitar o mapeamento reverso. - Removido o bloco de codigo obsoleto que disparava o evento 'UnexpectedEvent', centralizando a regra de negocio no fluxo do 'PidProviderXMLRegistration' conforme arquitetura planejada. --- pid_provider/models.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pid_provider/models.py b/pid_provider/models.py index c2c41bc87..37fe6662c 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -633,7 +633,7 @@ def build_readable_data(xml_with_pre): surnames = [p.get("surname") for p in persons if p.get("surname")] except Exception: surnames = [] - partial = (xml_with_pre.partial_body or "")[: PidProviderXML.PARTIAL_BODY_MAX] + partial = (xml_with_pre.partial_body or "")[: PARTIAL_BODY_MAX] return { "surnames": surnames, "collab": xml_with_pre.collab, @@ -683,12 +683,15 @@ def register( try: response["input_data"] = xml_with_pre.data response["input_data"].update({"origin": origin}) + print(f"response: {response}") xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) response["xml_adapter_data"] = xml_adapter.data + print(f"response: {response}") # dados legíveis do XML entrando (mesmo formato do readable_data) readable_input = cls.build_readable_data(xml_with_pre) + print(f"readable_input: {readable_input}") pkg_name = xml_adapter.sps_pkg_name # consulta se documento já está registrado @@ -754,8 +757,12 @@ def register( response.update(registered.data) except Exception as exc: - if event_status != "skipped": - exc_type, exc_value, exc_traceback = sys.exc_info() + exc_type, exc_value, exc_traceback = sys.exc_info() + logging.exception(exc) + response["event_status"] = event_status + if event_status == "skipped": + response["message"] = str(exc) + else: error_type = str(type(exc)) response.update({"error_msg": str(exc), "error_type": error_type}) @@ -1021,6 +1028,7 @@ def best_matches(cls, results, xml_adapter): for item in results: item_data = item.data_to_compare response = compare(item_data, input_data) + response["id"] = item.id responses[item.id] = response items[item.id] = item found.append( @@ -1032,7 +1040,7 @@ def best_matches(cls, results, xml_adapter): ok = [] failed = [] for percentual_score, updated, item_id in found: - if percentual_score > 0.5: + if percentual_score > 0.6: ok.append(responses[item_id]) else: failed.append(responses[item_id]) @@ -1041,9 +1049,6 @@ def best_matches(cls, results, xml_adapter): detail["failed"] = failed if ok: detail["registered"] = items.get(ok[0]["id"]) - # NOTA: removido o UnexpectedEvent daqui (disciplina item 4). - # O caso "failed sem ok" passa a ser refletido no PidProviderXMLRegistration - # com event_status="unmatched" no register/is_registered. return detail @profile_method From b40fe984c6420514d06df44f92b1de6b09b805d0 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 17:13:19 -0300 Subject: [PATCH 08/13] migration(pid_provider): cria estrutura para armazenamento de dados legiveis do XML - Gerada a migracao 0014 para refletir as alteracoes de campos e metadados necessarios a persistencia do 'readable_data' no modelo 'PidProviderXML'. - Garante a integridade e sincronismo entre a definicao da classe de negocio e o esquema do banco de dados relacional. --- ...4_pidproviderxml_readable_data_and_more.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py diff --git a/pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py b/pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py new file mode 100644 index 000000000..8e67f8451 --- /dev/null +++ b/pid_provider/migrations/0014_pidproviderxml_readable_data_and_more.py @@ -0,0 +1,144 @@ +# Generated by Django 5.2.3 on 2026-06-27 16:16 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pid_provider", "0013_alter_xmlurl_status"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="pidproviderxml", + name="readable_data", + field=models.JSONField(blank=True, null=True, verbose_name="Readable data"), + ), + migrations.CreateModel( + name="PidProviderXMLRegistration", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pkg_name", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Package name", + ), + ), + ( + "event_status", + models.CharField( + blank=True, + choices=[ + ("created", "created"), + ("updated", "updated"), + ("skipped", "skipped"), + ("forbidden", "forbidden"), + ("conflict", "conflict"), + ("unmatched", "unmatched"), + ("bad_request", "bad_request"), + ("error", "error"), + ], + max_length=15, + null=True, + verbose_name="Event status", + ), + ), + ( + "input_data", + models.JSONField( + blank=True, null=True, verbose_name="Input data (readable)" + ), + ), + ( + "best_matches", + models.JSONField( + blank=True, null=True, verbose_name="Best matches detail" + ), + ), + ( + "error_type", + models.CharField( + blank=True, max_length=255, null=True, verbose_name="Error type" + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "pid_provider_xml", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="registration_events", + to="pid_provider.pidproviderxml", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "verbose_name": "PidProviderXML Registration", + "verbose_name_plural": "PidProviderXML Registrations", + "ordering": ["-created"], + "indexes": [ + models.Index( + fields=["pkg_name"], name="pid_provide_pkg_nam_2db0b2_idx" + ), + models.Index( + fields=["event_status"], name="pid_provide_event_s_3c9ae7_idx" + ), + models.Index( + fields=["-created"], name="pid_provide_created_94fb08_idx" + ), + models.Index( + fields=["pid_provider_xml"], + name="pid_provide_pid_pro_c9fb0e_idx", + ), + ], + }, + ), + ] From a44d953c5b863b97c4b82d8fa7ec8d1bacacc1c9 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 17:13:19 -0300 Subject: [PATCH 09/13] test(pid_provider): implementa suite de testes automatizados para novos fluxos - Adicionada a estrutura de testes dentro de 'pid_provider/tests/' cobrindo os modulos cruciais do sistema. - Incluidos testes especificos para 'test_best_matches.py', 'test_controller.py', 'test_get_records.py', 'test_models.py' e 'test_register.py'. - Garante a cobertura de codigo, consistencia das novas regras de score (0.6) e previne regressoes no comportamento do motor de busca e registro de PIDs. --- pid_provider/tests/__init__.py | 0 pid_provider/tests/test_best_matches.py | 104 +++ pid_provider/tests/test_controller.py | 87 ++ pid_provider/tests/test_get_records.py | 145 ++++ pid_provider/tests/test_models.py | 1029 +++++++++++++++++++++++ pid_provider/tests/test_register.py | 313 +++++++ 6 files changed, 1678 insertions(+) create mode 100644 pid_provider/tests/__init__.py create mode 100644 pid_provider/tests/test_best_matches.py create mode 100644 pid_provider/tests/test_controller.py create mode 100644 pid_provider/tests/test_get_records.py create mode 100644 pid_provider/tests/test_models.py create mode 100644 pid_provider/tests/test_register.py diff --git a/pid_provider/tests/__init__.py b/pid_provider/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pid_provider/tests/test_best_matches.py b/pid_provider/tests/test_best_matches.py new file mode 100644 index 000000000..b85861955 --- /dev/null +++ b/pid_provider/tests/test_best_matches.py @@ -0,0 +1,104 @@ +from django.test import TestCase +from unittest.mock import MagicMock, patch +from pid_provider.models import PidProviderXML + + +class PidProviderXMLBestMatchesTests(TestCase): + + def setUp(self): + # Mock do xml_adapter e das propriedades exigidas por best_matches + self.xml_adapter_mock = MagicMock() + self.xml_adapter_mock.xml_with_pre.article_titles_texts = "Titulo Original" + self.xml_adapter_mock.z_surnames = "Silva; Santos" + self.xml_adapter_mock.z_collab = "Grupo SpS" + self.xml_adapter_mock.z_links = "http://link1.com" + self.xml_adapter_mock.z_partial_body = "Texto do corpo do artigo..." + + @patch("pid_provider.models.compare") + def test_best_matches_success_with_valid_and_invalid_scores(self, mock_compare): + """Deve ordenar os resultados por score descrescente e separar entre 'ok' (>0.5) e 'failed' (<=0.5).""" + + # Cria mocks para simular instâncias de PidProviderXML do banco de dados + item_bom = MagicMock(spec=PidProviderXML) + item_bom.id = 101 + item_bom.updated.isoformat.return_value = "2026-06-27T12:00:00" + item_bom.data_to_compare = {"title": "Titulo Original", "z_surnames": "Silva; Santos"} + + item_ruim = MagicMock(spec=PidProviderXML) + item_ruim.id = 102 + item_ruim.updated.isoformat.return_value = "2026-06-27T13:00:00" + item_ruim.data_to_compare = {"title": "Outro Titulo Completamente Diferente", "z_surnames": "Alves"} + + # Configura o efeito colateral do mock 'compare' baseado no item recebido + def side_effect_compare(item_data, input_data): + if item_data["title"] == "Titulo Original": + return {"id": 101, "percentual_score": 0.95, "match": True} + return {"id": 102, "percentual_score": 0.20, "match": False} + + mock_compare.side_effect = side_effect_compare + + # Executa o método passando a lista materializada de candidatos + candidates = [item_ruim, item_bom] # Enviados fora de ordem propositalmente + result = PidProviderXML.best_matches(candidates, self.xml_adapter_mock) + + # Validações de estrutura e contagem + self.assertEqual(result["total_results"], 2) + self.assertEqual(len(result["ok"]), 1) + self.assertEqual(len(result["failed"]), 1) + + # Valida a ordenação e separação (o de score 0.95 deve ser o 'registered') + self.assertEqual(result["registered"], item_bom) + self.assertEqual(result["ok"][0]["id"], 101) + self.assertEqual(result["failed"][0]["id"], 102) + + @patch("pid_provider.models.compare") + def test_best_matches_no_candidates_approved(self, mock_compare): + """Quando nenhum candidato atinge score > 0.5, a chave 'registered' não deve existir no retorno.""" + + item_fraco = MagicMock(spec=PidProviderXML) + item_fraco.id = 201 + item_fraco.updated.isoformat.return_value = "2026-06-27T14:00:00" + item_fraco.data_to_compare = {"title": "Quase igual, mas nao o suficiente"} + + # Mock retorna score abaixo da linha de corte (0.5) + mock_compare.return_value = {"id": 201, "percentual_score": 0.48, "match": False} + + result = PidProviderXML.best_matches([item_fraco], self.xml_adapter_mock) + + # Validações + self.assertEqual(result["total_results"], 1) + self.assertEqual(len(result["ok"]), 0) + self.assertEqual(len(result["failed"]), 1) + self.assertNotIn("registered", result) # Não pode haver match oficial + + @patch("pid_provider.models.compare") + def test_best_matches_tie_breaking_by_updated_date(self, mock_compare): + """Em caso de empate no percentual_score, o critério de desempate do sorted() deve usar a data 'updated' descrescente.""" + + # Dois itens com o mesmo score, mas datas de atualização diferentes + item_antigo = MagicMock(spec=PidProviderXML) + item_antigo.id = 301 + item_antigo.updated.isoformat.return_value = "2026-01-01T00:00:00" + item_antigo.data_to_compare = {"title": "Clone"} + + item_recente = MagicMock(spec=PidProviderXML) + item_recente.id = 302 + item_recente.updated.isoformat.return_value = "2026-06-27T00:00:00" # Mais recente + item_recente.data_to_compare = {"title": "Clone"} + + # Mock retorna o mesmo score alto para ambos + def side_effect_compare(item_data, input_data): + items = [ + {"label": "title", "score": 0.9}, + {"label": "title", "score": 0.9}, + ] + return {"items": items, "percentual_score": 0.90, "total_score": 0.90} + + mock_compare.side_effect = side_effect_compare + + # Executa passando o antigo primeiro + result = PidProviderXML.best_matches([item_antigo, item_recente], self.xml_adapter_mock) + + # Como a ordenação usa reverse=True no par (score, data.isoformat(), id), + # o item_recente ("2026-06-27...") deve ficar em primeiro lugar na ordenação. + self.assertEqual(result["registered"], item_recente) \ No newline at end of file diff --git a/pid_provider/tests/test_controller.py b/pid_provider/tests/test_controller.py new file mode 100644 index 000000000..f4235cec6 --- /dev/null +++ b/pid_provider/tests/test_controller.py @@ -0,0 +1,87 @@ +from unittest.mock import ANY, Mock, patch + +from django.contrib.auth import get_user_model +from django.test import TestCase + +from pid_provider.provider import PidProvider +from pid_provider.models import PidProviderXML + +User = get_user_model() + + +# def get_mock_config(): +# config = object() +# config.host = '' +# config.access_key = '' +# config.secret_key = '' +# config.bucket_root = '' +# config.bucket_app_subdir = 'bucket-app-subdir' +# config.secure = '' +# return config + + +class PidProviderTest(TestCase): + @patch("pid_provider.models.XMLSPS.save") + @patch("pid_provider.models.XMLVersion.save") + @patch("pid_provider.models.XMLIssue.save") + @patch("pid_provider.models.XMLJournal.save") + @patch("pid_provider.models.PidProviderXML.save") + @patch( + "pid_provider.models.PidProviderXML._get_unique_v3", + return_value="SJLD63mRxz9nTXtyMj7SLwk", + ) + @patch( + "pid_provider.models.PidProviderXML._get_unique_v2", + return_value="S2236-89062022061645340", + ) + @patch("pid_provider.controller.PidProviderConfig.get_or_create") + def test_provide_pid_for_xml_zip( + self, + mock_pid_provider_config, + mock_get_unique_v2, + mock_get_unique_v3, + mock_pid_provider_xml_save, + mock_xml_journal_save, + mock_xml_issue_save, + mock_xml_version_save, + mock_xmlsps_save, + ): + pid_provider = PidProvider() + result = pid_provider.provide_pid_for_xml_zip( + zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", + user=User.objects.first(), + ) + result = list(result) + self.assertEqual("SJLD63mRxz9nTXtyMj7SLwk", result[0]["v3"]) + self.assertEqual("S2236-89062022061645340", result[0]["v2"]) + self.assertIsNone(result[0]["aop_pid"]) + self.assertIsNotNone(result[0]["created"]) + self.assertIsNone(result[0]["updated"]) + self.assertEqual("2236-8906-hoehnea-49-e1082020.xml", result[0]["filename"]) + self.assertEqual("created", result[0]["record_status"]) + self.assertEqual(True, result[0]["xml_changed"]) + + @patch("pid_provider.models.PidProviderXML._query_document") + @patch("pid_provider.models.PidProviderXML.is_equal_to", return_value=True) + def test_provide_pid_for_xml_with_pre_do_nothing_because_it_is_already_updated( + self, + mock_is_equal, + mock_query_document, + ): + # dubla o registro encontrado + pid_provider_xml = Mock(PidProviderXML) + pid_provider_xml.data = {"v3": ""} + mock_query_document.return_value = pid_provider_xml + + pid_provider_ = PidProvider() + result = pid_provider_.provide_pid_for_xml_zip( + zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", + user=User.objects.first(), + ) + result = list(result) + expected = { + "filename": "2236-8906-hoehnea-49-e1082020.xml", + "v3": "", + "xml_with_pre": ANY, + } + self.assertDictEqual(result[0], expected) diff --git a/pid_provider/tests/test_get_records.py b/pid_provider/tests/test_get_records.py new file mode 100644 index 000000000..7b0f9e981 --- /dev/null +++ b/pid_provider/tests/test_get_records.py @@ -0,0 +1,145 @@ +from unittest.mock import MagicMock, patch + +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.test import TestCase + +from pid_provider.models import PidProviderXML +from pid_provider.exceptions import UnmatchedPidProviderXMLError + + +User = get_user_model() + + +class PidProviderXMLGetRecordsTests(TestCase): + + def setUp(self): + # Cria um usuário para satisfazer a restrição de 'creator_id' + self.user = User.objects.create_user(username="testuser", password="password") + + # Mocks existentes... + self.xml_adapter_mock = MagicMock() + self.xml_adapter_mock.xml_with_pre.article_titles_texts = "Titulo de Teste" + self.xml_adapter_mock.z_surnames = "Silva" + self.xml_adapter_mock.z_collab = None + self.xml_adapter_mock.z_links = None + self.xml_adapter_mock.z_partial_body = "Corpo parcial do artigo" + self.xml_adapter_mock.sps_pkg_name = "test_package" + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_by_identifiers_success(self, mock_best_matches, mock_qbuilder_cls): + """1) Deve retornar o registro quando encontrado por identificadores diretos.""" + # Configura o mock do QueryBuilder + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="12345") + + # Cria um registro no banco que case com a query + record = PidProviderXML.objects.create(creator=self.user, v3="12345", registered_in_core=True) + + # Mock do retorno do best_matches + expected_result = {"registered": record, "total_results": 1} + mock_best_matches.return_value = expected_result + + # Executa o método + result = PidProviderXML.get_records(self.xml_adapter_mock) + + # Validações + self.assertEqual(result, expected_result) + mock_best_matches.assert_called_once_with([record], self.xml_adapter_mock) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_by_journal_and_issue_and_article_success(self, mock_best_matches, mock_qbuilder_cls): + """2) Deve encontrar o registro por Journal + Issue + Dados do Artigo quando os IDs falharem.""" + mock_qbuilder = mock_qbuilder_cls.return_value + # Força falha no passo 1 + mock_qbuilder.identifier_queries = Q(v3="id_inexistente") + + # Configura os passos do fluxo de fallback + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + mock_qbuilder.issue_params = {"pub_year": 2026} + mock_qbuilder.article_data_query = Q(z_surnames="Silva") + + # Cria o registro correspondente no banco + record = PidProviderXML.objects.create(creator=self.user, + issn_print="1234-5678", + z_surnames="Silva", + v3="outro_id" + ) + + expected_result = {"registered": record, "total_results": 1} + mock_best_matches.return_value = expected_result + + result = PidProviderXML.get_records(self.xml_adapter_mock) + + self.assertEqual(result, expected_result) + mock_best_matches.assert_called_once_with([record], self.xml_adapter_mock) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_by_journal_and_article_only_success(self, mock_best_matches, mock_qbuilder_cls): + """3) Deve encontrar o registro por Journal + Dados do Artigo (ignorando Issue) se os passos anteriores falharem.""" + mock_qbuilder = mock_qbuilder_cls.return_value + # Força falha nos passos 1 e 2 + mock_qbuilder.identifier_queries = Q(v3="id_inexistente") + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + mock_qbuilder.issue_params = {"pub_year": 9999} # Ano errado para falhar o passo 2 + mock_qbuilder.article_data_query = Q(z_surnames="Silva") + + # Registro no banco compartilha apenas o ISSN e o dado do artigo (o ano/issue seria diferente) + record = PidProviderXML.objects.create(creator=self.user, + issn_print="1234-5678", + z_surnames="Silva" + ) + + expected_result = {"registered": record, "total_results": 1} + mock_best_matches.return_value = expected_result + + result = PidProviderXML.get_records(self.xml_adapter_mock) + + self.assertEqual(result, expected_result) + mock_best_matches.assert_called_once_with([record], self.xml_adapter_mock) + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_raises_does_not_exist(self, mock_best_matches, mock_qbuilder_cls): + """4) Deve levantar PidProviderXML.DoesNotExist se nenhuma das estratégias encontrar candidatos.""" + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="nao_existe") + mock_qbuilder.issn_query = Q(issn_print="0000-0000") + mock_qbuilder.issue_params = {} + mock_qbuilder.article_data_query = Q(z_surnames="Ninguém") + + with self.assertRaises(PidProviderXML.DoesNotExist): + PidProviderXML.get_records(self.xml_adapter_mock) + + # O best_matches nunca deve ter sido chamado porque nenhuma lista de candidatos foi gerada + mock_best_matches.assert_not_called() + + @patch("pid_provider.models.QueryBuilderPidProviderXML") + @patch.object(PidProviderXML, "best_matches") + def test_get_records_fallback_when_best_matches_returns_no_registered(self, mock_best_matches, mock_qbuilder_cls): + """5) Se o passo 1 achar candidatos mas o best_matches não validar um 'registered', deve prosseguir para o passo seguinte.""" + mock_qbuilder = mock_qbuilder_cls.return_value + mock_qbuilder.identifier_queries = Q(v3="id_com_score_baixo") + mock_qbuilder.issn_query = Q(issn_print="1234-5678") + mock_qbuilder.issue_params = {"pub_year": 2026} + mock_qbuilder.article_data_query = Q(z_surnames="Silva") + + # Cria candidato para o passo 1 e o alvo real para o passo 2 + candidato_ruim = PidProviderXML.objects.create(creator=self.user, v3="id_com_score_baixo") + alvo_correto = PidProviderXML.objects.create(creator=self.user, issn_print="1234-5678", z_surnames="Silva") + + # Configura o mock do best_matches para simular comportamento diferente por chamada + # 1ª chamada (Passo 1): Encontra resultado mas sem a chave 'registered' validada (percentual baixo) + # 2ª chamada (Passo 2): Encontra e valida o 'registered' + mock_best_matches.side_effect = [ + {"registered": None, "failed": [candidato_ruim]}, + {"registered": alvo_correto, "total_results": 1} + ] + + result = PidProviderXML.get_records(self.xml_adapter_mock) + + self.assertEqual(result["registered"], alvo_correto) + self.assertEqual(mock_best_matches.call_count, 2) \ No newline at end of file diff --git a/pid_provider/tests/test_models.py b/pid_provider/tests/test_models.py new file mode 100644 index 000000000..f874ff514 --- /dev/null +++ b/pid_provider/tests/test_models.py @@ -0,0 +1,1029 @@ +import logging +from datetime import datetime +from unittest import mock +from unittest.mock import ANY, MagicMock, Mock, call, patch + +from django.contrib.auth import get_user_model +from django.test import TestCase +from lxml import etree +from packtools.xmlsps.xml_sps_lib import XMLWithPre + +from pid_provider import exceptions, models +from packtools.pid_provider.xml_sps_adapter import PidProviderXMLAdapter + +User = get_user_model() + + +def _get_xml_adapter_from_file(path): + for xml_with_pre in XMLWithPre.create(path=path): + obj = PidProviderXMLAdapter(xml_with_pre) + return obj + + +def _get_xml_with_pre(xml=None): + xml = xml or "
" + return XMLWithPre("", etree.fromstring(xml)) + + +def _get_xml_adapter(xml=None): + xml = xml or "
" + xml_with_pre = XMLWithPre("", etree.fromstring(xml)) + obj = PidProviderXMLAdapter(xml_with_pre) + return obj + + +def _get_xml_adapter_with_issue_data(): + xml_adapter = _get_xml_adapter() + xml_adapter.journal_issn_electronic = "data-issn-e" + xml_adapter.journal_issn_print = "data-issn-p" + xml_adapter.volume = "data-vol" + xml_adapter.number = "data-num" + xml_adapter.suppl = "data-suppl" + xml_adapter.pub_year = "data-year" + xml_adapter.issue = models.XMLIssue.get_or_create( + models.XMLJournal.get_or_create("data-issn-e", "data-issn-p"), + "data-vol", + "data-num", + "data-suppl", + "data-year", + ) + xml_adapter.fpage = "data-fpage" + xml_adapter.fpage_seq = "data-fpage-seq" + xml_adapter.lpage = "data-lpage" + + xml_adapter.article_pub_year = "data-pub-year" + xml_adapter.v3 = "123456789012345678901v3" + xml_adapter.v2 = "123456789012345678901v2" + xml_adapter.aop_pid = "12345678901234567890aop" + + xml_adapter.main_doi = "data-main_doi" + xml_adapter.elocation_id = "data-elocation_id" + return xml_adapter + + +class PidProviderXMLValidateQueryParamsTest(TestCase): + def setUp(self): + self.article_params = { + "z_collab": "VALUE", + "z_links": "Links", + "z_partial_body": "Body", + "z_surnames": "Z_SURNAMES", + "article_pub_year": "2020", + "elocation_id": "e19347", + "journal__issn_electronic": "issn electronic", + "journal__issn_print": "issn print", + "main_doi": "DOI", + "pkg_name": "pkgName", + } + + self.issue_params = { + "issue__pub_year": "year", + "issue__volume": "vol", + "issue__number": "num", + "issue__suppl": "suppl", + "fpage": "1", + "fpage_seq": "a", + "lpage": "11", + } + + def test_validate_query_params_all_present(self): + params = self.article_params + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_all_present_plus_issue_params(self): + params = self.article_params + params.update(self.issue_params) + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_issue_params_only(self): + params = {} + params.update(self.issue_params) + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + result = models.PidProviderXML.validate_query_params(params) + + def test_validate_query_params_journal_issns_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["journal__issn_print"] + del params["journal__issn_electronic"] + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + result = models.PidProviderXML.validate_query_params(params) + + def test_validate_query_params_pub_year_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["article_pub_year"] + del params["issue__pub_year"] + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + result = models.PidProviderXML.validate_query_params(params) + + def test_validate_query_params_main_doi_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_fpage_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["fpage"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_elocation_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["elocation_id"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_main_doi_fpage_elocation_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + del params["fpage"] + del params["elocation_id"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_z_surnames_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + del params["fpage"] + del params["elocation_id"] + del params["z_surnames"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_z_collab_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + del params["fpage"] + del params["elocation_id"] + del params["z_collab"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_z_collab_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + del params["fpage"] + del params["elocation_id"] + del params["z_links"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_z_collab_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + del params["fpage"] + del params["elocation_id"] + del params["pkg_name"] + result = models.PidProviderXML.validate_query_params(params) + self.assertTrue(result) + + def test_validate_query_params_z_collab_id_absence(self): + params = self.article_params + params.update(self.issue_params) + del params["main_doi"] + del params["fpage"] + del params["elocation_id"] + del params["pkg_name"] + del params["z_surnames"] + del params["z_collab"] + del params["z_links"] + + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + result = models.PidProviderXML.validate_query_params(params) + + +@patch( + "pid_provider.xml_sps_adapter.PidProviderXMLAdapter.query_list", + new_callable=mock.PropertyMock, +) +@patch( + "pid_provider.models.PidProviderXML.validate_query_params", + return_value=True, +) +@patch("pid_provider.models.PidProviderXML.objects.get") +class PidProviderXMLQueryDocumentTest(TestCase): + def test_query_document_is_called_with_query_params( + self, + mock_get, + mock_validate_params, + mock_query_list, + ): + """ + PidProviderXML._query_document is called with parameters returned by + PidProviderXML.query_list + """ + params_list = [ + {"key": "value"}, + ] + mock_query_list.return_value = params_list + mock_get.side_effect = models.PidProviderXML.DoesNotExist + xml_adapter = _get_xml_adapter() + result = models.PidProviderXML._query_document(xml_adapter) + mock_get.assert_called_once_with(**{"key": "value"}) + + def test_query_document_returns_none_if_document_does_not_exist( + self, + mock_get, + mock_validate_params, + mock_query_list, + ): + params_list = [ + {"key": "value"}, + ] + mock_query_list.return_value = params_list + mock_get.side_effect = models.PidProviderXML.DoesNotExist + xml_adapter = _get_xml_adapter() + result = models.PidProviderXML._query_document(xml_adapter) + self.assertIsNone(result) + + def test_query_document_returns_found_document( + self, + mock_get, + mock_validate_params, + mock_query_list, + ): + params_list = [ + {"key": "value"}, + ] + mock_query_list.return_value = params_list + mock_get.return_value = models.PidProviderXML() + xml_adapter = _get_xml_adapter() + result = models.PidProviderXML._query_document(xml_adapter) + self.assertEqual(models.PidProviderXML, type(result)) + + def test_query_document_returns_found_item_at_the_second_round( + self, + mock_get, + mock_validate_params, + mock_query_list, + ): + params_list = [ + {"key": "value"}, + {"key": "value2"}, + ] + mock_query_list.return_value = params_list + mock_get.side_effect = [ + models.PidProviderXML.DoesNotExist, + models.PidProviderXML(), + ] + xml_adapter = _get_xml_adapter() + result = models.PidProviderXML._query_document(xml_adapter) + self.assertEqual(models.PidProviderXML, type(result)) + + def test_query_document_raises_query_document_error_because_multiple_objects_returned( + self, + mock_get, + mock_validate_params, + mock_query_list, + ): + params_list = [ + {"key": "value"}, + ] + mock_query_list.return_value = params_list + mock_get.side_effect = models.PidProviderXML.MultipleObjectsReturned + with self.assertRaises( + exceptions.QueryDocumentMultipleObjectsReturnedError + ) as exc: + xml_adapter = _get_xml_adapter() + result = models.PidProviderXML._query_document(xml_adapter) + + def test_query_document_raises_error( + self, + mock_get, + mock_validate_params, + mock_query_list, + ): + """ + PidProviderXML._query_document is called with parameters returned by + PidProviderXML.query_list + """ + params_list = [ + {"key": "value"}, + ] + mock_query_list.return_value = params_list + mock_validate_params.side_effect = ( + exceptions.NotEnoughParametersToGetPidProviderXMLError + ) + + with self.assertRaises(exceptions.NotEnoughParametersToGetPidProviderXMLError): + xml_adapter = _get_xml_adapter() + result = models.PidProviderXML._query_document(xml_adapter) + + +@patch("pid_provider.models.PidProviderXML._query_document") +class PidProviderXMLGetRegisteredTest(TestCase): + def setUp(self): + self.xml_with_pre = _get_xml_with_pre() + + def test_get_registered_returns_dict_with_registered_data( + self, + mock_query_document, + ): + pid_req_xml = models.PidProviderXML() + pid_req_xml.pkg_name = "registered_pkg_name" + pid_req_xml.v2 = "registered_v2" + pid_req_xml.v3 = "registered_v3" + pid_req_xml.aop_pid = "registered_aop_pid" + pid_req_xml.created = datetime(2023, 2, 20) + pid_req_xml.updated = datetime(2023, 2, 20) + + mock_query_document.return_value = pid_req_xml + + result = models.PidProviderXML.get_registered(self.xml_with_pre) + expected = { + "v3": "registered_v3", + "v2": "registered_v2", + "aop_pid": "registered_aop_pid", + "pkg_name": "registered_pkg_name", + "created": "2023-02-20T00:00:00", + "updated": "2023-02-20T00:00:00", + "record_status": "updated", + } + self.assertDictEqual(expected, result) + + def test_get_registered_returns_none( + self, + mock_query_document, + ): + mock_query_document.return_value = None + + result = models.PidProviderXML.get_registered(self.xml_with_pre) + self.assertIsNone(result) + + def test_get_registered_returns_error_multiple_return( + self, + mock_query_document, + ): + mock_query_document.side_effect = ( + exceptions.QueryDocumentMultipleObjectsReturnedError + ) + + result = models.PidProviderXML.get_registered(self.xml_with_pre) + self.assertIn("error_type", result.keys()) + self.assertIn("error_msg", result.keys()) + + def test_get_registered_returns_error_not_enough_params( + self, + mock_query_document, + ): + mock_query_document.side_effect = ( + exceptions.NotEnoughParametersToGetPidProviderXMLError + ) + + result = models.PidProviderXML.get_registered(self.xml_with_pre) + self.assertIn("error_type", result.keys()) + self.assertIn("error_msg", result.keys()) + + +class PidProviderXMLEvaluateRegistrationTest(TestCase): + def setUp(self): + self.xml_adapter = _get_xml_adapter() + + def test_evaluate_registration_accepts_xml_is_aop_and_registered_is_aop(self): + registered = Mock(spec=models.PidProviderXML) + registered.is_aop = True + + self.xml_adapter.is_aop = True + + result = models.PidProviderXML.evaluate_registration( + self.xml_adapter, registered + ) + self.assertTrue(result) + + def test_evaluate_registration_accepts_xml_is_not_aop_and_registered_is_aop(self): + registered = Mock(spec=models.PidProviderXML) + registered.is_aop = True + + self.xml_adapter.is_aop = False + + result = models.PidProviderXML.evaluate_registration( + self.xml_adapter, registered + ) + self.assertTrue(result) + + def test_evaluate_registration_raises_error(self): + registered = Mock(spec=models.PidProviderXML) + registered.is_aop = False + + self.xml_adapter.is_aop = True + + with self.assertRaises(exceptions.ForbiddenPidProviderXMLRegistrationError): + result = models.PidProviderXML.evaluate_registration( + self.xml_adapter, registered + ) + + +@patch("pid_provider.models.PidProviderXML._get_unique_v2") +class PidProviderXMLAddV2Test(TestCase): + def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): + v2 = ( + v2 + and f'{v2}' + or "" + ) + v3 = ( + v3 + and f'{v3}' + or "" + ) + aop_pid = ( + aop_pid + and f'{aop_pid}' + or "" + ) + + return _get_xml_adapter( + f"""
+ + {v2} + {v3} + {aop_pid} + 10.36416/1806-3756/e20220072 + 01100 + +
""" + ) + + # TODO + # def test_add_pid_v2_uses_registered_pid_v2( + # self, + # mock_get_unique_v2, + # ): + # found = models.PidProviderXML() + # found.v2 = "registered_v2" + + # xml_adapter = self._get_xml_adapter(v2='xml_v2') + + # mock_get_unique_v2.return_value = "generated_v2" + + # models.PidProviderXML._add_pid_v2(xml_adapter, found) + # self.assertEqual("registered_v2", xml_adapter.v2) + + def test_add_pid_v2_replace_xml_v2_because_its_value_is_invalid_length_is_not_23( + self, + mock_get_unique_v2, + ): + found = models.PidProviderXML() + found.v2 = None + + xml_adapter = self._get_xml_adapter(v2="bad_size_not_23") + + mock_get_unique_v2.return_value = "S1806-37132022000201100" + + models.PidProviderXML._add_pid_v2(xml_adapter, found) + self.assertEqual("S1806-37132022000201100", xml_adapter.v2) + + def test_add_pid_v2_keeps_xml_v2( + self, + mock_get_unique_v2, + ): + found = models.PidProviderXML() + found.v2 = None + + xml_adapter = self._get_xml_adapter(v2="S1806-37132022000199999") + + mock_get_unique_v2.return_value = "S1806-37132022000300001" + + models.PidProviderXML._add_pid_v2(xml_adapter, found) + self.assertEqual("S1806-37132022000199999", xml_adapter.v2) + + def test_add_pid_v2_uses_unique_v2( + self, + mock_get_unique_v2, + ): + found = models.PidProviderXML() + found.v2 = None + + xml_adapter = self._get_xml_adapter() + + mock_get_unique_v2.return_value = "S1806-37132022000201100" + + models.PidProviderXML._add_pid_v2(xml_adapter, found) + self.assertEqual("S1806-37132022000201100", xml_adapter.v2) + + +class PidProviderXMLAddAopPidTest(TestCase): + def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): + v2 = ( + v2 + and f'{v2}' + or "" + ) + v3 = ( + v3 + and f'{v3}' + or "" + ) + aop_pid = ( + aop_pid + and f'{aop_pid}' + or "" + ) + + return _get_xml_adapter( + f"""
+ + {v2} + {v3} + {aop_pid} + 10.36416/1806-3756/e20220072 + 01100 + +
""" + ) + + def test_add_aop_pid_uses_registered_aop_pid( + self, + ): + found = models.PidProviderXML() + found.aop_pid = "12345678901234567890aop" + + xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") + + models.PidProviderXML._add_aop_pid(xml_adapter, found) + self.assertEqual("12345678901234567890aop", xml_adapter.aop_pid) + + def test_add_aop_pid_does_not_replace_by_none( + self, + ): + found = models.PidProviderXML() + found.aop_pid = None + + xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") + + models.PidProviderXML._add_aop_pid(xml_adapter, found) + self.assertEqual("xml_aop_pid", xml_adapter.aop_pid) + + +@patch("pid_provider.models.PidProviderXML._is_registered_pid") +@patch("pid_provider.models.PidProviderXML._get_unique_v3") +class PidProviderXMLAddPidV3Test(TestCase): + def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): + v2 = ( + v2 + and f'{v2}' + or "" + ) + v3 = ( + v3 + and f'{v3}' + or "" + ) + aop_pid = ( + aop_pid + and f'{aop_pid}' + or "" + ) + + return _get_xml_adapter( + f"""
+ + {v2} + {v3} + {aop_pid} + 10.36416/1806-3756/e20220072 + 01100 + +
""" + ) + + def test_add_pid_v3_uses_registered_v3( + self, + mock__get_unique_v3, + mock__is_registered_pid, + ): + found = models.PidProviderXML() + found.v3 = "123456789012345678901v3" + + xml_adapter = self._get_xml_adapter(v3="xml_v3") + + models.PidProviderXML._add_pid_v3(xml_adapter, found) + self.assertEqual("123456789012345678901v3", xml_adapter.v3) + + def test_add_pid_v3_replaced_by_generated( + self, + mock__get_unique_v3, + mock__is_registered_pid, + ): + mock__is_registered_pid.return_value = True + mock__get_unique_v3.return_value = "gen456789012345678901v3" + + found = None + + xml_adapter = self._get_xml_adapter(v3="xml_v3") + + models.PidProviderXML._add_pid_v3(xml_adapter, found) + self.assertEqual("gen456789012345678901v3", xml_adapter.v3) + + def test_add_pid_v3_keeps_xml_v3( + self, + mock__get_unique_v3, + mock__is_registered_pid, + ): + mock__is_registered_pid.return_value = False + mock__get_unique_v3.return_value = "gen456789012345678901v3" + + found = None + + xml_adapter = self._get_xml_adapter(v3="xml456789012345678901v3") + + models.PidProviderXML._add_pid_v3(xml_adapter, found) + self.assertEqual("xml456789012345678901v3", xml_adapter.v3) + + +class PidProviderXMLIsEqualToTest(TestCase): + def test_is_equal_to_returns_false(self): + xml_adapter = _get_xml_adapter_from_file( + "./pid_provider/fixtures/article/ex-aop.xml" + ) + registered = models.PidProviderXML() + + result = registered.is_equal_to(xml_adapter) + self.assertFalse(result) + + +class PidProviderXMLAddDataForRegularArticleTest(TestCase): + def setUp(self): + user = User() + xml_adapter = _get_xml_adapter_from_file( + "./pid_provider/fixtures/article/ex-aop.xml" + ) + self.registered = models.PidProviderXML() + self.registered._add_data(xml_adapter, user) + + def test_v3(self): + self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", self.registered.v3) + + def test_v2(self): + self.assertEqual("S1413-41522020000400627", self.registered.v2) + + def test_aop_pid(self): + self.assertEqual("S1413-41522020005000111", self.registered.aop_pid) + + def test_main_doi(self): + self.assertEqual("10.1590/S1413-4152202020180029", self.registered.main_doi) + + def test_fpage(self): + self.assertEqual("627", self.registered.fpage) + + def test_fpage_seq(self): + self.assertEqual(None, self.registered.fpage_seq) + + def test_lpage(self): + self.assertEqual("634", self.registered.lpage) + + def test_elocation_id(self): + self.assertEqual(None, self.registered.elocation_id) + + def test_article_pub_year(self): + self.assertEqual("2020", self.registered.article_pub_year) + + def test_z_surnames(self): + self.assertEqual( + "544700df348a47fdd7c55713054e12663a0c530e60e7a166395a496f77de9d36", + self.registered.z_surnames, + ) + + def test_z_collab(self): + self.assertIsNone(self.registered.z_collab) + + def test_z_links(self): + self.assertIsNone(self.registered.z_links) + + def test_z_partial_body(self): + self.assertEqual( + "2e07675bfe91c65e1544ada450ff2e956fef9b492d30e997ebd47687e0f7afa2", + self.registered.z_partial_body, + ) + + +@patch( + "pid_provider.models.utcnow", + side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], +) +@patch("pid_provider.models.XMLVersion.save_file") +@patch("pid_provider.models.XMLVersion.save") +@patch("pid_provider.models.XMLIssue.save") +@patch("pid_provider.models.XMLJournal.save") +@patch("pid_provider.models.PidProviderXML.save") +@patch("pid_provider.models.PidRequest.save") +class PidProviderXMLRegisterTest(TestCase): + def test_register_returns_error( + self, + mock_pid_request_save, + mock_pid_provider_xml_save, + mock_xml_journal_save, + mock_xml_issue_save, + mock_xml_version_save, + mock_xml_version_save_file, + mock_now, + ): + expected = { + "result_type": "", + "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", + "origin": "filename.xml", + "xml": "
", + } + + user = User() + xml_with_pre = _get_xml_with_pre() + result = models.PidProviderXML.register( + xml_with_pre=xml_with_pre, + filename="filename.xml", + user=user, + ) + print(result) + self.assertEqual(expected["result_type"], result["result_type"]) + self.assertIsNotNone(result["result_msg"]) + # self.assertEqual(expected["result_message"], result["result_msg"]) + self.assertEqual(expected["origin"], result["origin"]) + self.assertEqual(expected["xml"], result["detail"]["xml"]) + mock_pid_provider_xml_save.assert_not_called() + mock_pid_request_save.assert_called_once_with() + + +@patch( + "pid_provider.models.utcnow", + side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], +) +@patch("pid_provider.models.XMLSPS.save") +@patch("pid_provider.models.XMLVersion.save_file") +@patch("pid_provider.models.XMLVersion.save") +@patch("pid_provider.models.XMLIssue.save") +@patch("pid_provider.models.XMLJournal.save") +@patch("pid_provider.models.PidProviderXML.save") +@patch("pid_provider.models.PidRequest.save") +class PidProviderXMLRegisterTest(TestCase): + def test_register_with_success( + self, + mock_pid_request_save, + mock_pid_provider_xml_save, + mock_xml_journal_save, + mock_xml_issue_save, + mock_xml_version_save, + mock_xml_version_save_file, + mock_xml_sps_save, + mock_now, + ): + expected = { + "result_type": "", + "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", + "origin": "filename.xml", + "xml": "
", + } + + user = User() + xml_adapter = _get_xml_adapter_from_file( + "./pid_provider/fixtures/article/ex-aop.xml" + ) + result = models.PidProviderXML.register( + xml_with_pre=xml_adapter.xml_with_pre, + filename="ex-aop.xml", + user=user, + ) + self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", result["v3"]) + self.assertEqual("S1413-41522020000400627", result["v2"]) + self.assertEqual("S1413-41522020005000111", result["aop_pid"]) + self.assertEqual("1809-4457-esa-25-04-627", result["pkg_name"]) + self.assertEqual(False, result["xml_changed"]) + self.assertEqual("created", result["record_status"]) + self.assertIsNone(result["updated"]) + self.assertIsNotNone(result["created"]) + mock_pid_request_save.assert_not_called() + + +class XMLURLTest(TestCase): + """Tests for XMLURL model""" + + def setUp(self): + self.user = User.objects.create_user(username="testuser", password="testpass") + self.test_url = "http://example.com/article.xml" + self.test_pid = "ABC123XYZ456" + + def test_create_xmlurl(self): + """Test creating a new XMLURL instance""" + xmlurl = models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + self.assertIsNotNone(xmlurl) + self.assertEqual(xmlurl.url, self.test_url) + self.assertEqual(xmlurl.status, "pending") + self.assertEqual(xmlurl.pid, self.test_pid) + self.assertEqual(xmlurl.creator, self.user) + + def test_get_xmlurl(self): + """Test getting an XMLURL by URL""" + models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + xmlurl = models.XMLURL.get(url=self.test_url) + self.assertIsNotNone(xmlurl) + self.assertEqual(xmlurl.url, self.test_url) + + def test_create_or_update_existing(self): + """Test updating an existing XMLURL""" + # Create initial record + xmlurl = models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=None, + ) + + # Update it + updated_xmlurl = models.XMLURL.create_or_update( + user=self.user, + url=self.test_url, + status="success", + pid=self.test_pid, + ) + + self.assertEqual(updated_xmlurl.id, xmlurl.id) + self.assertEqual(updated_xmlurl.status, "success") + self.assertEqual(updated_xmlurl.pid, self.test_pid) + self.assertEqual(updated_xmlurl.updated_by, self.user) + + def test_create_or_update_new(self): + """Test creating a new XMLURL when it doesn't exist""" + xmlurl = models.XMLURL.create_or_update( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + self.assertIsNotNone(xmlurl) + self.assertEqual(xmlurl.url, self.test_url) + self.assertEqual(xmlurl.status, "pending") + + def test_save_file_with_string_content(self): + """Test save_file method with string XML content""" + xmlurl = models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + xml_content = "
Test Article
" + result = xmlurl.save_file(xml_content, filename="test.xml") + + self.assertTrue(result) + self.assertTrue(xmlurl.zipfile.name) + + def test_save_file_with_bytes_content(self): + """Test save_file method with bytes XML content""" + xmlurl = models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + xml_content = b"
Test Article
" + result = xmlurl.save_file(xml_content, filename="test.xml") + + self.assertTrue(result) + self.assertTrue(xmlurl.zipfile.name) + + def test_save_file_default_filename(self): + """Test save_file method with default filename""" + xmlurl = models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + xml_content = "
Test Article
" + result = xmlurl.save_file(xml_content) + + self.assertTrue(result) + self.assertTrue(xmlurl.zipfile.name) + + def test_str_method(self): + """Test __str__ method""" + xmlurl = models.XMLURL.create( + user=self.user, + url=self.test_url, + status="pending", + pid=self.test_pid, + ) + + expected_str = f"{self.test_url} - pending" + self.assertEqual(str(xmlurl), expected_str) + + +class BasePidProviderXMLURITest(TestCase): + """Tests for BasePidProvider.provide_pid_for_xml_uri method""" + + def setUp(self): + self.user = User.objects.create_user(username="testuser", password="testpass") + + @patch("pid_provider.base_pid_provider.XMLWithPre.create") + def test_provide_pid_for_xml_uri_fetch_failure(self, mock_create): + """Test exception type a) - Failure to obtain XML""" + from pid_provider.base_pid_provider import BasePidProvider + + # Mock XMLWithPre.create to raise an exception + mock_create.side_effect = Exception("Connection timeout") + + provider = BasePidProvider() + result = provider.provide_pid_for_xml_uri( + xml_uri="http://example.com/article.xml", + name="test.xml", + user=self.user, + ) + + # Should return error details + self.assertIn("error_msg", result) + self.assertIn("error_type", result) + + # Should create XMLURL with failed status + xmlurl = models.XMLURL.get(url="http://example.com/article.xml") + self.assertEqual(xmlurl.status, "xml_fetch_failed") + self.assertIsNone(xmlurl.pid) + + @patch("pid_provider.base_pid_provider.XMLWithPre.create") + @patch.object(models.PidProviderXML, "register") + def test_provide_pid_for_xml_uri_success(self, mock_register, mock_create): + """Test successful processing with XMLURL creation""" + from pid_provider.base_pid_provider import BasePidProvider + + # Mock XMLWithPre.create + xml_with_pre = _get_xml_with_pre("
Test
") + mock_create.return_value = [xml_with_pre] + + # Mock successful registration + mock_register.return_value = { + "v3": "test_v3_pid", + "v2": "test_v2_pid", + "created": datetime.now(), + } + + provider = BasePidProvider() + result = provider.provide_pid_for_xml_uri( + xml_uri="http://example.com/article.xml", + name="test.xml", + user=self.user, + ) + + # Should return success response + self.assertEqual(result.get("v3"), "test_v3_pid") + + # Should create XMLURL with success status + xmlurl = models.XMLURL.get(url="http://example.com/article.xml") + self.assertEqual(xmlurl.status, "success") + self.assertEqual(xmlurl.pid, "test_v3_pid") + + @patch("pid_provider.base_pid_provider.XMLWithPre.create") + @patch.object(models.PidProviderXML, "register") + def test_provide_pid_for_xml_uri_registration_failure(self, mock_register, mock_create): + """Test exception type b) - XML obtained but registration failed""" + from pid_provider.base_pid_provider import BasePidProvider + + # Mock XMLWithPre.create + xml_with_pre = _get_xml_with_pre("
Test
") + mock_create.return_value = [xml_with_pre] + + # Mock failed registration + mock_register.return_value = { + "error_type": "ValidationError", + "error_message": "Invalid XML structure", + "v3": "test_v3_pid", + } + + provider = BasePidProvider() + result = provider.provide_pid_for_xml_uri( + xml_uri="http://example.com/article2.xml", + name="test2.xml", + user=self.user, + ) + + # Should return error response + self.assertIn("error_type", result) + + # Should create XMLURL with failed status and save zipfile + xmlurl = models.XMLURL.get(url="http://example.com/article2.xml") + self.assertEqual(xmlurl.status, "pid_provider_xml_failed") + self.assertEqual(xmlurl.pid, "test_v3_pid") diff --git a/pid_provider/tests/test_register.py b/pid_provider/tests/test_register.py new file mode 100644 index 000000000..a8bdccc32 --- /dev/null +++ b/pid_provider/tests/test_register.py @@ -0,0 +1,313 @@ +""" +Testes unitários para PidProviderXML.register. + +Estratégia +---------- +register() é orquestrador: delega a get_records, complete_missing_xml_pids, +is_updated e _save. Os testes isolam register desses colaboradores via mock e +verificam, para cada caminho, DOIS contratos: + + 1. o event_status efetivamente gravado em PidProviderXMLRegistration.record + 2. a forma do response retornado + +Como register grava SEMPRE (no finally), o ponto de observação central é o +mock de PidProviderXMLRegistration.record: inspecionamos seus kwargs. + +Caminhos cobertos (event_status): + - created : sem registro existente -> _save cria + - updated : registro existente -> _save atualiza + - skipped : is_updated retorna data (já atualizado / igual) + - conflict : complete_missing_xml_pids levanta PidProviderXMLPidV3ConflictError + - forbidden : is_updated levanta ForbiddenPidProviderXMLRegistrationError + - unmatched : get_records levanta UnmatchedPidProviderXMLError / MultipleObjectsReturned + - bad_request : get_records levanta RequiredISSNError etc. + - error : exceção inesperada -> UnexpectedEvent + event_status=error + +Ajuste os caminhos de import (PATCH_BASE) conforme a estrutura do seu projeto. +""" + +from unittest.mock import patch, MagicMock + +from django.test import TestCase + +from pid_provider import exceptions +from pid_provider.models import ( + PidProviderXML, + PidProviderXMLPidV3ConflictError, +) + +# Caminho do módulo onde register está definido (para os patches "where used"). +PATCH_BASE = "pid_provider.models" + + +def make_xml_with_pre(**overrides): + """ + XMLWithPre falso, com os atributos que register/build_readable_data tocam. + """ + m = MagicMock(name="xml_with_pre") + m.data = {"pid_v3": overrides.get("v3"), "sps_pkg_name": "pkg-fake"} + m.sps_pkg_name = overrides.get("sps_pkg_name", "pkg-fake") + # build_readable_data: + m.authors = {"person": [{"surname": "SILVA"}]} + m.collab = None + m.links = [] + m.article_titles_texts = ["Some title"] + m.partial_body = "corpo parcial" + return m + + +class RegisterTestBase(TestCase): + """ + Mocka todos os colaboradores de register e o gravador de auditoria. + Cada teste configura os side_effects/returns conforme o caminho. + """ + + def setUp(self): + self.user = MagicMock(name="user") + self.xml = make_xml_with_pre(v3="ABCDEFGHIJKLMNOPQRSTUVW") + + # patch do adapter para não depender de packtools real + self.p_adapter = patch( + "packtools.sps.pid_provider.xml_sps_adapter.PidProviderXMLAdapter" + ) + self.m_adapter_cls = self.p_adapter.start() + self.m_adapter = self.m_adapter_cls.return_value + self.m_adapter.data = {"pkg_name": "pkg-fake"} + self.m_adapter.sps_pkg_name = "pkg-fake" + self.m_adapter.xml_with_pre = self.xml + self.addCleanup(self.p_adapter.stop) + + # patch do gravador de auditoria — ponto central de verificação + self.p_record = patch(f"{PATCH_BASE}.PidProviderXMLRegistration.record") + self.m_record = self.p_record.start() + self.addCleanup(self.p_record.stop) + + # build_readable_data é staticmethod; deixamos rodar (usa o xml fake), + # mas se preferir isolar, dá para mockar também. + + # -- helper de asserção ------------------------------------------------- + def assert_recorded_status(self, expected_status): + self.assertTrue( + self.m_record.called, "PidProviderXMLRegistration.record não foi chamado" + ) + kwargs = self.m_record.call_args.kwargs + self.assertEqual(kwargs.get("event_status"), expected_status) + return kwargs + + +class CreatedPathTest(RegisterTestBase): + def test_created_when_no_existing_record(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.side_effect = PidProviderXML.DoesNotExist + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "created"} + m_save.return_value = (saved, "created") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("created") + self.assertEqual(response.get("v3"), "ABC") + self.assertNotIn("error_msg", response) + + +class UpdatedPathTest(RegisterTestBase): + def test_updated_when_existing_record(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.return_value = {"pid_v3": "NEW"} + m_upd.return_value = None + saved = MagicMock(name="saved_ppx") + saved.data = {"v3": "ABC", "record_status": "updated"} + m_save.return_value = (saved, "updated") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + kwargs = self.assert_recorded_status("updated") + # o evento referencia o objeto salvo + self.assertIs(kwargs.get("pid_provider_xml"), saved) + self.assertIn("xml_changed", response) + + +class SkippedPathTest(RegisterTestBase): + def test_skipped_returns_data_and_does_not_flag_error(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.return_value = {} + m_upd.return_value = {"v3": "ABC", "record_status": "updated"} # já atualizado + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + # _save NÃO deve ser chamado no skip + m_save.assert_not_called() + + self.assert_recorded_status("skipped") + self.assertTrue(response.get("skip_update")) + # skip é sucesso de negócio: não deve marcar erro + self.assertNotIn("error_msg", response) + + +class ConflictPathTest(RegisterTestBase): + def test_conflict_when_pid_v3_conflict(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.side_effect = PidProviderXMLPidV3ConflictError("conflict!") + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + self.assert_recorded_status("conflict") + self.assertIn("error_msg", response) + self.assertIn("error_type", response) + + +class ForbiddenPathTest(RegisterTestBase): + def test_forbidden_when_aop_over_vor(self): + existing = MagicMock(name="existing_ppx") + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.return_value = {"registered": existing, "failed": []} + m_cmp.return_value = {} + m_upd.side_effect = ( + exceptions.ForbiddenPidProviderXMLRegistrationError("forbidden") + ) + + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_save.assert_not_called() + + self.assert_recorded_status("forbidden") + self.assertIn("error_msg", response) + + +class UnmatchedPathTest(RegisterTestBase): + def test_unmatched_when_get_records_raises_unmatched(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = exceptions.UnmatchedPidProviderXMLError("unmatched") + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + self.assertIn("error_msg", response) + + def test_unmatched_when_failed_without_registered(self): + # get_records retorna dict com failed e sem registered -> register levanta + # UnmatchedPidProviderXMLError internamente + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.return_value = {"registered": None, "failed": [{"id": 1}]} + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + + def test_multiple_objects_returned_is_unmatched(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = PidProviderXML.MultipleObjectsReturned() + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("unmatched") + + +class BadRequestPathTest(RegisterTestBase): + """ + ATENÇÃO — MUDANÇA DE CONTRATO: + Na versão atual, as exceções de bad_request NÃO propagam mais ao chamador; + viram response com event_status='bad_request'. Estes testes DOCUMENTAM o + comportamento atual. Se a decisão for propagar (opção B), troque por + assertRaises e remova o return no finally. + """ + + def test_required_issn_becomes_response_not_raise(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = ( + exceptions.RequiredISSNErrorToGetPidProviderXMLError("no issn") + ) + # NÃO levanta — retorna response + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + self.assertIn("error_msg", response) + + def test_required_pub_year_becomes_response(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = ( + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError("no year") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + + def test_not_enough_parameters_becomes_response(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get: + m_get.side_effect = ( + exceptions.NotEnoughParametersToGetPidProviderXMLError("not enough") + ) + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("bad_request") + + +class ErrorPathTest(RegisterTestBase): + def test_unexpected_exception_records_error_and_logs_unexpected_event(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.UnexpectedEvent.create") as m_unexpected: + + m_get.side_effect = ValueError("algo totalmente inesperado") + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assert_recorded_status("error") + self.assertIn("error_msg", response) + # erro inesperado deve registrar UnexpectedEvent + m_unexpected.assert_called_once() + + +class RecordAlwaysCalledTest(RegisterTestBase): + """Garante a premissa 'grava SEMPRE': record é chamado exatamente 1 vez.""" + + def test_record_called_exactly_once_on_success(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp, \ + patch(f"{PATCH_BASE}.PidProviderXML.is_updated") as m_upd, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + + m_get.side_effect = PidProviderXML.DoesNotExist + m_cmp.return_value = {} + m_upd.return_value = None + saved = MagicMock() + saved.data = {"v3": "ABC"} + m_save.return_value = (saved, "created") + + PidProviderXML.register(self.xml, "file.xml", self.user) + + self.assertEqual(self.m_record.call_count, 1) + + def test_record_called_exactly_once_on_conflict(self): + with patch(f"{PATCH_BASE}.PidProviderXML.get_records") as m_get, \ + patch(f"{PATCH_BASE}.PidProviderXML.complete_missing_xml_pids") as m_cmp: + m_get.return_value = {"registered": MagicMock(), "failed": []} + m_cmp.side_effect = PidProviderXMLPidV3ConflictError("x") + PidProviderXML.register(self.xml, "file.xml", self.user) + + # antes havia risco de gravar 2x (except + finally); deve ser 1 + self.assertEqual(self.m_record.call_count, 1) \ No newline at end of file From 904784ec77eb87773bcb176acfe1f4a05aaccda3 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 18:58:02 -0300 Subject: [PATCH 10/13] fix(query_params): corrige calculo da pontuacao percentual de correspondencia - Removida a multiplicacao por 100 no calculo do 'percentual_score'. - Otimizacao/Correcao: Ajusta o retorno para uma escala decimal (0 a 1) em vez de percentual (0 a 100), compatibilizando o resultado com a nova regra de corte de score estabelecida em 0.6. --- pid_provider/query_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index f8e71ec4a..79d786968 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -19,7 +19,7 @@ def compare(registered_items, input_data): return { "items": items, "total_score": total_score, - "percentual_score": total_score * 100 / len(items) + "percentual_score": total_score / len(items) } From b5c88e9bde8a7979a39cb0423ecfeda53c8e6dd1 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 18:58:03 -0300 Subject: [PATCH 11/13] chore(wagtail): inclui PidProviderXMLRegistrationViewSet no grupo de snippets - Comentado o metodo 'get_queryset' antigo e incorreto que quebrava a assinatura do Wagtail/Django. - Adicionado o 'PidProviderXMLRegistrationViewSet' a tupla de viewsets do 'PidProviderViewSetGroup'. - Melhora o gerenciamento e a visibilidade dos logs de auditoria de registros XML diretamente na interface administrativa. --- pid_provider/wagtail_hooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pid_provider/wagtail_hooks.py b/pid_provider/wagtail_hooks.py index d90bb6b3b..08b1b8329 100644 --- a/pid_provider/wagtail_hooks.py +++ b/pid_provider/wagtail_hooks.py @@ -232,9 +232,8 @@ class PidProviderXMLRegistrationViewSet(CommonControlFieldViewSet): list_per_page = 50 # ---- READ ONLY: auditoria não é editada pela UI ---- - def get_queryset(self, request): - return super().get_queryset(request).select_related("pid_provider_xml") - + # def get_queryset(self): + # return super().get_queryset().select_related("pid_provider_xml") # Grupo de ViewSets class PidProviderViewSetGroup(SnippetViewSetGroup): @@ -248,6 +247,7 @@ class PidProviderViewSetGroup(SnippetViewSetGroup): PidProviderConfigViewSet, XMLVersionViewSet, XMLURLViewSet, + PidProviderXMLRegistrationViewSet, ) From 205970d8820b74e23e8ede9a029a74499b21e40f Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 27 Jun 2026 18:58:03 -0300 Subject: [PATCH 12/13] refactor(test): conclui movimentacao de testes para nova estrutura de diretorios - Concluida a remocao dos arquivos 'test_controller.py' e 'test_models.py' da raiz do modulo pid_provider. - Organizacao: Conecta a delecao no antigo escopo com a migracao dos arquivos para dentro da pasta consolidada 'pid_provider/tests/', finalizando a reestruturacao da suite de testes automatizados. --- pid_provider/test_controller.py | 87 --- pid_provider/test_models.py | 1029 ------------------------------- 2 files changed, 1116 deletions(-) delete mode 100644 pid_provider/test_controller.py delete mode 100644 pid_provider/test_models.py diff --git a/pid_provider/test_controller.py b/pid_provider/test_controller.py deleted file mode 100644 index 3ff177825..000000000 --- a/pid_provider/test_controller.py +++ /dev/null @@ -1,87 +0,0 @@ -from unittest.mock import ANY, Mock, patch - -from django.contrib.auth import get_user_model -from django.test import TestCase - -from pid_provider.controller import PidProvider -from pid_provider.models import PidProviderXML - -User = get_user_model() - - -# def get_mock_config(): -# config = object() -# config.host = '' -# config.access_key = '' -# config.secret_key = '' -# config.bucket_root = '' -# config.bucket_app_subdir = 'bucket-app-subdir' -# config.secure = '' -# return config - - -class PidProviderTest(TestCase): - @patch("pid_provider.models.XMLSPS.save") - @patch("pid_provider.models.XMLVersion.save") - @patch("pid_provider.models.XMLIssue.save") - @patch("pid_provider.models.XMLJournal.save") - @patch("pid_provider.models.PidProviderXML.save") - @patch( - "pid_provider.models.PidProviderXML._get_unique_v3", - return_value="SJLD63mRxz9nTXtyMj7SLwk", - ) - @patch( - "pid_provider.models.PidProviderXML._get_unique_v2", - return_value="S2236-89062022061645340", - ) - @patch("pid_provider.controller.PidProviderConfig.get_or_create") - def test_provide_pid_for_xml_zip( - self, - mock_pid_provider_config, - mock_get_unique_v2, - mock_get_unique_v3, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xmlsps_save, - ): - pid_provider = PidProvider() - result = pid_provider.provide_pid_for_xml_zip( - zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", - user=User.objects.first(), - ) - result = list(result) - self.assertEqual("SJLD63mRxz9nTXtyMj7SLwk", result[0]["v3"]) - self.assertEqual("S2236-89062022061645340", result[0]["v2"]) - self.assertIsNone(result[0]["aop_pid"]) - self.assertIsNotNone(result[0]["created"]) - self.assertIsNone(result[0]["updated"]) - self.assertEqual("2236-8906-hoehnea-49-e1082020.xml", result[0]["filename"]) - self.assertEqual("created", result[0]["record_status"]) - self.assertEqual(True, result[0]["xml_changed"]) - - @patch("pid_provider.models.PidProviderXML._query_document") - @patch("pid_provider.models.PidProviderXML.is_equal_to", return_value=True) - def test_provide_pid_for_xml_with_pre_do_nothing_because_it_is_already_updated( - self, - mock_is_equal, - mock_query_document, - ): - # dubla o registro encontrado - pid_provider_xml = Mock(PidProviderXML) - pid_provider_xml.data = {"v3": ""} - mock_query_document.return_value = pid_provider_xml - - pid_provider_ = PidProvider() - result = pid_provider_.provide_pid_for_xml_zip( - zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", - user=User.objects.first(), - ) - result = list(result) - expected = { - "filename": "2236-8906-hoehnea-49-e1082020.xml", - "v3": "", - "xml_with_pre": ANY, - } - self.assertDictEqual(result[0], expected) diff --git a/pid_provider/test_models.py b/pid_provider/test_models.py deleted file mode 100644 index 4249c7c2a..000000000 --- a/pid_provider/test_models.py +++ /dev/null @@ -1,1029 +0,0 @@ -import logging -from datetime import datetime -from unittest import mock -from unittest.mock import ANY, MagicMock, Mock, call, patch - -from django.contrib.auth import get_user_model -from django.test import TestCase -from lxml import etree -from xmlsps.xml_sps_lib import XMLWithPre - -from pid_provider import exceptions, models -from pid_provider.xml_sps_adapter import PidProviderXMLAdapter - -User = get_user_model() - - -def _get_xml_adapter_from_file(path): - for xml_with_pre in XMLWithPre.create(path=path): - obj = PidProviderXMLAdapter(xml_with_pre) - return obj - - -def _get_xml_with_pre(xml=None): - xml = xml or "
" - return XMLWithPre("", etree.fromstring(xml)) - - -def _get_xml_adapter(xml=None): - xml = xml or "
" - xml_with_pre = XMLWithPre("", etree.fromstring(xml)) - obj = PidProviderXMLAdapter(xml_with_pre) - return obj - - -def _get_xml_adapter_with_issue_data(): - xml_adapter = _get_xml_adapter() - xml_adapter.journal_issn_electronic = "data-issn-e" - xml_adapter.journal_issn_print = "data-issn-p" - xml_adapter.volume = "data-vol" - xml_adapter.number = "data-num" - xml_adapter.suppl = "data-suppl" - xml_adapter.pub_year = "data-year" - xml_adapter.issue = models.XMLIssue.get_or_create( - models.XMLJournal.get_or_create("data-issn-e", "data-issn-p"), - "data-vol", - "data-num", - "data-suppl", - "data-year", - ) - xml_adapter.fpage = "data-fpage" - xml_adapter.fpage_seq = "data-fpage-seq" - xml_adapter.lpage = "data-lpage" - - xml_adapter.article_pub_year = "data-pub-year" - xml_adapter.v3 = "123456789012345678901v3" - xml_adapter.v2 = "123456789012345678901v2" - xml_adapter.aop_pid = "12345678901234567890aop" - - xml_adapter.main_doi = "data-main_doi" - xml_adapter.elocation_id = "data-elocation_id" - return xml_adapter - - -class PidProviderXMLValidateQueryParamsTest(TestCase): - def setUp(self): - self.article_params = { - "z_collab": "VALUE", - "z_links": "Links", - "z_partial_body": "Body", - "z_surnames": "Z_SURNAMES", - "article_pub_year": "2020", - "elocation_id": "e19347", - "journal__issn_electronic": "issn electronic", - "journal__issn_print": "issn print", - "main_doi": "DOI", - "pkg_name": "pkgName", - } - - self.issue_params = { - "issue__pub_year": "year", - "issue__volume": "vol", - "issue__number": "num", - "issue__suppl": "suppl", - "fpage": "1", - "fpage_seq": "a", - "lpage": "11", - } - - def test_validate_query_params_all_present(self): - params = self.article_params - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_all_present_plus_issue_params(self): - params = self.article_params - params.update(self.issue_params) - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_issue_params_only(self): - params = {} - params.update(self.issue_params) - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_journal_issns_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["journal__issn_print"] - del params["journal__issn_electronic"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_pub_year_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["article_pub_year"] - del params["issue__pub_year"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_main_doi_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_fpage_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["fpage"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_elocation_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["elocation_id"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_main_doi_fpage_elocation_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_surnames_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_surnames"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_collab"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_links"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["pkg_name"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["pkg_name"] - del params["z_surnames"] - del params["z_collab"] - del params["z_links"] - - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - -@patch( - "pid_provider.xml_sps_adapter.PidProviderXMLAdapter.query_list", - new_callable=mock.PropertyMock, -) -@patch( - "pid_provider.models.PidProviderXML.validate_query_params", - return_value=True, -) -@patch("pid_provider.models.PidProviderXML.objects.get") -class PidProviderXMLQueryDocumentTest(TestCase): - def test_query_document_is_called_with_query_params( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - """ - PidProviderXML._query_document is called with parameters returned by - PidProviderXML.query_list - """ - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.DoesNotExist - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - mock_get.assert_called_once_with(**{"key": "value"}) - - def test_query_document_returns_none_if_document_does_not_exist( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.DoesNotExist - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertIsNone(result) - - def test_query_document_returns_found_document( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.return_value = models.PidProviderXML() - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertEqual(models.PidProviderXML, type(result)) - - def test_query_document_returns_found_item_at_the_second_round( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - {"key": "value2"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = [ - models.PidProviderXML.DoesNotExist, - models.PidProviderXML(), - ] - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertEqual(models.PidProviderXML, type(result)) - - def test_query_document_raises_query_document_error_because_multiple_objects_returned( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.MultipleObjectsReturned - with self.assertRaises( - exceptions.QueryDocumentMultipleObjectsReturnedError - ) as exc: - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - - def test_query_document_raises_error( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - """ - PidProviderXML._query_document is called with parameters returned by - PidProviderXML.query_list - """ - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_validate_params.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError - ) - - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - - -@patch("pid_provider.models.PidProviderXML._query_document") -class PidProviderXMLGetRegisteredTest(TestCase): - def setUp(self): - self.xml_with_pre = _get_xml_with_pre() - - def test_get_registered_returns_dict_with_registered_data( - self, - mock_query_document, - ): - pid_req_xml = models.PidProviderXML() - pid_req_xml.pkg_name = "registered_pkg_name" - pid_req_xml.v2 = "registered_v2" - pid_req_xml.v3 = "registered_v3" - pid_req_xml.aop_pid = "registered_aop_pid" - pid_req_xml.created = datetime(2023, 2, 20) - pid_req_xml.updated = datetime(2023, 2, 20) - - mock_query_document.return_value = pid_req_xml - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - expected = { - "v3": "registered_v3", - "v2": "registered_v2", - "aop_pid": "registered_aop_pid", - "pkg_name": "registered_pkg_name", - "created": "2023-02-20T00:00:00", - "updated": "2023-02-20T00:00:00", - "record_status": "updated", - } - self.assertDictEqual(expected, result) - - def test_get_registered_returns_none( - self, - mock_query_document, - ): - mock_query_document.return_value = None - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIsNone(result) - - def test_get_registered_returns_error_multiple_return( - self, - mock_query_document, - ): - mock_query_document.side_effect = ( - exceptions.QueryDocumentMultipleObjectsReturnedError - ) - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIn("error_type", result.keys()) - self.assertIn("error_msg", result.keys()) - - def test_get_registered_returns_error_not_enough_params( - self, - mock_query_document, - ): - mock_query_document.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError - ) - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIn("error_type", result.keys()) - self.assertIn("error_msg", result.keys()) - - -class PidProviderXMLEvaluateRegistrationTest(TestCase): - def setUp(self): - self.xml_adapter = _get_xml_adapter() - - def test_evaluate_registration_accepts_xml_is_aop_and_registered_is_aop(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = True - - self.xml_adapter.is_aop = True - - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - self.assertTrue(result) - - def test_evaluate_registration_accepts_xml_is_not_aop_and_registered_is_aop(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = True - - self.xml_adapter.is_aop = False - - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - self.assertTrue(result) - - def test_evaluate_registration_raises_error(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = False - - self.xml_adapter.is_aop = True - - with self.assertRaises(exceptions.ForbiddenPidProviderXMLRegistrationError): - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - - -@patch("pid_provider.models.PidProviderXML._get_unique_v2") -class PidProviderXMLAddV2Test(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - # TODO - # def test_add_pid_v2_uses_registered_pid_v2( - # self, - # mock_get_unique_v2, - # ): - # found = models.PidProviderXML() - # found.v2 = "registered_v2" - - # xml_adapter = self._get_xml_adapter(v2='xml_v2') - - # mock_get_unique_v2.return_value = "generated_v2" - - # models.PidProviderXML._add_pid_v2(xml_adapter, found) - # self.assertEqual("registered_v2", xml_adapter.v2) - - def test_add_pid_v2_replace_xml_v2_because_its_value_is_invalid_length_is_not_23( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter(v2="bad_size_not_23") - - mock_get_unique_v2.return_value = "S1806-37132022000201100" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000201100", xml_adapter.v2) - - def test_add_pid_v2_keeps_xml_v2( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter(v2="S1806-37132022000199999") - - mock_get_unique_v2.return_value = "S1806-37132022000300001" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000199999", xml_adapter.v2) - - def test_add_pid_v2_uses_unique_v2( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter() - - mock_get_unique_v2.return_value = "S1806-37132022000201100" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000201100", xml_adapter.v2) - - -class PidProviderXMLAddAopPidTest(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - def test_add_aop_pid_uses_registered_aop_pid( - self, - ): - found = models.PidProviderXML() - found.aop_pid = "12345678901234567890aop" - - xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") - - models.PidProviderXML._add_aop_pid(xml_adapter, found) - self.assertEqual("12345678901234567890aop", xml_adapter.aop_pid) - - def test_add_aop_pid_does_not_replace_by_none( - self, - ): - found = models.PidProviderXML() - found.aop_pid = None - - xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") - - models.PidProviderXML._add_aop_pid(xml_adapter, found) - self.assertEqual("xml_aop_pid", xml_adapter.aop_pid) - - -@patch("pid_provider.models.PidProviderXML._is_registered_pid") -@patch("pid_provider.models.PidProviderXML._get_unique_v3") -class PidProviderXMLAddPidV3Test(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - def test_add_pid_v3_uses_registered_v3( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - found = models.PidProviderXML() - found.v3 = "123456789012345678901v3" - - xml_adapter = self._get_xml_adapter(v3="xml_v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("123456789012345678901v3", xml_adapter.v3) - - def test_add_pid_v3_replaced_by_generated( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - mock__is_registered_pid.return_value = True - mock__get_unique_v3.return_value = "gen456789012345678901v3" - - found = None - - xml_adapter = self._get_xml_adapter(v3="xml_v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("gen456789012345678901v3", xml_adapter.v3) - - def test_add_pid_v3_keeps_xml_v3( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - mock__is_registered_pid.return_value = False - mock__get_unique_v3.return_value = "gen456789012345678901v3" - - found = None - - xml_adapter = self._get_xml_adapter(v3="xml456789012345678901v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("xml456789012345678901v3", xml_adapter.v3) - - -class PidProviderXMLIsEqualToTest(TestCase): - def test_is_equal_to_returns_false(self): - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - registered = models.PidProviderXML() - - result = registered.is_equal_to(xml_adapter) - self.assertFalse(result) - - -class PidProviderXMLAddDataForRegularArticleTest(TestCase): - def setUp(self): - user = User() - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - self.registered = models.PidProviderXML() - self.registered._add_data(xml_adapter, user) - - def test_v3(self): - self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", self.registered.v3) - - def test_v2(self): - self.assertEqual("S1413-41522020000400627", self.registered.v2) - - def test_aop_pid(self): - self.assertEqual("S1413-41522020005000111", self.registered.aop_pid) - - def test_main_doi(self): - self.assertEqual("10.1590/S1413-4152202020180029", self.registered.main_doi) - - def test_fpage(self): - self.assertEqual("627", self.registered.fpage) - - def test_fpage_seq(self): - self.assertEqual(None, self.registered.fpage_seq) - - def test_lpage(self): - self.assertEqual("634", self.registered.lpage) - - def test_elocation_id(self): - self.assertEqual(None, self.registered.elocation_id) - - def test_article_pub_year(self): - self.assertEqual("2020", self.registered.article_pub_year) - - def test_z_surnames(self): - self.assertEqual( - "544700df348a47fdd7c55713054e12663a0c530e60e7a166395a496f77de9d36", - self.registered.z_surnames, - ) - - def test_z_collab(self): - self.assertIsNone(self.registered.z_collab) - - def test_z_links(self): - self.assertIsNone(self.registered.z_links) - - def test_z_partial_body(self): - self.assertEqual( - "2e07675bfe91c65e1544ada450ff2e956fef9b492d30e997ebd47687e0f7afa2", - self.registered.z_partial_body, - ) - - -@patch( - "pid_provider.models.utcnow", - side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], -) -@patch("pid_provider.models.XMLVersion.save_file") -@patch("pid_provider.models.XMLVersion.save") -@patch("pid_provider.models.XMLIssue.save") -@patch("pid_provider.models.XMLJournal.save") -@patch("pid_provider.models.PidProviderXML.save") -@patch("pid_provider.models.PidRequest.save") -class PidProviderXMLRegisterTest(TestCase): - def test_register_returns_error( - self, - mock_pid_request_save, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xml_version_save_file, - mock_now, - ): - expected = { - "result_type": "", - "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", - "origin": "filename.xml", - "xml": "
", - } - - user = User() - xml_with_pre = _get_xml_with_pre() - result = models.PidProviderXML.register( - xml_with_pre=xml_with_pre, - filename="filename.xml", - user=user, - ) - print(result) - self.assertEqual(expected["result_type"], result["result_type"]) - self.assertIsNotNone(result["result_msg"]) - # self.assertEqual(expected["result_message"], result["result_msg"]) - self.assertEqual(expected["origin"], result["origin"]) - self.assertEqual(expected["xml"], result["detail"]["xml"]) - mock_pid_provider_xml_save.assert_not_called() - mock_pid_request_save.assert_called_once_with() - - -@patch( - "pid_provider.models.utcnow", - side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], -) -@patch("pid_provider.models.XMLSPS.save") -@patch("pid_provider.models.XMLVersion.save_file") -@patch("pid_provider.models.XMLVersion.save") -@patch("pid_provider.models.XMLIssue.save") -@patch("pid_provider.models.XMLJournal.save") -@patch("pid_provider.models.PidProviderXML.save") -@patch("pid_provider.models.PidRequest.save") -class PidProviderXMLRegisterTest(TestCase): - def test_register_with_success( - self, - mock_pid_request_save, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xml_version_save_file, - mock_xml_sps_save, - mock_now, - ): - expected = { - "result_type": "", - "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", - "origin": "filename.xml", - "xml": "
", - } - - user = User() - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - result = models.PidProviderXML.register( - xml_with_pre=xml_adapter.xml_with_pre, - filename="ex-aop.xml", - user=user, - ) - self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", result["v3"]) - self.assertEqual("S1413-41522020000400627", result["v2"]) - self.assertEqual("S1413-41522020005000111", result["aop_pid"]) - self.assertEqual("1809-4457-esa-25-04-627", result["pkg_name"]) - self.assertEqual(False, result["xml_changed"]) - self.assertEqual("created", result["record_status"]) - self.assertIsNone(result["updated"]) - self.assertIsNotNone(result["created"]) - mock_pid_request_save.assert_not_called() - - -class XMLURLTest(TestCase): - """Tests for XMLURL model""" - - def setUp(self): - self.user = User.objects.create_user(username="testuser", password="testpass") - self.test_url = "http://example.com/article.xml" - self.test_pid = "ABC123XYZ456" - - def test_create_xmlurl(self): - """Test creating a new XMLURL instance""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - self.assertEqual(xmlurl.status, "pending") - self.assertEqual(xmlurl.pid, self.test_pid) - self.assertEqual(xmlurl.creator, self.user) - - def test_get_xmlurl(self): - """Test getting an XMLURL by URL""" - models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xmlurl = models.XMLURL.get(url=self.test_url) - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - - def test_create_or_update_existing(self): - """Test updating an existing XMLURL""" - # Create initial record - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=None, - ) - - # Update it - updated_xmlurl = models.XMLURL.create_or_update( - user=self.user, - url=self.test_url, - status="success", - pid=self.test_pid, - ) - - self.assertEqual(updated_xmlurl.id, xmlurl.id) - self.assertEqual(updated_xmlurl.status, "success") - self.assertEqual(updated_xmlurl.pid, self.test_pid) - self.assertEqual(updated_xmlurl.updated_by, self.user) - - def test_create_or_update_new(self): - """Test creating a new XMLURL when it doesn't exist""" - xmlurl = models.XMLURL.create_or_update( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - self.assertEqual(xmlurl.status, "pending") - - def test_save_file_with_string_content(self): - """Test save_file method with string XML content""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = "
Test Article
" - result = xmlurl.save_file(xml_content, filename="test.xml") - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_save_file_with_bytes_content(self): - """Test save_file method with bytes XML content""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = b"
Test Article
" - result = xmlurl.save_file(xml_content, filename="test.xml") - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_save_file_default_filename(self): - """Test save_file method with default filename""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = "
Test Article
" - result = xmlurl.save_file(xml_content) - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_str_method(self): - """Test __str__ method""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - expected_str = f"{self.test_url} - pending" - self.assertEqual(str(xmlurl), expected_str) - - -class BasePidProviderXMLURITest(TestCase): - """Tests for BasePidProvider.provide_pid_for_xml_uri method""" - - def setUp(self): - self.user = User.objects.create_user(username="testuser", password="testpass") - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - def test_provide_pid_for_xml_uri_fetch_failure(self, mock_create): - """Test exception type a) - Failure to obtain XML""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create to raise an exception - mock_create.side_effect = Exception("Connection timeout") - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article.xml", - name="test.xml", - user=self.user, - ) - - # Should return error details - self.assertIn("error_msg", result) - self.assertIn("error_type", result) - - # Should create XMLURL with failed status - xmlurl = models.XMLURL.get(url="http://example.com/article.xml") - self.assertEqual(xmlurl.status, "xml_fetch_failed") - self.assertIsNone(xmlurl.pid) - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - @patch.object(models.PidProviderXML, "register") - def test_provide_pid_for_xml_uri_success(self, mock_register, mock_create): - """Test successful processing with XMLURL creation""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create - xml_with_pre = _get_xml_with_pre("
Test
") - mock_create.return_value = [xml_with_pre] - - # Mock successful registration - mock_register.return_value = { - "v3": "test_v3_pid", - "v2": "test_v2_pid", - "created": datetime.now(), - } - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article.xml", - name="test.xml", - user=self.user, - ) - - # Should return success response - self.assertEqual(result.get("v3"), "test_v3_pid") - - # Should create XMLURL with success status - xmlurl = models.XMLURL.get(url="http://example.com/article.xml") - self.assertEqual(xmlurl.status, "success") - self.assertEqual(xmlurl.pid, "test_v3_pid") - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - @patch.object(models.PidProviderXML, "register") - def test_provide_pid_for_xml_uri_registration_failure(self, mock_register, mock_create): - """Test exception type b) - XML obtained but registration failed""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create - xml_with_pre = _get_xml_with_pre("
Test
") - mock_create.return_value = [xml_with_pre] - - # Mock failed registration - mock_register.return_value = { - "error_type": "ValidationError", - "error_message": "Invalid XML structure", - "v3": "test_v3_pid", - } - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article2.xml", - name="test2.xml", - user=self.user, - ) - - # Should return error response - self.assertIn("error_type", result) - - # Should create XMLURL with failed status and save zipfile - xmlurl = models.XMLURL.get(url="http://example.com/article2.xml") - self.assertEqual(xmlurl.status, "pid_provider_xml_failed") - self.assertEqual(xmlurl.pid, "test_v3_pid") From 92d27e279ed7cc04cba78912d53d3e6cae675d70 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Fri, 3 Jul 2026 17:48:02 -0300 Subject: [PATCH 13/13] Draft --- pid_provider/models.py | 110 ++++++++++++++++++++--------------- pid_provider/query_params.py | 4 +- requirements/base.txt | 2 +- 3 files changed, 66 insertions(+), 50 deletions(-) diff --git a/pid_provider/models.py b/pid_provider/models.py index 37fe6662c..8689b0d72 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -103,26 +103,52 @@ def xml_directory_path(instance, filename): return f"pid_provider/{subdir_sps_pkg_name}/{filename}" +def get_collection_from_article_id_authority(article_id_authority): + article_id_authority = (article_id_authority or "").split("-")[-1] + if article_id_authority: + try: + return Collection.objects.get( + acronym__iexact=article_id_authority.lower() + ) + except Collection.DoesNotExist: + return None + return None + + class XMLVersion(CommonControlField): """ Tem função de guardar a versão do XML """ - pid_provider_xml = models.ForeignKey( "PidProviderXML", null=True, blank=True, on_delete=models.SET_NULL ) file = models.FileField(upload_to=xml_directory_path, null=True, blank=True, max_length=300) finger_print = models.CharField(max_length=64, null=True, blank=True) + body_fragment_fingerprint = models.CharField(_("body fragment fingerprint"), max_length=300, null=True, blank=True) + + pkg_name = models.CharField(_("Package name"), max_length=100, null=True, blank=True) + pid_v3 = models.CharField(_("pid_v3"), max_length=23, null=True, blank=True) + pid_v2 = models.CharField(_("pid_v2"), max_length=24, null=True, blank=True) + aop_pid = models.CharField(_("AOP PID"), max_length=64, null=True, blank=True) + collection = models.ForeignKey("Collection", null=True, blank=True, on_delete=models.SET_NULL) class Meta: ordering = ["-created"] indexes = [ models.Index(fields=["pid_provider_xml"]), models.Index(fields=["finger_print"]), + models.Index(fields=["pid_v2"]), + models.Index(fields=["collection"]), + models.Index(fields=["pid_v3"]), + models.Index(fields=["aop_pid"]), + models.Index(fields=["pkg_name"]), + models.Index(fields=["body_fragment_fingerprint"]), ] def __str__(self): - return f"{self.pid_provider_xml.pkg_name} {self.created}" + if self.collection and self.pkg_name and self.created: + return f"{self.collection} {self.pkg_name} {self.created}" + return f"{self.pid_provider_xml} {self.created}" @classmethod @profile_classmethod @@ -131,11 +157,21 @@ def create( user, pid_provider_xml, xml_with_pre, + collection, ): + finger_print = xml_with_pre.finger_print + body_fragment_fingerprint = xml_with_pre.body_fragment_fingerprint try: obj = cls() obj.pid_provider_xml = pid_provider_xml - obj.finger_print = xml_with_pre.finger_print + obj.collection = collection + obj.finger_print = finger_print + obj.body_fragment_fingerprint = body_fragment_fingerprint + obj.pid_v3 = xml_with_pre.v3 + obj.pid_v2 = xml_with_pre.v2 + obj.collection = get_collection_from_article_id_authority(xml_with_pre.v2_authority) + obj.aop_pid = xml_with_pre.aop_pid + obj.pkg_name = xml_with_pre.pkg_name obj.creator = user obj.save() obj.save_file( @@ -144,7 +180,7 @@ def create( obj.save() return obj except IntegrityError: - return cls.get(pid_provider_xml, xml_with_pre.finger_print) + return cls.get(pid_provider_xml, finger_print, body_fragment_fingerprint) def save_file(self, filename, content): try: @@ -181,24 +217,31 @@ def xml(self): @classmethod @profile_classmethod - def get(cls, pid_provider_xml, finger_print): + def get(cls, pid_provider_xml, finger_print=None, body_fragment_fingerprint=None): """ Retorna última versão se finger_print corresponde """ - if not pid_provider_xml or not finger_print: + if not pid_provider_xml or not (body_fragment_fingerprint or finger_print): raise XMLVersionGetError( "XMLVersion.get requires pid_provider_xml and xml_with_pre parameters" ) # .latest() já levanta DoesNotExist se vazio + if body_fragment_fingerprint: + return cls.objects.filter( + pid_provider_xml=pid_provider_xml, body_fragment_fingerprint=body_fragment_fingerprint + ).latest("created") + return cls.objects.filter( - pid_provider_xml=pid_provider_xml, finger_print=finger_print + pid_provider_xml=pid_provider_xml, finger_print=finger_print, ).latest("created") @classmethod @profile_classmethod def get_or_create(cls, user, pid_provider_xml, xml_with_pre): try: - latest = cls.get(pid_provider_xml, xml_with_pre.finger_print) + finger_print = xml_with_pre.finger_print + body_fragment_fingerprint = xml_with_pre.body_fragment_fingerprint + latest = cls.get(pid_provider_xml, finger_print, body_fragment_fingerprint) try: file_exist = os.path.isfile(latest.file.path) except (AttributeError, TypeError, ValueError) as e: @@ -216,6 +259,7 @@ def get_or_create(cls, user, pid_provider_xml, xml_with_pre): user=user, pid_provider_xml=pid_provider_xml, xml_with_pre=xml_with_pre, + collection=get_collection_from_article_id_authority(xml_with_pre.v2_authority) ) @@ -395,7 +439,7 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): z_links = models.CharField(_("links"), max_length=64, null=True, blank=True) z_partial_body = models.CharField( _("partial_body"), max_length=64, null=True, blank=True - ) + ) # data de atualização / criação do registro fonte origin_date = models.CharField( _("Origin date"), max_length=10, null=True, blank=True @@ -414,8 +458,6 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): _("Readable data"), null=True, blank=True ) - - base_form_class = CoreAdminModelForm panel_a = [ @@ -438,10 +480,10 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): InlinePanel("other_pid", label=_("Other PID")), ] panel_c = [ - FieldPanel("z_surnames"), - FieldPanel("z_collab"), - FieldPanel("z_links"), - FieldPanel("z_partial_body"), + FieldPanel("z_surnames", read_only=True), + FieldPanel("z_collab", read_only=True), + FieldPanel("z_links", read_only=True), + FieldPanel("z_partial_body", read_only=True), FieldPanel("readable_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), ] @@ -626,35 +668,18 @@ def is_aop(self): return False return True - @staticmethod - def build_readable_data(xml_with_pre): - try: - persons = xml_with_pre.authors.get("person") or [] - surnames = [p.get("surname") for p in persons if p.get("surname")] - except Exception: - surnames = [] - partial = (xml_with_pre.partial_body or "")[: PARTIAL_BODY_MAX] - return { - "surnames": surnames, - "collab": xml_with_pre.collab, - "links": xml_with_pre.links, - "article_titles": xml_with_pre.article_titles_texts, - "partial_body": partial or None, - } - @property def data_to_compare(self): readable = self.readable_data or {} titles = readable.get("article_titles") - if titles is None: - # legado: sem readable_data -> recai no XML (caro, lê arquivo) - titles = self.xml_with_pre.article_titles_texts + body_fragment = readable.get("body_fragment") return { - "title": titles, + "article_titles": titles or self.xml_with_pre.article_titles_texts, "z_surnames": self.z_surnames, "z_collab": self.z_collab, "z_links": self.z_links, "z_partial_body": self.z_partial_body, + "body_fragment": body_fragment or self.xml_with_pre.get_body_fragment(300), } @classmethod @@ -683,15 +708,12 @@ def register( try: response["input_data"] = xml_with_pre.data response["input_data"].update({"origin": origin}) - print(f"response: {response}") xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) response["xml_adapter_data"] = xml_adapter.data - print(f"response: {response}") # dados legíveis do XML entrando (mesmo formato do readable_data) - readable_input = cls.build_readable_data(xml_with_pre) - print(f"readable_input: {readable_input}") + readable_input = xml_with_pre.get_article_data() pkg_name = xml_adapter.sps_pkg_name # consulta se documento já está registrado @@ -1011,13 +1033,7 @@ def get_record_by_pid_v3(cls, xml_adapter): @classmethod def best_matches(cls, results, xml_adapter): # results agora é uma LISTA materializada (não queryset) - input_data = { - "title": xml_adapter.xml_with_pre.article_titles_texts, - "z_surnames": xml_adapter.z_surnames, - "z_collab": xml_adapter.z_collab, - "z_links": xml_adapter.z_links, - "z_partial_body": xml_adapter.z_partial_body, - } + input_data = xml_adapter.get_data_to_compare() detail = { "total_results": len(results), "input_data": input_data, @@ -1074,7 +1090,7 @@ def _add_data(self, xml_adapter, registered_in_core): self.z_partial_body = xml_adapter.z_partial_body # NOVO: dados legíveis (somente inspeção) - self.readable_data = self.build_readable_data(xml_adapter.xml_with_pre) + self.readable_data = xml_adapter.xml_with_pre.get_article_data() @profile_method diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index 79d786968..d25297e20 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -45,7 +45,7 @@ def compare_items(label, registered, input_data): elif (input_data or None) == (registered or None): score = 1 else: - score = 0 + score = how_similar(input_data, registered) response = {"label": label, "score": score} if score != 1: response["registered"] = registered @@ -90,7 +90,7 @@ def __init__(self, xml_adapter): self.xml_adapter = xml_adapter # ========== Cached Properties para Atributos do XML Adapter ========== - + @cached_property def v3(self): """PID v3 do documento.""" diff --git a/requirements/base.txt b/requirements/base.txt index ff4a4eea1..0bb791a54 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -90,7 +90,7 @@ iso639-lang==2.6.3 # Mantendo versão maior # SciELO Specific Packages # ======================================== # Using specific versions for stability --e git+https://github.com/scieloorg/packtools.git@4.16.4#egg=packtools +-e git+https://github.com/scieloorg/packtools.git@4.16.6#egg=packtools -e git+https://github.com/scieloorg/scielo_scholarly_data#egg=scielo_scholarly_data -e git+https://github.com/scieloorg/opac_schema.git@v2.66#egg=opac_schema -e git+https://github.com/scieloorg/scielo_migration.git@1.10.8#egg=scielo_classic_website