Coverage for sites/ptf_tools/ptf_tools/views/base_views.py: 20%
1574 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-05-19 19:20 +0000
1import io
2import json
3import os
4import re
5from datetime import datetime
6from itertools import groupby
8import jsonpickle
9import requests
10from braces.views import CsrfExemptMixin
11from braces.views import LoginRequiredMixin
12from braces.views import StaffuserRequiredMixin
13from celery import Celery
14from celery import current_app
15from django_celery_results.models import TaskResult
16from extra_views import CreateWithInlinesView
17from extra_views import InlineFormSetFactory
18from extra_views import NamedFormsetsMixin
19from extra_views import UpdateWithInlinesView
20from requests import Timeout
22from django.conf import settings
23from django.contrib import messages
24from django.contrib.auth.mixins import UserPassesTestMixin
25from django.db.models import Q
26from django.http import Http404
27from django.http import HttpRequest
28from django.http import HttpResponse
29from django.http import HttpResponseRedirect
30from django.http import HttpResponseServerError
31from django.http import JsonResponse
32from django.shortcuts import get_object_or_404
33from django.shortcuts import redirect
34from django.shortcuts import render
35from django.urls import reverse
36from django.urls import reverse_lazy
37from django.utils import timezone
38from django.views.decorators.http import require_http_methods
39from django.views.generic import ListView
40from django.views.generic import TemplateView
41from django.views.generic import View
42from django.views.generic.base import RedirectView
43from django.views.generic.detail import SingleObjectMixin
44from django.views.generic.edit import CreateView
45from django.views.generic.edit import DeleteView
46from django.views.generic.edit import FormView
47from django.views.generic.edit import UpdateView
49from comments_moderation.utils import get_comments_for_home
50from comments_moderation.utils import is_comment_moderator
51from history import models as history_models
52from history import views as history_views
53from ptf import model_data_converter
54from ptf import model_helpers
55from ptf import tex
56from ptf import utils
57from ptf.cmds import ptf_cmds
58from ptf.cmds import xml_cmds
59from ptf.cmds.base_cmds import make_int
60from ptf.cmds.xml.jats import jats_parser
61from ptf.cmds.xml.xml_utils import replace_html_entities
62from ptf.display import resolver
63from ptf.exceptions import DOIException
64from ptf.exceptions import PDFException
65from ptf.exceptions import ServerUnderMaintenance
66from ptf.model_data import create_issuedata
67from ptf.model_data import create_publisherdata
68from ptf.models import Abstract
69from ptf.models import Article
70from ptf.models import BibItem
71from ptf.models import BibItemId
72from ptf.models import Collection
73from ptf.models import Container
74from ptf.models import ExtId
75from ptf.models import ExtLink
76from ptf.models import Resource
77from ptf.models import ResourceId
78from ptf.views import ArticleEditAPIView
79from ptf_tools.doaj import doaj_pid_register
80from ptf_tools.doi import get_or_create_doibatch
81from ptf_tools.doi import recordDOI
82from ptf_tools.forms import BibItemIdForm
83from ptf_tools.forms import CollectionForm
84from ptf_tools.forms import ContainerForm
85from ptf_tools.forms import DiffContainerForm
86from ptf_tools.forms import ExtIdForm
87from ptf_tools.forms import ExtLinkForm
88from ptf_tools.forms import FormSetHelper
89from ptf_tools.forms import ImportArticleForm
90from ptf_tools.forms import ImportContainerForm
91from ptf_tools.forms import PtfFormHelper
92from ptf_tools.forms import PtfLargeModalFormHelper
93from ptf_tools.forms import PtfModalFormHelper
94from ptf_tools.forms import RegisterPubmedForm
95from ptf_tools.forms import ResourceIdForm
96from ptf_tools.forms import get_article_choices
97from ptf_tools.models import ResourceInNumdam
98from ptf_tools.tasks import archive_numdam_collection
99from ptf_tools.tasks import archive_numdam_issue
100from ptf_tools.tasks import archive_trammel_collection
101from ptf_tools.tasks import archive_trammel_resource
102from ptf_tools.templatetags.tools_helpers import get_authorized_collections
103from ptf_tools.utils import is_authorized_editor
104from pubmed.views import recordPubmed
107def view_404(request: HttpRequest):
108 """
109 Dummy view raising HTTP 404 exception.
110 """
111 raise Http404
114def check_collection(collection, server_url, server_type):
115 """
116 Check if a collection exists on a serveur (test/prod)
117 and upload the collection (XML, image) if necessary
118 """
120 url = server_url + reverse("collection_status", kwargs={"colid": collection.pid})
121 response = requests.get(url, verify=False)
122 # First, upload the collection XML
123 xml = ptf_cmds.exportPtfCmd({"pid": collection.pid}).do()
124 body = xml.encode("utf8")
126 url = server_url + reverse("upload-serials")
127 if response.status_code == 200:
128 # PUT http verb is used for update
129 response = requests.put(url, data=body, verify=False)
130 else:
131 # POST http verb is used for creation
132 response = requests.post(url, data=body, verify=False)
134 # Second, copy the collection images
135 # There is no need to copy files for the test server
136 # Files were already copied in /mersenne_test_data during the ptf_tools import
137 # We only need to copy files from /mersenne_test_data to
138 # /mersenne_prod_data during an upload to prod
139 if server_type == "website":
140 resolver.copy_binary_files(
141 collection, settings.MERSENNE_TEST_DATA_FOLDER, settings.MERSENNE_PROD_DATA_FOLDER
142 )
143 elif server_type == "numdam":
144 from_folder = settings.MERSENNE_PROD_DATA_FOLDER
145 if collection.pid in settings.NUMDAM_COLLECTIONS:
146 from_folder = settings.MERSENNE_TEST_DATA_FOLDER
148 resolver.copy_binary_files(collection, from_folder, settings.NUMDAM_DATA_ROOT)
151def check_lock():
152 return hasattr(settings, "LOCK_FILE") and os.path.isfile(settings.LOCK_FILE)
155def load_cedrics_article_choices(request):
156 colid = request.GET.get("colid")
157 issue = request.GET.get("issue")
158 article_choices = get_article_choices(colid, issue)
159 return render(
160 request, "cedrics_article_dropdown_list_options.html", {"article_choices": article_choices}
161 )
164class ImportCedricsArticleFormView(FormView):
165 template_name = "import_article.html"
166 form_class = ImportArticleForm
168 def dispatch(self, request, *args, **kwargs):
169 self.colid = self.kwargs["colid"]
170 return super().dispatch(request, *args, **kwargs)
172 def get_success_url(self):
173 if self.colid:
174 return reverse("collection-detail", kwargs={"pid": self.colid})
175 return "/"
177 def get_context_data(self, **kwargs):
178 context = super().get_context_data(**kwargs)
179 context["colid"] = self.colid
180 context["helper"] = PtfModalFormHelper
181 return context
183 def get_form_kwargs(self):
184 kwargs = super().get_form_kwargs()
185 kwargs["colid"] = self.colid
186 return kwargs
188 def form_valid(self, form):
189 self.issue = form.cleaned_data["issue"]
190 self.article = form.cleaned_data["article"]
191 return super().form_valid(form)
193 def import_cedrics_article(self, *args, **kwargs):
194 cmd = xml_cmds.addorUpdateCedricsArticleXmlCmd(
195 {"container_pid": self.issue_pid, "article_folder_name": self.article_pid}
196 )
197 cmd.do()
199 def post(self, request, *args, **kwargs):
200 self.colid = self.kwargs.get("colid", None)
201 issue = request.POST["issue"]
202 self.article_pid = request.POST["article"]
203 self.issue_pid = os.path.basename(os.path.dirname(issue))
205 import_args = [self]
206 import_kwargs = {}
208 try:
209 _, status, message = history_views.execute_and_record_func(
210 "import",
211 f"{self.issue_pid} / {self.article_pid}",
212 self.colid,
213 self.import_cedrics_article,
214 "",
215 False,
216 *import_args,
217 **import_kwargs,
218 )
220 messages.success(
221 self.request, f"L'article {self.article_pid} a été importé avec succès"
222 )
224 except Exception as exception:
225 messages.error(
226 self.request,
227 f"Echec de l'import de l'article {self.article_pid} : {str(exception)}",
228 )
230 return redirect(self.get_success_url())
233class ImportCedricsIssueView(FormView):
234 template_name = "import_container.html"
235 form_class = ImportContainerForm
237 def dispatch(self, request, *args, **kwargs):
238 self.colid = self.kwargs["colid"]
239 self.to_appear = self.request.GET.get("to_appear", False)
240 return super().dispatch(request, *args, **kwargs)
242 def get_success_url(self):
243 if self.filename:
244 return reverse(
245 "diff_cedrics_issue", kwargs={"colid": self.colid, "filename": self.filename}
246 )
247 return "/"
249 def get_context_data(self, **kwargs):
250 context = super().get_context_data(**kwargs)
251 context["colid"] = self.colid
252 context["helper"] = PtfModalFormHelper
253 return context
255 def get_form_kwargs(self):
256 kwargs = super().get_form_kwargs()
257 kwargs["colid"] = self.colid
258 kwargs["to_appear"] = self.to_appear
259 return kwargs
261 def form_valid(self, form):
262 self.filename = form.cleaned_data["filename"].split("/")[-1]
263 return super().form_valid(form)
266class DiffCedricsIssueView(FormView):
267 template_name = "diff_container_form.html"
268 form_class = DiffContainerForm
269 diffs = None
270 xissue = None
271 xissue_encoded = None
273 def get_success_url(self):
274 return reverse("collection-detail", kwargs={"pid": self.colid})
276 def dispatch(self, request, *args, **kwargs):
277 self.colid = self.kwargs["colid"]
278 # self.filename = self.kwargs['filename']
279 return super().dispatch(request, *args, **kwargs)
281 def get(self, request, *args, **kwargs):
282 self.filename = request.GET["filename"]
283 self.remove_mail = request.GET["remove_email"]
284 self.remove_date_prod = request.GET["remove_date_prod"]
286 try:
287 result, status, message = history_views.execute_and_record_func(
288 "import",
289 os.path.basename(self.filename),
290 self.colid,
291 self.diff_cedrics_issue,
292 "",
293 True,
294 )
295 except Exception as exception:
296 pid = self.filename.split("/")[-1]
297 messages.error(self.request, f"Echec de l'import du volume {pid} : {exception}")
298 return HttpResponseRedirect(self.get_success_url())
300 no_conflict = result[0]
301 self.diffs = result[1]
302 self.xissue = result[2]
304 if no_conflict:
305 # Proceed with the import
306 self.form_valid(self.get_form())
307 return redirect(self.get_success_url())
308 else:
309 # Display the diff template
310 self.xissue_encoded = jsonpickle.encode(self.xissue)
312 return super().get(request, *args, **kwargs)
314 def post(self, request, *args, **kwargs):
315 self.filename = request.POST["filename"]
316 data = request.POST["xissue_encoded"]
317 self.xissue = jsonpickle.decode(data)
319 return super().post(request, *args, **kwargs)
321 def get_context_data(self, **kwargs):
322 context = super().get_context_data(**kwargs)
323 context["colid"] = self.colid
324 context["diff"] = self.diffs
325 context["filename"] = self.filename
326 context["xissue_encoded"] = self.xissue_encoded
327 return context
329 def get_form_kwargs(self):
330 kwargs = super().get_form_kwargs()
331 kwargs["colid"] = self.colid
332 return kwargs
334 def diff_cedrics_issue(self, *args, **kwargs):
335 params = {
336 "colid": self.colid,
337 "input_file": self.filename,
338 "remove_email": self.remove_mail,
339 "remove_date_prod": self.remove_date_prod,
340 "diff_only": True,
341 }
343 if settings.IMPORT_CEDRICS_DIRECTLY:
344 params["is_seminar"] = self.colid in settings.MERSENNE_SEMINARS
345 params["force_dois"] = self.colid not in settings.NUMDAM_COLLECTIONS
346 cmd = xml_cmds.importCedricsIssueDirectlyXmlCmd(params)
347 else:
348 cmd = xml_cmds.importCedricsIssueXmlCmd(params)
350 result = cmd.do()
351 if len(cmd.warnings) > 0 and self.request.user.is_superuser:
352 messages.warning(
353 self.request, message="Balises non parsées lors de l'import : %s" % cmd.warnings
354 )
356 return result
358 def import_cedrics_issue(self, *args, **kwargs):
359 # modify xissue with data_issue if params to override
360 if "import_choice" in kwargs and kwargs["import_choice"] == "1":
361 issue = model_helpers.get_container(self.xissue.pid)
362 if issue:
363 data_issue = model_data_converter.db_to_issue_data(issue)
364 for xarticle in self.xissue.articles:
365 filter_articles = [
366 article for article in data_issue.articles if article.doi == xarticle.doi
367 ]
368 if len(filter_articles) > 0:
369 db_article = filter_articles[0]
370 xarticle.coi_statement = db_article.coi_statement
371 xarticle.kwds = db_article.kwds
372 xarticle.contrib_groups = db_article.contrib_groups
374 params = {
375 "colid": self.colid,
376 "xissue": self.xissue,
377 "input_file": self.filename,
378 }
380 if settings.IMPORT_CEDRICS_DIRECTLY:
381 params["is_seminar"] = self.colid in settings.MERSENNE_SEMINARS
382 params["add_body_html"] = self.colid not in settings.NUMDAM_COLLECTIONS
383 cmd = xml_cmds.importCedricsIssueDirectlyXmlCmd(params)
384 else:
385 cmd = xml_cmds.importCedricsIssueXmlCmd(params)
387 cmd.do()
389 def form_valid(self, form):
390 if "import_choice" in self.kwargs and self.kwargs["import_choice"] == "1":
391 import_kwargs = {"import_choice": form.cleaned_data["import_choice"]}
392 else:
393 import_kwargs = {}
394 import_args = [self]
396 try:
397 _, status, message = history_views.execute_and_record_func(
398 "import",
399 self.xissue.pid,
400 self.kwargs["colid"],
401 self.import_cedrics_issue,
402 "",
403 False,
404 *import_args,
405 **import_kwargs,
406 )
407 except Exception as exception:
408 messages.error(
409 self.request, f"Echec de l'import du volume {self.xissue.pid} : " + str(exception)
410 )
411 return super().form_invalid(form)
413 messages.success(self.request, f"Le volume {self.xissue.pid} a été importé avec succès")
414 return super().form_valid(form)
417class BibtexAPIView(View):
418 def get(self, request, *args, **kwargs):
419 pid = self.kwargs.get("pid", None)
420 all_bibtex = ""
421 if pid:
422 article = model_helpers.get_article(pid)
423 if article:
424 for bibitem in article.bibitem_set.all():
425 bibtex_array = bibitem.get_bibtex()
426 last = len(bibtex_array)
427 i = 1
428 for bibtex in bibtex_array:
429 if i > 1 and i < last:
430 all_bibtex += " "
431 all_bibtex += bibtex + "\n"
432 i += 1
434 data = {"bibtex": all_bibtex}
435 return JsonResponse(data)
438class MatchingAPIView(View):
439 def get(self, request, *args, **kwargs):
440 pid = self.kwargs.get("pid", None)
442 url = settings.MATCHING_URL
443 headers = {"Content-Type": "application/xml"}
445 body = ptf_cmds.exportPtfCmd({"pid": pid, "with_body": False}).do()
447 if settings.DEBUG:
448 print("Issue exported to /tmp/issue.xml")
449 f = open("/tmp/issue.xml", "w")
450 f.write(body.encode("utf8"))
451 f.close()
453 r = requests.post(url, data=body.encode("utf8"), headers=headers)
454 body = r.text.encode("utf8")
455 data = {"status": r.status_code, "message": body[:1000]}
457 if settings.DEBUG:
458 print("Matching received, new issue exported to /tmp/issue1.xml")
459 f = open("/tmp/issue1.xml", "w")
460 text = body
461 f.write(text)
462 f.close()
464 resource = model_helpers.get_resource(pid)
465 obj = resource.cast()
466 colid = obj.get_collection().pid
468 full_text_folder = settings.CEDRAM_XML_FOLDER + colid + "/plaintext/"
470 cmd = xml_cmds.addOrUpdateIssueXmlCmd(
471 {"body": body, "assign_doi": True, "full_text_folder": full_text_folder}
472 )
473 cmd.do()
475 print("Matching finished")
476 return JsonResponse(data)
479class ImportAllAPIView(View):
480 def internal_do(self, *args, **kwargs):
481 pid = self.kwargs.get("pid", None)
483 root_folder = os.path.join(settings.MATHDOC_ARCHIVE_FOLDER, pid)
484 if not os.path.isdir(root_folder):
485 raise ValueError(root_folder + " does not exist")
487 resource = model_helpers.get_resource(pid)
488 if not resource:
489 file = os.path.join(root_folder, pid + ".xml")
490 body = utils.get_file_content_in_utf8(file)
491 journals = xml_cmds.addCollectionsXmlCmd(
492 {
493 "body": body,
494 "from_folder": settings.MATHDOC_ARCHIVE_FOLDER,
495 "to_folder": settings.MERSENNE_TEST_DATA_FOLDER,
496 }
497 ).do()
498 if not journals:
499 raise ValueError(file + " does not contain a collection")
500 resource = journals[0]
501 # resolver.copy_binary_files(
502 # resource,
503 # settings.MATHDOC_ARCHIVE_FOLDER,
504 # settings.MERSENNE_TEST_DATA_FOLDER)
506 obj = resource.cast()
508 if obj.classname != "Collection":
509 raise ValueError(pid + " does not contain a collection")
511 cmd = xml_cmds.collectEntireCollectionXmlCmd(
512 {"pid": pid, "folder": settings.MATHDOC_ARCHIVE_FOLDER}
513 )
514 pids = cmd.do()
516 return pids
518 def get(self, request, *args, **kwargs):
519 pid = self.kwargs.get("pid", None)
521 try:
522 pids, status, message = history_views.execute_and_record_func(
523 "import", pid, pid, self.internal_do
524 )
525 except Timeout as exception:
526 return HttpResponse(exception, status=408)
527 except Exception as exception:
528 return HttpResponseServerError(exception)
530 data = {"message": message, "ids": pids, "status": status}
531 return JsonResponse(data)
534class DeployAllAPIView(View):
535 def internal_do(self, *args, **kwargs):
536 pid = self.kwargs.get("pid", None)
537 site = self.kwargs.get("site", None)
539 pids = []
541 collection = model_helpers.get_collection(pid)
542 if not collection:
543 raise RuntimeError(pid + " does not exist")
545 if site == "numdam":
546 server_url = settings.NUMDAM_PRE_URL
547 elif site != "ptf_tools":
548 server_url = getattr(collection, site)()
549 if not server_url:
550 raise RuntimeError("The collection has no " + site)
552 if site != "ptf_tools":
553 # check if the collection exists on the server
554 # if not, check_collection will upload the collection (XML,
555 # image...)
556 check_collection(collection, server_url, site)
558 for issue in collection.content.all():
559 if site != "website" or (site == "website" and issue.are_all_articles_published()):
560 pids.append(issue.pid)
562 return pids
564 def get(self, request, *args, **kwargs):
565 pid = self.kwargs.get("pid", None)
566 site = self.kwargs.get("site", None)
568 try:
569 pids, status, message = history_views.execute_and_record_func(
570 "deploy", pid, pid, self.internal_do, site
571 )
572 except Timeout as exception:
573 return HttpResponse(exception, status=408)
574 except Exception as exception:
575 return HttpResponseServerError(exception)
577 data = {"message": message, "ids": pids, "status": status}
578 return JsonResponse(data)
581class AddIssuePDFView(View):
582 def __init(self, *args, **kwargs):
583 super().__init__(*args, **kwargs)
584 self.pid = None
585 self.issue = None
586 self.collection = None
587 self.site = "test_website"
589 def post_to_site(self, url):
590 response = requests.post(url, verify=False)
591 status = response.status_code
592 if not (199 < status < 205):
593 messages.error(self.request, response.text)
594 if status == 503:
595 raise ServerUnderMaintenance(response.text)
596 else:
597 raise RuntimeError(response.text)
599 def internal_do(self, *args, **kwargs):
600 """
601 Called by history_views.execute_and_record_func to do the actual job.
602 """
604 issue_pid = self.issue.pid
605 colid = self.collection.pid
607 if self.site == "website":
608 # Copy the PDF from the test to the production folder
609 resolver.copy_binary_files(
610 self.issue, settings.MERSENNE_TEST_DATA_FOLDER, settings.MERSENNE_PROD_DATA_FOLDER
611 )
612 else:
613 # Copy the PDF from the cedram to the test folder
614 from_folder = resolver.get_cedram_issue_tex_folder(colid, issue_pid)
615 from_path = os.path.join(from_folder, issue_pid + ".pdf")
616 if not os.path.isfile(from_path):
617 raise Http404(f"{from_path} does not exist")
619 to_path = resolver.get_disk_location(
620 settings.MERSENNE_TEST_DATA_FOLDER, colid, "pdf", issue_pid
621 )
622 resolver.copy_file(from_path, to_path)
624 url = reverse("issue_pdf_upload", kwargs={"pid": self.issue.pid})
626 if self.site == "test_website":
627 # Post to ptf-tools: it will add a Datastream to the issue
628 absolute_url = self.request.build_absolute_uri(url)
629 self.post_to_site(absolute_url)
631 server_url = getattr(self.collection, self.site)()
632 absolute_url = server_url + url
633 # Post to the test or production website
634 self.post_to_site(absolute_url)
636 def get(self, request, *args, **kwargs):
637 """
638 Send an issue PDF to the test or production website
639 :param request: pid (mandatory), site (optional) "test_website" (default) or 'website'
640 :param args:
641 :param kwargs:
642 :return:
643 """
644 if check_lock():
645 m = "Trammel is under maintenance. Please try again later."
646 messages.error(self.request, m)
647 return JsonResponse({"message": m, "status": 503})
649 self.pid = self.kwargs.get("pid", None)
650 self.site = self.kwargs.get("site", "test_website")
652 self.issue = model_helpers.get_container(self.pid)
653 if not self.issue:
654 raise Http404(f"{self.pid} does not exist")
655 self.collection = self.issue.get_top_collection()
657 try:
658 pids, status, message = history_views.execute_and_record_func(
659 "deploy",
660 self.pid,
661 self.collection.pid,
662 self.internal_do,
663 f"add issue PDF to {self.site}",
664 )
666 except Timeout as exception:
667 return HttpResponse(exception, status=408)
668 except Exception as exception:
669 return HttpResponseServerError(exception)
671 data = {"message": message, "status": status}
672 return JsonResponse(data)
675class ArchiveAllAPIView(View):
676 """
677 - archive le xml de la collection ainsi que les binaires liés
678 - renvoie une liste de pid des issues de la collection qui seront ensuite archivés par appel JS
679 @return array of issues pid
680 """
682 def internal_do(self, *args, **kwargs):
683 collection = kwargs["collection"]
684 pids = []
685 colid = collection.pid
687 logfile = os.path.join(settings.LOG_DIR, "archive.log")
688 if os.path.isfile(logfile):
689 os.remove(logfile)
691 ptf_cmds.exportPtfCmd(
692 {
693 "pid": colid,
694 "export_folder": settings.MATHDOC_ARCHIVE_FOLDER,
695 "with_binary_files": True,
696 "for_archive": True,
697 "binary_files_folder": settings.MERSENNE_PROD_DATA_FOLDER,
698 }
699 ).do()
701 cedramcls = os.path.join(settings.CEDRAM_TEX_FOLDER, "cedram.cls")
702 if os.path.isfile(cedramcls):
703 dest_folder = os.path.join(settings.MATHDOC_ARCHIVE_FOLDER, collection.pid, "src/tex")
704 resolver.create_folder(dest_folder)
705 resolver.copy_file(cedramcls, dest_folder)
707 for issue in collection.content.all():
708 qs = issue.article_set.filter(
709 date_online_first__isnull=True, date_published__isnull=True
710 )
711 if qs.count() == 0:
712 pids.append(issue.pid)
714 return pids
716 def get(self, request, *args, **kwargs):
717 pid = self.kwargs.get("pid", None)
719 collection = model_helpers.get_collection(pid)
720 if not collection:
721 return HttpResponse(f"{pid} does not exist", status=400)
723 dict_ = {"collection": collection}
724 args_ = [self]
726 try:
727 pids, status, message = history_views.execute_and_record_func(
728 "archive", pid, pid, self.internal_do, "", False, *args_, **dict_
729 )
730 except Timeout as exception:
731 return HttpResponse(exception, status=408)
732 except Exception as exception:
733 return HttpResponseServerError(exception)
735 data = {"message": message, "ids": pids, "status": status}
736 return JsonResponse(data)
739class CreateAllDjvuAPIView(View):
740 def internal_do(self, *args, **kwargs):
741 issue = kwargs["issue"]
742 pids = [issue.pid]
744 for article in issue.article_set.all():
745 pids.append(article.pid)
747 return pids
749 def get(self, request, *args, **kwargs):
750 pid = self.kwargs.get("pid", None)
751 issue = model_helpers.get_container(pid)
752 if not issue:
753 raise Http404(f"{pid} does not exist")
755 try:
756 dict_ = {"issue": issue}
757 args_ = [self]
759 pids, status, message = history_views.execute_and_record_func(
760 "numdam",
761 pid,
762 issue.get_collection().pid,
763 self.internal_do,
764 "",
765 False,
766 *args_,
767 **dict_,
768 )
769 except Exception as exception:
770 return HttpResponseServerError(exception)
772 data = {"message": message, "ids": pids, "status": status}
773 return JsonResponse(data)
776class ImportJatsContainerAPIView(View):
777 def internal_do(self, *args, **kwargs):
778 pid = self.kwargs.get("pid", None)
779 colid = self.kwargs.get("colid", None)
781 if pid and colid:
782 body = resolver.get_archive_body(settings.MATHDOC_ARCHIVE_FOLDER, colid, pid)
784 cmd = xml_cmds.addOrUpdateContainerXmlCmd(
785 {
786 "body": body,
787 "from_folder": settings.MATHDOC_ARCHIVE_FOLDER,
788 "to_folder": settings.MERSENNE_TEST_DATA_FOLDER,
789 "backup_folder": settings.MATHDOC_ARCHIVE_FOLDER,
790 }
791 )
792 container = cmd.do()
793 if len(cmd.warnings) > 0:
794 messages.warning(
795 self.request,
796 message="Balises non parsées lors de l'import : %s" % cmd.warnings,
797 )
799 if not container:
800 raise RuntimeError("Error: the container " + pid + " was not imported")
802 # resolver.copy_binary_files(
803 # container,
804 # settings.MATHDOC_ARCHIVE_FOLDER,
805 # settings.MERSENNE_TEST_DATA_FOLDER)
806 #
807 # for article in container.article_set.all():
808 # resolver.copy_binary_files(
809 # article,
810 # settings.MATHDOC_ARCHIVE_FOLDER,
811 # settings.MERSENNE_TEST_DATA_FOLDER)
812 else:
813 raise RuntimeError("colid or pid are not defined")
815 def get(self, request, *args, **kwargs):
816 pid = self.kwargs.get("pid", None)
817 colid = self.kwargs.get("colid", None)
819 try:
820 _, status, message = history_views.execute_and_record_func(
821 "import", pid, colid, self.internal_do
822 )
823 except Timeout as exception:
824 return HttpResponse(exception, status=408)
825 except Exception as exception:
826 return HttpResponseServerError(exception)
828 data = {"message": message, "status": status}
829 return JsonResponse(data)
832class DeployCollectionAPIView(View):
833 # Update collection.xml on a site (with its images)
835 def internal_do(self, *args, **kwargs):
836 colid = self.kwargs.get("colid", None)
837 site = self.kwargs.get("site", None)
839 collection = model_helpers.get_collection(colid)
840 if not collection:
841 raise RuntimeError(f"{colid} does not exist")
843 if site == "numdam":
844 server_url = settings.NUMDAM_PRE_URL
845 else:
846 server_url = getattr(collection, site)()
847 if not server_url:
848 raise RuntimeError(f"The collection has no {site}")
850 # check_collection creates or updates the collection (XML, image...)
851 check_collection(collection, server_url, site)
853 def get(self, request, *args, **kwargs):
854 colid = self.kwargs.get("colid", None)
855 site = self.kwargs.get("site", None)
857 try:
858 _, status, message = history_views.execute_and_record_func(
859 "deploy", colid, colid, self.internal_do, site
860 )
861 except Timeout as exception:
862 return HttpResponse(exception, status=408)
863 except Exception as exception:
864 return HttpResponseServerError(exception)
866 data = {"message": message, "status": status}
867 return JsonResponse(data)
870class DeployJatsResourceAPIView(View):
871 # A RENOMMER aussi DeleteJatsContainerAPIView (mais fonctionne tel quel)
873 def internal_do(self, *args, **kwargs):
874 pid = self.kwargs.get("pid", None)
875 colid = self.kwargs.get("colid", None)
876 site = self.kwargs.get("site", None)
878 if site == "ptf_tools":
879 raise RuntimeError("Do not choose to deploy on PTF Tools")
881 resource = model_helpers.get_resource(pid)
882 if not resource:
883 raise RuntimeError(f"{pid} does not exist")
885 obj = resource.cast()
886 article = None
887 if obj.classname == "Article":
888 article = obj
889 container = article.my_container
890 articles_to_deploy = [article]
891 else:
892 container = obj
893 articles_to_deploy = container.article_set.exclude(do_not_publish=True)
895 if site == "website" and article is not None and article.do_not_publish:
896 raise RuntimeError(f"{pid} is marked as Do not publish")
897 if site == "numdam" and article is not None:
898 raise RuntimeError("You can only deploy issues to Numdam")
900 collection = container.get_top_collection()
901 colid = collection.pid
902 djvu_exception = None
904 if site == "numdam":
905 server_url = settings.NUMDAM_PRE_URL
906 ResourceInNumdam.objects.get_or_create(pid=container.pid)
908 # 06/12/2022: DjVu are no longer added with Mersenne articles
909 # Add Djvu (before exporting the XML)
910 if False and int(container.year) < 2020:
911 for art in container.article_set.all():
912 try:
913 cmd = ptf_cmds.addDjvuPtfCmd()
914 cmd.set_resource(art)
915 cmd.do()
916 except Exception as e:
917 # Djvu are optional.
918 # Allow the deployment, but record the exception in the history
919 djvu_exception = e
920 else:
921 server_url = getattr(collection, site)()
922 if not server_url:
923 raise RuntimeError(f"The collection has no {site}")
925 # check if the collection exists on the server
926 # if not, check_collection will upload the collection (XML,
927 # image...)
928 if article is None:
929 check_collection(collection, server_url, site)
931 with open(os.path.join(settings.LOG_DIR, "cmds.log"), "w", encoding="utf-8") as file_:
932 # Create/update deployed date and published date on all container articles
933 if site == "website":
934 file_.write(
935 "Create/Update deployed_date and date_published on all articles for {}\n".format(
936 pid
937 )
938 )
940 # create date_published on articles without date_published (ou date_online_first pour le volume 0)
941 cmd = ptf_cmds.publishResourcePtfCmd()
942 cmd.set_resource(resource)
943 updated_articles = cmd.do()
945 tex.create_frontpage(colid, container, updated_articles, test=False)
947 mersenneSite = model_helpers.get_site_mersenne(colid)
948 # create or update deployed_date on container and articles
949 model_helpers.update_deployed_date(obj, mersenneSite, None, file_)
951 for art in articles_to_deploy:
952 if art.doi and (art.date_published or art.date_online_first):
953 if art.my_container.year is None:
954 art.my_container.year = datetime.now().strftime("%Y")
955 # BUG ? update the container but no save() ?
957 file_.write(
958 "Publication date of {} : Online First: {}, Published: {}\n".format(
959 art.pid, art.date_online_first, art.date_published
960 )
961 )
962 elif site == "test_website":
963 # create date_pre_published on articles without date_pre_published
964 cmd = ptf_cmds.publishResourcePtfCmd({"pre_publish": True})
965 cmd.set_resource(resource)
966 updated_articles = cmd.do()
968 tex.create_frontpage(colid, container, updated_articles)
970 export_to_website = site == "website"
972 if article is None:
973 with_djvu = site == "numdam"
974 xml = ptf_cmds.exportPtfCmd(
975 {"pid": pid, "with_djvu": with_djvu, "export_to_website": export_to_website}
976 ).do()
977 body = xml.encode("utf8")
979 # if container = issue
980 if container.ctype == "issue":
981 url = server_url + reverse("issue_upload")
982 else:
983 url = server_url + reverse("book_upload")
985 # verify=False: ignore TLS certificate
986 response = requests.post(url, data=body, verify=False)
987 else:
988 xml = ptf_cmds.exportPtfCmd(
989 {
990 "pid": pid,
991 "with_djvu": False,
992 "article_standalone": True,
993 "collection_pid": collection.pid,
994 "export_to_website": export_to_website,
995 "export_folder": settings.LOG_DIR,
996 }
997 ).do()
998 # Unlike containers that send their XML as the body of the POST request,
999 # articles send their XML as a file, because PCJ editor sends multiple files (XML, PDF, img)
1000 xml_file = io.StringIO(xml)
1001 files = {"xml": xml_file}
1003 url = server_url + reverse(
1004 "article_in_issue_upload", kwargs={"pid": container.pid}
1005 )
1006 # verify=False: ignore TLS certificate
1007 header = {}
1008 response = requests.post(url, headers=header, files=files, verify=False)
1010 status = response.status_code
1012 if 199 < status < 205:
1013 # There is no need to copy files for the test server
1014 # Files were already copied in /mersenne_test_data during the ptf_tools import
1015 # We only need to copy files from /mersenne_test_data to
1016 # /mersenne_prod_data during an upload to prod
1017 if site == "website":
1018 if article is None:
1019 resolver.copy_binary_files(
1020 container,
1021 settings.MERSENNE_TEST_DATA_FOLDER,
1022 settings.MERSENNE_PROD_DATA_FOLDER,
1023 )
1025 for art in articles_to_deploy:
1026 resolver.copy_binary_files(
1027 art,
1028 settings.MERSENNE_TEST_DATA_FOLDER,
1029 settings.MERSENNE_PROD_DATA_FOLDER,
1030 )
1031 # record DOI automatically when deploying in prod
1033 if art.doi and art.allow_crossref():
1034 recordDOI(art)
1036 if colid == "CRBIOL":
1037 recordPubmed(
1038 art, force_update=False, updated_articles=updated_articles
1039 )
1041 if colid == "PCJ":
1042 self.update_pcj_editor(updated_articles)
1044 # Archive the container or the article
1045 if article is None:
1046 archive_trammel_resource.delay(
1047 colid=colid,
1048 pid=pid,
1049 mathdoc_archive=settings.MATHDOC_ARCHIVE_FOLDER,
1050 binary_files_folder=settings.MERSENNE_PROD_DATA_FOLDER,
1051 )
1052 else:
1053 archive_trammel_resource.delay(
1054 colid=colid,
1055 pid=pid,
1056 mathdoc_archive=settings.MATHDOC_ARCHIVE_FOLDER,
1057 binary_files_folder=settings.MERSENNE_PROD_DATA_FOLDER,
1058 article_doi=article.doi,
1059 )
1060 # cmd = ptf_cmds.archiveIssuePtfCmd({
1061 # "pid": pid,
1062 # "export_folder": settings.MATHDOC_ARCHIVE_FOLDER,
1063 # "binary_files_folder": settings.MERSENNE_PROD_DATA_FOLDER})
1064 # cmd.set_article(article) # set_article allows archiving only the article
1065 # cmd.do()
1067 elif site == "numdam":
1068 from_folder = settings.MERSENNE_PROD_DATA_FOLDER
1069 if colid in settings.NUMDAM_COLLECTIONS:
1070 from_folder = settings.MERSENNE_TEST_DATA_FOLDER
1072 resolver.copy_binary_files(container, from_folder, settings.NUMDAM_DATA_ROOT)
1073 for article in container.article_set.all():
1074 resolver.copy_binary_files(article, from_folder, settings.NUMDAM_DATA_ROOT)
1076 elif status == 503:
1077 raise ServerUnderMaintenance(response.text)
1078 else:
1079 raise RuntimeError(response.text)
1081 if djvu_exception:
1082 raise djvu_exception
1084 def get(self, request, *args, **kwargs):
1085 pid = self.kwargs.get("pid", None)
1086 colid = self.kwargs.get("colid", None)
1087 site = self.kwargs.get("site", None)
1089 try:
1090 _, status, message = history_views.execute_and_record_func(
1091 "deploy", pid, colid, self.internal_do, site
1092 )
1093 except Timeout as exception:
1094 return HttpResponse(exception, status=408)
1095 except Exception as exception:
1096 return HttpResponseServerError(exception)
1098 data = {"message": message, "status": status}
1099 return JsonResponse(data)
1101 def update_pcj_editor(self, updated_articles):
1102 for article in updated_articles:
1103 data = {
1104 "date_published": article.date_published.strftime("%Y-%m-%d"),
1105 "article_number": article.article_number,
1106 }
1107 url = "http://pcj-editor.u-ga.fr/submit/api-article-publish/" + article.doi + "/"
1108 requests.post(url, json=data, verify=False)
1111class DeployTranslatedArticleAPIView(CsrfExemptMixin, View):
1112 article = None
1114 def internal_do(self, *args, **kwargs):
1115 lang = self.kwargs.get("lang", None)
1117 translation = None
1118 for trans_article in self.article.translations.all():
1119 if trans_article.lang == lang:
1120 translation = trans_article
1122 if translation is None:
1123 raise RuntimeError(f"{self.article.doi} does not exist in {lang}")
1125 collection = self.article.get_top_collection()
1126 colid = collection.pid
1127 container = self.article.my_container
1129 if translation.date_published is None:
1130 # Add date posted
1131 cmd = ptf_cmds.publishResourcePtfCmd()
1132 cmd.set_resource(translation)
1133 updated_articles = cmd.do()
1135 # Recompile PDF to add the date posted
1136 try:
1137 tex.create_frontpage(colid, container, updated_articles, test=False, lang=lang)
1138 except Exception:
1139 raise PDFException(
1140 "Unable to compile the article PDF. Please contact the centre Mersenne"
1141 )
1143 # Unlike regular articles, binary files of translations need to be copied before uploading the XML.
1144 # The full text in HTML is read by the JATS parser, so the HTML file needs to be present on disk
1145 resolver.copy_binary_files(
1146 self.article, settings.MERSENNE_TEST_DATA_FOLDER, settings.MERSENNE_PROD_DATA_FOLDER
1147 )
1149 # Deploy in prod
1150 xml = ptf_cmds.exportPtfCmd(
1151 {
1152 "pid": self.article.pid,
1153 "with_djvu": False,
1154 "article_standalone": True,
1155 "collection_pid": colid,
1156 "export_to_website": True,
1157 "export_folder": settings.LOG_DIR,
1158 }
1159 ).do()
1160 xml_file = io.StringIO(xml)
1161 files = {"xml": xml_file}
1163 server_url = getattr(collection, "website")()
1164 if not server_url:
1165 raise RuntimeError("The collection has no website")
1166 url = server_url + reverse("article_in_issue_upload", kwargs={"pid": container.pid})
1167 header = {}
1169 try:
1170 response = requests.post(
1171 url, headers=header, files=files, verify=False
1172 ) # verify: ignore TLS certificate
1173 status = response.status_code
1174 except requests.exceptions.ConnectionError:
1175 raise ServerUnderMaintenance(
1176 "The journal is under maintenance. Please try again later."
1177 )
1179 # Register translation in Crossref
1180 if 199 < status < 205:
1181 if self.article.allow_crossref():
1182 try:
1183 recordDOI(translation)
1184 except Exception:
1185 raise DOIException(
1186 "Error while recording the DOI. Please contact the centre Mersenne"
1187 )
1189 def get(self, request, *args, **kwargs):
1190 doi = kwargs.get("doi", None)
1191 self.article = model_helpers.get_article_by_doi(doi)
1192 if self.article is None:
1193 raise Http404(f"{doi} does not exist")
1195 try:
1196 _, status, message = history_views.execute_and_record_func(
1197 "deploy",
1198 self.article.pid,
1199 self.article.get_top_collection().pid,
1200 self.internal_do,
1201 "website",
1202 )
1203 except Timeout as exception:
1204 return HttpResponse(exception, status=408)
1205 except Exception as exception:
1206 return HttpResponseServerError(exception)
1208 data = {"message": message, "status": status}
1209 return JsonResponse(data)
1212class DeleteJatsIssueAPIView(View):
1213 # TODO ? rename in DeleteJatsContainerAPIView mais fonctionne tel quel pour book*
1214 def get(self, request, *args, **kwargs):
1215 pid = self.kwargs.get("pid", None)
1216 colid = self.kwargs.get("colid", None)
1217 site = self.kwargs.get("site", None)
1218 message = "Le volume a bien été supprimé"
1219 status = 200
1221 issue = model_helpers.get_container(pid)
1222 if not issue:
1223 raise Http404(f"{pid} does not exist")
1224 try:
1225 mersenneSite = model_helpers.get_site_mersenne(colid)
1227 if site == "ptf_tools":
1228 if issue.is_deployed(mersenneSite):
1229 issue.undeploy(mersenneSite)
1230 for article in issue.article_set.all():
1231 article.undeploy(mersenneSite)
1233 p = model_helpers.get_provider("mathdoc-id")
1235 cmd = ptf_cmds.addContainerPtfCmd(
1236 {
1237 "pid": issue.pid,
1238 "ctype": "issue",
1239 "to_folder": settings.MERSENNE_TEST_DATA_FOLDER,
1240 }
1241 )
1242 cmd.set_provider(p)
1243 cmd.add_collection(issue.get_collection())
1244 cmd.set_object_to_be_deleted(issue)
1245 cmd.undo()
1247 else:
1248 if site == "numdam":
1249 server_url = settings.NUMDAM_PRE_URL
1250 else:
1251 collection = issue.get_collection()
1252 server_url = getattr(collection, site)()
1254 if not server_url:
1255 message = "The collection has no " + site
1256 status = 500
1257 else:
1258 url = server_url + reverse("issue_delete", kwargs={"pid": pid})
1259 response = requests.delete(url, verify=False)
1260 status = response.status_code
1262 if status == 404:
1263 message = "Le serveur retourne un code 404. Vérifier que le volume soit bien sur le serveur"
1264 elif status > 204:
1265 body = response.text.encode("utf8")
1266 message = body[:1000]
1267 else:
1268 status = 200
1269 # unpublish issue in collection site (site_register.json)
1270 if site == "website":
1271 if issue.is_deployed(mersenneSite):
1272 issue.undeploy(mersenneSite)
1273 for article in issue.article_set.all():
1274 article.undeploy(mersenneSite)
1275 # delete article binary files
1276 folder = article.get_relative_folder()
1277 resolver.delete_object_folder(
1278 folder,
1279 to_folder=settings.MERSENNE_PROD_DATA_FORLDER,
1280 )
1281 # delete issue binary files
1282 folder = issue.get_relative_folder()
1283 resolver.delete_object_folder(
1284 folder, to_folder=settings.MERSENNE_PROD_DATA_FORLDER
1285 )
1287 except Timeout as exception:
1288 return HttpResponse(exception, status=408)
1289 except Exception as exception:
1290 return HttpResponseServerError(exception)
1292 data = {"message": message, "status": status}
1293 return JsonResponse(data)
1296class ArchiveIssueAPIView(View):
1297 def get(self, request, *args, **kwargs):
1298 try:
1299 pid = kwargs["pid"]
1300 colid = kwargs["colid"]
1301 except IndexError:
1302 raise Http404
1304 try:
1305 cmd = ptf_cmds.archiveIssuePtfCmd(
1306 {
1307 "pid": pid,
1308 "export_folder": settings.MATHDOC_ARCHIVE_FOLDER,
1309 "binary_files_folder": settings.MERSENNE_PROD_DATA_FOLDER,
1310 }
1311 )
1312 result_, status, message = history_views.execute_and_record_func(
1313 "archive", pid, colid, cmd.do
1314 )
1315 except Exception as exception:
1316 return HttpResponseServerError(exception)
1318 data = {"message": message, "status": 200}
1319 return JsonResponse(data)
1322class CreateDjvuAPIView(View):
1323 def internal_do(self, *args, **kwargs):
1324 pid = self.kwargs.get("pid", None)
1326 resource = model_helpers.get_resource(pid)
1327 cmd = ptf_cmds.addDjvuPtfCmd()
1328 cmd.set_resource(resource)
1329 cmd.do()
1331 def get(self, request, *args, **kwargs):
1332 pid = self.kwargs.get("pid", None)
1333 colid = pid.split("_")[0]
1335 try:
1336 _, status, message = history_views.execute_and_record_func(
1337 "numdam", pid, colid, self.internal_do
1338 )
1339 except Exception as exception:
1340 return HttpResponseServerError(exception)
1342 data = {"message": message, "status": status}
1343 return JsonResponse(data)
1346class PTFToolsHomeView(LoginRequiredMixin, View):
1347 """
1348 Home Page.
1349 - Admin & staff -> Render blank home.html
1350 - User with unique authorized collection -> Redirect to collection details page
1351 - User with multiple authorized collections -> Render home.html with data
1352 - Comment moderator -> Comments dashboard
1353 - Others -> 404 response
1354 """
1356 def get(self, request, *args, **kwargs) -> HttpResponse:
1357 # Staff or user with authorized collections
1358 if request.user.is_staff or request.user.is_superuser:
1359 return render(request, "home.html")
1361 colids = get_authorized_collections(request.user)
1362 is_mod = is_comment_moderator(request.user)
1364 # The user has no rights
1365 if not (colids or is_mod):
1366 raise Http404("No collections associated with your account.")
1367 # Comment moderator only
1368 elif not colids:
1369 return HttpResponseRedirect(reverse("comment_list"))
1371 # User with unique collection -> Redirect to collection detail page
1372 if len(colids) == 1 or getattr(settings, "COMMENTS_DISABLED", False):
1373 return HttpResponseRedirect(reverse("collection-detail", kwargs={"pid": colids[0]}))
1375 # User with multiple authorized collections - Special home
1376 context = {}
1377 context["overview"] = True
1379 all_collections = Collection.objects.filter(pid__in=colids).values("pid", "title_html")
1380 all_collections = {c["pid"]: c for c in all_collections}
1382 # Comments summary
1383 error, comments_data = get_comments_for_home(request.user)
1384 context["comment_server_ok"] = False
1386 if not error:
1387 context["comment_server_ok"] = True
1388 if comments_data:
1389 for col_id, comment_nb in comments_data.items():
1390 if col_id.upper() in all_collections: 1390 ↛ 1389line 1390 didn't jump to line 1389, because the condition on line 1390 was never false
1391 all_collections[col_id.upper()]["pending_comments"] = comment_nb
1393 # TODO: Translations summary
1394 context["translation_server_ok"] = False
1396 # Sort the collections according to the number of pending comments
1397 context["collections"] = sorted(
1398 all_collections.values(), key=lambda col: col.get("pending_comments", -1), reverse=True
1399 )
1401 return render(request, "home.html", context)
1404class MersenneDashboardView(TemplateView, history_views.HistoryContextMixin):
1405 template_name = "mersenne.html"
1407 def get_context_data(self, **kwargs):
1408 context = super().get_context_data(**kwargs)
1410 containers = []
1411 last_col_events = []
1412 now = timezone.now()
1414 curyear = now.year
1415 published_articles = []
1416 published_articles2 = []
1417 columns = 5
1418 years = range(curyear - columns + 1, curyear + 1)
1420 mersenne_col_ids = settings.MERSENNE_COLLECTIONS
1421 total_published_articles = [
1422 {"year": year, "total_articles": 0, "total_pages": 0} for year in years
1423 ]
1424 total_published_articles2 = [
1425 {"year": year, "total_articles": 0, "total_pages": 0} for year in years
1426 ]
1428 for pid in mersenne_col_ids:
1429 if pid != "MERSENNE":
1430 articles_in_col = []
1431 articles2_in_col = []
1432 for year in years:
1433 articles = Article.objects.filter(
1434 Q(my_container__my_collection__pid=pid)
1435 & (
1436 Q(date_published__year=year, date_online_first__isnull=True)
1437 | Q(date_online_first__year=year)
1438 )
1439 ).prefetch_related("resourcecount_set")
1441 page_count = sum(article.get_article_page_count() for article in articles)
1442 articles_count = articles.count()
1443 articles_in_col.append(
1444 {"year": year, "articles": articles_count, "pages": page_count}
1445 )
1446 total_published_articles[year - curyear + columns - 1][
1447 "total_articles"
1448 ] += articles_count
1449 total_published_articles[year - curyear + +columns - 1][
1450 "total_pages"
1451 ] += page_count
1453 issues = Container.objects.filter(my_collection__pid=pid, year=year)
1454 page_count = 0
1455 articles_count = 0
1456 for issue in issues:
1457 for article in issue.article_set.filter(
1458 Q(date_published__isnull=False) | Q(date_online_first__isnull=False)
1459 ).prefetch_related("resourcecount_set"):
1460 page_count += article.get_article_page_count()
1461 articles_count += 1
1462 articles2_in_col.append(
1463 {"year": year, "articles": articles_count, "pages": page_count}
1464 )
1465 total_published_articles2[year - curyear + 2][
1466 "total_articles"
1467 ] += articles_count
1468 total_published_articles2[year - curyear + 2]["total_pages"] += page_count
1470 published_articles.append({"pid": pid, "years": articles_in_col})
1471 published_articles2.append({"pid": pid, "years": articles2_in_col})
1473 # last_events_types = []
1474 #
1475 # if container:
1476 # containers.append(container)
1477 #
1478 # for type in ['matching', 'edit', 'deploy', 'archive']:
1479 # event = history_models.get_history_last_event_by(type, pid)
1480 # gap = history_models.get_gap(now, event)
1481 #
1482 # last_events_types.append(gap)
1483 #
1484 # last_col_events.append({'pid': pid, 'last_events': last_events_types})
1486 event = history_models.get_history_last_event_by("clockss", "ALL")
1487 clockss_gap = history_models.get_gap(now, event)
1489 context["collections"] = settings.MERSENNE_COLLECTIONS
1490 context["containers_to_be_published"] = containers
1491 context["last_col_events"] = last_col_events
1493 context["years"] = years
1494 context["published_articles"] = published_articles
1495 context["total_published_articles"] = total_published_articles
1496 context["published_articles2"] = published_articles2
1497 context["total_published_articles2"] = total_published_articles2
1499 context["clockss_gap"] = clockss_gap
1501 return context
1504class DOAJResourceRegisterView(View):
1505 def get(self, request, *args, **kwargs):
1506 pid = kwargs.get("pid", None)
1507 resource = model_helpers.get_resource(pid)
1508 if resource is None:
1509 raise Http404
1511 try:
1512 data = {}
1513 doaj_meta, response = doaj_pid_register(pid)
1514 if response is None:
1515 return HttpResponse(status=204)
1516 elif doaj_meta and 200 <= response.status_code <= 299:
1517 data.update(doaj_meta)
1518 else:
1519 return HttpResponse(status=response.status_code, reason=response.text)
1520 except Timeout as exception:
1521 return HttpResponse(exception, status=408)
1522 except Exception as exception:
1523 return HttpResponseServerError(exception)
1524 return JsonResponse(data)
1527class CROSSREFResourceRegisterView(View):
1528 def get(self, request, *args, **kwargs):
1529 pid = kwargs.get("pid", None)
1530 # option force for registering doi of articles without date_published (ex; TSG from Numdam)
1531 force = kwargs.get("force", None)
1532 if not request.user.is_superuser:
1533 force = None
1535 resource = model_helpers.get_resource(pid)
1536 if resource is None:
1537 raise Http404
1539 resource = resource.cast()
1540 meth = getattr(self, "recordDOI" + resource.classname)
1541 try:
1542 data = meth(resource, force)
1543 except Timeout as exception:
1544 return HttpResponse(exception, status=408)
1545 except Exception as exception:
1546 return HttpResponseServerError(exception)
1547 return JsonResponse(data)
1549 def recordDOIArticle(self, article, force=None):
1550 result = {"status": 404}
1551 if (
1552 article.doi
1553 and not article.do_not_publish
1554 and (article.date_published or article.date_online_first or force == "force")
1555 ):
1556 if article.my_container.year is None: # or article.my_container.year == '0':
1557 article.my_container.year = datetime.now().strftime("%Y")
1558 result = recordDOI(article)
1559 return result
1561 def recordDOICollection(self, collection, force=None):
1562 return recordDOI(collection)
1564 def recordDOIContainer(self, container, force=None):
1565 data = {"status": 200, "message": "tout va bien"}
1567 if container.ctype == "issue":
1568 if container.doi:
1569 result = recordDOI(container)
1570 if result["status"] != 200:
1571 return result
1572 if force == "force":
1573 articles = container.article_set.exclude(
1574 doi__isnull=True, do_not_publish=True, date_online_first__isnull=True
1575 )
1576 else:
1577 articles = container.article_set.exclude(
1578 doi__isnull=True,
1579 do_not_publish=True,
1580 date_published__isnull=True,
1581 date_online_first__isnull=True,
1582 )
1584 for article in articles:
1585 result = self.recordDOIArticle(article, force)
1586 if result["status"] != 200:
1587 data = result
1588 else:
1589 return recordDOI(container)
1590 return data
1593class CROSSREFResourceCheckStatusView(View):
1594 def get(self, request, *args, **kwargs):
1595 pid = kwargs.get("pid", None)
1596 resource = model_helpers.get_resource(pid)
1597 if resource is None:
1598 raise Http404
1599 resource = resource.cast()
1600 meth = getattr(self, "checkDOI" + resource.classname)
1601 try:
1602 meth(resource)
1603 except Timeout as exception:
1604 return HttpResponse(exception, status=408)
1605 except Exception as exception:
1606 return HttpResponseServerError(exception)
1608 data = {"status": 200, "message": "tout va bien"}
1609 return JsonResponse(data)
1611 def checkDOIArticle(self, article):
1612 if article.my_container.year is None or article.my_container.year == "0":
1613 article.my_container.year = datetime.now().strftime("%Y")
1614 get_or_create_doibatch(article)
1616 def checkDOICollection(self, collection):
1617 get_or_create_doibatch(collection)
1619 def checkDOIContainer(self, container):
1620 if container.doi is not None:
1621 get_or_create_doibatch(container)
1622 for article in container.article_set.all():
1623 self.checkDOIArticle(article)
1626class RegisterPubmedFormView(FormView):
1627 template_name = "record_pubmed_dialog.html"
1628 form_class = RegisterPubmedForm
1630 def get_context_data(self, **kwargs):
1631 context = super().get_context_data(**kwargs)
1632 context["pid"] = self.kwargs["pid"]
1633 context["helper"] = PtfLargeModalFormHelper
1634 return context
1637class RegisterPubmedView(View):
1638 def get(self, request, *args, **kwargs):
1639 pid = kwargs.get("pid", None)
1640 update_article = self.request.GET.get("update_article", "on") == "on"
1642 article = model_helpers.get_article(pid)
1643 if article is None:
1644 raise Http404
1645 try:
1646 recordPubmed(article, update_article)
1647 except Exception as exception:
1648 messages.error("Unable to register the article in PubMed")
1649 return HttpResponseServerError(exception)
1651 return HttpResponseRedirect(
1652 reverse("issue-items", kwargs={"pid": article.my_container.pid})
1653 )
1656class PTFToolsContainerView(TemplateView):
1657 template_name = ""
1659 def get_context_data(self, **kwargs):
1660 context = super().get_context_data(**kwargs)
1662 container = model_helpers.get_container(self.kwargs.get("pid"))
1663 if container is None:
1664 raise Http404
1665 citing_articles = container.citations()
1666 source = self.request.GET.get("source", None)
1667 if container.ctype.startswith("book"):
1668 book_parts = (
1669 container.article_set.filter(sites__id=settings.SITE_ID).all().order_by("seq")
1670 )
1671 references = False
1672 if container.ctype == "book-monograph":
1673 # on regarde si il y a au moins une bibliographie
1674 for art in container.article_set.all():
1675 if art.bibitem_set.count() > 0:
1676 references = True
1677 context.update(
1678 {
1679 "book": container,
1680 "book_parts": list(book_parts),
1681 "source": source,
1682 "citing_articles": citing_articles,
1683 "references": references,
1684 "test_website": container.get_top_collection()
1685 .extlink_set.get(rel="test_website")
1686 .location,
1687 "prod_website": container.get_top_collection()
1688 .extlink_set.get(rel="website")
1689 .location,
1690 }
1691 )
1692 self.template_name = "book-toc.html"
1693 else:
1694 articles = container.article_set.all().order_by("seq")
1695 for article in articles:
1696 try:
1697 last_match = (
1698 history_models.HistoryEvent.objects.filter(
1699 pid=article.pid,
1700 type="matching",
1701 )
1702 .only("created_on")
1703 .latest("created_on")
1704 )
1705 except history_models.HistoryEvent.DoesNotExist as _:
1706 article.last_match = None
1707 else:
1708 article.last_match = last_match.created_on
1710 # article1 = articles.first()
1711 # date = article1.deployed_date()
1712 # TODO next_issue, previous_issue
1714 # check DOI est maintenant une commande à part
1715 # # specific PTFTools : on regarde pour chaque article l'état de l'enregistrement DOI
1716 # articlesWithStatus = []
1717 # for article in articles:
1718 # get_or_create_doibatch(article)
1719 # articlesWithStatus.append(article)
1721 test_location = prod_location = ""
1722 qs = container.get_top_collection().extlink_set.filter(rel="test_website")
1723 if qs:
1724 test_location = qs.first().location
1725 qs = container.get_top_collection().extlink_set.filter(rel="website")
1726 if qs:
1727 prod_location = qs.first().location
1728 context.update(
1729 {
1730 "issue": container,
1731 "articles": articles,
1732 "source": source,
1733 "citing_articles": citing_articles,
1734 "test_website": test_location,
1735 "prod_website": prod_location,
1736 }
1737 )
1738 self.template_name = "issue-items.html"
1740 context["allow_crossref"] = container.allow_crossref()
1741 context["coltype"] = container.my_collection.coltype
1742 return context
1745class ExtLinkInline(InlineFormSetFactory):
1746 model = ExtLink
1747 form_class = ExtLinkForm
1748 factory_kwargs = {"extra": 0}
1751class ResourceIdInline(InlineFormSetFactory):
1752 model = ResourceId
1753 form_class = ResourceIdForm
1754 factory_kwargs = {"extra": 0}
1757class IssueDetailAPIView(View):
1758 def get(self, request, *args, **kwargs):
1759 issue = get_object_or_404(Container, pid=kwargs["pid"])
1760 deployed_date = issue.deployed_date()
1761 result = {
1762 "deployed_date": timezone.localtime(deployed_date).strftime("%Y-%m-%d %H:%M")
1763 if deployed_date
1764 else None,
1765 "last_modified": timezone.localtime(issue.last_modified).strftime("%Y-%m-%d %H:%M"),
1766 "all_doi_are_registered": issue.all_doi_are_registered(),
1767 "registered_in_doaj": issue.registered_in_doaj(),
1768 "doi": issue.my_collection.doi,
1769 "has_articles_excluded_from_publication": issue.has_articles_excluded_from_publication(),
1770 }
1771 try:
1772 latest = history_models.HistoryEvent.objects.get_last_unsolved_error(
1773 pid=issue.pid, strict=False
1774 )
1775 except history_models.HistoryEvent.DoesNotExist as _:
1776 pass
1777 else:
1778 result["latest"] = latest.data["message"]
1779 result["latest_target"] = latest.data.get("target", "")
1780 result["latest_date"] = timezone.localtime(latest.created_on).strftime(
1781 "%Y-%m-%d %H:%M"
1782 )
1784 result["latest_type"] = latest.type.capitalize()
1785 for event_type in ["matching", "edit", "deploy", "archive", "import"]:
1786 try:
1787 result[event_type] = timezone.localtime(
1788 history_models.HistoryEvent.objects.filter(
1789 type=event_type,
1790 status="OK",
1791 pid__startswith=issue.pid,
1792 )
1793 .latest("created_on")
1794 .created_on
1795 ).strftime("%Y-%m-%d %H:%M")
1796 except history_models.HistoryEvent.DoesNotExist as _:
1797 result[event_type] = ""
1798 return JsonResponse(result)
1801class CollectionFormView(LoginRequiredMixin, StaffuserRequiredMixin, NamedFormsetsMixin, View):
1802 model = Collection
1803 form_class = CollectionForm
1804 inlines = [ResourceIdInline, ExtLinkInline]
1805 inlines_names = ["resource_ids_form", "ext_links_form"]
1807 def get_context_data(self, **kwargs):
1808 context = super().get_context_data(**kwargs)
1809 context["helper"] = PtfFormHelper
1810 context["formset_helper"] = FormSetHelper
1811 return context
1813 def add_description(self, collection, description, lang, seq):
1814 if description:
1815 la = Abstract(
1816 resource=collection,
1817 tag="description",
1818 lang=lang,
1819 seq=seq,
1820 value_xml=f'<description xml:lang="{lang}">{replace_html_entities(description)}</description>',
1821 value_html=description,
1822 value_tex=description,
1823 )
1824 la.save()
1826 def form_valid(self, form):
1827 if form.instance.abbrev:
1828 form.instance.title_xml = f"<title-group><title>{form.instance.title_tex}</title><abbrev-title>{form.instance.abbrev}</abbrev-title></title-group>"
1829 else:
1830 form.instance.title_xml = (
1831 f"<title-group><title>{form.instance.title_tex}</title></title-group>"
1832 )
1834 form.instance.title_html = form.instance.title_tex
1835 form.instance.title_sort = form.instance.title_tex
1836 result = super().form_valid(form)
1838 collection = self.object
1839 collection.abstract_set.all().delete()
1841 seq = 1
1842 description = form.cleaned_data["description_en"]
1843 if description:
1844 self.add_description(collection, description, "en", seq)
1845 seq += 1
1846 description = form.cleaned_data["description_fr"]
1847 if description:
1848 self.add_description(collection, description, "fr", seq)
1850 return result
1852 def get_success_url(self):
1853 messages.success(self.request, "La Collection a été modifiée avec succès")
1854 return reverse("collection-detail", kwargs={"pid": self.object.pid})
1857class CollectionCreate(CollectionFormView, CreateWithInlinesView):
1858 """
1859 Warning : Not yet finished
1860 Automatic site membership creation is still missing
1861 """
1864class CollectionUpdate(CollectionFormView, UpdateWithInlinesView):
1865 slug_field = "pid"
1866 slug_url_kwarg = "pid"
1869def suggest_load_journal_dois(colid):
1870 articles = (
1871 Article.objects.filter(my_container__my_collection__pid=colid)
1872 .filter(doi__isnull=False)
1873 .filter(Q(date_published__isnull=False) | Q(date_online_first__isnull=False))
1874 .values_list("doi", flat=True)
1875 )
1877 try:
1878 articles = sorted(
1879 articles,
1880 key=lambda d: (
1881 re.search(r"([a-zA-Z]+).\d+$", d).group(1),
1882 int(re.search(r".(\d+)$", d).group(1)),
1883 ),
1884 )
1885 except:
1886 pass
1887 return [f'<option value="{doi}">' for doi in articles]
1890def get_context_with_volumes(journal):
1891 result = model_helpers.get_volumes_in_collection(journal)
1892 volume_count = result["volume_count"]
1893 collections = []
1894 for ancestor in journal.ancestors.all():
1895 item = model_helpers.get_volumes_in_collection(ancestor)
1896 volume_count = max(0, volume_count)
1897 item.update({"journal": ancestor})
1898 collections.append(item)
1900 # add the parent collection to its children list and sort it by date
1901 result.update({"journal": journal})
1902 collections.append(result)
1904 collections = [c for c in collections if c["sorted_issues"]]
1905 collections.sort(
1906 key=lambda ancestor: ancestor["sorted_issues"][0]["volumes"][0]["lyear"],
1907 reverse=True,
1908 )
1910 context = {
1911 "journal": journal,
1912 "sorted_issues": result["sorted_issues"],
1913 "volume_count": volume_count,
1914 "max_width": result["max_width"],
1915 "collections": collections,
1916 "choices": "\n".join(suggest_load_journal_dois(journal.pid)),
1917 }
1918 return context
1921class CollectionDetail(
1922 UserPassesTestMixin, SingleObjectMixin, ListView, history_views.HistoryContextMixin
1923):
1924 model = Collection
1925 slug_field = "pid"
1926 slug_url_kwarg = "pid"
1927 template_name = "ptf/collection_detail.html"
1929 def test_func(self):
1930 return is_authorized_editor(self.request.user, self.kwargs.get("pid"))
1932 def get(self, request, *args, **kwargs):
1933 self.object = self.get_object(queryset=Collection.objects.all())
1934 return super().get(request, *args, **kwargs)
1936 def get_context_data(self, **kwargs):
1937 context = super().get_context_data(**kwargs)
1938 context.update(get_context_with_volumes(self.object))
1940 if self.object.pid in settings.ISSUE_TO_APPEAR_PIDS:
1941 context["issue_to_appear_pid"] = settings.ISSUE_TO_APPEAR_PIDS[self.object.pid]
1942 context["issue_to_appear"] = Container.objects.filter(
1943 pid=context["issue_to_appear_pid"]
1944 ).exists()
1945 try:
1946 latest_error = history_models.HistoryEvent.objects.get_last_unsolved_error(
1947 self.object.pid,
1948 strict=True,
1949 )
1950 except history_models.HistoryEvent.DoesNotExist as _:
1951 pass
1952 else:
1953 message = latest_error.data["message"]
1954 i = message.find(" - ")
1955 latest_exception = message[:i]
1956 latest_error_message = message[i + 3 :]
1957 context["latest_exception"] = latest_exception
1958 context["latest_exception_date"] = latest_error.created_on
1959 context["latest_exception_type"] = latest_error.type
1960 context["latest_error_message"] = latest_error_message
1961 return context
1963 def get_queryset(self):
1964 query = self.object.content.all()
1966 for ancestor in self.object.ancestors.all():
1967 query |= ancestor.content.all()
1969 return query.order_by("-year", "-vseries", "-volume", "-volume_int", "-number_int")
1972class ContainerEditView(FormView):
1973 template_name = "container_form.html"
1974 form_class = ContainerForm
1976 def get_success_url(self):
1977 if self.kwargs["pid"]:
1978 return reverse("issue-items", kwargs={"pid": self.kwargs["pid"]})
1979 return reverse("mersenne_dashboard")
1981 def set_success_message(self): # pylint: disable=no-self-use
1982 messages.success(self.request, "Le fascicule a été modifié")
1984 def get_form_kwargs(self):
1985 kwargs = super().get_form_kwargs()
1986 if "pid" not in self.kwargs:
1987 self.kwargs["pid"] = None
1988 if "colid" not in self.kwargs:
1989 self.kwargs["colid"] = None
1990 if "data" in kwargs and "colid" in kwargs["data"]:
1991 # colid is passed as a hidden param in the form.
1992 # It is used when you submit a new container
1993 self.kwargs["colid"] = kwargs["data"]["colid"]
1995 self.kwargs["container"] = kwargs["container"] = model_helpers.get_container(
1996 self.kwargs["pid"]
1997 )
1998 return kwargs
2000 def get_context_data(self, **kwargs):
2001 context = super().get_context_data(**kwargs)
2003 context["pid"] = self.kwargs["pid"]
2004 context["colid"] = self.kwargs["colid"]
2005 context["container"] = self.kwargs["container"]
2007 context["edit_container"] = context["pid"] is not None
2009 return context
2011 def form_valid(self, form):
2012 new_pid = form.cleaned_data.get("pid")
2013 new_title = form.cleaned_data.get("title")
2014 new_trans_title = form.cleaned_data.get("trans_title")
2015 new_publisher = form.cleaned_data.get("publisher")
2016 new_year = form.cleaned_data.get("year")
2017 new_volume = form.cleaned_data.get("volume")
2018 new_number = form.cleaned_data.get("number")
2020 collection = None
2021 issue = self.kwargs["container"]
2022 if issue is not None:
2023 collection = issue.my_collection
2024 elif self.kwargs["colid"] is not None:
2025 collection = model_helpers.get_collection(self.kwargs["colid"])
2027 if collection is None:
2028 raise ValueError("Collection for " + new_pid + " does not exist")
2030 # Icon
2031 new_icon_location = ""
2032 if "icon" in self.request.FILES:
2033 filename = os.path.basename(self.request.FILES["icon"].name)
2034 file_extension = filename.split(".")[1]
2036 icon_filename = resolver.get_disk_location(
2037 settings.MERSENNE_TEST_DATA_FOLDER,
2038 collection.pid,
2039 file_extension,
2040 new_pid,
2041 None,
2042 True,
2043 )
2045 with open(icon_filename, "wb+") as destination:
2046 for chunk in self.request.FILES["icon"].chunks():
2047 destination.write(chunk)
2049 folder = resolver.get_relative_folder(collection.pid, new_pid)
2050 new_icon_location = os.path.join(folder, new_pid + "." + file_extension)
2052 if self.kwargs["container"]:
2053 # Edit Issue
2054 issue = self.kwargs["container"]
2055 if issue is None:
2056 raise ValueError(self.kwargs["pid"] + " does not exist")
2058 if new_trans_title and (not issue.trans_lang or issue.trans_lang == "und"):
2059 issue.trans_lang = "fr" if issue.lang == "en" else "en"
2061 issue.pid = new_pid
2062 issue.title_tex = issue.title_html = new_title
2063 issue.trans_title_tex = issue.trans_title_html = new_trans_title
2064 issue.title_xml = jats_parser.get_issue_title_xml(
2065 new_title, issue.lang, new_trans_title, issue.trans_lang
2066 )
2067 issue.year = new_year
2068 issue.volume = new_volume
2069 issue.volume_int = make_int(new_volume)
2070 issue.number = new_number
2071 issue.number_int = make_int(new_number)
2072 issue.save()
2073 else:
2074 xissue = create_issuedata()
2075 xissue.ctype = "issue"
2076 xissue.pid = new_pid
2077 # TODO: add lang + trans_lang
2078 xissue.title_tex = new_title
2079 xissue.trans_title_tex = new_trans_title
2080 xissue.title_xml = jats_parser.get_title_xml(new_title)
2081 # new_title, new_trans_title, issue.trans_lang
2082 # )
2083 xissue.year = new_year
2084 xissue.volume = new_volume
2085 xissue.number = new_number
2086 xissue.last_modified_iso_8601_date_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
2088 cmd = ptf_cmds.addContainerPtfCmd({"xobj": xissue})
2089 cmd.add_collection(collection)
2090 cmd.set_provider(model_helpers.get_provider_by_name("mathdoc"))
2091 issue = cmd.do()
2093 self.kwargs["pid"] = new_pid
2095 # Add objects related to the article: contribs, datastream, counts...
2096 params = {
2097 "icon_location": new_icon_location,
2098 }
2099 cmd = ptf_cmds.updateContainerPtfCmd(params)
2100 cmd.set_resource(issue)
2101 cmd.do()
2103 publisher = model_helpers.get_publisher(new_publisher)
2104 if not publisher:
2105 xpub = create_publisherdata()
2106 xpub.name = new_publisher
2107 publisher = ptf_cmds.addPublisherPtfCmd({"xobj": xpub}).do()
2108 issue.my_publisher = publisher
2109 issue.save()
2111 self.set_success_message()
2113 return super().form_valid(form)
2116# class ArticleEditView(FormView):
2117# template_name = 'article_form.html'
2118# form_class = ArticleForm
2119#
2120# def get_success_url(self):
2121# if self.kwargs['pid']:
2122# return reverse('article', kwargs={'aid': self.kwargs['pid']})
2123# return reverse('mersenne_dashboard')
2124#
2125# def set_success_message(self): # pylint: disable=no-self-use
2126# messages.success(self.request, "L'article a été modifié")
2127#
2128# def get_form_kwargs(self):
2129# kwargs = super(ArticleEditView, self).get_form_kwargs()
2130#
2131# if 'pid' not in self.kwargs or self.kwargs['pid'] == 'None':
2132# # Article creation: pid is None
2133# self.kwargs['pid'] = None
2134# if 'issue_id' not in self.kwargs:
2135# # Article edit: issue_id is not passed
2136# self.kwargs['issue_id'] = None
2137# if 'data' in kwargs and 'issue_id' in kwargs['data']:
2138# # colid is passed as a hidden param in the form.
2139# # It is used when you submit a new container
2140# self.kwargs['issue_id'] = kwargs['data']['issue_id']
2141#
2142# self.kwargs['article'] = kwargs['article'] = model_helpers.get_article(self.kwargs['pid'])
2143# return kwargs
2144#
2145# def get_context_data(self, **kwargs):
2146# context = super(ArticleEditView, self).get_context_data(**kwargs)
2147#
2148# context['pid'] = self.kwargs['pid']
2149# context['issue_id'] = self.kwargs['issue_id']
2150# context['article'] = self.kwargs['article']
2151#
2152# context['edit_article'] = context['pid'] is not None
2153#
2154# article = context['article']
2155# if article:
2156# context['author_contributions'] = article.get_author_contributions()
2157# context['kwds_fr'] = None
2158# context['kwds_en'] = None
2159# kwd_gps = article.get_non_msc_kwds()
2160# for kwd_gp in kwd_gps:
2161# if kwd_gp.lang == 'fr' or (kwd_gp.lang == 'und' and article.lang == 'fr'):
2162# if kwd_gp.value_xml:
2163# kwd_ = types.SimpleNamespace()
2164# kwd_.value = kwd_gp.value_tex
2165# context['kwd_unstructured_fr'] = kwd_
2166# context['kwds_fr'] = kwd_gp.kwd_set.all()
2167# elif kwd_gp.lang == 'en' or (kwd_gp.lang == 'und' and article.lang == 'en'):
2168# if kwd_gp.value_xml:
2169# kwd_ = types.SimpleNamespace()
2170# kwd_.value = kwd_gp.value_tex
2171# context['kwd_unstructured_en'] = kwd_
2172# context['kwds_en'] = kwd_gp.kwd_set.all()
2173#
2174# # Article creation: init pid
2175# if context['issue_id'] and context['pid'] is None:
2176# issue = model_helpers.get_container(context['issue_id'])
2177# context['pid'] = issue.pid + '_A' + str(issue.article_set.count() + 1) + '_0'
2178#
2179# return context
2180#
2181# def form_valid(self, form):
2182#
2183# new_pid = form.cleaned_data.get('pid')
2184# new_title = form.cleaned_data.get('title')
2185# new_fpage = form.cleaned_data.get('fpage')
2186# new_lpage = form.cleaned_data.get('lpage')
2187# new_page_range = form.cleaned_data.get('page_range')
2188# new_page_count = form.cleaned_data.get('page_count')
2189# new_coi_statement = form.cleaned_data.get('coi_statement')
2190# new_show_body = form.cleaned_data.get('show_body')
2191# new_do_not_publish = form.cleaned_data.get('do_not_publish')
2192#
2193# # TODO support MathML
2194# # 27/10/2020: title_xml embeds the trans_title_group in JATS.
2195# # We need to pass trans_title to get_title_xml
2196# # Meanwhile, ignore new_title_xml
2197# new_title_xml = jats_parser.get_title_xml(new_title)
2198# new_title_html = new_title
2199#
2200# authors_count = int(self.request.POST.get('authors_count', "0"))
2201# i = 1
2202# new_authors = []
2203# old_author_contributions = []
2204# if self.kwargs['article']:
2205# old_author_contributions = self.kwargs['article'].get_author_contributions()
2206#
2207# while authors_count > 0:
2208# prefix = self.request.POST.get('contrib-p-' + str(i), None)
2209#
2210# if prefix is not None:
2211# addresses = []
2212# if len(old_author_contributions) >= i:
2213# old_author_contribution = old_author_contributions[i - 1]
2214# addresses = [contrib_address.address for contrib_address in
2215# old_author_contribution.get_addresses()]
2216#
2217# first_name = self.request.POST.get('contrib-f-' + str(i), None)
2218# last_name = self.request.POST.get('contrib-l-' + str(i), None)
2219# suffix = self.request.POST.get('contrib-s-' + str(i), None)
2220# orcid = self.request.POST.get('contrib-o-' + str(i), None)
2221# deceased = self.request.POST.get('contrib-d-' + str(i), None)
2222# deceased_before_publication = deceased == 'on'
2223# equal_contrib = self.request.POST.get('contrib-e-' + str(i), None)
2224# equal_contrib = equal_contrib == 'on'
2225# corresponding = self.request.POST.get('corresponding-' + str(i), None)
2226# corresponding = corresponding == 'on'
2227# email = self.request.POST.get('email-' + str(i), None)
2228#
2229# params = jats_parser.get_name_params(first_name, last_name, prefix, suffix, orcid)
2230# params['deceased_before_publication'] = deceased_before_publication
2231# params['equal_contrib'] = equal_contrib
2232# params['corresponding'] = corresponding
2233# params['addresses'] = addresses
2234# params['email'] = email
2235#
2236# params['contrib_xml'] = xml_utils.get_contrib_xml(params)
2237#
2238# new_authors.append(params)
2239#
2240# authors_count -= 1
2241# i += 1
2242#
2243# kwds_fr_count = int(self.request.POST.get('kwds_fr_count', "0"))
2244# i = 1
2245# new_kwds_fr = []
2246# while kwds_fr_count > 0:
2247# value = self.request.POST.get('kwd-fr-' + str(i), None)
2248# new_kwds_fr.append(value)
2249# kwds_fr_count -= 1
2250# i += 1
2251# new_kwd_uns_fr = self.request.POST.get('kwd-uns-fr-0', None)
2252#
2253# kwds_en_count = int(self.request.POST.get('kwds_en_count', "0"))
2254# i = 1
2255# new_kwds_en = []
2256# while kwds_en_count > 0:
2257# value = self.request.POST.get('kwd-en-' + str(i), None)
2258# new_kwds_en.append(value)
2259# kwds_en_count -= 1
2260# i += 1
2261# new_kwd_uns_en = self.request.POST.get('kwd-uns-en-0', None)
2262#
2263# if self.kwargs['article']:
2264# # Edit article
2265# container = self.kwargs['article'].my_container
2266# else:
2267# # New article
2268# container = model_helpers.get_container(self.kwargs['issue_id'])
2269#
2270# if container is None:
2271# raise ValueError(self.kwargs['issue_id'] + " does not exist")
2272#
2273# collection = container.my_collection
2274#
2275# # Copy PDF file & extract full text
2276# body = ''
2277# pdf_filename = resolver.get_disk_location(settings.MERSENNE_TEST_DATA_FOLDER,
2278# collection.pid,
2279# "pdf",
2280# container.pid,
2281# new_pid,
2282# True)
2283# if 'pdf' in self.request.FILES:
2284# with open(pdf_filename, 'wb+') as destination:
2285# for chunk in self.request.FILES['pdf'].chunks():
2286# destination.write(chunk)
2287#
2288# # Extract full text from the PDF
2289# body = utils.pdf_to_text(pdf_filename)
2290#
2291# # Icon
2292# new_icon_location = ''
2293# if 'icon' in self.request.FILES:
2294# filename = os.path.basename(self.request.FILES['icon'].name)
2295# file_extension = filename.split('.')[1]
2296#
2297# icon_filename = resolver.get_disk_location(settings.MERSENNE_TEST_DATA_FOLDER,
2298# collection.pid,
2299# file_extension,
2300# container.pid,
2301# new_pid,
2302# True)
2303#
2304# with open(icon_filename, 'wb+') as destination:
2305# for chunk in self.request.FILES['icon'].chunks():
2306# destination.write(chunk)
2307#
2308# folder = resolver.get_relative_folder(collection.pid, container.pid, new_pid)
2309# new_icon_location = os.path.join(folder, new_pid + '.' + file_extension)
2310#
2311# if self.kwargs['article']:
2312# # Edit article
2313# article = self.kwargs['article']
2314# article.fpage = new_fpage
2315# article.lpage = new_lpage
2316# article.page_range = new_page_range
2317# article.coi_statement = new_coi_statement
2318# article.show_body = new_show_body
2319# article.do_not_publish = new_do_not_publish
2320# article.save()
2321#
2322# else:
2323# # New article
2324# params = {
2325# 'pid': new_pid,
2326# 'title_xml': new_title_xml,
2327# 'title_html': new_title_html,
2328# 'title_tex': new_title,
2329# 'fpage': new_fpage,
2330# 'lpage': new_lpage,
2331# 'page_range': new_page_range,
2332# 'seq': container.article_set.count() + 1,
2333# 'body': body,
2334# 'coi_statement': new_coi_statement,
2335# 'show_body': new_show_body,
2336# 'do_not_publish': new_do_not_publish
2337# }
2338#
2339# xarticle = create_articledata()
2340# xarticle.pid = new_pid
2341# xarticle.title_xml = new_title_xml
2342# xarticle.title_html = new_title_html
2343# xarticle.title_tex = new_title
2344# xarticle.fpage = new_fpage
2345# xarticle.lpage = new_lpage
2346# xarticle.page_range = new_page_range
2347# xarticle.seq = container.article_set.count() + 1
2348# xarticle.body = body
2349# xarticle.coi_statement = new_coi_statement
2350# params['xobj'] = xarticle
2351#
2352# cmd = ptf_cmds.addArticlePtfCmd(params)
2353# cmd.set_container(container)
2354# cmd.add_collection(container.my_collection)
2355# article = cmd.do()
2356#
2357# self.kwargs['pid'] = new_pid
2358#
2359# # Add objects related to the article: contribs, datastream, counts...
2360# params = {
2361# # 'title_xml': new_title_xml,
2362# # 'title_html': new_title_html,
2363# # 'title_tex': new_title,
2364# 'authors': new_authors,
2365# 'page_count': new_page_count,
2366# 'icon_location': new_icon_location,
2367# 'body': body,
2368# 'use_kwds': True,
2369# 'kwds_fr': new_kwds_fr,
2370# 'kwds_en': new_kwds_en,
2371# 'kwd_uns_fr': new_kwd_uns_fr,
2372# 'kwd_uns_en': new_kwd_uns_en
2373# }
2374# cmd = ptf_cmds.updateArticlePtfCmd(params)
2375# cmd.set_article(article)
2376# cmd.do()
2377#
2378# self.set_success_message()
2379#
2380# return super(ArticleEditView, self).form_valid(form)
2383@require_http_methods(["POST"])
2384def do_not_publish_article(request, *args, **kwargs):
2385 next = request.headers.get("referer")
2387 pid = kwargs.get("pid", "")
2389 article = model_helpers.get_article(pid)
2390 if article:
2391 article.do_not_publish = not article.do_not_publish
2392 article.save()
2393 else:
2394 raise Http404
2396 return HttpResponseRedirect(next)
2399@require_http_methods(["POST"])
2400def show_article_body(request, *args, **kwargs):
2401 next = request.headers.get("referer")
2403 pid = kwargs.get("pid", "")
2405 article = model_helpers.get_article(pid)
2406 if article:
2407 article.show_body = not article.show_body
2408 article.save()
2409 else:
2410 raise Http404
2412 return HttpResponseRedirect(next)
2415class ArticleEditWithVueAPIView(CsrfExemptMixin, ArticleEditAPIView):
2416 """
2417 API to get/post article metadata
2418 The class is derived from ArticleEditAPIView (see ptf.views)
2419 """
2421 def __init__(self, *args, **kwargs):
2422 super().__init__(*args, **kwargs)
2423 self.fields_to_update = [
2424 "lang",
2425 "title_xml",
2426 "title_tex",
2427 "title_html",
2428 "trans_lang",
2429 "trans_title_html",
2430 "trans_title_tex",
2431 "trans_title_xml",
2432 "atype",
2433 "contributors",
2434 "abstracts",
2435 "subjs",
2436 "kwds",
2437 "ext_links",
2438 ]
2440 def convert_data_for_editor(self, data_article):
2441 super().convert_data_for_editor(data_article)
2442 data_article.is_staff = self.request.user.is_staff
2444 def save_data(self, data_article):
2445 # On sauvegarde les données additionnelles (extid, deployed_date,...) dans un json
2446 # The icons are not preserved since we can add/edit/delete them in VueJs
2447 params = {
2448 "pid": data_article.pid,
2449 "export_folder": settings.MERSENNE_TMP_FOLDER,
2450 "export_all": True,
2451 "with_binary_files": False,
2452 }
2453 ptf_cmds.exportExtraDataPtfCmd(params).do()
2455 def restore_data(self, article):
2456 ptf_cmds.importExtraDataPtfCmd(
2457 {
2458 "pid": article.pid,
2459 "import_folder": settings.MERSENNE_TMP_FOLDER,
2460 }
2461 ).do()
2464class ArticleEditWithVueView(LoginRequiredMixin, TemplateView):
2465 template_name = "article_form.html"
2467 def get_success_url(self):
2468 if self.kwargs["doi"]:
2469 return reverse("article", kwargs={"aid": self.kwargs["doi"]})
2470 return reverse("mersenne_dashboard")
2472 def get_context_data(self, **kwargs):
2473 context = super().get_context_data(**kwargs)
2474 if "doi" in self.kwargs:
2475 context["article"] = model_helpers.get_article_by_doi(self.kwargs["doi"])
2476 context["pid"] = context["article"].pid
2478 return context
2481class ArticleDeleteView(View):
2482 def get(self, request, *args, **kwargs):
2483 pid = self.kwargs.get("pid", None)
2484 article = get_object_or_404(Article, pid=pid)
2486 try:
2487 mersenneSite = model_helpers.get_site_mersenne(article.get_collection().pid)
2488 article.undeploy(mersenneSite)
2490 cmd = ptf_cmds.addArticlePtfCmd(
2491 {"pid": article.pid, "to_folder": settings.MERSENNE_TEST_DATA_FOLDER}
2492 )
2493 cmd.set_container(article.my_container)
2494 cmd.set_object_to_be_deleted(article)
2495 cmd.undo()
2496 except Exception as exception:
2497 return HttpResponseServerError(exception)
2499 data = {"message": "L'article a bien été supprimé de ptf-tools", "status": 200}
2500 return JsonResponse(data)
2503def get_messages_in_queue():
2504 app = Celery("ptf-tools")
2505 # tasks = list(current_app.tasks)
2506 tasks = list(sorted(name for name in current_app.tasks if name.startswith("celery")))
2507 print(tasks)
2508 # i = app.control.inspect()
2510 with app.connection_or_acquire() as conn:
2511 remaining = conn.default_channel.queue_declare(queue="celery", passive=True).message_count
2512 return remaining
2515class FailedTasksListView(ListView):
2516 model = TaskResult
2517 queryset = TaskResult.objects.filter(
2518 status="FAILURE",
2519 task_name="ptf_tools.tasks.archive_numdam_issue",
2520 )
2523class FailedTasksDeleteView(DeleteView):
2524 model = TaskResult
2525 success_url = reverse_lazy("tasks-failed")
2528class FailedTasksRetryView(SingleObjectMixin, RedirectView):
2529 model = TaskResult
2531 @staticmethod
2532 def retry_task(task):
2533 colid, pid = (arg.strip("'") for arg in task.task_args.strip("()").split(", "))
2534 archive_numdam_issue.delay(colid, pid)
2535 task.delete()
2537 def get_redirect_url(self, *args, **kwargs):
2538 self.retry_task(self.get_object())
2539 return reverse("tasks-failed")
2542class NumdamView(TemplateView, history_views.HistoryContextMixin):
2543 template_name = "numdam.html"
2545 def get_context_data(self, **kwargs):
2546 context = super().get_context_data(**kwargs)
2548 context["objs"] = ResourceInNumdam.objects.all()
2550 pre_issues = []
2551 prod_issues = []
2552 url = f"{settings.NUMDAM_PRE_URL}/api-all-issues/"
2553 try:
2554 response = requests.get(url)
2555 if response.status_code == 200:
2556 data = response.json()
2557 if "issues" in data:
2558 pre_issues = data["issues"]
2559 except Exception:
2560 pass
2562 url = f"{settings.NUMDAM_URL}/api-all-issues/"
2563 response = requests.get(url)
2564 if response.status_code == 200:
2565 data = response.json()
2566 if "issues" in data:
2567 prod_issues = data["issues"]
2569 new = sorted(list(set(pre_issues).difference(prod_issues)))
2570 removed = sorted(list(set(prod_issues).difference(pre_issues)))
2571 grouped = [
2572 {"colid": k, "issues": list(g)} for k, g in groupby(new, lambda x: x.split("_")[0])
2573 ]
2574 grouped_removed = [
2575 {"colid": k, "issues": list(g)} for k, g in groupby(removed, lambda x: x.split("_")[0])
2576 ]
2577 context["added_issues"] = grouped
2578 context["removed_issues"] = grouped_removed
2580 context["numdam_collections"] = settings.NUMDAM_COLLECTIONS
2581 return context
2584class TasksProgressView(View):
2585 def get(self, *args, **kwargs):
2586 task_name = self.kwargs.get("task", "archive_numdam_issue")
2587 successes = TaskResult.objects.filter(
2588 task_name=f"ptf_tools.tasks.{task_name}", status="SUCCESS"
2589 ).count()
2590 fails = TaskResult.objects.filter(
2591 task_name=f"ptf_tools.tasks.{task_name}", status="FAILURE"
2592 ).count()
2593 last_task = (
2594 TaskResult.objects.filter(
2595 task_name=f"ptf_tools.tasks.{task_name}",
2596 status="SUCCESS",
2597 )
2598 .order_by("-date_done")
2599 .first()
2600 )
2601 if last_task:
2602 last_task = " : ".join([last_task.date_done.strftime("%Y-%m-%d"), last_task.task_args])
2603 remaining = get_messages_in_queue()
2604 all = successes + remaining
2605 progress = int(successes * 100 / all) if all else 0
2606 error_rate = int(fails * 100 / all) if all else 0
2607 status = "consuming_queue" if (successes or fails) and not progress == 100 else "polling"
2608 data = {
2609 "status": status,
2610 "progress": progress,
2611 "total": all,
2612 "remaining": remaining,
2613 "successes": successes,
2614 "fails": fails,
2615 "error_rate": error_rate,
2616 "last_task": last_task,
2617 }
2618 return JsonResponse(data)
2621class TasksView(TemplateView):
2622 template_name = "tasks.html"
2624 def get_context_data(self, **kwargs):
2625 context = super().get_context_data(**kwargs)
2626 context["tasks"] = TaskResult.objects.all()
2627 return context
2630class NumdamArchiveView(RedirectView):
2631 @staticmethod
2632 def reset_task_results():
2633 TaskResult.objects.all().delete()
2635 def get_redirect_url(self, *args, **kwargs):
2636 self.colid = kwargs["colid"]
2638 if self.colid != "ALL" and self.colid in settings.MERSENNE_COLLECTIONS:
2639 return Http404
2641 # we make sure archiving is not already running
2642 if not get_messages_in_queue():
2643 self.reset_task_results()
2644 response = requests.get(f"{settings.NUMDAM_URL}/api-all-collections/")
2645 if response.status_code == 200:
2646 data = sorted(response.json()["collections"])
2648 if self.colid != "ALL" and self.colid not in data:
2649 return Http404
2651 colids = [self.colid] if self.colid != "ALL" else data
2653 with open(
2654 os.path.join(settings.LOG_DIR, "archive.log"), "w", encoding="utf-8"
2655 ) as file_:
2656 file_.write("Archive " + " ".join([colid for colid in colids]) + "\n")
2658 for colid in colids:
2659 if colid not in settings.MERSENNE_COLLECTIONS:
2660 archive_numdam_collection.delay(colid)
2661 return reverse("numdam")
2664class DeployAllNumdamAPIView(View):
2665 def internal_do(self, *args, **kwargs):
2666 pids = []
2668 for obj in ResourceInNumdam.objects.all():
2669 pids.append(obj.pid)
2671 return pids
2673 def get(self, request, *args, **kwargs):
2674 try:
2675 pids, status, message = history_views.execute_and_record_func(
2676 "deploy", "numdam", "numdam", self.internal_do, "numdam"
2677 )
2678 except Exception as exception:
2679 return HttpResponseServerError(exception)
2681 data = {"message": message, "ids": pids, "status": status}
2682 return JsonResponse(data)
2685class NumdamDeleteAPIView(View):
2686 def get(self, request, *args, **kwargs):
2687 pid = self.kwargs.get("pid", None)
2689 try:
2690 obj = ResourceInNumdam.objects.get(pid=pid)
2691 obj.delete()
2692 except Exception as exception:
2693 return HttpResponseServerError(exception)
2695 data = {"message": "Le volume a bien été supprimé de la liste pour Numdam", "status": 200}
2696 return JsonResponse(data)
2699class ExtIdApiDetail(View):
2700 def get(self, request, *args, **kwargs):
2701 extid = get_object_or_404(
2702 ExtId,
2703 resource__pid=kwargs["pid"],
2704 id_type=kwargs["what"],
2705 )
2706 return JsonResponse(
2707 {
2708 "pk": extid.pk,
2709 "href": extid.get_href(),
2710 "fetch": reverse(
2711 "api-fetch-id",
2712 args=(
2713 extid.resource.pk,
2714 extid.id_value,
2715 extid.id_type,
2716 "extid",
2717 ),
2718 ),
2719 "check": reverse("update-extid", args=(extid.pk, "toggle-checked")),
2720 "uncheck": reverse("update-extid", args=(extid.pk, "toggle-false-positive")),
2721 "update": reverse("extid-update", kwargs={"pk": extid.pk}),
2722 "delete": reverse("update-extid", args=(extid.pk, "delete")),
2723 "is_valid": extid.checked,
2724 }
2725 )
2728class ExtIdFormTemplate(TemplateView):
2729 template_name = "common/externalid_form.html"
2731 def get_context_data(self, **kwargs):
2732 context = super().get_context_data(**kwargs)
2733 context["sequence"] = kwargs["sequence"]
2734 return context
2737class BibItemIdFormView(LoginRequiredMixin, StaffuserRequiredMixin, View):
2738 def get_context_data(self, **kwargs):
2739 context = super().get_context_data(**kwargs)
2740 context["helper"] = PtfFormHelper
2741 return context
2743 def get_success_url(self):
2744 self.post_process()
2745 return self.object.bibitem.resource.get_absolute_url()
2747 def post_process(self):
2748 cmd = xml_cmds.updateBibitemCitationXmlCmd()
2749 cmd.set_bibitem(self.object.bibitem)
2750 cmd.do()
2751 model_helpers.post_resource_updated(self.object.bibitem.resource)
2754class BibItemIdCreate(BibItemIdFormView, CreateView):
2755 model = BibItemId
2756 form_class = BibItemIdForm
2758 def get_context_data(self, **kwargs):
2759 context = super().get_context_data(**kwargs)
2760 context["bibitem"] = BibItem.objects.get(pk=self.kwargs["bibitem_pk"])
2761 return context
2763 def get_initial(self):
2764 initial = super().get_initial()
2765 initial["bibitem"] = BibItem.objects.get(pk=self.kwargs["bibitem_pk"])
2766 return initial
2768 def form_valid(self, form):
2769 form.instance.checked = False
2770 return super().form_valid(form)
2773class BibItemIdUpdate(BibItemIdFormView, UpdateView):
2774 model = BibItemId
2775 form_class = BibItemIdForm
2777 def get_context_data(self, **kwargs):
2778 context = super().get_context_data(**kwargs)
2779 context["bibitem"] = self.object.bibitem
2780 return context
2783class ExtIdFormView(LoginRequiredMixin, StaffuserRequiredMixin, View):
2784 def get_context_data(self, **kwargs):
2785 context = super().get_context_data(**kwargs)
2786 context["helper"] = PtfFormHelper
2787 return context
2789 def get_success_url(self):
2790 self.post_process()
2791 return self.object.resource.get_absolute_url()
2793 def post_process(self):
2794 model_helpers.post_resource_updated(self.object.resource)
2797class ExtIdCreate(ExtIdFormView, CreateView):
2798 model = ExtId
2799 form_class = ExtIdForm
2801 def get_context_data(self, **kwargs):
2802 context = super().get_context_data(**kwargs)
2803 context["resource"] = Resource.objects.get(pk=self.kwargs["resource_pk"])
2804 return context
2806 def get_initial(self):
2807 initial = super().get_initial()
2808 initial["resource"] = Resource.objects.get(pk=self.kwargs["resource_pk"])
2809 return initial
2811 def form_valid(self, form):
2812 form.instance.checked = False
2813 return super().form_valid(form)
2816class ExtIdUpdate(ExtIdFormView, UpdateView):
2817 model = ExtId
2818 form_class = ExtIdForm
2820 def get_context_data(self, **kwargs):
2821 context = super().get_context_data(**kwargs)
2822 context["resource"] = self.object.resource
2823 return context
2826class BibItemIdApiDetail(View):
2827 def get(self, request, *args, **kwargs):
2828 bibitemid = get_object_or_404(
2829 BibItemId,
2830 bibitem__resource__pid=kwargs["pid"],
2831 bibitem__sequence=kwargs["seq"],
2832 id_type=kwargs["what"],
2833 )
2834 return JsonResponse(
2835 {
2836 "pk": bibitemid.pk,
2837 "href": bibitemid.get_href(),
2838 "fetch": reverse(
2839 "api-fetch-id",
2840 args=(
2841 bibitemid.bibitem.pk,
2842 bibitemid.id_value,
2843 bibitemid.id_type,
2844 "bibitemid",
2845 ),
2846 ),
2847 "check": reverse("update-bibitemid", args=(bibitemid.pk, "toggle-checked")),
2848 "uncheck": reverse(
2849 "update-bibitemid", args=(bibitemid.pk, "toggle-false-positive")
2850 ),
2851 "update": reverse("bibitemid-update", kwargs={"pk": bibitemid.pk}),
2852 "delete": reverse("update-bibitemid", args=(bibitemid.pk, "delete")),
2853 "is_valid": bibitemid.checked,
2854 }
2855 )
2858class UpdateTexmfZipAPIView(View):
2859 def get(self, request, *args, **kwargs):
2860 def copy_zip_files(src_folder, dest_folder):
2861 os.makedirs(dest_folder, exist_ok=True)
2863 zip_files = [
2864 os.path.join(src_folder, f)
2865 for f in os.listdir(src_folder)
2866 if os.path.isfile(os.path.join(src_folder, f)) and f.endswith(".zip")
2867 ]
2868 for zip_file in zip_files:
2869 resolver.copy_file(zip_file, dest_folder)
2871 # Exceptions: specific zip/gz files
2872 zip_file = os.path.join(src_folder, "texmf-bsmf.zip")
2873 resolver.copy_file(zip_file, dest_folder)
2875 zip_file = os.path.join(src_folder, "texmf-cg.zip")
2876 resolver.copy_file(zip_file, dest_folder)
2878 gz_file = os.path.join(src_folder, "texmf-mersenne.tar.gz")
2879 resolver.copy_file(gz_file, dest_folder)
2881 src_folder = settings.CEDRAM_DISTRIB_FOLDER
2883 dest_folder = os.path.join(
2884 settings.MERSENNE_TEST_DATA_FOLDER, "MERSENNE", "media", "texmf"
2885 )
2887 try:
2888 copy_zip_files(src_folder, dest_folder)
2889 except Exception as exception:
2890 return HttpResponseServerError(exception)
2892 try:
2893 dest_folder = os.path.join(
2894 settings.MERSENNE_PROD_DATA_FOLDER, "MERSENNE", "media", "texmf"
2895 )
2896 copy_zip_files(src_folder, dest_folder)
2897 except Exception as exception:
2898 return HttpResponseServerError(exception)
2900 data = {"message": "Les texmf*.zip ont bien été mis à jour", "status": 200}
2901 return JsonResponse(data)
2904class TestView(TemplateView):
2905 template_name = "mersenne.html"
2907 def get_context_data(self, **kwargs):
2908 super().get_context_data(**kwargs)
2909 issue = model_helpers.get_container(pid="CRPHYS_0__0_0", prefetch=True)
2910 model_data_converter.db_to_issue_data(issue)
2913class TrammelArchiveView(RedirectView):
2914 @staticmethod
2915 def reset_task_results():
2916 TaskResult.objects.all().delete()
2918 def get_redirect_url(self, *args, **kwargs):
2919 self.colid = kwargs["colid"]
2920 self.mathdoc_archive = settings.MATHDOC_ARCHIVE_FOLDER
2921 self.binary_files_folder = settings.MERSENNE_PROD_DATA_FOLDER
2922 # Make sure archiving is not already running
2923 if not get_messages_in_queue():
2924 self.reset_task_results()
2925 if "progress/" in self.colid:
2926 self.colid = self.colid.replace("progress/", "")
2927 if "/progress" in self.colid:
2928 self.colid = self.colid.replace("/progress", "")
2930 if self.colid != "ALL" and self.colid not in settings.MERSENNE_COLLECTIONS:
2931 return Http404
2933 colids = [self.colid] if self.colid != "ALL" else settings.MERSENNE_COLLECTIONS
2935 with open(
2936 os.path.join(settings.LOG_DIR, "archive.log"), "w", encoding="utf-8"
2937 ) as file_:
2938 file_.write("Archive " + " ".join([colid for colid in colids]) + "\n")
2940 for colid in colids:
2941 archive_trammel_collection.delay(
2942 colid, self.mathdoc_archive, self.binary_files_folder
2943 )
2945 if self.colid == "ALL":
2946 return reverse("home")
2947 else:
2948 return reverse("collection-detail", kwargs={"pid": self.colid})
2951class TrammelTasksProgressView(View):
2952 def get(self, request, *args, **kwargs):
2953 """
2954 Return a JSON object with the progress of the archiving task Le code permet de récupérer l'état d'avancement
2955 de la tache celery (archive_trammel_resource) en SSE (Server-Sent Events)
2956 """
2957 task_name = self.kwargs.get("task", "archive_numdam_issue")
2959 def get_event_data():
2960 # Tasks are typically in the CREATED then SUCCESS or FAILURE state
2962 # Some messages (in case of many call to <task>.delay) have not been converted to TaskResult yet
2963 remaining_messages = get_messages_in_queue()
2965 all_tasks = TaskResult.objects.filter(task_name=f"ptf_tools.tasks.{task_name}")
2966 successed_tasks = all_tasks.filter(status="SUCCESS").order_by("-date_done")
2967 failed_tasks = all_tasks.filter(status="FAILURE")
2969 all_tasks_count = all_tasks.count()
2970 success_count = successed_tasks.count()
2971 fail_count = failed_tasks.count()
2973 all_count = all_tasks_count + remaining_messages
2974 remaining_count = all_count - success_count - fail_count
2976 success_rate = int(success_count * 100 / all_count) if all_count else 0
2977 error_rate = int(fail_count * 100 / all_count) if all_count else 0
2978 status = "consuming_queue" if remaining_count != 0 else "polling"
2980 last_task = successed_tasks.first()
2981 last_task = (
2982 " : ".join([last_task.date_done.strftime("%Y-%m-%d"), last_task.task_args])
2983 if last_task
2984 else ""
2985 )
2987 # SSE event format
2988 event_data = {
2989 "status": status,
2990 "success_rate": success_rate,
2991 "error_rate": error_rate,
2992 "all_count": all_count,
2993 "remaining_count": remaining_count,
2994 "success_count": success_count,
2995 "fail_count": fail_count,
2996 "last_task": last_task,
2997 }
2999 return event_data
3001 def stream_response(data):
3002 # Send initial response headers
3003 yield f"data: {json.dumps(data)}\n\n"
3005 data = get_event_data()
3006 format = request.GET.get("format", "stream")
3007 if format == "json":
3008 response = JsonResponse(data)
3009 else:
3010 response = HttpResponse(stream_response(data), content_type="text/event-stream")
3011 return response
3014class TrammelFailedTasksListView(ListView):
3015 model = TaskResult
3016 queryset = TaskResult.objects.filter(
3017 status="FAILURE",
3018 task_name="ptf_tools.tasks.archive_trammel_resource",
3019 )