Coverage for sites/ptf_tools/ptf_tools/tasks.py: 36%

1import os 

2import subprocess 

3 

4from celery import shared_task 

5 

6from django.conf import settings 

7 

8from history.views import insert_history_event 

9from history.views import manage_exceptions 

10from ptf.cmds.ptf_cmds import archiveCollectionPtfCmd 

11from ptf.cmds.ptf_cmds import archiveIssuePtfCmd 

12from ptf.cmds.ptf_cmds import archiveNumdamResourcePtfCmd 

13from ptf.cmds.ptf_cmds import get_numdam_issues_list 

14from ptf.models import Article 

15from ptf.models import Collection 

16from ptf.models import Container 

17 

18 

19@shared_task 

20def archive_numdam_collection(colid): 

21 """ 

22 Archive the files related to a collection (top level only, does not archive files of the issues) 

23 => col.xml and the collection images 

24 """ 

25 

26 if colid in settings.MERSENNE_COLLECTIONS: 

27 return 

28 

29 try: 

30 # First, check NFS mount. A Bad mount will result in a timeout with os.path.isfile or isdir 

31 # The code will hang and the Celery tasks will be killed at some point 

32 # It's better to check and raise an exception 

33 subprocess.check_call(["test", "-d", settings.NUMDAM_ISSUE_SRC_FOLDER], timeout=0.5) 

34 subprocess.check_call(["test", "-d", settings.NUMDAM_ARTICLE_SRC_FOLDER], timeout=0.5) 

35 subprocess.check_call( 

36 ["test", "-d", os.path.join(settings.NUMDAM_DATA_ROOT, colid)], timeout=0.5 

37 ) 

38 

39 archiveNumdamResourcePtfCmd({"colid": colid}).do() 

40 

41 pids = sorted(get_numdam_issues_list(colid)) 

42 for pid in pids: 

43 print("1 task (issue)") 

44 archive_numdam_issue.delay(colid, pid) 

45 

46 # except requests.exceptions.RequestException as exception: 

47 # manage_exceptions("archive", colid, colid, "WARNING", exception) 

48 # raise 

49 except Exception as exception: 

50 manage_exceptions("archive", colid, colid, "ERROR", exception) 

51 raise 

52 

53 insert_history_event( 

54 {"type": "archive", "pid": colid, "col": colid, "status": "OK", "data": {"message": ""}} 

55 ) 

56 

57 

58@shared_task 

59def archive_numdam_issue(colid, pid): 

60 """ 

61 Archive the files of an issue. Get the list of files from numdam.org 

62 """ 

63 try: 

64 archiveNumdamResourcePtfCmd({"colid": colid, "pid": pid}).do() 

65 # except requests.exceptions.RequestException as exception: 

66 # manage_exceptions("archive", pid, colid, "WARNING", exception) 

67 # raise 

68 except Exception as exception: 

69 manage_exceptions("archive", pid, colid, "ERROR", exception) 

70 raise 

71 

72 

73@shared_task 

74def archive_trammel_collection(colid, mathdoc_archive, binary_files_folder): 

75 try: 

76 # First, check NFS mount. A Bad mount will result in a timeout with os.path.isfile or isdir 

77 # The code will hang and the Celery tasks will be killed at some point 

78 # It's better to check and raise an exception 

79 subprocess.check_call(["test", "-d", settings.NUMDAM_ISSUE_SRC_FOLDER], timeout=0.5) 

80 subprocess.check_call(["test", "-d", settings.NUMDAM_ARTICLE_SRC_FOLDER], timeout=0.5) 

81 subprocess.check_call(["test", "-d", settings.CEDRAM_TEX_FOLDER], timeout=0.5) 

82 

83 collection = Collection.objects.get(pid=colid) 

84 issues = collection.content.all() 

85 

86 archiveCollectionPtfCmd({"colid": colid, "issues": issues}).do() 

87 for issue in issues: 

88 archive_trammel_resource.delay(colid, issue.pid, mathdoc_archive, binary_files_folder) 

89 

90 except Exception as exception: 

91 manage_exceptions("archive", colid, colid, "ERROR", exception) 

92 raise 

93 

94 insert_history_event( 

95 {"type": "archive", "pid": colid, "col": colid, "status": "OK", "data": {"message": ""}} 

96 ) 

97 

98 

99@shared_task 

100def archive_trammel_resource( 

101 colid=None, pid=None, mathdoc_archive=None, binary_files_folder=None, article_doi=None 

102): 

103 """ 

104 Archive the files of an issue or an article stored in the ptf-tools database (Trammel) 

105 """ 

106 try: 

107 if article_doi is not None: 

108 article = Article.objects.get(doi=article_doi) 

109 cmd = archiveIssuePtfCmd( 

110 { 

111 "pid": pid, 

112 "export_folder": mathdoc_archive, 

113 "binary_files_folder": binary_files_folder, 

114 "article": article, 

115 } 

116 ) 

117 else: 

118 issue = Container.objects.get(pid=pid) 

119 cmd = archiveIssuePtfCmd( 

120 { 

121 "pid": issue.pid, 

122 "export_folder": mathdoc_archive, 

123 "binary_files_folder": binary_files_folder, 

124 } 

125 ) 

126 cmd.do() 

127 except Exception as exception: 

128 manage_exceptions("archive", pid, colid, "ERROR", exception) 

129 raise 

130 

131 insert_history_event( 

132 {"type": "archive", "pid": pid, "col": colid, "status": "OK", "data": {"message": ""}} 

133 )