Coverage for sites/ptf_tools/history/models.py: 60%

240 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-06-19 08:36 +0000

1import datetime 

2from datetime import timedelta 

3from itertools import tee 

4 

5from django.db import models 

6from django.db.models import JSONField 

7from django.db.models import Q 

8from django.db.models.functions import TruncMonth 

9from django.utils import timezone 

10 

11 

12class HistoryEventQuerySet(models.QuerySet): 

13 def get_stale_events(self): 

14 distinct_pids = self.order_by("pid", "type").distinct("pid", "type") 

15 latest_events = [ 

16 self.filter(pid=event.pid, type=event.type).latest("created_on").pk 

17 for event in distinct_pids 

18 ] 

19 return self.exclude(pk__in=latest_events) 

20 

21 def get_last_unsolved_error(self, pid, strict): 

22 if strict: 

23 result = self.filter(status="ERROR", pid=pid).latest("created_on") 

24 else: 

25 result = self.filter(status="ERROR", pid__startswith=pid).latest("created_on") 

26 if "obsolete" in result.data: 

27 raise HistoryEvent.DoesNotExist 

28 return result 

29 

30 

31class HistoryEvent(models.Model): 

32 created_on = models.DateTimeField(db_index=True, default=timezone.now) 

33 type = models.CharField(max_length=200, db_index=True) 

34 pid = models.CharField(max_length=64, db_index=True, default="") 

35 col = models.CharField(max_length=64, db_index=True, default="") 

36 status = models.CharField(max_length=10, db_index=True, default="OK") 

37 data = JSONField(default=dict) 

38 objects = HistoryEventQuerySet.as_manager() 

39 

40 def __str__(self): 

41 return f"{self.pid}:{self.type} - {self.created_on}" 

42 

43 def is_expandable(self): 

44 result = ( 

45 ("ids_count" in self.data and self.data["ids_count"] > 1) 

46 or ("message" in self.data and len(self.data["message"]) > 0) 

47 or ("issues" in self.data and len(self.data["issues"]) > 0) 

48 ) 

49 

50 return result 

51 

52 

53def test_create(): 

54 HistoryEvent.objects.all().delete() 

55 

56 insert_history_event( 

57 { 

58 "type": "edit", 

59 "pid": "ALCO_2018__1_5_26_0", 

60 "col": "ALCO", 

61 "status": "OK", 

62 "created_on": datetime.datetime(2018, 12, 3, tzinfo=datetime.UTC), 

63 "data": {"message": "MR:mr1 false_positive"}, 

64 } 

65 ) 

66 

67 insert_history_event( 

68 { 

69 "type": "matching", 

70 "pid": "AIF_2015__65_6_2331_0", 

71 "col": "AIF", 

72 "status": "WARNING", 

73 "created_on": datetime.datetime(2018, 12, 10, tzinfo=datetime.UTC), 

74 "data": { 

75 "message": "MR ne répond pas", 

76 "ids_count": 2, 

77 "ids": [ 

78 {"type": "zbl", "id": "zbl1", "seq": 15}, 

79 {"type": "zbl", "id": "zbl2", "seq": 22}, 

80 ], 

81 }, 

82 } 

83 ) 

84 

85 insert_history_event( 

86 { 

87 "type": "matching", 

88 "pid": "AIF_2015__65_6_2331_0", 

89 "col": "AIF", 

90 "status": "OK", 

91 "created_on": datetime.datetime(2018, 12, 11, 8, tzinfo=datetime.UTC), 

92 "data": {"ids_count": 1, "ids": [{"type": "mr", "id": "mr1", "seq": 7}]}, 

93 } 

94 ) 

95 

96 insert_history_event( 

97 { 

98 "type": "edit", 

99 "pid": "AIF_2015__65_6_2331_0", 

100 "col": "AIF", 

101 "status": "OK", 

102 "created_on": datetime.datetime(2018, 12, 11, 17, tzinfo=datetime.UTC), 

103 "data": {"message": "MR:mr12 checked"}, 

104 } 

105 ) 

106 

107 insert_history_event( 

108 { 

109 "type": "edit", 

110 "pid": "AIF_2009__59_7_2593_0", 

111 "col": "AIF", 

112 "status": "OK", 

113 "created_on": datetime.datetime(2018, 12, 12, 10, tzinfo=datetime.UTC), 

114 "data": {"message": "[2] zbl:zbl4 false_positive"}, 

115 } 

116 ) 

117 

118 insert_history_event( 

119 { 

120 "type": "edit", 

121 "pid": "AIF_2009__59_7_2593_0", 

122 "col": "AIF", 

123 "status": "OK", 

124 "created_on": datetime.datetime(2018, 12, 12, 10, tzinfo=datetime.UTC), 

125 "data": {"message": "[7] zbl:zbl8 checked"}, 

126 } 

127 ) 

128 

129 insert_history_event( 

130 { 

131 "type": "edit", 

132 "pid": "AIF_2009__59_7_2700_0", 

133 "col": "AIF", 

134 "status": "OK", 

135 "created_on": datetime.datetime(2018, 12, 12, 11, tzinfo=datetime.UTC), 

136 "data": {"message": "[3] zbl:zbl5 checked"}, 

137 } 

138 ) 

139 

140 insert_history_event( 

141 { 

142 "type": "edit", 

143 "pid": "AIF_2009__59_7_2611_0", 

144 "col": "AIF", 

145 "status": "OK", 

146 "created_on": datetime.datetime(2018, 12, 12, 15, tzinfo=datetime.UTC), 

147 "data": {"message": "All ids checked"}, 

148 } 

149 ) 

150 

151 insert_history_event( 

152 { 

153 "type": "matching", 

154 "pid": "AIF_2007__57_7", 

155 "col": "AIF", 

156 "status": "ERROR", 

157 "created_on": datetime.datetime(2019, 1, 4, tzinfo=datetime.UTC), 

158 "data": { 

159 "message": "Internal Error", 

160 "ids_count": 3, 

161 "articles": [ 

162 { 

163 "pid": "AIF_2007__57_7_2143_0", 

164 "ids": [{"type": "zbl", "id": "zbl3", "seq": 5}], 

165 }, 

166 { 

167 "pid": "AIF_2007__57_7_2143_0", 

168 "ids": [ 

169 {"type": "zbl", "id": "zbl4", "seq": 8}, 

170 {"type": "mr", "id": "mr2", "seq": 12}, 

171 ], 

172 }, 

173 ], 

174 }, 

175 } 

176 ) 

177 

178 insert_history_event( 

179 { 

180 "type": "edit", 

181 "pid": "AIF_2007__57_7", 

182 "col": "AIF", 

183 "status": "OK", 

184 "created_on": datetime.datetime(2019, 1, 5, tzinfo=datetime.UTC), 

185 "data": {"message": "All ids checked"}, 

186 } 

187 ) 

188 

189 insert_history_event( 

190 { 

191 "type": "edit", 

192 "pid": "JEP_2014__1_7_352_0", 

193 "col": "JEP", 

194 "status": "OK", 

195 "data": {"message": "Zbl:zbl8 checked"}, 

196 } 

197 ) 

198 

199 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO"}) 

200 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_1"}) 

201 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_2"}) 

202 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_3"}) 

203 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_4"}) 

204 

205 insert_history_event({"type": "clockss", "pid": "ALL", "col": "ALL"}) 

206 

207 

208def insert_history_event(new_event): 

209 """ 

210 

211 :param new_event: 

212 :return: 

213 """ 

214 if "type" not in new_event or "pid" not in new_event or "col" not in new_event: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 raise ValueError("type, pid or col is nor set") 

216 

217 do_insert = True 

218 last_events = HistoryEvent.objects.all().order_by("-pk")[:1] 

219 last_event = last_events[0] if len(last_events) > 0 else None 

220 

221 if last_event: 

222 time_delta = timezone.now() - last_event.created_on 

223 # Check if we have to merge with the last event in the history 

224 # Match 1 article generates many events per bibitemid 

225 if ( 

226 ( 

227 last_event.type == new_event["type"] 

228 and last_event.type == "matching" 

229 and last_event.pid == new_event["pid"] 

230 and time_delta < timedelta(seconds=5) 

231 ) 

232 or 

233 # Deploy, Archive, Import a collection 

234 ( 

235 last_event.type == new_event["type"] 

236 and last_event.pid == new_event["col"] 

237 and time_delta < timedelta(seconds=5) 

238 ) 

239 or 

240 # Typically used when Edit one article (mark matched ids as valid) 

241 ( 

242 last_event.type == new_event["type"] 

243 and last_event.pid == new_event["pid"] 

244 and last_event.status == new_event["status"] 

245 ) 

246 ): 

247 do_insert = False 

248 new_fields = {} 

249 

250 # Downgrade the status 

251 if "status" in new_event and ( 

252 last_event.status == "OK" 

253 or (last_event.status == "WARNING" and new_event["status"] == "ERROR") 

254 or new_event["status"] == "ERROR" 

255 ): 

256 last_event.status = new_event["status"] 

257 

258 last_event_data = last_event.data 

259 new_event_data = new_event["data"] if "data" in new_event else None 

260 

261 if last_event.pid == new_event["pid"]: 

262 # Merge ids 

263 if new_event_data and "ids" in new_event["data"]: 263 ↛ 264line 263 didn't jump to line 264, because the condition on line 263 was never true

264 if "ids" in last_event_data: 

265 new_fields["ids"] = last_event_data["ids"] + new_event_data["ids"] 

266 else: 

267 new_fields["ids"] = new_event_data["ids"] 

268 elif new_event_data: 268 ↛ 269line 268 didn't jump to line 269, because the condition on line 268 was never true

269 if "articles" in last_event_data and "ids" in new_event_data: 

270 # Article inside issue 

271 # The last event was for an issue (ex: Matching) and 

272 # The new event is for an article. 

273 # last_event_data has 'articles' and new_even_data has 'ids' 

274 found_articles = [ 

275 item 

276 for item in last_event_data["articles"] 

277 if item["pid"] == new_event["pid"] 

278 ] 

279 if len(found_articles) > 0: 

280 article = found_articles[0] 

281 article["ids"] = article["ids"] + new_event_data["ids"] 

282 else: 

283 last_event_data["articles"].append( 

284 {"pid": new_event["pid"], "ids": new_event_data["ids"]} 

285 ) 

286 elif "ids" in new_event_data: 

287 last_event_data["articles"] = [ 

288 {"pid": new_event["pid"], "ids": new_event_data["ids"]} 

289 ] 

290 if "articles" in last_event_data: 

291 new_fields["articles"] = last_event_data["articles"] 

292 

293 # Merge *data['articles'] 

294 if new_event_data and "articles" in new_event_data: 294 ↛ 295line 294 didn't jump to line 295, because the condition on line 294 was never true

295 if "articles" in last_event_data: 

296 articles = last_event_data["articles"] 

297 for new_article in new_event_data["articles"]: 

298 found_articles = [ 

299 item for item in articles if item["pid"] == new_article["pid"] 

300 ] 

301 if len(found_articles) > 0: 

302 article = found_articles[0] 

303 article["ids"] = article["ids"] + new_article["ids"] 

304 else: 

305 articles.append(new_article) 

306 else: 

307 last_event_data["articles"] = new_event_data["articles"] 

308 new_fields["articles"] = last_event_data["articles"] 

309 

310 # Update ids_count 

311 if new_event_data and "ids_count" in new_event_data: 311 ↛ 312line 311 didn't jump to line 312, because the condition on line 311 was never true

312 new_count = 0 

313 if "ids_count" in last_event_data: 

314 new_count = last_event_data["ids_count"] 

315 new_count += new_event_data["ids_count"] 

316 new_fields["ids_count"] = new_count 

317 

318 # Deploy, Archive, Import a collection 

319 # Add 'issues' to the collection event 

320 if last_event.type == new_event["type"] and last_event.pid == new_event["col"]: 

321 if "issues" in last_event_data: 

322 if new_event["pid"] not in last_event_data["issues"]: 322 ↛ 326line 322 didn't jump to line 326, because the condition on line 322 was never false

323 last_event_data["issues"].append(new_event["pid"]) 

324 else: 

325 last_event_data["issues"] = [new_event["pid"]] 

326 new_fields["issues"] = last_event_data["issues"] 

327 

328 if ( 

329 new_event_data 

330 and "message" in new_event_data 

331 and len(new_event_data["message"]) > 0 

332 ): 

333 if ( 333 ↛ 342line 333 didn't jump to line 342

334 "message" in last_event_data 

335 and new_event["type"] == "edit" 

336 and last_event_data["message"] != "" 

337 ): 

338 new_fields["message"] = ( 

339 new_event_data["message"] + "<br/>" + last_event_data["message"] 

340 ) 

341 else: 

342 new_fields["message"] = new_event_data["message"] 

343 

344 for key, value in new_fields.items(): 

345 last_event.data[key] = value 

346 

347 if "created_on" in new_event: 

348 last_event.created_on = new_event["created_on"] 

349 else: 

350 last_event.created_on = timezone.now() 

351 

352 last_event.save() 

353 

354 if do_insert: 

355 filtered_events = ( 

356 HistoryEvent.objects.filter(type=new_event["type"], pid__startswith=new_event["pid"]) 

357 .exclude(status="OK") 

358 .order_by("-pk") 

359 ) 

360 

361 first = True 

362 for event in filtered_events: 

363 if not first or new_event["status"] == "OK": 363 ↛ 366line 363 didn't jump to line 366, because the condition on line 363 was never false

364 event.data["obsolete"] = True 

365 event.save() 

366 first = False 

367 

368 event = HistoryEvent(type=new_event["type"], pid=new_event["pid"], col=new_event["col"]) 

369 if "message" in new_event: 369 ↛ 370line 369 didn't jump to line 370, because the condition on line 369 was never true

370 event.message = new_event["message"] 

371 if "created_on" in new_event: 

372 event.created_on = new_event["created_on"] 

373 if "status" in new_event: 

374 event.status = new_event["status"] 

375 if "data" in new_event: 

376 event.data = new_event["data"] 

377 

378 event.save() 

379 

380 

381# Attempt to create a generator to group events. 

382# See https://docs.python.org/2/library/itertools.html#itertools.groupby 

383# It works fine in the model, the view (and the tests) 

384# Unfortunately, it does not work with the template. 

385# Django's render function converts the iterator into a list: 

386# See django/template/defaulttags.py, line 172: 

387# if not hasattr(values, '__len__'): 

388# values = list(values) 

389# len_values = len(values) 

390# The generator is completely traversed in list(values) and is not reset 

391# When the render arrives in the for loop, there is nothing left to iterate 

392class GroupEventsBy: 

393 # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B 

394 # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D 

395 def __init__(self, iterable, key=None): 

396 if key is None: 

397 key = lambda x: x 

398 self.keyfunc = key 

399 self.it = iter(iterable) 

400 self.tgtkey = self.currkey = self.currvalue = object() 

401 self.error_count = 0 

402 

403 def __iter__(self): 

404 return self 

405 

406 def __next__(self): 

407 while self.currkey == self.tgtkey: 

408 self.currvalue = next(self.it) # Exit on StopIteration 

409 self.currkey = self.keyfunc(self.currvalue) 

410 self.tgtkey = self.currkey 

411 

412 # make a copy of the iterator to fetch counts 

413 self.it, second_it = tee(self.it) 

414 value_in_gp = self.currvalue 

415 count = error_count = warning_count = 0 

416 gp_key = self.tgtkey 

417 while self.tgtkey == gp_key: 

418 count += 1 

419 if value_in_gp.status == "ERROR": 

420 error_count += 1 

421 if value_in_gp.status == "WARNING": 

422 warning_count += 1 

423 try: 

424 value_in_gp = next(second_it) 

425 gp_key = self.keyfunc(value_in_gp) 

426 except StopIteration: 

427 gp_key = object() 

428 

429 return (self.currkey, count, error_count, warning_count, self._grouper(self.tgtkey)) 

430 

431 def _grouper(self, tgtkey): 

432 while self.currkey == tgtkey: 

433 yield self.currvalue 

434 self.currvalue = next(self.it) # Exit on StopIteration 

435 self.currkey = self.keyfunc(self.currvalue) 

436 

437 

438def get_history_events(filters): 

439 data = HistoryEvent.objects.all() 

440 

441 filter_by_month = True 

442 kwargs = {} 

443 for key, value in filters.items(): 

444 if key == "status": 

445 if value == "error": 

446 kwargs["status"] = "ERROR" 

447 else: 

448 data = data.filter(Q(status="ERROR") | Q(status="WARNING")) 

449 elif key == "month": 

450 if value == "all": 450 ↛ 443line 450 didn't jump to line 443, because the condition on line 450 was never false

451 filter_by_month = False 

452 else: 

453 kwargs[key] = value 

454 

455 if filter_by_month: 

456 today = datetime.datetime.today() 

457 kwargs["created_on__year"] = today.year 

458 kwargs["created_on__month"] = today.month 

459 

460 data = data.filter(**kwargs).order_by("-pk").annotate(month=TruncMonth("created_on")) 

461 # grouped_events = GroupEventsBy(data, lambda t: t.month) 

462 grouped_events = [] 

463 events_in_month = [] 

464 curmonth = None 

465 error_count = warning_count = 0 

466 for event in data: 

467 month = event.month 

468 if month != curmonth and len(events_in_month) > 0: 

469 grouped_events.append( 

470 { 

471 "month": curmonth, 

472 "error_count": error_count, 

473 "warning_count": warning_count, 

474 "events_in_month": events_in_month, 

475 } 

476 ) 

477 events_in_month = [] 

478 curmonth = month 

479 error_count = warning_count = 0 

480 elif month != curmonth: 

481 curmonth = month 

482 events_in_month.append(event) 

483 is_obsolete = "obsolete" in event.data and event.data["obsolete"] 

484 if event.status == "ERROR" and not is_obsolete: 

485 error_count += 1 

486 if event.status == "WARNING" and not is_obsolete: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true

487 warning_count += 1 

488 if len(events_in_month) > 0: 488 ↛ 498line 488 didn't jump to line 498, because the condition on line 488 was never false

489 grouped_events.append( 

490 { 

491 "month": curmonth, 

492 "error_count": error_count, 

493 "warning_count": warning_count, 

494 "events_in_month": events_in_month, 

495 } 

496 ) 

497 

498 return grouped_events 

499 

500 # .values('month') #.annotate(count=Count('id')) 

501 

502 # https://stackoverflow.com/questions/8746014/django-group-by-date-day-month-year 

503 # data = test1.objects.annotate(month=TruncMonth('cdate')).values('month').annotate(c=Count('id')).order_by() 

504 

505 # https://docs.djangoproject.com/en/2.1/topics/db/aggregation/#values 

506 

507 

508def get_history_error_warning_counts(): 

509 # .exclude(data__obsolete=True) or ~Q(data__obsolete=True) do not work in Django 1.11 

510 # Need to count manually 

511 

512 error_count = warning_count = 0 

513 

514 events = HistoryEvent.objects.filter(status="ERROR") 

515 for event in events: 

516 if "obsolete" not in event.data or not event.data["obsolete"]: 516 ↛ 515line 516 didn't jump to line 515, because the condition on line 516 was never false

517 error_count += 1 

518 

519 events = HistoryEvent.objects.filter(status="WARNING") 

520 for event in events: 

521 if "obsolete" not in event.data or not event.data["obsolete"]: 521 ↛ 520line 521 didn't jump to line 520, because the condition on line 521 was never false

522 warning_count += 1 

523 

524 return error_count, warning_count 

525 

526 

527def delete_history_event(pk): 

528 HistoryEvent.objects.get(pk=pk).delete() 

529 

530 

531def get_history_last_event_by(type, pid=""): 

532 last_events = HistoryEvent.objects.filter(type=type, status="OK") 

533 if len(pid) > 0: 

534 last_events = last_events.filter(pid__startswith=pid) 

535 last_events = last_events.order_by("-pk")[:1] 

536 

537 last_event = last_events[0] if len(last_events) > 0 else None 

538 return last_event 

539 

540 

541def get_gap(now, event): 

542 gap = "" 

543 if event: 

544 timelapse = now - event.created_on 

545 if timelapse.days > 0: 

546 gap = str(timelapse.days) + " days ago" 

547 elif timelapse.seconds > 3600: 

548 gap = str(int(timelapse.seconds // 3600)) + " hours ago" 

549 else: 

550 gap = str(int(timelapse.seconds // 60)) + " minutes ago" 

551 return gap