Coverage for sites/ptf_tools/history/models.py: 60%

1import datetime

2from datetime import timedelta

3from itertools import tee

5from django.db import models

6from django.db.models import JSONField

7from django.db.models import Q

8from django.db.models.functions import TruncMonth

9from django.utils import timezone

12class HistoryEventQuerySet(models.QuerySet):

13 def get_stale_events(self):

14 distinct_pids = self.order_by("pid", "type").distinct("pid", "type")

15 latest_events = [

16 self.filter(pid=event.pid, type=event.type).latest("created_on").pk

17 for event in distinct_pids

18 ]

19 return self.exclude(pk__in=latest_events)

21 def get_last_unsolved_error(self, pid, strict):

22 if strict:

23 result = self.filter(status="ERROR", pid=pid).latest("created_on")

24 else:

25 result = self.filter(status="ERROR", pid__startswith=pid).latest("created_on")

26 if "obsolete" in result.data:

27 raise HistoryEvent.DoesNotExist

28 return result

31class HistoryEvent(models.Model):

32 created_on = models.DateTimeField(db_index=True, default=timezone.now)

33 type = models.CharField(max_length=200, db_index=True)

34 pid = models.CharField(max_length=64, db_index=True, default="")

35 col = models.CharField(max_length=64, db_index=True, default="")

36 status = models.CharField(max_length=10, db_index=True, default="OK")

37 data = JSONField(default=dict)

38 objects = HistoryEventQuerySet.as_manager()

40 def __str__(self):

41 return f"{self.pid}:{self.type} - {self.created_on}"

43 def is_expandable(self):

44 result = (

45 ("ids_count" in self.data and self.data["ids_count"] > 1)

46 or ("message" in self.data and len(self.data["message"]) > 0)

47 or ("issues" in self.data and len(self.data["issues"]) > 0)

48 )

50 return result

53def test_create():

54 HistoryEvent.objects.all().delete()

56 insert_history_event(

57 {

58 "type": "edit",

59 "pid": "ALCO_2018__1_5_26_0",

60 "col": "ALCO",

61 "status": "OK",

62 "created_on": datetime.datetime(2018, 12, 3, tzinfo=datetime.UTC),

63 "data": {"message": "MR:mr1 false_positive"},

64 }

65 )

67 insert_history_event(

68 {

69 "type": "matching",

70 "pid": "AIF_2015__65_6_2331_0",

71 "col": "AIF",

72 "status": "WARNING",

73 "created_on": datetime.datetime(2018, 12, 10, tzinfo=datetime.UTC),

74 "data": {

75 "message": "MR ne répond pas",

76 "ids_count": 2,

77 "ids": [

78 {"type": "zbl", "id": "zbl1", "seq": 15},

79 {"type": "zbl", "id": "zbl2", "seq": 22},

80 ],

81 },

82 }

83 )

85 insert_history_event(

86 {

87 "type": "matching",

88 "pid": "AIF_2015__65_6_2331_0",

89 "col": "AIF",

90 "status": "OK",

91 "created_on": datetime.datetime(2018, 12, 11, 8, tzinfo=datetime.UTC),

92 "data": {"ids_count": 1, "ids": [{"type": "mr", "id": "mr1", "seq": 7}]},

93 }

94 )

96 insert_history_event(

97 {

98 "type": "edit",

99 "pid": "AIF_2015__65_6_2331_0",

100 "col": "AIF",

101 "status": "OK",

102 "created_on": datetime.datetime(2018, 12, 11, 17, tzinfo=datetime.UTC),

103 "data": {"message": "MR:mr12 checked"},

104 }

105 )

106

107 insert_history_event(

108 {

109 "type": "edit",

110 "pid": "AIF_2009__59_7_2593_0",

111 "col": "AIF",

112 "status": "OK",

113 "created_on": datetime.datetime(2018, 12, 12, 10, tzinfo=datetime.UTC),

114 "data": {"message": "[2] zbl:zbl4 false_positive"},

115 }

116 )

117

118 insert_history_event(

119 {

120 "type": "edit",

121 "pid": "AIF_2009__59_7_2593_0",

122 "col": "AIF",

123 "status": "OK",

124 "created_on": datetime.datetime(2018, 12, 12, 10, tzinfo=datetime.UTC),

125 "data": {"message": "[7] zbl:zbl8 checked"},

126 }

127 )

128

129 insert_history_event(

130 {

131 "type": "edit",

132 "pid": "AIF_2009__59_7_2700_0",

133 "col": "AIF",

134 "status": "OK",

135 "created_on": datetime.datetime(2018, 12, 12, 11, tzinfo=datetime.UTC),

136 "data": {"message": "[3] zbl:zbl5 checked"},

137 }

138 )

139

140 insert_history_event(

141 {

142 "type": "edit",

143 "pid": "AIF_2009__59_7_2611_0",

144 "col": "AIF",

145 "status": "OK",

146 "created_on": datetime.datetime(2018, 12, 12, 15, tzinfo=datetime.UTC),

147 "data": {"message": "All ids checked"},

148 }

149 )

150

151 insert_history_event(

152 {

153 "type": "matching",

154 "pid": "AIF_2007__57_7",

155 "col": "AIF",

156 "status": "ERROR",

157 "created_on": datetime.datetime(2019, 1, 4, tzinfo=datetime.UTC),

158 "data": {

159 "message": "Internal Error",

160 "ids_count": 3,

161 "articles": [

162 {

163 "pid": "AIF_2007__57_7_2143_0",

164 "ids": [{"type": "zbl", "id": "zbl3", "seq": 5}],

165 },

166 {

167 "pid": "AIF_2007__57_7_2143_0",

168 "ids": [

169 {"type": "zbl", "id": "zbl4", "seq": 8},

170 {"type": "mr", "id": "mr2", "seq": 12},

171 ],

172 },

173 ],

174 },

175 }

176 )

177

178 insert_history_event(

179 {

180 "type": "edit",

181 "pid": "AIF_2007__57_7",

182 "col": "AIF",

183 "status": "OK",

184 "created_on": datetime.datetime(2019, 1, 5, tzinfo=datetime.UTC),

185 "data": {"message": "All ids checked"},

186 }

187 )

188

189 insert_history_event(

190 {

191 "type": "edit",

192 "pid": "JEP_2014__1_7_352_0",

193 "col": "JEP",

194 "status": "OK",

195 "data": {"message": "Zbl:zbl8 checked"},

196 }

197 )

198

199 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO"})

200 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_1"})

201 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_2"})

202 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_3"})

203 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_4"})

204

205 insert_history_event({"type": "clockss", "pid": "ALL", "col": "ALL"})

206

207

208def insert_history_event(new_event):

209 """

210

211 :param new_event:

212 :return:

213 """

214 if "type" not in new_event or "pid" not in new_event or "col" not in new_event: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 raise ValueError("type, pid or col is nor set")

216

217 do_insert = True

218 last_events = HistoryEvent.objects.all().order_by("-pk")[:1]

219 last_event = last_events[0] if len(last_events) > 0 else None

220

221 if last_event:

222 time_delta = timezone.now() - last_event.created_on

223 # Check if we have to merge with the last event in the history

224 # Match 1 article generates many events per bibitemid

225 if (

226 (

227 last_event.type == new_event["type"]

228 and last_event.type == "matching"

229 and last_event.pid == new_event["pid"]

230 and time_delta < timedelta(seconds=5)

231 )

232 or

233 # Deploy, Archive, Import a collection

234 (

235 last_event.type == new_event["type"]

236 and last_event.pid == new_event["col"]

237 and time_delta < timedelta(seconds=5)

238 )

239 or

240 # Typically used when Edit one article (mark matched ids as valid)

241 (

242 last_event.type == new_event["type"]

243 and last_event.pid == new_event["pid"]

244 and last_event.status == new_event["status"]

245 )

246 ):

247 do_insert = False

248 new_fields = {}

249

250 # Downgrade the status

251 if "status" in new_event and (

252 last_event.status == "OK"

253 or (last_event.status == "WARNING" and new_event["status"] == "ERROR")

254 or new_event["status"] == "ERROR"

255 ):

256 last_event.status = new_event["status"]

257

258 last_event_data = last_event.data

259 new_event_data = new_event["data"] if "data" in new_event else None

260

261 if last_event.pid == new_event["pid"]:

262 # Merge ids

263 if new_event_data and "ids" in new_event["data"]: 263 ↛ 264line 263 didn't jump to line 264, because the condition on line 263 was never true

264 if "ids" in last_event_data:

265 new_fields["ids"] = last_event_data["ids"] + new_event_data["ids"]

266 else:

267 new_fields["ids"] = new_event_data["ids"]

268 elif new_event_data: 268 ↛ 269line 268 didn't jump to line 269, because the condition on line 268 was never true

269 if "articles" in last_event_data and "ids" in new_event_data:

270 # Article inside issue

271 # The last event was for an issue (ex: Matching) and

272 # The new event is for an article.

273 # last_event_data has 'articles' and new_even_data has 'ids'

274 found_articles = [

275 item

276 for item in last_event_data["articles"]

277 if item["pid"] == new_event["pid"]

278 ]

279 if len(found_articles) > 0:

280 article = found_articles[0]

281 article["ids"] = article["ids"] + new_event_data["ids"]

282 else:

283 last_event_data["articles"].append(

284 {"pid": new_event["pid"], "ids": new_event_data["ids"]}

285 )

286 elif "ids" in new_event_data:

287 last_event_data["articles"] = [

288 {"pid": new_event["pid"], "ids": new_event_data["ids"]}

289 ]

290 if "articles" in last_event_data:

291 new_fields["articles"] = last_event_data["articles"]

292

293 # Merge *data['articles']

294 if new_event_data and "articles" in new_event_data: 294 ↛ 295line 294 didn't jump to line 295, because the condition on line 294 was never true

295 if "articles" in last_event_data:

296 articles = last_event_data["articles"]

297 for new_article in new_event_data["articles"]:

298 found_articles = [

299 item for item in articles if item["pid"] == new_article["pid"]

300 ]

301 if len(found_articles) > 0:

302 article = found_articles[0]

303 article["ids"] = article["ids"] + new_article["ids"]

304 else:

305 articles.append(new_article)

306 else:

307 last_event_data["articles"] = new_event_data["articles"]

308 new_fields["articles"] = last_event_data["articles"]

309

310 # Update ids_count

311 if new_event_data and "ids_count" in new_event_data: 311 ↛ 312line 311 didn't jump to line 312, because the condition on line 311 was never true

312 new_count = 0

313 if "ids_count" in last_event_data:

314 new_count = last_event_data["ids_count"]

315 new_count += new_event_data["ids_count"]

316 new_fields["ids_count"] = new_count

317

318 # Deploy, Archive, Import a collection

319 # Add 'issues' to the collection event

320 if last_event.type == new_event["type"] and last_event.pid == new_event["col"]:

321 if "issues" in last_event_data:

322 if new_event["pid"] not in last_event_data["issues"]: 322 ↛ 326line 322 didn't jump to line 326, because the condition on line 322 was never false

323 last_event_data["issues"].append(new_event["pid"])

324 else:

325 last_event_data["issues"] = [new_event["pid"]]

326 new_fields["issues"] = last_event_data["issues"]

327

328 if (

329 new_event_data

330 and "message" in new_event_data

331 and len(new_event_data["message"]) > 0

332 ):

333 if ( 333 ↛ 342line 333 didn't jump to line 342

334 "message" in last_event_data

335 and new_event["type"] == "edit"

336 and last_event_data["message"] != ""

337 ):

338 new_fields["message"] = (

339 new_event_data["message"] + "<br/>" + last_event_data["message"]

340 )

341 else:

342 new_fields["message"] = new_event_data["message"]

343

344 for key, value in new_fields.items():

345 last_event.data[key] = value

346

347 if "created_on" in new_event:

348 last_event.created_on = new_event["created_on"]

349 else:

350 last_event.created_on = timezone.now()

351

352 last_event.save()

353

354 if do_insert:

355 filtered_events = (

356 HistoryEvent.objects.filter(type=new_event["type"], pid__startswith=new_event["pid"])

357 .exclude(status="OK")

358 .order_by("-pk")

359 )

360

361 first = True

362 for event in filtered_events:

363 if not first or new_event["status"] == "OK": 363 ↛ 366line 363 didn't jump to line 366, because the condition on line 363 was never false

364 event.data["obsolete"] = True

365 event.save()

366 first = False

367

368 event = HistoryEvent(type=new_event["type"], pid=new_event["pid"], col=new_event["col"])

369 if "message" in new_event: 369 ↛ 370line 369 didn't jump to line 370, because the condition on line 369 was never true

370 event.message = new_event["message"]

371 if "created_on" in new_event:

372 event.created_on = new_event["created_on"]

373 if "status" in new_event:

374 event.status = new_event["status"]

375 if "data" in new_event:

376 event.data = new_event["data"]

377

378 event.save()

379

380

381# Attempt to create a generator to group events.

382# See https://docs.python.org/2/library/itertools.html#itertools.groupby

383# It works fine in the model, the view (and the tests)

384# Unfortunately, it does not work with the template.

385# Django's render function converts the iterator into a list:

386# See django/template/defaulttags.py, line 172:

387# if not hasattr(values, '__len__'):

388# values = list(values)

389# len_values = len(values)

390# The generator is completely traversed in list(values) and is not reset

391# When the render arrives in the for loop, there is nothing left to iterate

392class GroupEventsBy:

393 # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B

394 # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D

395 def __init__(self, iterable, key=None):

396 if key is None:

397 key = lambda x: x

398 self.keyfunc = key

399 self.it = iter(iterable)

400 self.tgtkey = self.currkey = self.currvalue = object()

401 self.error_count = 0

402

403 def __iter__(self):

404 return self

405

406 def __next__(self):

407 while self.currkey == self.tgtkey:

408 self.currvalue = next(self.it) # Exit on StopIteration

409 self.currkey = self.keyfunc(self.currvalue)

410 self.tgtkey = self.currkey

411

412 # make a copy of the iterator to fetch counts

413 self.it, second_it = tee(self.it)

414 value_in_gp = self.currvalue

415 count = error_count = warning_count = 0

416 gp_key = self.tgtkey

417 while self.tgtkey == gp_key:

418 count += 1

419 if value_in_gp.status == "ERROR":

420 error_count += 1

421 if value_in_gp.status == "WARNING":

422 warning_count += 1

423 try:

424 value_in_gp = next(second_it)

425 gp_key = self.keyfunc(value_in_gp)

426 except StopIteration:

427 gp_key = object()

428

429 return (self.currkey, count, error_count, warning_count, self._grouper(self.tgtkey))

430

431 def _grouper(self, tgtkey):

432 while self.currkey == tgtkey:

433 yield self.currvalue

434 self.currvalue = next(self.it) # Exit on StopIteration

435 self.currkey = self.keyfunc(self.currvalue)

436

437

438def get_history_events(filters):

439 data = HistoryEvent.objects.all()

440

441 filter_by_month = True

442 kwargs = {}

443 for key, value in filters.items():

444 if key == "status":

445 if value == "error":

446 kwargs["status"] = "ERROR"

447 else:

448 data = data.filter(Q(status="ERROR") | Q(status="WARNING"))

449 elif key == "month":

450 if value == "all": 450 ↛ 443line 450 didn't jump to line 443, because the condition on line 450 was never false

451 filter_by_month = False

452 else:

453 kwargs[key] = value

454

455 if filter_by_month:

456 today = datetime.datetime.today()

457 kwargs["created_on__year"] = today.year

458 kwargs["created_on__month"] = today.month

459

460 data = data.filter(**kwargs).order_by("-pk").annotate(month=TruncMonth("created_on"))

461 # grouped_events = GroupEventsBy(data, lambda t: t.month)

462 grouped_events = []

463 events_in_month = []

464 curmonth = None

465 error_count = warning_count = 0

466 for event in data:

467 month = event.month

468 if month != curmonth and len(events_in_month) > 0:

469 grouped_events.append(

470 {

471 "month": curmonth,

472 "error_count": error_count,

473 "warning_count": warning_count,

474 "events_in_month": events_in_month,

475 }

476 )

477 events_in_month = []

478 curmonth = month

479 error_count = warning_count = 0

480 elif month != curmonth:

481 curmonth = month

482 events_in_month.append(event)

483 is_obsolete = "obsolete" in event.data and event.data["obsolete"]

484 if event.status == "ERROR" and not is_obsolete:

485 error_count += 1

486 if event.status == "WARNING" and not is_obsolete: 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true

487 warning_count += 1

488 if len(events_in_month) > 0: 488 ↛ 498line 488 didn't jump to line 498, because the condition on line 488 was never false

489 grouped_events.append(

490 {

491 "month": curmonth,

492 "error_count": error_count,

493 "warning_count": warning_count,

494 "events_in_month": events_in_month,

495 }

496 )

497

498 return grouped_events

499

500 # .values('month') #.annotate(count=Count('id'))

501

502 # https://stackoverflow.com/questions/8746014/django-group-by-date-day-month-year

503 # data = test1.objects.annotate(month=TruncMonth('cdate')).values('month').annotate(c=Count('id')).order_by()

504

505 # https://docs.djangoproject.com/en/2.1/topics/db/aggregation/#values

506

507

508def get_history_error_warning_counts():

509 # .exclude(data__obsolete=True) or ~Q(data__obsolete=True) do not work in Django 1.11

510 # Need to count manually

511

512 error_count = warning_count = 0

513

514 events = HistoryEvent.objects.filter(status="ERROR")

515 for event in events:

516 if "obsolete" not in event.data or not event.data["obsolete"]: 516 ↛ 515line 516 didn't jump to line 515, because the condition on line 516 was never false

517 error_count += 1

518

519 events = HistoryEvent.objects.filter(status="WARNING")

520 for event in events:

521 if "obsolete" not in event.data or not event.data["obsolete"]: 521 ↛ 520line 521 didn't jump to line 520, because the condition on line 521 was never false

522 warning_count += 1

523

524 return error_count, warning_count

525

526

527def delete_history_event(pk):

528 HistoryEvent.objects.get(pk=pk).delete()

529

530

531def get_history_last_event_by(type, pid=""):

532 last_events = HistoryEvent.objects.filter(type=type, status="OK")

533 if len(pid) > 0:

534 last_events = last_events.filter(pid__startswith=pid)

535 last_events = last_events.order_by("-pk")[:1]

536

537 last_event = last_events[0] if len(last_events) > 0 else None

538 return last_event

539

540

541def get_gap(now, event):

542 gap = ""

543 if event:

544 timelapse = now - event.created_on

545 if timelapse.days > 0:

546 gap = str(timelapse.days) + " days ago"

547 elif timelapse.seconds > 3600:

548 gap = str(int(timelapse.seconds // 3600)) + " hours ago"

549 else:

550 gap = str(int(timelapse.seconds // 60)) + " minutes ago"

551 return gap