Coverage for portality/bll/services/application.py: 94%

519 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1import csv 

2import json 

3import logging 

4import re 

5from io import StringIO 

6 

7from portality import constants, lock, models 

8from portality.bll import DOAJ, exceptions 

9from portality.core import app 

10from portality.crosswalks.journal_form import JournalFormXWalk 

11from portality.crosswalks.journal_questions import Journal2PublisherUploadQuestionsXwalk, QuestionTransformError 

12from portality.forms.application_forms import ApplicationFormFactory 

13from portality.lib import csv_utils, dates, httputil 

14from portality.lib.argvalidate import argvalidate 

15from portality.ui.messages import Messages 

16from portality import datasets 

17 

18 

19class ApplicationService(object): 

20 """ 

21 ~~Application:Service->DOAJ:Service~~ 

22 """ 

23 

24 def auto_assign_ur_editor_group(self, ur: models.Application): 

25 """ 

26 Auto assign editor group to the update request 

27 :param ur: 

28 :return: 

29 """ 

30 if not app.config.get("AUTO_ASSIGN_UR_EDITOR_GROUP", False): 

31 return ur 

32 

33 target = None 

34 reason = "" 

35 

36 by_owner = models.URReviewRoute.by_account(ur.owner) 

37 if by_owner is not None: 

38 reason = Messages.AUTOASSIGN__OWNER_MAPPED.format(owner=ur.owner, target=by_owner.target) 

39 target = by_owner.target 

40 

41 if target is None: 

42 by_country = models.URReviewRoute.by_country(ur.bibjson().publisher_country) 

43 if by_country is not None: 

44 reason = Messages.AUTOASSIGN__COUNTRY_MAPPED.format(country=by_country.country_name, country_code=by_country.country_code, target=by_country.target) 

45 target = by_country.target 

46 

47 if target is None: 

48 return ur 

49 

50 if ur.editor_group == target: 

51 # UR is already assigned to the correct editor group 

52 return ur 

53 

54 id = models.EditorGroup.group_exists_by_name(target) 

55 if id is None: 

56 ur.add_note(Messages.AUTOASSIGN__NOTE__EDITOR_GROUP_MISSING.format(target=target)) 

57 return ur 

58 

59 ur.set_editor_group(target) 

60 ur.remove_editor() 

61 ur.add_note(Messages.AUTOASSIGN__NOTE__ASSIGN.format(target=target, reason=reason)) 

62 return ur 

63 

64 def retrieve_ur_editor_group_sheets(self, keep_history=1): 

65 start = dates.now() 

66 

67 publisher_sheet = app.config.get("AUTO_ASSIGN_EDITOR_BY_PUBLISHER_SHEET") 

68 country_sheet = app.config.get("AUTO_ASSIGN_EDITOR_BY_COUNTRY_SHEET") 

69 

70 presp = httputil.get(publisher_sheet) 

71 if presp.status_code != 200: 

72 raise exceptions.RemoteServiceException("Failed to retrieve publisher sheet from {x}".format(x=publisher_sheet)) 

73 

74 cresp = httputil.get(country_sheet) 

75 if cresp.status_code != 200: 

76 raise exceptions.RemoteServiceException("Failed to retrieve country sheet from {x}".format(x=country_sheet)) 

77 

78 presp.encoding = "utf-8" 

79 pdata = presp.text 

80 if pdata is None or pdata == "": 

81 raise ValueError("Publisher sheet is empty at {x}".format(x=publisher_sheet)) 

82 

83 cresp.encoding = "utf-8" 

84 cdata = cresp.text 

85 if cdata is None or cdata == "": 

86 raise ValueError("Country sheet is empty at {x}".format(x=country_sheet)) 

87 

88 preader = csv.reader(StringIO(pdata)) 

89 creader = csv.reader(StringIO(cdata)) 

90 

91 preader.__next__() 

92 creader.__next__() 

93 

94 routers = [] 

95 

96 for i, row in enumerate(preader): 

97 account = row[1].strip() 

98 group = row[2].strip() 

99 

100 if account is None or account == "": 

101 raise ValueError(f"Publisher Sheet: Account is empty on row {i+2}") 

102 if group is None or group == "": 

103 raise ValueError(f"Publisher Sheet: Group is empty on row {i+2}") 

104 

105 acc = models.Account.pull(account) 

106 if acc is None: 

107 raise ValueError(f"Publisher Sheet: Account `{account}` not found in DOAJ; row {i+2}") 

108 if models.EditorGroup.group_exists_by_name(group) is None: 

109 raise ValueError(f"Publisher Sheet: Group `{group}` not found in DOAJ; row {i+2}") 

110 

111 router = models.URReviewRoute() 

112 router.account_id = acc.id 

113 router.target = group 

114 router.pre_save_prep() 

115 routers.append(router) 

116 

117 for i, row in enumerate(creader): 

118 country = row[0].strip() 

119 group = row[1].strip() 

120 

121 if country is None or country == "": 

122 raise ValueError(f"Country Sheet: Country is empty on row {i+2}") 

123 if group is None or group == "": 

124 raise ValueError(f"Country Sheet: Group is empty on row {i+2}") 

125 

126 if models.EditorGroup.group_exists_by_name(group) is None: 

127 raise ValueError(f"Country Sheet: Group `{group}` not found in DOAJ; row {i+2}") 

128 

129 cc = datasets.get_country_code(country, fail_if_not_found=True) 

130 if cc is None: 

131 raise ValueError(f"Country Sheet: Country `{country}` does not match an existing ISO country name; row {i+2}") 

132 

133 router = models.URReviewRoute() 

134 router.country_code = cc 

135 router.country_name = country 

136 router.target = group 

137 router.pre_save_prep() 

138 routers.append(router) 

139 

140 # if we get to here we have two valid sheets, so we can update 

141 # the local data using an index rollover technique 

142 if len(routers) > 0: 

143 mapping = routers[0].mappings() 

144 docs = [router.data for router in routers] 

145 models.URReviewRoute.create_and_seed_index_and_rollover_alias(docs, mapping, keep_history) 

146 

147 return routers 

148 

149 

150 @staticmethod 

151 def prevent_concurrent_ur_submission(ur: models.Application, record_if_not_concurrent=True): 

152 """ 

153 Prevent duplicated update request submission 

154 :param ur: 

155 :param record_if_not_concurrent: 

156 :return: 

157 """ 

158 cs = DOAJ.concurrencyPreventionService() 

159 

160 if ur.current_journal is not None and ur.id is not None: 

161 if cs.check_concurrency(ur.current_journal, ur.id): 

162 raise exceptions.ConcurrentUpdateRequestException(Messages.CONCURRENT_UPDATE_REQUEST) 

163 

164 if record_if_not_concurrent: 

165 cs.store_concurrency(ur.current_journal, ur.id, timeout=app.config.get("UR_CONCURRENCY_TIMEOUT", 10)) 

166 

167 def reject_application(self, application, account, provenance=True, note=None, manual_update=True): 

168 """ 

169 Reject an application. This will: 

170 * set the application status to "rejected" (if not already) 

171 * remove the current_journal field, and move it to related_journal (if needed) 

172 * remove the current_application field from the related journal (if needed) 

173 * save the application 

174 * write a provenance record for the rejection (if requested) 

175 

176 :param application: 

177 :param account: 

178 :param provenance: 

179 :param note: 

180 :param manual_update: 

181 :return: 

182 """ 

183 # first validate the incoming arguments to ensure that we've got the right thing 

184 argvalidate("reject_application", [ 

185 {"arg": application, "instance" : models.Application, "allow_none" : False, "arg_name" : "application"}, 

186 {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"}, 

187 {"arg" : provenance, "instance" : bool, "allow_none" : False, "arg_name" : "provenance"}, 

188 {"arg" : note, "instance" : str, "allow_none" : True, "arg_name" : "note"}, 

189 {"arg" : manual_update, "instance" : bool, "allow_none" : False, "arg_name" : "manual_update"} 

190 ], exceptions.ArgumentException) 

191 

192 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering reject_application") 

193 

194 # ~~->Journal:Service~~ 

195 journalService = DOAJ.journalService() 

196 

197 # check we're allowed to carry out this action 

198 if not account.has_role("reject_application"): 

199 raise exceptions.AuthoriseException(message="This user is not allowed to reject applications", reason=exceptions.AuthoriseException.WRONG_ROLE) 

200 

201 # ensure the application status is "rejected" 

202 if application.application_status != constants.APPLICATION_STATUS_REJECTED: 

203 application.set_application_status(constants.APPLICATION_STATUS_REJECTED) 

204 

205 # set the rejection date 

206 application.date_rejected = dates.now_str() 

207 

208 # add the note to the application 

209 if note is not None: 

210 application.add_note(note) 

211 

212 # retrieve the id of the current journal if there is one 

213 cj_id = application.current_journal 

214 cj = None 

215 

216 # if there is a current_journal record, remove it, and record 

217 # it as a related journal. This will let us come back later and know 

218 # which journal record this was intended as an update against if needed. 

219 if cj_id is not None: 

220 cj, _ = journalService.journal(cj_id) 

221 application.remove_current_journal() 

222 if cj is not None: 

223 application.set_related_journal(cj_id) 

224 cj.remove_current_application() 

225 

226 # if there is a current journal, we will have modified it above, so save it 

227 if cj is not None: 

228 saved = cj.save() 

229 if saved is None: 

230 raise exceptions.SaveException("Save on current_journal in reject_application failed") 

231 

232 # if we were asked to record this as a manual update, record that on the application 

233 if manual_update: 

234 application.set_last_manual_update() 

235 

236 saved = application.save() 

237 if saved is None: 

238 raise exceptions.SaveException("Save on application in reject_application failed") 

239 

240 # record a provenance record that this action took place 

241 if provenance: 

242 # ~~->Provenance:Model~~ 

243 models.Provenance.make(account, constants.PROVENANCE_STATUS_REJECTED, application) 

244 

245 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed reject_application") 

246 

247 

248 def unreject_application(self, 

249 application: models.Application, 

250 account: models.Account, 

251 manual_update: bool = True, 

252 disallow_status: list = None): 

253 """ 

254 Un-reject an application. This will: 

255 * check that the application status is no longer "rejected" (throw an error if it is) 

256 * check for a related journal, and if one is present, promote that to current_journal (if no other UR exists) 

257 * save the application 

258 

259 :param application: 

260 :param account: 

261 :param manual_update: 

262 :param disallow_status: statuses that we are not allowed to unreject to (excluding rejected, which is always disallowed) 

263 :return: 

264 """ 

265 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering unreject_application") 

266 

267 if application is None: 

268 raise exceptions.ArgumentException(Messages.BLL__UNREJECT_APPLICATION__NO_APPLICATION) 

269 

270 if account is None: 

271 raise exceptions.ArgumentException(Messages.BLL__UNREJECT_APPLICATION__NO_ACCOUNT) 

272 

273 # check we're allowed to carry out this action 

274 if not account.has_role("unreject_application"): 

275 raise exceptions.AuthoriseException(message=Messages.BLL__UNREJECT_APPLICATION__WRONG_ROLE, 

276 reason=exceptions.AuthoriseException.WRONG_ROLE) 

277 

278 # ensure the application status is not "rejected" 

279 if application.application_status == constants.APPLICATION_STATUS_REJECTED: 

280 raise exceptions.IllegalStatusException(message=Messages.BLL__UNREJECT_APPLICATION__ILLEGAL_STATE_REJECTED.format(id=application.id)) 

281 

282 # by default reject transitions to the accepted status (because acceptance implies other processing that this 

283 # method does not handle). You can override this by passing in an empty list 

284 if disallow_status is None: 

285 disallow_status = [constants.APPLICATION_STATUS_ACCEPTED] 

286 if application.application_status in disallow_status: 

287 raise exceptions.IllegalStatusException( 

288 message=Messages.BLL__UNREJECT_APPLICATION__ILLEGAL_STATE_DISALLOWED.format( 

289 id=application.id, x=application.application_status 

290 )) 

291 

292 rjid = application.related_journal 

293 if rjid: 

294 ur = models.Application.find_latest_by_current_journal(rjid) 

295 if ur: 

296 raise exceptions.DuplicateUpdateRequest(message=Messages.BLL__UNREJECT_APPLICATION__DUPLICATE_UR.format( 

297 id=application.id, 

298 urid=ur.id, 

299 jid=rjid 

300 )) 

301 

302 rj = models.Journal.pull(rjid) 

303 if rj is None: 

304 raise exceptions.NoSuchObjectException(Messages.BLL__UNREJECT_APPLICATION__JOURNAL_MISSING.format( 

305 jid=rjid, id=application.id 

306 )) 

307 

308 # update the application's view of the current journal 

309 application.set_current_journal(rjid) 

310 application.remove_related_journal() 

311 

312 # update the journal's view of the current application 

313 rj.set_current_application(application.id) 

314 rj.remove_related_application(application.id) 

315 

316 saved = rj.save() 

317 if saved is None: 

318 raise exceptions.SaveException(Messages.BLL__UNREJECT_APPLICATION__SAVE_FAIL.format( 

319 obj="current_journal", 

320 id=rj.id 

321 )) 

322 

323 # if we were asked to record this as a manual update, record that on the application 

324 if manual_update: 

325 application.set_last_manual_update() 

326 

327 saved = application.save() 

328 if saved is None: 

329 raise exceptions.SaveException(Messages.BLL__UNREJECT_APPLICATION__SAVE_FAIL.format( 

330 obj="application", 

331 id=application.id 

332 )) 

333 

334 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed unreject_application") 

335 

336 

337 def accept_application(self, application, account, manual_update=True, provenance=True, save_journal=True, save_application=True): 

338 """ 

339 Take the given application and create the Journal object in DOAJ for it. 

340 

341 The account provided must have permission to create journals from applications. 

342 

343 :param application: The application to be converted 

344 :param account: The account doing the conversion 

345 :param manual_update: Whether to record this update as a manual update on both the application and journal objects 

346 :param provenance: Whether to write provenance records for this operation 

347 :param save_journal: Whether to save the journal that is produced 

348 :param save_application: Whether to save the application after it has been modified 

349 :return: The journal that was created 

350 """ 

351 # first validate the incoming arguments to ensure that we've got the right thing 

352 argvalidate("accept_application", [ 

353 {"arg": application, "instance" : models.Suggestion, "allow_none" : False, "arg_name" : "application"}, 

354 {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"}, 

355 {"arg" : manual_update, "instance" : bool, "allow_none" : False, "arg_name" : "manual_update"}, 

356 {"arg" : provenance, "instance" : bool, "allow_none" : False, "arg_name" : "provenance"}, 

357 {"arg" : save_journal, "instance" : bool, "allow_none" : False, "arg_name" : "save_journal"}, 

358 {"arg" : save_application, "instance" : bool, "allow_none" : False, "arg_name" : "save_application"} 

359 ], exceptions.ArgumentException) 

360 

361 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering accept_application") 

362 

363 # ensure that the account holder has a suitable role 

364 if not account.has_role("accept_application"): 

365 raise exceptions.AuthoriseException( 

366 message="User {x} is not permitted to accept application {y}".format(x=account.id, y=application.id), 

367 reason=exceptions.AuthoriseException.WRONG_ROLE) 

368 

369 # ensure the application status is "accepted" 

370 if application.application_status != constants.APPLICATION_STATUS_ACCEPTED: 

371 application.set_application_status(constants.APPLICATION_STATUS_ACCEPTED) 

372 

373 # make the resulting journal (and save it if requested) 

374 j = self.application_2_journal(application, manual_update=manual_update) 

375 if save_journal is True: 

376 saved = j.save() 

377 if saved is None: 

378 raise exceptions.SaveException("Save of resulting journal in accept_application failed") 

379 

380 # retrieve the id of the current journal if there is one 

381 cj = application.current_journal 

382 

383 # if there is a current_journal record, remove it 

384 if cj is not None: 

385 application.remove_current_journal() 

386 

387 # set the relationship with the journal 

388 application.set_related_journal(j.id) 

389 

390 # if we were asked to record this as a manual update, record that on the application 

391 # (the journal is done implicitly above) 

392 if manual_update: 

393 application.set_last_manual_update() 

394 

395 if provenance: 

396 # record the event in the provenance tracker 

397 models.Provenance.make(account, constants.PROVENANCE_STATUS_ACCEPTED, application) 

398 

399 # save the application if requested 

400 if save_application is True: 

401 application.save() 

402 

403 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed accept_application") 

404 

405 return j 

406 

407 def reject_update_request_of_journal(self, journal_id, account): 

408 """ 

409 Rejects update request associated with journal 

410 

411 :param journal_id: 

412 :param account: 

413 :return: Journal object 

414 """ 

415 ur = models.Application.find_latest_by_current_journal(journal_id) # ~~->Application:Model~~ 

416 if ur: 

417 self.reject_application(ur, account, note=Messages.AUTOMATICALLY_REJECTED_UPDATE_REQUEST_NOTE) 

418 return ur 

419 else: 

420 return None 

421 

422 def reject_update_request_of_journals(self, ids, account): 

423 """ 

424 Rejects update request associated with journal 

425 

426 :param ids: 

427 :param account: 

428 :return: Journal object 

429 """ 

430 ur_ids = [] 

431 for journal_id in ids: 

432 ur = self.reject_update_request_of_journal(journal_id, account) 

433 if ur: 

434 ur_ids.append(ur.id) 

435 return ur_ids 

436 

437 def update_request_for_journal(self, journal_id, account=None, lock_timeout=None, lock_records=True): 

438 """ 

439 Obtain an update request application object for the journal with the given journal_id 

440 

441 An update request may either be loaded from the database, if it already exists, or created 

442 in-memory if it has not previously been created. 

443 

444 If an account is provided, this will validate that the account holder is allowed to make 

445 the conversion from journal to application, if a conversion is required. 

446 

447 When this request runs, the journal will be locked to the provided account if an account is 

448 given. If the application is loaded from the database, this will also be locked for the account 

449 holder. 

450 

451 :param journal_id: 

452 :param account: 

453 :return: a tuple of (Application Object, Journal Lock, Application Lock) 

454 """ 

455 # first validate the incoming arguments to ensure that we've got the right thing 

456 argvalidate("update_request_for_journal", [ 

457 {"arg": journal_id, "instance" : str, "allow_none" : False, "arg_name" : "journal_id"}, 

458 {"arg" : account, "instance" : models.Account, "allow_none" : True, "arg_name" : "account"}, 

459 {"arg" : lock_timeout, "instance" : int, "allow_none" : True, "arg_name" : "lock_timeout"} 

460 ], exceptions.ArgumentException) 

461 

462 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering update_request_for_journal") 

463 

464 # ~~-> Journal:Service~~ 

465 # ~~-> AuthNZ:Service~~ 

466 journalService = DOAJ.journalService() 

467 authService = DOAJ.authorisationService() 

468 

469 # first retrieve the journal, and return empty if there isn't one. 

470 # We don't attempt to obtain a lock at this stage, as we want to check that the user is authorised first 

471 journal_lock = None 

472 journal, _ = journalService.journal(journal_id) 

473 if journal is None: 

474 app.logger.info("Request for journal {x} did not find anything in the database".format(x=journal_id)) 

475 return None, None, None 

476 

477 # if the journal is not in_doaj, we won't create an update request for it 

478 if not journal.is_in_doaj(): 

479 app.logger.info("Request for journal {x} found it is not in_doaj; will not create update request".format(x=journal_id)) 

480 return None, None, None 

481 

482 # retrieve the latest application attached to this journal 

483 application_lock = None 

484 application = models.Suggestion.find_latest_by_current_journal(journal_id) # ~~->Application:Model~~ 

485 

486 # if no such application exists, create one in memory (this will check that the user is permitted to create one) 

487 # at the same time, create the lock for the journal. This will throw an AuthorisedException or a Locked exception 

488 # (in that order of preference) if any problems arise. 

489 if application is None: 

490 app.logger.info("No existing update request for journal {x}; creating one".format(x=journal.id)) 

491 application = journalService.journal_2_application(journal, account=account) 

492 application.set_is_update_request(True) 

493 if account is not None: 

494 if lock_records: 

495 journal_lock = lock.lock("journal", journal_id, account.id) 

496 

497 # otherwise check that the user (if given) has the rights to edit the application 

498 # then lock the application and journal to the account. 

499 # If a lock cannot be obtained, unlock the journal and application before we return 

500 elif account is not None: 

501 try: 

502 authService.can_edit_application(account, application) 

503 if lock_records: 

504 application_lock = lock.lock("suggestion", application.id, account.id) 

505 journal_lock = lock.lock("journal", journal_id, account.id) 

506 except lock.Locked as e: 

507 if application_lock is not None: application_lock.delete() 

508 if journal_lock is not None: journal_lock.delete() 

509 raise 

510 except exceptions.AuthoriseException as e: 

511 msg = "Account {x} is not permitted to edit the current update request on journal {y}".format(x=account.id, y=journal.id) 

512 app.logger.info(msg) 

513 e.args += (msg,) 

514 raise 

515 

516 app.logger.info("Using existing application {y} as update request for journal {x}".format(y=application.id, x=journal.id)) 

517 

518 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed update_request_for_journal; return application object") 

519 

520 return application, journal_lock, application_lock 

521 

522 def application_2_journal(self, application, manual_update=True): 

523 # first validate the incoming arguments to ensure that we've got the right thing 

524 argvalidate("application_2_journal", [ 

525 {"arg": application, "instance" : models.Suggestion, "allow_none" : False, "arg_name" : "application"}, 

526 {"arg" : manual_update, "instance" : bool, "allow_none" : False, "arg_name" : "manual_update"} 

527 ], exceptions.ArgumentException) 

528 

529 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering application_2_journal") 

530 

531 # create a new blank journal record, which we can build up 

532 journal = models.Journal() # ~~->Journal:Model~~ 

533 

534 # first thing is to copy the bibjson as-is wholesale, 

535 abj = application.bibjson() 

536 journal.set_bibjson(abj) 

537 jbj = journal.bibjson() 

538 

539 # now carry over key administrative properties from the application itself 

540 # * notes 

541 # * editor 

542 # * editor_group 

543 # * owner 

544 # * application date 

545 # notes = application.notes 

546 notes = application.get_detached_notes() 

547 

548 if application.editor is not None: 

549 journal.set_editor(application.editor) 

550 if application.editor_group is not None: 

551 journal.set_editor_group(application.editor_group) 

552 for note in notes: 

553 journal.add_note_by_dict(note) 

554 if application.owner is not None: 

555 journal.set_owner(application.owner) 

556 if application.date_applied is not None: 

557 journal.set_date_applied(application.date_applied) 

558 

559 b = application.bibjson() 

560 if b.pissn == "": 

561 b.add_identifier("pissn", None) 

562 if b.eissn == "": 

563 b.add_identifier("eissn", None) 

564 

565 # no relate the journal to the application and place it in_doaj 

566 journal.add_related_application(application.id, dates.now_str()) 

567 journal.set_in_doaj(True) 

568 

569 # if we've been called in the context of a manual update, record that 

570 if manual_update: 

571 journal.set_last_manual_update() 

572 

573 # if this is an update to an existing journal, then we can also port information from 

574 # that journal 

575 if application.current_journal is not None: 

576 cj = models.Journal.pull(application.current_journal) 

577 if cj is not None: 

578 # carry the id and the created date 

579 journal.set_id(cj.id) 

580 journal.set_created(cj.created_date) 

581 

582 # bring forward any notes from the old journal record 

583 old_notes = cj.get_detached_notes() 

584 for note in old_notes: 

585 journal.add_note_by_dict(note) 

586 

587 # bring forward any related applications 

588 related = cj.related_applications 

589 for r in related: 

590 journal.add_related_application(r.get("application_id"), r.get("date_accepted"), r.get("status")) 

591 

592 # carry over any properties that are not already set from the application 

593 # * editor & editor_group (together or not at all) 

594 # * owner 

595 if journal.editor is None and journal.editor_group is None: 

596 journal.set_editor(cj.editor) 

597 journal.set_editor_group(cj.editor_group) 

598 if journal.owner is None: 

599 journal.set_owner(cj.owner) 

600 

601 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completing application_2_journal") 

602 

603 return journal 

604 

605 def application(self, application_id, lock_application=False, lock_account=None, lock_timeout=None): 

606 """ 

607 Function to retrieve an application by its id 

608 

609 :param application_id: the id of the application 

610 :param: lock_application: should we lock the resource on retrieval 

611 :param: lock_account: which account is doing the locking? Must be present if lock_journal=True 

612 :param: lock_timeout: how long to lock the resource for. May be none, in which case it will default 

613 :return: Tuple of (Suggestion Object, Lock Object) 

614 """ 

615 # first validate the incoming arguments to ensure that we've got the right thing 

616 argvalidate("application", [ 

617 {"arg": application_id, "allow_none" : False, "arg_name" : "application_id"}, 

618 {"arg": lock_application, "instance" : bool, "allow_none" : False, "arg_name" : "lock_journal"}, 

619 {"arg": lock_account, "instance" : models.Account, "allow_none" : True, "arg_name" : "lock_account"}, 

620 {"arg": lock_timeout, "instance" : int, "allow_none" : True, "arg_name" : "lock_timeout"} 

621 ], exceptions.ArgumentException) 

622 

623 # pull the application from the database 

624 application = models.Suggestion.pull(application_id) 

625 

626 # if we've retrieved the journal, and a lock is requested, request it 

627 the_lock = None 

628 if application is not None and lock_application: 

629 if lock_account is not None: 

630 # ~~->Lock:Feature~~ 

631 the_lock = lock.lock(constants.LOCK_APPLICATION, application_id, lock_account.id, lock_timeout) 

632 else: 

633 raise exceptions.ArgumentException("If you specify lock_application on application retrieval, you must also provide lock_account") 

634 

635 return application, the_lock 

636 

637 def delete_application(self, application_id, account): 

638 """ 

639 Function to delete an application, and all references to it in other objects (current and related journals) 

640 

641 The application and all related journals need to be locked before this process can proceed, so you may get a 

642 lock.Locked exception 

643 

644 :param application_id: 

645 :param account: 

646 :return: 

647 """ 

648 # first validate the incoming arguments to ensure that we've got the right thing 

649 argvalidate("delete_application", [ 

650 {"arg": application_id, "instance" : str, "allow_none" : False, "arg_name" : "application_id"}, 

651 {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"} 

652 ], exceptions.ArgumentException) 

653 

654 # ~~-> Journal:Service~~ 

655 # ~~-> AuthNZ:Service~~ 

656 journalService = DOAJ.journalService() 

657 authService = DOAJ.authorisationService() 

658 

659 # get hold of a copy of the application. If there isn't one, our work here is done 

660 # (note the application could be locked, in which case this will raise a lock.Locked exception) 

661 

662 # get the application 

663 application, _ = self.application(application_id) 

664 if application is None: 

665 raise exceptions.NoSuchObjectException 

666 

667 # determine if the user can edit the application 

668 authService.can_edit_application(account, application) 

669 

670 # attempt to lock the record (this may raise a Locked exception) 

671 # ~~-> Lock:Feature~~ 

672 alock = lock.lock(constants.LOCK_APPLICATION, application_id, account.id) 

673 

674 # obtain the current journal, with associated lock 

675 current_journal = None 

676 cjlock = None 

677 if application.current_journal is not None: 

678 try: 

679 current_journal, cjlock = journalService.journal(application.current_journal, lock_journal=True, lock_account=account) 

680 except lock.Locked as e: 

681 # if the resource is locked, we have to back out 

682 if alock is not None: alock.delete() 

683 raise 

684 

685 # obtain the related journal, with associated lock 

686 related_journal = None 

687 rjlock = None 

688 if application.related_journal is not None: 

689 try: 

690 related_journal, rjlock = journalService.journal(application.related_journal, lock_journal=True, lock_account=account) 

691 except lock.Locked: 

692 # if the resource is locked, we have to back out 

693 if alock is not None: alock.delete() 

694 if cjlock is not None: cjlock.delete() 

695 raise 

696 

697 try: 

698 if current_journal is not None: 

699 current_journal.remove_current_application() 

700 saved = current_journal.save() 

701 if saved is None: 

702 raise exceptions.SaveException("Unable to save journal record") 

703 

704 if related_journal is not None: 

705 relation_record = related_journal.related_application_record(application_id) 

706 if relation_record is None: 

707 relation_record = {} 

708 related_journal.add_related_application(application_id, relation_record.get("date_accepted"), "deleted") 

709 saved = related_journal.save() 

710 if saved is None: 

711 raise exceptions.SaveException("Unable to save journal record") 

712 

713 application.delete() 

714 

715 finally: 

716 if alock is not None: alock.delete() 

717 if cjlock is not None: cjlock.delete() 

718 if rjlock is not None: rjlock.delete() 

719 

720 return 

721 

722 def validate_update_csv(self, file_path, account: models.Account): 

723 # Open with encoding that deals with the Byte Order Mark since we're given files from Windows. 

724 with open(file_path, "r", encoding='utf-8-sig') as file: 

725 reader = csv.DictReader(file) 

726 validation = CSVValidationReport() 

727 

728 # verify header row with current CSV headers, report errors 

729 header_row = reader.fieldnames 

730 allowed_headers = Journal2PublisherUploadQuestionsXwalk.question_list() 

731 lower_case_allowed_headers = list(map(str.lower, allowed_headers)) 

732 required_headers = Journal2PublisherUploadQuestionsXwalk.required_questions() 

733 

734 # Always perform a match check on supplied headers, not counting order 

735 for i, h in enumerate(header_row): 

736 if h and h not in allowed_headers: 

737 if h.lower() in lower_case_allowed_headers: 

738 expected = allowed_headers[lower_case_allowed_headers.index(h.lower())] 

739 validation.header(validation.WARN, i, Messages.JOURNAL_CSV_VALIDATE__HEADER_CASE_MISMATCH.format(h=h, expected=expected)) 

740 else: 

741 validation.header(validation.ERROR, i, Messages.JOURNAL_CSV_VALIDATE__INVALID_HEADER.format(h=h)) 

742 

743 missing_required = False 

744 for rh in required_headers: 

745 if rh not in header_row: 

746 validation.general(validation.ERROR, Messages.JOURNAL_CSV_VALIDATE__REQUIRED_HEADER_MISSING.format(h=rh)) 

747 missing_required = True 

748 if missing_required: 

749 return validation 

750 

751 # Talking about spreadsheets, so we start at 1 

752 row_ix = 1 

753 

754 # ~~ ->$JournalUpdateByCSV:Feature ~~ 

755 for row in csv_utils.trim_trailing_empty_rows(reader): 

756 row_ix += 1 

757 validation.log(f'Processing CSV row {row_ix}') 

758 

759 # Skip empty rows 

760 if csv_utils.is_empty_row(row): 

761 validation.log("Skipping empty row {x}.".format(x=row_ix)) 

762 continue 

763 

764 # Pull by ISSNs 

765 issns = [ 

766 row.get(Journal2PublisherUploadQuestionsXwalk.q("pissn")), 

767 row.get(Journal2PublisherUploadQuestionsXwalk.q("eissn")) 

768 ] 

769 issns = [issn for issn in issns if issn is not None and issn != ""] 

770 

771 try: 

772 j = models.Journal.find_by_issn(issns, in_doaj=True, max=1).pop(0) 

773 except IndexError: 

774 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__MISSING_JOURNAL.format(issns=", ".join(issns))) 

775 continue 

776 

777 # Confirm that the account is allowed to update the journal (is admin or owner) 

778 if not account.is_super and account.id != j.owner: 

779 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__OWNER_MISMATCH.format(acc=account.id, issns=", ".join(issns))) 

780 continue 

781 

782 # By this point the issns are confirmed as matching a journal that belongs to the publisher 

783 validation.log('Validating update for journal with ID ' + j.id) 

784 

785 # Load remaining rows into application form as an update 

786 # ~~ ^->JournalQuestions:Crosswalk ~~ 

787 journal_form = JournalFormXWalk.obj2form(j) 

788 journal_questions = Journal2PublisherUploadQuestionsXwalk.journal2question(j) 

789 

790 try: 

791 update_form, updates = Journal2PublisherUploadQuestionsXwalk.question2form(j, row) 

792 except QuestionTransformError as e: 

793 question = Journal2PublisherUploadQuestionsXwalk.q(e.key) 

794 journal_value = [y for x, y in journal_questions if x == question][0] 

795 validation.value(validation.ERROR, row_ix, header_row.index(question), 

796 Messages.JOURNAL_CSV_VALIDATE__INVALID_DATA.format(question=question), 

797 was=journal_value, now=e.value) 

798 continue 

799 

800 if len(updates) == 0: 

801 validation.row(validation.WARN, row_ix, Messages.JOURNAL_CSV_VALIDATE__NO_DATA_CHANGE) 

802 continue 

803 

804 # if we get to here, then there are updates 

805 [validation.log(upd) for upd in updates] 

806 

807 # If a field is disabled in the UR Form Context, then we must confirm that the form data from the 

808 # file has not changed from that provided in the source 

809 formulaic_context = ApplicationFormFactory.context("update_request") 

810 disabled_fields = formulaic_context.disabled_fields() 

811 trip_wire = False 

812 for field in disabled_fields: 

813 field_name = field.get("name") 

814 question = Journal2PublisherUploadQuestionsXwalk.q(field_name) 

815 update_value = update_form.get(field_name) 

816 journal_value = journal_form.get(field_name) 

817 if update_value != journal_value: 

818 trip_wire = True 

819 validation.value(validation.ERROR, row_ix, header_row.index(question), 

820 Messages.JOURNAL_CSV_VALIDATE__QUESTION_CANNOT_CHANGE.format(question=question), 

821 was=journal_value, now=update_value) 

822 

823 if trip_wire: 

824 continue 

825 

826 # Create an update request for this journal 

827 update_req = None 

828 jlock = None 

829 alock = None 

830 try: 

831 # ~~ ^->UpdateRequest:Feature ~~ 

832 update_req, jlock, alock = self.update_request_for_journal(j.id, account=account, lock_records=False) 

833 except exceptions.AuthoriseException as e: 

834 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__CANNOT_MAKE_UR.format(reason=e.reason)) 

835 continue 

836 

837 # If we don't have a UR, we can't continue 

838 if update_req is None: 

839 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__MISSING_JOURNAL.format(issns=", ".join(issns))) 

840 continue 

841 

842 # validate update_form - portality.forms.application_processors.PublisherUpdateRequest 

843 # ~~ ^->UpdateRequest:FormContext ~~ 

844 fc = formulaic_context.processor( 

845 formdata=update_form, 

846 source=update_req 

847 ) 

848 

849 if not fc.validate(): 

850 for k, v in fc.form.errors.items(): 

851 question = Journal2PublisherUploadQuestionsXwalk.q(k) 

852 try: 

853 pos = header_row.index(question) 

854 except ValueError: 

855 # this is because the validation is on a field which is not in the csv, so it must 

856 # be due to an existing validation error in the data, and not something the publisher 

857 # can do anything about 

858 continue 

859 now = row.get(question) 

860 was = [v for q, v in journal_questions if q == question][0] 

861 # If both the CSV value and the journal's current value are blank/null, 

862 # this is not actionable by the publisher - they haven't changed this field. 

863 # Suppress it as a false positive, similar to fields not in the CSV headers above. 

864 if not now and not was: 

865 continue 

866 if isinstance(v[0], dict): 

867 for sk, sv in v[0].items(): 

868 validation.value(validation.ERROR, row_ix, pos, ". ".join([str(x) for x in sv]), 

869 was=was, now=now) 

870 elif isinstance(v[0], list): 

871 # If we have a list, we must go a level deeper 

872 validation.value(validation.ERROR, row_ix, pos, ". ".join([str(x) for x in v[0]]), 

873 was=was, now=now) 

874 else: 

875 validation.value(validation.ERROR, row_ix, pos, ". ".join([str(x) for x in v]), 

876 was=was, now=now) 

877 

878 return validation 

879 

880 

881class CSVValidationReport: 

882 

883 WARN = "warn" 

884 ERROR = "error" 

885 

886 CLEANR = re.compile('<.*?>') 

887 

888 def __init__(self): 

889 self._general = [] 

890 self._headers = {} 

891 self._row = {} 

892 self._values = {} 

893 self._log = [] 

894 self._errors = False 

895 self._warnings = False 

896 

897 @property 

898 def general_errors(self): 

899 return self._general 

900 

901 @property 

902 def header_errors(self): 

903 return self._headers 

904 

905 @property 

906 def row_errors(self): 

907 return self._row 

908 

909 @property 

910 def value_errors(self): 

911 return self._values 

912 

913 @property 

914 def all_errors(self): 

915 """ (error_type, msg) for every general, header, row and value entry """ 

916 return self._general + list(self._headers.values()) + list(self._row.values()) + \ 

917 [(error_type, msg) for row in self._values.values() for error_type, msg, *_ in row.values()] 

918 

919 def has_errors_or_warnings(self): 

920 return self._errors or self._warnings 

921 

922 def has_errors(self): 

923 return self._errors 

924 

925 def has_warnings(self): 

926 return self._warnings 

927 

928 def record_error_type(self, error_type): 

929 if error_type == self.WARN: 

930 self._warnings = True 

931 elif error_type == self.ERROR: 

932 self._errors = True 

933 

934 def general(self, error_type, msg): 

935 msg = self._cleanhtml(msg) 

936 self.log("[" + error_type + "] " + msg) 

937 self._general.append((error_type, msg)) 

938 self.record_error_type(error_type) 

939 

940 def header(self, error_type, pos, msg): 

941 msg = self._cleanhtml(msg) 

942 self.log("[" + error_type + "] " + msg) 

943 self._headers[pos] = (error_type, msg) 

944 self.record_error_type(error_type) 

945 

946 def row(self, error_type, row, msg): 

947 msg = self._cleanhtml(msg) 

948 self.log("[" + error_type + "] " + msg) 

949 self._row[row] = (error_type, msg) 

950 self.record_error_type(error_type) 

951 

952 def value(self, error_type, row, pos, msg, was, now): 

953 msg = self._cleanhtml(msg) 

954 if row not in self._values: 

955 self._values[row] = {} 

956 self.log("[" + error_type + "] " + msg) 

957 self._values[row][pos] = (error_type, msg, was, now) 

958 self.record_error_type(error_type) 

959 

960 def log(self, msg): 

961 msg = self._cleanhtml(msg) 

962 self._log.append(msg) 

963 

964 def _cleanhtml(self, raw_html): 

965 cleantext = re.sub(self.CLEANR, '', raw_html) 

966 return cleantext 

967 

968 def json(self, indent=None): 

969 _repr = { 

970 "has_errors": self._errors, 

971 "has_warnings": self._warnings, 

972 "general": self._general, 

973 "headers": self._headers, 

974 "rows": self._row, 

975 "values": self._values, 

976 "log": self._log 

977 } 

978 return json.dumps(_repr, indent=indent)