Coverage for portality/bll/services/application.py: 94%
519 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1import csv
2import json
3import logging
4import re
5from io import StringIO
7from portality import constants, lock, models
8from portality.bll import DOAJ, exceptions
9from portality.core import app
10from portality.crosswalks.journal_form import JournalFormXWalk
11from portality.crosswalks.journal_questions import Journal2PublisherUploadQuestionsXwalk, QuestionTransformError
12from portality.forms.application_forms import ApplicationFormFactory
13from portality.lib import csv_utils, dates, httputil
14from portality.lib.argvalidate import argvalidate
15from portality.ui.messages import Messages
16from portality import datasets
19class ApplicationService(object):
20 """
21 ~~Application:Service->DOAJ:Service~~
22 """
24 def auto_assign_ur_editor_group(self, ur: models.Application):
25 """
26 Auto assign editor group to the update request
27 :param ur:
28 :return:
29 """
30 if not app.config.get("AUTO_ASSIGN_UR_EDITOR_GROUP", False):
31 return ur
33 target = None
34 reason = ""
36 by_owner = models.URReviewRoute.by_account(ur.owner)
37 if by_owner is not None:
38 reason = Messages.AUTOASSIGN__OWNER_MAPPED.format(owner=ur.owner, target=by_owner.target)
39 target = by_owner.target
41 if target is None:
42 by_country = models.URReviewRoute.by_country(ur.bibjson().publisher_country)
43 if by_country is not None:
44 reason = Messages.AUTOASSIGN__COUNTRY_MAPPED.format(country=by_country.country_name, country_code=by_country.country_code, target=by_country.target)
45 target = by_country.target
47 if target is None:
48 return ur
50 if ur.editor_group == target:
51 # UR is already assigned to the correct editor group
52 return ur
54 id = models.EditorGroup.group_exists_by_name(target)
55 if id is None:
56 ur.add_note(Messages.AUTOASSIGN__NOTE__EDITOR_GROUP_MISSING.format(target=target))
57 return ur
59 ur.set_editor_group(target)
60 ur.remove_editor()
61 ur.add_note(Messages.AUTOASSIGN__NOTE__ASSIGN.format(target=target, reason=reason))
62 return ur
64 def retrieve_ur_editor_group_sheets(self, keep_history=1):
65 start = dates.now()
67 publisher_sheet = app.config.get("AUTO_ASSIGN_EDITOR_BY_PUBLISHER_SHEET")
68 country_sheet = app.config.get("AUTO_ASSIGN_EDITOR_BY_COUNTRY_SHEET")
70 presp = httputil.get(publisher_sheet)
71 if presp.status_code != 200:
72 raise exceptions.RemoteServiceException("Failed to retrieve publisher sheet from {x}".format(x=publisher_sheet))
74 cresp = httputil.get(country_sheet)
75 if cresp.status_code != 200:
76 raise exceptions.RemoteServiceException("Failed to retrieve country sheet from {x}".format(x=country_sheet))
78 presp.encoding = "utf-8"
79 pdata = presp.text
80 if pdata is None or pdata == "":
81 raise ValueError("Publisher sheet is empty at {x}".format(x=publisher_sheet))
83 cresp.encoding = "utf-8"
84 cdata = cresp.text
85 if cdata is None or cdata == "":
86 raise ValueError("Country sheet is empty at {x}".format(x=country_sheet))
88 preader = csv.reader(StringIO(pdata))
89 creader = csv.reader(StringIO(cdata))
91 preader.__next__()
92 creader.__next__()
94 routers = []
96 for i, row in enumerate(preader):
97 account = row[1].strip()
98 group = row[2].strip()
100 if account is None or account == "":
101 raise ValueError(f"Publisher Sheet: Account is empty on row {i+2}")
102 if group is None or group == "":
103 raise ValueError(f"Publisher Sheet: Group is empty on row {i+2}")
105 acc = models.Account.pull(account)
106 if acc is None:
107 raise ValueError(f"Publisher Sheet: Account `{account}` not found in DOAJ; row {i+2}")
108 if models.EditorGroup.group_exists_by_name(group) is None:
109 raise ValueError(f"Publisher Sheet: Group `{group}` not found in DOAJ; row {i+2}")
111 router = models.URReviewRoute()
112 router.account_id = acc.id
113 router.target = group
114 router.pre_save_prep()
115 routers.append(router)
117 for i, row in enumerate(creader):
118 country = row[0].strip()
119 group = row[1].strip()
121 if country is None or country == "":
122 raise ValueError(f"Country Sheet: Country is empty on row {i+2}")
123 if group is None or group == "":
124 raise ValueError(f"Country Sheet: Group is empty on row {i+2}")
126 if models.EditorGroup.group_exists_by_name(group) is None:
127 raise ValueError(f"Country Sheet: Group `{group}` not found in DOAJ; row {i+2}")
129 cc = datasets.get_country_code(country, fail_if_not_found=True)
130 if cc is None:
131 raise ValueError(f"Country Sheet: Country `{country}` does not match an existing ISO country name; row {i+2}")
133 router = models.URReviewRoute()
134 router.country_code = cc
135 router.country_name = country
136 router.target = group
137 router.pre_save_prep()
138 routers.append(router)
140 # if we get to here we have two valid sheets, so we can update
141 # the local data using an index rollover technique
142 if len(routers) > 0:
143 mapping = routers[0].mappings()
144 docs = [router.data for router in routers]
145 models.URReviewRoute.create_and_seed_index_and_rollover_alias(docs, mapping, keep_history)
147 return routers
150 @staticmethod
151 def prevent_concurrent_ur_submission(ur: models.Application, record_if_not_concurrent=True):
152 """
153 Prevent duplicated update request submission
154 :param ur:
155 :param record_if_not_concurrent:
156 :return:
157 """
158 cs = DOAJ.concurrencyPreventionService()
160 if ur.current_journal is not None and ur.id is not None:
161 if cs.check_concurrency(ur.current_journal, ur.id):
162 raise exceptions.ConcurrentUpdateRequestException(Messages.CONCURRENT_UPDATE_REQUEST)
164 if record_if_not_concurrent:
165 cs.store_concurrency(ur.current_journal, ur.id, timeout=app.config.get("UR_CONCURRENCY_TIMEOUT", 10))
167 def reject_application(self, application, account, provenance=True, note=None, manual_update=True):
168 """
169 Reject an application. This will:
170 * set the application status to "rejected" (if not already)
171 * remove the current_journal field, and move it to related_journal (if needed)
172 * remove the current_application field from the related journal (if needed)
173 * save the application
174 * write a provenance record for the rejection (if requested)
176 :param application:
177 :param account:
178 :param provenance:
179 :param note:
180 :param manual_update:
181 :return:
182 """
183 # first validate the incoming arguments to ensure that we've got the right thing
184 argvalidate("reject_application", [
185 {"arg": application, "instance" : models.Application, "allow_none" : False, "arg_name" : "application"},
186 {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"},
187 {"arg" : provenance, "instance" : bool, "allow_none" : False, "arg_name" : "provenance"},
188 {"arg" : note, "instance" : str, "allow_none" : True, "arg_name" : "note"},
189 {"arg" : manual_update, "instance" : bool, "allow_none" : False, "arg_name" : "manual_update"}
190 ], exceptions.ArgumentException)
192 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering reject_application")
194 # ~~->Journal:Service~~
195 journalService = DOAJ.journalService()
197 # check we're allowed to carry out this action
198 if not account.has_role("reject_application"):
199 raise exceptions.AuthoriseException(message="This user is not allowed to reject applications", reason=exceptions.AuthoriseException.WRONG_ROLE)
201 # ensure the application status is "rejected"
202 if application.application_status != constants.APPLICATION_STATUS_REJECTED:
203 application.set_application_status(constants.APPLICATION_STATUS_REJECTED)
205 # set the rejection date
206 application.date_rejected = dates.now_str()
208 # add the note to the application
209 if note is not None:
210 application.add_note(note)
212 # retrieve the id of the current journal if there is one
213 cj_id = application.current_journal
214 cj = None
216 # if there is a current_journal record, remove it, and record
217 # it as a related journal. This will let us come back later and know
218 # which journal record this was intended as an update against if needed.
219 if cj_id is not None:
220 cj, _ = journalService.journal(cj_id)
221 application.remove_current_journal()
222 if cj is not None:
223 application.set_related_journal(cj_id)
224 cj.remove_current_application()
226 # if there is a current journal, we will have modified it above, so save it
227 if cj is not None:
228 saved = cj.save()
229 if saved is None:
230 raise exceptions.SaveException("Save on current_journal in reject_application failed")
232 # if we were asked to record this as a manual update, record that on the application
233 if manual_update:
234 application.set_last_manual_update()
236 saved = application.save()
237 if saved is None:
238 raise exceptions.SaveException("Save on application in reject_application failed")
240 # record a provenance record that this action took place
241 if provenance:
242 # ~~->Provenance:Model~~
243 models.Provenance.make(account, constants.PROVENANCE_STATUS_REJECTED, application)
245 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed reject_application")
248 def unreject_application(self,
249 application: models.Application,
250 account: models.Account,
251 manual_update: bool = True,
252 disallow_status: list = None):
253 """
254 Un-reject an application. This will:
255 * check that the application status is no longer "rejected" (throw an error if it is)
256 * check for a related journal, and if one is present, promote that to current_journal (if no other UR exists)
257 * save the application
259 :param application:
260 :param account:
261 :param manual_update:
262 :param disallow_status: statuses that we are not allowed to unreject to (excluding rejected, which is always disallowed)
263 :return:
264 """
265 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering unreject_application")
267 if application is None:
268 raise exceptions.ArgumentException(Messages.BLL__UNREJECT_APPLICATION__NO_APPLICATION)
270 if account is None:
271 raise exceptions.ArgumentException(Messages.BLL__UNREJECT_APPLICATION__NO_ACCOUNT)
273 # check we're allowed to carry out this action
274 if not account.has_role("unreject_application"):
275 raise exceptions.AuthoriseException(message=Messages.BLL__UNREJECT_APPLICATION__WRONG_ROLE,
276 reason=exceptions.AuthoriseException.WRONG_ROLE)
278 # ensure the application status is not "rejected"
279 if application.application_status == constants.APPLICATION_STATUS_REJECTED:
280 raise exceptions.IllegalStatusException(message=Messages.BLL__UNREJECT_APPLICATION__ILLEGAL_STATE_REJECTED.format(id=application.id))
282 # by default reject transitions to the accepted status (because acceptance implies other processing that this
283 # method does not handle). You can override this by passing in an empty list
284 if disallow_status is None:
285 disallow_status = [constants.APPLICATION_STATUS_ACCEPTED]
286 if application.application_status in disallow_status:
287 raise exceptions.IllegalStatusException(
288 message=Messages.BLL__UNREJECT_APPLICATION__ILLEGAL_STATE_DISALLOWED.format(
289 id=application.id, x=application.application_status
290 ))
292 rjid = application.related_journal
293 if rjid:
294 ur = models.Application.find_latest_by_current_journal(rjid)
295 if ur:
296 raise exceptions.DuplicateUpdateRequest(message=Messages.BLL__UNREJECT_APPLICATION__DUPLICATE_UR.format(
297 id=application.id,
298 urid=ur.id,
299 jid=rjid
300 ))
302 rj = models.Journal.pull(rjid)
303 if rj is None:
304 raise exceptions.NoSuchObjectException(Messages.BLL__UNREJECT_APPLICATION__JOURNAL_MISSING.format(
305 jid=rjid, id=application.id
306 ))
308 # update the application's view of the current journal
309 application.set_current_journal(rjid)
310 application.remove_related_journal()
312 # update the journal's view of the current application
313 rj.set_current_application(application.id)
314 rj.remove_related_application(application.id)
316 saved = rj.save()
317 if saved is None:
318 raise exceptions.SaveException(Messages.BLL__UNREJECT_APPLICATION__SAVE_FAIL.format(
319 obj="current_journal",
320 id=rj.id
321 ))
323 # if we were asked to record this as a manual update, record that on the application
324 if manual_update:
325 application.set_last_manual_update()
327 saved = application.save()
328 if saved is None:
329 raise exceptions.SaveException(Messages.BLL__UNREJECT_APPLICATION__SAVE_FAIL.format(
330 obj="application",
331 id=application.id
332 ))
334 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed unreject_application")
337 def accept_application(self, application, account, manual_update=True, provenance=True, save_journal=True, save_application=True):
338 """
339 Take the given application and create the Journal object in DOAJ for it.
341 The account provided must have permission to create journals from applications.
343 :param application: The application to be converted
344 :param account: The account doing the conversion
345 :param manual_update: Whether to record this update as a manual update on both the application and journal objects
346 :param provenance: Whether to write provenance records for this operation
347 :param save_journal: Whether to save the journal that is produced
348 :param save_application: Whether to save the application after it has been modified
349 :return: The journal that was created
350 """
351 # first validate the incoming arguments to ensure that we've got the right thing
352 argvalidate("accept_application", [
353 {"arg": application, "instance" : models.Suggestion, "allow_none" : False, "arg_name" : "application"},
354 {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"},
355 {"arg" : manual_update, "instance" : bool, "allow_none" : False, "arg_name" : "manual_update"},
356 {"arg" : provenance, "instance" : bool, "allow_none" : False, "arg_name" : "provenance"},
357 {"arg" : save_journal, "instance" : bool, "allow_none" : False, "arg_name" : "save_journal"},
358 {"arg" : save_application, "instance" : bool, "allow_none" : False, "arg_name" : "save_application"}
359 ], exceptions.ArgumentException)
361 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering accept_application")
363 # ensure that the account holder has a suitable role
364 if not account.has_role("accept_application"):
365 raise exceptions.AuthoriseException(
366 message="User {x} is not permitted to accept application {y}".format(x=account.id, y=application.id),
367 reason=exceptions.AuthoriseException.WRONG_ROLE)
369 # ensure the application status is "accepted"
370 if application.application_status != constants.APPLICATION_STATUS_ACCEPTED:
371 application.set_application_status(constants.APPLICATION_STATUS_ACCEPTED)
373 # make the resulting journal (and save it if requested)
374 j = self.application_2_journal(application, manual_update=manual_update)
375 if save_journal is True:
376 saved = j.save()
377 if saved is None:
378 raise exceptions.SaveException("Save of resulting journal in accept_application failed")
380 # retrieve the id of the current journal if there is one
381 cj = application.current_journal
383 # if there is a current_journal record, remove it
384 if cj is not None:
385 application.remove_current_journal()
387 # set the relationship with the journal
388 application.set_related_journal(j.id)
390 # if we were asked to record this as a manual update, record that on the application
391 # (the journal is done implicitly above)
392 if manual_update:
393 application.set_last_manual_update()
395 if provenance:
396 # record the event in the provenance tracker
397 models.Provenance.make(account, constants.PROVENANCE_STATUS_ACCEPTED, application)
399 # save the application if requested
400 if save_application is True:
401 application.save()
403 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed accept_application")
405 return j
407 def reject_update_request_of_journal(self, journal_id, account):
408 """
409 Rejects update request associated with journal
411 :param journal_id:
412 :param account:
413 :return: Journal object
414 """
415 ur = models.Application.find_latest_by_current_journal(journal_id) # ~~->Application:Model~~
416 if ur:
417 self.reject_application(ur, account, note=Messages.AUTOMATICALLY_REJECTED_UPDATE_REQUEST_NOTE)
418 return ur
419 else:
420 return None
422 def reject_update_request_of_journals(self, ids, account):
423 """
424 Rejects update request associated with journal
426 :param ids:
427 :param account:
428 :return: Journal object
429 """
430 ur_ids = []
431 for journal_id in ids:
432 ur = self.reject_update_request_of_journal(journal_id, account)
433 if ur:
434 ur_ids.append(ur.id)
435 return ur_ids
437 def update_request_for_journal(self, journal_id, account=None, lock_timeout=None, lock_records=True):
438 """
439 Obtain an update request application object for the journal with the given journal_id
441 An update request may either be loaded from the database, if it already exists, or created
442 in-memory if it has not previously been created.
444 If an account is provided, this will validate that the account holder is allowed to make
445 the conversion from journal to application, if a conversion is required.
447 When this request runs, the journal will be locked to the provided account if an account is
448 given. If the application is loaded from the database, this will also be locked for the account
449 holder.
451 :param journal_id:
452 :param account:
453 :return: a tuple of (Application Object, Journal Lock, Application Lock)
454 """
455 # first validate the incoming arguments to ensure that we've got the right thing
456 argvalidate("update_request_for_journal", [
457 {"arg": journal_id, "instance" : str, "allow_none" : False, "arg_name" : "journal_id"},
458 {"arg" : account, "instance" : models.Account, "allow_none" : True, "arg_name" : "account"},
459 {"arg" : lock_timeout, "instance" : int, "allow_none" : True, "arg_name" : "lock_timeout"}
460 ], exceptions.ArgumentException)
462 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering update_request_for_journal")
464 # ~~-> Journal:Service~~
465 # ~~-> AuthNZ:Service~~
466 journalService = DOAJ.journalService()
467 authService = DOAJ.authorisationService()
469 # first retrieve the journal, and return empty if there isn't one.
470 # We don't attempt to obtain a lock at this stage, as we want to check that the user is authorised first
471 journal_lock = None
472 journal, _ = journalService.journal(journal_id)
473 if journal is None:
474 app.logger.info("Request for journal {x} did not find anything in the database".format(x=journal_id))
475 return None, None, None
477 # if the journal is not in_doaj, we won't create an update request for it
478 if not journal.is_in_doaj():
479 app.logger.info("Request for journal {x} found it is not in_doaj; will not create update request".format(x=journal_id))
480 return None, None, None
482 # retrieve the latest application attached to this journal
483 application_lock = None
484 application = models.Suggestion.find_latest_by_current_journal(journal_id) # ~~->Application:Model~~
486 # if no such application exists, create one in memory (this will check that the user is permitted to create one)
487 # at the same time, create the lock for the journal. This will throw an AuthorisedException or a Locked exception
488 # (in that order of preference) if any problems arise.
489 if application is None:
490 app.logger.info("No existing update request for journal {x}; creating one".format(x=journal.id))
491 application = journalService.journal_2_application(journal, account=account)
492 application.set_is_update_request(True)
493 if account is not None:
494 if lock_records:
495 journal_lock = lock.lock("journal", journal_id, account.id)
497 # otherwise check that the user (if given) has the rights to edit the application
498 # then lock the application and journal to the account.
499 # If a lock cannot be obtained, unlock the journal and application before we return
500 elif account is not None:
501 try:
502 authService.can_edit_application(account, application)
503 if lock_records:
504 application_lock = lock.lock("suggestion", application.id, account.id)
505 journal_lock = lock.lock("journal", journal_id, account.id)
506 except lock.Locked as e:
507 if application_lock is not None: application_lock.delete()
508 if journal_lock is not None: journal_lock.delete()
509 raise
510 except exceptions.AuthoriseException as e:
511 msg = "Account {x} is not permitted to edit the current update request on journal {y}".format(x=account.id, y=journal.id)
512 app.logger.info(msg)
513 e.args += (msg,)
514 raise
516 app.logger.info("Using existing application {y} as update request for journal {x}".format(y=application.id, x=journal.id))
518 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completed update_request_for_journal; return application object")
520 return application, journal_lock, application_lock
522 def application_2_journal(self, application, manual_update=True):
523 # first validate the incoming arguments to ensure that we've got the right thing
524 argvalidate("application_2_journal", [
525 {"arg": application, "instance" : models.Suggestion, "allow_none" : False, "arg_name" : "application"},
526 {"arg" : manual_update, "instance" : bool, "allow_none" : False, "arg_name" : "manual_update"}
527 ], exceptions.ArgumentException)
529 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Entering application_2_journal")
531 # create a new blank journal record, which we can build up
532 journal = models.Journal() # ~~->Journal:Model~~
534 # first thing is to copy the bibjson as-is wholesale,
535 abj = application.bibjson()
536 journal.set_bibjson(abj)
537 jbj = journal.bibjson()
539 # now carry over key administrative properties from the application itself
540 # * notes
541 # * editor
542 # * editor_group
543 # * owner
544 # * application date
545 # notes = application.notes
546 notes = application.get_detached_notes()
548 if application.editor is not None:
549 journal.set_editor(application.editor)
550 if application.editor_group is not None:
551 journal.set_editor_group(application.editor_group)
552 for note in notes:
553 journal.add_note_by_dict(note)
554 if application.owner is not None:
555 journal.set_owner(application.owner)
556 if application.date_applied is not None:
557 journal.set_date_applied(application.date_applied)
559 b = application.bibjson()
560 if b.pissn == "":
561 b.add_identifier("pissn", None)
562 if b.eissn == "":
563 b.add_identifier("eissn", None)
565 # no relate the journal to the application and place it in_doaj
566 journal.add_related_application(application.id, dates.now_str())
567 journal.set_in_doaj(True)
569 # if we've been called in the context of a manual update, record that
570 if manual_update:
571 journal.set_last_manual_update()
573 # if this is an update to an existing journal, then we can also port information from
574 # that journal
575 if application.current_journal is not None:
576 cj = models.Journal.pull(application.current_journal)
577 if cj is not None:
578 # carry the id and the created date
579 journal.set_id(cj.id)
580 journal.set_created(cj.created_date)
582 # bring forward any notes from the old journal record
583 old_notes = cj.get_detached_notes()
584 for note in old_notes:
585 journal.add_note_by_dict(note)
587 # bring forward any related applications
588 related = cj.related_applications
589 for r in related:
590 journal.add_related_application(r.get("application_id"), r.get("date_accepted"), r.get("status"))
592 # carry over any properties that are not already set from the application
593 # * editor & editor_group (together or not at all)
594 # * owner
595 if journal.editor is None and journal.editor_group is None:
596 journal.set_editor(cj.editor)
597 journal.set_editor_group(cj.editor_group)
598 if journal.owner is None:
599 journal.set_owner(cj.owner)
601 if app.logger.isEnabledFor(logging.DEBUG): app.logger.debug("Completing application_2_journal")
603 return journal
605 def application(self, application_id, lock_application=False, lock_account=None, lock_timeout=None):
606 """
607 Function to retrieve an application by its id
609 :param application_id: the id of the application
610 :param: lock_application: should we lock the resource on retrieval
611 :param: lock_account: which account is doing the locking? Must be present if lock_journal=True
612 :param: lock_timeout: how long to lock the resource for. May be none, in which case it will default
613 :return: Tuple of (Suggestion Object, Lock Object)
614 """
615 # first validate the incoming arguments to ensure that we've got the right thing
616 argvalidate("application", [
617 {"arg": application_id, "allow_none" : False, "arg_name" : "application_id"},
618 {"arg": lock_application, "instance" : bool, "allow_none" : False, "arg_name" : "lock_journal"},
619 {"arg": lock_account, "instance" : models.Account, "allow_none" : True, "arg_name" : "lock_account"},
620 {"arg": lock_timeout, "instance" : int, "allow_none" : True, "arg_name" : "lock_timeout"}
621 ], exceptions.ArgumentException)
623 # pull the application from the database
624 application = models.Suggestion.pull(application_id)
626 # if we've retrieved the journal, and a lock is requested, request it
627 the_lock = None
628 if application is not None and lock_application:
629 if lock_account is not None:
630 # ~~->Lock:Feature~~
631 the_lock = lock.lock(constants.LOCK_APPLICATION, application_id, lock_account.id, lock_timeout)
632 else:
633 raise exceptions.ArgumentException("If you specify lock_application on application retrieval, you must also provide lock_account")
635 return application, the_lock
637 def delete_application(self, application_id, account):
638 """
639 Function to delete an application, and all references to it in other objects (current and related journals)
641 The application and all related journals need to be locked before this process can proceed, so you may get a
642 lock.Locked exception
644 :param application_id:
645 :param account:
646 :return:
647 """
648 # first validate the incoming arguments to ensure that we've got the right thing
649 argvalidate("delete_application", [
650 {"arg": application_id, "instance" : str, "allow_none" : False, "arg_name" : "application_id"},
651 {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"}
652 ], exceptions.ArgumentException)
654 # ~~-> Journal:Service~~
655 # ~~-> AuthNZ:Service~~
656 journalService = DOAJ.journalService()
657 authService = DOAJ.authorisationService()
659 # get hold of a copy of the application. If there isn't one, our work here is done
660 # (note the application could be locked, in which case this will raise a lock.Locked exception)
662 # get the application
663 application, _ = self.application(application_id)
664 if application is None:
665 raise exceptions.NoSuchObjectException
667 # determine if the user can edit the application
668 authService.can_edit_application(account, application)
670 # attempt to lock the record (this may raise a Locked exception)
671 # ~~-> Lock:Feature~~
672 alock = lock.lock(constants.LOCK_APPLICATION, application_id, account.id)
674 # obtain the current journal, with associated lock
675 current_journal = None
676 cjlock = None
677 if application.current_journal is not None:
678 try:
679 current_journal, cjlock = journalService.journal(application.current_journal, lock_journal=True, lock_account=account)
680 except lock.Locked as e:
681 # if the resource is locked, we have to back out
682 if alock is not None: alock.delete()
683 raise
685 # obtain the related journal, with associated lock
686 related_journal = None
687 rjlock = None
688 if application.related_journal is not None:
689 try:
690 related_journal, rjlock = journalService.journal(application.related_journal, lock_journal=True, lock_account=account)
691 except lock.Locked:
692 # if the resource is locked, we have to back out
693 if alock is not None: alock.delete()
694 if cjlock is not None: cjlock.delete()
695 raise
697 try:
698 if current_journal is not None:
699 current_journal.remove_current_application()
700 saved = current_journal.save()
701 if saved is None:
702 raise exceptions.SaveException("Unable to save journal record")
704 if related_journal is not None:
705 relation_record = related_journal.related_application_record(application_id)
706 if relation_record is None:
707 relation_record = {}
708 related_journal.add_related_application(application_id, relation_record.get("date_accepted"), "deleted")
709 saved = related_journal.save()
710 if saved is None:
711 raise exceptions.SaveException("Unable to save journal record")
713 application.delete()
715 finally:
716 if alock is not None: alock.delete()
717 if cjlock is not None: cjlock.delete()
718 if rjlock is not None: rjlock.delete()
720 return
722 def validate_update_csv(self, file_path, account: models.Account):
723 # Open with encoding that deals with the Byte Order Mark since we're given files from Windows.
724 with open(file_path, "r", encoding='utf-8-sig') as file:
725 reader = csv.DictReader(file)
726 validation = CSVValidationReport()
728 # verify header row with current CSV headers, report errors
729 header_row = reader.fieldnames
730 allowed_headers = Journal2PublisherUploadQuestionsXwalk.question_list()
731 lower_case_allowed_headers = list(map(str.lower, allowed_headers))
732 required_headers = Journal2PublisherUploadQuestionsXwalk.required_questions()
734 # Always perform a match check on supplied headers, not counting order
735 for i, h in enumerate(header_row):
736 if h and h not in allowed_headers:
737 if h.lower() in lower_case_allowed_headers:
738 expected = allowed_headers[lower_case_allowed_headers.index(h.lower())]
739 validation.header(validation.WARN, i, Messages.JOURNAL_CSV_VALIDATE__HEADER_CASE_MISMATCH.format(h=h, expected=expected))
740 else:
741 validation.header(validation.ERROR, i, Messages.JOURNAL_CSV_VALIDATE__INVALID_HEADER.format(h=h))
743 missing_required = False
744 for rh in required_headers:
745 if rh not in header_row:
746 validation.general(validation.ERROR, Messages.JOURNAL_CSV_VALIDATE__REQUIRED_HEADER_MISSING.format(h=rh))
747 missing_required = True
748 if missing_required:
749 return validation
751 # Talking about spreadsheets, so we start at 1
752 row_ix = 1
754 # ~~ ->$JournalUpdateByCSV:Feature ~~
755 for row in csv_utils.trim_trailing_empty_rows(reader):
756 row_ix += 1
757 validation.log(f'Processing CSV row {row_ix}')
759 # Skip empty rows
760 if csv_utils.is_empty_row(row):
761 validation.log("Skipping empty row {x}.".format(x=row_ix))
762 continue
764 # Pull by ISSNs
765 issns = [
766 row.get(Journal2PublisherUploadQuestionsXwalk.q("pissn")),
767 row.get(Journal2PublisherUploadQuestionsXwalk.q("eissn"))
768 ]
769 issns = [issn for issn in issns if issn is not None and issn != ""]
771 try:
772 j = models.Journal.find_by_issn(issns, in_doaj=True, max=1).pop(0)
773 except IndexError:
774 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__MISSING_JOURNAL.format(issns=", ".join(issns)))
775 continue
777 # Confirm that the account is allowed to update the journal (is admin or owner)
778 if not account.is_super and account.id != j.owner:
779 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__OWNER_MISMATCH.format(acc=account.id, issns=", ".join(issns)))
780 continue
782 # By this point the issns are confirmed as matching a journal that belongs to the publisher
783 validation.log('Validating update for journal with ID ' + j.id)
785 # Load remaining rows into application form as an update
786 # ~~ ^->JournalQuestions:Crosswalk ~~
787 journal_form = JournalFormXWalk.obj2form(j)
788 journal_questions = Journal2PublisherUploadQuestionsXwalk.journal2question(j)
790 try:
791 update_form, updates = Journal2PublisherUploadQuestionsXwalk.question2form(j, row)
792 except QuestionTransformError as e:
793 question = Journal2PublisherUploadQuestionsXwalk.q(e.key)
794 journal_value = [y for x, y in journal_questions if x == question][0]
795 validation.value(validation.ERROR, row_ix, header_row.index(question),
796 Messages.JOURNAL_CSV_VALIDATE__INVALID_DATA.format(question=question),
797 was=journal_value, now=e.value)
798 continue
800 if len(updates) == 0:
801 validation.row(validation.WARN, row_ix, Messages.JOURNAL_CSV_VALIDATE__NO_DATA_CHANGE)
802 continue
804 # if we get to here, then there are updates
805 [validation.log(upd) for upd in updates]
807 # If a field is disabled in the UR Form Context, then we must confirm that the form data from the
808 # file has not changed from that provided in the source
809 formulaic_context = ApplicationFormFactory.context("update_request")
810 disabled_fields = formulaic_context.disabled_fields()
811 trip_wire = False
812 for field in disabled_fields:
813 field_name = field.get("name")
814 question = Journal2PublisherUploadQuestionsXwalk.q(field_name)
815 update_value = update_form.get(field_name)
816 journal_value = journal_form.get(field_name)
817 if update_value != journal_value:
818 trip_wire = True
819 validation.value(validation.ERROR, row_ix, header_row.index(question),
820 Messages.JOURNAL_CSV_VALIDATE__QUESTION_CANNOT_CHANGE.format(question=question),
821 was=journal_value, now=update_value)
823 if trip_wire:
824 continue
826 # Create an update request for this journal
827 update_req = None
828 jlock = None
829 alock = None
830 try:
831 # ~~ ^->UpdateRequest:Feature ~~
832 update_req, jlock, alock = self.update_request_for_journal(j.id, account=account, lock_records=False)
833 except exceptions.AuthoriseException as e:
834 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__CANNOT_MAKE_UR.format(reason=e.reason))
835 continue
837 # If we don't have a UR, we can't continue
838 if update_req is None:
839 validation.row(validation.ERROR, row_ix, Messages.JOURNAL_CSV_VALIDATE__MISSING_JOURNAL.format(issns=", ".join(issns)))
840 continue
842 # validate update_form - portality.forms.application_processors.PublisherUpdateRequest
843 # ~~ ^->UpdateRequest:FormContext ~~
844 fc = formulaic_context.processor(
845 formdata=update_form,
846 source=update_req
847 )
849 if not fc.validate():
850 for k, v in fc.form.errors.items():
851 question = Journal2PublisherUploadQuestionsXwalk.q(k)
852 try:
853 pos = header_row.index(question)
854 except ValueError:
855 # this is because the validation is on a field which is not in the csv, so it must
856 # be due to an existing validation error in the data, and not something the publisher
857 # can do anything about
858 continue
859 now = row.get(question)
860 was = [v for q, v in journal_questions if q == question][0]
861 # If both the CSV value and the journal's current value are blank/null,
862 # this is not actionable by the publisher - they haven't changed this field.
863 # Suppress it as a false positive, similar to fields not in the CSV headers above.
864 if not now and not was:
865 continue
866 if isinstance(v[0], dict):
867 for sk, sv in v[0].items():
868 validation.value(validation.ERROR, row_ix, pos, ". ".join([str(x) for x in sv]),
869 was=was, now=now)
870 elif isinstance(v[0], list):
871 # If we have a list, we must go a level deeper
872 validation.value(validation.ERROR, row_ix, pos, ". ".join([str(x) for x in v[0]]),
873 was=was, now=now)
874 else:
875 validation.value(validation.ERROR, row_ix, pos, ". ".join([str(x) for x in v]),
876 was=was, now=now)
878 return validation
881class CSVValidationReport:
883 WARN = "warn"
884 ERROR = "error"
886 CLEANR = re.compile('<.*?>')
888 def __init__(self):
889 self._general = []
890 self._headers = {}
891 self._row = {}
892 self._values = {}
893 self._log = []
894 self._errors = False
895 self._warnings = False
897 @property
898 def general_errors(self):
899 return self._general
901 @property
902 def header_errors(self):
903 return self._headers
905 @property
906 def row_errors(self):
907 return self._row
909 @property
910 def value_errors(self):
911 return self._values
913 @property
914 def all_errors(self):
915 """ (error_type, msg) for every general, header, row and value entry """
916 return self._general + list(self._headers.values()) + list(self._row.values()) + \
917 [(error_type, msg) for row in self._values.values() for error_type, msg, *_ in row.values()]
919 def has_errors_or_warnings(self):
920 return self._errors or self._warnings
922 def has_errors(self):
923 return self._errors
925 def has_warnings(self):
926 return self._warnings
928 def record_error_type(self, error_type):
929 if error_type == self.WARN:
930 self._warnings = True
931 elif error_type == self.ERROR:
932 self._errors = True
934 def general(self, error_type, msg):
935 msg = self._cleanhtml(msg)
936 self.log("[" + error_type + "] " + msg)
937 self._general.append((error_type, msg))
938 self.record_error_type(error_type)
940 def header(self, error_type, pos, msg):
941 msg = self._cleanhtml(msg)
942 self.log("[" + error_type + "] " + msg)
943 self._headers[pos] = (error_type, msg)
944 self.record_error_type(error_type)
946 def row(self, error_type, row, msg):
947 msg = self._cleanhtml(msg)
948 self.log("[" + error_type + "] " + msg)
949 self._row[row] = (error_type, msg)
950 self.record_error_type(error_type)
952 def value(self, error_type, row, pos, msg, was, now):
953 msg = self._cleanhtml(msg)
954 if row not in self._values:
955 self._values[row] = {}
956 self.log("[" + error_type + "] " + msg)
957 self._values[row][pos] = (error_type, msg, was, now)
958 self.record_error_type(error_type)
960 def log(self, msg):
961 msg = self._cleanhtml(msg)
962 self._log.append(msg)
964 def _cleanhtml(self, raw_html):
965 cleantext = re.sub(self.CLEANR, '', raw_html)
966 return cleantext
968 def json(self, indent=None):
969 _repr = {
970 "has_errors": self._errors,
971 "has_warnings": self._warnings,
972 "general": self._general,
973 "headers": self._headers,
974 "rows": self._row,
975 "values": self._values,
976 "log": self._log
977 }
978 return json.dumps(_repr, indent=indent)