Coverage for portality/models/v2/journal.py: 93%
958 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1from __future__ import annotations
3import string
4import uuid
5from copy import deepcopy
6from datetime import datetime, timedelta
7from typing import Callable, Iterable, Union
9from unidecode import unidecode
11from portality.core import app
12from portality.dao import DomainObject
13from portality.lib import es_data_mapping, dates, coerce
14from portality.lib.coerce import COERCE_MAP
15from portality.lib.dates import DEFAULT_TIMESTAMP_VAL, find_earliest_date
16from portality.lib.seamless import SeamlessMixin
17from portality.models.account import Account
18from portality.models.note import Note
19from portality.models.v2 import shared_structs
20from portality.models.v2.bibjson import JournalLikeBibJSON
22from portality.lib.dates import FMT_DATE_STD
24JOURNAL_STRUCT = {
25 "objects": [
26 "admin", "index"
27 ],
29 "structs": {
30 "admin": {
31 "fields": {
32 "in_doaj": {"coerce": "bool"},
33 "ticked": {"coerce": "bool"},
34 "current_application": {"coerce": "unicode"},
35 "last_full_review": {"coerce": "bigenddate"},
36 "last_withdrawn": {"coerce": "utcdatetime"},
37 "last_reinstated": {"coerce": "utcdatetime"},
38 "last_owner_transfer": {"coerce": "utcdatetime"}
39 },
40 "lists": {
41 "related_applications": {"contains": "object"}
42 },
43 "structs": {
44 "related_applications": {
45 "fields": {
46 "application_id": {"coerce": "unicode"},
47 "date_accepted": {"coerce": "utcdatetime"},
48 "status": {"coerce": "unicode"}
49 }
50 },
51 "contact": {
52 "name": {"coerce": "unicode"},
53 "email": {"coerce": "unicode"}
54 }
55 }
56 },
57 "index": {
58 "fields": {
59 "publisher_ac": {"coerce": "unicode"},
60 "institution_ac": {"coerce": "unicode"}
61 }
62 }
63 }
64}
68class ContinuationException(Exception):
69 pass
72class JournalLikeObject(SeamlessMixin, DomainObject):
74 # During migration from the old data model to the new data model for journal-like objects, this allows
75 # the front-end to continue to work, even if the object sees data which is not in the struct.
76 # This can be commented out after any migration which changes the data model
77 __SEAMLESS_SILENT_PRUNE__ = app.config.get("SEAMLESS_JOURNAL_LIKE_SILENT_PRUNE", False)
78 __SEAMLESS_ALLOW_OTHER_FIELDS__ = app.config.get("SEAMLESS_JOURNAL_LIKE_OTHER_FIELDS", False)
80 def __init__(self, *args, **kwargs):
81 super(JournalLikeObject, self).__init__(*args, **kwargs)
82 self._notes = {}
83 self._notes_loaded = False
85 @classmethod
86 def find_by_issn(cls, issns, in_doaj=None, max=10):
87 if not isinstance(issns, list):
88 issns = [issns]
89 q = JournalQuery()
90 q.find_by_issn(issns, in_doaj=in_doaj, max=max)
91 result = cls.query(q=q.query)
92 # create an array of objects, using cls rather than Journal, which means subclasses can use it too
93 records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
94 return records
96 @classmethod
97 def find_by_issn_exact(cls, issns, in_doaj=None, max=2):
98 """
99 Finds journal that matches given issns exactly - if no data problems should always be only 1
100 """
101 if not isinstance(issns, list):
102 issns = [issns]
103 if len(issns) > 2:
104 return []
105 q = JournalQuery()
106 q.find_by_issn_exact(issns, in_doaj=in_doaj, max=max)
107 result = cls.query(q=q.query)
108 # create an array of objects, using cls rather than Journal, which means subclasses can use it too
109 records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
110 return records
112 @classmethod
113 def issns_by_owner(cls, owner, in_doaj=None, issn_field=None):
114 q = IssnQuery(owner, in_doaj=in_doaj, issn_field=issn_field)
115 res = cls.query(q=q.query())
116 issns = [term.get("key") for term in res.get("aggregations", {}).get("issns", {}).get("buckets", [])]
117 return issns
119 @classmethod
120 def get_by_owner(cls, owner):
121 q = OwnerQuery(owner)
122 res = cls.query(q=q.query())
123 # get_by_owner() in application.py predates this, but I've made it an override because it does application stuff
124 records = [cls(**r.get("_source")) for r in res.get("hits", {}).get("hits", [])]
125 return records
127 @classmethod
128 def issns_by_query(cls, query):
129 issns = []
130 for j in cls.iterate(query):
131 issns += j.known_issns()
132 return issns
134 @classmethod
135 def find_by_journal_url(cls, url, in_doaj=None, max=10):
136 q = JournalURLQuery(url, in_doaj, max)
137 result = cls.query(q=q.query())
138 # create an array of objects, using cls rather than Journal, which means subclasses can use it too
139 records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
140 return records
142 @classmethod
143 def recent(cls, max=10):
144 q = RecentJournalsQuery(max)
145 result = cls.query(q=q.query())
146 # create an array of objects, using cls rather than Journal, which means subclasses can use it too
147 records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
148 return records
150 ############################################
151 ## base property methods
153 @property
154 def data(self):
155 return self.__seamless__.data
157 @property
158 def has_apc(self):
159 return self.__seamless__.get_single("bibjson.apc.has_apc")
161 @property
162 def id(self):
163 return self.__seamless__.get_single("id")
165 def set_id(self, id=None):
166 if id is None:
167 id = self.makeid()
168 self.__seamless__.set_with_struct("id", id)
170 def set_created(self, date=None):
171 if date is None:
172 date = dates.now_str()
173 self.__seamless__.set_with_struct("created_date", date)
175 @property
176 def created_date(self):
177 return self.__seamless__.get_single("created_date")
179 @property
180 def created_timestamp(self):
181 return self.__seamless__.get_single("created_date", coerce=coerce.to_datestamp())
183 def set_last_updated(self, date=None):
184 if date is None:
185 date = dates.now_str()
186 self.__seamless__.set_with_struct("last_updated", date)
188 @property
189 def last_updated(self):
190 return self.__seamless__.get_single("last_updated")
192 @property
193 def last_updated_timestamp(self):
194 return self.__seamless__.get_single("last_updated", coerce=coerce.to_datestamp())
196 def last_updated_since(self, days=0):
197 return self.last_updated_timestamp > (dates.now() - timedelta(days=days))
199 def set_last_manual_update(self, date=None):
200 if date is None:
201 date = dates.now_str()
202 self.__seamless__.set_with_struct("last_manual_update", date)
204 @property
205 def last_manual_update(self):
206 return self.__seamless__.get_single("last_manual_update")
208 def last_manually_updated_since(self, days=0):
209 return self.last_manual_update_timestamp > (datetime.utcnow() - timedelta(days=days))
211 @property
212 def last_manual_update_timestamp(self):
213 return self.__seamless__.get_single("last_manual_update", coerce=coerce.to_datestamp())
215 @property
216 def most_urgent_flag_deadline_timestamp(self):
217 fn = coerce.to_datestamp()
218 return fn(self.most_urgent_flag_deadline)
219 # return self.__seamless__.get_single("most_urgent_flag_deadline", coerce=coerce.to_datestamp())
221 def has_been_manually_updated(self):
222 lmut = self.last_manual_update_timestamp
223 if lmut is None:
224 return False
225 return lmut > datetime.utcfromtimestamp(0)
227 def set_date_applied(self, date=None):
228 if date is None:
229 date = dates.now_str()
230 self.__seamless__.set_with_struct("admin.date_applied", date)
232 @property
233 def date_applied(self):
234 return self.__seamless__.get_single("admin.date_applied")
236 @property
237 def date_applied_timestamp(self):
238 return self.__seamless__.get_single("admin.date_applied", coerce=coerce.to_datestamp())
240 @date_applied.setter
241 def date_applied(self, val):
242 self.__seamless__.set_with_struct("admin.date_applied", val)
244 def has_oa_start_date(self):
245 return self.__seamless__.get_single("bibjson.oa_start", default=False)
247 @property
248 def owner(self):
249 return self.__seamless__.get_single("admin.owner")
251 def set_owner(self, owner):
252 self.__seamless__.set_with_struct("admin.owner", owner)
254 def remove_owner(self):
255 self.__seamless__.delete("admin.owner")
257 @property
258 def owner_account(self):
259 if self.owner:
260 return Account.pull(self.owner)
261 return None
263 @property
264 def editor_group(self):
265 return self.__seamless__.get_single("admin.editor_group")
267 def set_editor_group(self, eg):
268 self.__seamless__.set_with_struct("admin.editor_group", eg)
270 def remove_editor_group(self):
271 self.__seamless__.delete("admin.editor_group")
273 @property
274 def editor(self):
275 return self.__seamless__.get_single("admin.editor")
277 def set_editor(self, ed):
278 self.__seamless__.set_with_struct("admin.editor", ed)
280 def remove_editor(self):
281 self.__seamless__.delete('admin.editor')
283 @property
284 def contact(self):
285 return self.__seamless__.get_single("admin.contact")
287 @property
288 def contact_name(self):
289 return self.__seamless__.get_single("admin.contact.name")
291 @contact_name.setter
292 def contact_name(self, name):
293 self.__seamless__.set_with_struct("admin.contact.name", name)
295 @property
296 def contact_email(self):
297 return self.__seamless__.get_single("admin.contact.email")
299 @contact_email.setter
300 def contact_email(self, email):
301 self.__seamless__.set_with_struct("admin.contact.email", email)
303 def set_contact(self, name, email):
304 self.contact_name = name
305 self.contact_email = email
307 def remove_contact(self):
308 self.__seamless__.delete("admin.contact")
310 #### Notes methods
312 def add_note(self, note:str, date=None, id=None, author_id=None,
313 assigned_to=None, deadline=None):
314 if id == "": id = None
315 if not author_id: author_id = None
316 if not assigned_to: assigned_to = None
317 if not deadline: deadline = None
319 if not date:
320 date = dates.now_str()
322 note_obj = Note()
323 note_obj.set_created(date)
324 note_obj.note = note
325 if author_id is not None:
326 note_obj.author_id = author_id
328 note_obj.resource_type = self.__type__
329 if self.id is None:
330 self.set_id(self.makeid())
331 note_obj.resource_id = self.id
333 match_id = None
334 for existing in self.notes:
335 note_match = existing.get("note") == note_obj.note
336 author_match = existing.get("author_id") == note_obj.author_id
337 date_match = existing.get("date") == date
338 if note_match and author_match and date_match:
339 match_id = existing.get("id")
340 break
342 # If there is no duplicate note, just go ahead. Otherwise we're not going
343 # to set this note
344 if match_id is None:
345 if id is not None:
346 note_obj.set_id(id)
347 else:
348 note_obj.set_id(self.makeid())
350 self._notes[note_obj.id] = note_obj
351 self.__seamless__.delete_from_list("admin.note_ids", note_obj.id)
352 self.__seamless__.add_to_list_with_struct("admin.note_ids", note_obj.id)
353 else:
354 note_obj.set_id(match_id)
356 if assigned_to is not None or deadline is not None:
357 # this is a flag
358 self.set_flag(note_obj.id, assigned_to, deadline)
360 return note_obj
362 def add_note_by_dict(self, note):
363 return self.add_note(note=note.get("note"), date=note.get("date"),
364 id=note.get("id"), author_id=note.get("author_id"),
365 assigned_to=note.get("assigned_to"), deadline=note.get("deadline"))
367 def remove_note(self, note:Union[Note, dict]):
368 nid = None
369 if isinstance(note, Note):
370 nid = note.id
371 else:
372 nid = note["id"]
374 self.remove_note_by_id(nid)
376 def remove_note_by_id(self, note_id:str):
377 """
378 Remove a note by its ID.
379 :param note_id: The ID of the note to remove.
380 """
381 self._notes[note_id] = False
382 self.__seamless__.delete_from_list("admin.note_ids", note_id)
384 if self.flag_note_id == note_id:
385 self.delete_flag()
387 def get_note_by_id(self, note_id:str):
388 if not self._notes_loaded:
389 self._load_notes()
390 original_note = self._notes.get(note_id)
391 if not original_note:
392 return None
393 return self._note_to_legacy_dict(original_note)
395 def set_notes(self, notes:Union[list[Note], list[dict], Note, dict]):
396 if not isinstance(notes, list):
397 notes = [notes]
399 # if there were any notes, we need to mark them for removal
400 for nid in self._notes:
401 self._notes[nid] = False
403 self._notes_loaded = True
404 ids = []
406 for n in notes:
407 assigned_to = None
408 deadline = None
410 if isinstance(n, dict):
411 if "flag" in n:
412 assigned_to = n["flag"].get("assigned_to")
413 deadline = n["flag"].get("deadline")
414 del n["flag"]
415 if "date" in n:
416 n["created_date"] = n["date"]
417 del n["date"]
418 n = Note(**n)
420 n.resource_type = self.__type__
421 if self.id is None:
422 self.set_id(self.makeid())
423 n.resource_id = self.id
425 if not n.id:
426 n.set_id(self.makeid())
428 if assigned_to is not None or deadline is not None:
429 # this is a note with a flag
430 self.set_flag(n.id, assigned_to, deadline)
432 self._notes[n.id] = n
433 ids.append(n.id)
435 self.__seamless__.set_list("admin.note_ids", ids)
437 def remove_notes(self):
438 for n in self._notes.keys():
439 self.remove_note_by_id(n)
441 def _note_to_legacy_dict(self, note, flag=None):
442 d = deepcopy(note.data)
443 if "created_date" in d:
444 d["date"] = d["created_date"]
445 del d["created_date"]
446 if "last_updated" in d:
447 del d["last_updated"]
448 if "resource_type" in d:
449 del d["resource_type"]
450 if "resource_id" in d:
451 del d["resource_id"]
452 if flag is None:
453 flag = self.__seamless__.get_single("admin.flag")
454 if flag is not None:
455 if flag["note_id"] == note.id:
456 d.update({"flag": deepcopy(flag)})
457 del d["flag"]["note_id"]
458 return d
460 @property
461 def notes(self):
462 if not self._notes_loaded:
463 self._load_notes()
465 flag = self.__seamless__.get_single("admin.flag")
466 if flag is None:
467 return [self._note_to_legacy_dict(n) for n in self._notes.values() if n]
469 raw = []
470 for n in self._notes.values():
471 if not n:
472 continue
473 raw.append(self._note_to_legacy_dict(n, flag))
474 return raw
476 @property
477 def note_objects(self):
478 if not self._notes_loaded:
479 self._load_notes()
480 return [n for n in self._notes.values() if n]
482 @property
483 def note_ids(self):
484 return self.__seamless__.get_list("admin.note_ids")
486 @property
487 def notes_except_flags(self):
488 return [note for note in self.notes if not note.get("flag") or not note["flag"].get("assigned_to")]
490 @property
491 def flags(self):
492 return [note for note in self.notes if note.get("flag") and note["flag"].get("assigned_to")]
494 def get_detached_notes(self):
495 """
496 Get a copy of all this record's notes, with their identifying information removed
498 Useful for creating copies of objects, where we have copied the note content, but not the
499 references/ids
500 :return:
501 """
502 detached = []
503 for n in self.notes:
504 n_copy = deepcopy(n)
505 if "id" in n_copy:
506 del n_copy["id"]
507 if "flag" in n_copy:
508 del n_copy["flag"]
509 detached.append(n_copy)
510 return detached
512 @property
513 def flag_assignee(self):
514 return self.__seamless__.get_single("admin.flag.assigned_to")
516 @property
517 def flag_note_id(self):
518 return self.__seamless__.get_single("admin.flag.note_id")
520 @property
521 def flag_deadline(self):
522 return self.__seamless__.get_single("admin.flag.deadline")
524 @property
525 def is_flagged(self):
526 return self.__seamless__.get_single("admin.flag") is not None
528 def set_flag(self, note_id, assigned_to=None, deadline=None):
529 flag = {"note_id": note_id}
530 if assigned_to is not None:
531 flag["assigned_to"] = assigned_to
532 if deadline is not None:
533 flag["deadline"] = deadline
534 self.__seamless__.set_with_struct("admin.flag", flag)
535 is_it_there = self.__seamless__.get_single("admin.flag")
536 return flag
538 def resolve_flag(self, flag_id, updated_note, date, author):
539 flag = self.get_note_by_id(flag_id)
540 if flag:
541 self.remove_note_by_id(flag_id)
542 date = date if date else flag.get("date", dates.now_str())
543 author_id = author if author else flag.get("author_id")
544 return self.add_note(updated_note, date, flag_id, author_id)
545 else:
546 # FIXME: it's possible in some circumstances not resolving the note could leave behind some detritus
547 return None
549 def delete_flag_and_note(self):
550 nid = self.flag_note_id
551 if nid is not None:
552 self.remove_note_by_id(nid)
554 def delete_flag(self):
555 self.__seamless__.delete("admin.flag")
557 @property
558 def most_urgent_flag_deadline(self):
559 flag = self.__seamless__.get_single("admin.flag")
560 if flag is None:
561 return dates.far_in_the_future()
563 deadlines = [flag.get("deadline")] if flag.get("deadline") else []
565 # Find the flag with the earliest deadline
566 if not len(deadlines):
567 return dates.far_in_the_future() # Dummy date for least urgent date
569 earliest_flag_deadline = find_earliest_date(deadlines, dates_format=FMT_DATE_STD)
571 return earliest_flag_deadline
573 @property
574 def ordered_notes(self):
575 """Orders notes by newest first"""
576 notes = self.notes
577 return self._order_notes(notes)
579 @property
580 def ordered_notes_except_flags(self):
581 notes = self.notes_except_flags
582 return self._order_notes(notes)
584 def _order_notes(self, notes):
585 clusters = {}
586 for note in notes:
587 if "date" not in note:
588 note["date"] = DEFAULT_TIMESTAMP_VAL # this really means something is broken with note date setting, which needs to be fixed
589 if note["date"] not in clusters:
590 clusters[note["date"]] = [note]
591 else:
592 clusters[note["date"]].append(note)
594 ordered_keys = sorted(list(clusters.keys()), reverse=True)
595 ordered = []
596 for key in ordered_keys:
597 clusters[key].reverse()
598 ordered += clusters[key]
599 return ordered
601 def _persist_notes(self):
602 for nid, n in self._notes.items():
603 if n is False:
604 Note.remove_by_id(nid)
605 elif n is not None:
606 n.save()
608 def _load_notes(self):
609 for n in self.note_ids:
610 if n not in self._notes:
611 self._notes[n] = Note.pull(n)
612 self._notes_loaded = True
614 #### end of notes methods
616 def bibjson(self):
617 bj = self.__seamless__.get_single("bibjson")
618 if bj is None:
619 self.__seamless__.set_single("bibjson", {})
620 bj = self.__seamless__.get_single("bibjson")
621 return JournalLikeBibJSON(bj)
623 def set_bibjson(self, bibjson):
624 bibjson = bibjson.data if isinstance(bibjson, JournalLikeBibJSON) else bibjson
625 self.__seamless__.set_with_struct("bibjson", bibjson)
627 def save(self, *args, **kwargs):
628 res = super(JournalLikeObject, self).save(**kwargs)
629 self._persist_notes()
630 return res
632 ######################################################
633 ## DEPRECATED METHODS
635 def known_issns(self):
636 """
637 DEPRECATED
639 all issns this journal is known by
641 This used to mean "all issns the journal has ever been known by", but that definition has changed since
642 continuations have been separated from the single journal object model.
644 Now this is just a proxy for self.bibjson().issns()
645 """
646 return self.bibjson().issns()
648 def get_latest_contact_name(self):
649 return self.contact_name
651 def get_latest_contact_email(self):
652 return self.contact_email
654 def add_contact(self, name, email):
655 self.set_contact(name, email)
657 def remove_contacts(self):
658 self.remove_contact()
660 ######################################################
661 ## internal utility methods
663 def _generate_index(self):
664 # the index fields we are going to generate
665 titles = []
666 subjects = []
667 schema_subjects = []
668 schema_codes = []
669 schema_codes_tree = []
670 classification = []
671 langs = []
672 country = None
673 license = []
674 publisher = []
675 classification_paths = []
676 unpunctitle = None
677 asciiunpunctitle = None
678 continued = "No"
679 has_editor_group = "No"
680 has_editor = "No"
681 is_flagged = False
682 flag_assignees = []
683 most_urgent_flag_deadline = dates.far_in_the_future()
685 # the places we're going to get those fields from
686 cbib = self.bibjson()
688 # get the title out of the current bibjson
689 if cbib.title is not None:
690 titles.append(cbib.title)
691 if cbib.alternative_title:
692 titles.append(cbib.alternative_title)
694 # get the subjects and concatenate them with their schemes from the current bibjson
695 for subs in cbib.subject:
696 scheme = subs.get("scheme")
697 term = subs.get("term")
698 if term:
699 subjects.append(term)
700 schema_subjects.append(scheme + ":" + term)
701 classification.append(term)
702 if "code" in subs:
703 schema_codes.append(scheme + ":" + subs.get("code"))
705 # now expand the classification to hold all its parent terms too
706 additional = []
707 for c in classification:
708 tp = cbib.term_path(c)
709 if tp is not None:
710 additional += tp
711 classification += additional
713 # add the keywords to the non-schema subjects (but not the classification)
714 subjects += cbib.keywords
716 # get the bibjson object to convert the languages to the english form
717 langs = cbib.language_name()
719 # get the english name of the country
720 country = cbib.country_name()
722 # get the type of the licenses
723 for l in cbib.licences:
724 license.append(l.get("type"))
726 # check for any flags
727 is_flagged = self.is_flagged
729 # FIXME: essentially this first index entry is redundant now, as flag_assignee is
730 # singular in this updated model
731 flag_assignees = [self.flag_assignee]
732 most_urgent_flag_deadline = self.most_urgent_flag_deadline
734 # deduplicate the lists
735 titles = list(set(titles))
736 subjects = list(set(subjects))
737 schema_subjects = list(set(schema_subjects))
738 classification = list(set(classification))
739 license = list(set(license))
740 schema_codes = list(set(schema_codes))
742 # get the full classification paths for the subjects
743 classification_paths = cbib.lcc_paths()
744 schema_codes_tree = cbib.lcc_codes_full_list()
746 # create an unpunctitle
747 if cbib.title is not None:
748 throwlist = string.punctuation + '\n\t'
749 unpunctitle = "".join(c for c in cbib.title if c not in throwlist).strip()
750 try:
751 asciiunpunctitle = unidecode(unpunctitle)
752 except:
753 asciiunpunctitle = unpunctitle
755 # record if this journal object is a continuation
756 if len(cbib.replaces) > 0 or len(cbib.is_replaced_by) > 0:
757 continued = "Yes"
759 if self.editor_group is not None:
760 has_editor_group = "Yes"
762 if self.editor is not None:
763 has_editor = "Yes"
765 # build the index part of the object
766 index = {}
768 if country is not None:
769 index["country"] = country
770 if unpunctitle is not None:
771 index["unpunctitle"] = unpunctitle
772 if asciiunpunctitle is not None:
773 index["asciiunpunctitle"] = asciiunpunctitle
774 if is_flagged:
775 index["is_flagged"] = is_flagged
776 index["flag_assignees"] = flag_assignees
777 if most_urgent_flag_deadline:
778 index["most_urgent_flag_deadline"] = most_urgent_flag_deadline
779 index["continued"] = continued
780 index["has_editor_group"] = has_editor_group
781 index["has_editor"] = has_editor
783 index["issn"] = cbib.issns()
785 if len(titles) > 0:
786 index["title"] = titles
787 if len(subjects) > 0:
788 index["subject"] = subjects
789 if len(schema_subjects) > 0:
790 index["schema_subject"] = schema_subjects
791 if len(classification) > 0:
792 index["classification"] = classification
793 if len(langs) > 0:
794 index["language"] = langs
795 if len(license) > 0:
796 index["license"] = license
797 if len(classification_paths) > 0:
798 index["classification_paths"] = classification_paths
799 if len(schema_codes) > 0:
800 index["schema_code"] = schema_codes
801 if len(schema_codes_tree) > 0:
802 index["schema_codes_tree"] = schema_codes_tree
804 self.__seamless__.set_with_struct("index", index)
806 # also add the admin index fields
808 # index the content of the notes
809 admin_index = {}
811 notes = []
812 note_objects = self.note_objects
813 for n in note_objects:
814 notes.append(n.note)
816 if len(notes) > 0:
817 admin_index["notes"] = notes
819 self.__seamless__.set_with_struct("admin.index", admin_index)
821class Journal(JournalLikeObject):
822 __type__ = "journal"
824 __SEAMLESS_STRUCT__ = [
825 shared_structs.JOURNAL_BIBJSON,
826 shared_structs.SHARED_JOURNAL_LIKE,
827 JOURNAL_STRUCT
828 ]
830 __SEAMLESS_COERCE__ = COERCE_MAP
832 def __init__(self, **kwargs):
833 # FIXME: hack, to deal with ES integration layer being improperly abstracted
834 if "_source" in kwargs:
835 kwargs = kwargs["_source"]
836 # FIXME: I have taken this out for the moment, as I'm not sure it's what we should be doing
837 # if kwargs:
838 # self.add_autogenerated_fields(**kwargs)
839 super(Journal, self).__init__(raw=kwargs)
841 @classmethod
842 def add_autogenerated_fields(cls, **kwargs):
843 bib = kwargs["bibjson"]
844 if "apc" in bib and bib["apc"] != '':
845 bib["apc"]["has_apc"] = len(bib["apc"]["max"]) != 0
846 else:
847 bib["apc"] = {"has_apc": False}
848 if "deposit_policy" in bib and bib["deposit_policy"] != []:
849 bib["deposit_policy"]["has_policy"] = True
850 else:
851 ##change made in https://github.com/DOAJ/doaj/commit/e507123f423fe16fd270744055da0129e2b32005
852 bib["deposit_policy"] = {"has_policy": False}
853 if "other_charges" in bib and bib["other_charges"] != '':
854 bib["other_charges"]["has_other_charges"] = bib["other_charges"]["url"] is not None
855 else:
856 bib["other_charges"] = {"has_other_charges": False}
857 if "copyright" in bib and bib["copyright"]["url"] != '':
858 bib["copyright"]["author_retains"] = bib["copyright"]["url"] is not None
859 else:
860 bib["copyright"] = {"author_retains": False}
861 if "pid_scheme" in bib and bib["pid_scheme"] != '':
862 bib["pid_scheme"]["has_pid_scheme"] = len(bib["pid_scheme"]["scheme"]) != 0
863 else:
864 bib["pid_scheme"] = {"has_pid_scheme": False}
865 if "preservation" in bib and bib["preservation"] != '':
866 bib["preservation"]["has_preservation"] = (len(bib["preservation"]) != 0 or
867 bib["national_library"] is not None)
868 else:
869 bib["preservation"] = {"has_preservation": True}
871 #####################################################
872 ## Journal-specific data access methods
874 @classmethod
875 def all_in_doaj(cls, page_size=5000):
876 q = JournalQuery()
877 return cls.iterate(q.all_in_doaj(), page_size=page_size, wrap=True, keepalive='5m')
879 @classmethod
880 def find_by_publisher(cls, publisher, exact=True):
881 q = PublisherQuery(publisher, exact)
882 result = cls.query(q=q.query())
883 records = [Journal(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
884 return records
886 @classmethod
887 def find_by_title(cls, title):
888 q = TitleQuery(title)
889 result = cls.query(q=q.query())
890 records = [Journal(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
891 return records
893 @classmethod
894 def delete_selected(cls, query, articles=False, snapshot_journals=True, snapshot_articles=True):
895 if articles:
896 # list the issns of all the journals
897 issns = cls.issns_by_query(query)
899 # issue a delete request over all the articles by those issns
900 from portality.models import Article
901 Article.delete_by_issns(issns, snapshot=snapshot_articles)
903 # snapshot the journal record
904 if snapshot_journals:
905 js = cls.iterate(query, page_size=1000)
906 for j in js:
907 j.snapshot()
909 # finally issue a delete request against the journals
910 cls.delete_by_query(query)
912 @classmethod
913 def add_mapping_extensions(cls, default_mappings: dict):
914 default_mappings_copy = deepcopy(default_mappings)
915 mapping_extensions = app.config.get("DATAOBJ_TO_MAPPING_COPY_TO_EXTENSIONS")
916 for key, value in mapping_extensions.items():
917 if key in default_mappings_copy:
918 default_mappings_copy[key] = {**default_mappings_copy[key], **value}
919 return default_mappings_copy
921 def all_articles(self):
922 from portality.models import Article
923 return Article.find_by_issns(self.known_issns())
925 def article_stats(self):
926 from portality.models import Article
927 q = ArticleStatsQuery(self.known_issns())
928 data = Article.query(q=q.query())
929 hits = data.get("hits", {})
930 total = hits.get("total", {}).get('value', 0)
931 latest = None
932 if total > 0:
933 latest = hits.get("hits", [])[0].get("_source").get("created_date")
934 return {
935 "total": total,
936 "latest": latest
937 }
939 def mappings(self):
940 return es_data_mapping.create_mapping(self.__seamless_struct__.raw, MAPPING_OPTS)
942 ############################################
943 ## base property methods
945 @property
946 def toc_id(self):
947 id_ = self.bibjson().get_preferred_issn()
948 if not id_:
949 id_ = self.id
950 return id_
952 ############################################################
953 ## revision history methods
955 def snapshot(self):
956 from portality.models import JournalHistory
958 snap = deepcopy(self.data)
959 if "id" in snap:
960 snap["about"] = snap["id"]
961 del snap["id"]
962 if "index" in snap:
963 del snap["index"]
964 if "last_updated" in snap:
965 del snap["last_updated"]
966 if "created_date" in snap:
967 del snap["created_date"]
969 hist = JournalHistory(**snap)
970 hist.save()
972 #######################################################################
973 ## Conversion methods
975 def make_continuation(self, type, eissn=None, pissn=None, title=None):
976 # check that the type is one we know. Must be either 'replaces' or 'is_replaced_by'
977 if type not in ["replaces", "is_replaced_by"]:
978 raise ContinuationException("type must be one of 'replaces' or 'is_replaced_by'")
980 if eissn is None and pissn is None:
981 raise ContinuationException("You must create a continuation with at least one issn")
983 # take a copy of the raw data for this journal, and the issns for this journal
984 raw_cont = deepcopy(self.data)
985 bibjson = self.bibjson()
986 issns = bibjson.issns()
987 cissns = []
989 # make a new instance of the journal - this will be our continuation
990 del raw_cont["id"]
991 del raw_cont["created_date"]
992 del raw_cont["last_updated"]
993 j = Journal(**raw_cont)
995 # ensure that the journal is NOT in doaj. That will be for the admin to decide
996 j.set_in_doaj(False)
998 # get a copy of the continuation's bibjson, then remove the existing issns
999 cbj = j.bibjson()
1000 del cbj.eissn
1001 del cbj.pissn
1003 # also remove any existing continuation information
1004 del cbj.replaces
1005 del cbj.is_replaced_by
1006 del cbj.discontinued_date
1008 # now write the new identifiers
1009 if eissn is not None and eissn != "":
1010 cissns.append(eissn)
1011 cbj.eissn = eissn
1012 if pissn is not None and pissn != "":
1013 cissns.append(pissn)
1014 cbj.pissn = pissn
1016 # update the title
1017 if title is not None:
1018 cbj.title = title
1020 # now add the issns of the original journal in the appropriate field
1021 #
1022 # This is a bit confusing - because we're asking this of a Journal object, the relationship type we're asking
1023 # for relates to this journal, not to the continuation we are creating. This means that when setting the
1024 # new continuations properties, we have to do the opposite to what we do to the journal's properties
1025 #
1026 # "replaces" means that the current journal replaces the new continuation
1027 if type == "replaces":
1028 bibjson.replaces = cissns
1029 cbj.is_replaced_by = issns
1031 # "is_replaced_by" means that the current journal is replaced by the new continuation
1032 elif type == "is_replaced_by":
1033 bibjson.is_replaced_by = cissns
1034 cbj.replaces = issns
1036 # save this journal
1037 self.save()
1039 # save the continuation, and return a copy to the caller
1040 j.save()
1041 return j
1043 ####################################################
1044 ## admin data methods
1046 def is_in_doaj(self):
1047 return self.__seamless__.get_single("admin.in_doaj", default=False)
1049 def set_in_doaj(self, value):
1050 self.__seamless__.set_with_struct("admin.in_doaj", value)
1052 def is_ticked(self):
1053 return self.__seamless__.get_single("admin.ticked", default=False)
1055 def set_ticked(self, ticked):
1056 self.__seamless__.set_with_struct("admin.ticked", ticked)
1058 @property
1059 def current_application(self):
1060 return self.__seamless__.get_single("admin.current_application")
1062 def set_current_application(self, application_id):
1063 self.__seamless__.set_with_struct("admin.current_application", application_id)
1065 def remove_current_application(self):
1066 self.__seamless__.delete("admin.current_application")
1068 # Related Applications Functions
1069 ###########
1071 @property
1072 def related_applications(self):
1073 return self.__seamless__.get_list("admin.related_applications")
1075 def add_related_application(self, application_id, date_accepted=None, status=None):
1076 obj = {"application_id": application_id}
1077 self.__seamless__.delete_from_list("admin.related_applications", matchsub=obj)
1078 if date_accepted is not None:
1079 obj["date_accepted"] = date_accepted
1080 if status is not None:
1081 obj["status"] = status
1082 self.__seamless__.add_to_list_with_struct("admin.related_applications", obj)
1084 def set_related_applications(self, related_applications_records):
1085 self.__seamless__.set_with_struct("admin.related_applications", related_applications_records)
1087 def remove_related_applications(self):
1088 self.__seamless__.delete("admin.related_applications")
1090 def remove_related_application(self, application_id):
1091 self.set_related_applications([r for r in self.related_applications
1092 if r.get("application_id") != application_id])
1094 def related_application_record(self, application_id):
1095 for record in self.related_applications:
1096 if record.get("application_id") == application_id:
1097 return record
1098 return None
1100 def latest_related_application_id(self):
1101 related = self.related_applications
1102 if len(related) == 0:
1103 return None
1104 if len(related) == 1:
1105 return related[0].get("application_id")
1106 sorted(related, key=lambda x: x.get("date_accepted", DEFAULT_TIMESTAMP_VAL))
1107 return related[0].get("application_id")
1109 @property
1110 def last_update_request(self):
1111 related = self.related_applications_ordered
1112 if related is None:
1113 return None
1114 return related[0].get("date_accepted", DEFAULT_TIMESTAMP_VAL)
1116 @property
1117 def related_applications_ordered(self):
1118 related = self.related_applications
1119 if len(related) == 0:
1120 return None
1121 sorted(related, key=lambda x: x.get("date_accepted", DEFAULT_TIMESTAMP_VAL))
1122 return related
1124 ########
1126 @property
1127 def last_full_review(self):
1128 return self.__seamless__.get_single("admin.last_full_review")
1130 @property
1131 def last_full_review_timestamp(self):
1132 return self.__seamless__.get_single("admin.last_full_review", coerce=coerce.to_datestamp())
1134 @last_full_review.setter
1135 def last_full_review(self, value):
1136 self.__seamless__.set_with_struct("admin.last_full_review", value)
1138 @property
1139 def last_withdrawn(self):
1140 return self.__seamless__.get_single("admin.last_withdrawn")
1142 @property
1143 def last_withdrawn_timestamp(self):
1144 return self.__seamless__.get_single("admin.last_withdrawn", coerce=coerce.to_datestamp())
1146 @last_withdrawn.setter
1147 def last_withdrawn(self, value):
1148 self.__seamless__.set_with_struct("admin.last_withdrawn", value)
1150 @property
1151 def last_reinstated(self):
1152 return self.__seamless__.get_single("admin.last_reinstated")
1154 @property
1155 def last_reinstated_timestamp(self):
1156 return self.__seamless__.get_single("admin.last_reinstated", coerce=coerce.to_datestamp())
1158 @last_reinstated.setter
1159 def last_reinstated(self, value):
1160 self.__seamless__.set_with_struct("admin.last_reinstated", value)
1162 @property
1163 def last_owner_transfer(self):
1164 return self.__seamless__.get_single("admin.last_owner_transfer")
1166 @property
1167 def last_owner_transfer_timestamp(self):
1168 return self.__seamless__.get_single("admin.last_owner_transfer", coerce=coerce.to_datestamp())
1170 @last_owner_transfer.setter
1171 def last_owner_transfer(self, value):
1172 self.__seamless__.set_with_struct("admin.last_owner_transfer", value)
1174 ########################################################################
1175 ## Functions for handling continuations
1178 def _get_continuations(self, issns,
1179 get_sub_journals: Callable,
1180 journal_caches: set[str] = None) -> Iterable['Journal']:
1181 """
1183 Parameters
1184 ----------
1185 issns
1186 get_sub_journals
1187 journal_caches
1188 contain completed journals ids, avoid infinite recursion by passing a
1189 set of journal objects that have already been processed
1190 """
1191 journal_caches = journal_caches or set()
1192 journal_caches.add(self.id)
1193 journals = self.q2obj(q=ContinuationQuery(issns).query())
1194 journals = [j for j in journals if j.id not in journal_caches]
1195 journal_caches.update({j.id for j in journals})
1197 subjournals = []
1198 for j in journals:
1199 subjournals += get_sub_journals(j, journal_caches)
1201 return journals + subjournals
1203 def get_future_continuations(self, journal_caches: set[str]=None) -> Iterable['Journal']:
1204 return self._get_continuations(self.bibjson().is_replaced_by,
1205 lambda j, jc: j.get_future_continuations(jc),
1206 journal_caches=journal_caches)
1208 def get_past_continuations(self, journal_caches: set[str]=None) -> Iterable['Journal']:
1209 return self._get_continuations(self.bibjson().replaces,
1210 lambda j, jc: j.get_past_continuations(jc),
1211 journal_caches=journal_caches)
1213 #######################################################################
1215 #####################################################
1216 ## operations we can do to the journal
1218 def calculate_tick(self):
1219 created_date = self.created_date
1220 last_update_request = self.last_update_request
1222 tick_threshold = app.config.get("TICK_THRESHOLD", '2014-03-19T00:00:00Z')
1223 threshold = dates.parse(tick_threshold)
1225 if created_date is None: # don't worry about the last_update_request date - you can't update unless you've been created!
1226 # we haven't even saved the record yet. All we need to do is check that the tick
1227 # threshold is in the past (which I suppose theoretically it could not be), then
1228 # set it
1229 if dates.now() >= threshold:
1230 self.set_ticked(True)
1231 else:
1232 self.set_ticked(False)
1233 return
1235 # otherwise, this is an existing record, and we just need to update it
1237 # convert the strings to datetime objects
1238 created = dates.parse(created_date)
1239 lud = None
1240 if last_update_request is not None:
1241 lud = dates.parse(last_update_request)
1243 if created >= threshold and self.is_in_doaj():
1244 self.set_ticked(True)
1245 return
1247 if lud is not None and lud >= threshold and self.is_in_doaj():
1248 self.set_ticked(True)
1249 return
1251 self.set_ticked(False)
1253 def propagate_in_doaj_status_to_articles(self):
1254 for article in self.all_articles():
1255 article.set_in_doaj(self.is_in_doaj())
1256 article.save()
1258 def prep(self, is_update=True):
1259 self._ensure_in_doaj()
1260 self.calculate_tick()
1261 self._generate_index()
1262 self._calculate_has_apc()
1263 self._generate_autocompletes()
1264 if is_update:
1265 self.set_last_updated()
1267 def save(self, snapshot=True, sync_owner=True, update_last_updated=True, **kwargs):
1268 self.prep(is_update=update_last_updated)
1269 self.verify_against_struct()
1270 if sync_owner:
1271 self._sync_owner_to_application()
1272 res = super(Journal, self).save(update_last_updated=update_last_updated, **kwargs)
1273 if snapshot:
1274 self.snapshot()
1275 return res
1277 ######################################################
1278 ## internal utility methods
1280 def _generate_autocompletes(self):
1281 bj = self.bibjson()
1282 publisher = bj.publisher
1283 institution = bj.institution
1285 if publisher is not None:
1286 self.__seamless__.set_with_struct("index.publisher_ac", publisher.lower())
1288 if institution is not None:
1289 self.__seamless__.set_with_struct("index.institution_ac", institution.lower())
1291 def _ensure_in_doaj(self):
1292 if self.__seamless__.get_single("admin.in_doaj", default=None) is None:
1293 self.set_in_doaj(False)
1295 def _sync_owner_to_application(self):
1296 if self.current_application is None:
1297 return
1298 from portality.models.v2.application import Application
1299 ca = Application.pull(self.current_application)
1300 if ca is not None and ca.owner != self.owner:
1301 ca.set_owner(self.owner)
1302 ca.save(sync_owner=False)
1304 def _calculate_has_apc(self):
1305 # work out of the journal has an apc
1306 has_apc = "No Information"
1307 apc_present = self.bibjson().has_apc
1308 if apc_present:
1309 has_apc = "Yes"
1310 elif self.is_ticked(): # Because if an item is not ticked we want to say "No Information"
1311 has_apc = "No"
1313 self.__seamless__.set_with_struct("index.has_apc", has_apc)
1316MAPPING_OPTS = {
1317 "dynamic": None,
1318 "coerces": Journal.add_mapping_extensions(app.config["DATAOBJ_TO_MAPPING_DEFAULTS"]),
1319 "exceptions": {**app.config["ADMIN_NOTES_SEARCH_MAPPING"], **app.config["JOURNAL_EXCEPTION_MAPPING"]},
1320 "additional_mappings": app.config["ADMIN_NOTES_INDEX_ONLY_FIELDS"]
1321}
1324########################################################
1325## Data Access Queries
1327class JournalQuery(object):
1328 """
1329 wrapper around the kinds of queries we want to do against the journal type
1330 """
1331 issn_query = {
1332 "track_total_hits": True,
1333 "query": {
1334 "bool": {
1335 "must": [
1336 {
1337 "terms": {"index.issn.exact": "<issn>"}
1338 }
1339 ]
1340 }
1341 }
1342 }
1344 must_query = {
1345 "track_total_hits": True,
1346 "query": {
1347 "bool": {
1348 "must": [
1349 ]
1350 }
1351 }
1352 }
1354 all_doaj = {
1355 "track_total_hits": True,
1356 "query": {
1357 "bool": {
1358 "must": [
1359 {"term": {"admin.in_doaj": True}}
1360 ]
1361 }
1362 }
1363 }
1365 _minified_fields = ["id", "bibjson.title", "last_updated"]
1367 def __init__(self, minified=False, sort_by_title=False):
1368 self.query = None
1369 self.minified = minified
1370 self.sort_by_title = sort_by_title
1372 def find_by_issn(self, issns, in_doaj=None, max=10):
1373 self.query = deepcopy(self.issn_query)
1374 self.query["query"]["bool"]["must"][0]["terms"]["index.issn.exact"] = issns
1375 if in_doaj is not None:
1376 self.query["query"]["bool"]["must"].append({"term": {"admin.in_doaj": in_doaj}})
1377 self.query["size"] = max
1379 def find_by_issn_exact(self, issns, in_doaj=None, max=10):
1380 self.query = deepcopy(self.must_query)
1381 for issn in issns:
1382 self.query["query"]["bool"]["must"].append({"term": {"index.issn.exact": issn}})
1383 if in_doaj is not None:
1384 self.query["query"]["bool"]["must"].append({"term": {"admin.in_doaj": in_doaj}})
1385 self.query["size"] = max
1387 def all_in_doaj(self):
1388 q = deepcopy(self.all_doaj)
1389 if self.minified:
1390 q["fields"] = self._minified_fields
1391 if self.sort_by_title:
1392 q["sort"] = [{"bibjson.title.exact": {"order": "asc"}}]
1393 return q
1396class JournalURLQuery(object):
1397 def __init__(self, url, in_doaj=None, max=10):
1398 self.url = url
1399 self.in_doaj = in_doaj
1400 self.max = max
1402 def query(self):
1403 q = {
1404 "track_total_hits": True,
1405 "query": {
1406 "bool": {
1407 "must": [
1408 {
1409 "match": {"bibjson.ref.journal.exact": self.url}
1410 }
1411 ]
1412 }
1413 },
1414 "size": self.max
1415 }
1416 if self.in_doaj is not None:
1417 q["query"]["bool"]["must"].append({"term": {"admin.in_doaj": self.in_doaj}})
1418 return q
1421class IssnQuery(object):
1422 def __init__(self, owner, in_doaj=None, issn_field=None):
1423 self._owner = owner
1424 self._in_doaj = in_doaj
1425 self._issn_field = issn_field or 'index.issn.exact'
1427 def query(self):
1428 musts = [{"term": {"admin.owner.exact": self._owner}}]
1429 if self._in_doaj is not None:
1430 musts.append({"term": {"admin.in_doaj": self._in_doaj}})
1431 return {
1432 "track_total_hits": True,
1433 "query": {
1434 "bool": {
1435 "must": musts
1436 }
1437 },
1438 "size": 0,
1439 "aggs": {
1440 "issns": {
1441 "terms": {
1442 "field": self._issn_field,
1443 "size": 10000,
1444 "order": {"_key": "asc"}
1445 }
1446 }
1447 }
1448 }
1451class OwnerQuery(object):
1452 """ Query to supply all full journal sources by owner """
1453 base_query = {
1454 "track_total_hits": True,
1455 "query": {
1456 "term": {"admin.owner.exact": "<owner id here>"}
1457 },
1458 "size": 10000,
1459 }
1461 def __init__(self, owner):
1462 self._query = deepcopy(self.base_query)
1463 self._query["query"]["term"]["admin.owner.exact"] = owner
1465 def query(self):
1466 return self._query
1469class PublisherQuery(object):
1470 exact_query = {
1471 "track_total_hits": True,
1472 "query": {
1473 "term": {"bibjson.publisher.name.exact": "<publisher name here>"}
1474 },
1475 "size": 10000
1476 }
1478 inexact_query = {
1479 "track_total_hits": True,
1480 "query": {
1481 "term": {"bibjson.publisher.name": "<publisher name here>"}
1482 },
1483 "size": 10000
1484 }
1486 def __init__(self, publisher, exact=True):
1487 self.publisher = publisher
1488 self.exact = exact
1490 def query(self):
1491 q = None
1492 if self.exact:
1493 q = deepcopy(self.exact_query)
1494 q["query"]["term"]["bibjson.publisher.name.exact"] = self.publisher
1495 else:
1496 q = deepcopy(self.inexact_query)
1497 q["query"]["term"]["bibjson.publisher.name"] = self.publisher.lower()
1498 return q
1501class TitleQuery(object):
1502 base_query = {
1503 "track_total_hits": True,
1504 "query": {
1505 "term": {"index.title.exact": "<title here>"}
1506 },
1507 "size": 10000
1508 }
1510 def __init__(self, title):
1511 self.title = title
1513 def query(self):
1514 q = deepcopy(self.base_query)
1515 q["query"]["term"]["index.title.exact"] = self.title
1516 return q
1519class ContinuationQuery(object):
1520 def __init__(self, issns):
1521 self.issns = issns
1523 def query(self):
1524 return {
1525 "track_total_hits": True,
1526 "query": {
1527 "bool": {
1528 "must": [
1529 {"terms": {"index.issn.exact": self.issns}}
1530 ]
1531 }
1532 },
1533 "size": 10000
1534 }
1537class ArticleStatsQuery(object):
1538 def __init__(self, issns):
1539 self.issns = issns
1541 def query(self):
1542 return {
1543 "track_total_hits": True,
1544 "query": {
1545 "bool": {
1546 "must": [
1547 {"terms": {"index.issn.exact": self.issns}},
1548 {"term": {"admin.in_doaj": True}}
1549 ]
1550 }
1551 },
1552 "size": 1,
1553 "_source": {
1554 "include": ["created_date"]
1555 },
1556 "sort": [{"created_date": {"order": "desc"}}]
1557 }
1560class RecentJournalsQuery(object):
1561 def __init__(self, max):
1562 self.max = max
1564 def query(self):
1565 return {
1566 "track_total_hits": True,
1567 "query": {"match_all": {}},
1568 "size": self.max,
1569 "sort": [
1570 {"created_date": {"order": "desc"}}
1571 ]
1572 }