Coverage for portality/models/article.py: 83%
777 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1import string
3from unidecode import unidecode
4from functools import reduce
5from copy import deepcopy
6from datetime import datetime
8from portality import datasets, constants
9from portality.core import app
10from portality.dao import DomainObject
11from portality.lib import es_data_mapping
12from portality.lib.coerce import COERCE_MAP
13from portality.lib.dates import FMT_DATETIME_STD
14from portality.lib.seamless import SeamlessMixin
15from portality.models import Journal
16from portality.models.v1.bibjson import GenericBibJSON # NOTE that article specifically uses the v1 BibJSON
17from portality.models.v1 import shared_structs
18from portality.models.v2.shared_structs import ARTICLE_STRUCT
19from portality.lib import normalise, dates
22class NoJournalException(Exception):
23 pass
25class NoValidOwnerException(Exception):
26 pass
29ARTICLE_BIBJSON_EXTENSION = {
30 "objects" : ["bibjson"],
31 "structs" : {
32 "bibjson" : {
33 "fields" : {
34 "year" : {"coerce" : "unicode"},
35 "month" : {"coerce" : "unicode"},
36 "start_page" : {"coerce" : "unicode"},
37 "end_page" : {"coerce" : "unicode"},
38 "abstract" : {"coerce" : "unicode"}
39 },
40 "lists" : {
41 "author" : {"contains" : "object"}
42 },
43 "objects" : [
44 "journal"
45 ],
47 "structs" : {
48 "author" : {
49 "fields" : {
50 "name" : {"coerce" : "unicode"},
51 "affiliation" : {"coerce" : "unicode"},
52 "email" : {"coerce": "unicode"},
53 "orcid_id" : {"coerce" : "unicode"}
54 }
55 },
57 "journal" : {
58 "fields" : {
59 "volume" : {"coerce" : "unicode"},
60 "number" : {"coerce" : "unicode"},
61 "publisher" : {"coerce" : "unicode"},
62 "title" : {"coerce" : "unicode"},
63 "country" : {"coerce" : "unicode"}
64 },
65 "lists" : {
66 "language" : {"contains" : "field", "coerce" : "unicode"},
67 "issns" : {"contains" : "field", "coerce" : "unicode"}
68 }
69 }
70 }
72 }
73 }
74}
76MAPPING_OPTS = {
77 "dynamic": None,
78 "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"],
79 "exceptions": app.config["ARTICLE_EXCEPTION_MAPPING"],
80 "additional_mappings": {}
81}
84class Article(SeamlessMixin, DomainObject):
85 __type__ = "article"
87 __SEAMLESS_STRUCT__ = [
88 ARTICLE_STRUCT,
89 shared_structs.SHARED_BIBJSON,
90 ARTICLE_BIBJSON_EXTENSION
91 ]
93 __SEAMLESS_COERCE__ = COERCE_MAP
95 def mappings(self):
96 return es_data_mapping.create_mapping(self.__seamless_struct__.raw, MAPPING_OPTS)
98 @classmethod
99 def duplicates(cls, publisher_record_id=None, doi=None, fulltexts=None, title=None, volume=None, number=None, start=None, should_match=None, size=10):
100 # some input sanitisation
101 urls = fulltexts if isinstance(fulltexts, list) else [fulltexts] if isinstance(fulltexts, str) or isinstance(fulltexts, str) else []
103 # make sure that we're dealing with the normal form of the identifiers
104 norm_urls = []
105 for url in urls:
106 try:
107 norm = normalise.normalise_url(url)
108 norm_urls.append(norm)
109 except ValueError:
110 # use the non-normal form
111 norm_urls.append(url)
112 urls = norm_urls
114 try:
115 doi = normalise.normalise_doi(doi)
116 except ValueError:
117 # leave the doi as it is
118 pass
120 q = DuplicateArticleQuery(publisher_record_id=publisher_record_id,
121 doi=doi,
122 urls=urls,
123 title=title,
124 volume=volume,
125 number=number,
126 start=start,
127 should_match=should_match,
128 size=size)
130 # res = cls.query(q=q.query())
131 # return [cls(**hit.get("_source")) for hit in res.get("hits", {}).get("hits", [])]
132 return cls.q2obj(q=q.query())
134 @classmethod
135 def list_volumes(cls, issns):
136 q = ArticleVolumesQuery(issns)
137 result = cls.query(q=q.query())
138 return _human_sort([t.get("key") for t in result.get("aggregations", {}).get("vols", {}).get("buckets", [])])
140 @classmethod
141 def list_volume_issues(cls, issns, volume):
142 q = ArticleVolumesIssuesQuery(issns, volume)
143 result = cls.query(q=q.query())
144 return _human_sort([t.get("key") for t in result.get("aggregations", {}).get("issues", {}).get("buckets", [])])
146 @classmethod
147 def get_by_volume(cls, issns, volume):
148 q = ArticleQuery(issns=issns, volume=volume)
149 articles = cls.iterate(q.query(), page_size=1000)
150 return articles
152 @classmethod
153 def find_by_issns(cls, issns):
154 q = ArticleQuery(issns=issns)
155 articles = cls.iterate(q.query(), page_size=1000)
156 return articles
158 @classmethod
159 def count_by_issns(cls, issns, in_doaj=None):
160 q = ArticleQuery(issns=issns, in_doaj=in_doaj)
161 return cls.hit_count(q.query())
163 @classmethod
164 def delete_by_issns(cls, issns, snapshot=True):
165 q = ArticleQuery(issns=issns)
166 cls.delete_selected(query=q.query(), snapshot=snapshot)
168 @classmethod
169 def delete_selected(cls, query=None, owner=None, snapshot=True, tombstone=True):
170 if owner is not None:
171 from portality.models import Journal
172 issns = Journal.issns_by_owner(owner)
173 q = ArticleQuery(issns=issns)
174 query = q.query()
176 if snapshot or tombstone:
177 articles = cls.iterate(query, page_size=1000)
178 for article in articles:
179 if snapshot:
180 article.snapshot()
181 if tombstone:
182 article._tombstone()
183 return cls.delete_by_query(query)
185 def delete(self):
186 self._tombstone()
187 super(Article, self).delete()
189 def bibjson(self, **kwargs):
190 if "bibjson" not in self.data:
191 self.data["bibjson"] = {}
192 return ArticleBibJSON(self.data.get("bibjson"), **kwargs)
194 def set_bibjson(self, bibjson):
195 bibjson = bibjson.bibjson if isinstance(bibjson, ArticleBibJSON) else bibjson
196 self.data["bibjson"] = bibjson
198 def history(self):
199 hs = self.data.get("history", [])
200 tuples = []
201 for h in hs:
202 tuples.append((h.get("date"), ArticleBibJSON(h.get("bibjson"))))
203 return tuples
205 def snapshot(self):
206 from portality.models import ArticleHistory
208 snap = deepcopy(self.data)
209 if "id" in snap:
210 snap["about"] = snap["id"]
211 del snap["id"]
212 if "index" in snap:
213 del snap["index"]
214 if "last_updated" in snap:
215 del snap["last_updated"]
216 if "created_date" in snap:
217 del snap["created_date"]
219 hist = ArticleHistory(**snap)
220 hist.save()
221 return hist.id
223 def make_tombstone(self):
224 stone = ArticleTombstone()
225 stone.set_id(self.id)
226 sbj = stone.bibjson()
228 abj = self.bibjson()
230 # copy minimal essential metadata
231 sbj.title = abj.title
232 sbj.author = abj.author
233 sbj.volume = abj.volume
234 sbj.number = abj.number
235 sbj.start_page = abj.start_page
236 sbj.end_page = abj.end_page
237 sbj.journal_title = abj.journal_title
238 pissn = abj.get_one_identifier(abj.P_ISSN)
239 if pissn is not None:
240 sbj.add_identifier(sbj.P_ISSN, pissn)
241 eissn = abj.get_one_identifier(abj.E_ISSN)
242 if eissn is not None:
243 sbj.add_identifier(sbj.E_ISSN, eissn)
245 # Ensure subjects are present (for back-compat with any consumers expecting it)
246 subs = abj.subjects()
247 if subs is not None:
248 for s in subs:
249 sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code"))
251 # Add owner (at time of deletion) information
252 stone.owner = self.get_owner()
254 return stone
256 def _tombstone(self):
257 # make the tombstone
258 stone = self.make_tombstone()
260 # save and return
261 stone.save()
262 return stone
264 def add_history(self, bibjson, date=None):
265 """Deprecated"""
266 bibjson = bibjson.bibjson if isinstance(bibjson, ArticleBibJSON) else bibjson
267 if date is None:
268 date = dates.now_str()
269 snobj = {"date": date, "bibjson": bibjson}
270 if "history" not in self.data:
271 self.data["history"] = []
272 self.data["history"].append(snobj)
274 def is_in_doaj(self):
275 try:
276 return self.data['admin'].get("in_doaj", False)
277 except KeyError:
278 # If we have no admin section, return None instead
279 return None
281 def set_in_doaj(self, value):
282 if "admin" not in self.data:
283 self.data["admin"] = {}
284 self.data["admin"]["in_doaj"] = value
286 def publisher_record_id(self):
287 return self.data.get("admin", {}).get("publisher_record_id")
289 def set_publisher_record_id(self, pri):
290 if "admin" not in self.data:
291 self.data["admin"] = {}
292 self.data["admin"]["publisher_record_id"] = pri
294 def upload_id(self):
295 return self.data.get("admin", {}).get("upload_id")
297 def set_upload_id(self, uid):
298 if "admin" not in self.data:
299 self.data["admin"] = {}
300 self.data["admin"]["upload_id"] = uid
302 def get_normalised_doi(self):
303 if self.data.get("index", {}).get("doi") is not None:
304 return self.data["index"]["doi"]
305 doi = self.bibjson().get_one_identifier(constants.IDENT_TYPE_DOI)
306 if doi is None:
307 return None
308 try:
309 return normalise.normalise_doi(doi)
310 except ValueError:
311 # can't be normalised, so we just return the doi as-is
312 return doi
314 def get_normalised_fulltext(self):
315 if self.data.get("index", {}).get("fulltext") is not None:
316 return self.data["index"]["fulltext"]
317 fulltexts = self.bibjson().get_urls(constants.LINK_TYPE_FULLTEXT)
318 if len(fulltexts) == 0:
319 return None
320 try:
321 return normalise.normalise_url(fulltexts[0])
322 except ValueError:
323 # can't be normalised, so we just return the url as-is
324 return fulltexts[0]
326 def get_journal(self):
327 """
328 Get this article's associated journal
329 :return: A Journal, or None if this is an orphan article
330 """
331 bibjson = self.bibjson()
333 # first, get the ISSNs associated with the record
334 pissns = bibjson.get_identifiers(bibjson.P_ISSN)
335 eissns = bibjson.get_identifiers(bibjson.E_ISSN)
336 allissns = list(set(pissns + eissns))
338 # find a matching journal record from the index
339 best_match = None
341 for issn in allissns:
342 journals = Journal.find_by_issn(issn)
343 if len(journals) > 0:
344 # Get the best journal match:
345 # 1. Prefer the most recently updated journal that is in DOAJ.
346 # 2. If none are in DOAJ, fall back to the most recently updated journal outside DOAJ.
348 matches = [j for j in journals if j.is_in_doaj()]
350 if len(matches) == 0:
351 matches = journals
353 best_match = max(
354 matches,
355 key=lambda j: j.last_updated,
356 default=None
357 )
359 return best_match
361 def get_associated_journals(self):
362 # find all matching journal record from the index
363 allissns = self.bibjson().issns()
364 return Journal.find_by_issn(allissns)
366 def add_journal_metadata(self, j=None, reg=None):
367 """
368 this function makes sure the article is populated
369 with all the relevant info from its owning parent object
370 :param j: Pass in a Journal to bypass the (slow) locating step. MAKE SURE IT'S THE RIGHT ONE!
371 """
373 # Record the data that is copied into the article into the "reg"ister, in case the
374 # caller needs to know exactly and only which information was copied
375 if reg is None:
376 reg = Journal()
377 rbj = reg.bibjson()
379 if j is None:
380 journal = self.get_journal()
381 else:
382 journal = j
384 # we were unable to find a journal
385 if journal is None:
386 raise NoJournalException("Unable to find a journal associated with this article")
388 # if we get to here, we have a journal record we want to pull data from
389 jbib = journal.bibjson()
390 bibjson = self.bibjson()
392 # tripwire to be tripped if the journal makes changes to the article
393 trip = False
395 if bibjson.subjects() != jbib.subjects():
396 trip = True
397 bibjson.set_subjects(jbib.subjects())
398 rbj.set_subjects(jbib.subjects())
400 if jbib.title is not None:
401 if bibjson.journal_title != jbib.title:
402 trip = True
403 bibjson.journal_title = jbib.title
404 rbj.title = jbib.title
406 if len(jbib.language) > 0:
407 jlang = jbib.language
408 alang = bibjson.journal_language
409 jlang.sort()
410 alang.sort()
411 if jlang != alang:
412 bibjson.journal_language = jbib.language
413 trip = True
414 rbj.set_language(jbib.language)
416 if jbib.country is not None:
417 if jbib.country != bibjson.journal_country:
418 bibjson.journal_country = jbib.country
419 trip = True
420 rbj.country = jbib.country
422 if jbib.publisher:
423 if jbib.publisher != bibjson.publisher:
424 bibjson.publisher = jbib.publisher
425 trip = True
426 rbj.publisher = jbib.publisher
428 # Copy the in_doaj status and the journal's ISSNs
429 if journal.is_in_doaj() != self.is_in_doaj():
430 self.set_in_doaj(journal.is_in_doaj())
431 trip = True
432 reg.set_in_doaj(journal.is_in_doaj())
434 try:
435 aissns = bibjson.journal_issns
436 jissns = jbib.issns()
437 aissns.sort()
438 jissns.sort()
439 if aissns != jissns:
440 bibjson.journal_issns = jbib.issns()
441 trip = True
443 eissns = jbib.get_identifiers(jbib.E_ISSN)
444 pissns = jbib.get_identifiers(jbib.P_ISSN)
445 if eissns is not None and len(eissns) > 0:
446 rbj.add_identifier(rbj.E_ISSN, eissns[0])
447 if pissns is not None and len(pissns) > 0:
448 rbj.add_identifier(rbj.P_ISSN, pissns[0])
449 except KeyError:
450 # No issns, don't worry about it for now
451 pass
453 return trip
455 def merge(self, old, take_id=True):
456 # this takes an old version of the article and brings
457 # forward any useful information that is needed. The rules of merge are:
458 # - ignore "index" (it gets regenerated on save)
459 # - always take the "created_date"
460 # - any top level field that does not exist in the current item (esp "id" and "history")
461 # - in "admin", copy any field that does not already exist
463 # first thing to do is create a snapshot of the old record
464 old.snapshot()
466 # now go on and do the merge
468 # always take the created date
469 self.set_created(old.created_date)
471 # take the id
472 if self.id is None or take_id:
473 self.set_id(old.id)
475 # take the history (deprecated)
476 if len(self.data.get("history", [])) == 0:
477 self.data["history"] = deepcopy(old.data.get("history", []))
479 # take the bibjson
480 if "bibjson" not in self.data:
481 self.set_bibjson(deepcopy(old.bibjson()))
483 # take the admin if there isn't one
484 if "admin" not in self.data:
485 self.data["admin"] = deepcopy(old.data.get("admin", {}))
486 else:
487 # otherwise, copy any admin keys that don't exist on the current item
488 oa = old.data.get("admin", {})
489 for key in oa:
490 if key not in self.data["admin"]:
491 self.data["admin"][key] = deepcopy(oa[key])
493 def _generate_index(self):
494 # the index fields we are going to generate
495 issns = []
496 subjects = []
497 schema_subjects = []
498 schema_codes = []
499 schema_codes_tree = []
500 classification = []
501 langs = []
502 country = None
503 publisher = []
504 classification_paths = []
505 unpunctitle = None
506 asciiunpunctitle = None
507 doi = None
508 fulltext = None
510 # the places we're going to get those fields from
511 cbib = self.bibjson()
512 jindex = self.data.get('index', {})
513 hist = self.history()
515 # get the issns out of the current bibjson
516 issns += cbib.get_identifiers(cbib.P_ISSN)
517 issns += cbib.get_identifiers(cbib.E_ISSN)
519 # get the issn from the journal bibjson
520 if isinstance(cbib.journal_issns, list):
521 issns += cbib.journal_issns
523 # de-duplicate the issns
524 issns = list(set(issns))
526 # now get the issns out of the historic records
527 for date, hbib in hist:
528 issns += hbib.get_identifiers(hbib.P_ISSN)
529 issns += hbib.get_identifiers(hbib.E_ISSN)
531 # get the subjects and concatenate them with their schemes from the current bibjson
532 for subs in cbib.subjects():
533 scheme = subs.get("scheme")
534 term = subs.get("term")
535 subjects.append(term)
536 schema_subjects.append(scheme + ":" + term)
537 classification.append(term)
538 if "code" in subs:
539 schema_codes.append(scheme + ":" + subs.get("code"))
541 # copy the languages
542 if len(cbib.journal_language) > 0:
543 langs = [datasets.name_for_lang(l) for l in cbib.journal_language]
545 # Get the country name from the bibjson country code
546 if cbib.journal_country:
547 country = datasets.get_country_name(cbib.journal_country)
549 # copy the publisher/provider
550 if cbib.publisher:
551 publisher.append(cbib.publisher)
553 # deduplicate the lists
554 issns = list(set(issns))
555 subjects = list(set(subjects))
556 schema_subjects = list(set(schema_subjects))
557 classification = list(set(classification))
558 publisher = list(set(publisher))
559 langs = list(set(langs))
560 schema_codes = list(set(schema_codes))
562 # work out what the date of publication is
563 date = cbib.get_publication_date()
565 # calculate the classification paths
566 from portality.lcc import lcc # inline import since this hits the database
567 for subs in cbib.subjects():
568 scheme = subs.get("scheme")
569 term = subs.get("term")
570 if scheme == "LCC":
571 path = lcc.pathify(term)
572 if path is not None:
573 classification_paths.append(path)
575 # normalise the classification paths, so we only store the longest ones
576 classification_paths = lcc.longest(classification_paths)
577 schema_codes_tree = cbib.lcc_codes_full_list()
579 # create an unpunctitle
580 if cbib.title is not None:
581 throwlist = string.punctuation + '\n\t'
582 unpunctitle = "".join(c for c in cbib.title if c not in throwlist).strip()
583 try:
584 asciiunpunctitle = unidecode(unpunctitle)
585 except:
586 asciiunpunctitle = unpunctitle
588 # create a normalised version of the DOI for deduplication
589 source_doi = cbib.get_one_identifier(constants.IDENT_TYPE_DOI)
590 try:
591 doi = normalise.normalise_doi(source_doi)
592 except ValueError as e:
593 # if we can't normalise the DOI, just store it cast to lower case.
594 doi = source_doi.lower()
596 # create a normalised version of the fulltext URL for deduplication
597 fulltexts = cbib.get_urls(constants.LINK_TYPE_FULLTEXT)
598 if len(fulltexts) > 0:
599 source_fulltext = fulltexts[0]
600 try:
601 fulltext = normalise.normalise_url(source_fulltext)
602 except ValueError as e:
603 # if we can't normalise the fulltext store it as-is
604 fulltext = source_fulltext
606 # build the index part of the object
607 self.data["index"] = {}
608 if len(issns) > 0:
609 self.data["index"]["issn"] = issns
610 if date != "":
611 self.data["index"]["date"] = date
612 self.data["index"]["date_toc_fv_month"] = date # Duplicated so we can have year/month facets in fv2
613 if len(subjects) > 0:
614 self.data["index"]["subject"] = subjects
615 if len(schema_subjects) > 0:
616 self.data["index"]["schema_subject"] = schema_subjects
617 if len(classification) > 0:
618 self.data["index"]["classification"] = classification
619 if len(publisher) > 0:
620 self.data["index"]["publisher"] = publisher
621 if len(langs) > 0:
622 self.data["index"]["language"] = langs
623 if country is not None:
624 self.data["index"]["country"] = country
625 if len(schema_codes) > 0:
626 self.data["index"]["schema_code"] = schema_codes
627 if len(classification_paths) > 0:
628 self.data["index"]["classification_paths"] = classification_paths
629 if unpunctitle is not None:
630 self.data["index"]["unpunctitle"] = unpunctitle
631 if asciiunpunctitle is not None:
632 self.data["index"]["asciiunpunctitle"] = unpunctitle
633 if doi is not None:
634 self.data["index"]["doi"] = doi
635 if fulltext is not None:
636 self.data["index"]["fulltext"] = fulltext
637 if len(schema_codes_tree) > 0:
638 self.data["index"]["schema_codes_tree"] = schema_codes_tree
640 def prep(self):
641 self._generate_index()
642 self.data['last_updated'] = dates.now_str()
644 def save(self, *args, **kwargs):
645 self._generate_index()
646 return super(Article, self).save(*args, **kwargs)
648 def get_owner(self):
649 b = self.bibjson()
650 article_issns = b.get_identifiers(b.P_ISSN)
651 article_issns += b.get_identifiers(b.E_ISSN)
652 owners = []
654 seen_journal_issns = {}
655 for issn in article_issns:
656 journals = Journal.find_by_issn(issn)
657 if journals is not None and len(journals) > 0:
658 for j in journals:
659 owners.append(j.owner)
660 if j.owner not in seen_journal_issns:
661 seen_journal_issns[j.owner] = []
662 seen_journal_issns[j.owner] += j.bibjson().issns()
664 # deduplicate the list of owners
665 owners = list(set(owners))
667 # no owner means we can't confirm
668 if len(owners) == 0:
669 raise NoValidOwnerException
671 # multiple owners means ownership of this article is confused
672 if len(owners) > 1:
673 return NoValidOwnerException
675 return owners[0]
678class ArticleTombstone(Article):
679 __type__ = "article_tombstone"
681 def snapshot(self):
682 return None
684 def is_in_doaj(self):
685 return False
687 @property
688 def owner(self):
689 return self.data.get("admin", {}).get("owner")
691 @owner.setter
692 def owner(self, owner):
693 if "admin" not in self.data:
694 self.data["admin"] = {}
695 self.data["admin"]["owner"] = owner
697 def prep(self):
698 self.data['last_updated'] = dates.now_str()
700 def save(self, *args, **kwargs):
701 return super(ArticleTombstone, self).save(*args, **kwargs)
704class ArticleBibJSON(GenericBibJSON):
706 def __init__(self, bibjson=None, **kwargs):
707 self._add_struct(shared_structs.SHARED_BIBJSON.get("structs", {}).get("bibjson"))
708 self._add_struct(ARTICLE_BIBJSON_EXTENSION.get("structs", {}).get("bibjson"))
709 super(ArticleBibJSON, self).__init__(bibjson, **kwargs)
711 # article-specific simple getters and setters
712 @property
713 def year(self):
714 return self._get_single("year")
716 @year.setter
717 def year(self, val):
718 self._set_with_struct("year", val)
720 @year.deleter
721 def year(self):
722 self._delete("year")
724 @property
725 def month(self):
726 return self._get_single("month")
728 @month.setter
729 def month(self, val):
730 self._set_with_struct("month", val)
732 @month.deleter
733 def month(self):
734 self._delete("month")
736 @property
737 def start_page(self):
738 return self._get_single("start_page")
740 @start_page.setter
741 def start_page(self, val):
742 self._set_with_struct("start_page", val)
744 @property
745 def end_page(self):
746 return self._get_single("end_page")
748 @end_page.setter
749 def end_page(self, val):
750 self._set_with_struct("end_page", val)
752 @property
753 def abstract(self):
754 return self._get_single("abstract")
756 @abstract.setter
757 def abstract(self, val):
758 self._set_with_struct("abstract", val)
760 # article-specific complex part getters and setters
762 @property
763 def volume(self):
764 return self._get_single("journal.volume")
766 @volume.setter
767 def volume(self, value):
768 self._set_with_struct("journal.volume", value)
770 @property
771 def number(self):
772 return self._get_single("journal.number")
774 @number.setter
775 def number(self, value):
776 self._set_with_struct("journal.number", value)
778 @property
779 def journal_title(self):
780 return self._get_single("journal.title")
782 @journal_title.setter
783 def journal_title(self, title):
784 self._set_with_struct("journal.title", title)
786 @property
787 def journal_language(self):
788 return self._get_list("journal.language")
790 @journal_language.setter
791 def journal_language(self, lang):
792 self._set_with_struct("journal.language", lang)
794 @property
795 def journal_country(self):
796 return self._get_single("journal.country")
798 @journal_country.setter
799 def journal_country(self, country):
800 self._set_single("journal.country", country)
802 @property
803 def journal_issns(self):
804 return self._get_list("journal.issns")
806 @journal_issns.setter
807 def journal_issns(self, issns):
808 self._set_with_struct("journal.issns", issns)
810 @property
811 def publisher(self):
812 return self._get_single("journal.publisher")
814 @publisher.setter
815 def publisher(self, value):
816 self._set_with_struct("journal.publisher", value)
818 def add_author(self, name, affiliation=None, orcid_id=None):
819 aobj = {"name": name}
820 if affiliation is not None:
821 aobj["affiliation"] = affiliation
822 if orcid_id is not None:
823 aobj["orcid_id"] = orcid_id
824 self._add_to_list_with_struct("author", aobj)
826 @property
827 def author(self):
828 return self._get_list("author")
830 @author.setter
831 def author(self, authors):
832 self._set_with_struct("author", authors)
834 def get_publication_date(self, date_format=FMT_DATETIME_STD):
835 # work out what the date of publication is
836 date = ""
837 if self.year is not None:
838 if type(self.year) is str: # It should be, if the mappings are correct. but len() needs a sequence.
839 # fix 2 digit years
840 if len(self.year) == 2:
841 try:
842 intyear = int(self.year)
843 except ValueError:
844 # if it's 2 chars long and the 2 chars don't make an integer,
845 # forget it
846 return date
848 # In the case of truncated years, assume it's this century if before the current year
849 if intyear <= int(str(dates.now().year)[:-2]):
850 self.year = "20" + self.year # For readability over long-lasting code, I have refrained
851 else: # from using str(dates.now().year)[:2] here.
852 self.year = "19" + self.year # But don't come crying to me 90-ish years from now.
854 # if we still don't have a 4 digit year, forget it
855 if len(self.year) != 4:
856 return date
858 # build up our proposed datestamp
859 date += str(self.year)
860 if self.month is not None:
861 try:
862 if type(self.month) is int:
863 if 1 <= int(self.month) <= 12:
864 month_number = self.month
865 else:
866 month_number = 1
867 elif len(self.month) <= 2:
868 if 1 <= int(self.month) <= 12:
869 month_number = self.month
870 else:
871 month_number = '1'
872 elif len(self.month) == 3: # 'May' works with either case, obvz.
873 month_number = datetime.strptime(self.month, '%b').month
874 else:
875 month_number = datetime.strptime(self.month, '%B').month
878 # pad the month number to two digits. This accepts int or string
879 date += '-{:0>2}'.format(month_number)
880 except:
881 # If something goes wrong, just assume it's January
882 date += "-01"
883 else:
884 date += "-01"
885 date += "-01T00:00:00Z"
887 # attempt to confirm the format of our datestamp
888 try:
889 datecheck = dates.parse(date)
890 date = datecheck.strftime(date_format)
891 except:
892 return ""
893 return date
895 def remove_journal_metadata(self):
896 self._delete("journal")
898 def vancouver_citation(self):
899 jtitle = self.journal_title
900 year = self.year
901 vol = self.volume
902 iss = self.number
903 start = self.start_page
904 end = self.end_page
906 citation = ""
908 if year:
909 citation += year + ";"
911 if vol:
912 citation += vol
914 if iss:
915 citation += "(" + iss + ")"
917 if start or end:
918 if citation != "":
919 citation += ":"
920 if start:
921 citation += start
922 if end:
923 if start:
924 citation += "-"
925 citation += end
927 return jtitle.strip(), citation
929 def lcc_codes_full_list(self):
930 full_list = set()
932 from portality.lcc import lcc # inline import since this hits the database
933 for subs in self.subjects():
934 scheme = subs.get("scheme")
935 if scheme != "LCC":
936 continue
937 code = subs.get("code")
938 expanded = lcc.expand_codes(code)
939 full_list.update(expanded)
941 return ["LCC:" + x for x in full_list if x is not None]
944##################################################
946class ArticleQuery(object):
947 base_query = {
948 "track_total_hits" : True,
949 "query" : {
950 "bool" : {
951 "must" : []
952 }
953 }
954 }
956 _issn_terms = { "terms" : {"index.issn.exact" : ["<list of issns here>"]} }
957 _volume_term = { "term" : {"bibjson.journal.volume.exact" : "<volume here>"} }
959 def __init__(self, issns=None, volume=None, in_doaj=None):
960 self.issns = issns
961 self.volume = volume
962 self.in_doaj = in_doaj
964 def query(self):
965 q = deepcopy(self.base_query)
967 if self.issns is not None:
968 iq = deepcopy(self._issn_terms)
969 iq["terms"]["index.issn.exact"] = self.issns
970 q["query"]["bool"]["must"].append(iq)
972 if self.volume is not None:
973 vq = deepcopy(self._volume_term)
974 vq["term"]["bibjson.journal.volume.exact"] = self.volume
975 q["query"]["bool"]["must"].append(vq)
977 if self.in_doaj is not None:
978 q["query"]["bool"]["must"].append({"term": {"admin.in_doaj": self.in_doaj}})
980 return q
982class ArticleVolumesQuery(object):
983 base_query = {
984 "track_total_hits": True,
985 "query" : {
986 "bool": {
987 "filter": {
988 "terms" : {"index.issn.exact" : ["<list of issns here>"]}
989 }
990 }
991 },
992 "size" : 0,
993 "aggs" : {
994 "vols" : {
995 "terms" : {
996 "field" : "bibjson.journal.volume.exact",
997 "order": {"_key" : "desc"},
998 "size" : 1000
999 }
1000 }
1001 }
1002 }
1004 def __init__(self, issns=None):
1005 self.issns = issns
1007 def query(self):
1008 q = deepcopy(self.base_query)
1009 q["query"]["bool"]["filter"]["terms"]["index.issn.exact"] = self.issns
1010 return q
1013class ArticleVolumesIssuesQuery(object):
1014 base_query = {
1015 "track_total_hits": True,
1016 "query" : {
1017 "bool": {
1018 "filter": {
1019 "bool": {
1020 "must": [
1021 {"terms" : {"index.issn.exact" : ["<list of issns here>"]}},
1022 {"term" : {"bibjson.journal.volume.exact" : "<volume here>"}}
1023 ]
1024 }
1025 }
1026 }
1027 },
1028 "size" : 0,
1029 "aggs" : {
1030 "issues" : {
1031 "terms" : {
1032 "field" : "bibjson.journal.number.exact",
1033 "order": {"_key", "desc"},
1034 "size" : 1000
1035 }
1036 }
1037 }
1038 }
1040 def __init__(self, issns=None, volume=None):
1041 self.issns = issns
1042 self.volume = volume
1044 def query(self):
1045 q = deepcopy(self.base_query)
1046 q["query"]["bool"]["filter"]["bool"]["must"][0]["terms"]["index.issn.exact"] = self.issns
1047 q["query"]["bool"]["filter"]["bool"]["must"][1]["term"]["bibjson.journal.volume.exact"] = self.volume
1048 return q
1051class DuplicateArticleQuery(object):
1052 base_query = {
1053 "track_total_hits" : True,
1054 "query": {
1055 "bool": {
1056 "must": []
1057 }
1058 },
1059 "sort": [{"last_updated": {"order": "desc"}}]
1060 }
1062 _should = {
1063 "should" : [],
1064 "minimum_should_match" : 2
1065 }
1067 _volume_term = {"term" : {"bibjson.journal.volume.exact" : "<volume>"}}
1068 _number_term = {"term" : {"bibjson.journal.number.exact" : "<issue number>"}}
1069 _start_term = {"term" : {"bibjson.start_page.exact" : "<start page>"}}
1070 _issn_terms = {"terms" : { "index.issn.exact" : ["<list of issns>"] }}
1071 _pubrec_term = {"term" : {"admin.publisher_record_id.exact" : "<publisher record id>"}}
1072 _identifier_term = {"term" : {"bibjson.identifier.id.exact" : "<issn here>"}}
1073 _doi_term = {"term" : {"index.doi.exact" : "<doi here>"}}
1074 _fulltext_terms = {"terms" : {"index.fulltext.exact" : ["<fulltext here>"]}}
1075 _fuzzy_title = {"fuzzy" : {"bibjson.title.exact" : "<title here>"}}
1077 def __init__(self, issns=None, publisher_record_id=None, doi=None, urls=None, title=None, volume=None, number=None, start=None, should_match=None, size=10):
1078 self.issns = issns if isinstance(issns, list) else []
1079 self.publisher_record_id = publisher_record_id
1080 self.doi = doi
1081 self.urls = urls if isinstance(urls, list) else [urls] if isinstance(urls, str) or isinstance(urls, str) else []
1082 self.title = title
1083 self.volume = volume
1084 self.number = number
1085 self.start = start
1086 self.should_match = should_match
1087 self.size = size
1089 def query(self):
1090 # - MUST be from at least one of the ISSNs
1091 # - MUST have the publisher record id
1092 # - MUST have the doi unless should_match is set
1093 # - MUST have the one of the fulltext urls unless should_match is set
1094 # - MUST fuzzy match the title
1095 # - SHOULD have <should_match> of: volume, issue, start page, fulltext url, doi
1097 q = deepcopy(self.base_query)
1098 if len(self.issns) > 0:
1099 it = deepcopy(self._issn_terms)
1100 it["terms"]["index.issn.exact"] = self.issns
1101 q["query"]["bool"]["must"].append(it)
1103 if self.publisher_record_id is not None:
1104 pr = deepcopy(self._pubrec_term)
1105 pr["term"]["admin.publisher_record_id.exact"] = self.publisher_record_id
1106 q["query"]["bool"]["must"].append(pr)
1108 if self.doi is not None and self.should_match is None:
1109 idt = deepcopy(self._doi_term)
1110 # idt["term"]["bibjson.identifier.id.exact"] = self.doi
1111 idt["term"]["index.doi.exact"] = self.doi
1112 q["query"]["bool"]["must"].append(idt)
1114 if len(self.urls) > 0 and self.should_match is None:
1115 uq = deepcopy(self._fulltext_terms)
1116 # uq["terms"]["bibjson.link.url.exact"] = self.urls
1117 uq["terms"]["index.fulltext.exact"] = self.urls
1118 q["query"]["bool"]["must"].append(uq)
1120 if self.title is not None:
1121 ft = deepcopy(self._fuzzy_title)
1122 ft["fuzzy"]["bibjson.title.exact"] = self.title
1123 q["query"]["bool"]["must"].append(ft)
1125 if self.should_match is not None:
1126 term_count = 0
1127 s = deepcopy(self._should)
1129 if self.volume is not None:
1130 term_count += 1
1131 vt = deepcopy(self._volume_term)
1132 vt["term"]["bibjson.journal.volume.exact"] = self.volume
1133 s["should"].append(vt)
1135 if self.number is not None:
1136 term_count += 1
1137 nt = deepcopy(self._number_term)
1138 nt["term"]["bibjson.journal.number.exact"] = self.number
1139 s["should"].append(nt)
1141 if self.start is not None:
1142 term_count += 1
1143 st = deepcopy(self._start_term)
1144 st["term"]["bibjson.start_page.exact"] = self.start
1145 s["should"].append(st)
1147 if len(self.urls) > 0:
1148 term_count += 1
1149 uq = deepcopy(self._url_terms)
1150 uq["terms"]["bibjson.link.url.exact"] = self.urls
1151 s["should"].append(uq)
1153 if self.doi is not None:
1154 term_count += 1
1155 idt = deepcopy(self._identifier_term)
1156 idt["term"]["bibjson.identifier.id.exact"] = self.doi
1157 s["should"].append(idt)
1159 msm = self.should_match
1160 if msm > term_count:
1161 msm = term_count
1162 s["minimum_should_match"] = msm
1164 q["query"]["bool"].update(s)
1166 # Allow more results than the default
1167 q["size"] = self.size
1169 return q
1172def _human_sort(things, reverse=True):
1173 numeric = []
1174 non_numeric = []
1175 nmap = {}
1176 for v in things:
1177 try:
1178 # try to convert n to an int
1179 vint = int(v)
1181 # remember the original string (it may have leading 0s)
1182 try:
1183 nmap[vint].append(v)
1184 except KeyError:
1185 nmap[vint] = [v]
1186 numeric.append(vint)
1187 except:
1188 non_numeric.append(v)
1190 numeric.sort(reverse=reverse)
1191 non_numeric.sort(reverse=reverse)
1193 # convert the integers back to their string representation
1194 return reduce(lambda x, y: x+y, [nmap[n] for n in numeric], []) + non_numeric
1197def _sort_articles(articles):
1198 # first extract the array we want to sort on
1199 # and make a map of that value to the issue itself
1200 unsorted = []
1201 numbers = []
1202 imap = {}
1203 for art in articles:
1204 sp = art.get("bibjson.start_page", [None])[0]
1206 # can't sort anything that doesn't have a start page
1207 if sp is None:
1208 unsorted.append(art)
1209 continue
1211 # deal with start page clashes and record the start pages
1212 # to sort by
1213 if sp not in numbers:
1214 numbers.append(sp)
1215 if sp in imap:
1216 imap[sp].append(art)
1217 else:
1218 imap[sp] = [art]
1220 sorted_keys = _human_sort(numbers, reverse=False)
1222 s = []
1223 for n in sorted_keys:
1224 s += [x for x in imap[n]]
1225 s += [x for x in unsorted]
1227 return s
1230class ArticleTombstoneRecentlyDeletedQuery(object):
1231 def __init__(self, since=None):
1232 self.since = since
1234 def query(self):
1235 q = {
1236 "query": {
1237 "range": {
1238 "created_date": {"gte": dates.format(self.since)}
1239 }
1240 }
1241 }
1242 return q