Coverage for portality/models/openurl.py: 89%
326 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1import re, json
2from flask import url_for
3from portality.models import Journal, Article
4from portality.core import app
5from copy import deepcopy
6from portality.lib import dates
8JOURNAL_SCHEMA_KEYS = ['doi', 'aulast', 'aufirst', 'auinit', 'auinit1', 'auinitm', 'ausuffix', 'au', 'aucorp', 'atitle',
9 'jtitle', 'stitle', 'date', 'chron', 'ssn', 'quarter', 'volume', 'part', 'issue', 'spage',
10 'epage', 'pages', 'artnum', 'issn', 'eissn', 'isbn', 'coden', 'sici', 'genre']
12# The genres from the OpenURL schema we support
13SUPPORTED_GENRES = ['journal', 'article']
15# Mapping from OpenURL schema to both supported models (Journal, Article)
16OPENURL_TO_ES = {
17 'aulast': (None, 'bibjson.author.name.exact'),
18 'aucorp': (None, 'bibjson.author.affiliation.exact'),
19 'atitle': (None, 'bibjson.title.exact'),
20 'jtitle': ('index.title.exact', 'bibjson.journal.title.exact'), # Note we use index.title.exact for journals, to support continuations
21 'stitle': ('bibjson.alternative_title.exact', None),
22 'date': (None, 'bibjson.year.exact'),
23 'volume': (None, 'bibjson.journal.volume.exact'),
24 'issue': (None, 'bibjson.journal.number.exact'),
25 'spage': (None, 'bibjson.start_page.exact'),
26 'epage': (None, 'bibjson.end_page.exact'),
27 'issn': ('index.issn.exact', 'index.issn.exact'), # bibjson.identifier.id.exact
28 'eissn': ('index.issn.exact', 'index.issn.exact'),
29 'isbn': ('index.issn.exact', 'index.issn.exact'),
30 'doi': (None, 'index.doi.exact')
31}
33# Terms search template. Ensure all queries from OpenURL return publicly visible results with in_doaj : true
34IN_DOAJ_TERM = {"term": {"admin.in_doaj": True}}
35TERMS_SEARCH = {"query": {"bool": {"must": [IN_DOAJ_TERM]}}}
38def reformat_au_as_aulast(au_value):
39 """
40 Best-effort reformat of an OpenURL ``au`` (free-text full author name) into the
41 "Given Surname" order DOAJ's own indexing uses (see
42 ``portality.crosswalks.article_crossref_xml.extract_authors``), so it can be
43 searched via the same ES field as ``aulast``.
45 Real-world requests we've seen in production use ``au`` non-standardly to hold
46 *multiple* authors in one value, semicolon-separated, each usually in
47 "Surname, Given" order (e.g. ``"Barich, Howard;Kotler, Philip"``) - the
48 opposite order to how we index names. Only the first author is used if
49 several are supplied: exact-match search on a full name is already fragile
50 (a missing middle initial, a diacritic, or different formatting from the
51 source citation will still silently fail to match), and OR-ing several
52 full-name guesses together would only make that harder to reason about, not
53 more accurate.
55 :param au_value: the raw au string, e.g. "Barich, Howard;Kotler, Philip"
56 :return: a best-effort "Given Surname" string, e.g. "Howard Barich", or the
57 first author's value unchanged if it doesn't look like "Surname, Given"
58 (no comma present)
59 """
60 first_author = au_value.split(";")[0].strip()
61 if "," in first_author:
62 surname, given = first_author.split(",", 1)
63 return (given.strip() + " " + surname.strip()).strip()
64 return first_author
67class UnsupportedOpenURLQuery(Exception):
68 """
69 Raised when an OpenURL request cannot be turned into a usable search query -
70 e.g. every field supplied is one the OpenURL schema allows but that DOAJ does
71 not support searching on (such as ``rft.chron``), or the genre supplied isn't
72 one DOAJ recognises. Callers (the view) should treat this as a bad request
73 (400), as distinct from a well-formed query that simply found no results.
74 """
75 pass
78class OpenURLRequest(object):
79 """
80 Based on the fields from ofi/fmt:kev:mtx:journal schema for Journals in OpenURL 1.0
81 This is the only schema the DOAJ supports.
82 """
84 # ~~API:Feature~~
86 def __init__(self, **kwargs):
88 # Initialise the OpenURLRequest object with empty attributes
89 for key in JOURNAL_SCHEMA_KEYS:
90 setattr(self, key, None)
92 # set by query_es(): the supported genre searched as (lowercased, or None if unsupported), and the fields it
93 # had to skip
94 self._search_genre = None
95 self._unsupported_fields = []
97 # Save any attributes specified at creation time
98 if kwargs:
99 for key, value in kwargs.items():
100 setattr(self, key, value)
102 @staticmethod
103 def _total_hits(results):
104 return results.get('hits', {}).get('total', {}).get('value', 0)
106 def __str__(self):
107 return "OpenURLRequest{" + ", ".join(["%s : %s" % (x, getattr(self, x)) for x in JOURNAL_SCHEMA_KEYS if getattr(self, x)]) + "}"
109 def query_es(self):
110 """
111 Query Elasticsearch for a set of matches for this request.
113 Fields the OpenURL schema allows but that DOAJ has no ES mapping for
114 (e.g. ``au``, ``chron``, ``ssn`` ...), and a genre DOAJ doesn't recognise,
115 are logged and skipped rather than raising - unless skipping them leaves
116 no usable search term at all, in which case :class:`UnsupportedOpenURLQuery`
117 is raised so the caller can respond with a 400 rather than treating it as
118 a legitimate (if empty) search.
120 :return: The results of a query through the dao, a JSON object, or None
121 if no attributes were supplied on the request at all.
122 :raises UnsupportedOpenURLQuery: if the request had content, but none of
123 it could be mapped onto a search DOAJ supports.
124 """
125 # Copy to the template, which will be populated with terms
126 populated_query = deepcopy(TERMS_SEARCH)
128 # Get all of the attributes with values set. Guard against a schema key that
129 # somehow doesn't have a matching property (shouldn't happen given __init__,
130 # but defends against future schema drift rather than crashing on it).
131 set_attributes = []
132 for x in JOURNAL_SCHEMA_KEYS[:-1]:
133 try:
134 val = getattr(self, x)
135 except AttributeError as e:
136 app.logger.warning("OpenURL schema key '{x}' has no matching property, skipping: {e}".format(x=x, e=e))
137 continue
138 if val:
139 set_attributes.append((x, val))
141 # If we don't have a genre, guess journal FIXME: is it correct to assume journal?
142 if not self.genre:
143 self.genre = SUPPORTED_GENRES[0] # TODO: we may want to handle 404 instead
145 # Set i to use either our mapping for journals or articles. An unrecognised
146 # genre means we can't tell which half of OPENURL_TO_ES to use at all, so
147 # none of the supplied fields (even otherwise-valid ones) can be mapped.
148 unsupported_fields = []
149 try:
150 i = SUPPORTED_GENRES.index(str(self.genre).lower())
151 except (ValueError, AttributeError) as e:
152 app.logger.warning("OpenURL request supplied an unsupported genre '{x}': {e}".format(x=self.genre, e=e))
153 unsupported_fields.append(("genre", self.genre))
154 i = None
156 # Add the attributes to the query, skipping (and logging) any field that
157 # the OpenURL schema allows but DOAJ can't map to a search term, instead
158 # of letting one bad field crash the whole request.
159 # NOTE: populated_query's "must" list always starts with IN_DOAJ_TERM, so
160 # we can't tell "no real search terms added" just from its length - track
161 # that separately.
162 terms_added = 0
163 if i is not None:
164 # so 'au' can defer to an explicit 'aulast' if both are given, rather
165 # than adding two, possibly conflicting, author terms
166 attr_keys = {x for (x, _) in set_attributes}
168 for (k, v) in set_attributes:
169 lookup_key, term_value = k, v
171 # 'au' (free-text full author name) isn't in OPENURL_TO_ES - DOAJ
172 # only supports author search via 'aulast' (surname). Real-world
173 # requests use 'au' far more often than 'aulast', so rather than
174 # just dropping it, make a best-effort translation onto the same
175 # search 'aulast' already uses (see reformat_au_as_aulast).
176 if k == "au":
177 if "aulast" in attr_keys:
178 app.logger.debug("OpenURL 'au' field ignored in favour of explicit 'aulast'")
179 continue
180 lookup_key = "aulast"
181 term_value = reformat_au_as_aulast(v)
183 try:
184 es_term = OPENURL_TO_ES[lookup_key][i]
185 except (KeyError, IndexError, TypeError) as e:
186 app.logger.warning("OpenURL field '{x}' is not supported by DOAJ, skipping: {e}".format(x=k, e=e))
187 unsupported_fields.append((k, v))
188 continue
190 # known field, but not searchable for this genre (e.g. aulast for journals)
191 if es_term is None:
192 app.logger.warning("OpenURL field '{x}' is not searchable for genre '{g}', skipping".format(x=k, g=self.genre))
193 unsupported_fields.append((k, v))
194 continue
196 term = {"term": {es_term: term_value}}
197 populated_query["query"]["bool"]["must"].append(term)
198 terms_added += 1
200 # get_result_url() needs these to decide whether a match is trustworthy
201 self._search_genre = SUPPORTED_GENRES[i] if i is not None else None
202 self._unsupported_fields = unsupported_fields
204 # avoid doing an empty (unconstrained beyond in_doaj) query
205 if terms_added == 0:
206 if unsupported_fields:
207 # the request had content, but none of it could be turned into a
208 # usable search - this is a bad request, not just "no results"
209 msg = "OpenURL request contained no fields DOAJ can search on (unsupported: {x})".format(
210 x=", ".join(f for f, _ in unsupported_fields))
211 app.logger.warning(msg)
212 raise UnsupportedOpenURLQuery(msg)
214 app.logger.debug("No valid search terms in OpenURL object")
215 return None
217 try:
218 query_json = json.dumps(populated_query)
219 except TypeError as e:
220 query_json = "<unable to serialise query for logging: {e}>".format(e=e)
222 # Return the results of the query
223 if i == 0:
224 app.logger.debug("OpenURL query to journal: " + query_json)
225 return Journal.query(q=populated_query)
226 elif i == 1:
227 app.logger.debug("OpenURL query to article: " + query_json)
228 return Article.query(q=populated_query)
230 def get_result_url(self):
231 """
232 Get the URL for this OpenURLRequest's referent.
233 :return: The url as a string, or None if not found.
234 :raises UnsupportedOpenURLQuery: propagated from query_es(), or if an article
235 request only matched several articles once unsupported fields were skipped.
236 The caller (the view) should turn this into a 400 response.
237 """
238 results = self.query_es()
240 if results is None:
241 return None
243 if self._total_hits(results) == 0:
244 # No results found for query, retry
245 results = self.fallthrough_retry()
246 if results is None or self._total_hits(results) == 0:
247 # This time we've definitely failed
248 return None
250 # Skipping an unsupported field broadens the search. For an article, the skipped field (artnum, pages, part,
251 # chron, aufirst ...) may be the very thing that tells the matches apart, so taking the first of several hits
252 # would send the user to the wrong article with no sign anything was ignored. Only trust a unique match.
253 # (For a journal the skipped fields describe items within it, so they don't change which journal it is.)
254 if self._search_genre == "article" and self._unsupported_fields and self._total_hits(results) > 1:
255 msg = ("OpenURL article request matched {n} articles once unsupported fields were skipped "
256 "(unsupported: {x}), so it can't be resolved to one article").format(
257 n=self._total_hits(results), x=", ".join(f for f, _ in self._unsupported_fields))
258 raise UnsupportedOpenURLQuery(msg)
260 if results.get('hits', {}).get('hits', [{}])[0].get('_source', {}).get('es_type') == 'journal':
262 # construct a journal object around the result
263 journal = Journal(**results['hits']['hits'][0])
265 # the continuation is a first-class journal object, so if we have a journal we have the right continuation
266 # (assuming that the user gave us specific enough information
267 ident = journal.id
269 # construct the toc url using the ident only
270 jtoc_url = url_for("doaj.toc", identifier=ident)
271 return jtoc_url
273 #~~->Article:Page~~
274 elif results.get('hits', {}).get('hits', [{}])[0].get('_source', {}).get('es_type') == 'article':
275 return url_for("doaj.article_page", identifier=results['hits']['hits'][0]['_id'])
277 def query_for_vol(self, journalobj):
279 # The journal object will already be the correct continuation, if the user provided sufficient detail.
280 issns = journalobj.bibjson().issns()
282 # If there's no way to get the wanted issns, give up, else run the query
283 if issns is None:
284 return None
285 else:
286 volume_query = deepcopy(TERMS_SEARCH)
287 volume_query["size"] = 0
289 issn_term = {"terms": {"index.issn.exact": issns}}
290 volume_query["query"]["bool"]["must"].append(issn_term)
292 vol_term = {"term": {"bibjson.journal.volume.exact": self.volume}}
293 volume_query["query"]["bool"]["must"].append(vol_term)
295 # And if there's an issue, query that too. Note, issue does not make sense on its own.
296 if self.issue:
297 iss_term = {"term": {"bibjson.journal.number.exact": self.issue}}
298 volume_query["query"]["bool"]["must"].append(iss_term)
300 try:
301 query_json = json.dumps(volume_query)
302 except TypeError as e:
303 query_json = "<unable to serialise query for logging: {e}>".format(e=e)
304 app.logger.debug("OpenURL subsequent volume query to article: " + query_json)
305 return Article.query(q=volume_query)
307 def fallthrough_retry(self):
308 """
309 Some things to try differently if we get no results on first attempt
310 :return: a new result set, or None
311 """
312 results = None
314 # Search again for the title against alternative_title (may catch translations of titles).
315 # Journals only: stitle has no article mapping, so for an article this would just drop the title and re-run
316 # the remaining terms - a broader search that could redirect to an unrelated article.
317 if self.jtitle and not self.stitle and self._search_genre == "journal":
318 self.stitle = self.jtitle
319 self.jtitle = None
320 results = self.query_es()
322 return results
324 def validate_issn(self, issn_str):
325 """
326 If the ISSN is missing a dash, add it so it matches that in the index.
327 :param issn_str: The ISSN, or if None, this will skip.
328 :return: The ISSN with the dash added
329 """
330 if issn_str:
331 match_dash = re.compile('[-]')
332 if not match_dash.search(issn_str):
333 issn_str = issn_str[:4] + '-' + issn_str[4:]
334 return issn_str
336 @property
337 def doi(self):
338 """Digital Object Identifier"""
339 return self._doi
341 @doi.setter
342 def doi(self, val):
343 self._doi = val
345 @property
346 def aulast(self):
347 """First author's family name, may be more than one word"""
348 return self._aulast
350 @aulast.setter
351 def aulast(self, val):
352 self._aulast = val
354 @property
355 def aufirst(self):
356 """First author's given name or names or initials"""
357 return self._aufirst
359 @aufirst.setter
360 def aufirst(self, val):
361 self._aufirst = val
363 @property
364 def auinit(self):
365 """First author's first and middle initials"""
366 return self._auinit
368 @auinit.setter
369 def auinit(self, val):
370 self._auinit = val
372 @property
373 def auinit1(self):
374 """First author's first initial"""
375 return self._auinit1
377 @auinit1.setter
378 def auinit1(self, val):
379 self._auinit1 = val
381 @property
382 def auinitm(self):
383 """First author's middle initial"""
384 return self._auinitm
386 @auinitm.setter
387 def auinitm(self, val):
388 self._auinitm = val
390 @property
391 def ausuffix(self):
392 """First author's name suffix. e.g. 'Jr.', 'III'"""
393 return self._ausuffix
395 @ausuffix.setter
396 def ausuffix(self, val):
397 self._ausuffix = val
399 @property
400 def au(self):
401 """full name of a single author"""
402 return self._au
404 @au.setter
405 def au(self, val):
406 self._au = val
408 @property
409 def aucorp(self):
410 """Organisation or corporation that is the author or creator of the document"""
411 return self._aucorp
413 @aucorp.setter
414 def aucorp(self, val):
415 self._aucorp = val
417 @property
418 def atitle(self):
419 """Article title"""
420 return self._atitle
422 @atitle.setter
423 def atitle(self, val):
424 self._atitle = val
426 @property
427 def jtitle(self):
428 """Journal title"""
429 return self._jtitle
431 @jtitle.setter
432 def jtitle(self, val):
433 self._jtitle = val
435 @property
436 def stitle(self):
437 """Abbreviated or short journal title"""
438 return self._stitle
440 @stitle.setter
441 def stitle(self, val):
442 self._stitle = val
444 @property
445 def date(self):
446 """Date of publication"""
447 return self._date
449 @date.setter
450 def date(self, val):
451 if val:
452 try:
453 parsed_date = dates.parse(val)
454 val = parsed_date.year
455 except ValueError:
456 val = None
457 self._date = val
459 @property
460 def chron(self):
461 """Non-normalised enumeration / chronology, e.g. '1st quarter'"""
462 return self._chron
464 @chron.setter
465 def chron(self, val):
466 self._chron = val
468 @property
469 def ssn(self):
470 """Season (chronology). spring|summer|fall|autumn|winter"""
471 return self._ssn
473 @ssn.setter
474 def ssn(self, val):
475 self._ssn = val
477 @property
478 def quarter(self):
479 """Quarter (chronology). 1|2|3|4"""
480 return self._quarter
482 @quarter.setter
483 def quarter(self, val):
484 self._quarter = val
486 @property
487 def volume(self):
488 """Volume designation. e.g. '124', or 'VI'"""
489 return self._volume
491 @volume.setter
492 def volume(self, val):
493 self._volume = val
495 @property
496 def part(self):
497 """Subdivision of a volume or highest level division of the journal. e.g. 'B', 'Supplement'"""
498 return self._part
500 @part.setter
501 def part(self, val):
502 self._part = val
504 @property
505 def issue(self):
506 """Journal issue"""
507 return self._issue
509 @issue.setter
510 def issue(self, val):
511 self._issue = val
513 @property
514 def spage(self):
515 """Starting page"""
516 return self._spage
518 @spage.setter
519 def spage(self, val):
520 self._spage = val
522 @property
523 def epage(self):
524 """Ending page"""
525 return self._epage
527 @epage.setter
528 def epage(self, val):
529 self._epage = val
531 @property
532 def pages(self):
533 """Page range e.g. '53-58', 'C4-9'"""
534 return self._pages
536 @pages.setter
537 def pages(self, val):
538 self._pages = val
540 @property
541 def artnum(self):
542 """Article number"""
543 return self._artnum
545 @artnum.setter
546 def artnum(self, val):
547 self._artnum = val
549 @property
550 def issn(self):
551 """Journal ISSN"""
552 return self._issn
554 @issn.setter
555 def issn(self, val):
556 self._issn = self.validate_issn(val)
558 @property
559 def eissn(self):
560 """ISSN for electronic version of the journal"""
561 return self._eissn
563 @eissn.setter
564 def eissn(self, val):
565 self._eissn = self.validate_issn(val)
567 @property
568 def isbn(self):
569 """Journal ISBN"""
570 return self._isbn
572 @isbn.setter
573 def isbn(self, val):
574 self._isbn = val
576 @property
577 def coden(self):
578 """CODEN"""
579 return self._coden
581 @coden.setter
582 def coden(self, val):
583 self._coden = val
585 @property
586 def sici(self):
587 """Serial Item and Contribution Identifier (SICI)"""
588 return self._sici
590 @sici.setter
591 def sici(self, val):
592 self._sici = val
594 @property
595 def genre(self):
596 """journal|issue|article|proceeding|conference|preprint|unknown"""
597 return self._genre
599 @genre.setter
600 def genre(self, val):
601 self._genre = val