Coverage for portality/models/openurl.py: 89%

326 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1import re, json 

2from flask import url_for 

3from portality.models import Journal, Article 

4from portality.core import app 

5from copy import deepcopy 

6from portality.lib import dates 

7 

8JOURNAL_SCHEMA_KEYS = ['doi', 'aulast', 'aufirst', 'auinit', 'auinit1', 'auinitm', 'ausuffix', 'au', 'aucorp', 'atitle', 

9 'jtitle', 'stitle', 'date', 'chron', 'ssn', 'quarter', 'volume', 'part', 'issue', 'spage', 

10 'epage', 'pages', 'artnum', 'issn', 'eissn', 'isbn', 'coden', 'sici', 'genre'] 

11 

12# The genres from the OpenURL schema we support 

13SUPPORTED_GENRES = ['journal', 'article'] 

14 

15# Mapping from OpenURL schema to both supported models (Journal, Article) 

16OPENURL_TO_ES = { 

17 'aulast': (None, 'bibjson.author.name.exact'), 

18 'aucorp': (None, 'bibjson.author.affiliation.exact'), 

19 'atitle': (None, 'bibjson.title.exact'), 

20 'jtitle': ('index.title.exact', 'bibjson.journal.title.exact'), # Note we use index.title.exact for journals, to support continuations 

21 'stitle': ('bibjson.alternative_title.exact', None), 

22 'date': (None, 'bibjson.year.exact'), 

23 'volume': (None, 'bibjson.journal.volume.exact'), 

24 'issue': (None, 'bibjson.journal.number.exact'), 

25 'spage': (None, 'bibjson.start_page.exact'), 

26 'epage': (None, 'bibjson.end_page.exact'), 

27 'issn': ('index.issn.exact', 'index.issn.exact'), # bibjson.identifier.id.exact 

28 'eissn': ('index.issn.exact', 'index.issn.exact'), 

29 'isbn': ('index.issn.exact', 'index.issn.exact'), 

30 'doi': (None, 'index.doi.exact') 

31} 

32 

33# Terms search template. Ensure all queries from OpenURL return publicly visible results with in_doaj : true 

34IN_DOAJ_TERM = {"term": {"admin.in_doaj": True}} 

35TERMS_SEARCH = {"query": {"bool": {"must": [IN_DOAJ_TERM]}}} 

36 

37 

38def reformat_au_as_aulast(au_value): 

39 """ 

40 Best-effort reformat of an OpenURL ``au`` (free-text full author name) into the 

41 "Given Surname" order DOAJ's own indexing uses (see 

42 ``portality.crosswalks.article_crossref_xml.extract_authors``), so it can be 

43 searched via the same ES field as ``aulast``. 

44 

45 Real-world requests we've seen in production use ``au`` non-standardly to hold 

46 *multiple* authors in one value, semicolon-separated, each usually in 

47 "Surname, Given" order (e.g. ``"Barich, Howard;Kotler, Philip"``) - the 

48 opposite order to how we index names. Only the first author is used if 

49 several are supplied: exact-match search on a full name is already fragile 

50 (a missing middle initial, a diacritic, or different formatting from the 

51 source citation will still silently fail to match), and OR-ing several 

52 full-name guesses together would only make that harder to reason about, not 

53 more accurate. 

54 

55 :param au_value: the raw au string, e.g. "Barich, Howard;Kotler, Philip" 

56 :return: a best-effort "Given Surname" string, e.g. "Howard Barich", or the 

57 first author's value unchanged if it doesn't look like "Surname, Given" 

58 (no comma present) 

59 """ 

60 first_author = au_value.split(";")[0].strip() 

61 if "," in first_author: 

62 surname, given = first_author.split(",", 1) 

63 return (given.strip() + " " + surname.strip()).strip() 

64 return first_author 

65 

66 

67class UnsupportedOpenURLQuery(Exception): 

68 """ 

69 Raised when an OpenURL request cannot be turned into a usable search query - 

70 e.g. every field supplied is one the OpenURL schema allows but that DOAJ does 

71 not support searching on (such as ``rft.chron``), or the genre supplied isn't 

72 one DOAJ recognises. Callers (the view) should treat this as a bad request 

73 (400), as distinct from a well-formed query that simply found no results. 

74 """ 

75 pass 

76 

77 

78class OpenURLRequest(object): 

79 """ 

80 Based on the fields from ofi/fmt:kev:mtx:journal schema for Journals in OpenURL 1.0 

81 This is the only schema the DOAJ supports. 

82 """ 

83 

84 # ~~API:Feature~~ 

85 

86 def __init__(self, **kwargs): 

87 

88 # Initialise the OpenURLRequest object with empty attributes 

89 for key in JOURNAL_SCHEMA_KEYS: 

90 setattr(self, key, None) 

91 

92 # set by query_es(): the supported genre searched as (lowercased, or None if unsupported), and the fields it 

93 # had to skip 

94 self._search_genre = None 

95 self._unsupported_fields = [] 

96 

97 # Save any attributes specified at creation time 

98 if kwargs: 

99 for key, value in kwargs.items(): 

100 setattr(self, key, value) 

101 

102 @staticmethod 

103 def _total_hits(results): 

104 return results.get('hits', {}).get('total', {}).get('value', 0) 

105 

106 def __str__(self): 

107 return "OpenURLRequest{" + ", ".join(["%s : %s" % (x, getattr(self, x)) for x in JOURNAL_SCHEMA_KEYS if getattr(self, x)]) + "}" 

108 

109 def query_es(self): 

110 """ 

111 Query Elasticsearch for a set of matches for this request. 

112 

113 Fields the OpenURL schema allows but that DOAJ has no ES mapping for 

114 (e.g. ``au``, ``chron``, ``ssn`` ...), and a genre DOAJ doesn't recognise, 

115 are logged and skipped rather than raising - unless skipping them leaves 

116 no usable search term at all, in which case :class:`UnsupportedOpenURLQuery` 

117 is raised so the caller can respond with a 400 rather than treating it as 

118 a legitimate (if empty) search. 

119 

120 :return: The results of a query through the dao, a JSON object, or None 

121 if no attributes were supplied on the request at all. 

122 :raises UnsupportedOpenURLQuery: if the request had content, but none of 

123 it could be mapped onto a search DOAJ supports. 

124 """ 

125 # Copy to the template, which will be populated with terms 

126 populated_query = deepcopy(TERMS_SEARCH) 

127 

128 # Get all of the attributes with values set. Guard against a schema key that 

129 # somehow doesn't have a matching property (shouldn't happen given __init__, 

130 # but defends against future schema drift rather than crashing on it). 

131 set_attributes = [] 

132 for x in JOURNAL_SCHEMA_KEYS[:-1]: 

133 try: 

134 val = getattr(self, x) 

135 except AttributeError as e: 

136 app.logger.warning("OpenURL schema key '{x}' has no matching property, skipping: {e}".format(x=x, e=e)) 

137 continue 

138 if val: 

139 set_attributes.append((x, val)) 

140 

141 # If we don't have a genre, guess journal FIXME: is it correct to assume journal? 

142 if not self.genre: 

143 self.genre = SUPPORTED_GENRES[0] # TODO: we may want to handle 404 instead 

144 

145 # Set i to use either our mapping for journals or articles. An unrecognised 

146 # genre means we can't tell which half of OPENURL_TO_ES to use at all, so 

147 # none of the supplied fields (even otherwise-valid ones) can be mapped. 

148 unsupported_fields = [] 

149 try: 

150 i = SUPPORTED_GENRES.index(str(self.genre).lower()) 

151 except (ValueError, AttributeError) as e: 

152 app.logger.warning("OpenURL request supplied an unsupported genre '{x}': {e}".format(x=self.genre, e=e)) 

153 unsupported_fields.append(("genre", self.genre)) 

154 i = None 

155 

156 # Add the attributes to the query, skipping (and logging) any field that 

157 # the OpenURL schema allows but DOAJ can't map to a search term, instead 

158 # of letting one bad field crash the whole request. 

159 # NOTE: populated_query's "must" list always starts with IN_DOAJ_TERM, so 

160 # we can't tell "no real search terms added" just from its length - track 

161 # that separately. 

162 terms_added = 0 

163 if i is not None: 

164 # so 'au' can defer to an explicit 'aulast' if both are given, rather 

165 # than adding two, possibly conflicting, author terms 

166 attr_keys = {x for (x, _) in set_attributes} 

167 

168 for (k, v) in set_attributes: 

169 lookup_key, term_value = k, v 

170 

171 # 'au' (free-text full author name) isn't in OPENURL_TO_ES - DOAJ 

172 # only supports author search via 'aulast' (surname). Real-world 

173 # requests use 'au' far more often than 'aulast', so rather than 

174 # just dropping it, make a best-effort translation onto the same 

175 # search 'aulast' already uses (see reformat_au_as_aulast). 

176 if k == "au": 

177 if "aulast" in attr_keys: 

178 app.logger.debug("OpenURL 'au' field ignored in favour of explicit 'aulast'") 

179 continue 

180 lookup_key = "aulast" 

181 term_value = reformat_au_as_aulast(v) 

182 

183 try: 

184 es_term = OPENURL_TO_ES[lookup_key][i] 

185 except (KeyError, IndexError, TypeError) as e: 

186 app.logger.warning("OpenURL field '{x}' is not supported by DOAJ, skipping: {e}".format(x=k, e=e)) 

187 unsupported_fields.append((k, v)) 

188 continue 

189 

190 # known field, but not searchable for this genre (e.g. aulast for journals) 

191 if es_term is None: 

192 app.logger.warning("OpenURL field '{x}' is not searchable for genre '{g}', skipping".format(x=k, g=self.genre)) 

193 unsupported_fields.append((k, v)) 

194 continue 

195 

196 term = {"term": {es_term: term_value}} 

197 populated_query["query"]["bool"]["must"].append(term) 

198 terms_added += 1 

199 

200 # get_result_url() needs these to decide whether a match is trustworthy 

201 self._search_genre = SUPPORTED_GENRES[i] if i is not None else None 

202 self._unsupported_fields = unsupported_fields 

203 

204 # avoid doing an empty (unconstrained beyond in_doaj) query 

205 if terms_added == 0: 

206 if unsupported_fields: 

207 # the request had content, but none of it could be turned into a 

208 # usable search - this is a bad request, not just "no results" 

209 msg = "OpenURL request contained no fields DOAJ can search on (unsupported: {x})".format( 

210 x=", ".join(f for f, _ in unsupported_fields)) 

211 app.logger.warning(msg) 

212 raise UnsupportedOpenURLQuery(msg) 

213 

214 app.logger.debug("No valid search terms in OpenURL object") 

215 return None 

216 

217 try: 

218 query_json = json.dumps(populated_query) 

219 except TypeError as e: 

220 query_json = "<unable to serialise query for logging: {e}>".format(e=e) 

221 

222 # Return the results of the query 

223 if i == 0: 

224 app.logger.debug("OpenURL query to journal: " + query_json) 

225 return Journal.query(q=populated_query) 

226 elif i == 1: 

227 app.logger.debug("OpenURL query to article: " + query_json) 

228 return Article.query(q=populated_query) 

229 

230 def get_result_url(self): 

231 """ 

232 Get the URL for this OpenURLRequest's referent. 

233 :return: The url as a string, or None if not found. 

234 :raises UnsupportedOpenURLQuery: propagated from query_es(), or if an article 

235 request only matched several articles once unsupported fields were skipped. 

236 The caller (the view) should turn this into a 400 response. 

237 """ 

238 results = self.query_es() 

239 

240 if results is None: 

241 return None 

242 

243 if self._total_hits(results) == 0: 

244 # No results found for query, retry 

245 results = self.fallthrough_retry() 

246 if results is None or self._total_hits(results) == 0: 

247 # This time we've definitely failed 

248 return None 

249 

250 # Skipping an unsupported field broadens the search. For an article, the skipped field (artnum, pages, part, 

251 # chron, aufirst ...) may be the very thing that tells the matches apart, so taking the first of several hits 

252 # would send the user to the wrong article with no sign anything was ignored. Only trust a unique match. 

253 # (For a journal the skipped fields describe items within it, so they don't change which journal it is.) 

254 if self._search_genre == "article" and self._unsupported_fields and self._total_hits(results) > 1: 

255 msg = ("OpenURL article request matched {n} articles once unsupported fields were skipped " 

256 "(unsupported: {x}), so it can't be resolved to one article").format( 

257 n=self._total_hits(results), x=", ".join(f for f, _ in self._unsupported_fields)) 

258 raise UnsupportedOpenURLQuery(msg) 

259 

260 if results.get('hits', {}).get('hits', [{}])[0].get('_source', {}).get('es_type') == 'journal': 

261 

262 # construct a journal object around the result 

263 journal = Journal(**results['hits']['hits'][0]) 

264 

265 # the continuation is a first-class journal object, so if we have a journal we have the right continuation 

266 # (assuming that the user gave us specific enough information 

267 ident = journal.id 

268 

269 # construct the toc url using the ident only 

270 jtoc_url = url_for("doaj.toc", identifier=ident) 

271 return jtoc_url 

272 

273 #~~->Article:Page~~ 

274 elif results.get('hits', {}).get('hits', [{}])[0].get('_source', {}).get('es_type') == 'article': 

275 return url_for("doaj.article_page", identifier=results['hits']['hits'][0]['_id']) 

276 

277 def query_for_vol(self, journalobj): 

278 

279 # The journal object will already be the correct continuation, if the user provided sufficient detail. 

280 issns = journalobj.bibjson().issns() 

281 

282 # If there's no way to get the wanted issns, give up, else run the query 

283 if issns is None: 

284 return None 

285 else: 

286 volume_query = deepcopy(TERMS_SEARCH) 

287 volume_query["size"] = 0 

288 

289 issn_term = {"terms": {"index.issn.exact": issns}} 

290 volume_query["query"]["bool"]["must"].append(issn_term) 

291 

292 vol_term = {"term": {"bibjson.journal.volume.exact": self.volume}} 

293 volume_query["query"]["bool"]["must"].append(vol_term) 

294 

295 # And if there's an issue, query that too. Note, issue does not make sense on its own. 

296 if self.issue: 

297 iss_term = {"term": {"bibjson.journal.number.exact": self.issue}} 

298 volume_query["query"]["bool"]["must"].append(iss_term) 

299 

300 try: 

301 query_json = json.dumps(volume_query) 

302 except TypeError as e: 

303 query_json = "<unable to serialise query for logging: {e}>".format(e=e) 

304 app.logger.debug("OpenURL subsequent volume query to article: " + query_json) 

305 return Article.query(q=volume_query) 

306 

307 def fallthrough_retry(self): 

308 """ 

309 Some things to try differently if we get no results on first attempt 

310 :return: a new result set, or None 

311 """ 

312 results = None 

313 

314 # Search again for the title against alternative_title (may catch translations of titles). 

315 # Journals only: stitle has no article mapping, so for an article this would just drop the title and re-run 

316 # the remaining terms - a broader search that could redirect to an unrelated article. 

317 if self.jtitle and not self.stitle and self._search_genre == "journal": 

318 self.stitle = self.jtitle 

319 self.jtitle = None 

320 results = self.query_es() 

321 

322 return results 

323 

324 def validate_issn(self, issn_str): 

325 """ 

326 If the ISSN is missing a dash, add it so it matches that in the index. 

327 :param issn_str: The ISSN, or if None, this will skip. 

328 :return: The ISSN with the dash added 

329 """ 

330 if issn_str: 

331 match_dash = re.compile('[-]') 

332 if not match_dash.search(issn_str): 

333 issn_str = issn_str[:4] + '-' + issn_str[4:] 

334 return issn_str 

335 

336 @property 

337 def doi(self): 

338 """Digital Object Identifier""" 

339 return self._doi 

340 

341 @doi.setter 

342 def doi(self, val): 

343 self._doi = val 

344 

345 @property 

346 def aulast(self): 

347 """First author's family name, may be more than one word""" 

348 return self._aulast 

349 

350 @aulast.setter 

351 def aulast(self, val): 

352 self._aulast = val 

353 

354 @property 

355 def aufirst(self): 

356 """First author's given name or names or initials""" 

357 return self._aufirst 

358 

359 @aufirst.setter 

360 def aufirst(self, val): 

361 self._aufirst = val 

362 

363 @property 

364 def auinit(self): 

365 """First author's first and middle initials""" 

366 return self._auinit 

367 

368 @auinit.setter 

369 def auinit(self, val): 

370 self._auinit = val 

371 

372 @property 

373 def auinit1(self): 

374 """First author's first initial""" 

375 return self._auinit1 

376 

377 @auinit1.setter 

378 def auinit1(self, val): 

379 self._auinit1 = val 

380 

381 @property 

382 def auinitm(self): 

383 """First author's middle initial""" 

384 return self._auinitm 

385 

386 @auinitm.setter 

387 def auinitm(self, val): 

388 self._auinitm = val 

389 

390 @property 

391 def ausuffix(self): 

392 """First author's name suffix. e.g. 'Jr.', 'III'""" 

393 return self._ausuffix 

394 

395 @ausuffix.setter 

396 def ausuffix(self, val): 

397 self._ausuffix = val 

398 

399 @property 

400 def au(self): 

401 """full name of a single author""" 

402 return self._au 

403 

404 @au.setter 

405 def au(self, val): 

406 self._au = val 

407 

408 @property 

409 def aucorp(self): 

410 """Organisation or corporation that is the author or creator of the document""" 

411 return self._aucorp 

412 

413 @aucorp.setter 

414 def aucorp(self, val): 

415 self._aucorp = val 

416 

417 @property 

418 def atitle(self): 

419 """Article title""" 

420 return self._atitle 

421 

422 @atitle.setter 

423 def atitle(self, val): 

424 self._atitle = val 

425 

426 @property 

427 def jtitle(self): 

428 """Journal title""" 

429 return self._jtitle 

430 

431 @jtitle.setter 

432 def jtitle(self, val): 

433 self._jtitle = val 

434 

435 @property 

436 def stitle(self): 

437 """Abbreviated or short journal title""" 

438 return self._stitle 

439 

440 @stitle.setter 

441 def stitle(self, val): 

442 self._stitle = val 

443 

444 @property 

445 def date(self): 

446 """Date of publication""" 

447 return self._date 

448 

449 @date.setter 

450 def date(self, val): 

451 if val: 

452 try: 

453 parsed_date = dates.parse(val) 

454 val = parsed_date.year 

455 except ValueError: 

456 val = None 

457 self._date = val 

458 

459 @property 

460 def chron(self): 

461 """Non-normalised enumeration / chronology, e.g. '1st quarter'""" 

462 return self._chron 

463 

464 @chron.setter 

465 def chron(self, val): 

466 self._chron = val 

467 

468 @property 

469 def ssn(self): 

470 """Season (chronology). spring|summer|fall|autumn|winter""" 

471 return self._ssn 

472 

473 @ssn.setter 

474 def ssn(self, val): 

475 self._ssn = val 

476 

477 @property 

478 def quarter(self): 

479 """Quarter (chronology). 1|2|3|4""" 

480 return self._quarter 

481 

482 @quarter.setter 

483 def quarter(self, val): 

484 self._quarter = val 

485 

486 @property 

487 def volume(self): 

488 """Volume designation. e.g. '124', or 'VI'""" 

489 return self._volume 

490 

491 @volume.setter 

492 def volume(self, val): 

493 self._volume = val 

494 

495 @property 

496 def part(self): 

497 """Subdivision of a volume or highest level division of the journal. e.g. 'B', 'Supplement'""" 

498 return self._part 

499 

500 @part.setter 

501 def part(self, val): 

502 self._part = val 

503 

504 @property 

505 def issue(self): 

506 """Journal issue""" 

507 return self._issue 

508 

509 @issue.setter 

510 def issue(self, val): 

511 self._issue = val 

512 

513 @property 

514 def spage(self): 

515 """Starting page""" 

516 return self._spage 

517 

518 @spage.setter 

519 def spage(self, val): 

520 self._spage = val 

521 

522 @property 

523 def epage(self): 

524 """Ending page""" 

525 return self._epage 

526 

527 @epage.setter 

528 def epage(self, val): 

529 self._epage = val 

530 

531 @property 

532 def pages(self): 

533 """Page range e.g. '53-58', 'C4-9'""" 

534 return self._pages 

535 

536 @pages.setter 

537 def pages(self, val): 

538 self._pages = val 

539 

540 @property 

541 def artnum(self): 

542 """Article number""" 

543 return self._artnum 

544 

545 @artnum.setter 

546 def artnum(self, val): 

547 self._artnum = val 

548 

549 @property 

550 def issn(self): 

551 """Journal ISSN""" 

552 return self._issn 

553 

554 @issn.setter 

555 def issn(self, val): 

556 self._issn = self.validate_issn(val) 

557 

558 @property 

559 def eissn(self): 

560 """ISSN for electronic version of the journal""" 

561 return self._eissn 

562 

563 @eissn.setter 

564 def eissn(self, val): 

565 self._eissn = self.validate_issn(val) 

566 

567 @property 

568 def isbn(self): 

569 """Journal ISBN""" 

570 return self._isbn 

571 

572 @isbn.setter 

573 def isbn(self, val): 

574 self._isbn = val 

575 

576 @property 

577 def coden(self): 

578 """CODEN""" 

579 return self._coden 

580 

581 @coden.setter 

582 def coden(self, val): 

583 self._coden = val 

584 

585 @property 

586 def sici(self): 

587 """Serial Item and Contribution Identifier (SICI)""" 

588 return self._sici 

589 

590 @sici.setter 

591 def sici(self, val): 

592 self._sici = val 

593 

594 @property 

595 def genre(self): 

596 """journal|issue|article|proceeding|conference|preprint|unknown""" 

597 return self._genre 

598 

599 @genre.setter 

600 def genre(self, val): 

601 self._genre = val