Coverage for portality/models/article.py: 83%

777 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1import string 

2 

3from unidecode import unidecode 

4from functools import reduce 

5from copy import deepcopy 

6from datetime import datetime 

7 

8from portality import datasets, constants 

9from portality.core import app 

10from portality.dao import DomainObject 

11from portality.lib import es_data_mapping 

12from portality.lib.coerce import COERCE_MAP 

13from portality.lib.dates import FMT_DATETIME_STD 

14from portality.lib.seamless import SeamlessMixin 

15from portality.models import Journal 

16from portality.models.v1.bibjson import GenericBibJSON # NOTE that article specifically uses the v1 BibJSON 

17from portality.models.v1 import shared_structs 

18from portality.models.v2.shared_structs import ARTICLE_STRUCT 

19from portality.lib import normalise, dates 

20 

21 

22class NoJournalException(Exception): 

23 pass 

24 

25class NoValidOwnerException(Exception): 

26 pass 

27 

28 

29ARTICLE_BIBJSON_EXTENSION = { 

30 "objects" : ["bibjson"], 

31 "structs" : { 

32 "bibjson" : { 

33 "fields" : { 

34 "year" : {"coerce" : "unicode"}, 

35 "month" : {"coerce" : "unicode"}, 

36 "start_page" : {"coerce" : "unicode"}, 

37 "end_page" : {"coerce" : "unicode"}, 

38 "abstract" : {"coerce" : "unicode"} 

39 }, 

40 "lists" : { 

41 "author" : {"contains" : "object"} 

42 }, 

43 "objects" : [ 

44 "journal" 

45 ], 

46 

47 "structs" : { 

48 "author" : { 

49 "fields" : { 

50 "name" : {"coerce" : "unicode"}, 

51 "affiliation" : {"coerce" : "unicode"}, 

52 "email" : {"coerce": "unicode"}, 

53 "orcid_id" : {"coerce" : "unicode"} 

54 } 

55 }, 

56 

57 "journal" : { 

58 "fields" : { 

59 "volume" : {"coerce" : "unicode"}, 

60 "number" : {"coerce" : "unicode"}, 

61 "publisher" : {"coerce" : "unicode"}, 

62 "title" : {"coerce" : "unicode"}, 

63 "country" : {"coerce" : "unicode"} 

64 }, 

65 "lists" : { 

66 "language" : {"contains" : "field", "coerce" : "unicode"}, 

67 "issns" : {"contains" : "field", "coerce" : "unicode"} 

68 } 

69 } 

70 } 

71 

72 } 

73 } 

74} 

75 

76MAPPING_OPTS = { 

77 "dynamic": None, 

78 "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"], 

79 "exceptions": app.config["ARTICLE_EXCEPTION_MAPPING"], 

80 "additional_mappings": {} 

81} 

82 

83 

84class Article(SeamlessMixin, DomainObject): 

85 __type__ = "article" 

86 

87 __SEAMLESS_STRUCT__ = [ 

88 ARTICLE_STRUCT, 

89 shared_structs.SHARED_BIBJSON, 

90 ARTICLE_BIBJSON_EXTENSION 

91 ] 

92 

93 __SEAMLESS_COERCE__ = COERCE_MAP 

94 

95 def mappings(self): 

96 return es_data_mapping.create_mapping(self.__seamless_struct__.raw, MAPPING_OPTS) 

97 

98 @classmethod 

99 def duplicates(cls, publisher_record_id=None, doi=None, fulltexts=None, title=None, volume=None, number=None, start=None, should_match=None, size=10): 

100 # some input sanitisation 

101 urls = fulltexts if isinstance(fulltexts, list) else [fulltexts] if isinstance(fulltexts, str) or isinstance(fulltexts, str) else [] 

102 

103 # make sure that we're dealing with the normal form of the identifiers 

104 norm_urls = [] 

105 for url in urls: 

106 try: 

107 norm = normalise.normalise_url(url) 

108 norm_urls.append(norm) 

109 except ValueError: 

110 # use the non-normal form 

111 norm_urls.append(url) 

112 urls = norm_urls 

113 

114 try: 

115 doi = normalise.normalise_doi(doi) 

116 except ValueError: 

117 # leave the doi as it is 

118 pass 

119 

120 q = DuplicateArticleQuery(publisher_record_id=publisher_record_id, 

121 doi=doi, 

122 urls=urls, 

123 title=title, 

124 volume=volume, 

125 number=number, 

126 start=start, 

127 should_match=should_match, 

128 size=size) 

129 

130 # res = cls.query(q=q.query()) 

131 # return [cls(**hit.get("_source")) for hit in res.get("hits", {}).get("hits", [])] 

132 return cls.q2obj(q=q.query()) 

133 

134 @classmethod 

135 def list_volumes(cls, issns): 

136 q = ArticleVolumesQuery(issns) 

137 result = cls.query(q=q.query()) 

138 return _human_sort([t.get("key") for t in result.get("aggregations", {}).get("vols", {}).get("buckets", [])]) 

139 

140 @classmethod 

141 def list_volume_issues(cls, issns, volume): 

142 q = ArticleVolumesIssuesQuery(issns, volume) 

143 result = cls.query(q=q.query()) 

144 return _human_sort([t.get("key") for t in result.get("aggregations", {}).get("issues", {}).get("buckets", [])]) 

145 

146 @classmethod 

147 def get_by_volume(cls, issns, volume): 

148 q = ArticleQuery(issns=issns, volume=volume) 

149 articles = cls.iterate(q.query(), page_size=1000) 

150 return articles 

151 

152 @classmethod 

153 def find_by_issns(cls, issns): 

154 q = ArticleQuery(issns=issns) 

155 articles = cls.iterate(q.query(), page_size=1000) 

156 return articles 

157 

158 @classmethod 

159 def count_by_issns(cls, issns, in_doaj=None): 

160 q = ArticleQuery(issns=issns, in_doaj=in_doaj) 

161 return cls.hit_count(q.query()) 

162 

163 @classmethod 

164 def delete_by_issns(cls, issns, snapshot=True): 

165 q = ArticleQuery(issns=issns) 

166 cls.delete_selected(query=q.query(), snapshot=snapshot) 

167 

168 @classmethod 

169 def delete_selected(cls, query=None, owner=None, snapshot=True, tombstone=True): 

170 if owner is not None: 

171 from portality.models import Journal 

172 issns = Journal.issns_by_owner(owner) 

173 q = ArticleQuery(issns=issns) 

174 query = q.query() 

175 

176 if snapshot or tombstone: 

177 articles = cls.iterate(query, page_size=1000) 

178 for article in articles: 

179 if snapshot: 

180 article.snapshot() 

181 if tombstone: 

182 article._tombstone() 

183 return cls.delete_by_query(query) 

184 

185 def delete(self): 

186 self._tombstone() 

187 super(Article, self).delete() 

188 

189 def bibjson(self, **kwargs): 

190 if "bibjson" not in self.data: 

191 self.data["bibjson"] = {} 

192 return ArticleBibJSON(self.data.get("bibjson"), **kwargs) 

193 

194 def set_bibjson(self, bibjson): 

195 bibjson = bibjson.bibjson if isinstance(bibjson, ArticleBibJSON) else bibjson 

196 self.data["bibjson"] = bibjson 

197 

198 def history(self): 

199 hs = self.data.get("history", []) 

200 tuples = [] 

201 for h in hs: 

202 tuples.append((h.get("date"), ArticleBibJSON(h.get("bibjson")))) 

203 return tuples 

204 

205 def snapshot(self): 

206 from portality.models import ArticleHistory 

207 

208 snap = deepcopy(self.data) 

209 if "id" in snap: 

210 snap["about"] = snap["id"] 

211 del snap["id"] 

212 if "index" in snap: 

213 del snap["index"] 

214 if "last_updated" in snap: 

215 del snap["last_updated"] 

216 if "created_date" in snap: 

217 del snap["created_date"] 

218 

219 hist = ArticleHistory(**snap) 

220 hist.save() 

221 return hist.id 

222 

223 def make_tombstone(self): 

224 stone = ArticleTombstone() 

225 stone.set_id(self.id) 

226 sbj = stone.bibjson() 

227 

228 abj = self.bibjson() 

229 

230 # copy minimal essential metadata 

231 sbj.title = abj.title 

232 sbj.author = abj.author 

233 sbj.volume = abj.volume 

234 sbj.number = abj.number 

235 sbj.start_page = abj.start_page 

236 sbj.end_page = abj.end_page 

237 sbj.journal_title = abj.journal_title 

238 pissn = abj.get_one_identifier(abj.P_ISSN) 

239 if pissn is not None: 

240 sbj.add_identifier(sbj.P_ISSN, pissn) 

241 eissn = abj.get_one_identifier(abj.E_ISSN) 

242 if eissn is not None: 

243 sbj.add_identifier(sbj.E_ISSN, eissn) 

244 

245 # Ensure subjects are present (for back-compat with any consumers expecting it) 

246 subs = abj.subjects() 

247 if subs is not None: 

248 for s in subs: 

249 sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code")) 

250 

251 # Add owner (at time of deletion) information 

252 stone.owner = self.get_owner() 

253 

254 return stone 

255 

256 def _tombstone(self): 

257 # make the tombstone 

258 stone = self.make_tombstone() 

259 

260 # save and return 

261 stone.save() 

262 return stone 

263 

264 def add_history(self, bibjson, date=None): 

265 """Deprecated""" 

266 bibjson = bibjson.bibjson if isinstance(bibjson, ArticleBibJSON) else bibjson 

267 if date is None: 

268 date = dates.now_str() 

269 snobj = {"date": date, "bibjson": bibjson} 

270 if "history" not in self.data: 

271 self.data["history"] = [] 

272 self.data["history"].append(snobj) 

273 

274 def is_in_doaj(self): 

275 try: 

276 return self.data['admin'].get("in_doaj", False) 

277 except KeyError: 

278 # If we have no admin section, return None instead 

279 return None 

280 

281 def set_in_doaj(self, value): 

282 if "admin" not in self.data: 

283 self.data["admin"] = {} 

284 self.data["admin"]["in_doaj"] = value 

285 

286 def publisher_record_id(self): 

287 return self.data.get("admin", {}).get("publisher_record_id") 

288 

289 def set_publisher_record_id(self, pri): 

290 if "admin" not in self.data: 

291 self.data["admin"] = {} 

292 self.data["admin"]["publisher_record_id"] = pri 

293 

294 def upload_id(self): 

295 return self.data.get("admin", {}).get("upload_id") 

296 

297 def set_upload_id(self, uid): 

298 if "admin" not in self.data: 

299 self.data["admin"] = {} 

300 self.data["admin"]["upload_id"] = uid 

301 

302 def get_normalised_doi(self): 

303 if self.data.get("index", {}).get("doi") is not None: 

304 return self.data["index"]["doi"] 

305 doi = self.bibjson().get_one_identifier(constants.IDENT_TYPE_DOI) 

306 if doi is None: 

307 return None 

308 try: 

309 return normalise.normalise_doi(doi) 

310 except ValueError: 

311 # can't be normalised, so we just return the doi as-is 

312 return doi 

313 

314 def get_normalised_fulltext(self): 

315 if self.data.get("index", {}).get("fulltext") is not None: 

316 return self.data["index"]["fulltext"] 

317 fulltexts = self.bibjson().get_urls(constants.LINK_TYPE_FULLTEXT) 

318 if len(fulltexts) == 0: 

319 return None 

320 try: 

321 return normalise.normalise_url(fulltexts[0]) 

322 except ValueError: 

323 # can't be normalised, so we just return the url as-is 

324 return fulltexts[0] 

325 

326 def get_journal(self): 

327 """ 

328 Get this article's associated journal 

329 :return: A Journal, or None if this is an orphan article 

330 """ 

331 bibjson = self.bibjson() 

332 

333 # first, get the ISSNs associated with the record 

334 pissns = bibjson.get_identifiers(bibjson.P_ISSN) 

335 eissns = bibjson.get_identifiers(bibjson.E_ISSN) 

336 allissns = list(set(pissns + eissns)) 

337 

338 # find a matching journal record from the index 

339 best_match = None 

340 

341 for issn in allissns: 

342 journals = Journal.find_by_issn(issn) 

343 if len(journals) > 0: 

344 # Get the best journal match: 

345 # 1. Prefer the most recently updated journal that is in DOAJ. 

346 # 2. If none are in DOAJ, fall back to the most recently updated journal outside DOAJ. 

347 

348 matches = [j for j in journals if j.is_in_doaj()] 

349 

350 if len(matches) == 0: 

351 matches = journals 

352 

353 best_match = max( 

354 matches, 

355 key=lambda j: j.last_updated, 

356 default=None 

357 ) 

358 

359 return best_match 

360 

361 def get_associated_journals(self): 

362 # find all matching journal record from the index 

363 allissns = self.bibjson().issns() 

364 return Journal.find_by_issn(allissns) 

365 

366 def add_journal_metadata(self, j=None, reg=None): 

367 """ 

368 this function makes sure the article is populated 

369 with all the relevant info from its owning parent object 

370 :param j: Pass in a Journal to bypass the (slow) locating step. MAKE SURE IT'S THE RIGHT ONE! 

371 """ 

372 

373 # Record the data that is copied into the article into the "reg"ister, in case the 

374 # caller needs to know exactly and only which information was copied 

375 if reg is None: 

376 reg = Journal() 

377 rbj = reg.bibjson() 

378 

379 if j is None: 

380 journal = self.get_journal() 

381 else: 

382 journal = j 

383 

384 # we were unable to find a journal 

385 if journal is None: 

386 raise NoJournalException("Unable to find a journal associated with this article") 

387 

388 # if we get to here, we have a journal record we want to pull data from 

389 jbib = journal.bibjson() 

390 bibjson = self.bibjson() 

391 

392 # tripwire to be tripped if the journal makes changes to the article 

393 trip = False 

394 

395 if bibjson.subjects() != jbib.subjects(): 

396 trip = True 

397 bibjson.set_subjects(jbib.subjects()) 

398 rbj.set_subjects(jbib.subjects()) 

399 

400 if jbib.title is not None: 

401 if bibjson.journal_title != jbib.title: 

402 trip = True 

403 bibjson.journal_title = jbib.title 

404 rbj.title = jbib.title 

405 

406 if len(jbib.language) > 0: 

407 jlang = jbib.language 

408 alang = bibjson.journal_language 

409 jlang.sort() 

410 alang.sort() 

411 if jlang != alang: 

412 bibjson.journal_language = jbib.language 

413 trip = True 

414 rbj.set_language(jbib.language) 

415 

416 if jbib.country is not None: 

417 if jbib.country != bibjson.journal_country: 

418 bibjson.journal_country = jbib.country 

419 trip = True 

420 rbj.country = jbib.country 

421 

422 if jbib.publisher: 

423 if jbib.publisher != bibjson.publisher: 

424 bibjson.publisher = jbib.publisher 

425 trip = True 

426 rbj.publisher = jbib.publisher 

427 

428 # Copy the in_doaj status and the journal's ISSNs 

429 if journal.is_in_doaj() != self.is_in_doaj(): 

430 self.set_in_doaj(journal.is_in_doaj()) 

431 trip = True 

432 reg.set_in_doaj(journal.is_in_doaj()) 

433 

434 try: 

435 aissns = bibjson.journal_issns 

436 jissns = jbib.issns() 

437 aissns.sort() 

438 jissns.sort() 

439 if aissns != jissns: 

440 bibjson.journal_issns = jbib.issns() 

441 trip = True 

442 

443 eissns = jbib.get_identifiers(jbib.E_ISSN) 

444 pissns = jbib.get_identifiers(jbib.P_ISSN) 

445 if eissns is not None and len(eissns) > 0: 

446 rbj.add_identifier(rbj.E_ISSN, eissns[0]) 

447 if pissns is not None and len(pissns) > 0: 

448 rbj.add_identifier(rbj.P_ISSN, pissns[0]) 

449 except KeyError: 

450 # No issns, don't worry about it for now 

451 pass 

452 

453 return trip 

454 

455 def merge(self, old, take_id=True): 

456 # this takes an old version of the article and brings 

457 # forward any useful information that is needed. The rules of merge are: 

458 # - ignore "index" (it gets regenerated on save) 

459 # - always take the "created_date" 

460 # - any top level field that does not exist in the current item (esp "id" and "history") 

461 # - in "admin", copy any field that does not already exist 

462 

463 # first thing to do is create a snapshot of the old record 

464 old.snapshot() 

465 

466 # now go on and do the merge 

467 

468 # always take the created date 

469 self.set_created(old.created_date) 

470 

471 # take the id 

472 if self.id is None or take_id: 

473 self.set_id(old.id) 

474 

475 # take the history (deprecated) 

476 if len(self.data.get("history", [])) == 0: 

477 self.data["history"] = deepcopy(old.data.get("history", [])) 

478 

479 # take the bibjson 

480 if "bibjson" not in self.data: 

481 self.set_bibjson(deepcopy(old.bibjson())) 

482 

483 # take the admin if there isn't one 

484 if "admin" not in self.data: 

485 self.data["admin"] = deepcopy(old.data.get("admin", {})) 

486 else: 

487 # otherwise, copy any admin keys that don't exist on the current item 

488 oa = old.data.get("admin", {}) 

489 for key in oa: 

490 if key not in self.data["admin"]: 

491 self.data["admin"][key] = deepcopy(oa[key]) 

492 

493 def _generate_index(self): 

494 # the index fields we are going to generate 

495 issns = [] 

496 subjects = [] 

497 schema_subjects = [] 

498 schema_codes = [] 

499 schema_codes_tree = [] 

500 classification = [] 

501 langs = [] 

502 country = None 

503 publisher = [] 

504 classification_paths = [] 

505 unpunctitle = None 

506 asciiunpunctitle = None 

507 doi = None 

508 fulltext = None 

509 

510 # the places we're going to get those fields from 

511 cbib = self.bibjson() 

512 jindex = self.data.get('index', {}) 

513 hist = self.history() 

514 

515 # get the issns out of the current bibjson 

516 issns += cbib.get_identifiers(cbib.P_ISSN) 

517 issns += cbib.get_identifiers(cbib.E_ISSN) 

518 

519 # get the issn from the journal bibjson 

520 if isinstance(cbib.journal_issns, list): 

521 issns += cbib.journal_issns 

522 

523 # de-duplicate the issns 

524 issns = list(set(issns)) 

525 

526 # now get the issns out of the historic records 

527 for date, hbib in hist: 

528 issns += hbib.get_identifiers(hbib.P_ISSN) 

529 issns += hbib.get_identifiers(hbib.E_ISSN) 

530 

531 # get the subjects and concatenate them with their schemes from the current bibjson 

532 for subs in cbib.subjects(): 

533 scheme = subs.get("scheme") 

534 term = subs.get("term") 

535 subjects.append(term) 

536 schema_subjects.append(scheme + ":" + term) 

537 classification.append(term) 

538 if "code" in subs: 

539 schema_codes.append(scheme + ":" + subs.get("code")) 

540 

541 # copy the languages 

542 if len(cbib.journal_language) > 0: 

543 langs = [datasets.name_for_lang(l) for l in cbib.journal_language] 

544 

545 # Get the country name from the bibjson country code 

546 if cbib.journal_country: 

547 country = datasets.get_country_name(cbib.journal_country) 

548 

549 # copy the publisher/provider 

550 if cbib.publisher: 

551 publisher.append(cbib.publisher) 

552 

553 # deduplicate the lists 

554 issns = list(set(issns)) 

555 subjects = list(set(subjects)) 

556 schema_subjects = list(set(schema_subjects)) 

557 classification = list(set(classification)) 

558 publisher = list(set(publisher)) 

559 langs = list(set(langs)) 

560 schema_codes = list(set(schema_codes)) 

561 

562 # work out what the date of publication is 

563 date = cbib.get_publication_date() 

564 

565 # calculate the classification paths 

566 from portality.lcc import lcc # inline import since this hits the database 

567 for subs in cbib.subjects(): 

568 scheme = subs.get("scheme") 

569 term = subs.get("term") 

570 if scheme == "LCC": 

571 path = lcc.pathify(term) 

572 if path is not None: 

573 classification_paths.append(path) 

574 

575 # normalise the classification paths, so we only store the longest ones 

576 classification_paths = lcc.longest(classification_paths) 

577 schema_codes_tree = cbib.lcc_codes_full_list() 

578 

579 # create an unpunctitle 

580 if cbib.title is not None: 

581 throwlist = string.punctuation + '\n\t' 

582 unpunctitle = "".join(c for c in cbib.title if c not in throwlist).strip() 

583 try: 

584 asciiunpunctitle = unidecode(unpunctitle) 

585 except: 

586 asciiunpunctitle = unpunctitle 

587 

588 # create a normalised version of the DOI for deduplication 

589 source_doi = cbib.get_one_identifier(constants.IDENT_TYPE_DOI) 

590 try: 

591 doi = normalise.normalise_doi(source_doi) 

592 except ValueError as e: 

593 # if we can't normalise the DOI, just store it cast to lower case. 

594 doi = source_doi.lower() 

595 

596 # create a normalised version of the fulltext URL for deduplication 

597 fulltexts = cbib.get_urls(constants.LINK_TYPE_FULLTEXT) 

598 if len(fulltexts) > 0: 

599 source_fulltext = fulltexts[0] 

600 try: 

601 fulltext = normalise.normalise_url(source_fulltext) 

602 except ValueError as e: 

603 # if we can't normalise the fulltext store it as-is 

604 fulltext = source_fulltext 

605 

606 # build the index part of the object 

607 self.data["index"] = {} 

608 if len(issns) > 0: 

609 self.data["index"]["issn"] = issns 

610 if date != "": 

611 self.data["index"]["date"] = date 

612 self.data["index"]["date_toc_fv_month"] = date # Duplicated so we can have year/month facets in fv2 

613 if len(subjects) > 0: 

614 self.data["index"]["subject"] = subjects 

615 if len(schema_subjects) > 0: 

616 self.data["index"]["schema_subject"] = schema_subjects 

617 if len(classification) > 0: 

618 self.data["index"]["classification"] = classification 

619 if len(publisher) > 0: 

620 self.data["index"]["publisher"] = publisher 

621 if len(langs) > 0: 

622 self.data["index"]["language"] = langs 

623 if country is not None: 

624 self.data["index"]["country"] = country 

625 if len(schema_codes) > 0: 

626 self.data["index"]["schema_code"] = schema_codes 

627 if len(classification_paths) > 0: 

628 self.data["index"]["classification_paths"] = classification_paths 

629 if unpunctitle is not None: 

630 self.data["index"]["unpunctitle"] = unpunctitle 

631 if asciiunpunctitle is not None: 

632 self.data["index"]["asciiunpunctitle"] = unpunctitle 

633 if doi is not None: 

634 self.data["index"]["doi"] = doi 

635 if fulltext is not None: 

636 self.data["index"]["fulltext"] = fulltext 

637 if len(schema_codes_tree) > 0: 

638 self.data["index"]["schema_codes_tree"] = schema_codes_tree 

639 

640 def prep(self): 

641 self._generate_index() 

642 self.data['last_updated'] = dates.now_str() 

643 

644 def save(self, *args, **kwargs): 

645 self._generate_index() 

646 return super(Article, self).save(*args, **kwargs) 

647 

648 def get_owner(self): 

649 b = self.bibjson() 

650 article_issns = b.get_identifiers(b.P_ISSN) 

651 article_issns += b.get_identifiers(b.E_ISSN) 

652 owners = [] 

653 

654 seen_journal_issns = {} 

655 for issn in article_issns: 

656 journals = Journal.find_by_issn(issn) 

657 if journals is not None and len(journals) > 0: 

658 for j in journals: 

659 owners.append(j.owner) 

660 if j.owner not in seen_journal_issns: 

661 seen_journal_issns[j.owner] = [] 

662 seen_journal_issns[j.owner] += j.bibjson().issns() 

663 

664 # deduplicate the list of owners 

665 owners = list(set(owners)) 

666 

667 # no owner means we can't confirm 

668 if len(owners) == 0: 

669 raise NoValidOwnerException 

670 

671 # multiple owners means ownership of this article is confused 

672 if len(owners) > 1: 

673 return NoValidOwnerException 

674 

675 return owners[0] 

676 

677 

678class ArticleTombstone(Article): 

679 __type__ = "article_tombstone" 

680 

681 def snapshot(self): 

682 return None 

683 

684 def is_in_doaj(self): 

685 return False 

686 

687 @property 

688 def owner(self): 

689 return self.data.get("admin", {}).get("owner") 

690 

691 @owner.setter 

692 def owner(self, owner): 

693 if "admin" not in self.data: 

694 self.data["admin"] = {} 

695 self.data["admin"]["owner"] = owner 

696 

697 def prep(self): 

698 self.data['last_updated'] = dates.now_str() 

699 

700 def save(self, *args, **kwargs): 

701 return super(ArticleTombstone, self).save(*args, **kwargs) 

702 

703 

704class ArticleBibJSON(GenericBibJSON): 

705 

706 def __init__(self, bibjson=None, **kwargs): 

707 self._add_struct(shared_structs.SHARED_BIBJSON.get("structs", {}).get("bibjson")) 

708 self._add_struct(ARTICLE_BIBJSON_EXTENSION.get("structs", {}).get("bibjson")) 

709 super(ArticleBibJSON, self).__init__(bibjson, **kwargs) 

710 

711 # article-specific simple getters and setters 

712 @property 

713 def year(self): 

714 return self._get_single("year") 

715 

716 @year.setter 

717 def year(self, val): 

718 self._set_with_struct("year", val) 

719 

720 @year.deleter 

721 def year(self): 

722 self._delete("year") 

723 

724 @property 

725 def month(self): 

726 return self._get_single("month") 

727 

728 @month.setter 

729 def month(self, val): 

730 self._set_with_struct("month", val) 

731 

732 @month.deleter 

733 def month(self): 

734 self._delete("month") 

735 

736 @property 

737 def start_page(self): 

738 return self._get_single("start_page") 

739 

740 @start_page.setter 

741 def start_page(self, val): 

742 self._set_with_struct("start_page", val) 

743 

744 @property 

745 def end_page(self): 

746 return self._get_single("end_page") 

747 

748 @end_page.setter 

749 def end_page(self, val): 

750 self._set_with_struct("end_page", val) 

751 

752 @property 

753 def abstract(self): 

754 return self._get_single("abstract") 

755 

756 @abstract.setter 

757 def abstract(self, val): 

758 self._set_with_struct("abstract", val) 

759 

760 # article-specific complex part getters and setters 

761 

762 @property 

763 def volume(self): 

764 return self._get_single("journal.volume") 

765 

766 @volume.setter 

767 def volume(self, value): 

768 self._set_with_struct("journal.volume", value) 

769 

770 @property 

771 def number(self): 

772 return self._get_single("journal.number") 

773 

774 @number.setter 

775 def number(self, value): 

776 self._set_with_struct("journal.number", value) 

777 

778 @property 

779 def journal_title(self): 

780 return self._get_single("journal.title") 

781 

782 @journal_title.setter 

783 def journal_title(self, title): 

784 self._set_with_struct("journal.title", title) 

785 

786 @property 

787 def journal_language(self): 

788 return self._get_list("journal.language") 

789 

790 @journal_language.setter 

791 def journal_language(self, lang): 

792 self._set_with_struct("journal.language", lang) 

793 

794 @property 

795 def journal_country(self): 

796 return self._get_single("journal.country") 

797 

798 @journal_country.setter 

799 def journal_country(self, country): 

800 self._set_single("journal.country", country) 

801 

802 @property 

803 def journal_issns(self): 

804 return self._get_list("journal.issns") 

805 

806 @journal_issns.setter 

807 def journal_issns(self, issns): 

808 self._set_with_struct("journal.issns", issns) 

809 

810 @property 

811 def publisher(self): 

812 return self._get_single("journal.publisher") 

813 

814 @publisher.setter 

815 def publisher(self, value): 

816 self._set_with_struct("journal.publisher", value) 

817 

818 def add_author(self, name, affiliation=None, orcid_id=None): 

819 aobj = {"name": name} 

820 if affiliation is not None: 

821 aobj["affiliation"] = affiliation 

822 if orcid_id is not None: 

823 aobj["orcid_id"] = orcid_id 

824 self._add_to_list_with_struct("author", aobj) 

825 

826 @property 

827 def author(self): 

828 return self._get_list("author") 

829 

830 @author.setter 

831 def author(self, authors): 

832 self._set_with_struct("author", authors) 

833 

834 def get_publication_date(self, date_format=FMT_DATETIME_STD): 

835 # work out what the date of publication is 

836 date = "" 

837 if self.year is not None: 

838 if type(self.year) is str: # It should be, if the mappings are correct. but len() needs a sequence. 

839 # fix 2 digit years 

840 if len(self.year) == 2: 

841 try: 

842 intyear = int(self.year) 

843 except ValueError: 

844 # if it's 2 chars long and the 2 chars don't make an integer, 

845 # forget it 

846 return date 

847 

848 # In the case of truncated years, assume it's this century if before the current year 

849 if intyear <= int(str(dates.now().year)[:-2]): 

850 self.year = "20" + self.year # For readability over long-lasting code, I have refrained 

851 else: # from using str(dates.now().year)[:2] here. 

852 self.year = "19" + self.year # But don't come crying to me 90-ish years from now. 

853 

854 # if we still don't have a 4 digit year, forget it 

855 if len(self.year) != 4: 

856 return date 

857 

858 # build up our proposed datestamp 

859 date += str(self.year) 

860 if self.month is not None: 

861 try: 

862 if type(self.month) is int: 

863 if 1 <= int(self.month) <= 12: 

864 month_number = self.month 

865 else: 

866 month_number = 1 

867 elif len(self.month) <= 2: 

868 if 1 <= int(self.month) <= 12: 

869 month_number = self.month 

870 else: 

871 month_number = '1' 

872 elif len(self.month) == 3: # 'May' works with either case, obvz. 

873 month_number = datetime.strptime(self.month, '%b').month 

874 else: 

875 month_number = datetime.strptime(self.month, '%B').month 

876 

877 

878 # pad the month number to two digits. This accepts int or string 

879 date += '-{:0>2}'.format(month_number) 

880 except: 

881 # If something goes wrong, just assume it's January 

882 date += "-01" 

883 else: 

884 date += "-01" 

885 date += "-01T00:00:00Z" 

886 

887 # attempt to confirm the format of our datestamp 

888 try: 

889 datecheck = dates.parse(date) 

890 date = datecheck.strftime(date_format) 

891 except: 

892 return "" 

893 return date 

894 

895 def remove_journal_metadata(self): 

896 self._delete("journal") 

897 

898 def vancouver_citation(self): 

899 jtitle = self.journal_title 

900 year = self.year 

901 vol = self.volume 

902 iss = self.number 

903 start = self.start_page 

904 end = self.end_page 

905 

906 citation = "" 

907 

908 if year: 

909 citation += year + ";" 

910 

911 if vol: 

912 citation += vol 

913 

914 if iss: 

915 citation += "(" + iss + ")" 

916 

917 if start or end: 

918 if citation != "": 

919 citation += ":" 

920 if start: 

921 citation += start 

922 if end: 

923 if start: 

924 citation += "-" 

925 citation += end 

926 

927 return jtitle.strip(), citation 

928 

929 def lcc_codes_full_list(self): 

930 full_list = set() 

931 

932 from portality.lcc import lcc # inline import since this hits the database 

933 for subs in self.subjects(): 

934 scheme = subs.get("scheme") 

935 if scheme != "LCC": 

936 continue 

937 code = subs.get("code") 

938 expanded = lcc.expand_codes(code) 

939 full_list.update(expanded) 

940 

941 return ["LCC:" + x for x in full_list if x is not None] 

942 

943 

944################################################## 

945 

946class ArticleQuery(object): 

947 base_query = { 

948 "track_total_hits" : True, 

949 "query" : { 

950 "bool" : { 

951 "must" : [] 

952 } 

953 } 

954 } 

955 

956 _issn_terms = { "terms" : {"index.issn.exact" : ["<list of issns here>"]} } 

957 _volume_term = { "term" : {"bibjson.journal.volume.exact" : "<volume here>"} } 

958 

959 def __init__(self, issns=None, volume=None, in_doaj=None): 

960 self.issns = issns 

961 self.volume = volume 

962 self.in_doaj = in_doaj 

963 

964 def query(self): 

965 q = deepcopy(self.base_query) 

966 

967 if self.issns is not None: 

968 iq = deepcopy(self._issn_terms) 

969 iq["terms"]["index.issn.exact"] = self.issns 

970 q["query"]["bool"]["must"].append(iq) 

971 

972 if self.volume is not None: 

973 vq = deepcopy(self._volume_term) 

974 vq["term"]["bibjson.journal.volume.exact"] = self.volume 

975 q["query"]["bool"]["must"].append(vq) 

976 

977 if self.in_doaj is not None: 

978 q["query"]["bool"]["must"].append({"term": {"admin.in_doaj": self.in_doaj}}) 

979 

980 return q 

981 

982class ArticleVolumesQuery(object): 

983 base_query = { 

984 "track_total_hits": True, 

985 "query" : { 

986 "bool": { 

987 "filter": { 

988 "terms" : {"index.issn.exact" : ["<list of issns here>"]} 

989 } 

990 } 

991 }, 

992 "size" : 0, 

993 "aggs" : { 

994 "vols" : { 

995 "terms" : { 

996 "field" : "bibjson.journal.volume.exact", 

997 "order": {"_key" : "desc"}, 

998 "size" : 1000 

999 } 

1000 } 

1001 } 

1002 } 

1003 

1004 def __init__(self, issns=None): 

1005 self.issns = issns 

1006 

1007 def query(self): 

1008 q = deepcopy(self.base_query) 

1009 q["query"]["bool"]["filter"]["terms"]["index.issn.exact"] = self.issns 

1010 return q 

1011 

1012 

1013class ArticleVolumesIssuesQuery(object): 

1014 base_query = { 

1015 "track_total_hits": True, 

1016 "query" : { 

1017 "bool": { 

1018 "filter": { 

1019 "bool": { 

1020 "must": [ 

1021 {"terms" : {"index.issn.exact" : ["<list of issns here>"]}}, 

1022 {"term" : {"bibjson.journal.volume.exact" : "<volume here>"}} 

1023 ] 

1024 } 

1025 } 

1026 } 

1027 }, 

1028 "size" : 0, 

1029 "aggs" : { 

1030 "issues" : { 

1031 "terms" : { 

1032 "field" : "bibjson.journal.number.exact", 

1033 "order": {"_key", "desc"}, 

1034 "size" : 1000 

1035 } 

1036 } 

1037 } 

1038 } 

1039 

1040 def __init__(self, issns=None, volume=None): 

1041 self.issns = issns 

1042 self.volume = volume 

1043 

1044 def query(self): 

1045 q = deepcopy(self.base_query) 

1046 q["query"]["bool"]["filter"]["bool"]["must"][0]["terms"]["index.issn.exact"] = self.issns 

1047 q["query"]["bool"]["filter"]["bool"]["must"][1]["term"]["bibjson.journal.volume.exact"] = self.volume 

1048 return q 

1049 

1050 

1051class DuplicateArticleQuery(object): 

1052 base_query = { 

1053 "track_total_hits" : True, 

1054 "query": { 

1055 "bool": { 

1056 "must": [] 

1057 } 

1058 }, 

1059 "sort": [{"last_updated": {"order": "desc"}}] 

1060 } 

1061 

1062 _should = { 

1063 "should" : [], 

1064 "minimum_should_match" : 2 

1065 } 

1066 

1067 _volume_term = {"term" : {"bibjson.journal.volume.exact" : "<volume>"}} 

1068 _number_term = {"term" : {"bibjson.journal.number.exact" : "<issue number>"}} 

1069 _start_term = {"term" : {"bibjson.start_page.exact" : "<start page>"}} 

1070 _issn_terms = {"terms" : { "index.issn.exact" : ["<list of issns>"] }} 

1071 _pubrec_term = {"term" : {"admin.publisher_record_id.exact" : "<publisher record id>"}} 

1072 _identifier_term = {"term" : {"bibjson.identifier.id.exact" : "<issn here>"}} 

1073 _doi_term = {"term" : {"index.doi.exact" : "<doi here>"}} 

1074 _fulltext_terms = {"terms" : {"index.fulltext.exact" : ["<fulltext here>"]}} 

1075 _fuzzy_title = {"fuzzy" : {"bibjson.title.exact" : "<title here>"}} 

1076 

1077 def __init__(self, issns=None, publisher_record_id=None, doi=None, urls=None, title=None, volume=None, number=None, start=None, should_match=None, size=10): 

1078 self.issns = issns if isinstance(issns, list) else [] 

1079 self.publisher_record_id = publisher_record_id 

1080 self.doi = doi 

1081 self.urls = urls if isinstance(urls, list) else [urls] if isinstance(urls, str) or isinstance(urls, str) else [] 

1082 self.title = title 

1083 self.volume = volume 

1084 self.number = number 

1085 self.start = start 

1086 self.should_match = should_match 

1087 self.size = size 

1088 

1089 def query(self): 

1090 # - MUST be from at least one of the ISSNs 

1091 # - MUST have the publisher record id 

1092 # - MUST have the doi unless should_match is set 

1093 # - MUST have the one of the fulltext urls unless should_match is set 

1094 # - MUST fuzzy match the title 

1095 # - SHOULD have <should_match> of: volume, issue, start page, fulltext url, doi 

1096 

1097 q = deepcopy(self.base_query) 

1098 if len(self.issns) > 0: 

1099 it = deepcopy(self._issn_terms) 

1100 it["terms"]["index.issn.exact"] = self.issns 

1101 q["query"]["bool"]["must"].append(it) 

1102 

1103 if self.publisher_record_id is not None: 

1104 pr = deepcopy(self._pubrec_term) 

1105 pr["term"]["admin.publisher_record_id.exact"] = self.publisher_record_id 

1106 q["query"]["bool"]["must"].append(pr) 

1107 

1108 if self.doi is not None and self.should_match is None: 

1109 idt = deepcopy(self._doi_term) 

1110 # idt["term"]["bibjson.identifier.id.exact"] = self.doi 

1111 idt["term"]["index.doi.exact"] = self.doi 

1112 q["query"]["bool"]["must"].append(idt) 

1113 

1114 if len(self.urls) > 0 and self.should_match is None: 

1115 uq = deepcopy(self._fulltext_terms) 

1116 # uq["terms"]["bibjson.link.url.exact"] = self.urls 

1117 uq["terms"]["index.fulltext.exact"] = self.urls 

1118 q["query"]["bool"]["must"].append(uq) 

1119 

1120 if self.title is not None: 

1121 ft = deepcopy(self._fuzzy_title) 

1122 ft["fuzzy"]["bibjson.title.exact"] = self.title 

1123 q["query"]["bool"]["must"].append(ft) 

1124 

1125 if self.should_match is not None: 

1126 term_count = 0 

1127 s = deepcopy(self._should) 

1128 

1129 if self.volume is not None: 

1130 term_count += 1 

1131 vt = deepcopy(self._volume_term) 

1132 vt["term"]["bibjson.journal.volume.exact"] = self.volume 

1133 s["should"].append(vt) 

1134 

1135 if self.number is not None: 

1136 term_count += 1 

1137 nt = deepcopy(self._number_term) 

1138 nt["term"]["bibjson.journal.number.exact"] = self.number 

1139 s["should"].append(nt) 

1140 

1141 if self.start is not None: 

1142 term_count += 1 

1143 st = deepcopy(self._start_term) 

1144 st["term"]["bibjson.start_page.exact"] = self.start 

1145 s["should"].append(st) 

1146 

1147 if len(self.urls) > 0: 

1148 term_count += 1 

1149 uq = deepcopy(self._url_terms) 

1150 uq["terms"]["bibjson.link.url.exact"] = self.urls 

1151 s["should"].append(uq) 

1152 

1153 if self.doi is not None: 

1154 term_count += 1 

1155 idt = deepcopy(self._identifier_term) 

1156 idt["term"]["bibjson.identifier.id.exact"] = self.doi 

1157 s["should"].append(idt) 

1158 

1159 msm = self.should_match 

1160 if msm > term_count: 

1161 msm = term_count 

1162 s["minimum_should_match"] = msm 

1163 

1164 q["query"]["bool"].update(s) 

1165 

1166 # Allow more results than the default 

1167 q["size"] = self.size 

1168 

1169 return q 

1170 

1171 

1172def _human_sort(things, reverse=True): 

1173 numeric = [] 

1174 non_numeric = [] 

1175 nmap = {} 

1176 for v in things: 

1177 try: 

1178 # try to convert n to an int 

1179 vint = int(v) 

1180 

1181 # remember the original string (it may have leading 0s) 

1182 try: 

1183 nmap[vint].append(v) 

1184 except KeyError: 

1185 nmap[vint] = [v] 

1186 numeric.append(vint) 

1187 except: 

1188 non_numeric.append(v) 

1189 

1190 numeric.sort(reverse=reverse) 

1191 non_numeric.sort(reverse=reverse) 

1192 

1193 # convert the integers back to their string representation 

1194 return reduce(lambda x, y: x+y, [nmap[n] for n in numeric], []) + non_numeric 

1195 

1196 

1197def _sort_articles(articles): 

1198 # first extract the array we want to sort on 

1199 # and make a map of that value to the issue itself 

1200 unsorted = [] 

1201 numbers = [] 

1202 imap = {} 

1203 for art in articles: 

1204 sp = art.get("bibjson.start_page", [None])[0] 

1205 

1206 # can't sort anything that doesn't have a start page 

1207 if sp is None: 

1208 unsorted.append(art) 

1209 continue 

1210 

1211 # deal with start page clashes and record the start pages 

1212 # to sort by 

1213 if sp not in numbers: 

1214 numbers.append(sp) 

1215 if sp in imap: 

1216 imap[sp].append(art) 

1217 else: 

1218 imap[sp] = [art] 

1219 

1220 sorted_keys = _human_sort(numbers, reverse=False) 

1221 

1222 s = [] 

1223 for n in sorted_keys: 

1224 s += [x for x in imap[n]] 

1225 s += [x for x in unsorted] 

1226 

1227 return s 

1228 

1229 

1230class ArticleTombstoneRecentlyDeletedQuery(object): 

1231 def __init__(self, since=None): 

1232 self.since = since 

1233 

1234 def query(self): 

1235 q = { 

1236 "query": { 

1237 "range": { 

1238 "created_date": {"gte": dates.format(self.since)} 

1239 } 

1240 } 

1241 } 

1242 return q