Coverage for portality / settings.py: 99%

1""" 

2~~AppSettings:Config~~ 

3""" 

4import os 

5from portality import constants 

6from portality.lib import paths 

7from datetime import datetime 

8 

9########################################### 

10# Application Version information 

11# ~~->API:Feature~~ 

12 

13DOAJ_VERSION = "8.6.2" 

14API_VERSION = "4.0.1" 

15 

16###################################### 

17# Deployment configuration 

18 

19HOST = '0.0.0.0' 

20DEBUG = False 

21DEBUG_DEV_LOG = False # show all log of each module 

22PORT = 5004 

23SSL = True 

24VALID_ENVIRONMENTS = ['dev', 'test', 'staging', 'production', 'harvester'] 

25CMS_BUILD_ASSETS_ON_STARTUP = False 

26# Cookies security 

27SESSION_COOKIE_SAMESITE = 'Strict' 

28SESSION_COOKIE_SECURE = True 

29REMEMBER_COOKIE_SECURE = True 

30 

31#################################### 

32# Testdrive for setting up the test environment. 

33# CAUTION - this can modify the index so should NEVER be used in production! 

34TESTDRIVE_ENABLED = False 

35 

36#################################### 

37# Debug Mode 

38 

39# PyCharm debug settings 

40DEBUG_PYCHARM = False # do not try to connect to the PyCharm debugger by default 

41DEBUG_PYCHARM_SERVER = 'localhost' 

42DEBUG_PYCHARM_PORT = 6000 

43 

44# ~~->DebugToolbar:Framework~~ 

45DEBUG_TB_TEMPLATE_EDITOR_ENABLED = True 

46DEBUG_TB_INTERCEPT_REDIRECTS = False 

47 

48# set to True to enable the env list panel in the debug toolbar 

49DEBUG_TB_ENV_LIST_ENABLED = False 

50 

51####################################### 

52# Elasticsearch configuration 

53# ~~->Elasticsearch:Technology 

54 

55# elasticsearch settings # TODO: changing from single host / esprit to multi host on ES & correct the default 

56ELASTICSEARCH_HOSTS = [{'host': 'localhost', 'port': 9200}, {'host': 'localhost', 'port': 9201}] 

57ELASTIC_SEARCH_VERIFY_CERTS = True # Verify the SSL certificate of the ES host. Set to False in dev.cfg to avoid having to configure your local certificates 

58 

59# 2 sets of elasticsearch DB settings - index-per-project and index-per-type. Keep both for now so we can migrate. 

60# e.g. host:port/index/type/id 

61ELASTIC_SEARCH_DB = "doaj" 

62ELASTIC_SEARCH_TEST_DB = "doajtest" 

63 

64# e.g. host:port/type/doc/id 

65ELASTIC_SEARCH_INDEX_PER_TYPE = True 

66INDEX_PER_TYPE_SUBSTITUTE = '_doc' # Migrated from esprit 

67ELASTIC_SEARCH_DB_PREFIX = "doaj-" # note: include the separator 

68ELASTIC_SEARCH_TEST_DB_PREFIX = "doajtest-" 

69 

70INITIALISE_INDEX = True # whether or not to try creating the index and required index types on startup 

71ELASTIC_SEARCH_SNAPSHOT_REPOSITORY = None 

72ELASTIC_SEARCH_SNAPSHOT_TTL = 366 

73 

74ES_TERMS_LIMIT = 1024 

75ELASTICSEARCH_REQ_TIMEOUT = 20 # Seconds - used in core.py for whole ES connection request timeout 

76ES_READ_TIMEOUT = '2m' # Minutes - used in DAO for searches 

77 

78##################################################### 

79# Elastic APM config (MUST be configured in env file) 

80# ~~->APM:Feature~~ 

81 

82ENABLE_APM = False 

83 

84ELASTIC_APM = { 

85 # Set required service name. Allowed characters: 

86 # a-z, A-Z, 0-9, -, _, and space 

87 'SERVICE_NAME': '', 

88 

89 # Use if APM Server requires a token 

90 'SECRET_TOKEN': '', 

91 

92 # Set custom APM Server URL (default: http://localhost:8200) 

93 'SERVER_URL': '', 

94} 

95 

96########################################### 

97# Event handler 

98 

99# Process events immediately/synchronously 

100EVENT_SEND_FUNCTION = "portality.events.background.send_event" 

101 

102########################################### 

103# Read Only Mode 

104# 

105# Use these options to place the application into READ ONLY mode 

106 

107# This puts the UI into READ_ONLY mode 

108# ~~->ReadOnlyMode:Feature~~ 

109READ_ONLY_MODE = False 

110 

111# This puts the cron jobs into READ_ONLY mode 

112SCRIPTS_READ_ONLY_MODE = False 

113 

114########################################### 

115# Feature Toggles 

116 

117# ~~->OfflineMode:Feature~~ 

118OFFLINE_MODE = False 

119 

120# List the features we want to be active (API v1 and v2 remain with redirects to v3) 

121# ~~->API:Feature~~ 

122FEATURES = ['api1', 'api2', 'api3', 'api4'] 

123VALID_FEATURES = ['api1', 'api2', 'api3', 'api4'] 

124 

125######################################## 

126# File Path and URL Path settings 

127 

128# root of the git repo 

129ROOT_DIR = paths.rel2abs(__file__, "..") 

130 

131# base path, to the directory where this settings file lives 

132BASE_FILE_PATH = paths.abs_dir_path(__file__) 

133 

134BASE_URL = "https://doaj.org" 

135if BASE_URL.startswith('https://'): 

136 BASE_DOMAIN = BASE_URL[8:] 

137elif BASE_URL.startswith('http://'): 

138 BASE_DOMAIN = BASE_URL[7:] 

139else: 

140 BASE_DOMAIN = BASE_URL 

141 

142# ~~->API:Feature~~ 

143BASE_API_URL = "https://doaj.org/api/" 

144API_CURRENT_BLUEPRINT_NAME = "api_v4" # change if upgrading API to new version and creating new view 

145CURRENT_API_MAJOR_VERSION = "4" 

146 

147# URL used for the journal ToC URL in the journal CSV export 

148# NOTE: must be the correct route as configured in view/doaj.py 

149# ~~->ToC:WebRoute~~ 

150JOURNAL_TOC_URL_FRAG = BASE_URL + '/toc/' 

151 

152# Used when generating external links, e.g. in the API docs 

153PREFERRED_URL_SCHEME = 'https' 

154 

155# Whether the app is running behind a proxy, for generating URLs based on X-Forwarded-Proto 

156# ~~->ProxyFix:Framework~~ 

157PROXIED = False 

158 

159# directory to upload files to. MUST be full absolute path 

160# The default takes the directory above this, and then down in to "upload" 

161UPLOAD_DIR = os.path.join(ROOT_DIR, "upload") 

162UPLOAD_ASYNC_DIR = os.path.join(ROOT_DIR, "upload_async") 

163FAILED_ARTICLE_DIR = os.path.join(ROOT_DIR, "failed_articles") 

164 

165# directory where reports are output 

166REPORTS_BASE_DIR = "/home/cloo/reports/" 

167 

168################################## 

169# File store 

170# ~~->FileStore:Feature~~ 

171 

172# put this in your production.cfg, to store everything on S3: 

173# STORE_IMPL = "portality.store.StoreS3" 

174 

175STORE_IMPL = "portality.store.StoreLocal" 

176STORE_SCOPE_IMPL = { 

177 # Enable this by scope in order to have different scopes store via different storage implementations 

178 # constants.STORE__SCOPE__PUBLIC_DATA_DUMP: "portality.store.StoreS3", 

179 # constants.STORE__SCOPE__JOURNAL_CSV: "portality.store.StoreS3" 

180} 

181 

182STORE_TMP_IMPL = "portality.store.TempStore" 

183 

184STORE_LOCAL_DIR = paths.rel2abs(__file__, "..", "local_store", "main") 

185STORE_TMP_DIR = paths.rel2abs(__file__, "..", "local_store", "tmp") 

186STORE_LOCAL_EXPOSE = False # if you want to allow files in the local store to be exposed under /store/<path> urls. For dev only. 

187STORE_LOCAL_WRITE_BUFFER_SIZE = 16777216 

188STORE_TMP_WRITE_BUFFER_SIZE = 16777216 

189 

190# containers (buckets in AWS) where various content will be stored 

191# These values are placeholders, and must be overridden in live deployment 

192# this prevents test environments from accidentally writing to the production buckets 

193STORE_ANON_DATA_CONTAINER = "doaj-anon-data-placeholder" 

194STORE_CACHE_CONTAINER = "doaj-data-cache-placeholder" 

195STORE_PUBLIC_DATA_DUMP_CONTAINER = "doaj-data-dump-placeholder" 

196STORE_HARVESTER_CONTAINER = "doaj-harvester" 

197STORE_EXPORT_CONTAINER = "doaj-export-placeholder" 

198STORE_JOURNAL_CSV_CONTAINER = "doaj-journal-csv-placeholder" 

199 

200# S3 credentials for relevant scopes 

201# ~~->S3:Technology~~ 

202STORE_S3_SCOPES = { 

203 "anon_data": { 

204 "aws_access_key_id": "put this in your dev/test/production.cfg", 

205 "aws_secret_access_key": "put this in your dev/test/production.cfg" 

206 }, 

207 "cache": { 

208 "aws_access_key_id": "put this in your dev/test/production.cfg", 

209 "aws_secret_access_key": "put this in your dev/test/production.cfg" 

210 }, 

211 # Used by the api_export script to dump data from the api 

212 constants.STORE__SCOPE__PUBLIC_DATA_DUMP: { 

213 "aws_access_key_id": "put this in your dev/test/production.cfg", 

214 "aws_secret_access_key": "put this in your dev/test/production.cfg" 

215 }, 

216 # Used to store harvester run logs to S3 

217 "harvester": { 

218 "aws_access_key_id": "put this in your dev/test/production.cfg", 

219 "aws_secret_access_key": "put this in your dev/test/production.cfg" 

220 }, 

221 # Used to store the admin-generated CSV reports 

222 "export": { 

223 "aws_access_key_id": "put this in your dev/test/production.cfg", 

224 "aws_secret_access_key": "put this in your dev/test/production.cfg" 

225 }, 

226 constants.STORE__SCOPE__JOURNAL_CSV: { 

227 "aws_access_key_id": "put this in your dev/test/production.cfg", 

228 "aws_secret_access_key": "put this in your dev/test/production.cfg" 

229 } 

230} 

231 

232STORE_S3_MULTIPART_THRESHOLD = 5 * 1024 ** 3 # 5GB 

233 

234#################################### 

235# CMS configuration 

236 

237# paths where static content should be served from. 

238# * in the order you want them searched 

239# * relative to the portality directory 

240# ~~->CMS:DataStore~~ 

241STATIC_PATHS = [ 

242 "static", 

243 "../cms/assets" 

244] 

245 

246# GitHub base url where static content can be edited by the DOAJ team (you can leave out the trailing slash) 

247# ~~->GitHub:ExternalService~~ 

248CMS_EDIT_BASE_URL = "https://github.com/DOAJ/doaj/edit/static_pages/cms" 

249 

250# Where static files are served from - in case we need to serve a file 

251# from there ourselves using Flask instead of nginx (e.g. to support a 

252# legacy route to that file) 

253# Changing this will not change the actual folder that Flask serves 

254# static files from. 

255# http://flask.pocoo.org/snippets/102/ 

256STATIC_DIR = os.path.join(ROOT_DIR, "portality", "static") 

257 

258###################################### 

259# Service Descriptive Text 

260 

261SERVICE_NAME = "Directory of Open Access Journals" 

262 

263################################### 

264# Cookie settings 

265 

266# make this something secret in your overriding app.cfg 

267# ~~->Cookies:Feature~~ 

268SECRET_KEY = "default-key" 

269 

270# Consent Cookie and other Top-Level dismissable notes 

271# ~~->ConsentCookie:Feature~~ 

272CONSENT_COOKIE_KEY = "doaj-cookie-consent" 

273 

274# site notes, which can be configured to run any time with any content 

275# ~~-> SiteNote:Feature~~ 

276SITE_NOTE_ACTIVE = False 

277SITE_NOTE_KEY = "doaj-site-note" 

278SITE_NOTE_SLEEP = 259200 # every 3 days 

279SITE_NOTE_COOKIE_VALUE = "You have seen our most recent site wide announcement" 

280 

281#################################### 

282# Authorisation settings 

283# ~~->AuthNZ:Feature~~ 

284 

285# Can people register publicly? If false, only the superuser can create new accounts 

286PUBLIC_REGISTER = True 

287 

288SUPER_USER_ROLE = "admin" 

289 

290LOGIN_VIA_ACCOUNT_ID = True 

291 

292# amount of time a reset token is valid for (86400 is 24 hours) 

293PASSWORD_RESET_TIMEOUT = 86400 

294# amount of time a reset token for a new account is valid for 

295PASSWORD_CREATE_TIMEOUT = PASSWORD_RESET_TIMEOUT * 14 

296 

297# "api" top-level role is added to all accounts on creation; it can be revoked per account by removal of the role. 

298TOP_LEVEL_ROLES = [ 

299 "admin", 

300 "publisher", 

301 "editor", 

302 "associate_editor", 

303 "api", 

304 "ultra_bulk_delete", 

305 "preservation", 

306 constants.ROLE_PUBLIC_DATA_DUMP, 

307 constants.ROLE_PUBLISHER_JOURNAL_CSV, 

308 constants.ROLE_ADMIN_REPORT_WITH_NOTES, 

309 constants.ROLE_PREMIUM, 

310 constants.ROLE_PREMIUM_OAI, 

311 constants.ROLE_PREMIUM_PDD, 

312 constants.ROLE_PREMIUM_CSV 

313] 

314 

315ROLE_MAP = { 

316 "editor": [ 

317 "associate_editor", # note, these don't cascade, so we still need to list all the low-level roles 

318 "edit_journal", 

319 "edit_suggestion", 

320 "edit_application", # todo: switchover from suggestion to application 

321 "editor_area", 

322 "assign_to_associate", 

323 "list_group_journals", 

324 "list_group_suggestions", 

325 "read_notifications" 

326 ], 

327 "associate_editor": [ 

328 "edit_journal", 

329 "edit_suggestion", 

330 "editor_area", 

331 "read_notifications" 

332 ], 

333 constants.ROLE_PREMIUM: [ 

334 constants.ROLE_PREMIUM_OAI, 

335 constants.ROLE_PREMIUM_PDD, 

336 constants.ROLE_PREMIUM_CSV 

337 ] 

338} 

339 

340# If you change the reserved usernames, your data will likely need a migration to remove their 

341# existing use in the system. 

342SYSTEM_USERNAME = "system" 

343RESERVED_USERNAMES = [SYSTEM_USERNAME] # do not allow the creation of user accounts with this id 

344 

345# Role map to destination route on login (when no other destination page is present) 

346# checked in order, if the user has the role in the first tuple position, they will 

347# be redirected to the endpoint in the second tuple position 

348ROLE_LOGIN_DESTINATIONS = [ 

349 ("admin", "dashboard.top_todo"), 

350 ("editor", "editor.index"), 

351 ("associate_editor", "editor.index"), 

352 ("publisher", "publisher.index") 

353] 

354 

355# if the user doesn't have one of the above roles, where should they be sent after login 

356DEFAULT_LOGIN_DESTINATION = "doaj.home" 

357 

358#################################### 

359# Email Settings 

360# ~~->Email:ExternalService 

361 

362# Settings for Flask-Mail. Set in app.cfg 

363MAIL_SERVER = None # default localhost 

364MAIL_PORT = 25 # default 25 

365#MAIL_USE_TLS # default False 

366#MAIL_USE_SSL # default False 

367#MAIL_DEBUG # default app.debug 

368#MAIL_USERNAME # default None 

369#MAIL_PASSWORD # default None 

370#MAIL_DEFAULT_SENDER # default None 

371#MAIL_MAX_EMAILS # default None 

372#MAIL_SUPPRESS_SEND # default app.testing 

373 

374ENABLE_EMAIL = True 

375ENABLE_PUBLISHER_EMAIL = True 

376 

377ADMIN_NAME = "DOAJ" 

378ADMIN_EMAIL = "sysadmin@cottagelabs.com" 

379ADMINS = ["steve@cottagelabs.com", "mark@cottagelabs.com"] 

380 

381MANAGING_EDITOR_EMAIL = "managing-editors@doaj.org" 

382CONTACT_FORM_ADDRESS = "feedback+contactform@doaj.org" 

383SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS = ["helpdesk@doaj.org"] 

384 

385SYSTEM_EMAIL_FROM = 'helpdesk@doaj.org' 

386CC_ALL_EMAILS_TO = SYSTEM_EMAIL_FROM # DOAJ may get a dedicated inbox in the future 

387 

388# Error logging via email 

389SUPPRESS_ERROR_EMAILS = False 

390ERROR_LOGGING_EMAIL = 'doaj.internal@gmail.com' 

391ERROR_MAIL_HOSTNAME = 'smtp.mailgun.org' 

392ERROR_MAIL_USERNAME = None 

393ERROR_MAIL_PASSWORD = None 

394 

395# Reports email recipient 

396REPORTS_EMAIL_TO = ["helpdesk@doaj.org"] 

397 

398######################################## 

399# workflow email notification settings 

400# ~~->WorkflowNotifications:Feature~~ 

401 

402MAN_ED_IDLE_WEEKS = 4 # weeks before an application is considered reminder-worthy 

403ED_IDLE_WEEKS = 3 # weeks before the editor is warned about idle applications in their group 

404ASSOC_ED_IDLE_DAYS = 10 

405ASSOC_ED_IDLE_WEEKS = 3 

406 

407# Which statuses the notification queries should be filtered to show 

408# ~~-> ApplicationStatuses:Config~~ 

409MAN_ED_NOTIFICATION_STATUSES = [ 

410 constants.APPLICATION_STATUS_PENDING, 

411 constants.APPLICATION_STATUS_IN_PROGRESS, constants.APPLICATION_STATUS_COMPLETED, 

412 constants.APPLICATION_STATUS_ON_HOLD 

413] 

414ED_NOTIFICATION_STATUSES = [ 

415 constants.APPLICATION_STATUS_PENDING, 

416 constants.APPLICATION_STATUS_IN_PROGRESS, 

417 constants.APPLICATION_STATUS_COMPLETED 

418] 

419ASSOC_ED_NOTIFICATION_STATUSES = [ 

420 constants.APPLICATION_STATUS_PENDING, 

421 constants.APPLICATION_STATUS_IN_PROGRESS 

422] 

423 

424################################### 

425# status endpoint configuration 

426# ~~->StatusEndpoint:Feature~~ 

427 

428# /status endpoint connection to all app machines 

429APP_MACHINES_INTERNAL_IPS = [HOST + ':' + str(PORT)] # This should be set in production.cfg (or dev.cfg etc) 

430 

431########################################### 

432# Background Jobs settings 

433# ~~->Huey:Technology~~ 

434# ~~->Redis:Technology~~ 

435# ~~->BackgroundTasks:Feature~~ 

436 

437# huey/redis settings 

438REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1') 

439REDIS_PORT = os.getenv('REDIS_PORT', 6379) 

440HUEY_IMMEDIATE = False 

441HUEY_ASYNC_DELAY = 10 

442 

443# Crontab for never running a job - February 31st (use to disable tasks) 

444CRON_NEVER = {"month": "2", "day": "31", "day_of_week": "*", "hour": "*", "minute": "*"} 

445 

446# Crontab schedules must be for unique times to avoid delays due to perceived race conditions 

447HUEY_SCHEDULE = { 

448 "sitemap": {"month": "*", "day": "*", "day_of_week": "*", "hour": "8", "minute": "0"}, 

449 "reporting": {"month": "*", "day": "1", "day_of_week": "*", "hour": "0", "minute": "0"}, 

450 "journal_csv": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "20"}, 

451 "read_news": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "30"}, 

452 "article_cleanup_sync": {"month": "*", "day": "2", "day_of_week": "*", "hour": "0", "minute": "0"}, 

453 "async_workflow_notifications": {"month": "*", "day": "*", "day_of_week": "1", "hour": "5", "minute": "0"}, 

454 "request_es_backup": {"month": "*", "day": "*", "day_of_week": "*", "hour": "6", "minute": "0"}, 

455 "check_latest_es_backup": {"month": "*", "day": "*", "day_of_week": "*", "hour": "9", "minute": "0"}, 

456 "prune_es_backups": {"month": "*", "day": "*", "day_of_week": "*", "hour": "9", "minute": "15"}, 

457 "public_data_dump": {"month": "*", "day": "*", "day_of_week": "*", "hour": "10", "minute": "0"}, 

458 "harvest": {"month": "*", "day": "*", "day_of_week": "*", "hour": "5", "minute": "30"}, 

459 "anon_export": {"month": "*", "day": "10", "day_of_week": "*", "hour": "1", "minute": "10"}, 

460 "old_data_cleanup": {"month": "*", "day": "12", "day_of_week": "*", "hour": "6", "minute": "30"}, 

461 "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"}, 

462 "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "0", "minute": "3"}, 

463 "datalog_journal_added_update": {"month": "*", "day": "*", "day_of_week": "*", "hour": "4", "minute": "30"}, 

464 "auto_assign_editor_group_data": {"month": "*", "day": "*/7", "day_of_week": "*", "hour": "3", "minute": "30"}, 

465 "ris_export": {"month": "*", "day": "15", "day_of_week": "*", "hour": "3", "minute": "30"}, 

466 "site_statistics": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "40"}, 

467} 

468 

469 

470HUEY_TASKS = { 

471 "ingest_articles": {"retries": 10, "retry_delay": 15}, 

472 "preserve": {"retries": 0, "retry_delay": 15}, 

473 "application_autochecks": {"retries": 0, "retry_delay": 15}, 

474 "journal_autochecks": {"retries": 0, "retry_delay": 15} 

475} 

476 

477#################################### 

478# publisher area settings 

479 

480# the earliest date accepted on the publisher's 'enter article metadata' form. 

481# code will default to 15 years before current year if commented out. 

482# ~~->ArticleMetadata:Page~~ 

483METADATA_START_YEAR = 1960 

484 

485# tick (on toc) and doaj seal settings 

486# ~~->Tick:Feature~~ 

487TICK_THRESHOLD = '2014-03-19T00:00:00Z' 

488 

489# ~~->UpdateRequests:Feature~~ 

490UPDATE_REQUESTS_SHOW_OLDEST = "2018-01-01T00:00:00Z" 

491 

492############################################## 

493# Elasticsearch Mappings 

494# ~~->Elasticsearch:Technology~~ 

495 

496FACET_FIELD = ".exact" 

497 

498# an array of DAO classes from which to retrieve the type-specific ES mappings 

499# to be loaded into the index during initialisation. 

500ELASTIC_SEARCH_MAPPINGS = [ 

501 "portality.models.Article", # ~~->Article:Model~~ 

502 "portality.models.Journal", # ~~->Journal:Model~~ 

503 "portality.models.Application", # ~~->Application:Model~~ 

504 "portality.models.DraftApplication", # ~~-> DraftApplication:Model~~ 

505 "portality.models.harvester.HarvestState", # ~~->HarvestState:Model~~ 

506 "portality.models.background.BackgroundJob", # ~~-> BackgroundJob:Model~~ 

507 "portality.models.autocheck.Autocheck", # ~~-> Autocheck:Model~~ 

508 "portality.models.export.Export", # ~~-> Export:Model~~ 

509 "portality.models.DataDump", # ~~-> DataDump:Model~~ 

510 "portality.models.JournalCSV", # ~~-> JournalCSV:Model~~ 

511 "portality.models.ur_review_route.URReviewRoute", # ~~-> URReviewRoute:Model~~ 

512 "portality.models.admin_alert.AdminAlert", # ~~-> AdminAlert:Model~~ 

513 "portality.models.ris_export.RISExport", 

514] 

515 

516# Map from dataobj coercion declarations to ES mappings 

517# ~~->DataObj:Library~~ 

518# ~~->Seamless:Library~~ 

519DATAOBJ_TO_MAPPING_DEFAULTS = { 

520 "unicode": { 

521 "type": "text", 

522 "fields": { 

523 "exact": { 

524 "type": "keyword", 

525 # "index": False, 

526 "store": True 

527 } 

528 } 

529 }, 

530 "str": { 

531 "type": "text", 

532 "fields": { 

533 "exact": { 

534 "type": "keyword", 

535 # "index": False, 

536 "store": True 

537 } 

538 } 

539 }, 

540 "unicode_upper": { 

541 "type": "text", 

542 "fields": { 

543 "exact": { 

544 "type": "keyword", 

545 # "index": False, 

546 "store": True 

547 } 

548 } 

549 }, 

550 "unicode_lower": { 

551 "type": "text", 

552 "fields": { 

553 "exact": { 

554 "type": "keyword", 

555 # "index": False, 

556 "store": True 

557 } 

558 } 

559 }, 

560 "isolang": { 

561 "type": "text", 

562 "fields": { 

563 "exact": { 

564 "type": "keyword", 

565 # "index": False, 

566 "store": True 

567 } 

568 } 

569 }, 

570 "isolang_2letter_strict": { 

571 "type": "text", 

572 "fields": { 

573 "exact": { 

574 "type": "keyword", 

575 # "index": False, 

576 "store": True 

577 } 

578 } 

579 }, 

580 "isolang_2letter_lax": { 

581 "type": "text", 

582 "fields": { 

583 "exact": { 

584 "type": "keyword", 

585 # "index": False, 

586 "store": True 

587 } 

588 } 

589 }, 

590 "country_code": { 

591 "type": "text", 

592 "fields": { 

593 "exact": { 

594 "type": "keyword", 

595 # "index": False, 

596 "store": True 

597 } 

598 } 

599 }, 

600 "currency_code_strict": { 

601 "type": "text", 

602 "fields": { 

603 "exact": { 

604 "type": "keyword", 

605 # "index": False, 

606 "store": True 

607 } 

608 } 

609 }, 

610 "currency_code_lax": { 

611 "type": "text", 

612 "fields": { 

613 "exact": { 

614 "type": "keyword", 

615 # "index": False, 

616 "store": True 

617 } 

618 } 

619 }, 

620 "issn": { 

621 "type": "text", 

622 "fields": { 

623 "exact": { 

624 "type": "keyword", 

625 # "index": False, 

626 "store": True 

627 } 

628 } 

629 }, 

630 "url": { 

631 "type": "text", 

632 "fields": { 

633 "exact": { 

634 "type": "keyword", 

635 # "index": False, 

636 "store": True 

637 } 

638 } 

639 }, 

640 "utcdatetimemicros": { 

641 "type": "date", 

642 "format": "date_optional_time" 

643 }, 

644 "utcdatetime": { 

645 "type": "date", 

646 "format": "date_optional_time" 

647 }, 

648 "bool": { 

649 "type": "boolean" 

650 }, 

651 "integer": { 

652 "type": "long" 

653 }, 

654 "bigenddate": { 

655 "type": "date", 

656 "format": "date_optional_time" 

657 }, 

658 "year": { 

659 "type": "date", 

660 "format": "year" 

661 } 

662} 

663 

664# Extension Map from dataobj coercion declarations to ES mappings. 

665# This is useful when some extensions required for some objects additional to defaults. 

666# ~~->DataObj:Library~~ 

667# ~~->Seamless:Library~~ 

668DATAOBJ_TO_MAPPING_COPY_TO_EXTENSIONS = { 

669 "unicode": {"copy_to": ["all_meta"]}, 

670 "str": {"copy_to": ["all_meta"]}, 

671 "unicode_upper": {"copy_to": ["all_meta"]}, 

672 "unicode_lower": {"copy_to": ["all_meta"]}, 

673 "issn": {"copy_to": ["all_meta"]}, 

674 "url": {"copy_to": ["all_meta"]} 

675} 

676 

677# TODO: we may want a big-type and little-type setting 

678DEFAULT_INDEX_SETTINGS = \ 

679 { 

680 'number_of_shards': 4, 

681 'number_of_replicas': 1, 

682 'analysis': { 

683 'analyzer': { 

684 'ascii_folded': { 

685 'tokenizer': 'standard', 

686 'filter': ['lowercase', 'asciifolding'] 

687 } 

688 } 

689 } 

690 } 

691 

692DEFAULT_DYNAMIC_MAPPING = { 

693 'dynamic_templates': [ 

694 { 

695 "strings": { 

696 "match_mapping_type": "string", 

697 "mapping": { 

698 "type": "text", 

699 "fields": { 

700 "exact": { 

701 "type": "keyword", 

702 # "normalizer": "lowercase" 

703 } 

704 } 

705 } 

706 } 

707 } 

708 ] 

709} 

710 

711# LEGACY MAPPINGS 

712# a dict of the ES mappings. identify by name, and include name as first object name 

713# and identifier for how non-analyzed fields for faceting are differentiated in the mappings 

714MAPPINGS = { 

715 'account': { # ~~->Account:Model~~ 

716 # 'aliases': { 

717 # 'account': {} 

718 # }, 

719 'mappings': DEFAULT_DYNAMIC_MAPPING, 

720 'settings': DEFAULT_INDEX_SETTINGS 

721 } 

722} 

723 

724MAPPINGS['upload'] = MAPPINGS["account"] # ~~->Upload:Model~~ 

725MAPPINGS['bulk_articles'] = MAPPINGS["account"] # ~~->BulkArticles:Model~~ 

726MAPPINGS['cache'] = MAPPINGS["account"] # ~~->Cache:Model~~ 

727MAPPINGS['lcc'] = MAPPINGS["account"] # ~~->LCC:Model~~ 

728MAPPINGS['editor_group'] = MAPPINGS["account"] # ~~->EditorGroup:Model~~ 

729MAPPINGS['news'] = MAPPINGS["account"] # ~~->News:Model~~ 

730MAPPINGS['lock'] = MAPPINGS["account"] # ~~->Lock:Model~~ 

731MAPPINGS['provenance'] = MAPPINGS["account"] # ~~->Provenance:Model~~ 

732MAPPINGS['preserve'] = MAPPINGS["account"] # ~~->Preservation:Model~~ 

733MAPPINGS['notification'] = MAPPINGS["account"] # ~~->Notification:Model~~ 

734MAPPINGS['article_tombstone'] = MAPPINGS["account"] # ~~->ArticleTombstone:Model~~ 

735MAPPINGS['shortened_url'] = MAPPINGS["account"] #~~->URLShortener:Model~~ 

736 

737######################################### 

738# Query Routes 

739# ~~->Query:WebRoute~~ 

740 

741QUERY_ROUTE = { 

742 "query": { 

743 # ~~->PublicJournalQuery:Endpoint~~ 

744 "journal": { 

745 "auth": False, 

746 "role": None, 

747 "query_validators": ["non_public_fields_validator", "public_query_validator"], 

748 "query_filters": ["only_in_doaj", "last_update_fallback", "search_all_meta"], 

749 "result_filters": ["public_result_filter"], 

750 "dao": "portality.models.Journal", # ~~->Journal:Model~~ 

751 "required_parameters": {"ref": ["ssw", "public_journal", "subject_page"]} 

752 }, 

753 # ~~->PublicArticleQuery:Endpoint~~ 

754 "article": { 

755 "auth": False, 

756 "role": None, 

757 "query_validators": ["non_public_fields_validator", "public_query_validator"], 

758 "query_filters": ["only_in_doaj"], 

759 "result_filters": ["public_result_filter"], 

760 "dao": "portality.models.Article", # ~~->Article:Model~~ 

761 "required_parameters": {"ref": ["public_article", "toc", "subject_page"]} 

762 }, 

763 # back-compat for fixed query widget 

764 # ~~->PublicJournalArticleQuery:Endpoint~~ 

765 "journal,article": { 

766 "auth": False, 

767 "role": None, 

768 "query_validators": ["non_public_fields_validator", "public_query_validator"], 

769 "query_filters": ["only_in_doaj", "strip_facets", "es_type_fix", "journal_article_filter"], 

770 "result_filters": ["public_result_filter", "add_fqw_facets", "fqw_back_compat"], 

771 "dao": "portality.models.JournalArticle", # ~~->JournalArticle:Model~~ 

772 "required_parameters": {"ref": ["fqw"]} 

773 } 

774 }, 

775 "publisher_query": { 

776 # ~~->PublisherJournalQuery:Endpoint~~ 

777 "journal": { 

778 "auth": True, 

779 "role": "publisher", 

780 "query_validators": ["non_public_fields_validator"], 

781 "query_filters": ["owner", "only_in_doaj", "search_all_meta"], 

782 "result_filters": ["publisher_result_filter"], 

783 "dao": "portality.models.Journal" # ~~->Journal:Model~~ 

784 }, 

785 # ~~->PublisherApplicationQuery:Endpoint~~ 

786 "applications": { 

787 "auth": True, 

788 "role": "publisher", 

789 "query_validators": ["non_public_fields_validator"], 

790 "query_filters": ["owner", "not_update_request", "search_all_meta"], 

791 "result_filters": ["publisher_result_filter"], 

792 "dao": "portality.models.AllPublisherApplications" # ~~->AllPublisherApplications:Model~~ 

793 }, 

794 # ~~->PublisherUpdateRequestsQuery:Endpoint~~ 

795 "update_requests": { 

796 "auth": True, 

797 "role": "publisher", 

798 "query_validators": ["non_public_fields_validator"], 

799 "query_filters": ["owner", "update_request", "search_all_meta"], 

800 "result_filters": ["publisher_result_filter"], 

801 "dao": "portality.models.Application" # ~~->Application:Model~~ 

802 } 

803 }, 

804 "admin_query": { 

805 # ~~->AdminJournalQuery:Endpoint~~ 

806 "journal": { 

807 "auth": True, 

808 "role": "admin", 

809 "dao": "portality.models.Journal" # ~~->Journal:Model~~ 

810 }, 

811 # ~~->AdminApplicationQuery:Endpoint~~ 

812 "suggestion": { 

813 "auth": True, 

814 "role": "admin", 

815 "query_filters": ["not_update_request"], 

816 "dao": "portality.models.Application" # ~~->Application:Model~~ 

817 }, 

818 # ~~->AdminUpdateRequestQuery:Endpoint~~ 

819 "update_requests": { 

820 "auth": True, 

821 "role": "admin", 

822 "query_filters": ["update_request"], 

823 "dao": "portality.models.Application" # ~~->Application:Model~~ 

824 }, 

825 # ~~->AdminEditorGroupQuery:Endpoint~~ 

826 "editor,group": { 

827 "auth": True, 

828 "role": "admin", 

829 "dao": "portality.models.EditorGroup" # ~~->EditorGroup:Model~~ 

830 }, 

831 # ~~->AdminAccountQuery:Endpoint~~ 

832 "account": { 

833 "auth": True, 

834 "role": "admin", 

835 "dao": "portality.models.Account" # ~~->Account:Model~~ 

836 }, 

837 # ~~->AdminJournalArticleQuery:Endpoint~~ 

838 "journal,article": { 

839 "auth": True, 

840 "role": "admin", 

841 "dao": "portality.models.search.JournalArticle" # ~~->JournalArticle:Model~~ 

842 }, 

843 # ~~->AdminBackgroundJobQuery:Endpoint~~ 

844 "background,job": { 

845 "auth": True, 

846 "role": "admin", 

847 "dao": "portality.models.BackgroundJob" # ~~->BackgroundJob:Model~~ 

848 }, 

849 # ~~->APINotificationQuery:Endpoint~~ 

850 "notifications": { 

851 "auth": True, 

852 "role": "admin", 

853 "dao": "portality.models.Notification", # ~~->Notification:Model~~ 

854 "required_parameters": None 

855 }, 

856 "reports": { 

857 "auth": True, 

858 "role": "admin", 

859 "dao": "portality.models.Export" 

860 }, 

861 "journal_csv": { 

862 "auth": True, 

863 "role": "admin", 

864 "dao": "portality.models.JournalCSV" # ~~->JournalCSV:Model~~ 

865 }, 

866 "pdd": { 

867 "auth": True, 

868 "role": "admin", 

869 "dao": "portality.models.DataDump" # ~~->JournalCSV:Model~~ 

870 }, 

871 "alerts": { 

872 "auth": True, 

873 "role": "admin", 

874 "dao": "portality.models.AdminAlert", # ~~->AdminAlert:Model~~ 

875 "required_parameters": None 

876 }, 

877 "autoassign": { 

878 "auth": True, 

879 "role": "admin", 

880 "dao": "portality.models.URReviewRoute", # ~~->AdminAlert:Model~~ 

881 "required_parameters": None 

882 }, 

883 "ris": { 

884 "auth": True, 

885 "role": "admin", 

886 "dao": "portality.models.RISExport", # ~~->AdminAlert:Model~~ 

887 "required_parameters": None 

888 } 

889 }, 

890 "associate_query": { 

891 # ~~->AssEdJournalQuery:Endpoint~~ 

892 "journal": { 

893 "auth": True, 

894 "role": "associate_editor", 

895 "query_validators": ["non_public_fields_validator"], 

896 "query_filters": ["associate", "search_all_meta", "flagged"], 

897 "dao": "portality.models.Journal" # ~~->Journal:Model~~ 

898 }, 

899 # ~~->AssEdApplicationQuery:Endpoint~~ 

900 "suggestion": { 

901 "auth": True, 

902 "role": "associate_editor", 

903 "query_validators": ["non_public_fields_validator"], 

904 "query_filters": ["associate", "search_all_meta"], 

905 "dao": "portality.models.Application" # ~~->Application:Model~~ 

906 } 

907 }, 

908 "editor_query": { 

909 # ~~->EditorJournalQuery:Endpoint~~ 

910 "journal": { 

911 "auth": True, 

912 "role": "editor", 

913 "query_validators": ["non_public_fields_validator"], 

914 "query_filters": ["editor", "search_all_meta"], 

915 "dao": "portality.models.Journal" # ~~->Journal:Model~~ 

916 }, 

917 # ~~->EditorApplicationQuery:Endpoint~~ 

918 "suggestion": { 

919 "auth": True, 

920 "role": "editor", 

921 "query_validators": ["non_public_fields_validator"], 

922 "query_filters": ["editor", "search_all_meta"], 

923 "dao": "portality.models.Application" # ~~->Application:Model~~ 

924 } 

925 }, 

926 "api_query": { 

927 # ~~->APIArticleQuery:Endpoint~~ 

928 "article": { 

929 "auth": False, 

930 "role": None, 

931 "query_filters": ["only_in_doaj", "public_source"], 

932 "dao": "portality.models.Article", # ~~->Article:Model~~ 

933 "required_parameters": None, 

934 "keepalive": "10m" 

935 }, 

936 # ~~->APIJournalQuery:Endpoint~~ 

937 "journal": { 

938 "auth": False, 

939 "role": None, 

940 "query_validators": ["non_public_fields_validator"], 

941 "query_filters": ["only_in_doaj", "public_source", "search_all_meta"], 

942 "dao": "portality.models.Journal", # ~~->Journal:Model~~ 

943 "required_parameters": None 

944 }, 

945 # ~~->APIApplicationQuery:Endpoint~~ 

946 "application": { 

947 "auth": True, 

948 "role": None, 

949 "query_validators": ["non_public_fields_validator"], 

950 "query_filters": ["owner", "private_source", "search_all_meta"], 

951 "dao": "portality.models.Suggestion", # ~~->Application:Model~~ 

952 "required_parameters": None 

953 } 

954 }, 

955 "dashboard_query": { 

956 # ~~->APINotificationQuery:Endpoint~~ 

957 "notifications": { 

958 "auth": True, 

959 "role": "read_notifications", 

960 "query_filters": ["who_current_user"], # ~~-> WhoCurrentUser:Query 

961 "dao": "portality.models.Notification", # ~~->Notification:Model~~ 

962 "required_parameters": None 

963 } 

964 } 

965} 

966 

967QUERY_FILTERS = { 

968 # sanitisers 

969 "public_query_validator": "portality.lib.query_filters.public_query_validator", 

970 "non_public_fields_validator": "portality.lib.query_filters.non_public_fields_validator", 

971 

972 # query filters 

973 "only_in_doaj": "portality.lib.query_filters.only_in_doaj", 

974 "owner": "portality.lib.query_filters.owner", 

975 "update_request": "portality.lib.query_filters.update_request",