Coverage for portality/core.py: 88%
145 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-09-05 21:15 +0100
1import os
2import threading
3import yaml
5from flask import Flask
6from flask_login import LoginManager
7from flask_cors import CORS
8from jinja2 import FileSystemLoader
9from lxml import etree
11from portality import settings, constants, datasets
12from portality.bll import exceptions
13from portality.error_handler import setup_error_logging
14from portality.lib import es_data_mapping
15from portality.ui.debug_toolbar import DoajDebugToolbar
17import esprit
18import elasticsearch
20login_manager = LoginManager()
23@login_manager.user_loader
24def load_account_for_login_manager(userid):
25 """
26 ~~LoginManager:Feature->Account:Model~~
27 :param userid:
28 :return:
29 """
30 from portality import models
31 out = models.Account.pull(userid)
32 return out
35def create_app():
36 """
37 ~~CreateApp:Framework->Flask:Technology~~
38 :return:
39 """
40 app = Flask(__name__)
41 # ~~->AppSettings:Config~~
42 configure_app(app)
43 #~~->ErrorHandler:Feature~~
44 setup_error_logging(app)
45 #~~->Jinja2:Environment~~
46 setup_jinja(app)
47 #~~->CrossrefXML:Feature~~
48 app.config["LOAD_CROSSREF_THREAD"] = threading.Thread(target=load_crossref_schema, args=(app, ), daemon=True)
49 app.config["LOAD_CROSSREF_THREAD"].start()
50 #~~->LoginManager:Feature~~
51 login_manager.init_app(app)
52 #~~->CORS:Framework~~
53 CORS(app)
54 #~~->APM:Feature~~
55 initialise_apm(app)
56 #~~->DebugToolbar:Framework~~
57 DoajDebugToolbar(app)
58 #~~->ProxyFix:Framework~~
59 proxyfix(app)
60 #~~->CMS:Build~~
61 build_statics(app)
62 return app
65##################################################
66# Configure the App
68def configure_app(app):
69 """
70 Configure the DOAJ from:
71 a) the settings.py file
72 b) the <env>.cfg file
73 c) the local secrets config in app.cfg
75 Later imports have precedence, so e.g. app.cfg will override the same setting in production.cfg and settings.py.
76 """
78 # import for settings.py
79 app.config.from_object(settings)
81 # import from <env>.cfg
82 here = os.path.dirname(os.path.abspath(__file__))
83 app.config['DOAJENV'] = get_app_env(app)
84 config_path = os.path.join(os.path.dirname(here), app.config['DOAJENV'] + '.cfg')
85 print('Running in ' + app.config['DOAJENV']) # the app.logger is not set up yet (?)
86 if os.path.exists(config_path):
87 app.config.from_pyfile(config_path)
88 print('Loaded environment config from ' + config_path)
90 # import from app.cfg
91 config_path = os.path.join(os.path.dirname(here), 'app.cfg')
92 if os.path.exists(config_path):
93 app.config.from_pyfile(config_path)
94 print('Loaded secrets config from ' + config_path)
97def get_app_env(app):
98 if not app.config.get('VALID_ENVIRONMENTS'):
99 raise Exception('VALID_ENVIRONMENTS must be set in the config. There shouldn\'t be a reason to change it in different set ups, or not have it.')
101 env = os.getenv('DOAJENV')
102 if not env:
103 envpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../.env')
104 if os.path.exists(envpath):
105 with open(envpath, 'r') as f:
106 env = f.readline().strip()
108 if not env or env not in app.config['VALID_ENVIRONMENTS']:
109 raise Exception(
110"""
111Set the DOAJENV environment variable when running the app, guessing is futile and fraught with peril.
112DOAJENV=test python portality/app.py
113to run the app will do.
114Or use the supervisord options - put this in the config: environment= DOAJENV="test" .
116Finally, you can create a file called .env with the text e.g. 'dev' in the root of the repo.
117Recommended only for dev environments so you don't have to bother specifying it each time you run a script or test.
119Valid values are: {valid_doajenv_vals}
121You can put environment-specific secret settings in <environment>.cfg , e.g. dev.cfg .
123The environment specified in the DOAJENV environment variable will override that specified in the
124application configuration (settings.py or app.cfg).
125""".format(valid_doajenv_vals=', '.join(app.config['VALID_ENVIRONMENTS']))
126 )
127 return env
130################################################
131# Crossref setup
133def load_crossref_schema(app):
134 """
135 ~~CrossrefXML:Feature->CrossrefXML:Schema~~
136 :param app:
137 :return:
138 """
139 schema442_path = app.config["SCHEMAS"].get("crossref442")
140 schema531_path = app.config["SCHEMAS"].get("crossref531")
142 if not app.config.get("CROSSREF442_SCHEMA"):
143 path = schema442_path
144 try:
145 schema_doc = etree.parse(schema442_path)
146 schema = etree.XMLSchema(schema_doc)
147 app.config["CROSSREF442_SCHEMA"] = schema
148 except Exception as e:
149 raise exceptions.IngestException(
150 message="There was an error attempting to load schema from " + path, inner=e)
152 if not app.config.get("CROSSREF531_SCHEMA"):
153 path = schema531_path
154 try:
155 schema_doc = etree.parse(schema531_path)
156 schema = etree.XMLSchema(schema_doc)
157 app.config["CROSSREF531_SCHEMA"] = schema
158 except Exception as e:
159 raise exceptions.IngestException(
160 message="There was an error attempting to load schema from " + path, inner=e)
164############################################
165# Elasticsearch initialisation
167def create_es_connection(app):
168 # ~~ElasticConnection:Framework->Elasticsearch:Technology~~
169 # temporary logging config for debugging index-per-type
170 #import logging
171 #esprit.raw.configure_logging(logging.DEBUG)
173 # FIXME: we are removing esprit conn in favour of elasticsearch lib
174 # make a connection to the index
175 # if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE']:
176 # conn = esprit.raw.Connection(host=app.config['ELASTIC_SEARCH_HOST'], index='')
177 # else:
178 # conn = esprit.raw.Connection(app.config['ELASTIC_SEARCH_HOST'], app.config['ELASTIC_SEARCH_DB'])
180 conn = elasticsearch.Elasticsearch(app.config['ELASTICSEARCH_HOSTS'], verify_certs=app.config.get("ELASTIC_SEARCH_VERIFY_CERTS", True))
182 return conn
184# FIXME: deprecated no longer necessary
185# def mutate_mapping(conn, type, mapping):
186# """ When we are using an index-per-type connection change the mappings to be keyed 'doc' rather than the type """
187# if conn.index_per_type:
188# try:
189# mapping[esprit.raw.INDEX_PER_TYPE_SUBSTITUTE] = mapping.pop(type)
190# except KeyError:
191# # Allow this mapping through unaltered if it isn't keyed by type
192# pass
193#
194# # Add the index prefix to the mapping as we create the type
195# type = app.config['ELASTIC_SEARCH_DB_PREFIX'] + type
196# return type
199def put_mappings(conn, mappings):
200 # get the ES version that we're working with
201 #es_version = app.config.get("ELASTIC_SEARCH_VERSION", "1.7.5")
203 # for each mapping (a class may supply multiple), create a mapping, or mapping and index
204 # for key, mapping in iter(mappings.items()):
205 # altered_key = mutate_mapping(conn, key, mapping)
206 # ix = conn.index or altered_key
207 # if not esprit.raw.type_exists(conn, altered_key, es_version=es_version):
208 # r = esprit.raw.put_mapping(conn, altered_key, mapping, es_version=es_version)
209 # print("Creating ES Type + Mapping in index {0} for {1}; status: {2}".format(ix, key, r.status_code))
210 # else:
211 # print("ES Type + Mapping already exists in index {0} for {1}".format(ix, key))
213 for key, mapping in iter(mappings.items()):
214 altered_key = app.config['ELASTIC_SEARCH_DB_PREFIX'] + key
215 if not conn.indices.exists(altered_key):
216 r = conn.indices.create(index=altered_key, body=mapping)
217 print("Creating ES Type + Mapping in index {0} for {1}; status: {2}".format(altered_key, key, r))
218 else:
219 print("ES Type + Mapping already exists in index {0} for {1}".format(altered_key, key))
222def initialise_index(app, conn, only_mappings=None):
223 """
224 ~~InitialiseIndex:Framework->Elasticsearch:Technology~~
225 :param app:
226 :param conn:
227 :return:
228 """
229 if not app.config['INITIALISE_INDEX']:
230 app.logger.warn('INITIALISE_INDEX config var is not True, initialise_index command cannot run')
231 return
233 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
234 app.logger.warn("System is in READ-ONLY mode, initialise_index command cannot run")
235 return
237 # get the app mappings
238 mappings = es_data_mapping.get_mappings(app)
240 if only_mappings is not None:
241 mappings = {key:value for (key, value) in mappings.items() if key in only_mappings}
243 # Send the mappings to ES
244 put_mappings(conn, mappings)
247##################################################
248# APM
250def initialise_apm(app):
251 """
252 ~~APM:Feature->ElasticAPM:Technology~~
253 :param app:
254 :return:
255 """
256 if app.config.get('ENABLE_APM', False):
257 from elasticapm.contrib.flask import ElasticAPM
258 app.logger.info("Configuring Elastic APM")
259 apm = ElasticAPM(app, logging=True)
262##################################################
263# proxyfix
265def proxyfix(app):
266 """
267 ~~ProxyFix:Framework~~
268 :param app:
269 :return:
270 """
271 if app.config.get('PROXIED', False):
272 from werkzeug.middleware.proxy_fix import ProxyFix
273 app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
276##################################################
277# Jinja2
279def setup_jinja(app):
280 """
281 Jinja2:Environment->Jinja2:Technology
282 :param app:
283 :return:
284 """
285 '''Add jinja extensions and other init-time config as needed.'''
287 app.jinja_env.add_extension('jinja2.ext.do')
288 app.jinja_env.add_extension('jinja2.ext.loopcontrols')
289 app.jinja_env.globals['getattr'] = getattr
290 app.jinja_env.globals['type'] = type
291 #~~->Constants:Config~~
292 app.jinja_env.globals['constants'] = constants
293 #~~->Datasets:Data~~
294 app.jinja_env.globals['datasets'] = datasets
295 _load_data(app)
296 #~~->CMS:DataStore~~
297 app.jinja_env.loader = FileSystemLoader([app.config['BASE_FILE_PATH'] + '/templates',
298 os.path.dirname(app.config['BASE_FILE_PATH']) + '/cms/fragments'])
300 # a jinja filter that prints to the Flask log
301 def jinja_debug(text):
302 print(text)
303 return ''
304 app.jinja_env.filters['debug']=jinja_debug
307def _load_data(app):
308 if not "data" in app.jinja_env.globals:
309 app.jinja_env.globals["data"] = {}
310 datadir = os.path.join(app.config["BASE_FILE_PATH"], "..", "cms", "data")
311 for datafile in os.listdir(datadir):
312 with open(os.path.join(datadir, datafile)) as f:
313 data = yaml.load(f, Loader=yaml.FullLoader)
314 dataname = datafile.split(".")[0]
315 dataname = dataname.replace("-", "_")
316 app.jinja_env.globals["data"][dataname] = data
319##################################################
320# Static Content
322def build_statics(app):
323 """
324 ~~CMS:Build->CMSFragments:Build~~
325 ~~->CMSSASS:Build~~
326 :param app:
327 :return:
328 """
329 if not app.config.get("CMS_BUILD_ASSETS_ON_STARTUP", False):
330 return
331 from portality.cms import build_fragments, build_sass
333 here = os.path.dirname(os.path.abspath(__file__))
334 base_path = os.path.dirname(here)
336 print("Compiling static content")
337 build_fragments.build(base_path)
338 print("Compiling main SASS")
339 build_sass.build(build_sass.MAIN_SETTINGS, base_path=base_path)
342app = create_app()
343es_connection = create_es_connection(app)