Coverage for portality/bll/services/query.py: 84%
223 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1from portality.core import app, es_connection
2from portality.util import ipt_prefix
3from portality.bll import exceptions
4from portality.lib import plugin
5from portality.dao import ESMappingMissingError
6from copy import deepcopy
8class QueryService(object):
9 """
10 ~~Query:Service~~
11 """
12 def _get_config_for_search(self, domain, index_type, account):
13 # load the query route config and the path we are being requested for
14 # ~~-> Query:Config~~
15 qrs = app.config.get("QUERY_ROUTE", {})
17 # get the configuration for this url route
18 route_cfg = None
19 for key in qrs:
20 if domain == key:
21 route_cfg = qrs.get(key)
22 break
24 if route_cfg is None:
25 raise exceptions.AuthoriseException(exceptions.AuthoriseException.NOT_AUTHORISED)
27 cfg = route_cfg.get(index_type)
28 if cfg is None:
29 raise exceptions.AuthoriseException(exceptions.AuthoriseException.NOT_AUTHORISED)
31 # does the user have to be authenticated
32 if cfg.get("auth", True):
33 if account is None:
34 raise exceptions.AuthoriseException(exceptions.AuthoriseException.NOT_AUTHORISED)
36 # if so, does the user require a role
37 role = cfg.get("role")
38 if role is not None and not account.has_role(role):
39 raise exceptions.AuthoriseException(exceptions.AuthoriseException.WRONG_ROLE)
41 return cfg
43 def _validate_query(self, cfg, query):
44 validators = cfg.get("query_validators")
45 if validators:
46 for validator in validators:
47 filters = app.config.get("QUERY_FILTERS", {})
48 validator_path = filters.get(validator)
49 fn = plugin.load_function(validator_path)
50 if fn is None:
51 msg = "Unable to load query validator for {x}".format(x=validator)
52 raise exceptions.ConfigurationException(msg)
54 if not fn(query):
55 return False
56 return True
59 def _pre_filter_search_query(self, cfg, query):
60 # now run the query through the filters
61 filters = app.config.get("QUERY_FILTERS", {})
62 filter_names = cfg.get("query_filters", [])
63 for filter_name in filter_names:
64 # because of back-compat, we have to do a few tricky things here...
65 # filter may be the name of a filter in the list of query filters
66 fn = plugin.load_function(filters.get(filter_name))
67 if fn is None:
68 msg = "Unable to load query filter for {x}".format(x=filter_name)
69 raise exceptions.ConfigurationException(msg)
71 # run the filter
72 fn(query)
74 return query
76 def _post_filter_search_results(self, cfg, res, unpacked=False):
77 filters = app.config.get("QUERY_FILTERS", {})
78 result_filter_names = cfg.get("result_filters", [])
79 for result_filter_name in result_filter_names:
80 fn = plugin.load_function(filters.get(result_filter_name))
81 if fn is None:
82 msg = "Unable to load result filter for {x}".format(x=result_filter_name)
83 raise exceptions.ConfigurationException(msg)
85 # apply the result filter
86 res = fn(res, unpacked=unpacked)
88 return res
90 def _get_query(self, cfg, raw_query):
91 query = Query()
92 if raw_query is not None:
93 query = Query(raw_query)
95 # validate the query, to make sure it is of a permitted form
96 if not self._validate_query(cfg, query):
97 raise exceptions.AuthoriseException()
99 # add any required filters to the query
100 query = self._pre_filter_search_query(cfg, query)
101 return query
103 def _get_dao_klass(self, cfg):
104 # get the name of the model that will handle this query, and then look up
105 # the class that will handle it
106 dao_name = cfg.get("dao")
107 dao_klass = plugin.load_class(dao_name)
108 if dao_klass is None:
109 raise exceptions.NoSuchObjectException(dao_name)
110 return dao_klass
112 def search(self, domain, index_type, raw_query, account, additional_parameters):
113 cfg = self._get_config_for_search(domain, index_type, account)
115 # check that the request values permit a query to this endpoint
116 required_parameters = cfg.get("required_parameters")
117 if required_parameters is not None:
118 for k, vs in required_parameters.items():
119 val = additional_parameters.get(k)
120 if val is None or val not in vs:
121 raise exceptions.AuthoriseException()
123 dao_klass = self._get_dao_klass(cfg)
125 # get the query
126 query = self._get_query(cfg, raw_query)
128 # send the query. These are user-facing requests, so retry sparingly: a gunicorn worker can't wait out
129 # the DAO's default of 50 retries, and mass retries add load to a cluster that's already struggling.
130 send_kwargs = {"retry": app.config.get("QUERY_SERVICE_ES_RETRIES", 3)}
131 if cfg.get("request_cache", False):
132 # use the shard request cache even though size > 0 (by default ES only caches size=0 requests)
133 send_kwargs["request_cache"] = True
134 try:
135 res = dao_klass.query(q=query.as_dict(), send_kwargs=send_kwargs)
136 except ESMappingMissingError:
137 if not cfg.get("tolerate_missing_mapping", False):
138 raise
139 return {"hits": {"hits": [], "total": {"value": 0}}, "aggregations": {}}
141 # filter the results as needed
142 res = self._post_filter_search_results(cfg, res)
144 return res
146 def scroll(self, domain, index_type, raw_query, account, page_size, scan=False):
147 cfg = self._get_config_for_search(domain, index_type, account)
149 dao_klass = self._get_dao_klass(cfg)
151 # get the query
152 query = self._get_query(cfg, raw_query)
154 # get the scroll parameters
155 if page_size is None:
156 page_size = cfg.get("page_size", 1000)
157 limit = cfg.get("limit", None)
158 keepalive = cfg.get("keepalive", "1m")
160 # ~~->Elasticsearch:Technology~~
161 for result in dao_klass.iterate(q=query.as_dict(), page_size=page_size, limit=limit, wrap=False, keepalive=keepalive):
162 res = self._post_filter_search_results(cfg, result, unpacked=True)
163 yield res
165 def make_actionable_query(self, domain, index_type, account, raw_query):
166 cfg = self._get_config_for_search(domain, index_type, account)
167 query = self._get_query(cfg, raw_query)
168 return query
170class Query(object):
171 """
172 ~~Query:Query -> Elasticsearch:Technology~~
173 """
174 def __init__(self, raw=None, filtered=False):
175 self.q = {"track_total_hits" : True, "query": {"match_all": {}}} if raw is None else raw
176 self.filtered = filtered is True or self.q.get("query", {}).get("filtered") is not None
177 if self.filtered:
178 # FIXME: this is just to help us catch filtered queries during development. Once we have them
179 # all, all the filtering logic in this class can come out
180 raise Exception("Filtered queries are no longer supported")
182 def convert_to_bool(self):
183 if self.filtered is True:
184 return
186 current_query = None
187 if "query" in self.q:
188 if "bool" in self.q["query"]:
189 return
190 current_query = deepcopy(self.q["query"])
191 del self.q["query"]
192 if len(list(current_query.keys())) == 0:
193 current_query = None
195 if "query" not in self.q:
196 self.q["query"] = {}
197 if "bool" not in self.q["query"]:
198 self.q["query"]["bool"] = {}
200 if current_query is not None:
201 if "must" not in self.q["query"]["bool"]:
202 self.q["query"]["bool"]["must"] = []
204 self.q["query"]["bool"]["must"].append(current_query)
206 def add_must(self, filter):
207 # self.convert_to_filtered()
208 self.convert_to_bool()
209 context = self.q["query"]["bool"]
210 if "must" not in context:
211 context["must"] = []
212 context["must"].append(filter)
214 def get_field_context(self):
215 """Get query string context"""
216 context = None
217 if "query_string" in self.q["query"]:
218 context = self.q["query"]["query_string"]
220 elif "bool" in self.q["query"]:
221 if "must" in self.q["query"]["bool"]:
222 context = self.q["query"]["bool"]["must"]
223 return context
225 def add_default_field(self, value: str):
226 """ Add a default field to the query string, if one is not already present"""
227 context = self.get_field_context()
229 if context:
230 if isinstance(context, dict):
231 if "default_field" not in context:
232 context["default_field"] = value
233 elif isinstance(context, list):
234 for item in context:
235 if "query_string" in item:
236 if "default_field" not in item["query_string"]:
237 item["query_string"]["default_field"] = value
238 break
240 def add_should(self, filter, minimum_should_match=1):
241 self.convert_to_bool()
242 context = self.q["query"]["bool"]
243 if "should" not in context:
244 context["should"] = []
245 if isinstance(filter, list):
246 context["should"].extend(filter)
247 else:
248 context["should"].append(filter)
249 context["minimum_should_match"] = minimum_should_match
251 def add_must_filter(self, filter):
252 self.convert_to_bool()
253 context = self.q["query"]["bool"]
254 if "filter" not in context:
255 context["filter"] = []
257 context["filter"].append(filter)
259 def add_must_not(self, filter):
260 self.convert_to_bool()
261 context = self.q["query"]["bool"]
262 if "must_not" not in context:
263 context["must_not"] = []
265 context["must_not"].append(filter)
267 def clear_match_all(self):
268 if "match_all" in self.q["query"]:
269 del self.q["query"]["match_all"]
271 def has_facets(self):
272 return "facets" in self.q or "aggregations" in self.q or "aggs" in self.q
274 def clear_facets(self):
275 if "facets" in self.q:
276 del self.q["facets"]
277 if "aggregations" in self.q:
278 del self.q["aggregations"]
279 if "aggs" in self.q:
280 del self.q["aggs"]
282 def size(self):
283 if "size" in self.q:
284 try:
285 return int(self.q["size"])
286 except ValueError:
287 app.logger.warn("Invalid size parameter in query: [{x}], "
288 "expected integer value".format(x=self.q["size"]))
289 return 10
291 def from_result(self):
292 if "from" in self.q:
293 try:
294 return int(self.q["from"])
295 except ValueError:
296 app.logger.warn("Invalid from parameter in query: [{x}], "
297 "expected integer value".format(x=self.q["from"]))
298 return 0
300 def as_dict(self):
301 return self.q
303 def add_include(self, fields):
304 if "_source" not in self.q:
305 self.q["_source"] = {}
306 if "includes" not in self.q["_source"]:
307 self.q["_source"]["includes"] = []
308 if not isinstance(fields, list):
309 fields = [fields]
310 self.q["_source"]["includes"] = list(set(self.q["_source"]["includes"] + fields))
312 def sort(self):
313 return self.q.get("sort")
315 def set_sort(self, s):
316 self.q["sort"] = s
319class QueryFilterException(Exception):
320 pass