Coverage for portality/bll/services/query.py: 84%

223 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1from portality.core import app, es_connection 

2from portality.util import ipt_prefix 

3from portality.bll import exceptions 

4from portality.lib import plugin 

5from portality.dao import ESMappingMissingError 

6from copy import deepcopy 

7 

8class QueryService(object): 

9 """ 

10 ~~Query:Service~~ 

11 """ 

12 def _get_config_for_search(self, domain, index_type, account): 

13 # load the query route config and the path we are being requested for 

14 # ~~-> Query:Config~~ 

15 qrs = app.config.get("QUERY_ROUTE", {}) 

16 

17 # get the configuration for this url route 

18 route_cfg = None 

19 for key in qrs: 

20 if domain == key: 

21 route_cfg = qrs.get(key) 

22 break 

23 

24 if route_cfg is None: 

25 raise exceptions.AuthoriseException(exceptions.AuthoriseException.NOT_AUTHORISED) 

26 

27 cfg = route_cfg.get(index_type) 

28 if cfg is None: 

29 raise exceptions.AuthoriseException(exceptions.AuthoriseException.NOT_AUTHORISED) 

30 

31 # does the user have to be authenticated 

32 if cfg.get("auth", True): 

33 if account is None: 

34 raise exceptions.AuthoriseException(exceptions.AuthoriseException.NOT_AUTHORISED) 

35 

36 # if so, does the user require a role 

37 role = cfg.get("role") 

38 if role is not None and not account.has_role(role): 

39 raise exceptions.AuthoriseException(exceptions.AuthoriseException.WRONG_ROLE) 

40 

41 return cfg 

42 

43 def _validate_query(self, cfg, query): 

44 validators = cfg.get("query_validators") 

45 if validators: 

46 for validator in validators: 

47 filters = app.config.get("QUERY_FILTERS", {}) 

48 validator_path = filters.get(validator) 

49 fn = plugin.load_function(validator_path) 

50 if fn is None: 

51 msg = "Unable to load query validator for {x}".format(x=validator) 

52 raise exceptions.ConfigurationException(msg) 

53 

54 if not fn(query): 

55 return False 

56 return True 

57 

58 

59 def _pre_filter_search_query(self, cfg, query): 

60 # now run the query through the filters 

61 filters = app.config.get("QUERY_FILTERS", {}) 

62 filter_names = cfg.get("query_filters", []) 

63 for filter_name in filter_names: 

64 # because of back-compat, we have to do a few tricky things here... 

65 # filter may be the name of a filter in the list of query filters 

66 fn = plugin.load_function(filters.get(filter_name)) 

67 if fn is None: 

68 msg = "Unable to load query filter for {x}".format(x=filter_name) 

69 raise exceptions.ConfigurationException(msg) 

70 

71 # run the filter 

72 fn(query) 

73 

74 return query 

75 

76 def _post_filter_search_results(self, cfg, res, unpacked=False): 

77 filters = app.config.get("QUERY_FILTERS", {}) 

78 result_filter_names = cfg.get("result_filters", []) 

79 for result_filter_name in result_filter_names: 

80 fn = plugin.load_function(filters.get(result_filter_name)) 

81 if fn is None: 

82 msg = "Unable to load result filter for {x}".format(x=result_filter_name) 

83 raise exceptions.ConfigurationException(msg) 

84 

85 # apply the result filter 

86 res = fn(res, unpacked=unpacked) 

87 

88 return res 

89 

90 def _get_query(self, cfg, raw_query): 

91 query = Query() 

92 if raw_query is not None: 

93 query = Query(raw_query) 

94 

95 # validate the query, to make sure it is of a permitted form 

96 if not self._validate_query(cfg, query): 

97 raise exceptions.AuthoriseException() 

98 

99 # add any required filters to the query 

100 query = self._pre_filter_search_query(cfg, query) 

101 return query 

102 

103 def _get_dao_klass(self, cfg): 

104 # get the name of the model that will handle this query, and then look up 

105 # the class that will handle it 

106 dao_name = cfg.get("dao") 

107 dao_klass = plugin.load_class(dao_name) 

108 if dao_klass is None: 

109 raise exceptions.NoSuchObjectException(dao_name) 

110 return dao_klass 

111 

112 def search(self, domain, index_type, raw_query, account, additional_parameters): 

113 cfg = self._get_config_for_search(domain, index_type, account) 

114 

115 # check that the request values permit a query to this endpoint 

116 required_parameters = cfg.get("required_parameters") 

117 if required_parameters is not None: 

118 for k, vs in required_parameters.items(): 

119 val = additional_parameters.get(k) 

120 if val is None or val not in vs: 

121 raise exceptions.AuthoriseException() 

122 

123 dao_klass = self._get_dao_klass(cfg) 

124 

125 # get the query 

126 query = self._get_query(cfg, raw_query) 

127 

128 # send the query. These are user-facing requests, so retry sparingly: a gunicorn worker can't wait out 

129 # the DAO's default of 50 retries, and mass retries add load to a cluster that's already struggling. 

130 send_kwargs = {"retry": app.config.get("QUERY_SERVICE_ES_RETRIES", 3)} 

131 if cfg.get("request_cache", False): 

132 # use the shard request cache even though size > 0 (by default ES only caches size=0 requests) 

133 send_kwargs["request_cache"] = True 

134 try: 

135 res = dao_klass.query(q=query.as_dict(), send_kwargs=send_kwargs) 

136 except ESMappingMissingError: 

137 if not cfg.get("tolerate_missing_mapping", False): 

138 raise 

139 return {"hits": {"hits": [], "total": {"value": 0}}, "aggregations": {}} 

140 

141 # filter the results as needed 

142 res = self._post_filter_search_results(cfg, res) 

143 

144 return res 

145 

146 def scroll(self, domain, index_type, raw_query, account, page_size, scan=False): 

147 cfg = self._get_config_for_search(domain, index_type, account) 

148 

149 dao_klass = self._get_dao_klass(cfg) 

150 

151 # get the query 

152 query = self._get_query(cfg, raw_query) 

153 

154 # get the scroll parameters 

155 if page_size is None: 

156 page_size = cfg.get("page_size", 1000) 

157 limit = cfg.get("limit", None) 

158 keepalive = cfg.get("keepalive", "1m") 

159 

160 # ~~->Elasticsearch:Technology~~ 

161 for result in dao_klass.iterate(q=query.as_dict(), page_size=page_size, limit=limit, wrap=False, keepalive=keepalive): 

162 res = self._post_filter_search_results(cfg, result, unpacked=True) 

163 yield res 

164 

165 def make_actionable_query(self, domain, index_type, account, raw_query): 

166 cfg = self._get_config_for_search(domain, index_type, account) 

167 query = self._get_query(cfg, raw_query) 

168 return query 

169 

170class Query(object): 

171 """ 

172 ~~Query:Query -> Elasticsearch:Technology~~ 

173 """ 

174 def __init__(self, raw=None, filtered=False): 

175 self.q = {"track_total_hits" : True, "query": {"match_all": {}}} if raw is None else raw 

176 self.filtered = filtered is True or self.q.get("query", {}).get("filtered") is not None 

177 if self.filtered: 

178 # FIXME: this is just to help us catch filtered queries during development. Once we have them 

179 # all, all the filtering logic in this class can come out 

180 raise Exception("Filtered queries are no longer supported") 

181 

182 def convert_to_bool(self): 

183 if self.filtered is True: 

184 return 

185 

186 current_query = None 

187 if "query" in self.q: 

188 if "bool" in self.q["query"]: 

189 return 

190 current_query = deepcopy(self.q["query"]) 

191 del self.q["query"] 

192 if len(list(current_query.keys())) == 0: 

193 current_query = None 

194 

195 if "query" not in self.q: 

196 self.q["query"] = {} 

197 if "bool" not in self.q["query"]: 

198 self.q["query"]["bool"] = {} 

199 

200 if current_query is not None: 

201 if "must" not in self.q["query"]["bool"]: 

202 self.q["query"]["bool"]["must"] = [] 

203 

204 self.q["query"]["bool"]["must"].append(current_query) 

205 

206 def add_must(self, filter): 

207 # self.convert_to_filtered() 

208 self.convert_to_bool() 

209 context = self.q["query"]["bool"] 

210 if "must" not in context: 

211 context["must"] = [] 

212 context["must"].append(filter) 

213 

214 def get_field_context(self): 

215 """Get query string context""" 

216 context = None 

217 if "query_string" in self.q["query"]: 

218 context = self.q["query"]["query_string"] 

219 

220 elif "bool" in self.q["query"]: 

221 if "must" in self.q["query"]["bool"]: 

222 context = self.q["query"]["bool"]["must"] 

223 return context 

224 

225 def add_default_field(self, value: str): 

226 """ Add a default field to the query string, if one is not already present""" 

227 context = self.get_field_context() 

228 

229 if context: 

230 if isinstance(context, dict): 

231 if "default_field" not in context: 

232 context["default_field"] = value 

233 elif isinstance(context, list): 

234 for item in context: 

235 if "query_string" in item: 

236 if "default_field" not in item["query_string"]: 

237 item["query_string"]["default_field"] = value 

238 break 

239 

240 def add_should(self, filter, minimum_should_match=1): 

241 self.convert_to_bool() 

242 context = self.q["query"]["bool"] 

243 if "should" not in context: 

244 context["should"] = [] 

245 if isinstance(filter, list): 

246 context["should"].extend(filter) 

247 else: 

248 context["should"].append(filter) 

249 context["minimum_should_match"] = minimum_should_match 

250 

251 def add_must_filter(self, filter): 

252 self.convert_to_bool() 

253 context = self.q["query"]["bool"] 

254 if "filter" not in context: 

255 context["filter"] = [] 

256 

257 context["filter"].append(filter) 

258 

259 def add_must_not(self, filter): 

260 self.convert_to_bool() 

261 context = self.q["query"]["bool"] 

262 if "must_not" not in context: 

263 context["must_not"] = [] 

264 

265 context["must_not"].append(filter) 

266 

267 def clear_match_all(self): 

268 if "match_all" in self.q["query"]: 

269 del self.q["query"]["match_all"] 

270 

271 def has_facets(self): 

272 return "facets" in self.q or "aggregations" in self.q or "aggs" in self.q 

273 

274 def clear_facets(self): 

275 if "facets" in self.q: 

276 del self.q["facets"] 

277 if "aggregations" in self.q: 

278 del self.q["aggregations"] 

279 if "aggs" in self.q: 

280 del self.q["aggs"] 

281 

282 def size(self): 

283 if "size" in self.q: 

284 try: 

285 return int(self.q["size"]) 

286 except ValueError: 

287 app.logger.warn("Invalid size parameter in query: [{x}], " 

288 "expected integer value".format(x=self.q["size"])) 

289 return 10 

290 

291 def from_result(self): 

292 if "from" in self.q: 

293 try: 

294 return int(self.q["from"]) 

295 except ValueError: 

296 app.logger.warn("Invalid from parameter in query: [{x}], " 

297 "expected integer value".format(x=self.q["from"])) 

298 return 0 

299 

300 def as_dict(self): 

301 return self.q 

302 

303 def add_include(self, fields): 

304 if "_source" not in self.q: 

305 self.q["_source"] = {} 

306 if "includes" not in self.q["_source"]: 

307 self.q["_source"]["includes"] = [] 

308 if not isinstance(fields, list): 

309 fields = [fields] 

310 self.q["_source"]["includes"] = list(set(self.q["_source"]["includes"] + fields)) 

311 

312 def sort(self): 

313 return self.q.get("sort") 

314 

315 def set_sort(self, s): 

316 self.q["sort"] = s 

317 

318 

319class QueryFilterException(Exception): 

320 pass