Coverage for portality/dao.py: 69%
930 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1from __future__ import annotations
3import json
4import os
5import re
6import sys
7import time
8import urllib.parse
9import uuid
10from collections import UserDict
11from copy import deepcopy
12from datetime import timedelta
13from typing import List, Iterable, Tuple
15import elasticsearch
16from elasticsearch.exceptions import ConnectionTimeout, TransportError
18from portality.core import app, es_connection as ES
19from portality.lib import dates
20from portality.lib.dates import FMT_DATETIME_STD
22# All models in models.py should inherit this DomainObject to know how to save themselves in the index and so on.
23# You can overwrite and add to the DomainObject functions as required. See models.py for some examples.
26ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_\.]+?\] in order to sort on.*', re.DOTALL)
27CONTENT_TYPE_JSON = {'Content-Type': 'application/json'}
30def is_client_error(e: Exception) -> bool:
31 """True if ES rejected the request itself (4xx), so retrying it cannot succeed. 429 is excluded, since it
32 means 'too many requests' and may succeed later."""
33 status = getattr(e, "status_code", None)
34 return isinstance(e, TransportError) and isinstance(status, int) and 400 <= status < 500 and status != 429
37class ElasticSearchWriteException(Exception):
38 pass
41class ScrollException(Exception):
42 pass
45class ScrollInitialiseException(ScrollException):
46 pass
49class ScrollTimeoutException(ScrollException):
50 pass
53class BulkException(Exception):
54 pass
57class DomainObject(UserDict, object):
58 """
59 ~~DomainObject:Model->Elasticsearch:Technology~~
61 base models class for Elasticsearch(ES) index,
62 which provide interaction with ES index such as (save, delete, query, etc.)
63 """
65 # set the type on the model that inherits this# which also is ES index name of the model
66 __type__ = None
68 def __init__(self, **kwargs):
69 # if self.data is already set, don't do anything here
70 try:
71 object.__getattribute__(self, "data")
72 except:
73 if '_source' in kwargs:
74 self.data = dict(kwargs['_source'])
75 self.meta = dict(kwargs)
76 del self.meta['_source']
77 else:
78 self.data = dict(kwargs)
79 # FIXME: calling super() breaks everything, even thought this is the correct thing to do
80 # this is because the DomainObject incorrectly overrides properties of the super class
81 # super(DomainObject, self).__init__()
83 @classmethod
84 def index_name(cls, override_index_name=None, **kwargs):
85 if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE'] and cls.__type__ is not None:
86 if override_index_name is not None:
87 name = app.config['ELASTIC_SEARCH_DB_PREFIX'] + override_index_name
88 else:
89 name = ','.join([app.config['ELASTIC_SEARCH_DB_PREFIX'] + t for t in cls.__type__.split(',')])
90 else:
91 if override_index_name is not None:
92 name = override_index_name
93 else:
94 name = app.config['ELASTIC_SEARCH_DB']
95 return name
97 @classmethod
98 def doc_type(cls):
99 if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE']:
100 return None
101 else:
102 return cls.__type__
104 @classmethod
105 def makeid(cls):
106 """Create a new id for data object overwrite this in specific model types if required"""
107 return str(uuid.uuid4().hex)
109 @property
110 def id(self):
111 rawid = self.data.get("id", None)
112 if rawid is not None:
113 return str(rawid)
114 return None
116 def set_id(self, id=None):
117 if id is None:
118 id = self.makeid()
119 self.data["id"] = str(id)
121 @property
122 def version(self):
123 return self.meta.get('_version', None)
125 @property
126 def json(self):
127 return json.dumps(self.data)
129 def set_created(self, date=None):
130 if date is None:
131 self.data['created_date'] = dates.now_str()
132 else:
133 self.data['created_date'] = date
135 @property
136 def created_date(self):
137 return self.data.get("created_date")
139 @property
140 def created_timestamp(self):
141 return dates.parse(self.data.get("created_date"))
143 @property
144 def last_updated(self):
145 return self.data.get("last_updated")
147 @property
148 def last_updated_timestamp(self):
149 lu = self.last_updated
150 if lu is None:
151 return None
152 return dates.parse(self.last_updated)
154 def pre_save_prep(self, blocking=False, differentiate=False, update_last_updated=True):
155 if 'id' not in self.data:
156 self.data['id'] = self.makeid()
158 self.data['es_type'] = self.__type__
160 now = dates.now_str()
161 if (blocking or differentiate) and "last_updated" in self.data:
162 diff = dates.now() - dates.parse(self.data["last_updated"])
164 # we need the new last_updated time to be later than the new one
165 if diff.total_seconds() < 1:
166 soon = dates.now() + timedelta(seconds=1)
167 now = soon.strftime(FMT_DATETIME_STD)
169 if update_last_updated:
170 self.data['last_updated'] = now
172 if 'created_date' not in self.data:
173 self.data['created_date'] = now
175 def save(self, retries=0, back_off_factor=1, differentiate=False, blocking=False, block_wait=0.25, update_last_updated=True):
176 """
177 ~~->ReadOnlyMode:Feature~~
178 :param retries:
179 :param back_off_factor:
180 :param differentiate:
181 :param blocking:
182 :return:
183 """
184 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
185 app.logger.warning("System is in READ-ONLY mode, save command cannot run")
186 return
188 if retries > app.config.get("ES_RETRY_HARD_LIMIT", 1000): # an arbitrary large number
189 retries = app.config.get("ES_RETRY_HARD_LIMIT", 1000)
191 if app.config.get("ES_BLOCK_WAIT_OVERRIDE") is not None:
192 block_wait = app.config["ES_BLOCK_WAIT_OVERRIDE"]
194 self.pre_save_prep(blocking=blocking, differentiate=differentiate, update_last_updated=update_last_updated)
195 now = self.data.get("last_updated", dates.now_str())
197 attempt = 0
198 d = json.dumps(self.data)
199 r = None
200 while attempt <= retries:
201 try:
202 r = ES.index(self.index_name(), d, doc_type=self.doc_type(),
203 id=self.data.get("id"),
204 headers=CONTENT_TYPE_JSON,
205 timeout=app.config.get('ES_READ_TIMEOUT', None), )
206 break
208 except (elasticsearch.ConnectionError, elasticsearch.ConnectionTimeout):
209 app.logger.exception("Failed to connect to ES")
210 attempt += 1
212 except elasticsearch.TransportError as e:
213 # Retries depend on which end the error lies.
214 if 400 <= e.status_code < 500:
215 # Bad request, do not retry as it won't work. Fail with ElasticSearchWriteException.
216 app.logger.exception("Bad Request to ES, save failed. Details: {0}".format(e.error))
217 raise ElasticSearchWriteException(e.error)
218 elif e.status_code >= 500:
219 # Server error, this could be temporary so we may want to retry
220 app.logger.exception("Server Error from ES, retrying. Details: {0}".format(e.error))
221 attempt += 1
222 except Exception as e:
223 # if any other exception occurs, make sure it's at least logged.
224 app.logger.exception("Unhandled exception in save method of DAO")
225 raise ElasticSearchWriteException(e)
227 # wait before retrying
228 time.sleep((2 ** attempt) * back_off_factor)
230 if attempt > retries:
231 raise DAOSaveExceptionMaxRetriesReached(
232 "After {attempts} attempts the record with "
233 "id {id} failed to save.".format(
234 attempts=attempt, id=self.data['id']))
236 if blocking:
237 self.__class__.block(self.id, last_updated=now, sleep=block_wait)
239 return r
241 def delete(self):
242 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
243 app.logger.warning("System is in READ-ONLY mode, delete command cannot run")
244 return
246 # r = requests.delete(self.target() + self.id)
247 try:
248 ES.delete(self.index_name(), self.id, doc_type=self.doc_type())
249 except elasticsearch.NotFoundError:
250 pass # This is to preserve the old behaviour
252 @staticmethod
253 def make_query(theq=None, should_terms=None, consistent_order=True, **kwargs):
254 """
255 Generate a query object based on parameters but don't send to
256 backend - return it instead. Must always have the same
257 parameters as the query method. See query method for explanation
258 of parameters.
259 """
260 if theq is None:
261 theq = ""
262 q = deepcopy(theq)
263 if isinstance(q, dict):
264 query = q
265 if 'bool' not in query['query']:
266 boolean = {'bool': {'must': []}}
267 boolean['bool']['must'].append(query['query'])
268 query['query'] = boolean
269 if 'must' not in query['query']['bool']:
270 query['query']['bool']['must'] = []
271 elif q:
272 query = {
273 'query': {
274 'bool': {
275 'must': [
276 {'query_string': {'query': q}}
277 ]
278 }
279 }
280 }
281 else:
282 query = {
283 'query': {
284 'bool': {
285 'must': [
286 {'match_all': {}}
287 ]
288 }
289 }
290 }
292 if should_terms is not None and len(should_terms) > 0:
293 for s in should_terms:
294 if not isinstance(should_terms[s], list):
295 should_terms[s] = [should_terms[s]]
296 query["query"]["bool"]["must"].append({"terms": {s: should_terms[s]}})
298 sort_specified = False
299 for k, v in kwargs.items():
300 if k == '_from':
301 query['from'] = v
302 elif k == 'sort':
303 sort_specified = True
304 query['sort'] = v
305 else:
306 query[k] = v
307 if "sort" in query:
308 sort_specified = True
310 if not sort_specified and consistent_order:
311 # FIXME: review this - where is default sort necessary, and why do we want this in ID order?
312 query['sort'] = [{"id.exact": {"order": "asc", "unmapped_type": "keyword"}}]
314 return query
316 @classmethod
317 def _unwrap_search_result(cls, res):
318 return [i.get("_source") if "_source" in i else i.get("fields") for i in
319 res.get('hits', {}).get('hits', [])]
321 @classmethod
322 def bulk_delete(cls, id_list, idkey='id', refresh=False):
323 return cls.bulk(documents=[{'id': i} for i in id_list], idkey=idkey, refresh=refresh, action='delete')
325 @classmethod
326 def bulk(cls, documents: List[dict], idkey='id', refresh=False, action='index', req_timeout=10, override_index_name=None, **kwargs):
327 """
328 :param documents: a list of objects to perform bulk actions on (list of dicts)
329 :param idkey: The path to extract an ID from the object, e.g. 'id', 'identifiers.id'
330 :param refresh: Refresh the index in each operation (make immediately available for search) - expensive!
331 :param req_timeout: Request timeout for bulk operation
332 :param kwargs: kwargs are passed into the bulk instruction for each record
333 """
334 # ~~->ReadOnlyMode:Feature~~
335 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
336 app.logger.warning("System is in READ-ONLY mode, bulk command cannot run")
337 return
339 if action not in ['index', 'update', 'delete']:
340 raise Exception("Unrecognised bulk action '{0}'".format(action))
342 data = ''
343 for d in documents:
344 data += cls.to_bulk_single_rec(d, idkey=idkey, action=action, **kwargs)
345 resp = ES.bulk(body=data, index=cls.index_name(override_index_name), doc_type=cls.doc_type(), refresh=refresh,
346 request_timeout=req_timeout)
347 return resp
349 @staticmethod
350 def to_bulk_single_rec(record, idkey="id", action="index", **kwargs):
351 """ Adapted from esprit. Create a bulk instruction from a single record. """
352 data = ''
353 idpath = idkey.split(".")
355 # traverse down the object in search of the supplied ID key
356 context = record
357 for pathseg in idpath:
358 if pathseg in context:
359 context = context[pathseg]
360 else:
361 raise BulkException(
362 "'{0}' not available in record to generate bulk _id: {1}".format(idkey, json.dumps(record)))
364 datadict = {action: {'_id': context}}
365 datadict[action].update(kwargs)
367 data += json.dumps(datadict) + '\n'
369 if action == 'delete':
370 return data
372 # For update, we wrap the document in {doc: document} if not already supplied
373 if action == 'update' and not (record.get('doc') and len(record.keys()) == 1):
374 data += json.dumps({'doc': record}) + '\n'
375 else:
376 data += json.dumps(record) + '\n'
377 return data
379 @classmethod
380 def refresh(cls):
381 """
382 ~~->ReadOnlyMode:Feature~~
383 :return:
384 """
385 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
386 app.logger.warning("System is in READ-ONLY mode, refresh command cannot run")
387 return
389 # r = requests.post(cls.target() + '_refresh', headers=CONTENT_TYPE_JSON)
390 # return r.json()
392 return ES.indices.refresh(index=cls.index_name())
394 @classmethod
395 def pull(cls, id_):
396 """Retrieve object by id."""
397 if id_ is None or id_ == '':
398 return None
400 try:
401 # out = requests.get(cls.target() + id_)
402 out = ES.get(cls.index_name(), id_, doc_type=cls.doc_type())
403 except elasticsearch.NotFoundError:
404 return None
405 except elasticsearch.TransportError as e:
406 raise Exception("ES returned an error: {x}".format(x=e.info))
407 except Exception as e:
408 app.logger.exception(f"Unexpected error in pull(): {e}")
409 return None
410 if out is None:
411 return None
413 return cls(**out)
415 @classmethod
416 def pull_by_key(cls, key, value):
417 res = cls.query(q={"query": {"term": {key + app.config['FACET_FIELD']: value}}})
418 if res.get('hits', {}).get('total', {}).get('value', 0) == 1:
419 return cls.pull(res['hits']['hits'][0]['_source']['id'])
420 else:
421 return None
423 @classmethod
424 def object_query(cls, q=None, **kwargs):
425 result = cls.query(q, **kwargs)
426 return [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])]
428 @classmethod
429 def query(cls, q=None, send_kwargs=None, **kwargs):
430 """Perform a query on backend.
432 :param q: maps to query_string parameter if string, or query dict if dict.
433 :param send_kwargs: keyword args for send_query, e.g. retry, request_cache
434 :param kwargs: any keyword args as per
435 http://www.elasticsearch.org/guide/reference/api/search/uri-request.html
436 """
437 query = cls.make_query(q, **kwargs)
438 return cls.send_query(query, **(send_kwargs or {}))
440 @classmethod
441 def send_query(cls, qobj, retry=50, pit_query=False, **kwargs):
442 """Actually send a query object to the backend.
443 :param kwargs are passed directly to Elasticsearch search() function
444 """
446 if retry > app.config.get("ES_RETRY_HARD_LIMIT", 1000) + 1: # an arbitrary large number
447 retry = app.config.get("ES_RETRY_HARD_LIMIT", 1000) + 1
449 r = None
450 count = 0
451 exception = None
452 while count < retry:
453 count += 1
454 try:
455 # ES 7.10 updated target to whole index, since specifying type for search is deprecated
456 # r = requests.post(cls.target_whole_index() + recid + "_search", data=json.dumps(qobj), headers=CONTENT_TYPE_JSON)
457 if kwargs.get('timeout') is None:
458 kwargs['timeout'] = app.config.get('ES_READ_TIMEOUT', None)
459 index = cls.index_name() if pit_query is False else None
460 r = ES.search(body=json.dumps(qobj), index=index, doc_type=cls.doc_type(),
461 headers=CONTENT_TYPE_JSON, **kwargs)
462 break
463 except ConnectionTimeout as e:
464 continue # retry on timeout
465 except Exception as e:
466 try:
467 exception = ESMappingMissingError(e) if ES_MAPPING_MISSING_REGEX.match(json.dumps(e.args[2])) else e
468 if isinstance(exception, ESMappingMissingError):
469 raise exception
470 except TypeError:
471 raise e
472 if is_client_error(e):
473 # 4xx (e.g. a query_string parse error): the request itself is bad, so retrying can't succeed
474 raise e
476 time.sleep(0.5)
478 if r is not None:
479 return r
480 if exception is not None:
481 raise exception
482 raise Exception("Couldn't get the ES query endpoint to respond. Also, you shouldn't be seeing this.")
484 @classmethod
485 def remove_by_id(cls, id):
486 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
487 app.logger.warning("System is in READ-ONLY mode, delete_by_id command cannot run")
488 return
490 # r = requests.delete(cls.target() + id)
491 try:
492 ES.delete(cls.index_name(), id)
493 except elasticsearch.NotFoundError:
494 return
496 @classmethod
497 def delete_by_query(cls, query):
498 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
499 app.logger.warning("System is in READ-ONLY mode, delete_by_query command cannot run")
500 return
502 # r = requests.delete(cls.target() + "_query", data=json.dumps(query))
503 # return r
504 return ES.delete_by_query(cls.index_name(), json.dumps(query), doc_type=cls.doc_type())
506 @classmethod
507 def destroy_index(cls, **es_kwargs):
508 default_es_kwargs = {
509 'timeout': '5m',
510 'request_timeout': 1001,
511 }
512 default_es_kwargs.update(es_kwargs)
513 es_kwargs = default_es_kwargs
515 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False):
516 app.logger.warning("System is in READ-ONLY mode, destroy_index command cannot run")
517 return
519 # if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE']:
520 # return esprit.raw.delete_index_by_prefix(es_connection, app.config['ELASTIC_SEARCH_DB_PREFIX'])
521 # else:
522 # return esprit.raw.delete_index(es_connection)
523 print('Destroying indexes with prefix ' + app.config['ELASTIC_SEARCH_DB_PREFIX'] + '*')
524 return ES.indices.delete(app.config['ELASTIC_SEARCH_DB_PREFIX'] + '*', **es_kwargs)
526 @classmethod
527 def check_es_raw_response(cls, res, extra_trace_info=''):
528 if 'error' in res:
529 es_resp = json.dumps(res, indent=2)
531 error_to_raise = ESMappingMissingError if ES_MAPPING_MISSING_REGEX.match(es_resp) else ESError
533 raise error_to_raise(
534 (
535 "Elasticsearch returned an error:"
536 "\nES HTTP Response status: {es_status}"
537 "\nES Response:{es_resp}"
538 .format(es_status=res.get('status', 'unknown'), es_resp=es_resp)
539 ) + extra_trace_info
540 )
542 if 'hits' not in res and 'hits' not in res['hits']: # i.e. if res['hits']['hits'] does not exist
543 raise ESResponseCannotBeInterpreted(
544 (
545 "Elasticsearch did not return any records. "
546 "It probably returned an error we did not understand instead."
547 "\nES HTTP Response status: {es_status}"
548 "\nES Response:{es_resp}\n"
549 .format(es_status=res.get('status', 'unknown'), es_resp=json.dumps(res, indent=2))
550 ) + extra_trace_info
551 )
552 return True
554 @classmethod
555 def handle_es_raw_response(cls, res, wrap, extra_trace_info=''):
556 """
557 Handles the JSON returned by ES, raising errors as needed. If no problems are detected it returns its input
558 unchanged.
560 :param res: The full ES raw response to a query in a Python dict (this method does not handle the raw JSON ES
561 outputs). Usually this parameter is the return value of the .query or .send_query methods.
562 :param wrap: Did the caller request wrapping of each ES record inside a model object? This matters for handling
563 records that have no '_source' or 'fields' keys, but do have an '_id' key. Such records should raise an error
564 if wrapping was requested, since there is nothing to wrap. If no wrapping was requested, perhaps the caller
565 simply needed the object IDs and nothing else, so we do not need to raise an error.
566 :param extra_trace_info: A string with additional diagnostic information to be put into exceptions.
567 """
569 cls.check_es_raw_response(res)
571 rs = []
572 for i, each in enumerate(res['hits']['hits']):
573 if '_source' in each:
574 rs.append(each['_source'])
575 elif 'fields' in each:
576 rs.append(each['fields'])
577 elif '_id' in each and not wrap:
578 # "_id" is a sibling (not child) of "_source" so it can only be used with unwrapped raw responses.
579 # wrap = True only makes sense if "_source" or "fields" were returned.
580 # So, if "_id" is the only one present, extract it into an object that's shaped the same as the item
581 # in the raw response.
582 rs.append({"_id": each['_id']})
583 else:
584 msg1 = "Can't find any useful data in the ES response.\n" + extra_trace_info
585 msg2 = "\nItem {i}.\nItem data:\n{each}".format(i=i, each=json.dumps(each, indent=2))
586 raise ESResponseCannotBeInterpreted(msg1 + msg2)
588 return rs
590 @classmethod
591 def iterate(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True,
592 keepalive: str = '1m'):
593 """ Provide an iterable of all items in a model, use
594 :param q: The query to scroll results on
595 :param page_size: limited by ElasticSearch, check settings to override
596 :param limit: Limit the number of results returned (e.g. to take a slice)
597 :param wrap: Whether to return the results in raw json or wrapped as an object
598 :param keepalive: scroll timeout
600 TODO: this is the old method, we should evaluate and aim to make iterate_pit the default scroll method.
601 """
602 theq = {"query": {"match_all": {}}} if q is None else deepcopy(q)
603 theq["size"] = page_size
604 theq["from"] = 0
605 if "sort" not in theq:
606 # This gives the same performance enhancement as scan, use it by default. This is the order of indexing like sort by ID
607 theq["sort"] = ["_doc"]
609 # Initialise the scroll
610 try:
611 res = cls.send_query(theq, scroll=keepalive)
612 except Exception as e:
613 raise ScrollInitialiseException("Unable to initialise scroll - could be your mappings are broken", e)
615 # unpack scroll response
616 scroll_id = res.get('_scroll_id')
617 total_results = res.get('hits', {}).get('total', {}).get('value')
619 # Supply the first set of results
620 counter = 0
621 for r in cls.handle_es_raw_response(
622 res,
623 wrap=wrap,
624 extra_trace_info=
625 "\nScroll initialised:\n{q}\n"
626 "\n\nPage #{counter} of the ES response with size {page_size}."
627 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)):
629 # apply the limit
630 if limit is not None and counter >= int(limit):
631 break
632 counter += 1
633 if wrap:
634 yield cls(**r)
635 else:
636 yield r
638 # Continue to scroll through the rest of the results
639 while True:
640 # apply the limit
641 if limit is not None and counter >= int(limit):
642 break
644 # if we consumed all the results we were expecting, we can just stop here
645 if counter >= total_results:
646 break
648 # get the next page and check that we haven't timed out
649 try:
650 res = ES.scroll(scroll_id=scroll_id, scroll=keepalive)
651 except elasticsearch.exceptions.NotFoundError as e:
652 raise ScrollTimeoutException(
653 "Scroll timed out; {status} - {message}".format(status=e.status_code, message=e.info))
654 except Exception as e:
655 # if any other exception occurs, make sure it's at least logged.
656 app.logger.exception("Unhandled exception in scroll method of DAO")
657 raise ScrollException(e)
659 # if we didn't get any results back, this means we're at the end
660 if len(res.get('hits', {}).get('hits')) == 0:
661 break
663 for r in cls.handle_es_raw_response(
664 res,
665 wrap=wrap,
666 extra_trace_info=
667 "\nScroll:\n{q}\n"
668 "\n\nPage #{counter} of the ES response with size {page_size}."
669 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)):
671 # apply the limit
672 if limit is not None and counter >= int(limit):
673 break
674 counter += 1
675 if wrap:
676 yield cls(**r)
677 else:
678 yield r
680 @classmethod
681 def iterate_pit(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True,
682 keepalive: str = '1m'):
683 """ Provide an iterable of all items in a model, reimplemented using point-in-time queries
684 :param q: The query to scroll results on
685 :param page_size: limited by ElasticSearch, check settings to override
686 :param limit: Limit the number of results returned (e.g. to take a slice)
687 :param wrap: Whether to return the results in raw json or wrapped as an object
688 :param keepalive: scroll timeout
689 """
690 theq = {"query": {"match_all": {}}} if q is None else deepcopy(q)
691 theq["size"] = page_size
692 theq["from"] = 0
693 if "sort" not in theq:
694 # This gives the same performance enhancement as scan, use it by default. This is the order of indexing like sort by ID
695 theq["sort"] = ["_doc"]
697 # Open a point in time query context
698 res = ES.open_point_in_time(index=cls.index_name(), keep_alive=keepalive)
699 pit_id = res.get("id")
701 theq["pit"] = {
702 "id": pit_id,
703 "keep_alive": keepalive
704 }
705 theq["track_total_hits"] = True
707 first_resp = cls.send_query(theq, pit_query=True)
708 if len(first_resp.get('hits', {}).get('hits', [])) == 0:
709 return
711 search_after = first_resp.get('hits', {}).get('hits', [])[-1].get('sort', [])
712 total_results = first_resp.get('hits', {}).get('total', {}).get('value')
714 # Supply the first set of results
715 counter = 0
716 for r in cls.handle_es_raw_response(
717 first_resp,
718 wrap=wrap,
719 extra_trace_info=
720 "\nPIT Initialised:\n{q}\n"
721 "\n\nPage #{counter} of the ES response with size {page_size}."
722 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)):
724 # apply the limit
725 if limit is not None and counter >= int(limit):
726 break
727 counter += 1
728 if wrap:
729 yield cls(**r)
730 else:
731 yield r
733 del theq["track_total_hits"]
735 # Continue to scroll through the rest of the results
736 while True:
737 # apply the limit
738 if limit is not None and counter >= int(limit):
739 break
741 # if we consumed all the results we were expecting, we can just stop here
742 if counter >= total_results:
743 break
745 theq["search_after"] = search_after
747 # get the next page and check that we haven't timed out
748 try:
749 res = cls.send_query(theq, pit_query=True)
750 if len(res.get('hits', {}).get('hits', [])) == 0:
751 break
752 search_after = first_resp.get('hits', {}).get('hits', [])[-1].get('sort', [])
753 except elasticsearch.exceptions.NotFoundError as e:
754 raise ScrollTimeoutException(
755 "PIT timed out; {status} - {message}".format(status=e.status_code, message=e.info))
756 except Exception as e:
757 # if any other exception occurs, make sure it's at least logged.
758 app.logger.exception("Unhandled exception in iterate_pit method of DAO")
759 try:
760 ES.close_point_in_time({"id": pit_id})
761 except:
762 pass
763 raise ScrollException(e)
765 for r in cls.handle_es_raw_response(
766 res,
767 wrap=wrap,
768 extra_trace_info=
769 "\nPIT:\n{q}\n"
770 "\n\nPage #{counter} of the ES response with size {page_size}."
771 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)):
773 # apply the limit
774 if limit is not None and counter >= int(limit):
775 break
776 counter += 1
777 if wrap:
778 yield cls(**r)
779 else:
780 yield r
782 ES.close_point_in_time({"id": pit_id})
784 @classmethod
785 def iterate_unstable(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True, logger=None):
786 """ Provide an iterable of all items in a model, using search_after but with no scroll context or
787 PIT. This means that if the index changes as the iterate is happening, there may be repeated or
788 missed elements. This is useful for cases where the index is not changing during the iteration, or the
789 exact export is not important (e.g. for anon_export for testing purposes)
791 :param q: The query to scroll results on
792 :param page_size: limited by ElasticSearch, check settings to override
793 :param limit: Limit the number of results returned (e.g. to take a slice)
794 :param wrap: Whether to return the results in raw json or wrapped as an object
795 """
796 theq = {"query": {"match_all": {}}} if q is None else deepcopy(q)
797 theq["size"] = page_size
798 if "from" in theq:
799 del theq["from"]
801 if "sort" not in theq:
802 # This gives the same performance enhancement as scan, use it by default. This is the order of indexing like sort by ID
803 theq["sort"] = [{"_id": "desc"}]
805 theq["track_total_hits"] = True
807 # if logger: logger(json.dumps(theq))
809 first_resp = cls.send_query(theq)
810 if len(first_resp.get('hits', {}).get('hits', [])) == 0:
811 # if logger: logger("No results found")
812 return
814 search_after = first_resp.get('hits', {}).get('hits', [])[-1].get('sort', [])
815 total_results = first_resp.get('hits', {}).get('total', {}).get('value')
816 if logger: logger(f"Expecting total {total_results}")
818 # Supply the first set of results
819 counter = 0
820 for r in cls.handle_es_raw_response(
821 first_resp,
822 wrap=wrap,
823 extra_trace_info=
824 "\nUnstable Iterate Initialised:\n{q}\n"
825 "\n\nPage #{counter} of the ES response with size {page_size}."
826 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)):
828 # apply the limit
829 if limit is not None and counter >= int(limit):
830 break
831 counter += 1
832 if wrap:
833 yield cls(**r)
834 else:
835 yield r
836 # if logger: logger(f"Iterated {counter} records")
838 del theq["track_total_hits"]
840 # Continue to scroll through the rest of the results
841 while True:
842 # apply the limit
843 if limit is not None and counter >= int(limit):
844 break
846 # if we consumed all the results we were expecting, we can just stop here
847 if counter >= total_results:
848 break
850 theq["search_after"] = search_after
851 # if logger: logger(json.dumps(theq))
852 # if logger: logger("search_after: " + str(search_after))
853 try:
854 res = cls.send_query(theq)
855 if len(res.get('hits', {}).get('hits', [])) == 0:
856 break
857 search_after = res.get('hits', {}).get('hits', [])[-1].get('sort', [])
858 except Exception as e:
859 # if any exception occurs, make sure it's at least logged.
860 app.logger.exception("Unhandled exception in iterate_unstable method of DAO")
861 if logger: logger(f"Iterate failed on {json.dumps(theq)}")
862 raise ScrollException(e)
864 for r in cls.handle_es_raw_response(
865 res,
866 wrap=wrap,
867 extra_trace_info=
868 "\nUnstable Iterate:\n{q}\n"
869 "\n\nPage #{counter} of the ES response with size {page_size}."
870 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)):
872 # apply the limit
873 if limit is not None and counter >= int(limit):
874 break
875 counter += 1
876 if wrap:
877 yield cls(**r)
878 else:
879 yield r
881 # if logger: logger(f"Iterated {counter} records")
883 @classmethod
884 def iterall(cls, page_size=1000, limit=None, **kwargs):
885 # TODO: Another candidate for swapping to iterate_pit (or rename iterate_pit to iterate when we're happy)
886 return cls.iterate(MatchAllQuery().query(), page_size, limit, **kwargs)
888 @classmethod
889 def iterall_unstable(cls, page_size=1000,
890 stripe_field="id",
891 striped=False,
892 prefix_generator=None,
893 prefix_size=4,
894 limit=None,
895 logger=None,
896 must=None,
897 **kwargs):
898 def hex_prefixes(n=4):
899 """ Generate a list of hex prefixes of length n """
900 return [str(hex(i))[2:].zfill(n) for i in range(0, 16 ** n)]
902 if striped:
903 q = None
904 if must is not None:
905 q = {
906 "query": {
907 "bool": {
908 "must": must
909 }
910 }
911 }
912 total = cls.count(q)
913 if total == 0:
914 return
916 count = 0
917 prefixes = prefix_generator(prefix_size) if prefix_generator is not None else hex_prefixes(prefix_size)
918 empty_prefixes = []
919 for prefix in prefixes:
920 if must is None:
921 q = {
922 "query": {
923 "prefix": {stripe_field: prefix}
924 }
925 }
926 else:
927 q = {
928 "query": {
929 "bool": {
930 "must": must + [
931 {
932 "prefix": {stripe_field: prefix}
933 }
934 ]
935 }
936 }
937 }
939 first = True
940 for record in cls.iterate_unstable(q, page_size, limit=limit, logger=logger, **kwargs):
941 count += 1
942 if logger and first:
943 if len(empty_prefixes) > 0:
944 logger(f"Skipped empty prefixes: {empty_prefixes}")
945 empty_prefixes = []
946 logger(f"Exporting prefix: {prefix}")
947 first = False
948 if limit is not None:
949 if count > limit:
950 if logger: logger(f"Limit reached: {count} / {limit}")
951 return
952 yield record
954 if first:
955 empty_prefixes.append(prefix)
956 else:
957 if logger: logger(f"Finished prefix: {prefix}; {count} total records")
958 if len(empty_prefixes) > 0:
959 if logger: logger(f"Skipped empty prefixes: {empty_prefixes}")
960 else:
961 if logger: logger("Exporting without prefix striping")
962 for record in cls.iterate_unstable(q=None, page_size=page_size, limit=limit, logger=logger, **kwargs):
963 yield record
965 # Aliases for the iterate functions
966 scroll = iterate
967 scroll_pit = iterate_pit
969 @classmethod
970 def dump(cls, q=None, page_size=1000, limit=None, out=None, out_template=None, out_batch_sizes=100000,
971 out_rollover_callback=None, transform=None, es_bulk_format=True, idkey='id', es_bulk_fields=None,
972 stripe_field="id", striped=False, prefix_generator=None, prefix_size=3, logger=None):
973 """ Export to file, bulk format or just a json dump of the record """
975 filenames = []
976 n = 1
977 current_file = None
978 if out_template is not None:
979 current_file = out_template + "." + str(n)
980 filenames.append(current_file)
981 if out is None and current_file is not None:
982 out = open(current_file, "w")
983 else:
984 out = sys.stdout
986 count = 0
987 if q is None:
988 iterator = cls.iterall_unstable(page_size=page_size, stripe_field=stripe_field,
989 striped=striped, prefix_generator=prefix_generator,
990 prefix_size=prefix_size,
991 limit=limit, wrap=False, logger=logger)
992 else:
993 iterator = cls.iterate_unstable(q, page_size=page_size, limit=limit, wrap=False)
995 for record in iterator:
996 if transform is not None:
997 record = transform(record)
998 if record is None:
999 # transform declined to alter (e.g. anonymise) this record - drop it rather than
1000 # writing raw/partially-anonymised data to the export
1001 continue
1003 if es_bulk_format:
1004 kwargs = {}
1005 if es_bulk_fields is None:
1006 es_bulk_fields = ["_id", "_index", "_type"]
1007 for key in es_bulk_fields:
1008 if key == "_id":
1009 kwargs["idkey"] = idkey
1010 if key == "_index":
1011 kwargs["index"] = cls.index_name()
1012 if key == "_type":
1013 kwargs["type_"] = type
1014 data = cls.to_bulk_single_rec(record, **kwargs)
1015 else:
1016 data = json.dumps(record) + "\n"
1018 out.write(data)
1019 if out_template is not None:
1020 count += 1
1021 if count > out_batch_sizes:
1022 out.close()
1023 if out_rollover_callback is not None:
1024 out_rollover_callback(current_file)
1026 count = 0
1027 n += 1
1028 current_file = out_template + "." + str(n)
1029 filenames.append(current_file)
1030 out = open(current_file, "w")
1032 if out_template is not None:
1033 out.close()
1034 if out_rollover_callback is not None:
1035 out_rollover_callback(current_file)
1037 return filenames
1039 @classmethod
1040 def bulk_load_from_file(cls, source_file, index=None, limit=None, max_content_length=100000000):
1041 """ ported from esprit.tasks - bulk load to index from file
1042 :param source_file
1043 :param index: index name for target
1044 :param limit: number of records to load (integer)
1045 :param max_content_length: Upload chunk size in bytes
1046 """
1047 index = index or cls.index_name()
1049 source_size = os.path.getsize(source_file)
1050 with open(source_file, "r") as f:
1051 if limit is None and source_size < max_content_length:
1052 # if we aren't selecting a portion of the file, and the file is below the max content length, then
1053 # we can just serve it directly
1054 ES.bulk(body=f.read(), index=index, doc_type=cls.doc_type(), request_timeout=120)
1055 return -1
1056 else:
1057 count = 0
1058 while True:
1059 chunk = DomainObject._make_next_chunk(f, max_content_length)
1060 if chunk == "":
1061 break
1063 finished = False
1064 if limit is not None:
1065 newlines = chunk.count("\n")
1066 records = newlines // 2
1067 if count + records > limit:
1068 max = (limit - count) * 2
1069 lines = chunk.split("\n")
1070 allowed = lines[:max]
1071 chunk = "\n".join(allowed) + "\n"
1072 count += max
1073 finished = True
1074 else:
1075 count += records
1077 ES.bulk(body=chunk, index=index, doc_type=cls.doc_type(), request_timeout=120)
1078 if finished:
1079 break
1080 if limit is not None:
1081 return count
1082 else:
1083 return -1
1085 @staticmethod
1086 def make_bulk_chunk_files(source_file, out_file_prefix, max_content_length=100000000):
1087 """ ported from esprit.tasks - break out a bulk file into smaller chunks """
1089 source_size = os.path.getsize(source_file)
1090 with open(source_file, "r") as f:
1091 if source_size < max_content_length:
1092 return [source_file]
1093 else:
1094 filenames = []
1095 count = 0
1096 while True:
1097 count += 1
1098 chunk = DomainObject._make_next_chunk(f, max_content_length)
1099 if chunk == "":
1100 break
1102 filename = out_file_prefix + "." + str(count)
1103 with open(filename, "w") as g:
1104 g.write(chunk)
1105 filenames.append(filename)
1107 return filenames
1109 @staticmethod
1110 def _make_next_chunk(f, max_content_length):
1111 """ ported from esprit.tasks - create a bulk chunk, ensuring it's not a partial instruction """
1113 def is_command(line):
1114 try:
1115 command = json.loads(line)
1116 except (json.JSONDecodeError, TypeError):
1117 return False
1118 keys = list(command.keys())
1119 if len(keys) > 1:
1120 return False
1121 if "index" not in keys:
1122 return False
1123 subkeys = list(command["index"].keys())
1124 for sk in subkeys:
1125 if sk not in ["_id"]:
1126 return False
1128 return True
1130 offset = f.tell()
1131 chunk = f.read(max_content_length)
1132 while True:
1133 last_newline = chunk.rfind("\n")
1134 tail = chunk[last_newline + 1:]
1135 chunk = chunk[:last_newline]
1137 if is_command(tail):
1138 f.seek(offset + last_newline)
1139 if chunk.startswith("\n"):
1140 chunk = chunk[1:]
1141 return chunk
1142 else:
1143 continue
1145 @classmethod
1146 def prefix_query(cls, field, prefix, filter_condition=None, size=5, facet_field=None, analyzed_field=True):
1147 # example of a prefix query
1148 # {
1149 # "query": {"prefix" : { "bibjson.publisher" : "ope" } },
1150 # "size": 0,
1151 # "facets" : {
1152 # "publisher" : { "terms" : {"field" : "bibjson.publisher.exact", "size": 5} }
1153 # }
1154 # }
1156 suffix = app.config['FACET_FIELD']
1157 query_field = field
1158 if analyzed_field:
1159 if field.endswith(suffix):
1160 # strip .exact (or whatever it's configured as) off the end
1161 query_field = field[:field.rfind(suffix)]
1162 else:
1163 if not field.endswith(suffix):
1164 query_field = field + suffix
1166 # the actual terms should come from the .exact version of the
1167 # field - we are suggesting whole values, not fragments
1168 if facet_field is None:
1169 facet_field = query_field + suffix
1170 if not facet_field.endswith(suffix):
1171 facet_field = facet_field + suffix
1173 q = PrefixAutocompleteQuery(query_field, prefix, field, facet_field, size, filter_condition)
1174 return cls.send_query(q.query())
1176 @classmethod
1177 def wildcard_autocomplete_query(cls, field, substring, before=True, after=True, facet_size=5, facet_field=None):
1178 """
1179 Example of a wildcard query
1180 Works only on .exact fields
1182 {
1183 "query" : {
1184 "wildcard" : {"bibjson.publisher.exact" : "De *"}
1185 },
1186 "size" : 0,
1187 "facets" : {
1188 "bibjson.publisher.exact" : {
1189 "terms" : {"field" : "bibjson.publisher.exact", "size" : 5}
1190 }
1191 }
1192 }
1193 :param field:
1194 :param substring:
1195 :param facet_size:
1196 :return:
1197 """
1198 # wildcard queries need to be on unanalyzed fields
1199 suffix = app.config['FACET_FIELD']
1200 filter_field = field
1201 if not filter_field.endswith(suffix):
1202 filter_field = filter_field + suffix
1204 # add the wildcard before/after
1205 if before:
1206 substring = "*" + substring
1207 if after:
1208 substring = substring + "*"
1210 # sort out the facet field
1211 if facet_field is None:
1212 facet_field = filter_field
1213 if not facet_field.endswith(suffix):
1214 facet_field = facet_field + suffix
1216 # build the query
1217 q = WildcardAutocompleteQuery(filter_field, substring, field, facet_field, facet_size)
1218 return cls.send_query(q.query())
1220 @classmethod
1221 def advanced_autocomplete(cls, filter_field, facet_field, substring, size=5, prefix_only=True):
1222 analyzed = True
1223 if " " in substring:
1224 analyzed = False
1226 substring = substring.lower()
1228 if " " in substring and not prefix_only:
1229 res = cls.wildcard_autocomplete_query(filter_field, substring, before=True, after=True, facet_size=size,
1230 facet_field=facet_field)
1231 else:
1232 res = cls.prefix_query(filter_field, substring, size=size, facet_field=facet_field, analyzed_field=analyzed)
1234 result = []
1235 for term in res['aggregations'][filter_field]['buckets']:
1236 # keep ordering - it's by count by default, so most frequent
1237 # terms will now go to the front of the result list
1238 result.append({"id": term['key'], "text": term['key']})
1239 return result
1241 @classmethod
1242 def autocomplete(cls, field, prefix, filter_condition=None, size=5):
1243 res = None
1244 # if there is a space in the prefix, the prefix query won't work, so we fall back to a wildcard
1245 # we only do this if we have to, because the wildcard query is a little expensive
1246 if " " in prefix:
1247 res = cls.wildcard_autocomplete_query(field, prefix, before=False, after=True, facet_size=size)
1248 else:
1249 prefix = prefix.lower()
1250 res = cls.prefix_query(field, prefix, filter_condition, size=size)
1252 result = []
1253 for term in res['aggregations'][field]['buckets']:
1254 # keep ordering - it's by count by default, so most frequent
1255 # terms will now go to the front of the result list
1256 result.append({"id": term['key'], "text": term['key']})
1257 return result
1259 @classmethod
1260 def q2obj(cls, **kwargs):
1261 extra_trace_info = ''
1262 if 'q' in kwargs:
1263 extra_trace_info = "\nQuery sent to ES (before manipulation in DomainObject.query):\n{}\n".format(
1264 json.dumps(kwargs['q'], indent=2))
1266 res = cls.query(**kwargs)
1267 rs = cls.handle_es_raw_response(res, wrap=True, extra_trace_info=extra_trace_info)
1268 results = [cls(**r) for r in rs]
1269 return results
1271 @classmethod
1272 def all(cls, size=10000, **kwargs):
1273 """ This is a shortcut to a match_all query with a large size, to return all records """
1274 # FIXME: is this only used in tests? ES now limits size so we can't guarantee ALL without using scroll / scan
1275 return cls.q2obj(size=size, **kwargs)
1277 @classmethod
1278 def count(cls, query=None):
1279 res = ES.count(index=cls.index_name(), doc_type=cls.doc_type(), body=query)
1280 return res.get("count")
1281 # return requests.get(cls.target() + '_count').json()['count']
1283 @classmethod
1284 def hit_count(cls, query, **kwargs) -> int:
1285 countable_query = deepcopy(query)
1286 if "track_total_hits" not in countable_query:
1287 countable_query["track_total_hits"] = True
1289 res = cls.query(q=countable_query, **kwargs)
1290 return res.get("hits", {}).get("total", {}).get("value", 0)
1292 @classmethod
1293 def count_updated_since(cls, last_update):
1294 """
1295 Count the number of records updated since a given date
1296 :param last_update: The date to count from
1297 :return: The number of records updated since the given date
1298 """
1299 q = {
1300 "query": {
1301 "range": {
1302 "last_updated": {
1303 "gte": last_update
1304 }
1305 }
1306 }
1307 }
1308 return cls.count(q)
1310 @classmethod
1311 def block(cls, id, last_updated=None, sleep=0.5, max_retry_seconds=30):
1312 if app.config.get("ES_BLOCK_WAIT_OVERRIDE") is not None:
1313 sleep = app.config["ES_BLOCK_WAIT_OVERRIDE"]
1315 q = BlockQuery(id)
1316 start_time = dates.now()
1317 while True:
1318 res = cls.query(q=q.query())
1319 hits = res.get("hits", {}).get("hits", [])
1320 if len(hits) > 0:
1321 if len(hits) > 1:
1322 raise Exception("More than one record with id {x}".format(x=id))
1323 obj = hits[0].get("_source", {})
1324 if last_updated is not None:
1325 lu = obj.get("last_updated")
1326 if lu:
1327 threshold = dates.parse(last_updated)
1328 lud = dates.parse(lu)
1329 if lud >= threshold:
1330 return
1331 else:
1332 return
1334 if (dates.now() - start_time).total_seconds() >= max_retry_seconds:
1335 raise (BlockTimeOutException(
1336 "Attempting to block until record with id {id} appears in the index, but this has not happened after {limit} seconds"
1337 .format(
1338 id=id, limit=max_retry_seconds)))
1340 time.sleep(sleep)
1342 @classmethod
1343 def blockall(cls, ids_and_last_updateds, sleep=0.05, individual_max_retry_seconds=30):
1344 for id, lu in ids_and_last_updateds:
1345 cls.block(id, lu, sleep=sleep, max_retry_seconds=individual_max_retry_seconds)
1347 @classmethod
1348 def blockdeleted(cls, id, sleep=0.5, max_retry_seconds=30):
1349 if app.config.get("ES_BLOCK_WAIT_OVERRIDE") is not None:
1350 sleep = app.config["ES_BLOCK_WAIT_OVERRIDE"]
1352 q = BlockQuery(id)
1353 start_time = dates.now()
1354 while True:
1355 res = cls.query(q=q.query())
1356 hits = res.get("hits", {}).get("hits", [])
1357 if len(hits) == 0:
1358 return
1359 else:
1360 if (dates.now() - start_time).total_seconds() >= max_retry_seconds:
1361 raise BlockTimeOutException(
1362 "Attempting to block until record with id {id} deleted from Elasticsearch, but this has not happened after {limit}".format(
1363 id=id, limit=max_retry_seconds))
1365 time.sleep(sleep)
1367 @classmethod
1368 def blockalldeleted(cls, ids, sleep=0.05, individual_max_retry_seconds=30):
1369 for id in ids:
1370 cls.blockdeleted(id, sleep, individual_max_retry_seconds)
1372 @classmethod
1373 def save_all(cls, models, blocking=False):
1374 for m in models:
1375 m.save()
1376 if blocking:
1377 cls.blockall((m.id, getattr(m, "last_updated", None)) for m in models)
1379 @classmethod
1380 def create_and_seed_index_and_rollover_alias(cls, documents: List[dict], mapping=None, keep_history=0):
1381 """
1382 Create a new index based on this object's type, put the mapping if provided, and seed it with the documents.
1383 The indexes alias is then repointed to the new index.
1384 If a keep_history value is provided, the previous indexes with the same prefix as the alias will be deleted,
1386 :param documents: may be empty, to roll the alias over onto a new empty index (then a mapping is required)
1387 :param mapping: the mappings, or a full index body ({"mappings": ..., "settings": ...}) to also apply index settings
1388 :param keep_history: supply -1 to keep all history, 0 to keep no history, or a positive integer to keep that many previous indexes.
1389 :return:
1390 """
1391 has_documents = len(documents) > 0
1392 if not has_documents and mapping is None:
1393 raise ValueError("A mapping is required to roll over onto an empty index")
1395 full_alias = cls.index_name()
1396 new_name = cls.__type__ + "-" + dates.now_str(dates.FMT_DATETIME_LONG)
1397 if mapping is not None:
1398 cls.put_mapping(mapping, override_index_name=new_name)
1399 if has_documents:
1400 resp = cls.bulk(documents, refresh=True, override_index_name=new_name)
1401 if resp.get('errors', False):
1402 raise ESError("Error creating index {}: {}".format(new_name, resp))
1403 cls.move_alias(full_alias, new_name)
1405 if keep_history > -1:
1406 # Find all indexes with the same prefix as full_alias
1407 all_indexes = find_indexes_by_prefix(full_alias)
1408 all_indexes.sort()
1409 keep_history += 1
1410 for idx in all_indexes[:-keep_history]:
1411 try:
1412 ES.indices.delete(index=idx)
1413 except elasticsearch.exceptions.NotFoundError:
1414 pass
1415 except elasticsearch.exceptions.RequestError as e:
1416 raise ESError(e)
1419 @classmethod
1420 def put_mapping(cls, mapping: dict, override_index_name: str = None):
1421 """
1422 Put a mapping for the index
1423 :param mapping: The mapping to put. If None then use cls.mapping()
1424 """
1425 try:
1426 if "mappings" not in mapping:
1427 mapping = {"mappings": mapping}
1428 return ES.indices.create(index=cls.index_name(override_index_name),
1429 body=mapping,
1430 request_timeout=app.config.get("ES_SOCKET_TIMEOUT", None))
1431 except elasticsearch.exceptions.RequestError as e:
1432 raise ESError(e)
1434 @classmethod
1435 def move_alias(cls, alias_name: str, target_index: str):
1436 """
1437 Create or update an alias for the index
1438 :param alias_name: The name of the alias to create or update
1439 :param target_index: The name of the index to point the alias to
1440 """
1442 try:
1443 alias_info = ES.indices.get_alias(name=alias_name)
1444 old_index_name = list(alias_info.keys())[0]
1445 full_index_name = cls.index_name(target_index)
1446 body = {
1447 "actions": [
1448 {
1449 "add": {
1450 "index": full_index_name,
1451 "alias": alias_name
1452 }
1453 },
1454 {
1455 "remove": {
1456 "index": old_index_name,
1457 "alias": alias_name
1458 }
1459 }
1460 ]
1461 }
1462 ES.indices.update_aliases(body=body)
1463 except elasticsearch.exceptions.RequestError as e:
1464 raise ESError(e)
1469def any_pending_tasks():
1470 """ Check if there are any pending tasks in the elasticsearch task queue """
1471 results = ES.cluster.pending_tasks()
1472 return len(results["tasks"]) > 0
1475def query_data_tasks(timeout='30s'):
1476 """ Check if there are any pending tasks in the elasticsearch task queue """
1477 results = ES.tasks.list(params={
1478 "actions": 'indices:data*',
1479 "timeout": timeout,
1480 "wait_for_completion": 'true',
1481 })
1482 tasks = []
1483 for node in results['nodes'].values():
1484 tasks.extend(node['tasks'].values())
1485 return tasks
1488def refresh():
1489 """
1490 refresh all indexes to make newly added or deleted documents immediately searchable
1491 """
1492 return ES.indices.refresh()
1495def find_indexes_by_prefix(index_prefix) -> list[str]:
1496 data = ES.indices.get(f'{index_prefix}*')
1497 return list(data.keys())
1500def find_index_aliases(alias_prefixes=None) -> Iterable[Tuple[str, str]]:
1501 def _yield_index_alias():
1502 data = ES.indices.get_alias()
1503 for index, d in data.items():
1504 for alias in d['aliases'].keys():
1505 yield index, alias
1507 index_aliases = _yield_index_alias()
1508 if alias_prefixes:
1509 index_aliases = ((index, alias) for index, alias in index_aliases
1510 if any(alias.startswith(p) for p in alias_prefixes))
1511 return index_aliases
1514def is_exist(query: dict, index):
1515 query['size'] = 1
1516 query['_source'] = False
1517 res = ES.search(body=query, index=index, size=1, ignore=[404])
1519 return res.get('hits', {}).get('total',{}).get('value', 0) > 0
1522class BlockTimeOutException(Exception):
1523 pass
1526class DAOSaveExceptionMaxRetriesReached(Exception):
1527 pass
1530class ESResponseCannotBeInterpreted(Exception):
1531 pass
1534class ESMappingMissingError(Exception):
1535 pass
1538class ESError(Exception):
1539 pass
1542########################################################################
1543# Some useful ES queries
1544########################################################################
1547class MatchAllQuery(object):
1548 def query(self):
1549 return {
1550 "track_total_hits": True,
1551 "query": {
1552 "match_all": {}
1553 }
1554 }
1557class BlockQuery(object):
1558 def __init__(self, id):
1559 self._id = id
1561 def query(self):
1562 return {
1563 "query": {
1564 "ids": {
1565 "values": [self._id]
1566 }
1567 },
1568 "_source": ["last_updated"],
1569 "size": 2
1570 }
1573class PrefixAutocompleteQuery(object):
1574 def __init__(self, query_field, prefix, agg_name, agg_field, agg_size, filter_condition=None):
1575 self._query_field = query_field
1576 self._prefix = prefix
1577 self._agg_name = agg_name
1578 self._agg_field = agg_field
1579 self._agg_size = agg_size
1580 self._filter_condition = filter_condition
1581 def query(self):
1582 query_body = {
1583 "track_total_hits": True,
1584 "query": {
1585 "bool": {
1586 "must": [
1587 {"prefix": {self._query_field: self._prefix.lower()}} # Keep the prefix query
1588 ]
1589 }
1590 },
1591 "size": 0,
1592 "aggs": {
1593 self._agg_name: {"terms": {"field": self._agg_field, "size": self._agg_size}}
1594 }
1595 }
1597 if self._filter_condition:
1598 query_body["query"]["bool"]["filter"] = [
1599 {"term": self._filter_condition}
1600 ]
1602 return query_body
1606class WildcardAutocompleteQuery(object):
1607 def __init__(self, wildcard_field, wildcard_query, agg_name, agg_field, agg_size):
1608 self._wildcard_field = wildcard_field
1609 self._wildcard_query = wildcard_query
1610 self._agg_name = agg_name
1611 self._agg_field = agg_field
1612 self._agg_size = agg_size
1614 def query(self):
1615 return {
1616 "track_total_hits": True,
1617 "query": {
1618 "wildcard": {self._wildcard_field: self._wildcard_query}
1619 },
1620 "size": 0,
1621 "aggs": {
1622 self._agg_name: {
1623 "terms": {"field": self._agg_field, "size": self._agg_size}
1624 }
1625 }
1626 }
1629#########################################################################
1630# A query handler that knows how to speak facetview2
1631#########################################################################
1634class Facetview2(object):
1635 """
1636 ~~SearchURLGenerator:Feature->Elasticsearch:Technology~~
1638 # Examples of queries
1639 # {"query":{"filtered":{"filter":{"bool":{"must":[{"term":{"_type":"article"}}]}},"query":{"query_string":{"query":"richard","default_operator":"OR"}}}},"from":0,"size":10}
1640 # {"query":{"query_string":{"query":"richard","default_operator":"OR"}},"from":0,"size":10}
1641 """
1642 @staticmethod
1643 def make_term_filter(term, value):
1644 return {"term": {term: value}}
1646 @staticmethod
1647 def make_query(query_string=None, filters=None, default_operator="OR", sort_parameter=None, sort_order="asc",
1648 default_field=None):
1649 query_part = {"match_all": {}}
1650 if query_string is not None:
1651 query_part = {"query_string": {"query": query_string, "default_operator": default_operator}}
1652 if default_field is not None:
1653 query_part["query_string"]["default_field"] = default_field
1654 query = {"query": query_part}
1656 if filters is not None:
1657 if not isinstance(filters, list):
1658 filters = [filters]
1659 filters.append(query_part)
1660 bool_part = {"bool": {"must": filters}}
1661 query = {"query": query_part}
1663 if sort_parameter is not None:
1664 # For facetview we can only have one sort parameter, but ES actually supports lists
1665 sort_part = [{sort_parameter: {"order": sort_order}}]
1666 query["sort"] = sort_part
1668 return query
1670 @staticmethod
1671 def url_encode_query(query):
1672 return urllib.parse.quote(json.dumps(query).replace(' ', ''))
1675def patch_model_for_bulk(obj: DomainObject):
1676 obj.data['es_type'] = obj.__type__
1677 obj.data['id'] = obj.makeid()
1678 return obj