Coverage for portality/dao.py: 69%

930 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1from __future__ import annotations 

2 

3import json 

4import os 

5import re 

6import sys 

7import time 

8import urllib.parse 

9import uuid 

10from collections import UserDict 

11from copy import deepcopy 

12from datetime import timedelta 

13from typing import List, Iterable, Tuple 

14 

15import elasticsearch 

16from elasticsearch.exceptions import ConnectionTimeout, TransportError 

17 

18from portality.core import app, es_connection as ES 

19from portality.lib import dates 

20from portality.lib.dates import FMT_DATETIME_STD 

21 

22# All models in models.py should inherit this DomainObject to know how to save themselves in the index and so on. 

23# You can overwrite and add to the DomainObject functions as required. See models.py for some examples. 

24 

25 

26ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_\.]+?\] in order to sort on.*', re.DOTALL) 

27CONTENT_TYPE_JSON = {'Content-Type': 'application/json'} 

28 

29 

30def is_client_error(e: Exception) -> bool: 

31 """True if ES rejected the request itself (4xx), so retrying it cannot succeed. 429 is excluded, since it 

32 means 'too many requests' and may succeed later.""" 

33 status = getattr(e, "status_code", None) 

34 return isinstance(e, TransportError) and isinstance(status, int) and 400 <= status < 500 and status != 429 

35 

36 

37class ElasticSearchWriteException(Exception): 

38 pass 

39 

40 

41class ScrollException(Exception): 

42 pass 

43 

44 

45class ScrollInitialiseException(ScrollException): 

46 pass 

47 

48 

49class ScrollTimeoutException(ScrollException): 

50 pass 

51 

52 

53class BulkException(Exception): 

54 pass 

55 

56 

57class DomainObject(UserDict, object): 

58 """ 

59 ~~DomainObject:Model->Elasticsearch:Technology~~ 

60 

61 base models class for Elasticsearch(ES) index, 

62 which provide interaction with ES index such as (save, delete, query, etc.) 

63 """ 

64 

65 # set the type on the model that inherits this# which also is ES index name of the model 

66 __type__ = None 

67 

68 def __init__(self, **kwargs): 

69 # if self.data is already set, don't do anything here 

70 try: 

71 object.__getattribute__(self, "data") 

72 except: 

73 if '_source' in kwargs: 

74 self.data = dict(kwargs['_source']) 

75 self.meta = dict(kwargs) 

76 del self.meta['_source'] 

77 else: 

78 self.data = dict(kwargs) 

79 # FIXME: calling super() breaks everything, even thought this is the correct thing to do 

80 # this is because the DomainObject incorrectly overrides properties of the super class 

81 # super(DomainObject, self).__init__() 

82 

83 @classmethod 

84 def index_name(cls, override_index_name=None, **kwargs): 

85 if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE'] and cls.__type__ is not None: 

86 if override_index_name is not None: 

87 name = app.config['ELASTIC_SEARCH_DB_PREFIX'] + override_index_name 

88 else: 

89 name = ','.join([app.config['ELASTIC_SEARCH_DB_PREFIX'] + t for t in cls.__type__.split(',')]) 

90 else: 

91 if override_index_name is not None: 

92 name = override_index_name 

93 else: 

94 name = app.config['ELASTIC_SEARCH_DB'] 

95 return name 

96 

97 @classmethod 

98 def doc_type(cls): 

99 if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE']: 

100 return None 

101 else: 

102 return cls.__type__ 

103 

104 @classmethod 

105 def makeid(cls): 

106 """Create a new id for data object overwrite this in specific model types if required""" 

107 return str(uuid.uuid4().hex) 

108 

109 @property 

110 def id(self): 

111 rawid = self.data.get("id", None) 

112 if rawid is not None: 

113 return str(rawid) 

114 return None 

115 

116 def set_id(self, id=None): 

117 if id is None: 

118 id = self.makeid() 

119 self.data["id"] = str(id) 

120 

121 @property 

122 def version(self): 

123 return self.meta.get('_version', None) 

124 

125 @property 

126 def json(self): 

127 return json.dumps(self.data) 

128 

129 def set_created(self, date=None): 

130 if date is None: 

131 self.data['created_date'] = dates.now_str() 

132 else: 

133 self.data['created_date'] = date 

134 

135 @property 

136 def created_date(self): 

137 return self.data.get("created_date") 

138 

139 @property 

140 def created_timestamp(self): 

141 return dates.parse(self.data.get("created_date")) 

142 

143 @property 

144 def last_updated(self): 

145 return self.data.get("last_updated") 

146 

147 @property 

148 def last_updated_timestamp(self): 

149 lu = self.last_updated 

150 if lu is None: 

151 return None 

152 return dates.parse(self.last_updated) 

153 

154 def pre_save_prep(self, blocking=False, differentiate=False, update_last_updated=True): 

155 if 'id' not in self.data: 

156 self.data['id'] = self.makeid() 

157 

158 self.data['es_type'] = self.__type__ 

159 

160 now = dates.now_str() 

161 if (blocking or differentiate) and "last_updated" in self.data: 

162 diff = dates.now() - dates.parse(self.data["last_updated"]) 

163 

164 # we need the new last_updated time to be later than the new one 

165 if diff.total_seconds() < 1: 

166 soon = dates.now() + timedelta(seconds=1) 

167 now = soon.strftime(FMT_DATETIME_STD) 

168 

169 if update_last_updated: 

170 self.data['last_updated'] = now 

171 

172 if 'created_date' not in self.data: 

173 self.data['created_date'] = now 

174 

175 def save(self, retries=0, back_off_factor=1, differentiate=False, blocking=False, block_wait=0.25, update_last_updated=True): 

176 """ 

177 ~~->ReadOnlyMode:Feature~~ 

178 :param retries: 

179 :param back_off_factor: 

180 :param differentiate: 

181 :param blocking: 

182 :return: 

183 """ 

184 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

185 app.logger.warning("System is in READ-ONLY mode, save command cannot run") 

186 return 

187 

188 if retries > app.config.get("ES_RETRY_HARD_LIMIT", 1000): # an arbitrary large number 

189 retries = app.config.get("ES_RETRY_HARD_LIMIT", 1000) 

190 

191 if app.config.get("ES_BLOCK_WAIT_OVERRIDE") is not None: 

192 block_wait = app.config["ES_BLOCK_WAIT_OVERRIDE"] 

193 

194 self.pre_save_prep(blocking=blocking, differentiate=differentiate, update_last_updated=update_last_updated) 

195 now = self.data.get("last_updated", dates.now_str()) 

196 

197 attempt = 0 

198 d = json.dumps(self.data) 

199 r = None 

200 while attempt <= retries: 

201 try: 

202 r = ES.index(self.index_name(), d, doc_type=self.doc_type(), 

203 id=self.data.get("id"), 

204 headers=CONTENT_TYPE_JSON, 

205 timeout=app.config.get('ES_READ_TIMEOUT', None), ) 

206 break 

207 

208 except (elasticsearch.ConnectionError, elasticsearch.ConnectionTimeout): 

209 app.logger.exception("Failed to connect to ES") 

210 attempt += 1 

211 

212 except elasticsearch.TransportError as e: 

213 # Retries depend on which end the error lies. 

214 if 400 <= e.status_code < 500: 

215 # Bad request, do not retry as it won't work. Fail with ElasticSearchWriteException. 

216 app.logger.exception("Bad Request to ES, save failed. Details: {0}".format(e.error)) 

217 raise ElasticSearchWriteException(e.error) 

218 elif e.status_code >= 500: 

219 # Server error, this could be temporary so we may want to retry 

220 app.logger.exception("Server Error from ES, retrying. Details: {0}".format(e.error)) 

221 attempt += 1 

222 except Exception as e: 

223 # if any other exception occurs, make sure it's at least logged. 

224 app.logger.exception("Unhandled exception in save method of DAO") 

225 raise ElasticSearchWriteException(e) 

226 

227 # wait before retrying 

228 time.sleep((2 ** attempt) * back_off_factor) 

229 

230 if attempt > retries: 

231 raise DAOSaveExceptionMaxRetriesReached( 

232 "After {attempts} attempts the record with " 

233 "id {id} failed to save.".format( 

234 attempts=attempt, id=self.data['id'])) 

235 

236 if blocking: 

237 self.__class__.block(self.id, last_updated=now, sleep=block_wait) 

238 

239 return r 

240 

241 def delete(self): 

242 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

243 app.logger.warning("System is in READ-ONLY mode, delete command cannot run") 

244 return 

245 

246 # r = requests.delete(self.target() + self.id) 

247 try: 

248 ES.delete(self.index_name(), self.id, doc_type=self.doc_type()) 

249 except elasticsearch.NotFoundError: 

250 pass # This is to preserve the old behaviour 

251 

252 @staticmethod 

253 def make_query(theq=None, should_terms=None, consistent_order=True, **kwargs): 

254 """ 

255 Generate a query object based on parameters but don't send to 

256 backend - return it instead. Must always have the same 

257 parameters as the query method. See query method for explanation 

258 of parameters. 

259 """ 

260 if theq is None: 

261 theq = "" 

262 q = deepcopy(theq) 

263 if isinstance(q, dict): 

264 query = q 

265 if 'bool' not in query['query']: 

266 boolean = {'bool': {'must': []}} 

267 boolean['bool']['must'].append(query['query']) 

268 query['query'] = boolean 

269 if 'must' not in query['query']['bool']: 

270 query['query']['bool']['must'] = [] 

271 elif q: 

272 query = { 

273 'query': { 

274 'bool': { 

275 'must': [ 

276 {'query_string': {'query': q}} 

277 ] 

278 } 

279 } 

280 } 

281 else: 

282 query = { 

283 'query': { 

284 'bool': { 

285 'must': [ 

286 {'match_all': {}} 

287 ] 

288 } 

289 } 

290 } 

291 

292 if should_terms is not None and len(should_terms) > 0: 

293 for s in should_terms: 

294 if not isinstance(should_terms[s], list): 

295 should_terms[s] = [should_terms[s]] 

296 query["query"]["bool"]["must"].append({"terms": {s: should_terms[s]}}) 

297 

298 sort_specified = False 

299 for k, v in kwargs.items(): 

300 if k == '_from': 

301 query['from'] = v 

302 elif k == 'sort': 

303 sort_specified = True 

304 query['sort'] = v 

305 else: 

306 query[k] = v 

307 if "sort" in query: 

308 sort_specified = True 

309 

310 if not sort_specified and consistent_order: 

311 # FIXME: review this - where is default sort necessary, and why do we want this in ID order? 

312 query['sort'] = [{"id.exact": {"order": "asc", "unmapped_type": "keyword"}}] 

313 

314 return query 

315 

316 @classmethod 

317 def _unwrap_search_result(cls, res): 

318 return [i.get("_source") if "_source" in i else i.get("fields") for i in 

319 res.get('hits', {}).get('hits', [])] 

320 

321 @classmethod 

322 def bulk_delete(cls, id_list, idkey='id', refresh=False): 

323 return cls.bulk(documents=[{'id': i} for i in id_list], idkey=idkey, refresh=refresh, action='delete') 

324 

325 @classmethod 

326 def bulk(cls, documents: List[dict], idkey='id', refresh=False, action='index', req_timeout=10, override_index_name=None, **kwargs): 

327 """ 

328 :param documents: a list of objects to perform bulk actions on (list of dicts) 

329 :param idkey: The path to extract an ID from the object, e.g. 'id', 'identifiers.id' 

330 :param refresh: Refresh the index in each operation (make immediately available for search) - expensive! 

331 :param req_timeout: Request timeout for bulk operation 

332 :param kwargs: kwargs are passed into the bulk instruction for each record 

333 """ 

334 # ~~->ReadOnlyMode:Feature~~ 

335 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

336 app.logger.warning("System is in READ-ONLY mode, bulk command cannot run") 

337 return 

338 

339 if action not in ['index', 'update', 'delete']: 

340 raise Exception("Unrecognised bulk action '{0}'".format(action)) 

341 

342 data = '' 

343 for d in documents: 

344 data += cls.to_bulk_single_rec(d, idkey=idkey, action=action, **kwargs) 

345 resp = ES.bulk(body=data, index=cls.index_name(override_index_name), doc_type=cls.doc_type(), refresh=refresh, 

346 request_timeout=req_timeout) 

347 return resp 

348 

349 @staticmethod 

350 def to_bulk_single_rec(record, idkey="id", action="index", **kwargs): 

351 """ Adapted from esprit. Create a bulk instruction from a single record. """ 

352 data = '' 

353 idpath = idkey.split(".") 

354 

355 # traverse down the object in search of the supplied ID key 

356 context = record 

357 for pathseg in idpath: 

358 if pathseg in context: 

359 context = context[pathseg] 

360 else: 

361 raise BulkException( 

362 "'{0}' not available in record to generate bulk _id: {1}".format(idkey, json.dumps(record))) 

363 

364 datadict = {action: {'_id': context}} 

365 datadict[action].update(kwargs) 

366 

367 data += json.dumps(datadict) + '\n' 

368 

369 if action == 'delete': 

370 return data 

371 

372 # For update, we wrap the document in {doc: document} if not already supplied 

373 if action == 'update' and not (record.get('doc') and len(record.keys()) == 1): 

374 data += json.dumps({'doc': record}) + '\n' 

375 else: 

376 data += json.dumps(record) + '\n' 

377 return data 

378 

379 @classmethod 

380 def refresh(cls): 

381 """ 

382 ~~->ReadOnlyMode:Feature~~ 

383 :return: 

384 """ 

385 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

386 app.logger.warning("System is in READ-ONLY mode, refresh command cannot run") 

387 return 

388 

389 # r = requests.post(cls.target() + '_refresh', headers=CONTENT_TYPE_JSON) 

390 # return r.json() 

391 

392 return ES.indices.refresh(index=cls.index_name()) 

393 

394 @classmethod 

395 def pull(cls, id_): 

396 """Retrieve object by id.""" 

397 if id_ is None or id_ == '': 

398 return None 

399 

400 try: 

401 # out = requests.get(cls.target() + id_) 

402 out = ES.get(cls.index_name(), id_, doc_type=cls.doc_type()) 

403 except elasticsearch.NotFoundError: 

404 return None 

405 except elasticsearch.TransportError as e: 

406 raise Exception("ES returned an error: {x}".format(x=e.info)) 

407 except Exception as e: 

408 app.logger.exception(f"Unexpected error in pull(): {e}") 

409 return None 

410 if out is None: 

411 return None 

412 

413 return cls(**out) 

414 

415 @classmethod 

416 def pull_by_key(cls, key, value): 

417 res = cls.query(q={"query": {"term": {key + app.config['FACET_FIELD']: value}}}) 

418 if res.get('hits', {}).get('total', {}).get('value', 0) == 1: 

419 return cls.pull(res['hits']['hits'][0]['_source']['id']) 

420 else: 

421 return None 

422 

423 @classmethod 

424 def object_query(cls, q=None, **kwargs): 

425 result = cls.query(q, **kwargs) 

426 return [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] 

427 

428 @classmethod 

429 def query(cls, q=None, send_kwargs=None, **kwargs): 

430 """Perform a query on backend. 

431 

432 :param q: maps to query_string parameter if string, or query dict if dict. 

433 :param send_kwargs: keyword args for send_query, e.g. retry, request_cache 

434 :param kwargs: any keyword args as per 

435 http://www.elasticsearch.org/guide/reference/api/search/uri-request.html 

436 """ 

437 query = cls.make_query(q, **kwargs) 

438 return cls.send_query(query, **(send_kwargs or {})) 

439 

440 @classmethod 

441 def send_query(cls, qobj, retry=50, pit_query=False, **kwargs): 

442 """Actually send a query object to the backend. 

443 :param kwargs are passed directly to Elasticsearch search() function 

444 """ 

445 

446 if retry > app.config.get("ES_RETRY_HARD_LIMIT", 1000) + 1: # an arbitrary large number 

447 retry = app.config.get("ES_RETRY_HARD_LIMIT", 1000) + 1 

448 

449 r = None 

450 count = 0 

451 exception = None 

452 while count < retry: 

453 count += 1 

454 try: 

455 # ES 7.10 updated target to whole index, since specifying type for search is deprecated 

456 # r = requests.post(cls.target_whole_index() + recid + "_search", data=json.dumps(qobj), headers=CONTENT_TYPE_JSON) 

457 if kwargs.get('timeout') is None: 

458 kwargs['timeout'] = app.config.get('ES_READ_TIMEOUT', None) 

459 index = cls.index_name() if pit_query is False else None 

460 r = ES.search(body=json.dumps(qobj), index=index, doc_type=cls.doc_type(), 

461 headers=CONTENT_TYPE_JSON, **kwargs) 

462 break 

463 except ConnectionTimeout as e: 

464 continue # retry on timeout 

465 except Exception as e: 

466 try: 

467 exception = ESMappingMissingError(e) if ES_MAPPING_MISSING_REGEX.match(json.dumps(e.args[2])) else e 

468 if isinstance(exception, ESMappingMissingError): 

469 raise exception 

470 except TypeError: 

471 raise e 

472 if is_client_error(e): 

473 # 4xx (e.g. a query_string parse error): the request itself is bad, so retrying can't succeed 

474 raise e 

475 

476 time.sleep(0.5) 

477 

478 if r is not None: 

479 return r 

480 if exception is not None: 

481 raise exception 

482 raise Exception("Couldn't get the ES query endpoint to respond. Also, you shouldn't be seeing this.") 

483 

484 @classmethod 

485 def remove_by_id(cls, id): 

486 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

487 app.logger.warning("System is in READ-ONLY mode, delete_by_id command cannot run") 

488 return 

489 

490 # r = requests.delete(cls.target() + id) 

491 try: 

492 ES.delete(cls.index_name(), id) 

493 except elasticsearch.NotFoundError: 

494 return 

495 

496 @classmethod 

497 def delete_by_query(cls, query): 

498 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

499 app.logger.warning("System is in READ-ONLY mode, delete_by_query command cannot run") 

500 return 

501 

502 # r = requests.delete(cls.target() + "_query", data=json.dumps(query)) 

503 # return r 

504 return ES.delete_by_query(cls.index_name(), json.dumps(query), doc_type=cls.doc_type()) 

505 

506 @classmethod 

507 def destroy_index(cls, **es_kwargs): 

508 default_es_kwargs = { 

509 'timeout': '5m', 

510 'request_timeout': 1001, 

511 } 

512 default_es_kwargs.update(es_kwargs) 

513 es_kwargs = default_es_kwargs 

514 

515 if app.config.get("READ_ONLY_MODE", False) and app.config.get("SCRIPTS_READ_ONLY_MODE", False): 

516 app.logger.warning("System is in READ-ONLY mode, destroy_index command cannot run") 

517 return 

518 

519 # if app.config['ELASTIC_SEARCH_INDEX_PER_TYPE']: 

520 # return esprit.raw.delete_index_by_prefix(es_connection, app.config['ELASTIC_SEARCH_DB_PREFIX']) 

521 # else: 

522 # return esprit.raw.delete_index(es_connection) 

523 print('Destroying indexes with prefix ' + app.config['ELASTIC_SEARCH_DB_PREFIX'] + '*') 

524 return ES.indices.delete(app.config['ELASTIC_SEARCH_DB_PREFIX'] + '*', **es_kwargs) 

525 

526 @classmethod 

527 def check_es_raw_response(cls, res, extra_trace_info=''): 

528 if 'error' in res: 

529 es_resp = json.dumps(res, indent=2) 

530 

531 error_to_raise = ESMappingMissingError if ES_MAPPING_MISSING_REGEX.match(es_resp) else ESError 

532 

533 raise error_to_raise( 

534 ( 

535 "Elasticsearch returned an error:" 

536 "\nES HTTP Response status: {es_status}" 

537 "\nES Response:{es_resp}" 

538 .format(es_status=res.get('status', 'unknown'), es_resp=es_resp) 

539 ) + extra_trace_info 

540 ) 

541 

542 if 'hits' not in res and 'hits' not in res['hits']: # i.e. if res['hits']['hits'] does not exist 

543 raise ESResponseCannotBeInterpreted( 

544 ( 

545 "Elasticsearch did not return any records. " 

546 "It probably returned an error we did not understand instead." 

547 "\nES HTTP Response status: {es_status}" 

548 "\nES Response:{es_resp}\n" 

549 .format(es_status=res.get('status', 'unknown'), es_resp=json.dumps(res, indent=2)) 

550 ) + extra_trace_info 

551 ) 

552 return True 

553 

554 @classmethod 

555 def handle_es_raw_response(cls, res, wrap, extra_trace_info=''): 

556 """ 

557 Handles the JSON returned by ES, raising errors as needed. If no problems are detected it returns its input 

558 unchanged. 

559 

560 :param res: The full ES raw response to a query in a Python dict (this method does not handle the raw JSON ES 

561 outputs). Usually this parameter is the return value of the .query or .send_query methods. 

562 :param wrap: Did the caller request wrapping of each ES record inside a model object? This matters for handling 

563 records that have no '_source' or 'fields' keys, but do have an '_id' key. Such records should raise an error 

564 if wrapping was requested, since there is nothing to wrap. If no wrapping was requested, perhaps the caller 

565 simply needed the object IDs and nothing else, so we do not need to raise an error. 

566 :param extra_trace_info: A string with additional diagnostic information to be put into exceptions. 

567 """ 

568 

569 cls.check_es_raw_response(res) 

570 

571 rs = [] 

572 for i, each in enumerate(res['hits']['hits']): 

573 if '_source' in each: 

574 rs.append(each['_source']) 

575 elif 'fields' in each: 

576 rs.append(each['fields']) 

577 elif '_id' in each and not wrap: 

578 # "_id" is a sibling (not child) of "_source" so it can only be used with unwrapped raw responses. 

579 # wrap = True only makes sense if "_source" or "fields" were returned. 

580 # So, if "_id" is the only one present, extract it into an object that's shaped the same as the item 

581 # in the raw response. 

582 rs.append({"_id": each['_id']}) 

583 else: 

584 msg1 = "Can't find any useful data in the ES response.\n" + extra_trace_info 

585 msg2 = "\nItem {i}.\nItem data:\n{each}".format(i=i, each=json.dumps(each, indent=2)) 

586 raise ESResponseCannotBeInterpreted(msg1 + msg2) 

587 

588 return rs 

589 

590 @classmethod 

591 def iterate(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True, 

592 keepalive: str = '1m'): 

593 """ Provide an iterable of all items in a model, use 

594 :param q: The query to scroll results on 

595 :param page_size: limited by ElasticSearch, check settings to override 

596 :param limit: Limit the number of results returned (e.g. to take a slice) 

597 :param wrap: Whether to return the results in raw json or wrapped as an object 

598 :param keepalive: scroll timeout 

599 

600 TODO: this is the old method, we should evaluate and aim to make iterate_pit the default scroll method. 

601 """ 

602 theq = {"query": {"match_all": {}}} if q is None else deepcopy(q) 

603 theq["size"] = page_size 

604 theq["from"] = 0 

605 if "sort" not in theq: 

606 # This gives the same performance enhancement as scan, use it by default. This is the order of indexing like sort by ID 

607 theq["sort"] = ["_doc"] 

608 

609 # Initialise the scroll 

610 try: 

611 res = cls.send_query(theq, scroll=keepalive) 

612 except Exception as e: 

613 raise ScrollInitialiseException("Unable to initialise scroll - could be your mappings are broken", e) 

614 

615 # unpack scroll response 

616 scroll_id = res.get('_scroll_id') 

617 total_results = res.get('hits', {}).get('total', {}).get('value') 

618 

619 # Supply the first set of results 

620 counter = 0 

621 for r in cls.handle_es_raw_response( 

622 res, 

623 wrap=wrap, 

624 extra_trace_info= 

625 "\nScroll initialised:\n{q}\n" 

626 "\n\nPage #{counter} of the ES response with size {page_size}." 

627 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)): 

628 

629 # apply the limit 

630 if limit is not None and counter >= int(limit): 

631 break 

632 counter += 1 

633 if wrap: 

634 yield cls(**r) 

635 else: 

636 yield r 

637 

638 # Continue to scroll through the rest of the results 

639 while True: 

640 # apply the limit 

641 if limit is not None and counter >= int(limit): 

642 break 

643 

644 # if we consumed all the results we were expecting, we can just stop here 

645 if counter >= total_results: 

646 break 

647 

648 # get the next page and check that we haven't timed out 

649 try: 

650 res = ES.scroll(scroll_id=scroll_id, scroll=keepalive) 

651 except elasticsearch.exceptions.NotFoundError as e: 

652 raise ScrollTimeoutException( 

653 "Scroll timed out; {status} - {message}".format(status=e.status_code, message=e.info)) 

654 except Exception as e: 

655 # if any other exception occurs, make sure it's at least logged. 

656 app.logger.exception("Unhandled exception in scroll method of DAO") 

657 raise ScrollException(e) 

658 

659 # if we didn't get any results back, this means we're at the end 

660 if len(res.get('hits', {}).get('hits')) == 0: 

661 break 

662 

663 for r in cls.handle_es_raw_response( 

664 res, 

665 wrap=wrap, 

666 extra_trace_info= 

667 "\nScroll:\n{q}\n" 

668 "\n\nPage #{counter} of the ES response with size {page_size}." 

669 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)): 

670 

671 # apply the limit 

672 if limit is not None and counter >= int(limit): 

673 break 

674 counter += 1 

675 if wrap: 

676 yield cls(**r) 

677 else: 

678 yield r 

679 

680 @classmethod 

681 def iterate_pit(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True, 

682 keepalive: str = '1m'): 

683 """ Provide an iterable of all items in a model, reimplemented using point-in-time queries 

684 :param q: The query to scroll results on 

685 :param page_size: limited by ElasticSearch, check settings to override 

686 :param limit: Limit the number of results returned (e.g. to take a slice) 

687 :param wrap: Whether to return the results in raw json or wrapped as an object 

688 :param keepalive: scroll timeout 

689 """ 

690 theq = {"query": {"match_all": {}}} if q is None else deepcopy(q) 

691 theq["size"] = page_size 

692 theq["from"] = 0 

693 if "sort" not in theq: 

694 # This gives the same performance enhancement as scan, use it by default. This is the order of indexing like sort by ID 

695 theq["sort"] = ["_doc"] 

696 

697 # Open a point in time query context 

698 res = ES.open_point_in_time(index=cls.index_name(), keep_alive=keepalive) 

699 pit_id = res.get("id") 

700 

701 theq["pit"] = { 

702 "id": pit_id, 

703 "keep_alive": keepalive 

704 } 

705 theq["track_total_hits"] = True 

706 

707 first_resp = cls.send_query(theq, pit_query=True) 

708 if len(first_resp.get('hits', {}).get('hits', [])) == 0: 

709 return 

710 

711 search_after = first_resp.get('hits', {}).get('hits', [])[-1].get('sort', []) 

712 total_results = first_resp.get('hits', {}).get('total', {}).get('value') 

713 

714 # Supply the first set of results 

715 counter = 0 

716 for r in cls.handle_es_raw_response( 

717 first_resp, 

718 wrap=wrap, 

719 extra_trace_info= 

720 "\nPIT Initialised:\n{q}\n" 

721 "\n\nPage #{counter} of the ES response with size {page_size}." 

722 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)): 

723 

724 # apply the limit 

725 if limit is not None and counter >= int(limit): 

726 break 

727 counter += 1 

728 if wrap: 

729 yield cls(**r) 

730 else: 

731 yield r 

732 

733 del theq["track_total_hits"] 

734 

735 # Continue to scroll through the rest of the results 

736 while True: 

737 # apply the limit 

738 if limit is not None and counter >= int(limit): 

739 break 

740 

741 # if we consumed all the results we were expecting, we can just stop here 

742 if counter >= total_results: 

743 break 

744 

745 theq["search_after"] = search_after 

746 

747 # get the next page and check that we haven't timed out 

748 try: 

749 res = cls.send_query(theq, pit_query=True) 

750 if len(res.get('hits', {}).get('hits', [])) == 0: 

751 break 

752 search_after = first_resp.get('hits', {}).get('hits', [])[-1].get('sort', []) 

753 except elasticsearch.exceptions.NotFoundError as e: 

754 raise ScrollTimeoutException( 

755 "PIT timed out; {status} - {message}".format(status=e.status_code, message=e.info)) 

756 except Exception as e: 

757 # if any other exception occurs, make sure it's at least logged. 

758 app.logger.exception("Unhandled exception in iterate_pit method of DAO") 

759 try: 

760 ES.close_point_in_time({"id": pit_id}) 

761 except: 

762 pass 

763 raise ScrollException(e) 

764 

765 for r in cls.handle_es_raw_response( 

766 res, 

767 wrap=wrap, 

768 extra_trace_info= 

769 "\nPIT:\n{q}\n" 

770 "\n\nPage #{counter} of the ES response with size {page_size}." 

771 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)): 

772 

773 # apply the limit 

774 if limit is not None and counter >= int(limit): 

775 break 

776 counter += 1 

777 if wrap: 

778 yield cls(**r) 

779 else: 

780 yield r 

781 

782 ES.close_point_in_time({"id": pit_id}) 

783 

784 @classmethod 

785 def iterate_unstable(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True, logger=None): 

786 """ Provide an iterable of all items in a model, using search_after but with no scroll context or 

787 PIT. This means that if the index changes as the iterate is happening, there may be repeated or 

788 missed elements. This is useful for cases where the index is not changing during the iteration, or the 

789 exact export is not important (e.g. for anon_export for testing purposes) 

790 

791 :param q: The query to scroll results on 

792 :param page_size: limited by ElasticSearch, check settings to override 

793 :param limit: Limit the number of results returned (e.g. to take a slice) 

794 :param wrap: Whether to return the results in raw json or wrapped as an object 

795 """ 

796 theq = {"query": {"match_all": {}}} if q is None else deepcopy(q) 

797 theq["size"] = page_size 

798 if "from" in theq: 

799 del theq["from"] 

800 

801 if "sort" not in theq: 

802 # This gives the same performance enhancement as scan, use it by default. This is the order of indexing like sort by ID 

803 theq["sort"] = [{"_id": "desc"}] 

804 

805 theq["track_total_hits"] = True 

806 

807 # if logger: logger(json.dumps(theq)) 

808 

809 first_resp = cls.send_query(theq) 

810 if len(first_resp.get('hits', {}).get('hits', [])) == 0: 

811 # if logger: logger("No results found") 

812 return 

813 

814 search_after = first_resp.get('hits', {}).get('hits', [])[-1].get('sort', []) 

815 total_results = first_resp.get('hits', {}).get('total', {}).get('value') 

816 if logger: logger(f"Expecting total {total_results}") 

817 

818 # Supply the first set of results 

819 counter = 0 

820 for r in cls.handle_es_raw_response( 

821 first_resp, 

822 wrap=wrap, 

823 extra_trace_info= 

824 "\nUnstable Iterate Initialised:\n{q}\n" 

825 "\n\nPage #{counter} of the ES response with size {page_size}." 

826 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)): 

827 

828 # apply the limit 

829 if limit is not None and counter >= int(limit): 

830 break 

831 counter += 1 

832 if wrap: 

833 yield cls(**r) 

834 else: 

835 yield r 

836 # if logger: logger(f"Iterated {counter} records") 

837 

838 del theq["track_total_hits"] 

839 

840 # Continue to scroll through the rest of the results 

841 while True: 

842 # apply the limit 

843 if limit is not None and counter >= int(limit): 

844 break 

845 

846 # if we consumed all the results we were expecting, we can just stop here 

847 if counter >= total_results: 

848 break 

849 

850 theq["search_after"] = search_after 

851 # if logger: logger(json.dumps(theq)) 

852 # if logger: logger("search_after: " + str(search_after)) 

853 try: 

854 res = cls.send_query(theq) 

855 if len(res.get('hits', {}).get('hits', [])) == 0: 

856 break 

857 search_after = res.get('hits', {}).get('hits', [])[-1].get('sort', []) 

858 except Exception as e: 

859 # if any exception occurs, make sure it's at least logged. 

860 app.logger.exception("Unhandled exception in iterate_unstable method of DAO") 

861 if logger: logger(f"Iterate failed on {json.dumps(theq)}") 

862 raise ScrollException(e) 

863 

864 for r in cls.handle_es_raw_response( 

865 res, 

866 wrap=wrap, 

867 extra_trace_info= 

868 "\nUnstable Iterate:\n{q}\n" 

869 "\n\nPage #{counter} of the ES response with size {page_size}." 

870 .format(q=json.dumps(theq, indent=2), counter=counter, page_size=page_size)): 

871 

872 # apply the limit 

873 if limit is not None and counter >= int(limit): 

874 break 

875 counter += 1 

876 if wrap: 

877 yield cls(**r) 

878 else: 

879 yield r 

880 

881 # if logger: logger(f"Iterated {counter} records") 

882 

883 @classmethod 

884 def iterall(cls, page_size=1000, limit=None, **kwargs): 

885 # TODO: Another candidate for swapping to iterate_pit (or rename iterate_pit to iterate when we're happy) 

886 return cls.iterate(MatchAllQuery().query(), page_size, limit, **kwargs) 

887 

888 @classmethod 

889 def iterall_unstable(cls, page_size=1000, 

890 stripe_field="id", 

891 striped=False, 

892 prefix_generator=None, 

893 prefix_size=4, 

894 limit=None, 

895 logger=None, 

896 must=None, 

897 **kwargs): 

898 def hex_prefixes(n=4): 

899 """ Generate a list of hex prefixes of length n """ 

900 return [str(hex(i))[2:].zfill(n) for i in range(0, 16 ** n)] 

901 

902 if striped: 

903 q = None 

904 if must is not None: 

905 q = { 

906 "query": { 

907 "bool": { 

908 "must": must 

909 } 

910 } 

911 } 

912 total = cls.count(q) 

913 if total == 0: 

914 return 

915 

916 count = 0 

917 prefixes = prefix_generator(prefix_size) if prefix_generator is not None else hex_prefixes(prefix_size) 

918 empty_prefixes = [] 

919 for prefix in prefixes: 

920 if must is None: 

921 q = { 

922 "query": { 

923 "prefix": {stripe_field: prefix} 

924 } 

925 } 

926 else: 

927 q = { 

928 "query": { 

929 "bool": { 

930 "must": must + [ 

931 { 

932 "prefix": {stripe_field: prefix} 

933 } 

934 ] 

935 } 

936 } 

937 } 

938 

939 first = True 

940 for record in cls.iterate_unstable(q, page_size, limit=limit, logger=logger, **kwargs): 

941 count += 1 

942 if logger and first: 

943 if len(empty_prefixes) > 0: 

944 logger(f"Skipped empty prefixes: {empty_prefixes}") 

945 empty_prefixes = [] 

946 logger(f"Exporting prefix: {prefix}") 

947 first = False 

948 if limit is not None: 

949 if count > limit: 

950 if logger: logger(f"Limit reached: {count} / {limit}") 

951 return 

952 yield record 

953 

954 if first: 

955 empty_prefixes.append(prefix) 

956 else: 

957 if logger: logger(f"Finished prefix: {prefix}; {count} total records") 

958 if len(empty_prefixes) > 0: 

959 if logger: logger(f"Skipped empty prefixes: {empty_prefixes}") 

960 else: 

961 if logger: logger("Exporting without prefix striping") 

962 for record in cls.iterate_unstable(q=None, page_size=page_size, limit=limit, logger=logger, **kwargs): 

963 yield record 

964 

965 # Aliases for the iterate functions 

966 scroll = iterate 

967 scroll_pit = iterate_pit 

968 

969 @classmethod 

970 def dump(cls, q=None, page_size=1000, limit=None, out=None, out_template=None, out_batch_sizes=100000, 

971 out_rollover_callback=None, transform=None, es_bulk_format=True, idkey='id', es_bulk_fields=None, 

972 stripe_field="id", striped=False, prefix_generator=None, prefix_size=3, logger=None): 

973 """ Export to file, bulk format or just a json dump of the record """ 

974 

975 filenames = [] 

976 n = 1 

977 current_file = None 

978 if out_template is not None: 

979 current_file = out_template + "." + str(n) 

980 filenames.append(current_file) 

981 if out is None and current_file is not None: 

982 out = open(current_file, "w") 

983 else: 

984 out = sys.stdout 

985 

986 count = 0 

987 if q is None: 

988 iterator = cls.iterall_unstable(page_size=page_size, stripe_field=stripe_field, 

989 striped=striped, prefix_generator=prefix_generator, 

990 prefix_size=prefix_size, 

991 limit=limit, wrap=False, logger=logger) 

992 else: 

993 iterator = cls.iterate_unstable(q, page_size=page_size, limit=limit, wrap=False) 

994 

995 for record in iterator: 

996 if transform is not None: 

997 record = transform(record) 

998 if record is None: 

999 # transform declined to alter (e.g. anonymise) this record - drop it rather than 

1000 # writing raw/partially-anonymised data to the export 

1001 continue 

1002 

1003 if es_bulk_format: 

1004 kwargs = {} 

1005 if es_bulk_fields is None: 

1006 es_bulk_fields = ["_id", "_index", "_type"] 

1007 for key in es_bulk_fields: 

1008 if key == "_id": 

1009 kwargs["idkey"] = idkey 

1010 if key == "_index": 

1011 kwargs["index"] = cls.index_name() 

1012 if key == "_type": 

1013 kwargs["type_"] = type 

1014 data = cls.to_bulk_single_rec(record, **kwargs) 

1015 else: 

1016 data = json.dumps(record) + "\n" 

1017 

1018 out.write(data) 

1019 if out_template is not None: 

1020 count += 1 

1021 if count > out_batch_sizes: 

1022 out.close() 

1023 if out_rollover_callback is not None: 

1024 out_rollover_callback(current_file) 

1025 

1026 count = 0 

1027 n += 1 

1028 current_file = out_template + "." + str(n) 

1029 filenames.append(current_file) 

1030 out = open(current_file, "w") 

1031 

1032 if out_template is not None: 

1033 out.close() 

1034 if out_rollover_callback is not None: 

1035 out_rollover_callback(current_file) 

1036 

1037 return filenames 

1038 

1039 @classmethod 

1040 def bulk_load_from_file(cls, source_file, index=None, limit=None, max_content_length=100000000): 

1041 """ ported from esprit.tasks - bulk load to index from file 

1042 :param source_file 

1043 :param index: index name for target 

1044 :param limit: number of records to load (integer) 

1045 :param max_content_length: Upload chunk size in bytes 

1046 """ 

1047 index = index or cls.index_name() 

1048 

1049 source_size = os.path.getsize(source_file) 

1050 with open(source_file, "r") as f: 

1051 if limit is None and source_size < max_content_length: 

1052 # if we aren't selecting a portion of the file, and the file is below the max content length, then 

1053 # we can just serve it directly 

1054 ES.bulk(body=f.read(), index=index, doc_type=cls.doc_type(), request_timeout=120) 

1055 return -1 

1056 else: 

1057 count = 0 

1058 while True: 

1059 chunk = DomainObject._make_next_chunk(f, max_content_length) 

1060 if chunk == "": 

1061 break 

1062 

1063 finished = False 

1064 if limit is not None: 

1065 newlines = chunk.count("\n") 

1066 records = newlines // 2 

1067 if count + records > limit: 

1068 max = (limit - count) * 2 

1069 lines = chunk.split("\n") 

1070 allowed = lines[:max] 

1071 chunk = "\n".join(allowed) + "\n" 

1072 count += max 

1073 finished = True 

1074 else: 

1075 count += records 

1076 

1077 ES.bulk(body=chunk, index=index, doc_type=cls.doc_type(), request_timeout=120) 

1078 if finished: 

1079 break 

1080 if limit is not None: 

1081 return count 

1082 else: 

1083 return -1 

1084 

1085 @staticmethod 

1086 def make_bulk_chunk_files(source_file, out_file_prefix, max_content_length=100000000): 

1087 """ ported from esprit.tasks - break out a bulk file into smaller chunks """ 

1088 

1089 source_size = os.path.getsize(source_file) 

1090 with open(source_file, "r") as f: 

1091 if source_size < max_content_length: 

1092 return [source_file] 

1093 else: 

1094 filenames = [] 

1095 count = 0 

1096 while True: 

1097 count += 1 

1098 chunk = DomainObject._make_next_chunk(f, max_content_length) 

1099 if chunk == "": 

1100 break 

1101 

1102 filename = out_file_prefix + "." + str(count) 

1103 with open(filename, "w") as g: 

1104 g.write(chunk) 

1105 filenames.append(filename) 

1106 

1107 return filenames 

1108 

1109 @staticmethod 

1110 def _make_next_chunk(f, max_content_length): 

1111 """ ported from esprit.tasks - create a bulk chunk, ensuring it's not a partial instruction """ 

1112 

1113 def is_command(line): 

1114 try: 

1115 command = json.loads(line) 

1116 except (json.JSONDecodeError, TypeError): 

1117 return False 

1118 keys = list(command.keys()) 

1119 if len(keys) > 1: 

1120 return False 

1121 if "index" not in keys: 

1122 return False 

1123 subkeys = list(command["index"].keys()) 

1124 for sk in subkeys: 

1125 if sk not in ["_id"]: 

1126 return False 

1127 

1128 return True 

1129 

1130 offset = f.tell() 

1131 chunk = f.read(max_content_length) 

1132 while True: 

1133 last_newline = chunk.rfind("\n") 

1134 tail = chunk[last_newline + 1:] 

1135 chunk = chunk[:last_newline] 

1136 

1137 if is_command(tail): 

1138 f.seek(offset + last_newline) 

1139 if chunk.startswith("\n"): 

1140 chunk = chunk[1:] 

1141 return chunk 

1142 else: 

1143 continue 

1144 

1145 @classmethod 

1146 def prefix_query(cls, field, prefix, filter_condition=None, size=5, facet_field=None, analyzed_field=True): 

1147 # example of a prefix query 

1148 # { 

1149 # "query": {"prefix" : { "bibjson.publisher" : "ope" } }, 

1150 # "size": 0, 

1151 # "facets" : { 

1152 # "publisher" : { "terms" : {"field" : "bibjson.publisher.exact", "size": 5} } 

1153 # } 

1154 # } 

1155 

1156 suffix = app.config['FACET_FIELD'] 

1157 query_field = field 

1158 if analyzed_field: 

1159 if field.endswith(suffix): 

1160 # strip .exact (or whatever it's configured as) off the end 

1161 query_field = field[:field.rfind(suffix)] 

1162 else: 

1163 if not field.endswith(suffix): 

1164 query_field = field + suffix 

1165 

1166 # the actual terms should come from the .exact version of the 

1167 # field - we are suggesting whole values, not fragments 

1168 if facet_field is None: 

1169 facet_field = query_field + suffix 

1170 if not facet_field.endswith(suffix): 

1171 facet_field = facet_field + suffix 

1172 

1173 q = PrefixAutocompleteQuery(query_field, prefix, field, facet_field, size, filter_condition) 

1174 return cls.send_query(q.query()) 

1175 

1176 @classmethod 

1177 def wildcard_autocomplete_query(cls, field, substring, before=True, after=True, facet_size=5, facet_field=None): 

1178 """ 

1179 Example of a wildcard query 

1180 Works only on .exact fields 

1181 

1182 { 

1183 "query" : { 

1184 "wildcard" : {"bibjson.publisher.exact" : "De *"} 

1185 }, 

1186 "size" : 0, 

1187 "facets" : { 

1188 "bibjson.publisher.exact" : { 

1189 "terms" : {"field" : "bibjson.publisher.exact", "size" : 5} 

1190 } 

1191 } 

1192 } 

1193 :param field: 

1194 :param substring: 

1195 :param facet_size: 

1196 :return: 

1197 """ 

1198 # wildcard queries need to be on unanalyzed fields 

1199 suffix = app.config['FACET_FIELD'] 

1200 filter_field = field 

1201 if not filter_field.endswith(suffix): 

1202 filter_field = filter_field + suffix 

1203 

1204 # add the wildcard before/after 

1205 if before: 

1206 substring = "*" + substring 

1207 if after: 

1208 substring = substring + "*" 

1209 

1210 # sort out the facet field 

1211 if facet_field is None: 

1212 facet_field = filter_field 

1213 if not facet_field.endswith(suffix): 

1214 facet_field = facet_field + suffix 

1215 

1216 # build the query 

1217 q = WildcardAutocompleteQuery(filter_field, substring, field, facet_field, facet_size) 

1218 return cls.send_query(q.query()) 

1219 

1220 @classmethod 

1221 def advanced_autocomplete(cls, filter_field, facet_field, substring, size=5, prefix_only=True): 

1222 analyzed = True 

1223 if " " in substring: 

1224 analyzed = False 

1225 

1226 substring = substring.lower() 

1227 

1228 if " " in substring and not prefix_only: 

1229 res = cls.wildcard_autocomplete_query(filter_field, substring, before=True, after=True, facet_size=size, 

1230 facet_field=facet_field) 

1231 else: 

1232 res = cls.prefix_query(filter_field, substring, size=size, facet_field=facet_field, analyzed_field=analyzed) 

1233 

1234 result = [] 

1235 for term in res['aggregations'][filter_field]['buckets']: 

1236 # keep ordering - it's by count by default, so most frequent 

1237 # terms will now go to the front of the result list 

1238 result.append({"id": term['key'], "text": term['key']}) 

1239 return result 

1240 

1241 @classmethod 

1242 def autocomplete(cls, field, prefix, filter_condition=None, size=5): 

1243 res = None 

1244 # if there is a space in the prefix, the prefix query won't work, so we fall back to a wildcard 

1245 # we only do this if we have to, because the wildcard query is a little expensive 

1246 if " " in prefix: 

1247 res = cls.wildcard_autocomplete_query(field, prefix, before=False, after=True, facet_size=size) 

1248 else: 

1249 prefix = prefix.lower() 

1250 res = cls.prefix_query(field, prefix, filter_condition, size=size) 

1251 

1252 result = [] 

1253 for term in res['aggregations'][field]['buckets']: 

1254 # keep ordering - it's by count by default, so most frequent 

1255 # terms will now go to the front of the result list 

1256 result.append({"id": term['key'], "text": term['key']}) 

1257 return result 

1258 

1259 @classmethod 

1260 def q2obj(cls, **kwargs): 

1261 extra_trace_info = '' 

1262 if 'q' in kwargs: 

1263 extra_trace_info = "\nQuery sent to ES (before manipulation in DomainObject.query):\n{}\n".format( 

1264 json.dumps(kwargs['q'], indent=2)) 

1265 

1266 res = cls.query(**kwargs) 

1267 rs = cls.handle_es_raw_response(res, wrap=True, extra_trace_info=extra_trace_info) 

1268 results = [cls(**r) for r in rs] 

1269 return results 

1270 

1271 @classmethod 

1272 def all(cls, size=10000, **kwargs): 

1273 """ This is a shortcut to a match_all query with a large size, to return all records """ 

1274 # FIXME: is this only used in tests? ES now limits size so we can't guarantee ALL without using scroll / scan 

1275 return cls.q2obj(size=size, **kwargs) 

1276 

1277 @classmethod 

1278 def count(cls, query=None): 

1279 res = ES.count(index=cls.index_name(), doc_type=cls.doc_type(), body=query) 

1280 return res.get("count") 

1281 # return requests.get(cls.target() + '_count').json()['count'] 

1282 

1283 @classmethod 

1284 def hit_count(cls, query, **kwargs) -> int: 

1285 countable_query = deepcopy(query) 

1286 if "track_total_hits" not in countable_query: 

1287 countable_query["track_total_hits"] = True 

1288 

1289 res = cls.query(q=countable_query, **kwargs) 

1290 return res.get("hits", {}).get("total", {}).get("value", 0) 

1291 

1292 @classmethod 

1293 def count_updated_since(cls, last_update): 

1294 """ 

1295 Count the number of records updated since a given date 

1296 :param last_update: The date to count from 

1297 :return: The number of records updated since the given date 

1298 """ 

1299 q = { 

1300 "query": { 

1301 "range": { 

1302 "last_updated": { 

1303 "gte": last_update 

1304 } 

1305 } 

1306 } 

1307 } 

1308 return cls.count(q) 

1309 

1310 @classmethod 

1311 def block(cls, id, last_updated=None, sleep=0.5, max_retry_seconds=30): 

1312 if app.config.get("ES_BLOCK_WAIT_OVERRIDE") is not None: 

1313 sleep = app.config["ES_BLOCK_WAIT_OVERRIDE"] 

1314 

1315 q = BlockQuery(id) 

1316 start_time = dates.now() 

1317 while True: 

1318 res = cls.query(q=q.query()) 

1319 hits = res.get("hits", {}).get("hits", []) 

1320 if len(hits) > 0: 

1321 if len(hits) > 1: 

1322 raise Exception("More than one record with id {x}".format(x=id)) 

1323 obj = hits[0].get("_source", {}) 

1324 if last_updated is not None: 

1325 lu = obj.get("last_updated") 

1326 if lu: 

1327 threshold = dates.parse(last_updated) 

1328 lud = dates.parse(lu) 

1329 if lud >= threshold: 

1330 return 

1331 else: 

1332 return 

1333 

1334 if (dates.now() - start_time).total_seconds() >= max_retry_seconds: 

1335 raise (BlockTimeOutException( 

1336 "Attempting to block until record with id {id} appears in the index, but this has not happened after {limit} seconds" 

1337 .format( 

1338 id=id, limit=max_retry_seconds))) 

1339 

1340 time.sleep(sleep) 

1341 

1342 @classmethod 

1343 def blockall(cls, ids_and_last_updateds, sleep=0.05, individual_max_retry_seconds=30): 

1344 for id, lu in ids_and_last_updateds: 

1345 cls.block(id, lu, sleep=sleep, max_retry_seconds=individual_max_retry_seconds) 

1346 

1347 @classmethod 

1348 def blockdeleted(cls, id, sleep=0.5, max_retry_seconds=30): 

1349 if app.config.get("ES_BLOCK_WAIT_OVERRIDE") is not None: 

1350 sleep = app.config["ES_BLOCK_WAIT_OVERRIDE"] 

1351 

1352 q = BlockQuery(id) 

1353 start_time = dates.now() 

1354 while True: 

1355 res = cls.query(q=q.query()) 

1356 hits = res.get("hits", {}).get("hits", []) 

1357 if len(hits) == 0: 

1358 return 

1359 else: 

1360 if (dates.now() - start_time).total_seconds() >= max_retry_seconds: 

1361 raise BlockTimeOutException( 

1362 "Attempting to block until record with id {id} deleted from Elasticsearch, but this has not happened after {limit}".format( 

1363 id=id, limit=max_retry_seconds)) 

1364 

1365 time.sleep(sleep) 

1366 

1367 @classmethod 

1368 def blockalldeleted(cls, ids, sleep=0.05, individual_max_retry_seconds=30): 

1369 for id in ids: 

1370 cls.blockdeleted(id, sleep, individual_max_retry_seconds) 

1371 

1372 @classmethod 

1373 def save_all(cls, models, blocking=False): 

1374 for m in models: 

1375 m.save() 

1376 if blocking: 

1377 cls.blockall((m.id, getattr(m, "last_updated", None)) for m in models) 

1378 

1379 @classmethod 

1380 def create_and_seed_index_and_rollover_alias(cls, documents: List[dict], mapping=None, keep_history=0): 

1381 """ 

1382 Create a new index based on this object's type, put the mapping if provided, and seed it with the documents. 

1383 The indexes alias is then repointed to the new index. 

1384 If a keep_history value is provided, the previous indexes with the same prefix as the alias will be deleted, 

1385 

1386 :param documents: may be empty, to roll the alias over onto a new empty index (then a mapping is required) 

1387 :param mapping: the mappings, or a full index body ({"mappings": ..., "settings": ...}) to also apply index settings 

1388 :param keep_history: supply -1 to keep all history, 0 to keep no history, or a positive integer to keep that many previous indexes. 

1389 :return: 

1390 """ 

1391 has_documents = len(documents) > 0 

1392 if not has_documents and mapping is None: 

1393 raise ValueError("A mapping is required to roll over onto an empty index") 

1394 

1395 full_alias = cls.index_name() 

1396 new_name = cls.__type__ + "-" + dates.now_str(dates.FMT_DATETIME_LONG) 

1397 if mapping is not None: 

1398 cls.put_mapping(mapping, override_index_name=new_name) 

1399 if has_documents: 

1400 resp = cls.bulk(documents, refresh=True, override_index_name=new_name) 

1401 if resp.get('errors', False): 

1402 raise ESError("Error creating index {}: {}".format(new_name, resp)) 

1403 cls.move_alias(full_alias, new_name) 

1404 

1405 if keep_history > -1: 

1406 # Find all indexes with the same prefix as full_alias 

1407 all_indexes = find_indexes_by_prefix(full_alias) 

1408 all_indexes.sort() 

1409 keep_history += 1 

1410 for idx in all_indexes[:-keep_history]: 

1411 try: 

1412 ES.indices.delete(index=idx) 

1413 except elasticsearch.exceptions.NotFoundError: 

1414 pass 

1415 except elasticsearch.exceptions.RequestError as e: 

1416 raise ESError(e) 

1417 

1418 

1419 @classmethod 

1420 def put_mapping(cls, mapping: dict, override_index_name: str = None): 

1421 """ 

1422 Put a mapping for the index 

1423 :param mapping: The mapping to put. If None then use cls.mapping() 

1424 """ 

1425 try: 

1426 if "mappings" not in mapping: 

1427 mapping = {"mappings": mapping} 

1428 return ES.indices.create(index=cls.index_name(override_index_name), 

1429 body=mapping, 

1430 request_timeout=app.config.get("ES_SOCKET_TIMEOUT", None)) 

1431 except elasticsearch.exceptions.RequestError as e: 

1432 raise ESError(e) 

1433 

1434 @classmethod 

1435 def move_alias(cls, alias_name: str, target_index: str): 

1436 """ 

1437 Create or update an alias for the index 

1438 :param alias_name: The name of the alias to create or update 

1439 :param target_index: The name of the index to point the alias to 

1440 """ 

1441 

1442 try: 

1443 alias_info = ES.indices.get_alias(name=alias_name) 

1444 old_index_name = list(alias_info.keys())[0] 

1445 full_index_name = cls.index_name(target_index) 

1446 body = { 

1447 "actions": [ 

1448 { 

1449 "add": { 

1450 "index": full_index_name, 

1451 "alias": alias_name 

1452 } 

1453 }, 

1454 { 

1455 "remove": { 

1456 "index": old_index_name, 

1457 "alias": alias_name 

1458 } 

1459 } 

1460 ] 

1461 } 

1462 ES.indices.update_aliases(body=body) 

1463 except elasticsearch.exceptions.RequestError as e: 

1464 raise ESError(e) 

1465 

1466 

1467 

1468 

1469def any_pending_tasks(): 

1470 """ Check if there are any pending tasks in the elasticsearch task queue """ 

1471 results = ES.cluster.pending_tasks() 

1472 return len(results["tasks"]) > 0 

1473 

1474 

1475def query_data_tasks(timeout='30s'): 

1476 """ Check if there are any pending tasks in the elasticsearch task queue """ 

1477 results = ES.tasks.list(params={ 

1478 "actions": 'indices:data*', 

1479 "timeout": timeout, 

1480 "wait_for_completion": 'true', 

1481 }) 

1482 tasks = [] 

1483 for node in results['nodes'].values(): 

1484 tasks.extend(node['tasks'].values()) 

1485 return tasks 

1486 

1487 

1488def refresh(): 

1489 """ 

1490 refresh all indexes to make newly added or deleted documents immediately searchable 

1491 """ 

1492 return ES.indices.refresh() 

1493 

1494 

1495def find_indexes_by_prefix(index_prefix) -> list[str]: 

1496 data = ES.indices.get(f'{index_prefix}*') 

1497 return list(data.keys()) 

1498 

1499 

1500def find_index_aliases(alias_prefixes=None) -> Iterable[Tuple[str, str]]: 

1501 def _yield_index_alias(): 

1502 data = ES.indices.get_alias() 

1503 for index, d in data.items(): 

1504 for alias in d['aliases'].keys(): 

1505 yield index, alias 

1506 

1507 index_aliases = _yield_index_alias() 

1508 if alias_prefixes: 

1509 index_aliases = ((index, alias) for index, alias in index_aliases 

1510 if any(alias.startswith(p) for p in alias_prefixes)) 

1511 return index_aliases 

1512 

1513 

1514def is_exist(query: dict, index): 

1515 query['size'] = 1 

1516 query['_source'] = False 

1517 res = ES.search(body=query, index=index, size=1, ignore=[404]) 

1518 

1519 return res.get('hits', {}).get('total',{}).get('value', 0) > 0 

1520 

1521 

1522class BlockTimeOutException(Exception): 

1523 pass 

1524 

1525 

1526class DAOSaveExceptionMaxRetriesReached(Exception): 

1527 pass 

1528 

1529 

1530class ESResponseCannotBeInterpreted(Exception): 

1531 pass 

1532 

1533 

1534class ESMappingMissingError(Exception): 

1535 pass 

1536 

1537 

1538class ESError(Exception): 

1539 pass 

1540 

1541 

1542######################################################################## 

1543# Some useful ES queries 

1544######################################################################## 

1545 

1546 

1547class MatchAllQuery(object): 

1548 def query(self): 

1549 return { 

1550 "track_total_hits": True, 

1551 "query": { 

1552 "match_all": {} 

1553 } 

1554 } 

1555 

1556 

1557class BlockQuery(object): 

1558 def __init__(self, id): 

1559 self._id = id 

1560 

1561 def query(self): 

1562 return { 

1563 "query": { 

1564 "ids": { 

1565 "values": [self._id] 

1566 } 

1567 }, 

1568 "_source": ["last_updated"], 

1569 "size": 2 

1570 } 

1571 

1572 

1573class PrefixAutocompleteQuery(object): 

1574 def __init__(self, query_field, prefix, agg_name, agg_field, agg_size, filter_condition=None): 

1575 self._query_field = query_field 

1576 self._prefix = prefix 

1577 self._agg_name = agg_name 

1578 self._agg_field = agg_field 

1579 self._agg_size = agg_size 

1580 self._filter_condition = filter_condition 

1581 def query(self): 

1582 query_body = { 

1583 "track_total_hits": True, 

1584 "query": { 

1585 "bool": { 

1586 "must": [ 

1587 {"prefix": {self._query_field: self._prefix.lower()}} # Keep the prefix query 

1588 ] 

1589 } 

1590 }, 

1591 "size": 0, 

1592 "aggs": { 

1593 self._agg_name: {"terms": {"field": self._agg_field, "size": self._agg_size}} 

1594 } 

1595 } 

1596 

1597 if self._filter_condition: 

1598 query_body["query"]["bool"]["filter"] = [ 

1599 {"term": self._filter_condition} 

1600 ] 

1601 

1602 return query_body 

1603 

1604 

1605 

1606class WildcardAutocompleteQuery(object): 

1607 def __init__(self, wildcard_field, wildcard_query, agg_name, agg_field, agg_size): 

1608 self._wildcard_field = wildcard_field 

1609 self._wildcard_query = wildcard_query 

1610 self._agg_name = agg_name 

1611 self._agg_field = agg_field 

1612 self._agg_size = agg_size 

1613 

1614 def query(self): 

1615 return { 

1616 "track_total_hits": True, 

1617 "query": { 

1618 "wildcard": {self._wildcard_field: self._wildcard_query} 

1619 }, 

1620 "size": 0, 

1621 "aggs": { 

1622 self._agg_name: { 

1623 "terms": {"field": self._agg_field, "size": self._agg_size} 

1624 } 

1625 } 

1626 } 

1627 

1628 

1629######################################################################### 

1630# A query handler that knows how to speak facetview2 

1631######################################################################### 

1632 

1633 

1634class Facetview2(object): 

1635 """ 

1636 ~~SearchURLGenerator:Feature->Elasticsearch:Technology~~ 

1637 

1638 # Examples of queries 

1639 # {"query":{"filtered":{"filter":{"bool":{"must":[{"term":{"_type":"article"}}]}},"query":{"query_string":{"query":"richard","default_operator":"OR"}}}},"from":0,"size":10} 

1640 # {"query":{"query_string":{"query":"richard","default_operator":"OR"}},"from":0,"size":10} 

1641 """ 

1642 @staticmethod 

1643 def make_term_filter(term, value): 

1644 return {"term": {term: value}} 

1645 

1646 @staticmethod 

1647 def make_query(query_string=None, filters=None, default_operator="OR", sort_parameter=None, sort_order="asc", 

1648 default_field=None): 

1649 query_part = {"match_all": {}} 

1650 if query_string is not None: 

1651 query_part = {"query_string": {"query": query_string, "default_operator": default_operator}} 

1652 if default_field is not None: 

1653 query_part["query_string"]["default_field"] = default_field 

1654 query = {"query": query_part} 

1655 

1656 if filters is not None: 

1657 if not isinstance(filters, list): 

1658 filters = [filters] 

1659 filters.append(query_part) 

1660 bool_part = {"bool": {"must": filters}} 

1661 query = {"query": query_part} 

1662 

1663 if sort_parameter is not None: 

1664 # For facetview we can only have one sort parameter, but ES actually supports lists 

1665 sort_part = [{sort_parameter: {"order": sort_order}}] 

1666 query["sort"] = sort_part 

1667 

1668 return query 

1669 

1670 @staticmethod 

1671 def url_encode_query(query): 

1672 return urllib.parse.quote(json.dumps(query).replace(' ', '')) 

1673 

1674 

1675def patch_model_for_bulk(obj: DomainObject): 

1676 obj.data['es_type'] = obj.__type__ 

1677 obj.data['id'] = obj.makeid() 

1678 return obj