Coverage for portality/bll/services/background_task_status.py: 88%
110 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1"""
2~~BackgroundTask:Monitoring~~
3"""
4import itertools
5from typing import Iterable, Optional
7from portality import constants
8from portality.core import app
9from portality.lib import dates
10from portality.models.background import BackgroundJobQueryBuilder, BackgroundJob, SimpleBgjobQueue, \
11 LastCompletedJobQuery
12from portality.tasks.helpers import background_helper
15class BackgroundTaskStatusService:
16 """
17 "background_task_status" is concept for list current background task status for monitoring.
18 `create_background_status` is important function in service for create background_status data
19 """
21 @staticmethod
22 def is_stable(val):
23 return val == constants.BG_STATUS_STABLE
25 @staticmethod
26 def to_bg_status_str(stable_val: bool) -> str:
27 return constants.BG_STATUS_STABLE if stable_val else constants.BG_STATUS_UNSTABLE
29 def all_stable(self, items: Iterable, field_name='status') -> bool:
30 return all(self.is_stable(q.get(field_name)) for q in items)
32 def all_stable_str(self, items: Iterable, field_name='status') -> str:
33 return self.to_bg_status_str(self.all_stable(items, field_name))
35 def create_last_successfully_run_status(self, action, last_run_successful_in=0, **_) -> dict:
36 if last_run_successful_in == 0:
37 return dict(
38 status=constants.BG_STATUS_STABLE,
39 last_run=None,
40 last_run_status=None,
41 err_msgs=[]
42 )
44 lr_query = (BackgroundJobQueryBuilder().action(action)
45 .since(dates.before_now(last_run_successful_in))
46 .status_includes([constants.BGJOB_STATUS_COMPLETE, constants.BGJOB_STATUS_ERROR])
47 .size(1)
48 .order_by('created_date', 'desc')
49 .build_query_dict())
51 try:
52 lr_results = BackgroundJob.q2obj(q=lr_query)
53 lr_job = lr_results[0] if len(lr_results) > 0 else None
54 except Exception as e:
55 app.logger.error(f'Error querying last successfully run status for action {action}: {e}')
56 lr_job = None
58 status = constants.BG_STATUS_UNSTABLE
59 lr = None
60 last_run_status = None
61 msg = ["No background jobs completed or errored in the time period"]
63 if lr_job is not None:
64 lr = lr_job.created_date
65 last_run_status = lr_job.status
66 if lr_job.status == constants.BGJOB_STATUS_COMPLETE:
67 status = constants.BG_STATUS_STABLE
68 msg = []
69 else:
70 msg = ["Last job did not complete successfully"]
72 return dict(
73 status=status,
74 last_run=lr,
75 last_run_status=last_run_status,
76 err_msgs=msg
77 )
79 def create_errors_status(self, action, check_sec=3600, allowed_num_err: Optional[int] = 0, **_) -> dict:
80 if allowed_num_err is None:
81 # error-count check disabled for this action - status is determined solely by
82 # the last_run_successful check instead
83 return dict(
84 status=constants.BG_STATUS_STABLE,
85 total=BackgroundJob.hit_count(query=SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_ERROR).query()),
86 in_monitoring_period=None,
87 err_msgs=[],
88 )
90 in_monitoring_query = SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_ERROR, since=dates.before_now(check_sec))
91 num_err_in_monitoring = BackgroundJob.hit_count(query=in_monitoring_query.query())
93 # prepare errors messages
94 err_msgs = []
95 if num_err_in_monitoring > allowed_num_err:
96 err_msgs.append(f'too many error in monitoring period [{num_err_in_monitoring} > {allowed_num_err}]')
98 return dict(
99 status=self.to_bg_status_str(not err_msgs),
100 total=BackgroundJob.hit_count(query=SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_ERROR).query()),
101 in_monitoring_period=num_err_in_monitoring,
102 err_msgs=err_msgs,
103 )
105 def create_queued_status(self, action, total=2, oldest=1200, **_) -> dict:
106 total_queued = BackgroundJob.hit_count(query=SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_QUEUED).query())
107 oldest_query = (BackgroundJobQueryBuilder().action(action)
108 .status_includes(constants.BGJOB_STATUS_QUEUED).size(1)
109 .order_by('created_date', 'asc')
110 .build_query_dict())
111 try:
112 oldest_jobs = list(BackgroundJob.q2obj(q=oldest_query))
113 oldest_job = oldest_jobs and oldest_jobs[0]
114 except Exception as e:
115 app.logger.error(f'Error querying queued status for action {action}: {e}')
116 oldest_jobs = []
117 oldest_job = None
119 err_msgs = []
120 limited_oldest_date = dates.before_now(oldest)
121 if oldest_job and oldest_job.created_timestamp < limited_oldest_date:
122 err_msgs.append('outdated queued job found[{}]. created_timestamp[{} < {}]'.format(
123 oldest_job.id,
124 oldest_job.created_timestamp,
125 limited_oldest_date
126 ))
128 if total_queued > total:
129 err_msgs.append(f'too many queued job [{total_queued} > {total}]')
131 return dict(
132 status=self.to_bg_status_str(not err_msgs),
133 total=total_queued,
134 oldest=oldest_job.created_date if oldest_job else None,
135 err_msgs=err_msgs,
136 )
138 def create_queues_status(self, queue_name) -> dict:
139 # define last_completed_job
140 try:
141 bgjob_list = BackgroundJob.q2obj(q=LastCompletedJobQuery(queue_name).query())
142 bgjob_list = list(bgjob_list)
143 if bgjob_list:
144 last_completed_date = bgjob_list[0].last_updated_timestamp
145 else:
146 last_completed_date = None
147 except Exception as e:
148 app.logger.error(f'Error querying last completed job for queue {queue_name}: {e}')
149 bgjob_list = []
150 last_completed_date = None
152 errors = {action: self.create_errors_status(action, **config) for action, config
153 in self.get_config_dict_by_queue_name('BG_MONITOR_ERRORS_CONFIG', queue_name).items()}
155 queued = {action: self.create_queued_status(action, **config) for action, config
156 in self.get_config_dict_by_queue_name('BG_MONITOR_QUEUED_CONFIG', queue_name).items()}
158 last_run = {action: self.create_last_successfully_run_status(action, **config) for action, config
159 in self.get_config_dict_by_queue_name('BG_MONITOR_LAST_SUCCESSFULLY_RUN_CONFIG', queue_name).items()}
161 # prepare for err_msgs
162 limited_sec = app.config.get('BG_MONITOR_LAST_COMPLETED', {}).get(queue_name)
163 if limited_sec is None:
164 app.logger.warning(f'BG_MONITOR_LAST_COMPLETED for {queue_name} not found ')
166 err_msgs = []
167 if limited_sec is not None and last_completed_date:
168 limited_date = dates.before_now(limited_sec)
169 if last_completed_date < limited_date:
170 err_msgs.append(
171 f'last completed job is too old. [{last_completed_date} < {limited_date}]'
172 )
174 result_dict = dict(
175 status=self.to_bg_status_str(
176 not err_msgs and self.all_stable(itertools.chain(errors.values(), queued.values(), last_run.values()))),
177 last_completed_job=last_completed_date and dates.format(last_completed_date),
178 errors=errors,
179 queued=queued,
180 last_run_successful=last_run,
181 err_msgs=err_msgs,
182 )
183 return result_dict
185 @staticmethod
186 def get_config_dict_by_queue_name(config_name, queue_name):
187 bg_specs = background_helper.get_all_background_task_specs()
188 actions = {action for qn, action, _ in bg_specs
189 if qn == queue_name}
191 return {
192 k: app.config.get(config_name, {}).get(k, app.config.get('BG_MONITOR_DEFAULT_CONFIG'))
193 for k in actions
194 }
196 def create_background_status(self) -> dict:
197 queues = {
198 queue_name: self.create_queues_status(queue_name)
199 for queue_name in [constants.BGJOB_QUEUE_ID_EVENTS,
200 constants.BGJOB_QUEUE_ID_SCHEDULED_LONG,
201 constants.BGJOB_QUEUE_ID_SCHEDULED_SHORT]
202 }
204 result_dict = dict(
205 status=(self.all_stable_str(queues.values())),
206 queues=queues,
207 )
209 # sort the results in the order of unstable status
210 sorted_data = self.sort_dict_by_unstable_status(result_dict)
212 return sorted_data
214 def sort_dict_by_unstable_status(self, data):
215 """
216 Sorts each dictionary within the nested structure by prioritizing items with 'status': 'unstable'.
217 The overall structure of the input dictionary is preserved.
218 """
219 if isinstance(data, dict):
220 # Extract items with 'status': 'unstable' and other items
221 unstable_items = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('status') == 'unstable'}
222 other_items = {k: v for k, v in data.items() if k not in unstable_items}
224 # Recursively sort nested dictionaries
225 for k in unstable_items:
226 unstable_items[k] = self.sort_dict_by_unstable_status(unstable_items[k])
227 for k in other_items:
228 other_items[k] = self.sort_dict_by_unstable_status(other_items[k])
230 # Merge the dictionaries, with unstable items first
231 return {**unstable_items, **other_items}
232 else:
233 # Return the item as is if it's not a dict
234 return data