Coverage for portality/tasks/anon_export.py: 88%
184 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1import functools
2import gzip
3import os
4import shutil
5import uuid
6from typing import Callable
7from werkzeug.security import generate_password_hash
9from portality import models, dao
10from portality.background import BackgroundTask
11from portality.core import app, es_connection
12from portality.lib import dates
13from portality.lib.anon import basic_hash, anon_email
14from portality.lib.dataobj import DataStructureException
15from portality.lib.seamless import SeamlessException
16from portality.models import Note
17from portality.store import StoreFactory
18from portality.tasks.helpers import background_helper
19from portality.tasks.redis_huey import scheduled_long_queue as queue
21email_subs = {}
22email_counter = 0
23name_subs = {}
24name_counter = 0
25password = None
27def create_random_str(n_char=10):
28 import string, random
30 s = string.ascii_letters + string.digits
31 return ''.join(random.choices(s, k=n_char))
33def _anonymise_email(record):
34 if record.email not in email_subs:
35 global email_counter
36 email_counter += 1
37 email_subs[record.email] = str(email_counter) + "@example.com"
38 record.set_email(email_subs[record.email])
39 return record
42def _anonymise_name(record):
43 if record.name is None:
44 return record
45 if record.name not in name_subs:
46 global name_counter
47 name_counter += 1
48 name_subs[record.name] = "User " + str(name_counter)
49 record.set_name(name_subs[record.name])
50 return record
53def _anonymise_admin(record):
54 d = record.data
55 if "admin" in d and "index" in d["admin"] and "notes" in d["admin"]["index"]:
56 anon_notes = []
57 for note in record.data["admin"]["index"]["notes"]:
58 anon_notes.append("---note removed for data security---")
59 record.data["admin"]["index"]["notes"] = anon_notes
61 return record
64def _reset_api_key(record):
65 if record.api_key is not None:
66 record.generate_api_key()
67 return record
70def _reset_password(record):
71 global password
72 if password is None:
73 password = generate_password_hash(uuid.uuid4().hex)
74 record.set_password_hash(password)
75 return record
78def _drop_unanonymisable(type_, record):
79 # A record we can't safely construct/anonymise must never be exported raw - an "anon" export
80 # containing un-anonymised data isn't anonymous. Drop it and log so the gap is visible.
81 if not isinstance(record, dict):
82 app.logger.warning(f"anon_export: dropping {type_} record - could not be anonymised")
83 return None
85 app.logger.warning(f"anon_export: dropping {type_} record '{record.get('id')}' - could not be anonymised")
86 return None
89# transform functions - return the JSON data source since
90# esprit doesn't understand our model classes
91def anonymise_account(record):
92 try:
93 a = models.Account(**record)
94 except DataStructureException:
95 return _drop_unanonymisable("account", record)
97 a = _anonymise_email(a)
98 a = _anonymise_name(a)
99 a = _reset_api_key(a)
100 a = _reset_password(a)
102 return a.data
105def anonymise_journal(record):
106 try:
107 j = models.Journal(**record)
108 except (DataStructureException, SeamlessException):
109 return _drop_unanonymisable("journal", record)
111 return _anonymise_admin(j).data
114def anonymise_application(record):
115 try:
116 appl = models.Application(**record)
117 except (DataStructureException, SeamlessException):
118 return _drop_unanonymisable("application", record)
120 appl = _anonymise_admin(appl)
121 return appl.data
124def anonymise_background_job(record):
125 try:
126 bgjob = models.BackgroundJob(**record)
127 except DataStructureException:
128 return _drop_unanonymisable("background_job", record)
130 if bgjob.params and 'suggestion_bulk_edit__note' in bgjob.params:
131 bgjob.params['suggestion_bulk_edit__note'] = basic_hash(bgjob.params['suggestion_bulk_edit__note'])
133 return bgjob.data
135def anonymise_note(record):
136 try:
137 note = Note(**record)
138 except (DataStructureException, SeamlessException):
139 return _drop_unanonymisable("note", record)
141 note.note = "---note removed for data security---"
142 note.author_id = create_random_str()
143 return note.data
146anonymisation_procedures = {
147 'account': anonymise_account,
148 'background_job': anonymise_background_job,
149 'journal': anonymise_journal,
150 'application': anonymise_application,
151 "note": anonymise_note
152}
154# types that should use prefix queries to optimise performance for bulk exporting
155striped = {
156 # "application": True,
157 "article": True
158}
160skip = app.config.get("ANON_EXPORT_SKIP_LIST",[])
163def _copy_on_complete(path, logger_fn, tmpStore, mainStore, container):
164 name = os.path.basename(path)
165 raw_size = os.path.getsize(path)
166 logger_fn(("Compressing temporary file {x} (from {y} bytes)".format(x=path, y=raw_size)))
167 zipped_name = name + ".gz"
168 dir = os.path.dirname(path)
169 zipped_path = os.path.join(dir, zipped_name)
170 with open(path, "rb") as f_in, gzip.open(zipped_path, "wb") as f_out:
171 shutil.copyfileobj(f_in, f_out)
172 zipped_size = os.path.getsize(zipped_path)
173 logger_fn(("Storing from temporary file {x} ({y} bytes)".format(x=zipped_name, y=zipped_size)))
174 mainStore.store(container, name, source_path=zipped_path)
175 tmpStore.delete_file(container, name)
176 tmpStore.delete_file(container, zipped_name)
179def run_anon_export(tmpStore, mainStore, container, clean=False, limit=None, batch_size=100000,
180 logger_fn: Callable[[str], None] = None):
181 if logger_fn is None:
182 logger_fn = print
183 if clean:
184 mainStore.delete_container(container)
186 doaj_types = es_connection.indices.get(app.config['ELASTIC_SEARCH_DB_PREFIX'] + '*')
187 true_names = []
188 for t, d in doaj_types.items():
189 if "aliases" in d:
190 aliases = list(d["aliases"].keys())
191 if len(aliases) > 0:
192 true_names.append(aliases[0])
193 else:
194 true_names.append(t)
195 else:
196 true_names.append(t)
198 # aliases = es_connection.indices.get_alias(index=app.config['ELASTIC_SEARCH_DB_PREFIX'] + "*")
199 # aliased_types = [item for sublist in [list(v.get("aliases").keys()) for k, v in aliases.items()] for item in sublist]
200 # prefixed_types = list(set(doaj_types + aliased_types))
201 # type_list = [t[len(app.config['ELASTIC_SEARCH_DB_PREFIX']):] for t in doaj_types] + [a[len(app.config['ELASTIC_SEARCH_DB_PREFIX']):] for a in aliases]
202 type_list = [a[len(app.config['ELASTIC_SEARCH_DB_PREFIX']):] for a in true_names]
204 for type_ in type_list:
205 if type_ in skip:
206 logger_fn("skipping " + type_)
207 continue
208 model = models.lookup_models_by_type(type_, dao.DomainObject)
209 if not model:
210 logger_fn("unable to locate model for " + type_)
211 continue
213 filename = type_ + ".bulk"
214 output_file = tmpStore.path(container, filename, create_container=True, must_exist=False)
215 logger_fn((dates.now_str() + " " + type_ + " => " + output_file + ".*"))
216 # iter_q = {"query": {"match_all": {}}, "sort": [{"_id": {"order": "asc"}}]}
217 transform = None
218 if type_ in anonymisation_procedures:
219 transform = anonymisation_procedures[type_]
221 # Use the model's dump method to write out this type to file
222 out_rollover_fn = functools.partial(_copy_on_complete, logger_fn=logger_fn, tmpStore=tmpStore,
223 mainStore=mainStore, container=container)
225 s = striped.get(type_, False)
226 _ = model.dump(limit=limit, transform=transform, out_template=output_file, out_batch_sizes=batch_size,
227 out_rollover_callback=out_rollover_fn, es_bulk_fields=["_id"], striped=s, prefix_size=3, logger=logger_fn)
229 logger_fn((dates.now_str() + " done\n"))
231 tmpStore.delete_container(container)
234class AnonExportBackgroundTask(BackgroundTask):
235 """
236 ~~AnonExport:Feature->BackgroundTask:Process~~
237 """
238 __action__ = "anon_export"
240 def run(self):
241 kwargs = self.get_bgjob_params()
242 kwargs['logger_fn'] = self.background_job.add_audit_message
244 tmpStore = StoreFactory.tmp()
245 mainStore = StoreFactory.get("anon_data")
246 container = app.config.get("STORE_ANON_DATA_CONTAINER")
248 run_anon_export(tmpStore, mainStore, container, **kwargs)
249 self.background_job.add_audit_message("Anon export completed")
251 def cleanup(self):
252 pass
254 @classmethod
255 def prepare(cls, username, **kwargs):
256 params = {}
257 cls.set_param(params, 'clean', background_helper.get_value_safe('clean', False, kwargs))
258 cls.set_param(params, "limit", kwargs.get('limit'))
259 cls.set_param(params, "batch_size", background_helper.get_value_safe('batch_size', 100000, kwargs))
260 return background_helper.create_job(username=username,
261 action=cls.__action__,
262 params=params,
263 queue_id=huey_helper.queue_id, )
265 @classmethod
266 def submit(cls, background_job):
267 background_job.save()
268 anon_export.schedule(args=(background_job.id,), delay=app.config.get('HUEY_ASYNC_DELAY', 10))
271huey_helper = AnonExportBackgroundTask.create_huey_helper(queue)
274@huey_helper.register_schedule
275def scheduled_anon_export():
276 background_helper.submit_by_bg_task_type(AnonExportBackgroundTask,
277 clean=app.config.get("TASKS_ANON_EXPORT_CLEAN", False),
278 limit=app.config.get("TASKS_ANON_EXPORT_LIMIT", None),
279 batch_size=app.config.get("TASKS_ANON_EXPORT_BATCH_SIZE", 100000))
282@huey_helper.register_execute(is_load_config=False)
283def anon_export(job_id):
284 background_helper.execute_by_job_id(job_id, AnonExportBackgroundTask)