entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 37 Releases Activity

nettoyages sql.py (#73674) #47

Merged
fpeters merged 3 commits from wip/73674-sql-cleanup into main 2023-01-27 09:18:10 +01:00
Conversation 0 Commits 3 Files Changed 1 +16 -108
1 changed files with 16 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
wcs/sql.py
View File

@ -59,6 +59,8 @@ import wcs.users
from wcs.qommon import force_str
from . import qommon
from .carddef import CardDef
from .formdef import FormDef
from .publisher import UnpicklerClass
from .qommon import _, get_cfg
from .qommon.misc import JSONEncoder, strftime
@ -772,94 +774,26 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
)
existing_fields = {x[0] for x in cur.fetchall()}
needed_fields = {
'id',
'user_id',
'receipt_time',
'status',
'workflow_data',
'id_display',
'fts',
'page_no',
'anonymised',
'workflow_roles',
# workflow_merged_roles_dict combines workflow_roles from formdef and
# formdata and is used to filter on function assignment.
'workflow_merged_roles_dict',
# workflow_roles_array is created from workflow_roles to be used in
# get_ids_with_indexed_value
'workflow_roles_array',
'concerned_roles_array',
'tracking_code',
'actions_roles_array',
'backoffice_submission',
'submission_context',
'submission_agent_id',
'submission_channel',
'criticality_level',
'last_update_time',
'digests',
'user_label',
'prefilling_data',
'auto_geoloc',
}
needed_fields = {x[0] for x in formdef.data_class()._table_static_fields}
needed_fields.add('fts')
# migrations
if 'fts' not in existing_fields:
# full text search
# full text search, column and index
cur.execute('''ALTER TABLE %s ADD COLUMN fts tsvector''' % table_name)
cur.execute('''CREATE INDEX %s_fts ON %s USING gin(fts)''' % (table_name, table_name))
if 'workflow_roles' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN workflow_roles bytea''' % table_name)
cur.execute('''ALTER TABLE %s ADD COLUMN workflow_roles_array text[]''' % table_name)
if 'workflow_merged_roles_dict' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN workflow_merged_roles_dict jsonb''' % table_name)
if 'concerned_roles_array' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN concerned_roles_array text[]''' % table_name)
if 'actions_roles_array' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN actions_roles_array text[]''' % table_name)
if 'page_no' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN page_no varchar''' % table_name)
if 'anonymised' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN anonymised timestamptz''' % table_name)
if 'tracking_code' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN tracking_code varchar''' % table_name)
if 'backoffice_submission' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN backoffice_submission boolean''' % table_name)
if 'submission_context' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN submission_context bytea''' % table_name)
if 'submission_agent_id' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN submission_agent_id varchar''' % table_name)
if 'submission_channel' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN submission_channel varchar''' % table_name)
if 'criticality_level' not in existing_fields:
# criticality leve, with default value
existing_fields.add('criticality_level')
cur.execute(
'''ALTER TABLE %s ADD COLUMN criticality_level integer NOT NULL DEFAULT(0)''' % table_name
)
if 'last_update_time' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN last_update_time timestamp''' % table_name)
if 'digests' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN digests jsonb''' % table_name)
if 'user_label' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN user_label varchar''' % table_name)
if 'prefilling_data' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN prefilling_data bytea''' % table_name)
if 'auto_geoloc' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN auto_geoloc POINT''' % table_name)
# generic migration for new columns
for field_name, field_type in formdef.data_class()._table_static_fields:
if field_name not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN %s %s''' % (table_name, field_name, field_type))
# add new fields
for field in formdef.get_all_fields():
@ -1882,8 +1816,6 @@ def do_global_views(conn, cur):
def clean_global_views(conn, cur):
from .formdef import FormDef
# Purge of any dead data
valid_ids = [int(i) for i in FormDef.keys()]
if valid_ids:
@ -1893,8 +1825,6 @@ def clean_global_views(conn, cur):
def init_global_table(conn=None, cur=None):
from .formdef import FormDef
own_conn = False
if not conn:
own_conn = True
@ -2550,7 +2480,11 @@ class SqlDataMixin(SqlMixin):
('prefilling_data', 'bytea'),
('id_display', 'varchar'),
('workflow_roles', 'bytea'),
# workflow_merged_roles_dict combines workflow_roles from formdef and
# formdata and is used to filter on function assignment.
('workflow_merged_roles_dict', 'jsonb'),
# workflow_roles_array is created from workflow_roles to be used in
# get_ids_with_indexed_value
('workflow_roles_array', 'text[]'),
('concerned_roles_array', 'text[]'),
('actions_roles_array', 'text[]'),
@ -2563,6 +2497,7 @@ class SqlDataMixin(SqlMixin):
('last_update_time', 'timestamp'),
('digests', 'jsonb'),
('user_label', 'varchar'),
('auto_geoloc', 'point'),
]
def __init__(self, id=None):
@ -4639,8 +4574,6 @@ class AnyFormData(SqlMixin):
@classproperty
def _table_static_fields(self):
if not hasattr(self, '__table_static_fields'):
from wcs.formdef import FormDef
fake_formdef = FormDef()
common_fields = get_view_fields(fake_formdef)
self.__table_static_fields = [(x[1], x[0]) for x in common_fields]
@ -4663,8 +4596,6 @@ class AnyFormData(SqlMixin):
@classmethod
def _row2ob(cls, row, **kwargs):
formdef_id = row[1]
from wcs.formdef import FormDef
formdef = cls._formdef_cache.setdefault(formdef_id, FormDef.get(formdef_id))
o = formdef.data_class()()
for static_field, value in zip(cls._table_static_fields, tuple(row[: len(cls._table_static_fields)])):
@ -4696,8 +4627,6 @@ def get_period_query(
clause = [NotNull('receipt_time')]
table_name = 'wcs_all_forms'
if criterias:
from wcs.formdef import FormDef
formdef_class = FormDef
for criteria in criterias:
if criteria.__class__.__name__ == 'Equal' and criteria.attribute == 'formdef_klass':
@ -5120,9 +5049,6 @@ def set_reindex(index, value, conn=None, cur=None):
def migrate_views(conn, cur):
drop_views(None, conn, cur)
from wcs.carddef import CardDef
from wcs.formdef import FormDef
for formdef in FormDef.select() + CardDef.select():
# make sure all formdefs have up-to-date views
do_formdef_tables(formdef, conn=conn, cur=cur, rebuild_views=True, rebuild_global_views=False)
@ -5218,8 +5144,6 @@ def migrate():
migrate_views(conn, cur)
if sql_level < 6:
# 6: add actions_roles_array to tables and views
from wcs.formdef import FormDef
migrate_views(conn, cur)
for formdef in FormDef.select():
formdef.data_class().rebuild_security()
@ -5246,9 +5170,6 @@ def migrate():
# 62: use setweight on formdata & user indexation (reapply)
set_reindex('formdata', 'needed', conn=conn, cur=cur)
if sql_level < 74:
from wcs.carddef import CardDef
from wcs.formdef import FormDef
# 24: add index on evolution(formdata_id)
# 35: add indexes on formdata(receipt_time) and formdata(anonymised)
# 36: add index on formdata(user_id)
@ -5259,8 +5180,6 @@ def migrate():
do_formdef_indexes(formdef, created=False, conn=conn, cur=cur)
if sql_level < 30:
# 30: actually remove evo.who on anonymised formdatas
from wcs.formdef import FormDef
for formdef in FormDef.select():
for formdata in formdef.data_class().select_iterator(clause=[NotNull('anonymised')]):
if formdata.evolution:
@ -5269,9 +5188,6 @@ def migrate():
formdata.store()
if sql_level < 52:
# 52: store digests on formdata and carddata
from wcs.carddef import CardDef
from wcs.formdef import FormDef
for formdef in FormDef.select() + CardDef.select():
if not formdef.digest_templates:
continue
@ -5281,9 +5197,6 @@ def migrate():
# 58: add workflow_merged_roles_dict as a jsonb column with
# combined formdef and formdata value.
# 69: add auto_geoloc field to form/card tables
from wcs.carddef import CardDef
from wcs.formdef import FormDef
for formdef in FormDef.select() + CardDef.select():
do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
migrate_views(conn, cur)
@ -5291,8 +5204,6 @@ def migrate():
if sql_level < 60:
# 59: switch wcs_all_forms to a trigger-maintained table
# 60: rebuild triggers
from wcs.formdef import FormDef
init_global_table(conn, cur)
for formdef in FormDef.select():
do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
@ -5338,9 +5249,6 @@ def reindex():
user.store()
set_reindex('user', 'done', conn=conn, cur=cur)
from wcs.carddef import CardDef
from wcs.formdef import FormDef
if is_reindex_needed('formdata', conn=conn, cur=cur):
# load and store all formdatas
for formdef in FormDef.select() + CardDef.select():