formdata: aggregate field data for statistics in new column (#73770)

wcs/formdata.py

@@ -302,6 +302,7 @@ class FormData(StorableObject):
     workflow_data = None
     workflow_roles = None
     geolocations = None
+    statistics_data = None
 
     _formdef = None
 
@@ -521,8 +522,9 @@ class FormData(StorableObject):
 
         changed = False
 
-        def get_all_fields():
-            for field in self.formdef.fields:
+        def get_all_fields(with_backoffice_fields=False):
+            fields = self.formdef.get_all_fields() if with_backoffice_fields else self.formdef.fields
+            for field in fields:
                 yield field
                 if field.key == 'block':
                     try:
@@ -608,6 +610,33 @@ class FormData(StorableObject):
             if digests:
                 self.digests = digests
                 changed = True
+
+        new_statistics_data = {}
+        for field in get_all_fields(with_backoffice_fields=True):
+            if 'statistics' not in field.display_locations:
+                continue
+
+            if new_statistics_data.get(field.varname):
+                continue  # ignore fields with duplicated varname if we already have data
+
+            block = getattr(field, 'block', None)
+            if block:
+                sub_data = self.data.get(block.id) or {}
+                items = set()
+                for data in sub_data.get('data', []):
+                    value = data.get(field.id)
+                    items.add(value)
+                values = list(items)
+            else:
+                values = self.data.get(field.id)
+                if not isinstance(values, list):
+                    values = [values]
+            new_statistics_data[field.varname] = [x for x in values if x is not None]
+
+        if new_statistics_data != self.statistics_data:
+            self.statistics_data = new_statistics_data
+            changed = True
+
         return changed
 
     def get_lateral_block(self):

wcs/sql.py

@@ -925,7 +925,8 @@ BEGIN
             NEW.criticality_level - {criticality_levels},
             {geoloc_base_x},
             {geoloc_base_y},
-            NEW.anonymised);
+            NEW.anonymised,
+            NEW.statistics_data);
         RETURN NEW;
     ELSE
         UPDATE wcs_all_forms SET
@@ -948,7 +949,8 @@ BEGIN
                 criticality_level = NEW.criticality_level - {criticality_levels},
                 geoloc_base_x = {geoloc_base_x},
                 geoloc_base_y = {geoloc_base_y},
-                anonymised = NEW.anonymised
+                anonymised = NEW.anonymised,
+                statistics_data = NEW.statistics_data
             WHERE formdef_id = {formdef_id} AND id = OLD.id;
         RETURN NEW;
     END IF;
@@ -1790,7 +1792,8 @@ def do_global_views(conn, cur):
         criticality_level integer,
         geoloc_base_x double precision,
         geoloc_base_y double precision,
-        anonymised timestamp with time zone
+        anonymised timestamp with time zone,
+        statistics_data jsonb
         , PRIMARY KEY(formdef_id, id)
     )"""
     )
@@ -1894,7 +1897,8 @@ def init_global_table(conn=None, cur=None):
                 criticality_level - {criticality_levels},
                 {geoloc_base_x},
                 {geoloc_base_y},
-                anonymised
+                anonymised,
+                statistics_data
             FROM {table_name}
             ON CONFLICT DO NOTHING;
                 """.format(
@@ -2502,6 +2506,7 @@ class SqlDataMixin(SqlMixin):
         ('digests', 'jsonb'),
         ('user_label', 'varchar'),
         ('auto_geoloc', 'point'),
+        ('statistics_data', 'jsonb'),
     ]
 
     def __init__(self, id=None):
@@ -2598,7 +2603,8 @@ class SqlDataMixin(SqlMixin):
                 '''UPDATE %s
                                   SET id_display = %%(id_display)s,
                                       digests = %%(digests)s,
-                                      user_label = %%(user_label)s
+                                      user_label = %%(user_label)s,
+                                      statistics_data = %%(statistics_data)s
                                 WHERE id = %%(id)s'''
                 % self._table_name
             )
@@ -2609,6 +2615,7 @@ class SqlDataMixin(SqlMixin):
                     'id_display': self.id_display,
                     'digests': self.digests,
                     'user_label': self.user_label,
+                    'statistics_data': self.statistics_data,
                 },
             )
 
@@ -2630,6 +2637,7 @@ class SqlDataMixin(SqlMixin):
             'submission_channel': self.submission_channel,
             'criticality_level': self.criticality_level,
             'workflow_merged_roles_dict': self.workflow_merged_roles_dict,
+            'statistics_data': self.statistics_data or {},
         }
         if self.last_update_time:
             sql_dict['last_update_time'] = datetime.datetime.fromtimestamp(time.mktime(self.last_update_time))
@@ -4936,7 +4944,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (79, 'add translatable column to TranslatableMessage table')
+SQL_LEVEL = (80, 'add statistics data column to formdata')
 
 
 def migrate_global_views(conn, cur):
@@ -5199,10 +5207,12 @@ def migrate():
                 continue
             for formdata in formdef.data_class().select_iterator():
                 formdata._set_auto_fields(cur)  # build digests
-    if sql_level < 69:
+    if sql_level < 80:
         # 58: add workflow_merged_roles_dict as a jsonb column with
         #     combined formdef and formdata value.
         # 69: add auto_geoloc field to form/card tables
+        # 80: add jsonb column to hold statistics data
+
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
         migrate_views(conn, cur)