tests/admin_pages/test_block.py

@@ -501,3 +501,38 @@ def test_block_duplicate(pub):
 
     block_copy = BlockDef.get_by_slug('other_copy')
     assert len(block_copy.fields) == 2
+
+
+def test_block_field_statistics_data_update(pub):
+    create_superuser(pub)
+
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'Foobar'
+    block.fields = [fields.BoolField(id='1', label='Bool', varname='bool', type='comment')]
+    block.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.BlockField(id='0', label='test', type='block:%s' % block.slug),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['0'] = {'data': [{'1': True}]}
+    formdata.store()
+
+    assert not formdata.statistics_data
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/blocks/%s/1/' % block.id)
+
+    resp.form['display_locations$element3'] = True
+    resp = resp.form.submit('submit').follow()
+    assert 'Statistics data will be collected in the background.' in resp.text
+
+    formdata.refresh_from_storage()
+    assert formdata.statistics_data == {'bool': [True]}

tests/admin_pages/test_form.py

@@ -3765,3 +3765,30 @@ def test_form_import_fields(pub):
         ('5', 'field 4'),
         ('3', 'Page 2'),
     ]
+
+
+def test_form_field_statistics_data_update(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.BoolField(id='1', label='Bool', varname='bool', type='comment')]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['1'] = True
+    formdata.store()
+
+    assert not formdata.statistics_data
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/fields/1/')
+
+    resp.form['display_locations$element3'] = True
+    resp = resp.form.submit('submit').follow()
+    assert 'Statistics data will be collected in the background.' in resp.text
+
+    formdata.refresh_from_storage()
+    assert formdata.statistics_data == {'bool': [True]}

tests/admin_pages/test_workflow.py

@@ -3745,3 +3745,41 @@ def test_remove_tracking_code_details(pub):
     workflow.store()
     resp = app.get('/backoffice/workflows/%s/status/%s/' % (workflow.id, baz_status.id))
     assert 'Remove Tracking Code (replace with a new one)' in resp.text
+
+
+def test_workflow_backoffice_field_statistics_data_update(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    FormDef.wipe()
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.BoolField(id='1', label='Bool', varname='bool', type='comment')
+    ]
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.workflow = workflow
+    formdef.store()
+
+    app = login(get_app(pub))
+
+    formdata = formdef.data_class()()
+    formdata.data['1'] = True
+    formdata.store()
+
+    assert not formdata.statistics_data
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/1/backoffice-fields/fields/1/')
+
+    resp.form['display_locations$element2'] = True
+    resp = resp.form.submit('submit').follow()
+    assert 'Statistics data will be collected in the background.' in resp.text
+
+    formdata.refresh_from_storage()
+    assert formdata.statistics_data == {'bool': [True]}

tests/api/test_statistics.py

@@ -563,6 +563,23 @@ def test_statistics_forms_count_subfilters(pub, formdef):
     new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
     assert new_resp.json == resp.json
 
+    # add items field inside block field, it should not appear
+    items_field = fields.ItemsField(
+        id='2',
+        varname='items',
+        label='Block items',
+        type='items',
+        items=['foo', 'bar', 'baz'],
+        anonymise=False,
+        display_locations=['statistics'],
+    )
+    formdef.fields[2].block.fields.append(bool_field)
+    formdef.store()
+    formdata.data['4'] = {'data': [{'2': ['bar']}]}
+    formdata.store()
+    new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
+    assert new_resp.json == resp.json
+
     # remove fields and statuses
     workflow = Workflow(name='Empty wf')
     workflow.store()
@@ -587,7 +604,7 @@ def test_statistics_forms_count_subfilters_query(pub, formdef):
             formdata.data['1'] = True
             formdata.data['2'] = 'foo'
             formdata.data['3'] = ['bar', 'baz']
-            formdata.data['4'] = {'data': [{'1': True}]}
+            formdata.data['4'] = {'data': [{'1': True}, {'1': False}]}
         elif i % 2:
             formdata.data['1'] = False
             formdata.data['2'] = 'baz'
@@ -646,7 +663,7 @@ def test_statistics_forms_count_subfilters_query(pub, formdef):
     assert resp.json['data']['series'][0]['data'][0] == 13
 
     resp = get_app(pub).get(sign_uri(url + '&filter-blockdata_bool=false'))
-    assert resp.json['data']['series'][0]['data'][0] == 3
+    assert resp.json['data']['series'][0]['data'][0] == 16
 
     # filter on status
     resp = get_app(pub).get(sign_uri(url + '&filter-status=_all'))
@@ -691,6 +708,7 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
     ]
     formdef.store()
 
+    formdatas = []
     for i in range(5):
         formdata = formdef.data_class()()
         formdata.just_created()
@@ -701,6 +719,7 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
             formdata.data['1'] = 'bar'
         formdata.data['2'] = 'foo'
         formdata.store()
+        formdatas.append(formdata)
 
     url = '/api/statistics/forms/count/?form=%s' % formdef.url_name
     resp = get_app(pub).get(sign_uri(url + '&filter-test=foo'))
@@ -708,14 +727,34 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
 
     formdef.fields[0].display_locations = ['statistics']
     formdef.store()
+    for formdata in formdatas:
+        formdata.store()  # refresh statistics_data column
 
-    # filter criterias is "f1 == 'foo' and f2 == 'foo'" hence one result, this should be improved
+    # first non empty value is used : 4 are 'foo' and one is 'bar' hence 4 results
     resp = get_app(pub).get(sign_uri(url + '&filter-test=foo'))
+    assert resp.json['data']['series'] == [{'data': [4], 'label': 'Forms Count'}]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-test=bar'))
     assert resp.json['data']['series'] == [{'data': [1], 'label': 'Forms Count'}]
 
-    # filter criterias is "f1 == 'bar' and f2 == 'bar'" hence no results, this should be improved
-    resp = get_app(pub).get(sign_uri(url + '&filter-test=bar'))
-    assert resp.json['data']['series'] == [{'data': [], 'label': 'Forms Count'}]
+
+def test_statistics_forms_count_subfilters_query_integer_items(pub, formdef):
+    for i in range(10):
+        formdata = formdef.data_class()()
+        formdata.just_created()
+        if i % 2:
+            formdata.data['3'] = ['1', '2']
+        else:
+            formdata.data['3'] = ['1']
+        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.store()
+
+    url = '/api/statistics/forms/count/?form=%s' % formdef.url_name
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-items=1'))
+    assert resp.json['data']['series'][0]['data'][0] == 10
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-items=2'))
+    assert resp.json['data']['series'][0]['data'][0] == 5
 
 
 @pytest.mark.parametrize('anonymise', [False, True])
@@ -907,6 +946,7 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
 
     formdef.fields[0].display_locations = ['statistics']
     formdef.store()
+    formdata.store()  # refresh statistics_data column
 
     # group by uses first field marked for statistics
     resp = get_app(pub).get(sign_uri(url + '&group-by=test'))
@@ -918,9 +958,8 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
     formdata.data['2'] = 'foo'
     formdata.store()
 
-    # second field is ignored
     resp = get_app(pub).get(sign_uri(url + '&group-by=test'))
-    assert resp.json['data']['series'] == [{'data': [1], 'label': 'foo'}, {'data': [1], 'label': 'None'}]
+    assert resp.json['data']['series'] == [{'data': [2], 'label': 'foo'}]
 
 
 def test_statistics_cards_count(pub):

wcs/admin/blocks.py

@@ -24,6 +24,7 @@ from wcs.admin.fields import FieldDefPage, FieldsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.blocks import BlockDef, BlockdefImportError
 from wcs.categories import BlockCategory
+from wcs.formdef import UpdateStatisticsDataAfterJob
 from wcs.qommon import _, misc, template
 from wcs.qommon.backoffice.menu import html_top
 from wcs.qommon.errors import AccessForbiddenError, TraversalError
@@ -35,6 +36,11 @@ class BlockFieldDefPage(FieldDefPage):
         anchor = '#itemId_%s' % field.id if field else ''
         return redirect('../%s' % anchor)
 
+    def schedule_statistics_data_update(self):
+        get_response().add_after_job(
+            UpdateStatisticsDataAfterJob(formdefs=self.objectdef.get_usage_formdefs())
+        )
+
 
 class BlockDirectory(FieldsDirectory):
     _q_exports = [

wcs/admin/fields.py

@@ -25,7 +25,7 @@ from wcs import fields
 from wcs.admin import utils
 from wcs.carddef import CardDef
 from wcs.fields import BlockField, get_field_options
-from wcs.formdef import FormDef
+from wcs.formdef import FormDef, UpdateStatisticsDataAfterJob
 from wcs.qommon import _, errors, get_cfg, misc
 from wcs.qommon.admin.menu import command_icon
 from wcs.qommon.backoffice.menu import html_top
@@ -70,6 +70,7 @@ class FieldDefPage(Directory):
     def _q_index(self):
         form = self.form()
         redo = False
+        old_display_locations = self.field.display_locations.copy()
 
         if form.get_submit() == 'cancel':
             return redirect('../#itemId_%s' % self.field.id)
@@ -111,6 +112,11 @@ class FieldDefPage(Directory):
             return r.getvalue()
 
         self.submit(form)
+
+        if 'statistics' in self.field.display_locations and 'statistics' not in old_display_locations:
+            self.schedule_statistics_data_update()
+            get_session().message = ('info', _('Statistics data will be collected in the background.'))
+
         if form.get_widget('items') is None and self.field.type == 'item':
             return redirect('.')
 
@@ -128,6 +134,9 @@ class FieldDefPage(Directory):
 
         return redirect('../#itemId_%s' % self.field.id)
 
+    def schedule_statistics_data_update(self):
+        get_response().add_after_job(UpdateStatisticsDataAfterJob(formdefs=[self.objectdef]))
+
     def submit(self, form):
         for f in self.field.get_admin_attributes():
             widget = form.get_widget(f)

wcs/admin/workflows.py

@@ -32,7 +32,7 @@ from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.carddef import CardDef
 from wcs.categories import WorkflowCategory
 from wcs.formdata import Evolution
-from wcs.formdef import FormDef
+from wcs.formdef import FormDef, UpdateStatisticsDataAfterJob
 from wcs.qommon import _, errors, force_str, misc, template
 from wcs.qommon.admin.menu import command_icon
 from wcs.qommon.backoffice.menu import html_top
@@ -1083,6 +1083,14 @@ class WorkflowBackofficeFieldDefPage(FieldDefPage):
             display_locations.options = display_locations.options[1:]
         return form
 
+    def schedule_statistics_data_update(self):
+        formdefs = [
+            x
+            for x in FormDef.select(lightweight=True) + CardDef.select(lightweight=True)
+            if x.workflow_id == self.objectdef.workflow.id
+        ]
+        get_response().add_after_job(UpdateStatisticsDataAfterJob(formdefs=formdefs))
+
 
 class WorkflowVariablesFieldsDirectory(FieldsDirectory):
     _q_exports = ['', 'update_order', 'new']

wcs/formdata.py

@@ -302,6 +302,7 @@ class FormData(StorableObject):
     workflow_data = None
     workflow_roles = None
     geolocations = None
+    statistics_data = None
 
     _formdef = None
 
@@ -521,8 +522,9 @@ class FormData(StorableObject):
 
         changed = False
 
-        def get_all_fields():
-            for field in self.formdef.fields:
+        def get_all_fields(with_backoffice_fields=False):
+            fields = self.formdef.get_all_fields() if with_backoffice_fields else self.formdef.fields
+            for field in fields:
                 yield field
                 if field.key == 'block':
                     try:
@@ -608,6 +610,33 @@ class FormData(StorableObject):
             if digests:
                 self.digests = digests
                 changed = True
+
+        new_statistics_data = {}
+        for field in get_all_fields(with_backoffice_fields=True):
+            if 'statistics' not in field.display_locations:
+                continue
+
+            if new_statistics_data.get(field.varname):
+                continue  # ignore fields with duplicated varname if we already have data
+
+            block = getattr(field, 'block', None)
+            if block:
+                sub_data = self.data.get(block.id) or {}
+                items = set()
+                for data in sub_data.get('data', []):
+                    value = data.get(field.id)
+                    items.add(value)
+                values = list(items)
+            else:
+                values = self.data.get(field.id)
+                if not isinstance(values, list):
+                    values = [values]
+            new_statistics_data[field.varname] = [x for x in values if x is not None]
+
+        if new_statistics_data != self.statistics_data:
+            self.statistics_data = new_statistics_data
+            changed = True
+
         return changed
 
     def get_lateral_block(self):

wcs/formdef.py

@@ -2163,3 +2163,7 @@ class UpdateDigestAfterJob(AfterJob):
             formdef = formdef_class.get(formdef_id)
             for formdata in formdef.data_class().select(order_by='id'):
                 formdata.store()
+
+
+class UpdateStatisticsDataAfterJob(UpdateDigestAfterJob):
+    label = _('Updating statistics data')

wcs/sql.py

@@ -181,7 +181,9 @@ def pickle_loads(value):
 
 class Criteria(qommon.storage.Criteria):
     def __init__(self, attribute, value, **kwargs):
-        self.attribute = attribute.replace('-', '_')
+        self.attribute = attribute
+        if '->' not in attribute:
+            self.attribute = self.attribute.replace('-', '_')
         self.value = value
         self.field = kwargs.get('field')
 
@@ -925,7 +927,8 @@ BEGIN
             NEW.criticality_level - {criticality_levels},
             {geoloc_base_x},
             {geoloc_base_y},
-            NEW.anonymised);
+            NEW.anonymised,
+            NEW.statistics_data);
         RETURN NEW;
     ELSE
         UPDATE wcs_all_forms SET
@@ -948,7 +951,8 @@ BEGIN
                 criticality_level = NEW.criticality_level - {criticality_levels},
                 geoloc_base_x = {geoloc_base_x},
                 geoloc_base_y = {geoloc_base_y},
-                anonymised = NEW.anonymised
+                anonymised = NEW.anonymised,
+                statistics_data = NEW.statistics_data
             WHERE formdef_id = {formdef_id} AND id = OLD.id;
         RETURN NEW;
     END IF;
@@ -1790,7 +1794,8 @@ def do_global_views(conn, cur):
         criticality_level integer,
         geoloc_base_x double precision,
         geoloc_base_y double precision,
-        anonymised timestamp with time zone
+        anonymised timestamp with time zone,
+        statistics_data jsonb
         , PRIMARY KEY(formdef_id, id)
     )"""
     )
@@ -1894,7 +1899,8 @@ def init_global_table(conn=None, cur=None):
                 criticality_level - {criticality_levels},
                 {geoloc_base_x},
                 {geoloc_base_y},
-                anonymised
+                anonymised,
+                statistics_data
             FROM {table_name}
             ON CONFLICT DO NOTHING;
                 """.format(
@@ -2502,6 +2508,7 @@ class SqlDataMixin(SqlMixin):
         ('digests', 'jsonb'),
         ('user_label', 'varchar'),
         ('auto_geoloc', 'point'),
+        ('statistics_data', 'jsonb'),
     ]
 
     def __init__(self, id=None):
@@ -2598,7 +2605,8 @@ class SqlDataMixin(SqlMixin):
                 '''UPDATE %s
                                   SET id_display = %%(id_display)s,
                                       digests = %%(digests)s,
-                                      user_label = %%(user_label)s
+                                      user_label = %%(user_label)s,
+                                      statistics_data = %%(statistics_data)s
                                 WHERE id = %%(id)s'''
                 % self._table_name
             )
@@ -2609,6 +2617,7 @@ class SqlDataMixin(SqlMixin):
                     'id_display': self.id_display,
                     'digests': self.digests,
                     'user_label': self.user_label,
+                    'statistics_data': self.statistics_data,
                 },
             )
 
@@ -2630,6 +2639,7 @@ class SqlDataMixin(SqlMixin):
             'submission_channel': self.submission_channel,
             'criticality_level': self.criticality_level,
             'workflow_merged_roles_dict': self.workflow_merged_roles_dict,
+            'statistics_data': self.statistics_data or {},
         }
         if self.last_update_time:
             sql_dict['last_update_time'] = datetime.datetime.fromtimestamp(time.mktime(self.last_update_time))
@@ -4639,7 +4649,11 @@ def get_period_query(
                 formdef_class = criteria.value
                 continue
 
-            if criteria.__class__.__name__ == 'Equal' and criteria.attribute == 'formdef_id':
+            if (
+                formdef_class
+                and criteria.__class__.__name__ == 'Equal'
+                and criteria.attribute == 'formdef_id'
+            ):
                 # if there's a formdef_id specified, switch to using the
                 # specific table so we have access to all fields
                 table_name = get_formdef_table_name(formdef_class.get(criteria.value))
@@ -4936,7 +4950,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (79, 'add translatable column to TranslatableMessage table')
+SQL_LEVEL = (80, 'add statistics data column to formdata')
 
 
 def migrate_global_views(conn, cur):
@@ -5199,10 +5213,12 @@ def migrate():
                 continue
             for formdata in formdef.data_class().select_iterator():
                 formdata._set_auto_fields(cur)  # build digests
-    if sql_level < 69:
+    if sql_level < 80:
         # 58: add workflow_merged_roles_dict as a jsonb column with
         #     combined formdef and formdata value.
         # 69: add auto_geoloc field to form/card tables
+        # 80: add jsonb column to hold statistics data
+
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
         migrate_views(conn, cur)

wcs/statistics/views.py

@@ -30,7 +30,7 @@ from wcs.categories import Category
 from wcs.formdata import FormData
 from wcs.formdef import FormDef
 from wcs.qommon import _, misc, pgettext_lazy
-from wcs.qommon.storage import Contains, Equal, GreaterOrEqual, Less, NotNull, Null, Or, StrictNotEqual
+from wcs.qommon.storage import Contains, Equal, GreaterOrEqual, Less, Null, Or, StrictNotEqual
 
 
 class RestrictedView(View):
@@ -252,11 +252,7 @@ class FormsCountView(RestrictedView):
             except KeyError:
                 return HttpResponseBadRequest('invalid form')
             form_page = self.formpage_class(formdef=formdef, update_breadcrumbs=False)
-
-            # formdef_klass is a fake criteria, it will be used in time interval functions
-            # to switch to appropriate class, it must appear before formdef_id.
-            totals_kwargs['criterias'].append(Equal('formdef_klass', self.formdef_class))
-            totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
+            self.set_formdef_parameters(totals_kwargs, formdef)
             totals_kwargs['criterias'].extend(self.get_filters_criterias(formdef, form_page))
             self.set_group_by_parameters(group_by, formdef, form_page, totals_kwargs, group_labels)
             subfilters = self.get_subfilters(form_page, group_by)
@@ -314,8 +310,27 @@ class FormsCountView(RestrictedView):
             {'data': {'x_labels': x_labels, 'series': series, 'subfilters': subfilters}, 'err': 0}
         )
 
+    def set_formdef_parameters(self, totals_kwargs, formdef):
+        # set formdef_klass to None to deactivate switching to formdef specific table
+        totals_kwargs['criterias'].append(Equal('formdef_klass', None))
+        totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
+
+    def transform_criteria(self, criteria):
+        if not hasattr(criteria, 'field'):
+            return criteria
+
+        attribute = "statistics_data->'%s'" % criteria.field.varname
+
+        if isinstance(criteria.value, bool):
+            value = str(criteria.value).lower()
+        else:
+            value = '"%s"' % criteria.value
+
+        return sql.ArrayContains(attribute, value)
+
     def get_filters_criterias(self, formdef, form_page):
         criterias = form_page.get_criterias_from_query(statistics_fields_only=True)
+        criterias = [self.transform_criteria(criteria) for criteria in criterias]
 
         selected_status = self.request.GET.get('filter-status')
         applied_filters = None
@@ -471,10 +486,10 @@ class FormsCountView(RestrictedView):
         if group_by_field.type == 'status':
             totals_kwargs['group_by'] = 'status'
         else:
-            totals_kwargs['group_by'] = sql.get_field_id(group_by_field)
+            totals_kwargs['group_by'] = "statistics_data->'%s'" % group_by_field.varname
 
         if self.request.GET.get('hide_none_label') == 'true':
-            totals_kwargs['criterias'].append(NotNull(totals_kwargs['group_by']))
+            totals_kwargs['criterias'].append(StrictNotEqual(totals_kwargs['group_by'], '[]'))
 
         group_labels.update(self.get_group_labels(group_by_field, formdef, form_page, group_by))
 
@@ -494,6 +509,8 @@ class FormsCountView(RestrictedView):
             groups = total[1]
             if not isinstance(groups, list):
                 groups = [groups]
+            if not groups:
+                groups = [None]
             for group in groups:
                 totals_by_group[group] += total[2]
                 seen_group_values.add(group)
@@ -516,6 +533,8 @@ class FormsCountView(RestrictedView):
         for groups, total in totals:
             if not isinstance(groups, list):
                 groups = [groups]
+            if not groups:
+                groups = [None]
             for group in groups:
                 totals_by_group[group] += total
 
@@ -562,6 +581,12 @@ class CardsCountView(FormsCountView):
     has_global_count_support = False
     label = _('Cards Count')
 
+    def set_formdef_parameters(self, totals_kwargs, formdef):
+        # formdef_klass is a fake criteria, it will be used in time interval functions
+        # to switch to appropriate class, it must appear before formdef_id.
+        totals_kwargs['criterias'].append(Equal('formdef_klass', CardDef))
+        totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
+
 
 class ResolutionTimeView(RestrictedView):
     formdef_class = FormDef