statistiques, ajouter une colonne commune à wcs_all_form avec les données nécessaires (#73770) #63

Merged
vdeniaud merged 4 commits from wip/73770-statistiques-ajouter-une-colonne into main 2023-02-14 18:16:16 +01:00
11 changed files with 265 additions and 29 deletions

View File

@ -501,3 +501,38 @@ def test_block_duplicate(pub):
block_copy = BlockDef.get_by_slug('other_copy')
assert len(block_copy.fields) == 2
def test_block_field_statistics_data_update(pub):
create_superuser(pub)
BlockDef.wipe()
block = BlockDef()
block.name = 'Foobar'
block.fields = [fields.BoolField(id='1', label='Bool', varname='bool', type='comment')]
block.store()
FormDef.wipe()
formdef = FormDef()
formdef.name = 'form title'
formdef.fields = [
fields.BlockField(id='0', label='test', type='block:%s' % block.slug),
]
formdef.store()
formdata = formdef.data_class()()
formdata.just_created()
formdata.data['0'] = {'data': [{'1': True}]}
formdata.store()
assert not formdata.statistics_data
app = login(get_app(pub))
resp = app.get('/backoffice/forms/blocks/%s/1/' % block.id)
resp.form['display_locations$element3'] = True
resp = resp.form.submit('submit').follow()
assert 'Statistics data will be collected in the background.' in resp.text
formdata.refresh_from_storage()
assert formdata.statistics_data == {'bool': [True]}

View File

@ -3765,3 +3765,30 @@ def test_form_import_fields(pub):
('5', 'field 4'),
('3', 'Page 2'),
]
def test_form_field_statistics_data_update(pub):
create_superuser(pub)
FormDef.wipe()
formdef = FormDef()
formdef.name = 'form title'
formdef.fields = [fields.BoolField(id='1', label='Bool', varname='bool', type='comment')]
formdef.store()
formdata = formdef.data_class()()
formdata.just_created()
formdata.data['1'] = True
formdata.store()
assert not formdata.statistics_data
app = login(get_app(pub))
resp = app.get('/backoffice/forms/1/fields/1/')
resp.form['display_locations$element3'] = True
resp = resp.form.submit('submit').follow()
assert 'Statistics data will be collected in the background.' in resp.text
formdata.refresh_from_storage()
assert formdata.statistics_data == {'bool': [True]}

View File

@ -3745,3 +3745,41 @@ def test_remove_tracking_code_details(pub):
workflow.store()
resp = app.get('/backoffice/workflows/%s/status/%s/' % (workflow.id, baz_status.id))
assert 'Remove Tracking Code (replace with a new one)' in resp.text
def test_workflow_backoffice_field_statistics_data_update(pub):
create_superuser(pub)
CardDef.wipe()
FormDef.wipe()
Workflow.wipe()
workflow = Workflow(name='foo')
workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
workflow.backoffice_fields_formdef.fields = [
fields.BoolField(id='1', label='Bool', varname='bool', type='comment')
]
workflow.store()
FormDef.wipe()
formdef = FormDef()
formdef.name = 'form title'
formdef.workflow = workflow
formdef.store()
app = login(get_app(pub))
formdata = formdef.data_class()()
formdata.data['1'] = True
formdata.store()
assert not formdata.statistics_data
app = login(get_app(pub))
resp = app.get('/backoffice/workflows/1/backoffice-fields/fields/1/')
resp.form['display_locations$element2'] = True
resp = resp.form.submit('submit').follow()
assert 'Statistics data will be collected in the background.' in resp.text
formdata.refresh_from_storage()
assert formdata.statistics_data == {'bool': [True]}

View File

@ -563,6 +563,23 @@ def test_statistics_forms_count_subfilters(pub, formdef):
new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
assert new_resp.json == resp.json
# add items field inside block field, it should not appear
items_field = fields.ItemsField(
id='2',
varname='items',
label='Block items',
type='items',
items=['foo', 'bar', 'baz'],
anonymise=False,
display_locations=['statistics'],
)
formdef.fields[2].block.fields.append(bool_field)
formdef.store()
formdata.data['4'] = {'data': [{'2': ['bar']}]}
formdata.store()
new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
assert new_resp.json == resp.json
# remove fields and statuses
workflow = Workflow(name='Empty wf')
workflow.store()
@ -587,7 +604,7 @@ def test_statistics_forms_count_subfilters_query(pub, formdef):
formdata.data['1'] = True
formdata.data['2'] = 'foo'
formdata.data['3'] = ['bar', 'baz']
formdata.data['4'] = {'data': [{'1': True}]}
formdata.data['4'] = {'data': [{'1': True}, {'1': False}]}
elif i % 2:
formdata.data['1'] = False
formdata.data['2'] = 'baz'
@ -646,7 +663,7 @@ def test_statistics_forms_count_subfilters_query(pub, formdef):
assert resp.json['data']['series'][0]['data'][0] == 13
resp = get_app(pub).get(sign_uri(url + '&filter-blockdata_bool=false'))
assert resp.json['data']['series'][0]['data'][0] == 3
assert resp.json['data']['series'][0]['data'][0] == 16
# filter on status
resp = get_app(pub).get(sign_uri(url + '&filter-status=_all'))
@ -691,6 +708,7 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
]
formdef.store()
formdatas = []
for i in range(5):
formdata = formdef.data_class()()
formdata.just_created()
@ -701,6 +719,7 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
formdata.data['1'] = 'bar'
formdata.data['2'] = 'foo'
formdata.store()
formdatas.append(formdata)
url = '/api/statistics/forms/count/?form=%s' % formdef.url_name
resp = get_app(pub).get(sign_uri(url + '&filter-test=foo'))
@ -708,14 +727,34 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
formdef.fields[0].display_locations = ['statistics']
formdef.store()
for formdata in formdatas:
formdata.store() # refresh statistics_data column
# filter criterias is "f1 == 'foo' and f2 == 'foo'" hence one result, this should be improved
# first non empty value is used : 4 are 'foo' and one is 'bar' hence 4 results
resp = get_app(pub).get(sign_uri(url + '&filter-test=foo'))
assert resp.json['data']['series'] == [{'data': [4], 'label': 'Forms Count'}]
resp = get_app(pub).get(sign_uri(url + '&filter-test=bar'))
assert resp.json['data']['series'] == [{'data': [1], 'label': 'Forms Count'}]
# filter criterias is "f1 == 'bar' and f2 == 'bar'" hence no results, this should be improved
resp = get_app(pub).get(sign_uri(url + '&filter-test=bar'))
assert resp.json['data']['series'] == [{'data': [], 'label': 'Forms Count'}]
def test_statistics_forms_count_subfilters_query_integer_items(pub, formdef):
for i in range(10):
formdata = formdef.data_class()()
formdata.just_created()
if i % 2:
formdata.data['3'] = ['1', '2']
else:
formdata.data['3'] = ['1']
formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
formdata.store()
url = '/api/statistics/forms/count/?form=%s' % formdef.url_name
resp = get_app(pub).get(sign_uri(url + '&filter-test-items=1'))
assert resp.json['data']['series'][0]['data'][0] == 10
resp = get_app(pub).get(sign_uri(url + '&filter-test-items=2'))
assert resp.json['data']['series'][0]['data'][0] == 5
@pytest.mark.parametrize('anonymise', [False, True])
@ -907,6 +946,7 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
formdef.fields[0].display_locations = ['statistics']
formdef.store()
formdata.store() # refresh statistics_data column
# group by uses first field marked for statistics
resp = get_app(pub).get(sign_uri(url + '&group-by=test'))
@ -918,9 +958,8 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
formdata.data['2'] = 'foo'
formdata.store()
# second field is ignored
resp = get_app(pub).get(sign_uri(url + '&group-by=test'))
assert resp.json['data']['series'] == [{'data': [1], 'label': 'foo'}, {'data': [1], 'label': 'None'}]
assert resp.json['data']['series'] == [{'data': [2], 'label': 'foo'}]
def test_statistics_cards_count(pub):

View File

@ -24,6 +24,7 @@ from wcs.admin.fields import FieldDefPage, FieldsDirectory
from wcs.backoffice.snapshots import SnapshotsDirectory
from wcs.blocks import BlockDef, BlockdefImportError
from wcs.categories import BlockCategory
from wcs.formdef import UpdateStatisticsDataAfterJob
from wcs.qommon import _, misc, template
from wcs.qommon.backoffice.menu import html_top
from wcs.qommon.errors import AccessForbiddenError, TraversalError
@ -35,6 +36,11 @@ class BlockFieldDefPage(FieldDefPage):
anchor = '#itemId_%s' % field.id if field else ''
return redirect('../%s' % anchor)
def schedule_statistics_data_update(self):
get_response().add_after_job(
UpdateStatisticsDataAfterJob(formdefs=self.objectdef.get_usage_formdefs())

Plutôt que tel quel le UpdateDigestAfterJob je préfererais un nouveau job (très simple, ça peut juste être une classe qui hérite de UpdateDigestAfterJob et lui met un attribut label adapté aux statistiques).

Plutôt que tel quel le UpdateDigestAfterJob je préfererais un nouveau job (très simple, ça peut juste être une classe qui hérite de UpdateDigestAfterJob et lui met un attribut label adapté aux statistiques).

Ajouté « UpdateStatisticsDataAfterJob ».

Ajouté « UpdateStatisticsDataAfterJob ».
)
class BlockDirectory(FieldsDirectory):
_q_exports = [

View File

@ -25,7 +25,7 @@ from wcs import fields
from wcs.admin import utils
from wcs.carddef import CardDef
from wcs.fields import BlockField, get_field_options
from wcs.formdef import FormDef
from wcs.formdef import FormDef, UpdateStatisticsDataAfterJob
from wcs.qommon import _, errors, get_cfg, misc
from wcs.qommon.admin.menu import command_icon
from wcs.qommon.backoffice.menu import html_top
@ -70,6 +70,7 @@ class FieldDefPage(Directory):
def _q_index(self):
form = self.form()
redo = False
old_display_locations = self.field.display_locations.copy()
if form.get_submit() == 'cancel':
return redirect('../#itemId_%s' % self.field.id)
@ -111,6 +112,11 @@ class FieldDefPage(Directory):
return r.getvalue()
self.submit(form)
if 'statistics' in self.field.display_locations and 'statistics' not in old_display_locations:
self.schedule_statistics_data_update()
get_session().message = ('info', _('Statistics data will be collected in the background.'))
if form.get_widget('items') is None and self.field.type == 'item':
return redirect('.')
@ -128,6 +134,9 @@ class FieldDefPage(Directory):
return redirect('../#itemId_%s' % self.field.id)
def schedule_statistics_data_update(self):
get_response().add_after_job(UpdateStatisticsDataAfterJob(formdefs=[self.objectdef]))
def submit(self, form):
for f in self.field.get_admin_attributes():
widget = form.get_widget(f)

View File

@ -32,7 +32,7 @@ from wcs.backoffice.snapshots import SnapshotsDirectory
from wcs.carddef import CardDef
from wcs.categories import WorkflowCategory
from wcs.formdata import Evolution
from wcs.formdef import FormDef
from wcs.formdef import FormDef, UpdateStatisticsDataAfterJob
from wcs.qommon import _, errors, force_str, misc, template
from wcs.qommon.admin.menu import command_icon
from wcs.qommon.backoffice.menu import html_top
@ -1083,6 +1083,14 @@ class WorkflowBackofficeFieldDefPage(FieldDefPage):
display_locations.options = display_locations.options[1:]
return form
def schedule_statistics_data_update(self):
formdefs = [
x
for x in FormDef.select(lightweight=True) + CardDef.select(lightweight=True)
if x.workflow_id == self.objectdef.workflow.id
]
get_response().add_after_job(UpdateStatisticsDataAfterJob(formdefs=formdefs))
class WorkflowVariablesFieldsDirectory(FieldsDirectory):
_q_exports = ['', 'update_order', 'new']

View File

@ -302,6 +302,7 @@ class FormData(StorableObject):
workflow_data = None
workflow_roles = None
geolocations = None
statistics_data = None
_formdef = None
@ -521,8 +522,9 @@ class FormData(StorableObject):
changed = False
def get_all_fields():
for field in self.formdef.fields:
def get_all_fields(with_backoffice_fields=False):
fields = self.formdef.get_all_fields() if with_backoffice_fields else self.formdef.fields
for field in fields:
yield field
if field.key == 'block':
try:
@ -608,6 +610,33 @@ class FormData(StorableObject):
if digests:
self.digests = digests
changed = True
new_statistics_data = {}
for field in get_all_fields(with_backoffice_fields=True):
if 'statistics' not in field.display_locations:
continue
if new_statistics_data.get(field.varname):
continue # ignore fields with duplicated varname if we already have data
block = getattr(field, 'block', None)
if block:
sub_data = self.data.get(block.id) or {}
items = set()
for data in sub_data.get('data', []):
value = data.get(field.id)
items.add(value)
values = list(items)
else:
values = self.data.get(field.id)
if not isinstance(values, list):
values = [values]
new_statistics_data[field.varname] = [x for x in values if x is not None]
if new_statistics_data != self.statistics_data:
self.statistics_data = new_statistics_data
changed = True
return changed
def get_lateral_block(self):

View File

@ -2163,3 +2163,7 @@ class UpdateDigestAfterJob(AfterJob):
formdef = formdef_class.get(formdef_id)
for formdata in formdef.data_class().select(order_by='id'):
formdata.store()
class UpdateStatisticsDataAfterJob(UpdateDigestAfterJob):
label = _('Updating statistics data')

View File

@ -181,7 +181,9 @@ def pickle_loads(value):
class Criteria(qommon.storage.Criteria):
def __init__(self, attribute, value, **kwargs):
self.attribute = attribute.replace('-', '_')
self.attribute = attribute
if '->' not in attribute:
self.attribute = self.attribute.replace('-', '_')
self.value = value
self.field = kwargs.get('field')
@ -925,7 +927,8 @@ BEGIN
NEW.criticality_level - {criticality_levels},
{geoloc_base_x},
{geoloc_base_y},
NEW.anonymised);
NEW.anonymised,
NEW.statistics_data);
RETURN NEW;
ELSE
UPDATE wcs_all_forms SET
@ -948,7 +951,8 @@ BEGIN
criticality_level = NEW.criticality_level - {criticality_levels},
geoloc_base_x = {geoloc_base_x},
geoloc_base_y = {geoloc_base_y},
anonymised = NEW.anonymised
anonymised = NEW.anonymised,
statistics_data = NEW.statistics_data
WHERE formdef_id = {formdef_id} AND id = OLD.id;
RETURN NEW;
END IF;
@ -1790,7 +1794,8 @@ def do_global_views(conn, cur):
criticality_level integer,
geoloc_base_x double precision,
geoloc_base_y double precision,
anonymised timestamp with time zone
anonymised timestamp with time zone,
statistics_data jsonb
, PRIMARY KEY(formdef_id, id)
)"""
)
@ -1894,7 +1899,8 @@ def init_global_table(conn=None, cur=None):
criticality_level - {criticality_levels},
{geoloc_base_x},
{geoloc_base_y},
anonymised
anonymised,
statistics_data
FROM {table_name}
ON CONFLICT DO NOTHING;
""".format(
@ -2502,6 +2508,7 @@ class SqlDataMixin(SqlMixin):
('digests', 'jsonb'),
('user_label', 'varchar'),
('auto_geoloc', 'point'),
('statistics_data', 'jsonb'),
]
def __init__(self, id=None):
@ -2598,7 +2605,8 @@ class SqlDataMixin(SqlMixin):
'''UPDATE %s
SET id_display = %%(id_display)s,
digests = %%(digests)s,
user_label = %%(user_label)s
user_label = %%(user_label)s,
statistics_data = %%(statistics_data)s
WHERE id = %%(id)s'''
% self._table_name
)
@ -2609,6 +2617,7 @@ class SqlDataMixin(SqlMixin):
'id_display': self.id_display,
'digests': self.digests,
'user_label': self.user_label,
'statistics_data': self.statistics_data,
},
)
@ -2630,6 +2639,7 @@ class SqlDataMixin(SqlMixin):
'submission_channel': self.submission_channel,
'criticality_level': self.criticality_level,
'workflow_merged_roles_dict': self.workflow_merged_roles_dict,
'statistics_data': self.statistics_data or {},
}
if self.last_update_time:
sql_dict['last_update_time'] = datetime.datetime.fromtimestamp(time.mktime(self.last_update_time))
@ -4639,7 +4649,11 @@ def get_period_query(
formdef_class = criteria.value
continue
if criteria.__class__.__name__ == 'Equal' and criteria.attribute == 'formdef_id':
if (
formdef_class
and criteria.__class__.__name__ == 'Equal'
and criteria.attribute == 'formdef_id'
):
# if there's a formdef_id specified, switch to using the
# specific table so we have access to all fields
table_name = get_formdef_table_name(formdef_class.get(criteria.value))
@ -4936,7 +4950,7 @@ def get_period_total(
# latest migration, number + description (description is not used
# programmaticaly but will make sure git conflicts if two migrations are
# separately added with the same number)
SQL_LEVEL = (79, 'add translatable column to TranslatableMessage table')
SQL_LEVEL = (80, 'add statistics data column to formdata')
def migrate_global_views(conn, cur):
@ -5199,10 +5213,12 @@ def migrate():
continue
for formdata in formdef.data_class().select_iterator():
formdata._set_auto_fields(cur) # build digests
if sql_level < 69:
if sql_level < 80:
# 58: add workflow_merged_roles_dict as a jsonb column with
# combined formdef and formdata value.
# 69: add auto_geoloc field to form/card tables
# 80: add jsonb column to hold statistics data
for formdef in FormDef.select() + CardDef.select():
do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
migrate_views(conn, cur)

View File

@ -30,7 +30,7 @@ from wcs.categories import Category
from wcs.formdata import FormData
from wcs.formdef import FormDef
from wcs.qommon import _, misc, pgettext_lazy
from wcs.qommon.storage import Contains, Equal, GreaterOrEqual, Less, NotNull, Null, Or, StrictNotEqual
from wcs.qommon.storage import Contains, Equal, GreaterOrEqual, Less, Null, Or, StrictNotEqual
class RestrictedView(View):
@ -252,11 +252,7 @@ class FormsCountView(RestrictedView):
except KeyError:
return HttpResponseBadRequest('invalid form')
form_page = self.formpage_class(formdef=formdef, update_breadcrumbs=False)
# formdef_klass is a fake criteria, it will be used in time interval functions
# to switch to appropriate class, it must appear before formdef_id.
totals_kwargs['criterias'].append(Equal('formdef_klass', self.formdef_class))
totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
self.set_formdef_parameters(totals_kwargs, formdef)
totals_kwargs['criterias'].extend(self.get_filters_criterias(formdef, form_page))
self.set_group_by_parameters(group_by, formdef, form_page, totals_kwargs, group_labels)
subfilters = self.get_subfilters(form_page, group_by)
@ -314,8 +310,27 @@ class FormsCountView(RestrictedView):
{'data': {'x_labels': x_labels, 'series': series, 'subfilters': subfilters}, 'err': 0}
)
def set_formdef_parameters(self, totals_kwargs, formdef):
# set formdef_klass to None to deactivate switching to formdef specific table
totals_kwargs['criterias'].append(Equal('formdef_klass', None))
totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
def transform_criteria(self, criteria):
if not hasattr(criteria, 'field'):
return criteria
attribute = "statistics_data->'%s'" % criteria.field.varname
if isinstance(criteria.value, bool):
value = str(criteria.value).lower()
else:
value = '"%s"' % criteria.value
return sql.ArrayContains(attribute, value)
def get_filters_criterias(self, formdef, form_page):
criterias = form_page.get_criterias_from_query(statistics_fields_only=True)
criterias = [self.transform_criteria(criteria) for criteria in criterias]
selected_status = self.request.GET.get('filter-status')
applied_filters = None
@ -471,10 +486,10 @@ class FormsCountView(RestrictedView):
if group_by_field.type == 'status':
totals_kwargs['group_by'] = 'status'
else:
totals_kwargs['group_by'] = sql.get_field_id(group_by_field)
totals_kwargs['group_by'] = "statistics_data->'%s'" % group_by_field.varname
if self.request.GET.get('hide_none_label') == 'true':
totals_kwargs['criterias'].append(NotNull(totals_kwargs['group_by']))
totals_kwargs['criterias'].append(StrictNotEqual(totals_kwargs['group_by'], '[]'))
group_labels.update(self.get_group_labels(group_by_field, formdef, form_page, group_by))
@ -494,6 +509,8 @@ class FormsCountView(RestrictedView):
groups = total[1]
if not isinstance(groups, list):
groups = [groups]
if not groups:
groups = [None]
for group in groups:
totals_by_group[group] += total[2]
seen_group_values.add(group)
@ -516,6 +533,8 @@ class FormsCountView(RestrictedView):
for groups, total in totals:
if not isinstance(groups, list):
groups = [groups]
if not groups:
groups = [None]
for group in groups:
totals_by_group[group] += total
@ -562,6 +581,12 @@ class CardsCountView(FormsCountView):
has_global_count_support = False
label = _('Cards Count')
def set_formdef_parameters(self, totals_kwargs, formdef):
# formdef_klass is a fake criteria, it will be used in time interval functions
# to switch to appropriate class, it must appear before formdef_id.
totals_kwargs['criterias'].append(Equal('formdef_klass', CardDef))
totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
class ResolutionTimeView(RestrictedView):
formdef_class = FormDef