wcs/wcs/fields.py

# w.c.s. - web application for online forms
# Copyright (C) 2005-2010  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import base64
import collections
import copy
import datetime
import html
import os
import random
import re
import sys
import time

from django.utils.encoding import force_bytes, force_str, smart_str
from django.utils.formats import date_format as django_date_format
from django.utils.html import strip_tags, urlize
from lxml import etree as ET
from quixote import get_publisher, get_request, get_session
from quixote.html import TemplateIO, htmlescape, htmltag, htmltext

from wcs.sql_criterias import Contains

from . import data_sources, portfolio
from .blocks import BlockDef, BlockWidget
from .conditions import Condition
from .qommon import N_, _, evalutils, force_str, get_cfg, misc
from .qommon.form import (
    AutocompleteStringWidget,
    CheckboxesWidget,
    CheckboxWidget,
    CommentWidget,
    CompositeWidget,
    ComputedExpressionWidget,
    ConditionWidget,
    DateWidget,
    EmailWidget,
    FileSizeWidget,
    FileWithPreviewWidget,
    Form,
    HiddenWidget,
    HtmlWidget,
    IntWidget,
    JsonpSingleSelectWidget,
    MapMarkerSelectionWidget,
    MapWidget,
    MultiSelectWidget,
    PasswordEntryWidget,
    RadiobuttonsWidget,
    RankedItemsWidget,
    RichTextWidget,
    SingleSelectHintWidget,
    SingleSelectTableWidget,
    SingleSelectWidget,
    StringWidget,
    TableListRowsWidget,
    TableWidget,
    TextWidget,
    ValidationWidget,
    VarnameWidget,
    WcsExtraStringWidget,
    WidgetList,
    WidgetListAsTable,
    WysiwygTextWidget,
)
from .qommon.misc import (
    check_carddefs,
    check_formdefs,
    check_wscalls,
    date_format,
    ellipsize,
    get_as_datetime,
    get_document_type_value_options,
    strftime,
    strip_some_tags,
    xml_node_text,
)
from .qommon.ods import NS as OD_NS
from .qommon.ods import clean_text as od_clean_text
from .qommon.template import Template, TemplateError
from .qommon.upload_storage import PicklableUpload
from .sessions import BasicSession


class SetValueError(Exception):
    pass


class MissingBlockFieldError(Exception):
    def __init__(self, block_slug):
        self.block_slug = block_slug

    def __str__(self):
        return force_str(_('Missing block field: %s') % self.block_slug)


class PrefillSelectionWidget(CompositeWidget):
    def __init__(self, name, value=None, field=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)

        if not value:
            value = {}

        options = [
            ('none', _('None')),
            ('string', _('String / Template')),
        ]
        if value.get('type') == 'formula':
            options.append(('formula', _('Python Expression (deprecated)')))
        options += [
            ('user', _('User Field')),
            ('geolocation', _('Geolocation')),
        ]

        if field and field.key == 'items':
            # limit choices strings (must be templates giving complex data) or
            # python; items field are prefilled with list of strings
            options = [x for x in options if x[0] in ('none', 'string', 'formula')]
        elif field and field.key == 'map':
            # limit choices to geolocation
            options = [x for x in options if x[0] in ('none', 'string', 'geolocation')]

        self.add(
            SingleSelectWidget,
            'type',
            options=options,
            value=value.get('type') or 'none',
            attrs={'data-dynamic-display-parent': 'true'},
        )

        self.parse()
        if not self.value or self.value.get('type') == 'none':
            self.value = {}

        self.prefill_types = prefill_types = collections.OrderedDict(options)
        self.add(
            StringWidget,
            'value_string',
            size=80,
            value=value.get('value') if value.get('type') == 'string' else None,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('string'),
            },
        )
        self.add(
            StringWidget,
            'value_formula',
            size=80,
            value=value.get('value') if value.get('type') == 'formula' else None,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('formula'),
            },
        )

        formdef = get_publisher().user_class.get_formdef()
        users_cfg = get_cfg('users', {})
        if formdef:
            user_fields = []
            for user_field in formdef.fields:
                if user_field.label in [x[1] for x in user_fields]:
                    # do not allow duplicated field names
                    continue
                user_fields.append((user_field.id, user_field.label))
            if not users_cfg.get('field_email'):
                user_fields.append(('email', _('Email (builtin)')))
        else:
            user_fields = [('name', _('Name')), ('email', _('Email'))]
        self.add(
            SingleSelectWidget,
            'value_user',
            value=value.get('value') if value.get('type') == 'user' else None,
            options=user_fields,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('user'),
            },
        )

        if field and field.key == 'map':
            # different prefilling sources on map fields
            geoloc_fields = [('position', _('Position'))]
        else:
            geoloc_fields = [
                ('house', _('Number')),
                ('road', _('Street')),
                ('number-and-street', _('Number and street')),
                ('postcode', _('Post Code')),
                ('city', _('City')),
                ('country', _('Country')),
            ]
            if field and field.key == 'item':
                geoloc_fields.append(('address-id', _('Address Identifier')))
        self.add(
            SingleSelectWidget,
            'value_geolocation',
            value=value.get('value') if value.get('type') == 'geolocation' else None,
            options=geoloc_fields,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('geolocation'),
            },
        )

        # exclude geolocation from locked prefill as the data necessarily
        # comes from the user device.
        self.add(
            CheckboxWidget,
            'locked',
            value=value.get('locked'),
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value-in': '|'.join(
                    [str(x[1]) for x in options if x[0] not in ('none', 'geolocation')]
                ),
                'inline_title': _('Locked'),
            },
        )

        self._parsed = False

    def _parse(self, request):
        values = {}
        type_ = self.get('type')
        if type_ and type_ != 'none':
            values['type'] = type_
            values['locked'] = self.get('locked')
            value = self.get('value_%s' % type_)
            if value:
                values['value'] = value
        self.value = values or None
        if values and values['type'] == 'formula' and values.get('value'):
            try:
                compile(values.get('value', ''), '<string>', 'eval')
            except (SyntaxError, TypeError) as e:
                self.set_error(_('invalid expression: %s') % e)
        if values and values['type'] == 'string' and Template.is_template_string(values.get('value')):
            try:
                Template(values.get('value'), raises=True)
            except TemplateError as e:
                self.set_error(str(e))

    def render_content(self):
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render_content()
        return r.getvalue()


class Field:
    id = None
    varname = None
    label = None
    extra_css_class = None
    convert_value_from_str = None
    convert_value_to_str = None
    convert_value_from_anything = None
    allow_complex = False
    allow_statistics = False
    display_locations = []
    prefill = None
    keep_raw_value = True
    store_display_value = None
    store_structured_value = None
    get_opendocument_node_value = None
    condition = None

    # flag a field for removal by AnonymiseWorkflowStatusItem
    # can be overriden in field' settings
    anonymise = True
    stats = None

    # declarations for serialization, they are mostly for legacy files,
    # new exports directly include typing attributes.
    TEXT_ATTRIBUTES = ['label', 'type', 'hint', 'varname', 'extra_css_class']

    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k.replace('-', '_'), v)

    @classmethod
    def init(cls):
        pass

    def get_type_label(self):
        return self.description

    @property
    def include_in_listing(self):
        return 'listings' in (self.display_locations or [])

    @property
    def include_in_validation_page(self):
        return 'validation' in (self.display_locations or [])

    @property
    def include_in_summary_page(self):
        return 'summary' in (self.display_locations or [])

    @property
    def include_in_statistics(self):
        return self.allow_statistics and self.varname and 'statistics' in (self.display_locations or [])

    @property
    def unhtmled_label(self):
        return force_str(html.unescape(force_str(re.sub('<.*?>', ' ', self.label or ''))).strip())

    @property
    def ellipsized_label(self):
        return ellipsize(self.unhtmled_label)

    def get_admin_attributes(self):
        return ['label', 'condition']

    def export_to_json(self, include_id=False):
        field = {}
        if include_id:
            extra_fields = ['id']
        else:
            extra_fields = []
        for attribute in self.get_admin_attributes() + extra_fields:
            if attribute == 'display_locations':
                continue
            if hasattr(self, attribute) and getattr(self, attribute) is not None:
                val = getattr(self, attribute)
                field[attribute] = val
        field['type'] = self.key
        field['in_statistics'] = self.include_in_statistics
        return field

    def init_with_json(self, elem, include_id=False):
        if include_id:
            self.id = elem.get('id')
        for attribute in self.get_admin_attributes():
            if attribute in elem:
                setattr(self, attribute, elem.get(attribute))

    def export_to_xml(self, charset, include_id=False):
        field = ET.Element('field')
        if include_id:
            extra_fields = ['id']
        else:
            extra_fields = []
        ET.SubElement(field, 'type').text = self.key
        for attribute in self.get_admin_attributes() + extra_fields:
            if hasattr(self, '%s_export_to_xml' % attribute):
                getattr(self, '%s_export_to_xml' % attribute)(field, charset, include_id=include_id)
                continue
            if hasattr(self, attribute) and getattr(self, attribute) is not None:
                val = getattr(self, attribute)
                if isinstance(val, dict) and not val:
                    continue
                el = ET.SubElement(field, attribute)
                if isinstance(val, dict):
                    for k, v in sorted(val.items()):
                        if isinstance(v, str):
                            text_value = force_str(v, charset, errors='replace')
                        else:
                            # field having non str value in dictionnary field must overload
                            # import_to_xml to handle import
                            text_value = force_str(v)
                        ET.SubElement(el, k).text = text_value
                elif isinstance(val, list):
                    if attribute[-1] == 's':
                        atname = attribute[:-1]
                    else:
                        atname = 'item'
                    # noqa pylint: disable=not-an-iterable
                    for v in val:
                        ET.SubElement(el, atname).text = force_str(v, charset, errors='replace')
                elif isinstance(val, str):
                    el.attrib['type'] = 'str'
                    el.text = force_str(val, charset, errors='replace')
                else:
                    el.text = str(val)
                    if isinstance(val, bool):
                        el.attrib['type'] = 'bool'
                    elif isinstance(val, int):
                        el.attrib['type'] = 'int'
        return field

    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
        for attribute in self.get_admin_attributes():
            el = elem.find(attribute)
            if hasattr(self, '%s_init_with_xml' % attribute):
                getattr(self, '%s_init_with_xml' % attribute)(
                    el, charset, include_id=include_id, snapshot=False
                )
                continue
            if el is None:
                continue
            if list(el):
                if isinstance(getattr(self, attribute), list):
                    v = [xml_node_text(x) for x in el]
                elif isinstance(getattr(self, attribute), dict):
                    v = {}
                    for e in el:
                        v[e.tag] = xml_node_text(e)
                else:
                    print('currently:', self.__dict__)
                    print('  attribute:', attribute)
                    # ???
                    raise AssertionError
                setattr(self, attribute, v)
            else:
                if attribute in self.TEXT_ATTRIBUTES:
                    elem_type = 'str'
                else:
                    elem_type = el.attrib.get('type')
                if el.text is None:
                    if isinstance(getattr(self, attribute), list):
                        setattr(self, attribute, [])
                    else:
                        setattr(self, attribute, None)
                elif elem_type == 'bool' or (not elem_type and el.text in ('False', 'True')):
                    # boolean
                    setattr(self, attribute, el.text == 'True')
                elif elem_type == 'int' or (not elem_type and isinstance(getattr(self, attribute), int)):
                    setattr(self, attribute, int(el.text))
                else:
                    setattr(self, attribute, xml_node_text(el))
        if include_id:
            try:
                self.id = xml_node_text(elem.find('id'))
            except Exception:
                pass

    def condition_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        self.condition = None
        if node is None:
            return
        if node.findall('type'):
            self.condition = {
                'type': xml_node_text(node.find('type')),
                'value': xml_node_text(node.find('value')),
            }
        elif node.text:
            self.condition = {'type': 'python', 'value': force_str(node.text).strip()}

    def data_source_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        self.data_source = {}
        if node is None:
            return
        if node.findall('type'):
            self.data_source = {
                'type': xml_node_text(node.find('type')),
                'value': xml_node_text(node.find('value')),
            }
            if self.data_source.get('type') is None:
                self.data_source = {}
            elif self.data_source.get('value') is None:
                del self.data_source['value']

    def prefill_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        self.prefill = {}
        if node is not None and node.findall('type'):
            self.prefill = {
                'type': xml_node_text(node.find('type')),
            }
            if self.prefill['type'] and self.prefill['type'] != 'none':
                self.prefill['value'] = xml_node_text(node.find('value'))
                if xml_node_text(node.find('locked')) == 'True':
                    self.prefill['locked'] = True

    def get_rst_view_value(self, value, indent=''):
        return indent + self.get_view_value(value)

    def get_csv_heading(self):
        return []

    def get_csv_value(self, element, **kwargs):
        return []

    def get_structured_value(self, data):
        if not self.store_structured_value:
            return None
        return data.get('%s_structured' % self.id)

    def get_prefill_configuration(self):
        if self.prefill and self.prefill.get('type') == 'none':
            # make sure a 'none' prefill is not considered as a value
            self.prefill = None
        return self.prefill or {}

    def get_prefill_value(self, user=None, force_string=True):
        # returns a tuple with two items,
        #  1. value[str], the value that will be used to prefill
        #  2. locked[bool], a flag to know if this is a locked value
        #     (because it has been explicitely marked so or because it
        #     comes from verified identity data).
        t = self.prefill.get('type')
        explicit_lock = bool(self.prefill.get('locked'))
        if t == 'string':
            value = self.prefill.get('value')
            if not Template.is_template_string(value):
                return (value, explicit_lock)

            from wcs.workflows import WorkflowStatusItem

            try:
                with get_publisher().complex_data():
                    v = WorkflowStatusItem.compute(
                        value,
                        raises=True,
                        allow_complex=self.allow_complex and not force_string,
                        record_errors=False,
                    )
                    if v and self.allow_complex:
                        v = get_publisher().get_cached_complex_data(v)
                return (v, explicit_lock)
            except TemplateError:
                return ('', explicit_lock)
            except AttributeError as e:
                get_publisher().record_error(
                    _('Failed to evaluate prefill on field "%s"') % self.label,
                    formdef=getattr(self, 'formdef', None),
                    exception=e,
                )
                return ('', explicit_lock)

        elif t == 'user' and user:
            x = self.prefill.get('value')
            if x == 'phone':
                # get mapped field
                x = get_cfg('users', {}).get('field_phone') or x
            if x == 'email':
                return (user.email, explicit_lock or 'email' in (user.verified_fields or []))
            elif user.form_data:
                userform = user.get_formdef()
                for userfield in userform.fields:
                    if userfield.id == x:
                        value = user.form_data.get(x)
                        if (
                            value
                            and getattr(userfield, 'validation', None)
                            and userfield.validation['type'] in ('phone', 'phone-fr')
                        ):
                            country_code = None
                            if (
                                getattr(self, 'validation', None)
                                and self.validation.get('type') == 'phone-fr'
                            ):
                                country_code = 'FR'
                            value = misc.get_formatted_phone(user.form_data.get(x), country_code)
                        return (
                            value,
                            explicit_lock or str(userfield.id) in (user.verified_fields or []),
                        )

        elif t == 'formula':
            formula = self.prefill.get('value')
            try:
                ret = misc.eval_python(
                    formula,
                    get_publisher().get_global_eval_dict(),
                    get_publisher().substitutions.get_context_variables(),
                )
                if isinstance(ret, datetime.time):
                    ret = misc.site_encode(django_date_format(ret, format='TIME_FORMAT'))
                if isinstance(ret, datetime.date):
                    ret = ret.strftime(date_format())
                if ret:
                    if force_string:
                        # prefilling is done with strings for most fields so
                        # we default to forcing the value as a string.
                        # (items field are prefilled with list of strings, and
                        # will get the native python object)
                        ret = str(ret)
                    return (ret, explicit_lock)
            except Exception:
                pass

        elif t == 'geolocation':
            return (None, False)

        return (None, False)

    def get_prefill_attributes(self):
        if not self.get_prefill_configuration():
            return
        t = self.prefill.get('type')

        if t == 'geolocation':
            return {'geolocation': self.prefill.get('value')}

        if t == 'user':
            formdef = get_publisher().user_class.get_formdef()
            for user_field in formdef.fields or []:
                if user_field.id != self.prefill.get('value'):
                    continue
                try:
                    autocomplete_attribute = re.search(
                        r'\bautocomplete-([a-z0-9-]+)', user_field.extra_css_class
                    ).groups()[0]
                except (TypeError, IndexError, AttributeError):
                    continue
                return {'autocomplete': autocomplete_attribute}

        return None

    def feed_session(self, value, display_value):
        pass

    def migrate(self):
        changed = False
        if getattr(self, 'in_listing', None):  # 2019-09-28
            self.display_locations = self.display_locations[:]
            self.display_locations.append('listings')
            changed = True
            self.in_listing = None
        return changed

    @staticmethod
    def evaluate_condition(dict_vars, formdef, condition, record_errors=True):
        return PageCondition(
            condition, {'dict_vars': dict_vars, 'formdef': formdef}, record_errors
        ).evaluate()

    def is_visible(self, dict, formdef):
        try:
            return self.evaluate_condition(dict, formdef, self.condition)
        except RuntimeError:
            return True

    @classmethod
    def get_referenced_varnames(cls, formdef, value):
        return re.findall(
            r'\b(?:%s)[_\.]var[_\.]([a-zA-Z0-9_]+?)(?:_raw|_live_|_structured_|\b)'
            % '|'.join(formdef.var_prefixes),
            value or '',
        )

    def get_condition_varnames(self, formdef):
        return self.get_referenced_varnames(formdef, self.condition['value'])

    def has_live_conditions(self, formdef, hidden_varnames=None):
        varnames = self.get_condition_varnames(formdef)
        if not varnames:
            return False
        field_position = formdef.fields.index(self)
        # rewind to field page
        for field_position in range(field_position, -1, -1):
            if formdef.fields[field_position].key == 'page':
                break
        else:
            field_position = -1  # form with no page
        # start from there
        for field in formdef.fields[field_position + 1 :]:
            if field.key == 'page':
                # stop at next page
                break
            if field.varname in varnames and (
                hidden_varnames is None or field.varname not in hidden_varnames
            ):
                return True
        return False

    def from_json_value(self, value):
        if value is None:
            return value
        return str(value)

    def set_value(self, data, value, raise_on_error=False):
        data['%s' % self.id] = value
        if self.store_display_value:
            display_value = self.store_display_value(data, self.id)
            if raise_on_error and display_value is None:
                raise SetValueError('a datasource is unavailable (field id: %s)' % self.id)
            if display_value:
                data['%s_display' % self.id] = display_value
            elif '%s_display' % self.id in data:
                del data['%s_display' % self.id]
        if self.store_structured_value and value:
            structured_value = self.store_structured_value(data, self.id, raise_on_error=raise_on_error)
            if structured_value:
                if isinstance(structured_value, dict) and structured_value.get('id'):
                    # in case of list field, override id
                    data['%s' % self.id] = str(structured_value.get('id'))
                data['%s_structured' % self.id] = structured_value
            elif '%s_structured' % self.id in data:
                del data['%s_structured' % self.id]
        elif self.store_structured_value and '%s_structured' % self.id in data:
            del data['%s_structured' % self.id]

    def get_dependencies(self):
        if getattr(self, 'data_source', None):
            data_source_type = self.data_source.get('type')
            if data_source_type and data_source_type.startswith('carddef:'):
                from .carddef import CardDef

                carddef_slug = data_source_type.split(':')[1]
                try:
                    yield CardDef.get_by_urlname(carddef_slug)
                except KeyError:
                    pass
            else:
                from .data_sources import NamedDataSource

                yield NamedDataSource.get_by_slug(data_source_type, ignore_errors=True)
        if getattr(self, 'prefill', None):
            prefill = self.prefill
            if prefill:
                if prefill.get('type') == 'string':
                    yield from check_wscalls(prefill.get('value'))
                    yield from check_carddefs(prefill.get('value'))
                    yield from check_formdefs(prefill.get('value'))
        if getattr(self, 'condition', None):
            condition = self.condition
            if condition:
                if condition.get('type') == 'django':
                    yield from check_wscalls(condition.get('value'))
                    yield from check_carddefs(condition.get('value'))
                    yield from check_formdefs(condition.get('value'))

    def get_parameters_view(self):
        r = TemplateIO(html=True)
        form = Form()
        self.fill_admin_form(form)
        parameters = [x for x in self.get_admin_attributes() if getattr(self, x, None) is not None]
        r += htmltext('<ul>')
        for parameter in parameters:
            widget = form.get_widget(parameter)
            if not widget:
                continue
            label = self.get_parameter_view_label(widget, parameter)
            if not label:
                continue
            value = getattr(self, parameter, Ellipsis)
            if value is None or value == getattr(self.__class__, parameter, Ellipsis):
                continue
            parameter_view_value = self.get_parameter_view_value(widget, parameter)
            if parameter_view_value:
                r += htmltext('<li class="parameter-%s">' % parameter)
                r += htmltext('<span class="parameter">%s</span> ') % _('%s:') % label
                r += parameter_view_value
                r += htmltext('</li>')
        r += htmltext('</ul>')
        return r.getvalue()

    def get_parameter_view_label(self, widget, parameter):
        if hasattr(self, 'get_%s_parameter_view_label' % parameter):
            return getattr(self, 'get_%s_parameter_view_label' % parameter)()
        return widget.get_title()

    def get_parameter_view_value(self, widget, parameter):
        if hasattr(self, 'get_%s_parameter_view_value' % parameter):
            return getattr(self, 'get_%s_parameter_view_value' % parameter)(widget)
        value = getattr(self, parameter)
        if isinstance(value, bool):
            return str(_('Yes') if value else _('No'))
        elif hasattr(widget, 'options') and value:
            if not isinstance(widget, CheckboxesWidget):
                value = [value]
            value_labels = []
            for option in widget.options:
                if isinstance(option, tuple):
                    if option[0] in value:
                        value_labels.append(str(option[1]))
                else:
                    if option in value:
                        value_labels.append(str(option))
            return ', '.join(value_labels) if value_labels else '-'
        elif isinstance(value, list):
            return ', '.join(value)
        else:
            return str(value)

    def get_prefill_parameter_view_value(self, widget):
        value = self.get_prefill_configuration()
        if not value:
            return
        r = TemplateIO(html=True)
        r += htmltext('<ul>')
        r += htmltext('<li><span class="parameter">%s%s</span> %s</li>') % (
            _('Type'),
            _(':'),
            widget.prefill_types.get(value.get('type')),
        )
        if value.get('type') in ('user', 'geolocation'):
            select_widget = widget.get_widget('value_%s' % value['type'])
            labels = {x[0]: x[1] for x in select_widget.options}
            r += htmltext('<li><span class="parameter">%s%s</span> %s</li>') % (
                _('Value'),
                _(':'),
                labels.get(value.get('value'), '-'),
            )
        else:
            r += htmltext('<li><span class="parameter">%s%s</span> %s</li>') % (
                _('Value'),
                _(':'),
                value.get('value'),
            )
        if value.get('locked'):
            r += htmltext('<li>%s</li>') % _('Locked')
        r += htmltext('</ul>')
        return r.getvalue()

    def get_data_source_parameter_view_value(self, widget):
        value = getattr(self, 'data_source', None)
        if not value or value.get('type') == 'none':
            return

        if value.get('type').startswith('carddef:'):
            from wcs.carddef import CardDef

            parts = value['type'].split(':')
            try:
                carddef = CardDef.get_by_urlname(parts[1])
            except KeyError:
                return str(_('deleted card model'))
            custom_view = CardDef.get_data_source_custom_view(value['type'], carddef=carddef)
            r = htmltext('<a href="%(url)s">%(label)s</a>') % {
                'label': _('card model: %s') % carddef.name,
                'url': carddef.get_admin_url(),
            }
            if custom_view:
                r += ', '
                r += htmltext('<a href="%(url)s">%(label)s</a>') % {
                    'label': _('custom view: %s') % custom_view.title,
                    'url': '%s%s' % (carddef.get_url(), custom_view.get_url_slug()),
                }
            return r

        data_source_types = {
            'json': _('JSON URL'),
            'jsonp': _('JSONP URL'),
            'geojson': _('GeoJSON URL'),
            'formula': _('Python Expression (deprecated)'),
            'jsonvalue': _('JSON Expression'),
        }
        if value.get('type') in data_source_types:
            return '%s - %s' % (data_source_types[value.get('type')], value.get('value'))

        from wcs.data_sources import NamedDataSource

        data_source = NamedDataSource.get_by_slug(value['type'], stub_fallback=True)
        return htmltext('<a href="%(url)s">%(label)s</a>') % {
            'label': data_source.name,
            'url': data_source.get_admin_url(),
        }

    def get_condition_parameter_view_value(self, widget):
        if not self.condition or self.condition.get('type') == 'none':
            return
        return htmltext('<tt class="condition">%s</tt> <span class="condition-type">(%s)</span>') % (
            self.condition['value'],
            {'django': 'Django', 'python': 'Python'}.get(self.condition['type']),
        )

    def __repr__(self):
        return '<%s %s %r>' % (self.__class__.__name__, self.id, self.label and self.label[:64])

    def i18n_scan(self, base_location):
        location = '%s%s/' % (base_location, self.id)
        yield location, None, self.label
        yield location, None, getattr(self, 'hint', None)


class WidgetField(Field):
    hint = None
    required = True
    display_locations = ['validation', 'summary']
    extra_attributes = []
    prefill = {}

    widget_class = None
    use_live_server_validation = False

    def add_to_form(self, form, value=None):
        kwargs = {'required': self.required, 'render_br': False}
        if value:
            kwargs['value'] = value
        for k in self.extra_attributes:
            if hasattr(self, k):
                kwargs[k] = getattr(self, k)
        self.perform_more_widget_changes(form, kwargs)
        if self.hint and self.hint.startswith('<'):
            hint = htmltext(get_publisher().translate(self.hint))
        else:
            hint = get_publisher().translate(self.hint or '')
        form.add(self.widget_class, 'f%s' % self.id, title=self.label, hint=hint, **kwargs)
        widget = form.get_widget('f%s' % self.id)
        widget.field = self
        widget.use_live_server_validation = self.use_live_server_validation
        if self.extra_css_class:
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
            else:
                widget.extra_css_class = self.extra_css_class
        if self.varname:
            widget.div_id = 'var_%s' % self.varname
        return widget

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        pass

    def add_to_view_form(self, form, value=None):
        kwargs = {'render_br': False}

        self.field_key = 'f%s' % self.id
        self.perform_more_widget_changes(form, kwargs, False)

        for k in self.extra_attributes:
            if hasattr(self, k):
                kwargs[k] = getattr(self, k)

        if self.widget_class is StringWidget and 'size' not in kwargs and value:
            # set a size if there is not one already defined, this will be for
            # example the case with ItemField
            kwargs['size'] = len(value)

        form.add(
            self.widget_class, self.field_key, title=self.label, value=value, readonly='readonly', **kwargs
        )
        widget = form.get_widget(self.field_key)
        widget.transfer_form_value(get_request())
        widget.field = self
        if self.extra_css_class:
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
            else:
                widget.extra_css_class = self.extra_css_class
        return widget

    def get_display_locations_options(self):
        options = [
            ('validation', _('Validation Page')),
            ('summary', _('Summary Page')),
            ('listings', _('Management Listings')),
        ]

        if self.allow_statistics:
            options.append(('statistics', _('Statistics')))

        return options

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(
            CheckboxWidget,
            'required',
            title=_('Required'),
            value=self.required,
            default_value=self.__class__.required,
        )
        form.add(TextWidget, 'hint', title=_('Hint'), value=self.hint, cols=60, rows=3)
        form.add(
            VarnameWidget,
            'varname',
            title=_('Identifier'),
            value=self.varname,
            size=30,
            advanced=False,
            hint=_('This is used as suffix for variable names.'),
        )
        form.add(
            CheckboxesWidget,
            'display_locations',
            title=_('Display Locations'),
            options=self.get_display_locations_options(),
            value=self.display_locations,
            tab=('display', _('Display')),
            default_value=self.__class__.display_locations,
        )
        form.add(
            StringWidget,
            'extra_css_class',
            title=_('Extra classes for CSS styling'),
            value=self.extra_css_class,
            size=30,
            tab=('display', _('Display')),
        )
        form.add(
            PrefillSelectionWidget,
            'prefill',
            title=_('Prefill'),
            value=self.prefill,
            advanced=True,
            field=self,
        )
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
            allow_python=False,
            tab=('display', _('Display')),
        )
        if 'anonymise' in self.get_admin_attributes():
            # override anonymise flag default value
            form.add(
                CheckboxWidget,
                'anonymise',
                title=_('Anonymise'),
                value=self.anonymise,
                advanced=True,
                hint=_('Marks the field data for removal in the anonymisation processes.'),
                default_value=self.__class__.anonymise,
            )

    def check_admin_form(self, form):
        display_locations = form.get_widget('display_locations').parse()
        varname = form.get_widget('varname').parse()
        if 'statistics' in display_locations and not varname:
            form.set_error(
                'display_locations', _('Field must have a varname in order to be displayed in statistics.')
            )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + [
            'required',
            'hint',
            'varname',
            'display_locations',
            'extra_css_class',
            'prefill',
        ]

    def get_csv_heading(self):
        return [self.label]

    def get_value_info(self, data):
        # return the selected value and an optional dictionary that will be
        # passed to get_view_value() to provide additional details.
        value_details = {}
        if self.id not in data:
            value = None
        else:
            if self.store_display_value and ('%s_display' % self.id) in data:
                value = data['%s_display' % self.id]
                value_details['value_id'] = data[self.id]
            else:
                value = data[self.id]

            if value is None or value == '':
                value = None
        return (value, value_details)

    def get_view_value(self, value, **kwargs):
        return str(value) if value else ''

    def get_view_short_value(self, value, max_len=30, **kwargs):
        return self.get_view_value(value)

    def get_csv_value(self, element, **kwargs):
        if self.convert_value_to_str:
            return [self.convert_value_to_str(element)]
        return [element]

    def get_fts_value(self, data, **kwargs):
        if self.store_display_value:
            return data.get('%s_display' % self.id)
        return data.get(str(self.id))


field_classes = []
field_types = []


def register_field_class(klass):
    if klass not in field_classes:
        field_classes.append(klass)
        if not issubclass(klass, WidgetField):
            field_types.append((klass.key, klass.description))
        else:
            non_widgets = [x for x in field_classes if not issubclass(x, WidgetField)]
            if not non_widgets:
                field_types.append((klass.key, klass.description))
            else:
                idx = field_types.index([x for x in field_types if x[0] == non_widgets[0].key][0])
                field_types.insert(idx, (klass.key, klass.description))
        klass.init()


class TitleField(Field):
    key = 'title'
    description = _('Title')
    html_tag = 'h3'
    display_locations = ['validation', 'summary']

    def add_to_form(self, form, value=None):
        import wcs.workflows

        extra_attributes = ' data-field-id="%s"' % self.id
        if self.extra_css_class:
            extra_attributes += ' class="%s"' % self.extra_css_class
        title_markup = '<{html_tag}{extra_attributes}>%s</{html_tag}>'.format(
            html_tag=self.html_tag,
            extra_attributes=extra_attributes,
        )
        label = wcs.workflows.template_on_formdata(
            None, get_publisher().translate(self.label), autoescape=False
        )
        widget = HtmlWidget(htmltext(title_markup) % label)
        widget.field = self
        form.widgets.append(widget)
        return widget

    add_to_view_form = add_to_form

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(
            StringWidget,
            'extra_css_class',
            title=_('Extra classes for CSS styling'),
            value=self.extra_css_class,
            size=30,
            tab=('display', _('Display')),
        )
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
            allow_python=False,
            tab=('display', _('Display')),
        )
        form.add(
            CheckboxesWidget,
            'display_locations',
            title=_('Display Locations'),
            options=self.get_display_locations_options(),
            value=self.display_locations,
            default_value=self.__class__.display_locations,
            tab=('display', _('Display')),
        )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + ['extra_css_class', 'display_locations']

    def get_dependencies(self):
        yield from super().get_dependencies()
        yield from check_wscalls(self.label)
        yield from check_carddefs(self.label)
        yield from check_formdefs(self.label)


register_field_class(TitleField)


class SubtitleField(TitleField):
    key = 'subtitle'
    description = _('Subtitle')
    html_tag = 'h4'


register_field_class(SubtitleField)


class CommentField(Field):
    key = 'comment'
    description = _('Comment')
    display_locations = []

    def get_text(self):
        import wcs.workflows

        label = self.get_html_content()
        return wcs.workflows.template_on_html_string(label)

    def add_to_form(self, form, value=None):
        widget = CommentWidget(content=self.get_text(), extra_css_class=self.extra_css_class)
        form.widgets.append(widget)
        widget.field = self
        return widget

    def add_to_view_form(self, *args, **kwargs):
        if self.include_in_validation_page:
            return self.add_to_form(*args, **kwargs)
        return None

    def get_html_content(self):
        if not self.label:
            return ''
        label = get_publisher().translate(self.label)
        if label.startswith('<'):
            return label
        if '\n\n' in label:
            # blank lines to paragraphs
            label = '</p>\n<p>'.join([str(htmlescape(x)) for x in re.split('\n\n+', label)])
            return '<p>' + label + '</p>'
        return '<p>%s</p>' % str(htmlescape(label))

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        if self.label and (self.label[0] != '<' and '[end]' in self.label):
            form.add(
                TextWidget,
                'label',
                title=_('Label'),
                value=self.label,
                validation_function=ComputedExpressionWidget.validate_template,
                required=True,
                cols=70,
                rows=3,
                render_br=False,
            )
        else:
            form.add(
                WysiwygTextWidget,
                'label',
                title=_('Label'),
                validation_function=ComputedExpressionWidget.validate_template,
                value=self.get_html_content(),
                required=True,
            )
        form.add(
            StringWidget,
            'extra_css_class',
            title=_('Extra classes for CSS styling'),
            value=self.extra_css_class,
            size=30,
            advanced=True,
        )
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
            allow_python=False,
            advanced=True,
        )
        form.add(
            CheckboxesWidget,
            'display_locations',
            title=_('Display Locations'),
            options=self.get_display_locations_options(),
            value=self.display_locations,
            advanced=True,
        )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + ['extra_css_class', 'display_locations']

    def get_dependencies(self):
        yield from super().get_dependencies()
        yield from check_wscalls(self.label)
        yield from check_carddefs(self.label)
        yield from check_formdefs(self.label)


register_field_class(CommentField)


class StringField(WidgetField):
    key = 'string'
    description = _('Text (line)')

    widget_class = WcsExtraStringWidget
    size = None
    extra_attributes = ['size']
    validation = {}
    data_source = {}
    keep_raw_value = False

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        if self.data_source:
            data_source = data_sources.get_object(self.data_source)
            if data_source.can_jsonp():
                kwargs['url'] = data_source.get_jsonp_url()
                self.widget_class = AutocompleteStringWidget

    @property
    def use_live_server_validation(self):
        if self.validation and self.validation['type']:
            return bool(ValidationWidget.validation_methods.get(self.validation['type']))
        return False

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        if self.size:
            form.add(
                StringWidget,
                'size',
                title=_('Line length'),
                hint=_(
                    'Deprecated option, it is advised to use CSS classes '
                    'to size the fields in a manner compatible with all devices.'
                ),
                value=self.size,
            )
        else:
            form.add(HiddenWidget, 'size', value=None)
        form.add(
            ValidationWidget,
            'validation',
            title=_('Validation'),
            value=self.validation,
            advanced=True,
        )
        form.add(
            data_sources.DataSourceSelectionWidget,
            'data_source',
            value=self.data_source,
            title=_('Data Source'),
            hint=_('This will allow autocompletion from an external source.'),
            disallowed_source_types={'geojson'},
            advanced=True,
            required=False,
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + ['size', 'validation', 'data_source', 'anonymise']

    def get_view_value(self, value, **kwargs):
        value = value or ''
        if value.startswith('http://') or value.startswith('https://'):
            charset = get_publisher().site_charset
            value = force_str(value, charset)
            return htmltext(force_str(urlize(value, nofollow=True, autoescape=True)))
        return str(value)

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        if value.startswith('http://') or value.startswith('https://'):
            node = ET.Element('{%s}a' % OD_NS['text'])
            node.attrib['{%s}href' % OD_NS['xlink']] = value
        else:
            node = ET.Element('{%s}span' % OD_NS['text'])
        node.text = od_clean_text(force_str(value))
        return node

    def get_rst_view_value(self, value, indent=''):
        return indent + str(value or '')

    def convert_value_from_str(self, value):
        return value

    def convert_value_to_str(self, value):
        if value is None:
            return None
        if isinstance(value, (time.struct_time, datetime.date)):
            return strftime(date_format(), value)
        return str(value)

    @classmethod
    def convert_value_from_anything(cls, value):
        if value is None:
            return None
        return str(value)

    def get_fts_value(self, data, **kwargs):
        value = super().get_fts_value(data, **kwargs)
        if value and self.validation and self.validation['type']:
            validation_method = ValidationWidget.validation_methods.get(self.validation['type'])
            if validation_method and validation_method.get('normalize_for_fts'):
                # index both original and normalized value
                # in the case of phone numbers this makes sure the "international/E164"
                # format (ex: +33199001234) is indexed.
                value = '%s %s' % (value, validation_method.get('normalize_for_fts')(value))
        return value

    def migrate(self):
        changed = super().migrate()
        if isinstance(self.validation, str):  # 2019-08-10
            self.validation = {'type': 'regex', 'value': self.validation}
            changed = True
        return changed

    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
        super().init_with_xml(elem, charset, include_id=include_id)
        self.migrate()

    def get_validation_parameter_view_value(self, widget):
        if not self.validation:
            return
        validation_type = self.validation['type']
        validation_types = {x: y['title'] for x, y in ValidationWidget.validation_methods.items()}
        if validation_type in ('regex', 'django'):
            validation_value = self.validation.get('value')
            if not validation_value:
                return
            return '%s - %s' % (validation_types.get(validation_type), validation_value)
        return str(validation_types.get(validation_type))

    def i18n_scan(self, base_location):
        location = '%s%s/' % (base_location, self.id)
        yield from super().i18n_scan(base_location)
        if self.validation and self.validation.get('error_message'):
            yield location, None, self.validation.get('error_message')


register_field_class(StringField)


class TextField(WidgetField):
    key = 'text'
    description = _('Long Text')

    widget_class = TextWidget
    cols = None
    rows = None
    pre = None
    display_mode = 'plain'
    maxlength = None
    extra_attributes = ['cols', 'rows', 'maxlength']

    def migrate(self):
        changed = super().migrate()
        if isinstance(getattr(self, 'pre', None), bool):  # 2022-09-16
            if self.pre:
                self.display_mode = 'pre'
            else:
                self.display_mode = 'plain'
            self.pre = None
            changed = True
        return changed

    def perform_more_widget_changes(self, *args, **kwargs):
        if self.display_mode == 'rich':
            self.widget_class = RichTextWidget

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        if self.cols:
            form.add(
                StringWidget,
                'cols',
                title=_('Line length'),
                hint=_(
                    'Deprecated option, it is advised to use CSS classes '
                    'to size the fields in a manner compatible with all devices.'
                ),
                value=self.cols,
            )
        else:
            form.add(HiddenWidget, 'cols', value=None)
        form.add(StringWidget, 'rows', title=_('Number of rows'), value=self.rows)
        form.add(StringWidget, 'maxlength', title=_('Maximum number of characters'), value=self.maxlength)
        display_options = []
        if get_publisher().get_site_option('enable-richtext-field'):
            display_options += [
                ('rich', _('Rich Text')),
            ]
        display_options += [
            ('plain', _('Plain Text (with automatic paragraphs on blank lines)')),
            ('pre', _('Plain Text (with linebreaks as typed)')),
        ]
        form.add(
            RadiobuttonsWidget,
            'display_mode',
            title=_('Text display'),
            value=self.display_mode,
            default_value='plain',
            options=display_options,
            advanced=True,
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'cols',
            'rows',
            'display_mode',
            'maxlength',
            'anonymise',
        ]

    def convert_value_from_str(self, value):
        return value

    def get_view_value(self, value, **kwargs):
        if self.display_mode == 'pre':
            return htmltext('<p class="plain-text-pre">') + value + htmltext('</p>')
        elif self.display_mode == 'rich':
            return htmltext(strip_some_tags(value, RichTextWidget.ALL_TAGS))
        else:
            try:
                return (
                    htmltext('<p>')
                    + htmltext('\n').join([(x or htmltext('</p><p>')) for x in value.splitlines()])
                    + htmltext('</p>')
                )
            except Exception:
                return ''

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        if self.display_mode == 'rich':
            return
        paragraphs = []
        for paragraph in value.splitlines():
            if paragraph.strip():
                p = ET.Element('{%s}p' % OD_NS['text'])
                p.text = paragraph
                paragraphs.append(p)
        return paragraphs

    def get_view_short_value(self, value, max_len=30, **kwargs):
        if self.display_mode == 'rich':
            return ellipsize(str(strip_tags(value)), max_len)
        return ellipsize(str(value), max_len)

    def get_json_value(self, value, **kwargs):
        if self.display_mode == 'rich':
            return str(self.get_view_value(value))
        return value


register_field_class(TextField)


class EmailField(WidgetField):
    key = 'email'
    description = _('Email')
    use_live_server_validation = True

    widget_class = EmailWidget

    def convert_value_from_str(self, value):
        return value

    def get_view_value(self, value, **kwargs):
        return htmltext('<a href="mailto:%s">%s</a>') % (value, value)

    def get_rst_view_value(self, value, indent=''):
        return indent + value

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        a = ET.Element('{%s}a' % OD_NS['text'])
        a.text = od_clean_text(value)
        a.attrib['{%s}href' % OD_NS['xlink']] = 'mailto:' + a.text
        return a


register_field_class(EmailField)


class BoolField(WidgetField):
    key = 'bool'
    description = _('Check Box (single choice)')
    allow_complex = True
    allow_statistics = True

    widget_class = CheckboxWidget
    required = False
    anonymise = False

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        if not edit:
            kwargs['disabled'] = 'disabled'
            value = get_request().get_field(self.field_key)
            form.add_hidden(self.field_key, value=str(value))
            widget = form.get_widget(self.field_key)
            widget.field = self
            if value and not value == 'False':
                self.field_key = 'f%sdisabled' % self.id
                get_request().form[self.field_key] = 'yes'
            self.field_key = 'f%sdisabled' % self.id

    def get_view_value(self, value, **kwargs):
        if value is True or value == 'True':
            return str(_('Yes'))
        elif value is False or value == 'False':
            return str(_('No'))
        else:
            return ''

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(self.get_view_value(value))
        return span

    def convert_value_from_anything(self, value):
        if isinstance(value, str):
            return self.convert_value_from_str(value)
        return bool(value)

    def convert_value_from_str(self, value):
        if value is None:
            return None
        for true_word in (N_('True'), N_('Yes')):
            if str(value).lower() in (true_word.lower(), _(true_word).lower()):
                return True
        return False

    def convert_value_to_str(self, value):
        if value is True:
            return 'True'
        elif value is False:
            return 'False'
        return value

    def stats(self, values):
        no_records = len(values)
        if not no_records:
            return
        r = TemplateIO(html=True)
        r += htmltext('<table class="stats">')
        r += htmltext('<thead><tr><th colspan="4">')
        r += self.label
        r += htmltext('</th></tr></thead>')
        options = (True, False)
        r += htmltext('<tbody>')
        for o in options:
            r += htmltext('<tr>')
            r += htmltext('<td class="label">')
            if o is True:
                r += str(_('Yes'))
                value = True
            else:
                r += str(_('No'))
                value = False
            r += htmltext('</td>')
            no = len([None for x in values if self.convert_value_from_str(x.data.get(self.id)) is value])

            r += htmltext('<td class="percent">')
            r += htmltext(' %.2f&nbsp;%%') % (100.0 * no / no_records)
            r += htmltext('</td>')
            r += htmltext('<td class="total">')
            r += '(%d/%d)' % (no, no_records)
            r += htmltext('</td>')
            r += htmltext('</tr>')
            r += htmltext('<tr>')
            r += htmltext('<td class="bar" colspan="3">')
            r += htmltext('<span style="width: %d%%"></span>' % (100 * no / no_records))
            r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody>')
        r += htmltext('</table>')
        return r.getvalue()

    def from_json_value(self, value):
        if value is None:
            return value
        return bool(value)


register_field_class(BoolField)


class FileField(WidgetField):
    key = 'file'
    description = _('File Upload')
    allow_complex = True

    document_type = None
    max_file_size = None
    automatic_image_resize = False
    allow_portfolio_picking = False
    storage = 'default'

    widget_class = FileWithPreviewWidget
    extra_attributes = [
        'file_type',
        'max_file_size',
        'allow_portfolio_picking',
        'automatic_image_resize',
        'storage',
    ]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.document_type = self.document_type or {}

    @property
    def file_type(self):
        return (self.document_type or {}).get('mimetypes', [])

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        options = get_document_type_value_options(self.document_type)
        form.add(
            SingleSelectWidget,
            'document_type',
            title=_('File type suggestion'),
            value=self.document_type,
            options=options,
            advanced=True,
        )
        form.add(
            FileSizeWidget,
            'max_file_size',
            title=_('Max file size'),
            value=self.max_file_size,
            advanced=True,
        )
        form.add(
            CheckboxWidget,
            'automatic_image_resize',
            title=_('Automatically resize uploaded images'),
            value=self.automatic_image_resize,
            advanced=True,
        )
        if portfolio.has_portfolio():
            form.add(
                CheckboxWidget,
                'allow_portfolio_picking',
                title=_('Allow user to pick a file from a portfolio'),
                value=self.allow_portfolio_picking,
                advanced=True,
            )
        storages = get_publisher().get_site_storages()
        if storages:
            storage_options = [('default', '---', {})]
            storage_options += [(key, value['label'], key) for key, value in storages.items()]
            form.add(
                SingleSelectWidget,
                'storage',
                title=_('File storage system'),
                value=self.storage,
                options=storage_options,
                advanced=True,
            )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'document_type',
            'max_file_size',
            'allow_portfolio_picking',
            'automatic_image_resize',
            'storage',
        ]

    @classmethod
    def convert_value_from_anything(cls, value):
        if not value:
            return None
        from wcs.variables import LazyFieldVarFile

        if isinstance(value, LazyFieldVarFile):
            value = value.get_value()  # unbox
        if hasattr(value, 'base_filename'):
            upload = PicklableUpload(value.base_filename, value.content_type or 'application/octet-stream')
            upload.receive([value.get_content()])
            return upload
        from wcs.workflows import NamedAttachmentsSubstitutionProxy

        if isinstance(value, NamedAttachmentsSubstitutionProxy):
            upload = PicklableUpload(value.filename, value.content_type)
            upload.receive([value.content])
            return upload
        if isinstance(value, dict):
            # if value is a dictionary we expect it to have a content or
            # b64_content key and a filename keys and an optional
            # content_type key.
            if 'b64_content' in value:
                value_content = base64.decodebytes(force_bytes(value['b64_content']))
            else:
                value_content = value.get('content')
            if 'filename' in value and value_content:
                content_type = value.get('content_type') or 'application/octet-stream'
                if content_type.startswith('text/'):
                    charset = 'utf-8'
                else:
                    charset = None
                upload = PicklableUpload(value['filename'], content_type, charset)
                upload.receive([force_bytes(value_content)])
                return upload
        raise ValueError('invalid data for file type (%r)' % value)

    def get_view_short_value(self, value, max_len=30, **kwargs):
        return self.get_view_value(value, include_image_thumbnail=False, max_len=max_len, **kwargs)

    def get_prefill_value(self, user=None, force_string=True):
        return super().get_prefill_value(user=user, force_string=False)

    def get_download_query_string(self, **kwargs):
        if kwargs.get('file_value'):
            return 'hash=%s' % kwargs.get('file_value').file_digest()
        if kwargs.get('parent_field'):
            return 'f=%s$%s$%s' % (kwargs['parent_field'].id, kwargs['parent_field_index'], self.id)
        return 'f=%s' % self.id

    def get_view_value(self, value, include_image_thumbnail=True, max_len=None, **kwargs):
        show_link = True
        if value.has_redirect_url():
            is_in_backoffice = bool(get_request() and get_request().is_in_backoffice())
            show_link = bool(value.get_redirect_url(backoffice=is_in_backoffice))
        t = TemplateIO(html=True)
        t += htmltext('<div class="file-field">')
        if show_link or include_image_thumbnail:
            download_qs = self.get_download_query_string(**kwargs)
        if show_link:
            attrs = {
                'href': '[download]?%s' % download_qs,
            }
            if kwargs.get('label_id'):
                attrs['aria-describedby'] = kwargs.get('label_id')
            if max_len:
                attrs['title'] = value
            t += htmltag('a', **attrs)
        if include_image_thumbnail and value.can_thumbnail():
            t += htmltext('<img alt="" src="[download]?%s&thumbnail=1"/>') % download_qs
        filename = str(value)
        if max_len and len(filename) > max_len:
            basename, ext = os.path.splitext(filename)
            basename = ellipsize(basename, max_len - 5)
            filename = basename + ext
        t += htmltext('<span>%s</span>') % filename
        if show_link:
            t += htmltext('</a>')
        t += htmltext('</div>')
        return t.getvalue()

    def get_download_url(self, formdata, **kwargs):
        return '%s?%s' % (formdata.get_file_base_url(), self.get_download_query_string(**kwargs))

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        show_link = True
        if value.has_redirect_url():
            is_in_backoffice = bool(get_request() and get_request().is_in_backoffice())
            show_link = bool(value.get_redirect_url(backoffice=is_in_backoffice))
        if show_link and formdata:
            node = ET.Element('{%s}a' % OD_NS['text'])
            node.attrib['{%s}href' % OD_NS['xlink']] = self.get_download_url(formdata, **kwargs)
        else:
            node = ET.Element('{%s}span' % OD_NS['text'])
        node.text = od_clean_text(force_str(value))
        return node

    def get_csv_value(self, element, **kwargs):
        return [str(element) if element else '']

    def get_json_value(self, value, formdata=None, include_file_content=True, **kwargs):
        out = value.get_json_value(include_file_content=include_file_content)
        if formdata:
            out['url'] = self.get_download_url(formdata, file_value=value, **kwargs)
            if value and misc.can_thumbnail(value.content_type):
                out['thumbnail_url'] = out['url'] + '&thumbnail=1'
        out['field_id'] = self.id
        return out

    def from_json_value(self, value):
        if value and 'filename' in value and 'content' in value:
            content = base64.b64decode(value['content'])
            content_type = value.get('content_type', 'application/octet-stream')
            if content_type.startswith('text/'):
                charset = 'utf-8'
            else:
                charset = None
            upload = PicklableUpload(value['filename'], content_type, charset)
            upload.receive([content])
            return upload
        return None

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        if not edit:
            value = get_request().get_field(self.field_key)
            if value and hasattr(value, 'token'):
                get_request().form[self.field_key + '$token'] = value.token

    def export_to_xml(self, charset, include_id=False):
        # convert some sub-fields to strings as export_to_xml() only supports
        # dictionnaries with strings values
        if self.document_type and self.document_type.get('mimetypes'):
            old_value = self.document_type['mimetypes']
            self.document_type['mimetypes'] = '|'.join(self.document_type['mimetypes'])
        result = super().export_to_xml(charset, include_id=include_id)
        if self.document_type and self.document_type.get('mimetypes'):
            self.document_type['mimetypes'] = old_value
        return result

    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
        super().init_with_xml(elem, charset, include_id=include_id)
        # translate fields flattened to strings
        if self.document_type and self.document_type.get('mimetypes'):
            self.document_type['mimetypes'] = self.document_type['mimetypes'].split('|')
        if self.document_type and self.document_type.get('fargo'):
            self.document_type['fargo'] = self.document_type['fargo'] == 'True'


register_field_class(FileField)


class DateField(WidgetField):
    key = 'date'
    description = _('Date')

    widget_class = DateWidget
    minimum_date = None
    maximum_date = None
    minimum_is_future = False
    date_in_the_past = False
    date_can_be_today = False
    extra_attributes = [
        'minimum_date',
        'minimum_is_future',
        'maximum_date',
        'date_in_the_past',
        'date_can_be_today',
    ]

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        form.add(DateWidget, 'minimum_date', title=_('Minimum Date'), value=self.minimum_date)
        form.add(
            CheckboxWidget,
            'minimum_is_future',
            title=_('Date must be in the future'),
            value=self.minimum_is_future,
            hint=_('This option is obviously not compatible with setting a minimum date'),
        )
        form.add(DateWidget, 'maximum_date', title=_('Maximum Date'), value=self.maximum_date)
        form.add(
            CheckboxWidget,
            'date_in_the_past',
            title=_('Date must be in the past'),
            value=self.date_in_the_past,
            hint=_('This option is obviously not compatible with setting a maximum date'),
        )
        form.add(
            CheckboxWidget,
            'date_can_be_today',
            title=_('Date can be present day'),
            value=self.date_can_be_today,
            hint=_('This option is only useful combined with one of the previous checkboxes.'),
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'minimum_date',
            'minimum_is_future',
            'maximum_date',
            'date_in_the_past',
            'date_can_be_today',
            'anonymise',
        ]

    @classmethod
    def convert_value_from_anything(cls, value):
        if value is None or value == '':
            return None
        date_value = evalutils.make_date(value).timetuple()  # could raise ValueError
        return date_value

    def convert_value_from_str(self, value):
        if not value:
            return None
        try:
            return get_as_datetime(value).timetuple()
        except ValueError:
            return None

    def convert_value_to_str(self, value):
        if value is None:
            return ''
        if isinstance(value, str):
            return value
        try:
            return strftime(date_format(), value)
        except TypeError:
            return ''

    def add_to_form(self, form, value=None):
        if value and not isinstance(value, str):
            value = self.convert_value_to_str(value)
        return WidgetField.add_to_form(self, form, value=value)

    def add_to_view_form(self, form, value=None):
        value = strftime(misc.date_format(), value)
        return super().add_to_view_form(form, value=value)

    def get_view_value(self, value, **kwargs):
        try:
            return strftime(misc.date_format(), value)
        except TypeError:
            return value

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(self.get_view_value(value))
        return span

    def get_json_value(self, value, **kwargs):
        try:
            return strftime('%Y-%m-%d', value)
        except TypeError:
            return ''

    def from_json_value(self, value):
        try:
            return time.strptime(value, '%Y-%m-%d')
        except (TypeError, ValueError):
            return None


register_field_class(DateField)


def item_items_stats(field, values):
    if field.data_source:
        options = data_sources.get_items(field.data_source)
    else:
        options = field.items or []
    if len(options) == 0:
        return None
    no_records = len(values)
    if no_records == 0:
        return None
    r = TemplateIO(html=True)
    r += htmltext('<table class="stats">')
    r += htmltext('<thead><tr><th colspan="4">')
    r += field.label
    r += htmltext('</th></tr></thead>')
    r += htmltext('<tbody>')
    for option in options:
        if type(option) in (tuple, list):
            option_label = option[1]
            option_value = str(option[0])
        else:
            option_label = option
            option_value = option
        if field.key == 'item':
            no = len([None for x in values if x.data.get(field.id) == option_value])
        else:
            no = len([None for x in values if option_value in (x.data.get(field.id) or [])])
        r += htmltext('<tr>')
        r += htmltext('<td class="label">')
        r += option_label
        r += htmltext('</td>')

        r += htmltext('<td class="percent">')
        r += htmltext(' %.2f&nbsp;%%') % (100.0 * no / no_records)
        r += htmltext('</td>')
        r += htmltext('<td class="total">')
        r += '(%d/%d)' % (no, no_records)
        r += htmltext('</td>')
        r += htmltext('</tr>')
        r += htmltext('<tr>')
        r += htmltext('<td class="bar" colspan="3">')
        r += htmltext('<span style="width: %d%%"></span>' % (100 * no / no_records))
        r += htmltext('</td>')
        r += htmltext('</tr>')
    r += htmltext('</tbody>')
    r += htmltext('</table>')
    return r.getvalue()


class MapOptionsMixin:
    initial_zoom = None
    min_zoom = None
    max_zoom = None

    @classmethod
    def get_zoom_levels(cls):
        zoom_levels = [
            (None, '---'),
            ('0', _('Whole world')),
            ('6', _('Country')),
            ('9', _('Wide area')),
            ('11', _('Area')),
            ('13', _('Town')),
            ('16', _('Small road')),
            ('18', _('Neighbourhood')),
            ('19', _('Ant')),
        ]
        return zoom_levels

    def fill_zoom_admin_form(self, form, **kwargs):
        zoom_levels = self.get_zoom_levels()
        zoom_levels_dict = dict(zoom_levels)
        default_zoom_level = get_publisher().get_default_zoom_level()
        initial_zoom_levels = zoom_levels[:]
        initial_zoom_levels[0] = (None, _('Default (%s)') % zoom_levels_dict[default_zoom_level])
        form.add(
            SingleSelectWidget,
            'initial_zoom',
            title=_('Initial zoom level'),
            value=self.initial_zoom,
            options=initial_zoom_levels,
            **kwargs,
        )
        form.add(
            SingleSelectWidget,
            'min_zoom',
            title=_('Minimal zoom level'),
            value=self.min_zoom,
            options=zoom_levels,
            required=False,
            **kwargs,
        )
        form.add(
            SingleSelectWidget,
            'max_zoom',
            title=_('Maximal zoom level'),
            value=self.max_zoom,
            options=zoom_levels,
            required=False,
            **kwargs,
        )

    def check_zoom_admin_form(self, form):
        initial_zoom = form.get_widget('initial_zoom').parse()
        min_zoom = form.get_widget('min_zoom').parse()
        max_zoom = form.get_widget('max_zoom').parse()
        if min_zoom and max_zoom:
            if int(min_zoom) > int(max_zoom):
                form.get_widget('min_zoom').set_error(
                    _('Minimal zoom level cannot be greater than maximal zoom level.')
                )
        # noqa pylint: disable=too-many-boolean-expressions
        if (initial_zoom and min_zoom and int(initial_zoom) < int(min_zoom)) or (
            initial_zoom and max_zoom and int(initial_zoom) > int(max_zoom)
        ):
            form.get_widget('initial_zoom').set_error(
                _('Initial zoom level must be between minimal and maximal zoom levels.')
            )


class ItemFieldMixin:
    def get_real_data_source(self):
        return data_sources.get_real(self.data_source)

    def add_items_fields_admin_form(self, form):
        real_data_source = self.get_real_data_source()
        form.add(
            RadiobuttonsWidget,
            'data_mode',
            title=_('Data'),
            options=[
                ('simple-list', _('Simple List'), 'simple-list'),
                ('data-source', _('Data Source'), 'data-source'),
            ],
            value='data-source' if real_data_source else 'simple-list',
            attrs={'data-dynamic-display-parent': 'true'},
            extra_css_class='widget-inline-radio no-bottom-margin',
        )
        form.add(
            WidgetList,
            'items',
            element_type=StringWidget,
            value=self.items,
            required=False,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add item'),
            attrs={'data-dynamic-display-child-of': 'data_mode', 'data-dynamic-display-value': 'simple-list'},
            extra_css_class='sortable',
        )
        form.add(
            data_sources.DataSourceSelectionWidget,
            'data_source',
            value=self.data_source,
            required=False,
            hint=_('This will get the available items from an external source.'),
            disallowed_source_types={'geojson'},
            attrs={'data-dynamic-display-child-of': 'data_mode', 'data-dynamic-display-value': 'data-source'},
        )

    def check_items_admin_form(self, form):
        data_mode = form.get_widget('data_mode').parse()
        if data_mode == 'simple-list':
            items = form.get_widget('items').parse()
            d = {}
            for v in items or []:
                if v in d:
                    form.set_error('items', _('Duplicated Items'))
                    return
                d[v] = None

            data_source_type = form.get_widget('data_source').get_widget('type')
            data_source_type.set_value(None)
            data_source_type.transfer_form_value(get_request())

    def get_items_parameter_view_label(self):
        if self.data_source:
            # skip field if there's a data source
            return None
        return str(_('Choices'))

    def get_data_source_parameter_view_label(self):
        return str(_('Data source'))

    def get_carddef(self):
        from wcs.carddef import CardDef

        if (
            not self.data_source
            or not self.data_source.get('type')
            or not self.data_source['type'].startswith('carddef:')
        ):
            return None

        carddef_slug = self.data_source['type'].split(':')[1]
        try:
            return CardDef.get_by_urlname(carddef_slug)
        except KeyError:
            return None

    def get_extended_options(self):
        if self.data_source:
            return data_sources.get_structured_items(
                self.data_source, mode='lazy', include_disabled=self.display_disabled_items
            )
        if self.items:
            return [{'id': x, 'text': x} for x in self.items]
        return []

    def export_to_json_data_source(self, field):
        if 'items' in field:
            del field['items']
        data_source_type = self.data_source.get('type')
        if data_source_type and data_source_type.startswith('carddef:'):
            carddef_slug = data_source_type.split(':')[1]
            url = None
            if data_source_type.count(':') == 1:
                url = '/api/cards/%s/list' % carddef_slug
            else:
                custom_view_slug = data_source_type.split(':')[2]
                if not custom_view_slug.startswith('_'):
                    url = '/api/cards/%s/%s/list' % (carddef_slug, custom_view_slug)
            if url:
                field['items_url'] = get_request().build_absolute_uri(url)
        return field

    def i18n_scan(self, base_location):
        location = '%s%s/' % (base_location, self.id)
        real_data_source = self.get_real_data_source()
        if not real_data_source:
            for item in self.items or []:
                yield location, None, item

    def get_display_value(self, value):
        data_source = data_sources.get_object(self.data_source)
        if data_source is None:
            return get_publisher().translate(value) or ''

        if data_source.type == 'jsonp':
            if not get_session():
                return value
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            return get_session().jsonp_display_values.get('%s_%s' % (data_source.get_jsonp_url(), value))

        display_value = data_source.get_display_value(value)
        session = get_session()
        if (
            isinstance(session, BasicSession)
            and self.display_mode == 'autocomplete'
            and data_source
            and data_source.can_jsonp()
        ):
            carddef = self.get_carddef()
            url_kwargs = {}
            if self.key == 'item' and get_request().is_in_backoffice() and carddef:
                url_kwargs['with_related'] = True
            # store display value in session to be used by select2
            url = data_source.get_jsonp_url(**url_kwargs)
            if not session.jsonp_display_values:
                session.jsonp_display_values = {}
            session.jsonp_display_values['%s_%s' % (url, value)] = display_value

        return display_value


class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin):
    key = 'item'
    description = _('List')
    allow_complex = True
    allow_statistics = True

    items = []
    show_as_radio = None
    anonymise = False
    widget_class = SingleSelectHintWidget
    data_source = {}
    in_filters = False
    display_disabled_items = False
    display_mode = 'list'

    # <select> option
    use_hint_as_first_option = True

    # timetable option
    initial_date_alignment = None

    # map options
    initial_position = None
    default_position = None
    position_template = None

    def __init__(self, **kwargs):
        self.items = []
        WidgetField.__init__(self, **kwargs)

    def migrate(self):
        changed = super().migrate()
        if isinstance(getattr(self, 'show_as_radio', None), bool):  # 2019-03-19
            if self.show_as_radio:
                self.display_mode = 'radio'
            else:
                self.display_mode = 'list'
            self.show_as_radio = None
            changed = True
        return changed

    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
        super().init_with_xml(elem, charset, include_id=include_id)
        if getattr(elem.find('show_as_radio'), 'text', None) == 'True':
            self.display_mode = 'radio'

    @property
    def extra_attributes(self):
        if self.display_mode == 'map':
            return [
                'initial_zoom',
                'min_zoom',
                'max_zoom',
                'initial_position',
                'position_template',
                'data_source',
            ]
        return []

    def get_options(self, mode=None):
        if self.data_source:
            return [
                x[:3]
                for x in data_sources.get_items(
                    self.data_source, mode=mode, include_disabled=self.display_disabled_items
                )
            ]
        if self.items:
            return [(x, get_publisher().translate(x), x) for x in self.items]
        return []

    def get_id_by_option_text(self, text_value):
        if self.data_source:
            return data_sources.get_id_by_option_text(self.data_source, text_value)
        return text_value

    def get_display_mode(self, data_source=None):
        if not data_source:
            data_source = data_sources.get_object(self.data_source)

        if data_source and data_source.type == 'jsonp':
            # a source defined as JSONP can only be used in autocomplete mode
            return 'autocomplete'

        return self.display_mode

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        data_source = data_sources.get_object(self.data_source)
        display_mode = self.get_display_mode(data_source)

        if display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
            carddef = self.get_carddef()
            url_kwargs = {}
            if get_request().is_in_backoffice() and carddef:
                if carddef.can_user_add_cards(get_request().user):
                    kwargs['add_related_url'] = carddef.get_backoffice_submission_url()
                kwargs['with_related'] = True
                url_kwargs['with_related'] = True
            self.url = kwargs['url'] = data_source.get_jsonp_url(**url_kwargs)
            self.widget_class = JsonpSingleSelectWidget
            return

        if self.display_mode != 'map':
            if self.data_source:
                items = data_sources.get_items(self.data_source, include_disabled=self.display_disabled_items)
                kwargs['options'] = [x[:3] for x in items if not x[-1].get('disabled')]
                kwargs['options_with_attributes'] = items[:]
            else:
                kwargs['options'] = self.get_options()
            if not kwargs.get('options'):
                kwargs['options'] = [(None, '---', None)]
        if display_mode == 'list':
            kwargs['use_hint_as_first_option'] = self.use_hint_as_first_option
        elif display_mode == 'radio':
            self.widget_class = RadiobuttonsWidget
            if isinstance(kwargs['options'][0], str):
                first_items = [x for x in kwargs['options'][:3]]
            else:
                first_items = [x[1] for x in kwargs['options'][:3]]
            length_first_items = sum(len(x) for x in first_items)
            # display radio buttons on a single line if there's just a few
            # short options.
            # TODO: absence/presence of delimitor should be an option
            self.inline = bool(len(kwargs['options']) <= 3 and length_first_items <= 40)
        elif display_mode == 'autocomplete':
            kwargs['select2'] = True
        elif display_mode == 'map':
            self.widget_class = MapMarkerSelectionWidget
        elif display_mode == 'timetable':
            # SingleSelectHintWidget with custom template
            kwargs['template-name'] = 'qommon/forms/widgets/select-timetable.html'

    def get_view_value(self, value, value_id=None, **kwargs):
        data_source = data_sources.get_object(self.data_source)
        if value and data_source is None:
            return get_publisher().translate(value) or ''
        value = super().get_view_value(value)
        if not (value_id and self.data_source and self.data_source.get('type', '').startswith('carddef:')):
            return value
        carddef = self.get_carddef()
        if not carddef:
            return value
        try:
            carddata = carddef.data_class().get(value_id)
        except KeyError:
            return value
        parts = data_source.data_source['type'].split(':')
        digest_key = 'default'
        value = (carddata.digests or {}).get(digest_key) or value
        if len(parts) == 3:
            digest_key = 'custom-view:%s' % parts[-1]
            value = (carddata.digests or {}).get(digest_key) or value
        if get_publisher().has_i18n_enabled():
            digest_key += ':' + get_publisher().current_language
            value = (carddata.digests or {}).get(digest_key) or value
        if not (
            get_request()
            and get_request().is_in_backoffice()
            and carddef.is_user_allowed_read(get_request().user, carddata)
        ):
            return value
        return htmltext('<a href="%s">' % carddata.get_url(backoffice=True)) + htmltext('%s</a>') % value

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(force_str(value))
        return span

    def add_to_view_form(self, form, value=None):
        real_value = value
        label_value = ''
        if value is not None:
            label_value = self.get_display_value(value)
        self.field_key = 'f%s' % self.id

        form.add(
            StringWidget,
            self.field_key + '_label',
            title=self.label,
            value=label_value,
            readonly='readonly',
            size=len(label_value or '') + 2,
            render_br=False,
        )
        label_widget = form.get_widget(self.field_key + '_label')
        # don't let subwidget overwrite label widget value
        label_widget.secondary = True
        get_request().form[label_widget.name] = label_value
        label_widget.field = self
        form.add(HiddenWidget, self.field_key, value=real_value)
        form.get_widget(self.field_key).field = self
        widget = form.get_widget(self.field_key + '_label')
        if self.extra_css_class:
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
            else:
                widget.extra_css_class = self.extra_css_class
        return widget

    def store_display_value(self, data, field_id, raise_on_error=False):
        value = data.get(field_id)
        if not value:
            return ''
        data_source = data_sources.get_object(self.data_source)
        if data_source and data_source.type == 'jsonp':
            if get_request():
                display_value = get_request().form.get('f%s_display' % field_id)
                real_data_source = data_source.data_source
                if display_value is None:
                    if not get_session().jsonp_display_values:
                        get_session().jsonp_display_values = {}
                    display_value = get_session().jsonp_display_values.get(
                        '%s_%s' % (real_data_source.get('value'), value)
                    )
                else:
                    if not get_session().jsonp_display_values:
                        get_session().jsonp_display_values = {}
                    get_session().jsonp_display_values[
                        '%s_%s' % (real_data_source.get('value'), value)
                    ] = display_value
                return display_value
        with get_publisher().with_language('default'):
            return self.get_display_value(value)

    def store_structured_value(self, data, field_id, raise_on_error=False):
        data_source = data_sources.get_object(self.data_source)
        if data_source is None:
            return

        if data_source.type == 'jsonp':
            return

        with get_publisher().with_language('default'):
            value = data_source.get_structured_value(data.get(field_id))
        if value is None and raise_on_error:
            raise SetValueError('a datasource is unavailable (field id: %s)' % field_id)

        if value is None or set(value.keys()) == {'id', 'text'}:
            return
        return value

    def convert_value_from_anything(self, value):
        if not value:
            return None
        value = str(value)
        if self.data_source:
            data_source = data_sources.get_object(self.data_source)
            if data_source.type and data_source.type.startswith('carddef:'):
                card_value = data_source.get_card_structured_value_by_id(value)
                if card_value:
                    value = str(card_value['id'])
                else:
                    raise ValueError('unknown card value (%r)' % value)
        return value

    def convert_value_from_str(self, value):
        # caller should also call store_display_value and store_structured_value
        return value

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        form.add(
            CheckboxWidget,
            'in_filters',
            title=_('Display in default filters'),
            value=self.in_filters,
            advanced=True,
        )
        options = [
            ('list', _('List'), 'list'),
            ('radio', _('Radio buttons'), 'radio'),
            ('autocomplete', _('Autocomplete'), 'autocomplete'),
            ('map', _('Map (requires geographical data)'), 'map'),
            ('timetable', _('Timetable'), 'timetable'),
        ]
        form.add(
            RadiobuttonsWidget,
            'display_mode',
            title=_('Display Mode'),
            options=options,
            value=self.display_mode,
            attrs={'data-dynamic-display-parent': 'true'},
            extra_css_class='widget-inline-radio',
        )
        self.add_items_fields_admin_form(form)
        form.add(
            CheckboxWidget,
            'display_disabled_items',
            title=_('Display disabled items'),
            value=self.display_disabled_items,
            advanced=True,
        )
        form.add(
            StringWidget,
            'initial_date_alignment',
            title=_('Initial date alignment'),
            value=self.initial_date_alignment,
            validation_function=ComputedExpressionWidget.validate_template,
            attrs={
                'data-dynamic-display-child-of': 'display_mode',
                'data-dynamic-display-value': 'timetable',
            },
        )
        self.fill_zoom_admin_form(
            form, attrs={'data-dynamic-display-child-of': 'display_mode', 'data-dynamic-display-value': 'map'}
        )
        initial_position_widget = form.add(
            RadiobuttonsWidget,
            'initial_position',
            title=_('Initial Position'),
            options=(
                ('', _('Default position (from markers)'), ''),
                ('template', _('From template'), 'template'),
            ),
            value=self.initial_position or '',
            extra_css_class='widget-inline-radio',
            attrs={
                'data-dynamic-display-child-of': 'display_mode',
                'data-dynamic-display-value': 'map',
                'data-dynamic-display-parent': 'true',
            },
        )
        form.add(
            StringWidget,
            'position_template',
            value=self.position_template,
            size=80,
            required=False,
            hint=_('Positions (using latitute;longitude format) and addresses are supported.'),
            validation_function=ComputedExpressionWidget.validate_template,
            attrs={
                'data-dynamic-display-child-of': initial_position_widget.get_name(),
                'data-dynamic-display-value': 'template',
            },
        )
        form.add(
            CheckboxWidget,
            'use_hint_as_first_option',
            title=_('Use hint as first option'),
            value=self.use_hint_as_first_option,
            attrs={
                'data-dynamic-display-child-of': 'display_mode',
                'data-dynamic-display-value': 'list',
            },
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'display_mode',
            'items',
            'data_source',
            'in_filters',
            'anonymise',
            'display_disabled_items',
            'initial_zoom',
            'min_zoom',
            'max_zoom',
            'initial_position',
            'position_template',
            'initial_date_alignment',
            'use_hint_as_first_option',
        ]

    def check_admin_form(self, form):
        super().check_admin_form(form)
        self.check_items_admin_form(form)
        self.check_zoom_admin_form(form)

    def stats(self, values):
        return item_items_stats(self, values)

    def get_initial_date_alignment(self):
        if not self.initial_date_alignment:
            return
        import wcs.workflows

        try:
            date = wcs.workflows.template_on_formdata(None, self.initial_date_alignment, autoescape=False)
        except TemplateError:
            return
        try:
            return misc.get_as_datetime(date)
        except ValueError:
            return

    def feed_session(self, value, display_value):
        real_data_source = self.get_real_data_source()
        if real_data_source and real_data_source.get('type') == 'jsonp':
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            get_session().jsonp_display_values[
                '%s_%s' % (real_data_source.get('value'), value)
            ] = display_value

    def get_csv_heading(self):
        if self.data_source:
            return ['%s (%s)' % (self.label, _('identifier')), self.label]
        return [self.label]

    def get_csv_value(self, element, display_value=None, **kwargs):
        values = [element]
        if self.data_source:
            values.append(display_value)
        return values

    def export_to_json(self, include_id=False):
        field = super().export_to_json(include_id=include_id)
        if self.data_source:
            self.export_to_json_data_source(field)
        return field

    def i18n_scan(self, base_location):
        yield from super(WidgetField, self).i18n_scan(base_location)
        yield from ItemFieldMixin.i18n_scan(self, base_location)


register_field_class(ItemField)


class ItemsField(WidgetField, ItemFieldMixin):
    key = 'items'
    description = _('Multiple choice list')
    allow_complex = True
    allow_statistics = True

    items = []
    min_choices = 0
    max_choices = 0
    data_source = {}
    in_filters = False
    display_disabled_items = False
    display_mode = 'checkboxes'

    widget_class = CheckboxesWidget

    _cached_data_source = None

    def __init__(self, **kwargs):
        self.items = []
        WidgetField.__init__(self, **kwargs)

    def get_options(self):
        if self.data_source:
            if self._cached_data_source:
                return self._cached_data_source
            self._cached_data_source = [x[:3] for x in data_sources.get_items(self.data_source)]
            return self._cached_data_source[:]
        elif self.items:
            return [(x, get_publisher().translate(x)) for x in self.items]
        else:
            return []

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['options'] = self.get_options()
        kwargs['min_choices'] = self.min_choices
        kwargs['max_choices'] = self.max_choices
        if self.data_source:
            items = data_sources.get_items(self.data_source, include_disabled=self.display_disabled_items)
            kwargs['options'] = [x[:3] for x in items if not x[-1].get('disabled')]
            kwargs['options_with_attributes'] = items[:]

        if len(kwargs['options']) > 3:
            kwargs['inline'] = False

        if self.display_mode == 'autocomplete':
            self.widget_class = MultiSelectWidget

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        form.add(
            CheckboxWidget,
            'in_filters',
            title=_('Display in default filters'),
            value=self.in_filters,
            advanced=True,
        )
        options = [
            ('checkboxes', _('Checkboxes'), 'checkboxes'),
            ('autocomplete', _('Autocomplete'), 'autocomplete'),
        ]
        form.add(
            RadiobuttonsWidget,
            'display_mode',
            title=_('Display Mode'),
            options=options,
            value=self.display_mode,
            extra_css_class='widget-inline-radio',
        )
        self.add_items_fields_admin_form(form)
        form.add(
            IntWidget,
            'min_choices',
            title=_('Minimum number of choices'),
            value=self.min_choices,
            required=False,
            size=4,
        )
        form.add(
            IntWidget,
            'max_choices',
            title=_('Maximum number of choices'),
            value=self.max_choices,
            required=False,
            size=4,
        )
        form.add(
            CheckboxWidget,
            'display_disabled_items',
            title=_('Display disabled items'),
            value=self.display_disabled_items,
            advanced=True,
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'items',
            'display_mode',
            'min_choices',
            'max_choices',
            'data_source',
            'in_filters',
            'anonymise',
            'display_disabled_items',
        ]

    def check_admin_form(self, form):
        super().check_admin_form(form)
        self.check_items_admin_form(form)

    def get_prefill_value(self, user=None, force_string=True):
        value, explicit_lock = super().get_prefill_value(user=user, force_string=False)
        if value is not None and (
            not isinstance(value, (str, tuple, list)) or not all(isinstance(x, (int, str)) for x in value)
        ):
            get_publisher().record_error(
                _('Invalid value for items prefill on field "%s"') % self.label,
                formdef=getattr(self, 'formdef', None),
            )
            return (None, explicit_lock)
        return (value, explicit_lock)

    def convert_value_to_str(self, value):
        return value

    def convert_value_from_str(self, value):
        if not isinstance(value, str):
            return value
        if not value.strip():
            return None
        return [x.strip() for x in value.split('|') if x.strip()]

    def convert_value_from_anything(self, value):
        if isinstance(value, str):
            return self.convert_value_from_str(value)
        if isinstance(value, int):
            return [str(value)]
        if not value:
            return None
        try:
            return list(value)
        except TypeError:
            raise ValueError('invalid data for items type (%r)' % value)

    def get_value_info(self, data):
        value, value_details = super().get_value_info(data)
        labels = []
        if not self.data_source:
            value_id = value_details.get('value_id')
            if value_id:
                labels = value_id.copy()
        else:
            structured_values = self.get_structured_value(data)
            if structured_values:
                labels = [x['text'] for x in structured_values]
        value_details['labels'] = labels
        return (value, value_details)

    def get_view_value(self, value, **kwargs):
        if kwargs.get('labels'):
            # summary page and labels are available
            r = TemplateIO(html=True)
            r += htmltext('<ul>')
            for x in kwargs['labels']:
                r += htmltext('<li>%s</li>' % x)
            r += htmltext('</ul>')
            return r.getvalue()

        if isinstance(value, str):  # == display_value
            return value
        if value:
            try:
                return ', '.join([(x) for x in value])
            except TypeError:
                pass
        return ''

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(self.get_view_value(value))
        return span

    def stats(self, values):
        return item_items_stats(self, values)

    def get_csv_heading(self):
        if self.data_source:
            labels = ['%s (%s)' % (self.label, _('identifier')), '%s (%s)' % (self.label, _('label'))]
            next_columns = [_('(continued)')] * 2
        else:
            labels = [self.label]
            next_columns = [_('(continued)')]
        if self.max_choices:
            labels.extend(next_columns * (self.max_choices - 1))
        elif len(self.get_options()):
            labels.extend(next_columns * (len(self.get_options()) - 1))
        return labels

    def get_csv_value(self, element, structured_value=None, **kwargs):
        values = []

        if self.max_choices:
            nb_columns = self.max_choices
        elif len(self.get_options()):
            nb_columns = len(self.get_options())
        else:
            nb_columns = 1

        if self.data_source:
            nb_columns *= 2
            for one_value in structured_value or []:
                values.append(one_value.get('id'))
                values.append(one_value.get('text'))
        else:
            for one_value in element:
                values.append(one_value)

        if len(values) > nb_columns:
            # this would happen if max_choices is set after forms were already
            # filled with more values
            values = values[:nb_columns]
        elif len(values) < nb_columns:
            values.extend([''] * (nb_columns - len(values)))

        return values

    def store_display_value(self, data, field_id, raise_on_error=False):
        if not data.get(field_id):
            return ''
        options = self.get_options()
        if not options:
            return ''
        choices = []
        for choice in data.get(field_id) or []:
            if isinstance(options[0], str):
                choices.append(choice)
            elif type(options[0]) in (tuple, list):
                if len(options[0]) == 2:
                    for key, option_value in options:
                        if str(key) == str(choice):
                            choices.append(option_value)
                            break
                elif len(options[0]) == 3:
                    for key, option_value, dummy in options:
                        if str(key) == str(choice):
                            choices.append(option_value)
                            break
                    else:
                        if raise_on_error:
                            raise SetValueError('datasource is unavailable')
        return ', '.join(choices)

    def store_structured_value(self, data, field_id, raise_on_error=False):
        if not data.get(field_id):
            return
        if not self.data_source:
            return
        try:
            structured_options = data_sources.get_structured_items(
                self.data_source, raise_on_error=raise_on_error
            )
        except data_sources.DataSourceError as e:
            raise SetValueError(str(e))
        if not structured_options:
            return
        structured_value = []
        for structured_option in structured_options:
            for choice in data.get(field_id) or []:
                if str(structured_option.get('id')) == str(choice):
                    structured_value.append(structured_option)
        return structured_value

    def export_to_json(self, include_id=False):
        field = super().export_to_json(include_id=include_id)
        if self.data_source:
            self.export_to_json_data_source(field)
        return field

    def from_json_value(self, value):
        if isinstance(value, list):
            return value
        return []

    def get_exploded_options(self, options):
        carddef = self.get_carddef()
        if not carddef:
            # unnest key/values
            exploded_options = {}
            for option_keys, option_label in options:
                if option_keys and option_label:
                    for option_key, option_label in zip(option_keys, option_label.split(', ')):
                        exploded_options[option_key] = option_label
            return exploded_options.items()

        options_ids = set()
        for option in options:
            if option[0]:
                options_ids.update(set(option[0]))

        options_ids = [x for x in options_ids if misc.is_ascii_digit(str(x))]
        cards = carddef.data_class().select([Contains('id', options_ids)])

        return [(str(x.id), x.get_display_label()) for x in cards]

    def i18n_scan(self, base_location):
        yield from super(WidgetField, self).i18n_scan(base_location)
        yield from ItemFieldMixin.i18n_scan(self, base_location)


register_field_class(ItemsField)


class PostConditionsRowWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        if not value:
            value = {}
        self.add(
            ConditionWidget, name='condition', title=_('Condition'), value=value.get('condition'), size=50
        )
        self.add(
            StringWidget,
            name='error_message',
            title=_('Error message if condition is not met'),
            value=value.get('error_message'),
            size=50,
        )

    def _parse(self, request):
        if self.get('condition') or self.get('error_message'):
            self.value = {'condition': self.get('condition'), 'error_message': self.get('error_message')}
        else:
            self.value = None


class PostConditionsTableWidget(WidgetListAsTable):
    readonly = False

    def __init__(self, name, **kwargs):
        super().__init__(name, element_type=PostConditionsRowWidget, **kwargs)

    def parse(self, request=None):
        super().parse(request=request)
        for post_condition in self.value or []:
            if not (post_condition.get('error_message') and post_condition.get('condition')):
                self.set_error(_('Both condition and error message are required.'))
                break
        return self.value


class PageCondition(Condition):
    record_errors = False

    def get_data(self):
        dict_vars = self.context['dict_vars']
        formdef = self.context['formdef']

        # create variables with values currently being evaluated, not yet
        # available in the formdata.
        from .formdata import get_dict_with_varnames

        live_data = {}
        form_live_data = {}
        if dict_vars is not None and formdef:
            live_data = get_dict_with_varnames(formdef.fields, dict_vars)
            form_live_data = {'form_' + x: y for x, y in live_data.items()}

        # 1) feed the form_var_* variables in the global substitution system,
        # they will shadow formdata context variables with their new "live"
        # value, this may be useful when evaluating data sources.
        class ConditionVars:
            def __init__(self, id_dict_var):
                # keep track of reference dictionary
                self.id_dict_var = id_dict_var

            def get_substitution_variables(self):
                return {}

            def get_static_substitution_variables(self):
                # only for backward compatibility with python evaluations
                return form_live_data

            def __eq__(self, other):
                # Assume all ConditionVars are equal when initialized with
                # the same live data dictionary; this avoids filling
                # the substitution sources with duplicates and invalidating its
                # cache.
                return self.id_dict_var == getattr(other, 'id_dict_var', None)

        if dict_vars is not None:
            # Add them only if there is a real dict_vars in context,
            # ie do nothing on first page condition
            get_publisher().substitutions.feed(ConditionVars(id(dict_vars)))

            # alter top-of-stack formdata with data from submitted form
            from wcs.formdata import FormData

            for source in reversed(get_publisher().substitutions.sources):
                if isinstance(source, FormData):
                    source.data.update(dict_vars)
                    break

        data = super().get_data()
        # 2) add live data as var_ variables for local evaluation only, for
        # backward compatibility. They are not added globally as they would
        # interfere with the var_ prefixed variables used in dynamic jsonp
        # fields. (#9786)
        data = copy.copy(data)
        data.update(live_data)
        if dict_vars is None:
            # ConditionsVars is not set when evaluating first page condition,
            # but we need to have form_var_* variables already; add them from
            # form_live_data (where all variables will have been set to None).
            data.update(form_live_data)
        return data


class PageField(Field):
    key = 'page'
    description = _('Page')

    post_conditions = None

    def post_conditions_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        if node is None:
            return
        self.post_conditions = []
        for post_condition_node in node.findall('post_condition'):
            if post_condition_node.findall('condition/type'):
                condition = {
                    'type': xml_node_text(post_condition_node.find('condition/type')),
                    'value': xml_node_text(post_condition_node.find('condition/value')),
                }
            elif post_condition_node.find('condition').text:
                condition = {
                    'type': 'python',
                    'value': xml_node_text(post_condition_node.find('condition')),
                }
            else:
                continue
            self.post_conditions.append(
                {
                    'condition': condition,
                    'error_message': xml_node_text(post_condition_node.find('error_message')),
                }
            )

    def post_conditions_export_to_xml(self, node, charset, include_id=False):
        if not self.post_conditions:
            return

        conditions_node = ET.SubElement(node, 'post_conditions')
        for post_condition in self.post_conditions:
            post_condition_node = ET.SubElement(conditions_node, 'post_condition')
            condition_node = ET.SubElement(post_condition_node, 'condition')
            ET.SubElement(condition_node, 'type').text = force_str(
                (post_condition['condition'] or {}).get('type') or '', charset, errors='replace'
            )
            ET.SubElement(condition_node, 'value').text = force_str(
                (post_condition['condition'] or {}).get('value') or '', charset, errors='replace'
            )
            ET.SubElement(post_condition_node, 'error_message').text = force_str(
                post_condition['error_message'] or '', charset, errors='replace'
            )

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
        )
        form.add(
            PostConditionsTableWidget,
            'post_conditions',
            title=_('Post Conditions'),
            value=self.post_conditions,
            advanced=True,
        )
        form.add(
            VarnameWidget,
            'varname',
            title=_('Identifier'),
            value=self.varname,
            size=30,
            advanced=True,
            hint=_('This is used as reference in workflow edition action.'),
        )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + ['post_conditions', 'varname']

    def add_to_view_form(self, *args, **kwargs):
        pass

    def get_conditions(self):
        if self.condition:
            yield self.condition
        for post_condition in self.post_conditions or []:
            yield post_condition.get('condition')

    def get_post_conditions_parameter_view_value(self, widget):
        if not self.post_conditions:
            return
        r = TemplateIO(html=True)
        r += htmltext('<ul>')
        for post_condition in self.post_conditions:
            r += htmltext('<li>%s <span class="condition-type">(%s)</span> - %s</li>') % (
                post_condition.get('condition').get('value'),
                {'django': 'Django', 'python': 'Python'}.get(post_condition.get('condition').get('type')),
                post_condition.get('error_message'),
            )
        r += htmltext('</ul>')
        return r.getvalue()

    def get_dependencies(self):
        yield from super().get_dependencies()
        if getattr(self, 'post_conditions', None):
            post_conditions = self.post_conditions or []
            for post_condition in post_conditions:
                condition = post_condition.get('condition') or {}
                if condition.get('type') == 'django':
                    yield from check_wscalls(condition.get('value'))
                    yield from check_carddefs(condition.get('value'))
                    yield from check_formdefs(condition.get('value'))

    def i18n_scan(self, base_location):
        location = '%s%s/' % (base_location, self.id)
        yield location, None, self.label
        for post_condition in self.post_conditions or []:
            yield location, None, post_condition.get('error_message')


register_field_class(PageField)


class TableField(WidgetField):
    key = 'table'
    description = _('Table')
    allow_complex = True

    rows = None
    columns = None

    widget_class = TableWidget

    def __init__(self, **kwargs):
        self.rows = []
        self.columns = []
        WidgetField.__init__(self, **kwargs)

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['rows'] = self.rows
        kwargs['columns'] = self.columns

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        try:
            form.remove('prefill')
        except KeyError:  # perhaps it was already removed
            pass
        form.add(
            WidgetList,
            'rows',
            title=_('Rows'),
            element_type=StringWidget,
            value=self.rows,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add row'),
        )
        form.add(
            WidgetList,
            'columns',
            title=_('Columns'),
            element_type=StringWidget,
            value=self.columns,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add column'),
        )

    def get_admin_attributes(self):
        t = WidgetField.get_admin_attributes(self) + ['rows', 'columns']
        try:
            t.remove('prefill')
        except ValueError:
            pass
        return t

    def get_view_value(self, value, **kwargs):
        r = TemplateIO(html=True)
        r += htmltext('<table><thead><tr><td></td>')
        for column in self.columns:
            r += htmltext('<th>%s</th>') % column
        r += htmltext('</tr></thead><tbody>')
        for i, row in enumerate(self.rows):
            r += htmltext('<tr><th>%s</th>') % row
            for j, column in enumerate(self.columns):
                r += htmltext('<td>')
                if value:
                    try:
                        r += value[i][j]
                    except IndexError:
                        pass
                r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody></table>')

        return r.getvalue()

    def get_rst_view_value(self, value, indent=''):
        if not value:
            return indent
        r = []
        max_width = 0
        for column in self.columns:
            max_width = max(max_width, len(smart_str(column)))

        for i, row in enumerate(value):
            value[i] = [x or '' for x in row]

        def get_value(i, j):
            try:
                return smart_str(value[i][j])
            except IndexError:
                return '-'

        for i, row in enumerate(self.rows):
            max_width = max(max_width, len(row))
            for j, column in enumerate(self.columns):
                max_width = max(max_width, len(get_value(i, j)))

        r.append(' '.join(['=' * max_width] * (len(self.columns) + 1)))
        r.append(' '.join([smart_str(column).center(max_width) for column in ['/'] + self.columns]))
        r.append(' '.join(['=' * max_width] * (len(self.columns) + 1)))
        for i, row in enumerate(self.rows):
            r.append(
                ' '.join(
                    [
                        cell.center(max_width)
                        for cell in [smart_str(row)] + [get_value(i, x) for x in range(len(self.columns))]
                    ]
                )
            )
        r.append(' '.join(['=' * max_width] * (len(self.columns) + 1)))
        return misc.site_encode('\n'.join([indent + x for x in r]))

    def get_csv_heading(self):
        if not self.columns:
            return [self.label]
        labels = []
        for col in self.columns:
            for row in self.rows:
                t = '%s / %s' % (col, row)
                if len(labels) == 0:
                    labels.append('%s - %s' % (self.label, t))
                else:
                    labels.append(t)
        return labels

    def get_csv_value(self, element, **kwargs):
        if not self.columns:
            return ['']
        values = []
        for i in range(len(self.columns)):
            for j in range(len(self.rows)):
                try:
                    values.append(element[j][i])
                except IndexError:
                    values.append('')
        return values

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        table = ET.Element('{%s}table' % OD_NS['table'])
        ET.SubElement(table, '{%s}table-column' % OD_NS['table'])
        for col in self.columns:
            ET.SubElement(table, '{%s}table-column' % OD_NS['table'])
        row = ET.SubElement(table, '{%s}table-row' % OD_NS['table'])
        ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
        for col in self.columns:
            table_cell = ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
            cell_value = ET.SubElement(table_cell, '{%s}p' % OD_NS['text'])
            cell_value.text = col
        for i, row_label in enumerate(self.rows):
            row = ET.SubElement(table, '{%s}table-row' % OD_NS['table'])
            table_cell = ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
            cell_value = ET.SubElement(table_cell, '{%s}p' % OD_NS['text'])
            cell_value.text = row_label
            for j, col in enumerate(self.columns):
                table_cell = ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
                cell_value = ET.SubElement(table_cell, '{%s}p' % OD_NS['text'])
                try:
                    cell_value.text = value[i][j]
                except IndexError:
                    pass
        return table

    def from_json_value(self, value):
        return value


register_field_class(TableField)


class TableSelectField(TableField):
    key = 'table-select'
    description = _('Table of Lists')
    allow_complex = True

    items = None

    widget_class = SingleSelectTableWidget

    def __init__(self, **kwargs):
        self.items = []
        TableField.__init__(self, **kwargs)

    def fill_admin_form(self, form):
        TableField.fill_admin_form(self, form)
        form.add(
            WidgetList,
            'items',
            title=_('Items'),
            element_type=StringWidget,
            value=self.items,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add item'),
        )

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        TableField.perform_more_widget_changes(self, form, kwargs, edit=edit)
        if edit:
            kwargs['options'] = self.items or [(None, '---')]
        else:
            self.widget_class = TableWidget

    def get_admin_attributes(self):
        return TableField.get_admin_attributes(self) + ['items']

    def check_admin_form(self, form):
        items = form.get_widget('items').parse()
        d = {}
        for v in items or []:
            if v in d:
                form.set_error('items', _('Duplicated Items'))
                return
            d[v] = None


register_field_class(TableSelectField)


class TableRowsField(WidgetField):
    key = 'tablerows'
    description = _('Table with rows')
    allow_complex = True

    total_row = True
    columns = None

    widget_class = TableListRowsWidget

    def __init__(self, **kwargs):
        self.columns = []
        WidgetField.__init__(self, **kwargs)

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['columns'] = self.columns
        kwargs['add_element_label'] = _('Add row')

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        try:
            form.remove('prefill')
        except KeyError:  # perhaps it was already removed
            pass
        form.add(
            WidgetList,
            'columns',
            title=_('Columns'),
            element_type=StringWidget,
            value=self.columns,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add column'),
        )
        form.add(
            CheckboxWidget,
            'total_row',
            title=_('Total Row'),
            value=self.total_row,
            default_value=self.__class__.total_row,
        )

    def get_admin_attributes(self):
        t = WidgetField.get_admin_attributes(self) + ['columns', 'total_row']
        try:
            t.remove('prefill')
        except ValueError:
            pass
        return t

    def get_view_value(self, value, **kwargs):
        r = TemplateIO(html=True)
        r += htmltext('<table><thead><tr>')
        for column in self.columns:
            r += htmltext('<th>%s</th>') % column
        r += htmltext('</tr></thead><tbody>')
        for row in value:
            r += htmltext('<tr>')
            for j, column in enumerate(self.columns):
                r += htmltext('<td>')
                if value:
                    try:
                        r += row[j]
                    except IndexError:
                        pass
                r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody>')

        if self.total_row:
            sums_row = []
            for j, column in enumerate(self.columns):
                sum_column = 0
                for row_value in value:
                    try:
                        cell_value = row_value[j]
                    except IndexError:
                        continue
                    if cell_value in (None, ''):
                        continue
                    try:
                        sum_column += float(cell_value)
                    except ValueError:
                        sums_row.append(None)
                        break
                else:
                    sums_row.append(sum_column)
            if [x for x in sums_row if x is not None]:
                r += htmltext('<tfoot><tr>')
                for sum_column in sums_row:
                    if sum_column is None:
                        r += htmltext('<td></td>')
                    else:
                        r += htmltext('<td>%.2f</td>' % sum_column)
                r += htmltext('</tr></tfoot>')

        r += htmltext('</table>')

        return r.getvalue()

    def get_rst_view_value(self, value, indent=''):
        if not value:
            return indent
        r = []
        max_width = 0
        for column in self.columns:
            max_width = max(max_width, len(smart_str(column)))

        for i, row in enumerate(value):
            value[i] = [x or '' for x in row]

        def get_value(i, j):
            try:
                return smart_str(value[i][j])
            except IndexError:
                return '-'

        for i, row_value in enumerate(value):
            for j, column in enumerate(self.columns):
                try:
                    max_width = max(max_width, len(smart_str(row_value[j])))
                except IndexError:
                    # ignore errors for shorter than expected rows, this is
                    # typical of the field gaining new columns after some forms
                    # were already saved.
                    pass

        r.append(' '.join(['=' * max_width] * (len(self.columns))))
        r.append(' '.join([smart_str(column).center(max_width) for column in self.columns]))
        r.append(' '.join(['=' * max_width] * (len(self.columns))))
        for i, row_value in enumerate(value):
            r.append(
                ' '.join(
                    [cell.center(max_width) for cell in [get_value(i, x) for x in range(len(self.columns))]]
                )
            )
        r.append(' '.join(['=' * max_width] * (len(self.columns))))
        return misc.site_encode('\n'.join([indent + x for x in r]))

    def get_csv_value(self, element, **kwargs):
        return [_('unimplemented')]  # XXX


register_field_class(TableRowsField)


class MapField(WidgetField, MapOptionsMixin):
    key = 'map'
    description = _('Map')

    initial_position = None
    default_position = None
    position_template = None

    widget_class = MapWidget
    extra_attributes = [
        'initial_zoom',
        'min_zoom',
        'max_zoom',
        'initial_position',
        'default_position',
        'position_template',
    ]

    def migrate(self):
        changed = False
        if not self.initial_position:  # 2023-04-20
            if getattr(self, 'init_with_geoloc', False):
                self.initial_position = 'geoloc'
                changed = True
            elif self.default_position:
                self.initial_position = 'point'
                changed = True
        return changed

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        self.fill_zoom_admin_form(form, tab=('position', _('Position')))
        initial_position_widget = form.add(
            RadiobuttonsWidget,
            'initial_position',
            title=_('Initial Position'),
            options=(
                ('', _('Default position'), ''),
                ('point', _('Specific point'), 'point'),
                ('geoloc', _('Device geolocation'), 'geoloc'),
                ('template', _('From template'), 'template'),
            ),
            value=self.initial_position or '',
            extra_css_class='widget-inline-radio',
            tab=('position', _('Position')),
            attrs={'data-dynamic-display-parent': 'true'},
        )
        form.add(
            MapWidget,
            'default_position',
            value=self.default_position,
            default_zoom='9',
            required=False,
            tab=('position', _('Position')),
            attrs={
                'data-dynamic-display-child-of': initial_position_widget.get_name(),
                'data-dynamic-display-value': 'point',
            },
        )
        form.add(
            StringWidget,
            'position_template',
            value=self.position_template,
            size=80,
            required=False,
            hint=_('Positions (using latitute;longitude format) and addresses are supported.'),
            validation_function=ComputedExpressionWidget.validate_template,
            tab=('position', _('Position')),
            attrs={
                'data-dynamic-display-child-of': initial_position_widget.get_name(),
                'data-dynamic-display-value': 'template',
            },
        )

    def check_admin_form(self, form):
        self.check_zoom_admin_form(form)

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'initial_zoom',
            'min_zoom',
            'max_zoom',
            'initial_position',
            'default_position',
            'position_template',
        ]

    def get_prefill_value(self, user=None, force_string=True):
        if self.prefill.get('type') != 'string' or not self.prefill.get('value'):
            return (None, False)
        # template string must produce lat;lon to be interpreted as coordinates,
        # otherwise it will be interpreted as an address that will be geocoded.
        prefill_value, explicit_lock = super().get_prefill_value()
        if re.match(r'-?\d+(\.\d+)?;-?\d+(\.\d+)?$', prefill_value):
            return (prefill_value, explicit_lock)

        from wcs.wf.geolocate import GeolocateWorkflowStatusItem

        geolocate = GeolocateWorkflowStatusItem()
        geolocate.method = 'address_string'
        geolocate.address_string = prefill_value
        coords = geolocate.geolocate_address_string(None, compute_template=False)
        if not coords:
            return (None, False)
        return ('%(lat)s;%(lon)s' % coords, False)

    def get_view_value(self, value, **kwargs):
        widget = self.widget_class('x%s' % random.random(), value, readonly=True)
        return widget.render_widget_content()

    def get_rst_view_value(self, value, indent=''):
        return indent + value

    def convert_value_from_str(self, value):
        try:
            dummy, dummy = (float(x) for x in value.split(';'))
        except (AttributeError, ValueError):
            return None
        return value

    def get_json_value(self, value, **kwargs):
        if not value or ';' not in value:
            return None
        lat, lon = value.split(';')
        try:
            lat = float(lat)
            lon = float(lon)
        except ValueError:
            return None
        return {'lat': lat, 'lon': lon}

    def from_json_value(self, value):
        if 'lat' in value and 'lon' in value:
            return '%s;%s' % (float(value['lat']), float(value['lon']))
        else:
            return None

    def get_structured_value(self, data):
        return self.get_json_value(data.get(self.id))

    def set_value(self, data, value, raise_on_error=False):
        if value == '':
            value = None
        elif value and ';' not in value:
            raise SetValueError('invalid coordinates %r (missing ;)' % value)
        elif value:
            try:
                dummy, dummy = (float(x) for x in value.split(';'))
            except ValueError:
                # will catch both "too many values to unpack" and invalid float values
                raise SetValueError('invalid coordinates %r' % value)
        super().set_value(data, value)


register_field_class(MapField)


class RankedItemsField(WidgetField):
    key = 'ranked-items'
    description = _('Ranked Items')
    allow_complex = True

    items = []
    randomize_items = False
    widget_class = RankedItemsWidget
    anonymise = False

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['elements'] = self.items or []
        kwargs['randomize_items'] = self.randomize_items

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        try:
            form.remove('prefill')
        except KeyError:  # perhaps it was already removed
            pass
        form.add(
            WidgetList,
            'items',
            title=_('Items'),
            element_type=StringWidget,
            value=self.items,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add item'),
        )
        form.add(CheckboxWidget, 'randomize_items', title=_('Randomize Items'), value=self.randomize_items)

    def get_admin_attributes(self):
        attrs = WidgetField.get_admin_attributes(self) + ['items', 'randomize_items']
        if 'prefill' in attrs:
            attrs.remove('prefill')
        return attrs

    def get_view_value(self, value, **kwargs):
        r = TemplateIO(html=True)
        r += htmltext('<ul>')
        items = list(value.items())
        items.sort(key=lambda x: x[1] or sys.maxsize)
        counter = 0
        last_it = None
        for it in items:
            if it[1] is not None:
                if last_it != it[1]:
                    counter += 1
                    last_it = it[1]
                r += htmltext('<li>%s: %s</li>') % (counter, it[0])
        r += htmltext('</ul>')
        return r.getvalue()

    def get_rst_view_value(self, value, indent=''):
        items = list(value.items())
        items.sort(key=lambda x: x[1] or sys.maxsize)
        counter = 0
        last_it = None
        values = []
        for it in items:
            if it[1] is not None:
                if last_it != it[1]:
                    counter += 1
                    last_it = it[1]
                values.append('%s: %s' % (counter, it[0]))
        return indent + ' / '.join(values)

    def get_csv_heading(self):
        if not self.items:
            return [self.label]
        return [self.label] + [''] * (len(self.items) - 1)

    def get_csv_value(self, element, **kwargs):
        if not self.items:
            return ['']
        if not isinstance(element, dict):
            element = {}
        items = [x for x in element.items() if x[1] is not None]
        items.sort(key=lambda x: x[1])
        ranked = [x[0] for x in items]
        return ranked + ['' for x in range(len(self.items) - len(ranked))]


register_field_class(RankedItemsField)


class PasswordField(WidgetField):
    key = 'password'
    description = _('Password')

    min_length = 0
    max_length = 0
    count_uppercase = 0
    count_lowercase = 0
    count_digit = 0
    count_special = 0
    confirmation = True
    confirmation_title = None
    strength_indicator = True
    formats = ['sha1']
    extra_attributes = [
        'formats',
        'min_length',
        'max_length',
        'count_uppercase',
        'count_lowercase',
        'count_digit',
        'count_special',
        'confirmation',
        'confirmation_title',
        'strength_indicator',
    ]

    widget_class = PasswordEntryWidget

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + self.extra_attributes

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        formats = [
            ('cleartext', _('Clear text')),
            ('md5', _('MD5')),
            ('sha1', _('SHA1')),
        ]
        form.add(
            CheckboxesWidget,
            'formats',
            title=_('Storage formats'),
            value=self.formats,
            options=formats,
            inline=True,
        )
        form.add(IntWidget, 'min_length', title=_('Minimum length'), value=self.min_length)
        form.add(
            IntWidget,
            'max_length',
            title=_('Maximum password length'),
            value=self.max_length,
            hint=_('0 for unlimited length'),
        )
        form.add(
            IntWidget,
            'count_uppercase',
            title=_('Minimum number of uppercase characters'),
            value=self.count_uppercase,
        )
        form.add(
            IntWidget,
            'count_lowercase',
            title=_('Minimum number of lowercase characters'),
            value=self.count_lowercase,
        )
        form.add(IntWidget, 'count_digit', title=_('Minimum number of digits'), value=self.count_digit)
        form.add(
            IntWidget,
            'count_special',
            title=_('Minimum number of special characters'),
            value=self.count_special,
        )
        form.add(
            CheckboxWidget,
            'strength_indicator',
            title=_('Add a password strength indicator'),
            value=self.strength_indicator,
            default_value=self.__class__.strength_indicator,
        )
        form.add(
            CheckboxWidget,
            'confirmation',
            title=_('Add a confirmation input'),
            value=self.confirmation,
            default_value=self.__class__.confirmation,
        )
        form.add(
            StringWidget,
            'confirmation_title',
            size=50,
            title=_('Label for confirmation input'),
            value=self.confirmation_title,
        )

    def get_view_value(self, value, **kwargs):
        return '●' * 8

    def get_csv_value(self, element, **kwargs):
        return [self.get_view_value(element)]

    def get_rst_view_value(self, value, indent=''):
        return indent + self.get_view_value(value)


register_field_class(PasswordField)


class BlockRowValue:
    # a container for a value that will be added as a "line" of a block
    def __init__(self, append=False, merge=False, existing=None, **kwargs):
        self.append = append
        self.merge = merge
        self.attributes = kwargs
        self.rows = None
        if append is True:
            self.rows = getattr(existing, 'rows', None) or []
            self.rows.append(kwargs)

    def check_current_value(self, current_block_value):
        return (
            isinstance(current_block_value, dict)
            and 'data' in current_block_value
            and isinstance(current_block_value['data'], list)
        )

    def make_value(self, block, field, data):
        def make_row_data(attributes):
            row_data = {}
            for sub_field in block.fields:
                if sub_field.varname and sub_field.varname in attributes:
                    sub_value = attributes.get(sub_field.varname)
                    if sub_field.convert_value_from_anything:
                        sub_value = sub_field.convert_value_from_anything(sub_value)
                    sub_field.set_value(row_data, sub_value)
            return row_data

        row_data = make_row_data(self.attributes)

        current_block_value = data.get(field.id)
        if not self.check_current_value(current_block_value):
            current_block_value = None
        if self.append and current_block_value:
            block_value = current_block_value
            block_value['data'].append(row_data)
        elif self.merge is not False and field.id in data:
            block_value = current_block_value
            try:
                merge_index = -1 if self.merge is True else int(self.merge)
                block_value['data'][merge_index].update(row_data)
            except (ValueError, IndexError):
                # ValueError if self.merge is not an integer,
                # IndexError if merge_index is out of range.
                pass  # ignore
        elif self.rows:
            rows_data = [make_row_data(x) for x in self.rows if x]
            block_value = {'data': rows_data, 'schema': {x.id: x.key for x in block.fields}}
        else:
            block_value = {'data': [row_data], 'schema': {x.id: x.key for x in block.fields}}
        return block_value


class BlockField(WidgetField):
    key = 'block'
    allow_complex = True

    widget_class = BlockWidget
    default_items_count = 1
    max_items = 1
    extra_attributes = [
        'block',
        'default_items_count',
        'max_items',
        'add_element_label',
        'label_display',
        'remove_button',
    ]
    add_element_label = ''
    label_display = 'normal'
    remove_button = False
    block_slug = None

    # cache
    _block = None

    def migrate(self):
        changed = False
        if not self.block_slug:  # 2023-05-21
            self.block_slug = self.type.removeprefix('block:')
            changed = True
        return changed

    @property
    def block(self):
        if self._block:
            return self._block
        self._block = BlockDef.get_on_index(self.block_slug, 'slug')
        return self._block

    def get_type_label(self):
        try:
            return _('Field Block (%s)') % self.block.name
        except KeyError:
            return _('Field Block (%s, missing)') % self.block_slug

    def get_dependencies(self):
        yield from super().get_dependencies()
        yield self.block

    def add_to_form(self, form, value=None):
        try:
            self.block
        except KeyError:
            raise MissingBlockFieldError(self.block_slug)
        return super().add_to_form(form, value=value)

    def fill_admin_form(self, form):
        super().fill_admin_form(form)
        form.add(
            IntWidget,
            'default_items_count',
            title=_('Number of items to display by default'),
            value=self.default_items_count,
        )
        form.add(IntWidget, 'max_items', title=_('Maximum number of items'), value=self.max_items)
        form.add(
            StringWidget, 'add_element_label', title=_('Label of "Add" button'), value=self.add_element_label
        )
        display_options = [
            ('normal', _('Normal')),
            ('subtitle', _('Subtitle')),
            ('hidden', _('Hidden')),
        ]
        form.add(
            SingleSelectWidget,
            'label_display',
            title=_('Label display'),
            value=self.label_display or 'normal',
            options=display_options,
        )
        form.add(CheckboxWidget, 'remove_button', title=_('Include remove button'), value=self.remove_button)

    def get_admin_attributes(self):
        return super().get_admin_attributes() + [
            'default_items_count',
            'max_items',
            'add_element_label',
            'label_display',
            'remove_button',
            'block_slug',  # only mentioned for xml export/import
        ]

    def store_display_value(self, data, field_id, raise_on_error=False):
        value = data.get(field_id)
        parts = []
        if value and value.get('data'):
            for subvalue in value.get('data'):
                parts.append(self.block.get_display_value(subvalue))
        return ', '.join(parts)

    def get_view_value(self, value, summary=False, include_unset_required_fields=False, **kwargs):
        from wcs.workflows import template_on_formdata

        if 'value_id' not in kwargs:
            # when called from get_rst_view_value()
            return str(value or '')
        value = kwargs['value_id']
        if value is None:
            return ''
        r = TemplateIO(html=True)
        for i, row_value in enumerate(value['data']):
            context = self.block.get_substitution_counter_variables(i)
            for field in self.block.fields:
                if summary and not field.include_in_summary_page:
                    continue
                if not hasattr(field, 'get_value_info'):
                    # inert field
                    if field.include_in_summary_page:
                        with get_publisher().substitutions.temporary_feed(context):
                            if field.key == 'title':
                                label = template_on_formdata(None, field.label, autoescape=False)
                                r += htmltext('<div class="title %s"><h3>%s</h3></div>') % (
                                    field.extra_css_class or '',
                                    label,
                                )
                            elif field.key == 'subtitle':
                                label = template_on_formdata(None, field.label, autoescape=False)
                                r += htmltext('<div class="subtitle %s"><h4>%s</h4></div>') % (
                                    field.extra_css_class or '',
                                    label,
                                )
                            elif field.key == 'comment':
                                r += htmltext(
                                    '<div class="comment-field %s">%s</div>'
                                    % (field.extra_css_class or '', field.get_text())
                                )
                    continue
                css_classes = ['field', 'field-type-%s' % field.key]
                if field.extra_css_class:
                    css_classes.append(field.extra_css_class)
                sub_value, sub_value_details = field.get_value_info(row_value)
                if sub_value is None and not (field.required and include_unset_required_fields):
                    continue
                label_id = f'form-field-label-f{self.id}-r{i}-s{field.id}'
                r += htmltext('<div class="%s">' % ' '.join(css_classes))
                r += htmltext('<p id="%s" class="label">%s</p> ') % (label_id, field.label)
                if sub_value is None:
                    r += htmltext('<div class="value"><i>%s</i></div>') % _('Not set')
                else:
                    r += htmltext('<div class="value">')
                    kwargs = {'parent_field': self, 'parent_field_index': i, 'label_id': label_id}
                    kwargs.update(**sub_value_details)
                    r += field.get_view_value(sub_value, **kwargs)
                    r += htmltext('</div>')
                r += htmltext('</div>\n')
        return r.getvalue()

    def get_csv_heading(self, subfield_label=None):
        nb_items = self.max_items or 1
        label = self.label
        if subfield_label:
            label += ' - %s' % subfield_label
        if nb_items == 1:
            return [label]
        headings = ['%s - %s' % (label, x + 1) for x in range(nb_items)]
        return headings

    def get_csv_value(self, element, **kwargs):
        nb_items = self.max_items or 1
        cells = [''] * nb_items
        if element and element.get('data'):
            for i, subvalue in enumerate(element.get('data')[:nb_items]):
                if subvalue:
                    cells[i] = self.block.get_display_value(subvalue)
        return cells

    def set_value(self, data, value, **kwargs):
        if value == '':
            value = None
        if isinstance(value, BlockRowValue):
            value = value.make_value(block=self.block, field=self, data=data)
        elif value and not (isinstance(value, dict) and 'data' in value and 'schema' in value):
            raise SetValueError('invalid value for block')
        super().set_value(data, value, **kwargs)

    def get_json_value(self, value, **kwargs):
        from wcs.formdata import FormData

        result = []
        if not value or not value.get('data'):
            return result
        for subvalue_data in value.get('data'):
            result.append(
                FormData.get_json_data_dict(
                    subvalue_data,
                    self.block.fields,
                    formdata=kwargs.get('formdata'),
                    include_files=kwargs.get('include_file_content'),
                    include_unnamed_fields=True,
                )
            )
        return result

    def from_json_value(self, value):
        from wcs.api import posted_json_data_to_formdata_data

        result = []
        if isinstance(value, list):
            for subvalue_data in value or []:
                result.append(posted_json_data_to_formdata_data(self.block, subvalue_data))

        return {'data': result, 'schema': {x.id: x.key for x in self.block.fields}}

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        node = ET.Element('{%s}span' % OD_NS['text'])
        node.text = od_clean_text(force_str(value))
        return node

    def __getstate__(self):
        # do not store _block cache
        odict = copy.copy(self.__dict__)
        if '_block' in odict:
            del odict['_block']
        return odict

    def __setstate__(self, ndict):
        # make sure a cached copy of _block is not restored
        self.__dict__ = ndict
        self._block = None


class ComputedField(Field):
    key = 'computed'
    description = _('Computed Data')

    value_template = None
    freeze_on_initial_value = False
    data_source = {}

    add_to_form = None
    add_to_view_form = None
    get_opendocument_node_value = None

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        if not self.varname:
            self.varname = misc.simplify(self.label, space='_')

    def get_admin_attributes(self):
        attributes = super().get_admin_attributes()
        attributes.remove('condition')
        return attributes + ['varname', 'value_template', 'freeze_on_initial_value', 'data_source']

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(
            VarnameWidget,
            'varname',
            title=_('Identifier'),
            required=True,
            value=self.varname,
            size=30,
            hint=_('This is used as suffix for variable names.'),
        )
        form.add(
            StringWidget,
            'value_template',
            title=_('Value'),
            required=True,
            size=80,
            value=self.value_template,
            validation_function=ComputedExpressionWidget.validate_template,
            hint=_('As a Django template'),
        )
        form.add(
            CheckboxWidget,
            'freeze_on_initial_value',
            title=_('Freeze on initial value'),
            value=self.freeze_on_initial_value,
        )
        form.add(
            data_sources.DataSourceSelectionWidget,
            'data_source',
            value=self.data_source,
            allowed_source_types={'cards'},
            title=_('Data Source (cards only)'),
            hint=_('This will make linked card data available for expressions.'),
            required=False,
        )

    def get_real_data_source(self):
        return data_sources.get_real(self.data_source)

    def get_dependencies(self):
        yield from super().get_dependencies()
        yield from check_wscalls(self.value_template)


register_field_class(ComputedField)


def get_field_class_by_type(type):
    for k in field_classes:
        if k.key == type:
            return k
    if type.startswith('block:'):
        # make sure block type exists (raises KeyError on missing data)
        BlockDef.get_on_index(type[6:], 'slug')
        return BlockField
    raise KeyError()


def get_field_options(blacklisted_types):
    widgets, non_widgets = [], []
    disabled_fields = (get_publisher().get_site_option('disabled-fields') or '').split(',')
    disabled_fields = [f.strip() for f in disabled_fields if f.strip()]
    for klass in field_classes:
        if klass is ComputedField:
            continue
        if klass.key in blacklisted_types:
            continue
        if klass.key in disabled_fields:
            continue
        if issubclass(klass, WidgetField):
            widgets.append((klass.key, klass.description, klass.key))
        else:
            non_widgets.append((klass.key, klass.description, klass.key))
    options = widgets + [('', '—', '')] + non_widgets

    if 'computed' not in blacklisted_types:
        # add computed field in its own "section"
        options.extend([('', '—', ''), (ComputedField.key, ComputedField.description, ComputedField.key)])

    if not blacklisted_types or 'blocks' not in blacklisted_types:
        position = len(options)
        for blockdef in BlockDef.select(order_by='name'):
            options.append(('block:%s' % blockdef.slug, blockdef.name, 'block:%s' % blockdef.slug))
        if len(options) != position:
            # add separator
            options.insert(position, ('', '—', ''))
    return options