83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
import json
|
|
|
|
from django.conf import settings
|
|
|
|
from combo.utils import requests
|
|
|
|
from .models import Statistic
|
|
|
|
|
|
def update_available_statistics():
|
|
if not settings.KNOWN_SERVICES:
|
|
return
|
|
|
|
results = []
|
|
for provider in settings.STATISTICS_PROVIDERS:
|
|
if isinstance(provider, dict):
|
|
url = provider['url']
|
|
sites = {provider['id']: {'title': provider['name']}}
|
|
provider = provider['id']
|
|
else:
|
|
sites = settings.KNOWN_SERVICES.get(provider, {})
|
|
url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
|
|
|
|
for site_key, site_dict in sites.items():
|
|
response = requests.get(
|
|
url,
|
|
allow_redirects=False,
|
|
timeout=5,
|
|
remote_service=site_dict if provider in settings.KNOWN_SERVICES else {},
|
|
without_user=True,
|
|
headers={'accept': 'application/json'},
|
|
)
|
|
if response.status_code != 200:
|
|
continue
|
|
|
|
try:
|
|
result = response.json()
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(result, dict):
|
|
result = result['data'] # detect new api
|
|
|
|
for stat in result:
|
|
results.append(
|
|
Statistic(
|
|
slug=stat.get('slug') or stat['id'],
|
|
site_slug=site_key,
|
|
service_slug=provider,
|
|
label=stat['name'],
|
|
url=stat.get('data-url') or stat['url'],
|
|
site_title=site_dict.get('title', ''),
|
|
filters=stat.get('filters', []),
|
|
has_future_data=stat.get('future_data', False),
|
|
data_type=stat.get('data_type', ''),
|
|
available=True,
|
|
)
|
|
)
|
|
|
|
update_fields = ('label', 'url', 'site_title', 'filters', 'available', 'has_future_data', 'data_type')
|
|
all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()}
|
|
statistics_to_create = []
|
|
statistics_to_update = {}
|
|
for stat in results:
|
|
existing_stat = all_statistics.get(stat.natural_key())
|
|
if existing_stat:
|
|
for field in update_fields:
|
|
new_value = getattr(stat, field)
|
|
if getattr(existing_stat, field) != new_value:
|
|
setattr(existing_stat, field, new_value)
|
|
statistics_to_update[existing_stat.pk] = existing_stat
|
|
else:
|
|
statistics_to_create.append(stat)
|
|
|
|
Statistic.objects.bulk_create(statistics_to_create)
|
|
Statistic.objects.bulk_update(statistics_to_update.values(), update_fields)
|
|
|
|
available_stats = Statistic.objects.filter(available=True)
|
|
for stat in results:
|
|
available_stats = available_stats.exclude(
|
|
slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug
|
|
)
|
|
available_stats.update(available=False)
|